archives/pebble
2
0
Fork 0
mirror of https://github.com/google/pebble.git synced 2025-07-22 07:14:53 -04:00
Code Issues Projects Releases Packages Wiki Activity Actions

Import of the watch repository from Pebble

Browse source
This commit is contained in:
Matthieu Jeanson 2024-12-12 16:43:03 -08:00 • committed by Katharine Berry
commit 3b92768480
10334 changed files with 2564465 additions and 0 deletions
Download patch file Download diff file Expand all files Collapse all files

590
src/fw/services/normal/settings/settings_file.c Normal file
View file

@ -0,0 +1,590 @@
/*
* Copyright 2024 Google LLC
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#include "settings_file.h"
#include "settings_raw_iter.h"
#include "drivers/rtc.h"
#include "kernel/pbl_malloc.h"
#include "services/normal/filesystem/pfs.h"
#include "system/logging.h"
#include "system/passert.h"
#include "util/crc8.h"
#include <string.h>
#include <time.h>
static status_t bootup_check(SettingsFile *file);
static void compute_stats(SettingsFile *file);
static bool file_hdr_is_uninitialized(SettingsFileHeader *file_hdr) {
return (file_hdr->magic == 0xffffffff) && (file_hdr->version == 0xffff)
&& (file_hdr->flags == 0xffff);
}
static status_t prv_open(SettingsFile *file, const char *name, uint8_t flags, int max_used_space) {
// Making the max_space_total at least a little bit larger than the
// max_used_space allows us to avoid thrashing. Without it, if
// max_space_total == max_used_space, then if the file is full, changing a
// single value would force the whole file to be rewritten- every single
// time! It's probably worth it to "waste" a bit of flash space to avoid
// this pathalogical case.
int max_space_total = pfs_sector_optimal_size(max_used_space * 12 / 10, strlen(name));
// TODO: Dynamically sized files?
int fd = pfs_open(name, flags, FILE_TYPE_STATIC, max_space_total);
if (fd < 0) {
PBL_LOG(LOG_LEVEL_ERROR, "Could not open settings file '%s', %d", name, fd);
if (fd == E_BUSY) {
// This is very bad. Someone didn't use a mutex. There could already be
// silent corruption, so it's better to crash now rather than let things
// get even more scrambled.
PBL_CROAK("Settings file is already open!");
}
return fd;
}
*file = (SettingsFile) {
.name = kernel_strdup_check(name),
.max_used_space = max_used_space,
.max_space_total = max_space_total,
};
settings_raw_iter_init(&file->iter, fd, file->name);
SettingsFileHeader file_hdr = file->iter.file_hdr;
if (file_hdr_is_uninitialized(&file_hdr)) {
// Newly created file, create & write out header.
memcpy(&file_hdr.magic, SETTINGS_FILE_MAGIC, sizeof(file_hdr.magic));
file_hdr.version = SETTINGS_FILE_VERSION;
settings_raw_iter_write_file_header(&file->iter, &file_hdr);
}
if (memcmp(&file_hdr.magic, SETTINGS_FILE_MAGIC, sizeof(file_hdr.magic)) != 0) {
PBL_LOG(LOG_LEVEL_ERROR, "Attempted to open %s, not a settings file.", name);
pfs_close_and_remove(fd);
return E_INVALID_OPERATION;
}
if (file_hdr.version > SETTINGS_FILE_VERSION) {
PBL_LOG(LOG_LEVEL_WARNING,
"Unrecognized version %d for file %s, removing...",
file_hdr.version, name);
pfs_close_and_remove(fd);
return prv_open(file, name, flags, max_used_space);
}
status_t status = bootup_check(file);
if (status < 0) {
PBL_LOG(LOG_LEVEL_ERROR,
"Bootup check failed (%"PRId32"), not good. "
"Attempting to recover by deleting %s...", status, name);
pfs_close_and_remove(fd);
return prv_open(file, name, flags, max_used_space);
}
// There's a chance that the caller increased the desired size of the settings file since
// the file was originally created (i.e. the file was created in an earlier version of the
// firmware). If we detect that situation, let's re-write the file to the new larger requested
// size.
int actual_size = pfs_get_file_size(file->iter.fd);
if (actual_size < max_space_total) {
PBL_LOG(LOG_LEVEL_INFO, "Re-writing settings file %s to increase its size from %d to %d.",
name, actual_size, max_space_total);
// The settings_file_rewrite_filtered call creates a new file based on file->max_used_space
// and copies the contents of the existing file into it.
status = settings_file_rewrite_filtered(file, NULL, NULL);
if (status < 0) {
PBL_LOG(LOG_LEVEL_ERROR, "Could not resize file %s (error %"PRId32"). Creating new one",
name, status);
return prv_open(file, name, flags, max_used_space);
}
}
compute_stats(file);
return S_SUCCESS;
}
status_t settings_file_open(SettingsFile *file, const char *name,
int max_used_space) {
return prv_open(file, name, OP_FLAG_READ | OP_FLAG_WRITE, max_used_space);
}
void settings_file_close(SettingsFile *file) {
settings_raw_iter_deinit(&file->iter);
kernel_free(file->name);
file->name = NULL;
}
static int record_size(SettingsRecordHeader *hdr) {
return sizeof(*hdr) + hdr->key_len + hdr->val_len;
}
// Flags are stored in flash the inverse of how you might normally expect- a
// zero denotes that the flag is set, a 1 means it is not. This is because our
// flash chip is NOR flash, and thus is all 1's by default.
// Once setting a flag, we cannot unset it.
static void set_flag(SettingsRecordHeader *hdr, uint8_t flags) {
hdr->flags &= ~flags;
}
static void clear_flag(SettingsRecordHeader *hdr, uint8_t flags) {
hdr->flags |= flags;
}
static bool flag_is_set(SettingsRecordHeader *hdr, uint8_t flags) {
return (hdr->flags & flags) == 0;
}
// Records have 4 possible states:
// - EOF marker: Header is all 1s
// - partially_written: Some bits in the header have been changed to 0s, but
// the entire record has not been completely written yet. Records in this
// state are removed on bootup, since they are in an indeterminate state.
// - written: The typical state for a record. == !partially_written
// - partially_overwritten: This record has been superceeded by another, which
// we are currently in the process of writing out to flash. Records in
// this state are restored on bootup.
// - overwritten: This record has been superceeded by another, which has been
// completely written out to flash. We skip over and ignore overwritten
// records.
static bool partially_written(SettingsRecordHeader *hdr) {
return !flag_is_set(hdr, SETTINGS_FLAG_WRITE_COMPLETE);
}
static bool partially_overwritten(SettingsRecordHeader *hdr) {
return flag_is_set(hdr, SETTINGS_FLAG_OVERWRITE_STARTED)
&& !flag_is_set(hdr, SETTINGS_FLAG_OVERWRITE_COMPLETE);
}
static bool overwritten(SettingsRecordHeader *hdr) {
return flag_is_set(hdr, SETTINGS_FLAG_OVERWRITE_STARTED)
&& flag_is_set(hdr, SETTINGS_FLAG_OVERWRITE_COMPLETE);
}
static uint32_t utc_time() {
return rtc_get_time();
}
static bool deleted_and_expired(SettingsRecordHeader *hdr) {
return (hdr->val_len == 0)
&& (hdr->last_modified <= (utc_time() - DELETED_LIFETIME));
}
static void compute_stats(SettingsFile *file) {
file->dead_space = 0;
file->used_space = 0;
file->last_modified = 0;
file->used_space += sizeof(SettingsFileHeader);
file->used_space += sizeof(SettingsRecordHeader); // EOF Marker
for (settings_raw_iter_begin(&file->iter); !settings_raw_iter_end(&file->iter);
settings_raw_iter_next(&file->iter)) {
if (overwritten(&file->iter.hdr) || deleted_and_expired(&file->iter.hdr)) {
file->dead_space += record_size(&file->iter.hdr);
} else {
file->used_space += record_size(&file->iter.hdr);
}
if (file->iter.hdr.last_modified > file->last_modified) {
file->last_modified = file->iter.hdr.last_modified;
}
}
}
status_t settings_file_rewrite_filtered(
SettingsFile *file, SettingsFileRewriteFilterCallback filter_cb, void *context) {
SettingsFile new_file;
status_t status = prv_open(&new_file, file->name, OP_FLAG_OVERWRITE | OP_FLAG_READ,
file->max_used_space);
if (status < 0) {
PBL_LOG(LOG_LEVEL_ERROR,
"Could not open temporary file to compact settings file. Error %"PRIi32".",
status);
return status;
}
settings_raw_iter_begin(&new_file.iter);
for (settings_raw_iter_begin(&file->iter); !settings_raw_iter_end(&file->iter);
settings_raw_iter_next(&file->iter)) {
SettingsRecordHeader *hdr = &file->iter.hdr;
if (partially_written(hdr)) {
// This should only happen if we reboot in the middle of writing a new
// record, and it should always be the most recently written record, so
// we shouldn't lose any data here (except for the partially written
// record, but we should ignore it's garbage data anyway).
break;
}
if (overwritten(hdr) || deleted_and_expired(hdr)) {
continue;
}
if (partially_overwritten(hdr)) {
// The only case where we should hit this is if we are compacting a file
// which has a record which was in the middle of being overwritten, but
// the write of the new record didn't finish by the time we rebooted.
// There should only ever be one such record, and we shouldn't ever hit
// this if writing the new record actually completed, since we check
// for this case in bootup_check().
clear_flag(hdr, SETTINGS_FLAG_OVERWRITE_STARTED);
}
// Get the old key and value
uint8_t *key = kernel_malloc(hdr->key_len);
settings_raw_iter_read_key(&file->iter, key);
uint8_t *val = kernel_malloc(hdr->val_len);
settings_raw_iter_read_val(&file->iter, val, hdr->val_len);
// Include in re-written file if it passes the filter
if (!filter_cb || filter_cb(key, hdr->key_len, val, hdr->val_len, context)) {
settings_raw_iter_write_header(&new_file.iter, hdr);
settings_raw_iter_write_key(&new_file.iter, key);
settings_raw_iter_write_val(&new_file.iter, val);
settings_raw_iter_next(&new_file.iter);
}
kernel_free(key);
kernel_free(val);
}
settings_file_close(file);
// We have to close and reopen the new_file so that it's temp flag is cleared.
// Before the close succeeds, if we reboot, we will just end up reading the
// old file. After the close suceeds, we will end up reading the new
// (compacted) file.
char *name = kernel_strdup(new_file.name);
settings_file_close(&new_file);
status = prv_open(file, name, OP_FLAG_READ | OP_FLAG_WRITE, file->max_used_space);
kernel_free(name);
return status;
}
T_STATIC status_t settings_file_compact(SettingsFile *file) {
return settings_file_rewrite_filtered(file, NULL, NULL);
}
static bool key_matches(SettingsRawIter *iter, const uint8_t *key, int key_len) {
SettingsRecordHeader *hdr = &iter->hdr;
if (key_len != hdr->key_len) {
return false;
}
if (crc8_calculate_bytes(key, key_len, true /* big_endian */) != hdr->key_hash) {
return false;
}
uint8_t hdr_key[hdr->key_len];
settings_raw_iter_read_key(iter, hdr_key);
if (memcmp(key, hdr_key, hdr->key_len) == 0) {
return true;
}
return false;
}
static bool prv_is_desired_hdr(SettingsRawIter *iter, const uint8_t *key, int key_len) {
if (overwritten(&iter->hdr) || partially_written(&iter->hdr)) {
return false;
}
return key_matches(iter, key, key_len);
}
static bool search_forward(SettingsRawIter *iter, const uint8_t *key, int key_len) {
const int resumed_pos = settings_raw_iter_get_resumed_record_pos(iter);
// Resume searching at the current record
for (; !settings_raw_iter_end(iter); settings_raw_iter_next(iter)) {
if (prv_is_desired_hdr(iter, key, key_len)) {
return true;
}
}
// If we got here, we didn't find it between `resumed_pos` and the last record in the file.
// Wrap around to the beginning and search until we get to the `resumed_pos`
settings_raw_iter_begin(iter);
for (; settings_raw_iter_get_current_record_pos(iter) < resumed_pos;
settings_raw_iter_next(iter)) {
if (prv_is_desired_hdr(iter, key, key_len)) {
return true;
}
}
// No record found
return false;
}
static status_t cleanup_partial_transactions(SettingsFile *file) {
for (settings_raw_iter_begin(&file->iter); !settings_raw_iter_end(&file->iter);
settings_raw_iter_next(&file->iter)) {
if (partially_written(&file->iter.hdr)) {
// Compact will remove partially written records. We could be smarter,
// but this is something of an edge case.
return settings_file_compact(file);
}
if (!partially_overwritten(&file->iter.hdr)) {
continue;
}
int partially_overwritten_record_pos =
settings_raw_iter_get_current_record_pos(&file->iter);
uint8_t key[file->iter.hdr.key_len];
settings_raw_iter_read_key(&file->iter, key);
settings_raw_iter_next(&file->iter); // Skip the current record
bool found_another = search_forward(&file->iter, key, file->iter.hdr.key_len);
if (!found_another) {
// No other file->iter.hdr found, we must have rebooted in the middle of
// writing the new record. Compacting the file will copy over the
// previous record while clearing the overwrite bits for us, so that we
// can still find the previous record. We could be smarter here, but
// this should happen pretty rarely.
return settings_file_compact(file);
}
// The overwrite completed, we just rebooted before getting a chance
// to flip the completion bit on the previous record. Flip it now so
// that we don't have to keep checking on every boot.
settings_raw_iter_set_current_record_pos(&file->iter,
partially_overwritten_record_pos);
set_flag(&file->iter.hdr, SETTINGS_FLAG_OVERWRITE_COMPLETE);
settings_raw_iter_write_header(&file->iter, &file->iter.hdr);
}
return S_SUCCESS;
}
static status_t bootup_check(SettingsFile* file) {
return cleanup_partial_transactions(file);
}
int settings_file_get_len(SettingsFile *file, const void *key, size_t key_len) {
settings_raw_iter_resume(&file->iter);
if (search_forward(&file->iter, key, key_len)) {
return file->iter.hdr.val_len;
} else {
return 0;
}
}
bool settings_file_exists(SettingsFile *file, const void *key, size_t key_len) {
return (settings_file_get_len(file, key, key_len) > 0);
}
status_t settings_file_get(SettingsFile *file, const void *key, size_t key_len,
void *val_out, size_t val_out_len) {
settings_raw_iter_resume(&file->iter);
if (!search_forward(&file->iter, key, key_len)) {
memset(val_out, 0, val_out_len);
return E_DOES_NOT_EXIST;
}
if (deleted_and_expired(&file->iter.hdr)) {
memset(val_out, 0, val_out_len);
return E_DOES_NOT_EXIST;
}
size_t val_len = file->iter.hdr.val_len;
if (val_out_len > val_len) {
memset(val_out, 0, val_out_len);
return E_RANGE;
}
settings_raw_iter_read_val(&file->iter, val_out, val_out_len);
return S_SUCCESS;
}
status_t settings_file_set_byte(SettingsFile *file, const void *key,
size_t key_len, size_t offset, uint8_t byte) {
if (key_len > SETTINGS_KEY_MAX_LEN) {
return E_RANGE;
}
// Find the record
settings_raw_iter_resume(&file->iter);
if (!search_forward(&file->iter, key, key_len) ||
file->iter.hdr.val_len == 0) {
return E_DOES_NOT_EXIST;
}
PBL_ASSERTN(offset < file->iter.hdr.val_len);
settings_raw_iter_write_byte(&file->iter, offset, byte);
return S_SUCCESS;
}
// Note that this operation is designed to be atomic from the perspective of
// an outside observer. That is, either the new value will be completely
// written and returned for all future queries, or, if we reboot/loose power/
// run into an error, then we will continue to return the previous value.
// We should never run into a case where neither value exists.
status_t settings_file_set(SettingsFile *file, const void *key, size_t key_len,
const void *val, size_t val_len) {
// Cannot set keys while iterating (Try settings_file_rewrite)
PBL_ASSERTN(file->cur_record_pos == 0);
if (key_len > SETTINGS_KEY_MAX_LEN) {
return E_RANGE;
}
if (val_len > SETTINGS_VAL_MAX_LEN) {
return E_RANGE;
}
const bool is_delete = (val_len == 0);
const int rec_size = sizeof(SettingsRecordHeader) + key_len + val_len;
if (!is_delete && file->used_space + rec_size > file->max_used_space) {
return E_OUT_OF_STORAGE;
}
if (file->used_space + file->dead_space + rec_size > file->max_space_total) {
status_t status = settings_file_compact(file);
if (status < 0) {
return status;
}
}
int overwritten_record = -1;
// Find an existing record, if any, and mark it as overwrite-in-progress.
settings_raw_iter_resume(&file->iter);
if (search_forward(&file->iter, key, key_len)) {
set_flag(&file->iter.hdr, SETTINGS_FLAG_OVERWRITE_STARTED);
settings_raw_iter_write_header(&file->iter, &file->iter.hdr);
overwritten_record = settings_raw_iter_get_current_record_pos(&file->iter);
}
while (!settings_raw_iter_end(&file->iter)) {
settings_raw_iter_next(&file->iter);
}
// Create and write out a new record. Writing the header transitions us into
// the write-in-progress state, since at least once of the bits must be
// flipped from a 1 to a 0 in order for the header to be valid.
SettingsRecordHeader new_hdr;
memset(&new_hdr, 0xff, sizeof(new_hdr));
new_hdr.last_modified = utc_time();
new_hdr.key_hash = crc8_calculate_bytes(key, key_len, true /* big_endian */);
new_hdr.key_len = key_len;
new_hdr.val_len = val_len;
settings_raw_iter_write_header(&file->iter, &new_hdr);
settings_raw_iter_write_key(&file->iter, key);
settings_raw_iter_write_val(&file->iter, val);
// Mark the new record as write complete, now that we have completely written
// out the header, key, and value.
set_flag(&new_hdr, SETTINGS_FLAG_WRITE_COMPLETE);
settings_raw_iter_write_header(&file->iter, &new_hdr);
file->used_space += rec_size;
// Finally, mark the existing record, if any, as overwritten.
if (overwritten_record >= 0) {
settings_raw_iter_set_current_record_pos(&file->iter, overwritten_record);
set_flag(&file->iter.hdr, SETTINGS_FLAG_OVERWRITE_COMPLETE);
settings_raw_iter_write_header(&file->iter, &file->iter.hdr);
file->dead_space += record_size(&file->iter.hdr);
file->used_space -= record_size(&file->iter.hdr);
}
return S_SUCCESS;
}
status_t settings_file_mark_synced(SettingsFile *file, const void *key, size_t key_len) {
// Cannot set keys while iterating (Try settings_file_rewrite)
PBL_ASSERTN(file->cur_record_pos == 0);
if (key_len > SETTINGS_KEY_MAX_LEN) {
return E_RANGE;
}
// Find an existing record, if any, and mark it as synced
settings_raw_iter_resume(&file->iter);
if (search_forward(&file->iter, key, key_len)) {
set_flag(&file->iter.hdr, SETTINGS_FLAG_SYNCED);
settings_raw_iter_write_header(&file->iter, &file->iter.hdr);
return S_SUCCESS;
}
return E_DOES_NOT_EXIST;
}
status_t settings_file_delete(SettingsFile *file,
const void *key, size_t key_len) {
return settings_file_set(file, key, key_len, NULL, 0);
}
static void prv_get_key(SettingsFile *file, void *key, size_t key_len) {
PBL_ASSERTN(key_len <= file->iter.hdr.key_len);
settings_raw_iter_set_current_record_pos(&file->iter, file->cur_record_pos);
settings_raw_iter_read_key(&file->iter, key);
}
static void prv_get_val(SettingsFile *file, void *val, size_t val_len) {
PBL_ASSERTN(val_len <= file->iter.hdr.val_len);
settings_raw_iter_set_current_record_pos(&file->iter, file->cur_record_pos);
settings_raw_iter_read_val(&file->iter, val, val_len);
}
status_t settings_file_each(SettingsFile *file, SettingsFileEachCallback cb,
void *context) {
// Cannot set keys while iterating
PBL_ASSERTN(file->cur_record_pos == 0);
SettingsRecordInfo info;
for (settings_raw_iter_begin(&file->iter); !settings_raw_iter_end(&file->iter);
settings_raw_iter_next(&file->iter)) {
if (overwritten(&file->iter.hdr) || deleted_and_expired(&file->iter.hdr)) {
continue;
}
info = (SettingsRecordInfo) {
.last_modified = file->iter.hdr.last_modified,
.get_key = prv_get_key,
.key_len = file->iter.hdr.key_len,
.get_val = prv_get_val,
.val_len = file->iter.hdr.val_len,
.dirty = !flag_is_set(&file->iter.hdr, SETTINGS_FLAG_SYNCED),
};
file->cur_record_pos = settings_raw_iter_get_current_record_pos(&file->iter);
// if the callback returns false, stop iterating.
if (!cb(file, &info, context)) {
break;
}
settings_raw_iter_set_current_record_pos(&file->iter, file->cur_record_pos);
}
file->cur_record_pos = 0;
return S_SUCCESS;
}
typedef struct {
SettingsFileRewriteCallback cb;
SettingsFile *new_file;
void *user_context;
} RewriteCbContext;
static bool prv_rewrite_cb(SettingsFile *file, SettingsRecordInfo *info,
void *context) {
RewriteCbContext *cb_ctx = (RewriteCbContext*)context;
cb_ctx->cb(file, cb_ctx->new_file, info, cb_ctx->user_context);
return true; // continue iterating
}
status_t settings_file_rewrite(SettingsFile *file,
SettingsFileRewriteCallback cb, void *context) {
SettingsFile new_file;
status_t status = prv_open(&new_file, file->name,
OP_FLAG_OVERWRITE | OP_FLAG_READ,
file->max_used_space);
if (status < 0) {
return status;
}
RewriteCbContext cb_ctx = (RewriteCbContext) {
.cb = cb,
.new_file = &new_file,
.user_context = context,
};
settings_file_each(file, prv_rewrite_cb, &cb_ctx);
settings_file_close(file);
// We have to close and reopen the new_file so that it's temp flag is cleared.
// Before the close succeeds, if we reboot, we will just end up reading the
// old file. After the close suceeds, we will end up reading the new
// (compacted) file.
char *name = kernel_strdup(new_file.name);
settings_file_close(&new_file);
status = prv_open(file, name, OP_FLAG_READ | OP_FLAG_WRITE, file->max_used_space);
kernel_free(name);
return status;
}

168
src/fw/services/normal/settings/settings_file.h Normal file
View file

@ -0,0 +1,168 @@
/*
* Copyright 2024 Google LLC
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#pragma once
#include "settings_raw_iter.h"
// Deleted records have their key stick around for at least DELETED_LIFETIME
// before they can be garbage collected from the file in which they are
// contained, that way they have time to propegate to all devices we end up
// syncronizing with. For more information, refer to the sync protocol proposal:
// https://pebbletechnology.atlassian.net/wiki/pages/viewpage.action?pageId=26837564
//
// FIXME: See PBL-18945
#define DELETED_LIFETIME (0 * SECONDS_PER_DAY)
//! A SettingsFile is just a simple binary key-value store. Keys can be strings,
//! uint32_ts, or arbitrary bytes. Values are similarilly flexible. All
//! operations are atomic, so a reboot in the middle of changing the value for a
//! key will always either complete, returning the new value upon reboot, or
//! will just return the old value.
//! It also supports bidirection syncronization between the phone & watch,
//! using timestamps to resolve conflicts.
//! Note that although all operations are atomic, they are not thread-safe. If
//! you will be accessing a SettingsFile from multiple threads, make sure you
//! use locks!
// NOTE: These fields are internal, modify them at your own risk!
typedef struct SettingsFile {
SettingsRawIter iter;
char *name;
//! Maximum total space which can be used by this settings_file before a
//! compaction will be forced. (Must be >= max_used_space)
int max_space_total;
//! Maximum space that can be used by valid records within this settings_file.
//! Once this has been exceeded, attempting to add more keys or values will
//! fail.
int max_used_space;
//! Amount of space in the settings_file that is currently dead, i.e.
//! has been written to with some data, but that data is no longer valid.
//! (overwritten records get added to this)
int dead_space;
//! Amount of space in the settings_file that is currently used by valid
//! records.
int used_space;
//! When this file as a whole was last_modified.
//! Defined as records.max(&:last_modified)
uint32_t last_modified;
//! The position of the current record in the iteration (if any). Necessary
//! so that clients can read other records in the middle of iteration (i.e.
//! settings_file_each()/settings_file_rewrite()), without messing up the
//! state of the iteration. Set to 0 if not in use.
int cur_record_pos;
} SettingsFile;
//! max_used_space should be >= 5317 for persist files to make sure we can
//! always fit all of the records in the worst case (if the programmer stored
//! nothing but booleans).
//! Note: If the settings file already exists, the max_used_space parameter is
//! ignored. We could change this if the need arises.
status_t settings_file_open(SettingsFile *file, const char *name,
int max_used_space);
void settings_file_close(SettingsFile *file);
bool settings_file_exists(SettingsFile *file, const void *key, size_t key_len);
status_t settings_file_delete(SettingsFile *file,
const void *key, size_t key_len);
int settings_file_get_len(SettingsFile *file, const void *key, size_t key_len);
//! val_out_len must exactly match the length of the record on disk.
status_t settings_file_get(SettingsFile *file, const void *key, size_t key_len,
void *val_out, size_t val_out_len);
status_t settings_file_set(SettingsFile *file, const void *key, size_t key_len,
const void *val, size_t val_len);
//! Mark a record as synced. The flag will remain until the record is overwritten
//! @param file the settings_file that contains the record
//! @param key the key to the settings file. Note: keys can be up to 127 bytes
//! @param key_len the length of the key
status_t settings_file_mark_synced(SettingsFile *file, const void *key, size_t key_len);
//! set a byte in a setting. This can only be used a byte at a time to guarantee
//! atomicity. Do not use to modify several bytes in a row!
//! Note that only the reset bits will be applied (it writes flash directly)
status_t settings_file_set_byte(
SettingsFile *file, const void *key, size_t key_len,
size_t offset, uint8_t byte);
//////////////////
// Each/rewrite //
//////////////////
typedef void (*SettingsFileGetter)(SettingsFile *file,
void *buf, size_t buf_len);
typedef struct {
uint32_t last_modified;
SettingsFileGetter get_key;
int key_len;
SettingsFileGetter get_val;
int val_len;
bool dirty; // has the dirty flag set
} SettingsRecordInfo;
//! Callback used for using settings_file_each.
//! The bool returned is used to control the iteration.
//! - If a callback returns true, the iteration continues
//! - If a callback returns false, the ieration stops.
typedef bool (*SettingsFileEachCallback)(SettingsFile *file,
SettingsRecordInfo *info,
void *context);
//! Calls cb for each and every entry within the given file.
//! Note that you cannot modify the settings file while iterating. If you want
//! to do this, try settings_file_rewrite instead. (you can read other entries
//! without fault).
status_t settings_file_each(SettingsFile *file, SettingsFileEachCallback cb,
void *context);
typedef void (*SettingsFileRewriteCallback)(SettingsFile *old_file,
SettingsFile *new_file,
SettingsRecordInfo *info,
void *context);
//! Opens a new SettingsFile with the same name as the original SettingsFile,
//! in overwrite mode. This new file is passed into the given
//! SettingsFileRewriteCallback, which is called for each entry within the
//! original file. If you desire to preserve a key/value pair, you must write
//! it to the new file.
status_t settings_file_rewrite(SettingsFile *file,
SettingsFileRewriteCallback cb,
void *context);
//! Callback used for using settings_file_rewrite_filtered.
//! The bool returned is used to control whether or not the record is included in the file
//! after compaction. This callback is not allowed to use any other settings_file calls.
//! - If callback returns true, the record is included
//! - If callback returns false, the record is not included
typedef bool (*SettingsFileRewriteFilterCallback)(void *key, size_t key_len, void *value,
size_t value_len, void *context);
//! Opens a new SettingsFile with the same name as the original SettingsFile,
//! in overwrite mode. Any records from the old file which pass through the filter_cb with
//! a true result are included into the new file. This call is much faster than using
//! settings_file_rewrite if all you are doing is excluding specific records from the old file.
status_t settings_file_rewrite_filtered(SettingsFile *file,
SettingsFileRewriteFilterCallback filter_cb, void *context);

239
src/fw/services/normal/settings/settings_raw_iter.c Normal file
View file

@ -0,0 +1,239 @@
/*
* Copyright 2024 Google LLC
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#include "settings_raw_iter.h"
#include "kernel/pbl_malloc.h"
#include "services/normal/filesystem/pfs.h"
#include "system/logging.h"
#include "system/passert.h"
#include "util/math.h"
///////////////////////////////////////////////////
// Helper functions for handling internal errors //
///////////////////////////////////////////////////
#if UNITTEST
static uint32_t s_num_record_changes;
#endif
static uint8_t *read_file_into_ram(SettingsRawIter *iter) {
int pos = pfs_seek(iter->fd, 0, FSeekCur);
int file_size = pfs_get_file_size(iter->fd);
if (file_size < 0) {
return NULL;
}
int read_size = file_size;
uint8_t *contents = kernel_calloc(1, read_size);
while (contents == NULL && read_size > 0) {
// If we can't allocate enough RAM to read the whole file, we should
// at least try to read part of it.
read_size /= 2;
contents = kernel_calloc(1, read_size);
}
if (contents == NULL) {
PBL_LOG(LOG_LEVEL_ERROR,
"Could not allocate %d bytes for corrupt file of size %d.",
read_size, file_size);
return NULL;
}
// In case reading the whole file is not possible due to RAM limitations,
// read the portions nearest the current seek position, as they are most
// likely to be the culprit.
int end_offset = MIN(pos + read_size/2, file_size);
int start_offset = end_offset - read_size;
int status = pfs_seek(iter->fd, start_offset, FSeekSet);
if (status < 0) {
PBL_LOG(LOG_LEVEL_ERROR, "Debug seek failed: %d", status);
kernel_free(contents);
return NULL;
}
int actual_read_size = pfs_read(iter->fd, contents, read_size);
PBL_LOG(LOG_LEVEL_INFO,
"Read %d (expected %d) bytes of file %s (size %d), around offset %d.",
actual_read_size, read_size, iter->file_name, file_size, pos);
return contents;
}
static NORETURN fatal_logic_error(SettingsRawIter *iter) {
PBL_LOG(LOG_LEVEL_ERROR,
"settings_raw_iter logic error. "
"Attempting to read affected file into RAM for easier debugging...");
uint8_t *contents = read_file_into_ram(iter);
PBL_LOG(LOG_LEVEL_INFO, "Removing affected file %s...", iter->file_name);
// Remove the file that caused us to get into this state before we reboot,
// that way we should be able to avoid getting into a reboot loop.
pfs_close_and_remove(iter->fd);
PBL_LOG(LOG_LEVEL_INFO, "Data at address %p. Rebooting...", contents);
PBL_CROAK("Internal logic error.");
}
static int sfs_seek(SettingsRawIter *iter, int amount, int whence) {
status_t status = pfs_seek(iter->fd, amount, whence);
if (status >= 0) {
return status;
}
int pos = pfs_seek(iter->fd, 0, FSeekCur);
PBL_LOG(LOG_LEVEL_ERROR, "Could not seek by %d from whence %d at pos %d: %"PRId32,
amount, whence, pos, status);
fatal_logic_error(iter);
}
static int sfs_pos(SettingsRawIter *iter) {
return sfs_seek(iter, 0, FSeekCur);
}
static int sfs_read(SettingsRawIter *iter, uint8_t *data, int data_len) {
status_t status = pfs_read(iter->fd, data, data_len);
if (status >= 0) {
return status;
}
int pos = pfs_seek(iter->fd, 0, FSeekCur);
PBL_LOG(LOG_LEVEL_ERROR, "Could not read data to %p of length %d at pos %d: %"PRId32,
data, data_len, pos, status);
fatal_logic_error(iter);
}
static int sfs_write(SettingsRawIter *iter, const uint8_t *data, int data_len) {
status_t status = pfs_write(iter->fd, data, data_len);
if (status >= 0) {
return status;
}
int pos = pfs_seek(iter->fd, 0, FSeekCur);
PBL_LOG(LOG_LEVEL_ERROR, "Could not write from %p, %d bytes at pos %d: %"PRId32,
data, data_len, pos, status);
fatal_logic_error(iter);
}
///////////////////////////
// Actual Iteration Code //
///////////////////////////
void settings_raw_iter_init(SettingsRawIter *iter, int fd, const char *file_name) {
*iter = (SettingsRawIter){};
iter->fd = fd;
iter->file_name = file_name;
sfs_seek(iter, 0, FSeekSet);
sfs_read(iter, (uint8_t*)&iter->file_hdr, sizeof(iter->file_hdr));
iter->hdr_pos = -1;
iter->resumed_pos = -1;
}
void settings_raw_iter_write_file_header(SettingsRawIter *iter, SettingsFileHeader *file_hdr) {
sfs_seek(iter, 0, FSeekSet);
sfs_write(iter, (uint8_t*)file_hdr, sizeof(*file_hdr));
iter->file_hdr = *file_hdr;
iter->hdr_pos = -1;
iter->resumed_pos = -1;
}
void settings_raw_iter_begin(SettingsRawIter *iter) {
sfs_seek(iter, sizeof(iter->file_hdr), FSeekSet);
// Read header for first record.
iter->hdr_pos = sfs_pos(iter);
iter->resumed_pos = iter->hdr_pos;
sfs_read(iter, (uint8_t*)&iter->hdr, sizeof(iter->hdr));
}
void settings_raw_iter_resume(SettingsRawIter *iter) {
iter->resumed_pos = iter->hdr_pos;
}
void settings_raw_iter_next(SettingsRawIter *iter) {
// Seek to start of next record header.
int record_len = sizeof(iter->hdr) + iter->hdr.key_len + iter->hdr.val_len;
sfs_seek(iter, iter->hdr_pos + record_len, FSeekSet);
// Read the next header
iter->hdr_pos = sfs_pos(iter);
sfs_read(iter, (uint8_t*)&iter->hdr, sizeof(iter->hdr));
#if UNITTEST
s_num_record_changes++;
#endif
}
bool settings_raw_iter_end(SettingsRawIter *iter) {
SettingsRecordHeader *hdr = &iter->hdr;
return (hdr->last_modified == 0xffffffff) && (hdr->flags == ((1 << FLAGS_BITS) - 1) &&
(hdr->key_hash == 0xff) && (hdr->key_len == ((1 << KEY_LEN_BITS) - 1)) &&
(hdr->val_len == SETTINGS_EOF_MARKER));
}
int settings_raw_iter_get_current_record_pos(SettingsRawIter *iter) {
return iter->hdr_pos;
}
int settings_raw_iter_get_resumed_record_pos(SettingsRawIter *iter) {
return iter->resumed_pos;
}
void settings_raw_iter_set_current_record_pos(SettingsRawIter *iter, int pos) {
sfs_seek(iter, pos, FSeekSet);
iter->hdr_pos = pos;
sfs_read(iter, (uint8_t*)&iter->hdr, sizeof(iter->hdr));
}
void settings_raw_iter_read_key(SettingsRawIter *iter, uint8_t *key_out) {
if (iter->hdr.key_len == 0) return;
sfs_seek(iter, iter->hdr_pos + sizeof(SettingsRecordHeader), FSeekSet);
sfs_read(iter, key_out, iter->hdr.key_len);
}
void settings_raw_iter_read_val(SettingsRawIter *iter, uint8_t *val_out, int val_len) {
if (iter->hdr.val_len == 0) return;
sfs_seek(iter, iter->hdr_pos
+ sizeof(SettingsRecordHeader) + iter->hdr.key_len, FSeekSet);
sfs_read(iter, val_out, val_len);
}
void settings_raw_iter_write_header(SettingsRawIter *iter, SettingsRecordHeader *hdr) {
PBL_ASSERTN(hdr->key_len <= SETTINGS_KEY_MAX_LEN);
PBL_ASSERTN(hdr->val_len <= SETTINGS_VAL_MAX_LEN);
sfs_seek(iter, iter->hdr_pos, FSeekSet);
sfs_write(iter, (uint8_t*)hdr, sizeof(*hdr));
iter->hdr = *hdr;
}
void settings_raw_iter_write_key(SettingsRawIter *iter, const uint8_t *key) {
if (iter->hdr.key_len == 0) return;
sfs_seek(iter, iter->hdr_pos + sizeof(SettingsRecordHeader), FSeekSet);
sfs_write(iter, key, iter->hdr.key_len);
}
void settings_raw_iter_write_val(SettingsRawIter *iter, const uint8_t *val) {
if (iter->hdr.val_len == 0) return;
sfs_seek(iter, iter->hdr_pos + sizeof(SettingsRecordHeader) + iter->hdr.key_len, FSeekSet);
sfs_write(iter, val, iter->hdr.val_len);
}
void settings_raw_iter_write_byte(SettingsRawIter *iter, int offset, uint8_t byte) {
sfs_seek(iter, iter->hdr_pos +
sizeof(SettingsRecordHeader) + iter->hdr.key_len + offset, FSeekSet);
sfs_write(iter, &byte, 1);
}
void settings_raw_iter_deinit(SettingsRawIter *iter) {
int status = pfs_close(iter->fd);
if (status < 0) {
PBL_LOG(LOG_LEVEL_WARNING, "Could not close settings file");
}
}
#if UNITTEST
uint32_t settings_raw_iter_prv_get_num_record_searches(void) {
return s_num_record_changes;
}
#endif

137
src/fw/services/normal/settings/settings_raw_iter.h Normal file
View file

@ -0,0 +1,137 @@
/*
* Copyright 2024 Google LLC
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#pragma once
//! This file is not intended for consumption by the general firmware, try
//! settings_file_each() in settings_file.h.
#include <inttypes.h>
#include <stddef.h>
#include <stdbool.h>
#include "system/status_codes.h"
#include "util/attributes.h"
#define SETTINGS_FILE_MAGIC "set"
#define SETTINGS_FILE_VERSION 1
typedef struct PACKED {
uint32_t magic; // = "set"
uint16_t version;
uint16_t flags;
} SettingsFileHeader;
_Static_assert(
sizeof((SettingsFileHeader) {}.magic) == sizeof(SETTINGS_FILE_MAGIC),
"The magic has been broken!");
#define SETTINGS_FLAG_WRITE_COMPLETE (1 << 0)
#define SETTINGS_FLAG_OVERWRITE_STARTED (1 << 1)
#define SETTINGS_FLAG_OVERWRITE_COMPLETE (1 << 2)
// Indicate that a record is in sync with the phone
#define SETTINGS_FLAG_SYNCED (1 << 3)
#define SETTINGS_KEY_MAX_LEN 127
#define SETTINGS_VAL_MAX_LEN (SETTINGS_EOF_MARKER - 1) // we reserve the largest value for EOF
#define KEY_LEN_BITS 7
#define VAL_LEN_BITS 11
#define FLAGS_BITS 6
#define SETTINGS_EOF_MARKER ((1 << VAL_LEN_BITS) - 1)
_Static_assert(KEY_LEN_BITS + VAL_LEN_BITS + FLAGS_BITS == 24,
"The record header bitfields must add up to 24!");
typedef struct PACKED {
uint32_t last_modified;
uint8_t key_hash;
uint8_t flags:FLAGS_BITS;
unsigned int key_len:KEY_LEN_BITS;
unsigned int val_len:VAL_LEN_BITS;
} SettingsRecordHeader;
// A SettingsRawIter is just a more convenient interface for the underlying file
// which has two primary utilities.
// a) It has an exception handling scheme for when logic is bad or files
// are corrupted, ensuring we can do something, and that we *always*
// do something when such unexpected conditions occur
// b) It ensures the upper layers (i.e. settings_file) can never get confused
// as to their current position within the file, and end up reading data
// as a header, reading past the end of a key/value, or other nefarious
// things.
typedef struct {
const char *file_name;
int fd;
SettingsFileHeader file_hdr;
// Header for the record we are currently on.
SettingsRecordHeader hdr;
// - Offset within the file pointing to the beginning of a `SettingsRecordHeader`
// - The header it points to is the one where our iterator is.
// - Used to make sure we can always skip to the next record properly.
int hdr_pos;
// - Offset within the file pointing to the beginning of a `SettingsRecordHeader`
// - The header it points to is the one where we began/resumed searching from.
// - Only gets changed when calling `settings_raw_iter_(being|resume)`
// - Used to allow wrapping from the end to the beginning when searching
// for a specific record.
int resumed_pos;
} SettingsRawIter;
//! Initialize the iterator for use with the given fd.
void settings_raw_iter_init(SettingsRawIter *iter, int fd, const char *file_name);
//! Useful for newly opened files.
void settings_raw_iter_write_file_header(SettingsRawIter *iter, SettingsFileHeader *file_hdr);
//! Begin iteration from the first record
void settings_raw_iter_begin(SettingsRawIter *iter);
//! Resumes iteration from the current record
void settings_raw_iter_resume(SettingsRawIter *iter);
//! Skip to the next record
void settings_raw_iter_next(SettingsRawIter *iter);
//! Returns true if we are at the end of the records.
bool settings_raw_iter_end(SettingsRawIter *iter);
//! Return the current record position, for later restoration by
//! \ref settings_raw_iter_set_current_record_pos.
int settings_raw_iter_get_current_record_pos(SettingsRawIter *iter);
//! Restore a previous record position from
//! \ref setting_iter_get_current_record_pos
void settings_raw_iter_set_current_record_pos(SettingsRawIter *iter, int pos);
//! Return the resumed record position. This was set when we started searching for a record.
int settings_raw_iter_get_resumed_record_pos(SettingsRawIter *iter);
//! Read the key/val for the current record. (The header is read automatically
//! by settings_raw_iter_next() as part of iteration).
void settings_raw_iter_read_key(SettingsRawIter *iter, uint8_t *key);
void settings_raw_iter_read_val(SettingsRawIter *iter, uint8_t *val, int val_len);
//! Write (over top of) the header/key/val for the current record.
void settings_raw_iter_write_header(SettingsRawIter *iter, SettingsRecordHeader *hdr);
void settings_raw_iter_write_key(SettingsRawIter *iter, const uint8_t *key);
void settings_raw_iter_write_val(SettingsRawIter *iter, const uint8_t *val);
//! Write a byte in place for the current record
void settings_raw_iter_write_byte(SettingsRawIter *iter, int offset, uint8_t byte);
//! Close a settings file and stop iteration.
void settings_raw_iter_deinit(SettingsRawIter *iter);