Import of the watch repository from Pebble

This commit is contained in:
Matthieu Jeanson 2024-12-12 16:43:03 -08:00 committed by Katharine Berry
commit 3b92768480
10334 changed files with 2564465 additions and 0 deletions

View file

@ -0,0 +1,155 @@
# Run a fuzz test to verify robustness against corrupted/malicious data.
import sys
import time
import zipfile
import random
import subprocess
Import("env", "malloc_env")
def set_pkgname(src, dst, pkgname):
data = open(str(src)).read()
placeholder = '// package name placeholder'
assert placeholder in data
data = data.replace(placeholder, 'package %s;' % pkgname)
open(str(dst), 'w').write(data)
# We want both pointer and static versions of the AllTypes message
# Prefix them with package name.
env.Command("alltypes_static.proto", "#alltypes/alltypes.proto",
lambda target, source, env: set_pkgname(source[0], target[0], 'alltypes_static'))
env.Command("alltypes_pointer.proto", "#alltypes/alltypes.proto",
lambda target, source, env: set_pkgname(source[0], target[0], 'alltypes_pointer'))
env.NanopbProto(["alltypes_pointer", "alltypes_pointer.options"])
env.NanopbProto(["alltypes_static", "alltypes_static.options"])
# Do the same for proto3 versions
env.Command("alltypes_proto3_static.proto", "#alltypes_proto3/alltypes.proto",
lambda target, source, env: set_pkgname(source[0], target[0], 'alltypes_proto3_static'))
env.Command("alltypes_proto3_pointer.proto", "#alltypes_proto3/alltypes.proto",
lambda target, source, env: set_pkgname(source[0], target[0], 'alltypes_proto3_pointer'))
env.NanopbProto(["alltypes_proto3_pointer", "alltypes_proto3_pointer.options"])
env.NanopbProto(["alltypes_proto3_static", "alltypes_proto3_static.options"])
# And also a callback version
env.Command("alltypes_callback.proto", "#alltypes/alltypes.proto",
lambda target, source, env: set_pkgname(source[0], target[0], 'alltypes_callback'))
env.NanopbProto(["alltypes_callback", "alltypes_callback.options"])
common_objs = [env.Object("random_data.c"),
env.Object("validation.c"),
env.Object("flakystream.c"),
env.Object("alltypes_pointer.pb.c"),
env.Object("alltypes_static.pb.c"),
env.Object("alltypes_callback.pb.c"),
env.Object("alltypes_proto3_pointer.pb.c"),
env.Object("alltypes_proto3_static.pb.c"),
"$COMMON/malloc_wrappers.o"]
objs_malloc = ["$COMMON/pb_encode_with_malloc.o",
"$COMMON/pb_decode_with_malloc.o",
"$COMMON/pb_common_with_malloc.o"] + common_objs
objs_static = ["$COMMON/pb_encode.o",
"$COMMON/pb_decode.o",
"$COMMON/pb_common.o"] + common_objs
fuzz = malloc_env.Program(["fuzztest.c"] + objs_malloc)
# Run the stand-alone fuzz tester
seed = int(time.time())
if env.get('EMBEDDED'):
iterations = 100
else:
iterations = 1000
env.RunTest(fuzz, ARGS = [str(seed), str(iterations)])
generate_message = malloc_env.Program(["generate_message.c"] + objs_static)
# Test the message generator
env.RunTest(generate_message, ARGS = [str(seed)])
env.RunTest("generate_message.output.fuzzed", [fuzz, "generate_message.output"])
# Run against the latest corpus from ossfuzz
# This allows quick testing against regressions and also lets us more
# completely test slow embedded targets. To reduce runtime, only a subset
# of the corpus is fuzzed each time.
def run_against_corpus(target, source, env):
corpus = zipfile.ZipFile(str(source[1]), 'r')
count = 0
args = [str(source[0])]
if "TEST_RUNNER" in env:
args = [env["TEST_RUNNER"]] + args
if "FUZZTEST_CORPUS_SAMPLESIZE" in env:
samplesize = int(env["FUZZTEST_CORPUS_SAMPLESIZE"])
elif env.get('EMBEDDED'):
samplesize = 100
else:
samplesize = 4096
files = [n for n in corpus.namelist() if not n.endswith('/')]
files = random.sample(files, min(samplesize, len(files)))
for filename in files:
sys.stdout.write("Fuzzing: %5d/%5d: %-40.40s\r" % (count, len(files), filename))
sys.stdout.flush()
count += 1
maxsize = env.get('CPPDEFINES', {}).get('FUZZTEST_BUFSIZE', 256*1024)
data_in = corpus.read(filename)[:maxsize]
try:
process = subprocess.Popen(args, stdin=subprocess.PIPE,
stdout=subprocess.PIPE, stderr=subprocess.PIPE)
stdout, stderr = process.communicate(input = data_in)
result = process.wait()
except OSError as e:
if e.errno == 22:
print("Warning: OSError 22 when running with input " + filename)
result = process.wait()
else:
raise
if result != 0:
stdout += stderr
print(stdout)
print('\033[31m[FAIL]\033[0m Program ' + str(args) + ' returned ' + str(result) + ' with input ' + filename + ' from ' + str(source[1]))
return result
open(str(target[0]), 'w').write(str(count))
print('\033[32m[ OK ]\033[0m Ran ' + str(args) + " against " + str(source[1]) + " (" + str(count) + " entries)")
env.Command("corpus.zip.fuzzed", [fuzz, "corpus.zip"], run_against_corpus)
env.Command("regressions.zip.fuzzed", [fuzz, "regressions.zip"], run_against_corpus)
# Build separate fuzzers for each test case.
# Having them separate speeds up control flow based fuzzer engines.
# These are mainly used by oss-fuzz project.
env_proto2_static = env.Clone()
env_proto2_static.Append(CPPDEFINES = {'FUZZTEST_PROTO2_STATIC': '1'})
env_proto2_static.Program("fuzztest_proto2_static",
[env_proto2_static.Object("fuzztest_proto2_static.o", "fuzztest.c")] + objs_static)
env_proto2_pointer = malloc_env.Clone()
env_proto2_pointer.Append(CPPDEFINES = {'FUZZTEST_PROTO2_POINTER': '1'})
env_proto2_pointer.Program("fuzztest_proto2_pointer",
[env_proto2_pointer.Object("fuzztest_proto2_pointer.o", "fuzztest.c")] + objs_malloc)
env_proto3_static = env.Clone()
env_proto3_static.Append(CPPDEFINES = {'FUZZTEST_PROTO3_STATIC': '1'})
env_proto3_static.Program("fuzztest_proto3_static",
[env_proto3_static.Object("fuzztest_proto3_static.o", "fuzztest.c")] + objs_static)
env_proto3_pointer = malloc_env.Clone()
env_proto3_pointer.Append(CPPDEFINES = {'FUZZTEST_PROTO3_POINTER': '1'})
env_proto3_pointer.Program("fuzztest_proto3_pointer",
[env_proto3_pointer.Object("fuzztest_proto3_pointer.o", "fuzztest.c")] + objs_malloc)
env_io_errors = malloc_env.Clone()
env_io_errors.Append(CPPDEFINES = {'FUZZTEST_IO_ERRORS': '1'})
env_io_errors.Program("fuzztest_io_errors",
[env_io_errors.Object("fuzztest_io_errors.o", "fuzztest.c")] + objs_malloc)

View file

@ -0,0 +1,4 @@
*.rep_* type:FT_CALLBACK
*.oneof_* submsg_callback:true
*.Limits.int64_min type:FT_CALLBACK
*.DescriptorSize8 descriptorsize:DS_8

View file

@ -0,0 +1,9 @@
# Generate all fields as pointers.
* type:FT_POINTER
*.static_msg type:FT_STATIC
*.*fbytes fixed_length:true max_size:4
*.*farray fixed_count:true max_count:5
*.*farray2 fixed_count:true max_count:3
*.DescriptorSize8 descriptorsize:DS_8
*.IntSizes.*int8 int_size:IS_8
*.IntSizes.*int16 int_size:IS_16

View file

@ -0,0 +1,4 @@
* type:FT_POINTER
*.static_msg type:FT_STATIC
*.*fbytes fixed_length:true max_size:4
*.req_limits proto3_singular_msgs:true

View file

@ -0,0 +1,4 @@
* max_size:32
* max_count:8
*.*fbytes fixed_length:true max_size:4
*.req_limits proto3_singular_msgs:true

View file

@ -0,0 +1,8 @@
* max_size:32
* max_count:8
*.*fbytes fixed_length:true max_size:4
*.*farray fixed_count:true max_count:5
*.*farray2 fixed_count:true max_count:3
*.DescriptorSize8 descriptorsize:DS_8
*.IntSizes.*int8 int_size:IS_8
*.IntSizes.*int16 int_size:IS_16

Binary file not shown.

View file

@ -0,0 +1,33 @@
#include "flakystream.h"
#include <string.h>
bool flakystream_callback(pb_istream_t *stream, pb_byte_t *buf, size_t count)
{
flakystream_t *state = stream->state;
if (state->position + count > state->msglen)
{
stream->bytes_left = 0;
return false;
}
else if (state->position + count > state->fail_after)
{
PB_RETURN_ERROR(stream, "flaky error");
}
memcpy(buf, state->buffer + state->position, count);
state->position += count;
return true;
}
void flakystream_init(flakystream_t *stream, const uint8_t *buffer, size_t msglen, size_t fail_after)
{
memset(stream, 0, sizeof(*stream));
stream->stream.callback = flakystream_callback;
stream->stream.bytes_left = SIZE_MAX;
stream->stream.state = stream;
stream->buffer = buffer;
stream->position = 0;
stream->msglen = msglen;
stream->fail_after = fail_after;
}

View file

@ -0,0 +1,19 @@
/* This module implements a custom input stream that can be set to give IO error
* at specific point. */
#ifndef FLAKYSTREAM_H
#define FLAKYSTREAM_H
#include <pb_decode.h>
typedef struct {
pb_istream_t stream;
const uint8_t *buffer;
size_t position;
size_t msglen;
size_t fail_after;
} flakystream_t;
void flakystream_init(flakystream_t *stream, const uint8_t *buffer, size_t msglen, size_t fail_after);
#endif

View file

@ -0,0 +1,469 @@
/* Fuzz testing for the nanopb core.
* Attempts to verify all the properties defined in the security model document.
*
* This program can run in three configurations:
* - Standalone fuzzer, generating its own inputs and testing against them.
* - Fuzzing target, reading input on stdin.
* - LLVM libFuzzer target, taking input as a function argument.
*/
#include <pb_decode.h>
#include <pb_encode.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <assert.h>
#include <malloc_wrappers.h>
#include "random_data.h"
#include "validation.h"
#include "flakystream.h"
#include "test_helpers.h"
#include "alltypes_static.pb.h"
#include "alltypes_pointer.pb.h"
#include "alltypes_callback.pb.h"
#include "alltypes_proto3_static.pb.h"
#include "alltypes_proto3_pointer.pb.h"
/* Longer buffer size allows hitting more branches, but lowers performance. */
#ifndef FUZZTEST_BUFSIZE
#define FUZZTEST_BUFSIZE 256*1024
#endif
#ifndef FUZZTEST_MAX_STANDALONE_BUFSIZE
#define FUZZTEST_MAX_STANDALONE_BUFSIZE 16384
#endif
static size_t g_bufsize = FUZZTEST_BUFSIZE;
/* Focusing on a single test case at a time improves fuzzing performance.
* If no test case is specified, enable all tests.
*/
#if !defined(FUZZTEST_PROTO2_STATIC) && \
!defined(FUZZTEST_PROTO3_STATIC) && \
!defined(FUZZTEST_PROTO2_POINTER) && \
!defined(FUZZTEST_PROTO3_POINTER) && \
!defined(FUZZTEST_IO_ERRORS)
#define FUZZTEST_PROTO2_STATIC
#define FUZZTEST_PROTO3_STATIC
#define FUZZTEST_PROTO2_POINTER
#define FUZZTEST_PROTO3_POINTER
#define FUZZTEST_IO_ERRORS
#endif
static uint32_t xor32_checksum(const void *data, size_t len)
{
const uint8_t *buf = (const uint8_t*)data;
uint32_t checksum = 1234;
for (; len > 0; len--)
{
checksum ^= checksum << 13;
checksum ^= checksum >> 17;
checksum ^= checksum << 5;
checksum += *buf++;
}
return checksum;
}
static bool do_decode(const uint8_t *buffer, size_t msglen, size_t structsize, const pb_msgdesc_t *msgtype, unsigned flags, bool assert_success)
{
bool status;
pb_istream_t stream;
size_t initial_alloc_count = get_alloc_count();
uint8_t *buf2 = malloc_with_check(g_bufsize); /* This is just to match the amount of memory allocations in do_roundtrips(). */
void *msg = malloc_with_check(structsize);
alltypes_static_TestExtension extmsg = alltypes_static_TestExtension_init_zero;
pb_extension_t ext = pb_extension_init_zero;
assert(msg);
memset(msg, 0, structsize);
ext.type = &alltypes_static_TestExtension_testextension;
ext.dest = &extmsg;
ext.next = NULL;
if (msgtype == alltypes_static_AllTypes_fields)
{
((alltypes_static_AllTypes*)msg)->extensions = &ext;
}
else if (msgtype == alltypes_pointer_AllTypes_fields)
{
((alltypes_pointer_AllTypes*)msg)->extensions = &ext;
}
stream = pb_istream_from_buffer(buffer, msglen);
status = pb_decode_ex(&stream, msgtype, msg, flags);
if (status)
{
validate_message(msg, structsize, msgtype);
}
if (assert_success)
{
if (!status) fprintf(stderr, "pb_decode: %s\n", PB_GET_ERROR(&stream));
assert(status);
}
pb_release(msgtype, msg);
free_with_check(msg);
free_with_check(buf2);
assert(get_alloc_count() == initial_alloc_count);
return status;
}
static bool do_stream_decode(const uint8_t *buffer, size_t msglen, size_t fail_after, size_t structsize, const pb_msgdesc_t *msgtype, bool assert_success)
{
bool status;
flakystream_t stream;
size_t initial_alloc_count = get_alloc_count();
void *msg = malloc_with_check(structsize);
assert(msg);
memset(msg, 0, structsize);
flakystream_init(&stream, buffer, msglen, fail_after);
status = pb_decode(&stream.stream, msgtype, msg);
if (status)
{
validate_message(msg, structsize, msgtype);
}
if (assert_success)
{
if (!status) fprintf(stderr, "pb_decode: %s\n", PB_GET_ERROR(&stream.stream));
assert(status);
}
pb_release(msgtype, msg);
free_with_check(msg);
assert(get_alloc_count() == initial_alloc_count);
return status;
}
static int g_sentinel;
static bool field_callback(pb_istream_t *stream, const pb_field_t *field, void **arg)
{
assert(stream);
assert(field);
assert(*arg == &g_sentinel);
return pb_read(stream, NULL, stream->bytes_left);
}
static bool submsg_callback(pb_istream_t *stream, const pb_field_t *field, void **arg)
{
assert(stream);
assert(field);
assert(*arg == &g_sentinel);
return true;
}
bool do_callback_decode(const uint8_t *buffer, size_t msglen, bool assert_success)
{
bool status;
pb_istream_t stream;
size_t initial_alloc_count = get_alloc_count();
alltypes_callback_AllTypes *msg = malloc_with_check(sizeof(alltypes_callback_AllTypes));
assert(msg);
memset(msg, 0, sizeof(alltypes_callback_AllTypes));
stream = pb_istream_from_buffer(buffer, msglen);
msg->rep_int32.funcs.decode = &field_callback;
msg->rep_int32.arg = &g_sentinel;
msg->rep_string.funcs.decode = &field_callback;
msg->rep_string.arg = &g_sentinel;
msg->rep_farray.funcs.decode = &field_callback;
msg->rep_farray.arg = &g_sentinel;
msg->req_limits.int64_min.funcs.decode = &field_callback;
msg->req_limits.int64_min.arg = &g_sentinel;
msg->cb_oneof.funcs.decode = &submsg_callback;
msg->cb_oneof.arg = &g_sentinel;
status = pb_decode(&stream, alltypes_callback_AllTypes_fields, msg);
if (assert_success)
{
if (!status) fprintf(stderr, "pb_decode: %s\n", PB_GET_ERROR(&stream));
assert(status);
}
pb_release(alltypes_callback_AllTypes_fields, msg);
free_with_check(msg);
assert(get_alloc_count() == initial_alloc_count);
return status;
}
/* Do a decode -> encode -> decode -> encode roundtrip */
void do_roundtrip(const uint8_t *buffer, size_t msglen, size_t structsize, const pb_msgdesc_t *msgtype)
{
bool status;
uint32_t checksum2, checksum3;
size_t msglen2, msglen3;
uint8_t *buf2 = malloc_with_check(g_bufsize);
void *msg = malloc_with_check(structsize);
/* For proto2 types, we also test extension fields */
alltypes_static_TestExtension extmsg = alltypes_static_TestExtension_init_zero;
pb_extension_t ext = pb_extension_init_zero;
pb_extension_t **ext_field = NULL;
ext.type = &alltypes_static_TestExtension_testextension;
ext.dest = &extmsg;
ext.next = NULL;
assert(buf2 && msg);
if (msgtype == alltypes_static_AllTypes_fields)
{
ext_field = &((alltypes_static_AllTypes*)msg)->extensions;
}
else if (msgtype == alltypes_pointer_AllTypes_fields)
{
ext_field = &((alltypes_pointer_AllTypes*)msg)->extensions;
}
/* Decode and encode the input data.
* This will bring it into canonical format.
*/
{
pb_istream_t stream = pb_istream_from_buffer(buffer, msglen);
memset(msg, 0, structsize);
if (ext_field) *ext_field = &ext;
status = pb_decode(&stream, msgtype, msg);
if (!status) fprintf(stderr, "pb_decode: %s\n", PB_GET_ERROR(&stream));
assert(status);
validate_message(msg, structsize, msgtype);
}
{
pb_ostream_t stream = pb_ostream_from_buffer(buf2, g_bufsize);
status = pb_encode(&stream, msgtype, msg);
/* Some messages expand when re-encoding and might no longer fit
* in the buffer. */
if (!status && strcmp(PB_GET_ERROR(&stream), "stream full") != 0)
{
fprintf(stderr, "pb_encode: %s\n", PB_GET_ERROR(&stream));
assert(status);
}
msglen2 = stream.bytes_written;
checksum2 = xor32_checksum(buf2, msglen2);
}
pb_release(msgtype, msg);
/* Then decode from canonical format and re-encode. Result should remain the same. */
if (status)
{
pb_istream_t stream = pb_istream_from_buffer(buf2, msglen2);
memset(msg, 0, structsize);
if (ext_field) *ext_field = &ext;
status = pb_decode(&stream, msgtype, msg);
if (!status) fprintf(stderr, "pb_decode: %s\n", PB_GET_ERROR(&stream));
assert(status);
validate_message(msg, structsize, msgtype);
}
if (status)
{
pb_ostream_t stream = pb_ostream_from_buffer(buf2, g_bufsize);
status = pb_encode(&stream, msgtype, msg);
if (!status) fprintf(stderr, "pb_encode: %s\n", PB_GET_ERROR(&stream));
assert(status);
msglen3 = stream.bytes_written;
checksum3 = xor32_checksum(buf2, msglen3);
assert(msglen2 == msglen3);
assert(checksum2 == checksum3);
}
pb_release(msgtype, msg);
free_with_check(msg);
free_with_check(buf2);
}
/* Run all enabled test cases for a given input */
void do_roundtrips(const uint8_t *data, size_t size, bool expect_valid)
{
size_t initial_alloc_count = get_alloc_count();
PB_UNUSED(expect_valid); /* Potentially unused depending on configuration */
#ifdef FUZZTEST_PROTO2_STATIC
if (do_decode(data, size, sizeof(alltypes_static_AllTypes), alltypes_static_AllTypes_fields, 0, expect_valid))
{
do_roundtrip(data, size, sizeof(alltypes_static_AllTypes), alltypes_static_AllTypes_fields);
do_stream_decode(data, size, SIZE_MAX, sizeof(alltypes_static_AllTypes), alltypes_static_AllTypes_fields, true);
do_callback_decode(data, size, true);
}
#endif
#ifdef FUZZTEST_PROTO3_STATIC
if (do_decode(data, size, sizeof(alltypes_proto3_static_AllTypes), alltypes_proto3_static_AllTypes_fields, 0, expect_valid))
{
do_roundtrip(data, size, sizeof(alltypes_proto3_static_AllTypes), alltypes_proto3_static_AllTypes_fields);
do_stream_decode(data, size, SIZE_MAX, sizeof(alltypes_proto3_static_AllTypes), alltypes_proto3_static_AllTypes_fields, true);
}
#endif
#ifdef FUZZTEST_PROTO2_POINTER
if (do_decode(data, size, sizeof(alltypes_pointer_AllTypes), alltypes_pointer_AllTypes_fields, 0, expect_valid))
{
do_roundtrip(data, size, sizeof(alltypes_pointer_AllTypes), alltypes_pointer_AllTypes_fields);
do_stream_decode(data, size, SIZE_MAX, sizeof(alltypes_pointer_AllTypes), alltypes_pointer_AllTypes_fields, true);
}
#endif
#ifdef FUZZTEST_PROTO3_POINTER
if (do_decode(data, size, sizeof(alltypes_proto3_pointer_AllTypes), alltypes_proto3_pointer_AllTypes_fields, 0, expect_valid))
{
do_roundtrip(data, size, sizeof(alltypes_proto3_pointer_AllTypes), alltypes_proto3_pointer_AllTypes_fields);
do_stream_decode(data, size, SIZE_MAX, sizeof(alltypes_proto3_pointer_AllTypes), alltypes_proto3_pointer_AllTypes_fields, true);
}
#endif
#ifdef FUZZTEST_IO_ERRORS
{
size_t orig_max_alloc_bytes = get_max_alloc_bytes();
/* Test decoding when error conditions occur.
* The decoding will end either when running out of memory or when stream returns IO error.
* Testing proto2 is enough for good coverage here, as it has a superset of the field types of proto3.
*/
set_max_alloc_bytes(get_alloc_bytes() + 4096);
do_stream_decode(data, size, size - 16, sizeof(alltypes_static_AllTypes), alltypes_static_AllTypes_fields, false);
do_stream_decode(data, size, size - 16, sizeof(alltypes_pointer_AllTypes), alltypes_pointer_AllTypes_fields, false);
set_max_alloc_bytes(orig_max_alloc_bytes);
}
/* Test pb_decode_ex() modes */
do_decode(data, size, sizeof(alltypes_static_AllTypes), alltypes_static_AllTypes_fields, PB_DECODE_NOINIT | PB_DECODE_DELIMITED, false);
do_decode(data, size, sizeof(alltypes_static_AllTypes), alltypes_static_AllTypes_fields, PB_DECODE_NULLTERMINATED, false);
/* Test callbacks also when message is not valid */
do_callback_decode(data, size, false);
#endif
assert(get_alloc_count() == initial_alloc_count);
}
/* Fuzzer stub for Google OSSFuzz integration */
int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size)
{
if (size > g_bufsize)
return 0;
do_roundtrips(data, size, false);
return 0;
}
#ifndef LLVMFUZZER
static bool generate_base_message(uint8_t *buffer, size_t *msglen)
{
pb_ostream_t stream;
bool status;
static const alltypes_static_AllTypes initval = alltypes_static_AllTypes_init_default;
/* Allocate a message and fill it with defaults */
alltypes_static_AllTypes *msg = malloc_with_check(sizeof(alltypes_static_AllTypes));
memcpy(msg, &initval, sizeof(initval));
/* Apply randomness to the data before encoding */
while (rand_int(0, 7))
rand_mess((uint8_t*)msg, sizeof(alltypes_static_AllTypes));
msg->extensions = NULL;
stream = pb_ostream_from_buffer(buffer, g_bufsize);
status = pb_encode(&stream, alltypes_static_AllTypes_fields, msg);
assert(stream.bytes_written <= g_bufsize);
assert(stream.bytes_written <= alltypes_static_AllTypes_size);
*msglen = stream.bytes_written;
pb_release(alltypes_static_AllTypes_fields, msg);
free_with_check(msg);
return status;
}
/* Stand-alone fuzzer iteration, generates random data itself */
static void run_iteration()
{
uint8_t *buffer = malloc_with_check(g_bufsize);
size_t msglen;
/* Fill the whole buffer with noise, to prepare for length modifications */
rand_fill(buffer, g_bufsize);
if (generate_base_message(buffer, &msglen))
{
rand_protobuf_noise(buffer, g_bufsize, &msglen);
/* At this point the message should always be valid */
do_roundtrips(buffer, msglen, true);
/* Apply randomness to the encoded data */
while (rand_bool())
rand_mess(buffer, g_bufsize);
/* Apply randomness to encoded data length */
if (rand_bool())
msglen = rand_int(0, g_bufsize);
/* In this step the message may be valid or invalid */
do_roundtrips(buffer, msglen, false);
}
free_with_check(buffer);
assert(get_alloc_count() == 0);
}
int main(int argc, char **argv)
{
int i;
int iterations;
if (argc >= 2)
{
/* Run in stand-alone mode */
if (g_bufsize > FUZZTEST_MAX_STANDALONE_BUFSIZE)
g_bufsize = FUZZTEST_MAX_STANDALONE_BUFSIZE;
random_set_seed(strtoul(argv[1], NULL, 0));
iterations = (argc >= 3) ? atol(argv[2]) : 10000;
for (i = 0; i < iterations; i++)
{
printf("Iteration %d/%d, seed %lu\n", i, iterations, (unsigned long)random_get_seed());
run_iteration();
}
}
else
{
/* Run as a stub for afl-fuzz and similar */
uint8_t *buffer;
size_t msglen;
buffer = malloc_with_check(g_bufsize);
SET_BINARY_MODE(stdin);
msglen = fread(buffer, 1, g_bufsize, stdin);
LLVMFuzzerTestOneInput(buffer, msglen);
if (!feof(stdin))
{
/* Read any leftover input data if our buffer is smaller than
* message size. */
fprintf(stderr, "Warning: input message too long\n");
while (fread(buffer, 1, g_bufsize, stdin) == g_bufsize);
}
free_with_check(buffer);
}
return 0;
}
#endif

View file

@ -0,0 +1,88 @@
/* Generates a random, valid protobuf message. Useful to seed
* external fuzzers such as afl-fuzz.
*/
#include <pb_encode.h>
#include <pb_common.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <assert.h>
#include "alltypes_static.pb.h"
#include "random_data.h"
#ifndef FUZZTEST_BUFSIZE
#define FUZZTEST_BUFSIZE 4096
#endif
/* Check that size/count fields do not exceed their max size.
* Otherwise we would have to loop pretty long in generate_message().
* Note that there may still be a few encoding errors from submessages.
*/
static void limit_sizes(alltypes_static_AllTypes *msg)
{
pb_field_iter_t iter;
pb_field_iter_begin(&iter, alltypes_static_AllTypes_fields, msg);
while (pb_field_iter_next(&iter))
{
if (PB_LTYPE(iter.type) == PB_LTYPE_BYTES)
{
((pb_bytes_array_t*)iter.pData)->size %= iter.data_size - PB_BYTES_ARRAY_T_ALLOCSIZE(0);
}
if (PB_HTYPE(iter.type) == PB_HTYPE_REPEATED)
{
*((pb_size_t*)iter.pSize) %= iter.array_size;
}
if (PB_HTYPE(iter.type) == PB_HTYPE_ONEOF)
{
/* Set the oneof to this message type with 50% chance. */
if (rand_word() & 1)
{
*((pb_size_t*)iter.pSize) = iter.tag;
}
}
}
}
static void generate_message()
{
alltypes_static_AllTypes msg;
alltypes_static_TestExtension extmsg = alltypes_static_TestExtension_init_zero;
pb_extension_t ext = pb_extension_init_zero;
static uint8_t buf[FUZZTEST_BUFSIZE];
pb_ostream_t stream = {0};
do {
rand_fill((void*)&msg, sizeof(msg));
limit_sizes(&msg);
rand_fill((void*)&extmsg, sizeof(extmsg));
ext.type = &alltypes_static_TestExtension_testextension;
ext.dest = &extmsg;
ext.next = NULL;
msg.extensions = &ext;
stream = pb_ostream_from_buffer(buf, sizeof(buf));
} while (!pb_encode(&stream, alltypes_static_AllTypes_fields, &msg));
fwrite(buf, 1, stream.bytes_written, stdout);
}
int main(int argc, char **argv)
{
if (argc < 2)
{
fprintf(stderr, "Usage: generate_message <seed>\n");
return 1;
}
random_set_seed(atol(argv[1]));
generate_message();
return 0;
}

45
third_party/nanopb/tests/fuzztest/ossfuzz.sh vendored Executable file
View file

@ -0,0 +1,45 @@
#!/bin/bash -eu
# Go to tests folder
cd "$( dirname "${BASH_SOURCE[0]}" )/.."
# Build seed corpus.
# Generating it here ensures it will contain all of the fields in the AllTypes
# test case. The generators are built without fuzzing instrumentation.
rm -rf build
scons build/alltypes/encode_alltypes build/fuzztest/generate_message
mkdir fuzztest_seed_corpus
build/alltypes/encode_alltypes 0 > fuzztest_seed_corpus/alltypes0
build/alltypes/encode_alltypes 1 > fuzztest_seed_corpus/alltypes1
build/alltypes/encode_alltypes 2 > fuzztest_seed_corpus/alltypes2
build/fuzztest/generate_message $(date +%s) > fuzztest_seed_corpus/rndmsg 2>/dev/null
for f in fuzztest_seed_corpus/*; do
mv $f fuzztest_seed_corpus/$(sha1sum $f | cut -f 1 -d ' ')
done
zip -r "$OUT/corpus.zip" fuzztest_seed_corpus
# Build the fuzz testing stubs with instrumentation
rm -rf build
FUZZERS="build/fuzztest/fuzztest_proto2_static
build/fuzztest/fuzztest_proto2_pointer
build/fuzztest/fuzztest_proto3_static
build/fuzztest/fuzztest_proto3_pointer
build/fuzztest/fuzztest_io_errors"
scons CC="$CC" CXX="$CXX" LINK="$CXX" \
CCFLAGS="-Wall -Wextra -g -DLLVMFUZZER $CFLAGS" \
CXXFLAGS="-Wall -Wextra -g -DLLVMFUZZER $CXXFLAGS" \
NODEFARGS="1" \
LINKFLAGS="-std=c++11 $CXXFLAGS" \
LINKLIBS="$LIB_FUZZING_ENGINE" $FUZZERS
cp $FUZZERS "$OUT"
# The fuzzer test cases are closely related, so use the same seed corpus
# for all of them.
for fuzzer in $FUZZERS
do cp "$OUT/corpus.zip" "$OUT/$(basename $fuzzer)_seed_corpus.zip"
done
rm "$OUT/corpus.zip"

View file

@ -0,0 +1,198 @@
#include "random_data.h"
#include <string.h>
#include <malloc_wrappers.h>
#include <pb_encode.h>
#ifndef LLVMFUZZER
static uint32_t g_random_seed = 1234;
void random_set_seed(uint32_t seed)
{
g_random_seed = seed;
}
uint32_t random_get_seed()
{
return g_random_seed;
}
/* Uses xorshift64 here instead of rand() for both speed and
* reproducibility across platforms. */
uint32_t rand_word()
{
g_random_seed ^= g_random_seed << 13;
g_random_seed ^= g_random_seed >> 17;
g_random_seed ^= g_random_seed << 5;
return g_random_seed;
}
/* Get a random integer in range, with approximately flat distribution. */
int rand_int(int min, int max)
{
return rand_word() % (max + 1 - min) + min;
}
bool rand_bool()
{
return rand_word() & 1;
}
/* Get a random byte, with skewed distribution.
* Important corner cases like 0xFF, 0x00 and 0xFE occur more
* often than other values. */
uint8_t rand_byte()
{
uint32_t w = rand_word();
uint8_t b = w & 0xFF;
if (w & 0x100000)
b >>= (w >> 8) & 7;
if (w & 0x200000)
b <<= (w >> 12) & 7;
if (w & 0x400000)
b ^= 0xFF;
return b;
}
/* Get a random length, with skewed distribution.
* Favors the shorter lengths, but always at least 1. */
size_t rand_len(size_t max)
{
uint32_t w = rand_word();
size_t s;
if (w & 0x800000)
w &= 3;
else if (w & 0x400000)
w &= 15;
else if (w & 0x200000)
w &= 255;
s = (w % max);
if (s == 0)
s = 1;
return s;
}
/* Fills a buffer with random data with skewed distribution. */
void rand_fill(uint8_t *buf, size_t count)
{
for (; count > 0; count--)
{
*buf++ = rand_byte();
}
}
/* Fill with random protobuf-like data */
size_t rand_fill_protobuf(uint8_t *buf, size_t min_bytes, size_t max_bytes, int min_tag)
{
pb_ostream_t stream = pb_ostream_from_buffer(buf, max_bytes);
while(stream.bytes_written < min_bytes)
{
pb_wire_type_t wt = rand_int(0, 3);
if (wt == 3) wt = 5; /* Gap in values */
if (!pb_encode_tag(&stream, wt, rand_int(min_tag, min_tag + 512)))
break;
if (wt == PB_WT_VARINT)
{
uint64_t value;
rand_fill((uint8_t*)&value, sizeof(value));
pb_encode_varint(&stream, value);
}
else if (wt == PB_WT_64BIT)
{
uint64_t value;
rand_fill((uint8_t*)&value, sizeof(value));
pb_encode_fixed64(&stream, &value);
}
else if (wt == PB_WT_32BIT)
{
uint32_t value;
rand_fill((uint8_t*)&value, sizeof(value));
pb_encode_fixed32(&stream, &value);
}
else if (wt == PB_WT_STRING)
{
size_t len;
uint8_t *buf;
if (min_bytes > stream.bytes_written)
len = rand_len(min_bytes - stream.bytes_written);
else
len = 0;
buf = malloc(len);
pb_encode_varint(&stream, len);
rand_fill(buf, len);
pb_write(&stream, buf, len);
free(buf);
}
}
return stream.bytes_written;
}
/* Given a buffer of data, mess it up a bit */
void rand_mess(uint8_t *buf, size_t count)
{
int m = rand_int(0, 3);
if (m == 0)
{
/* Replace random substring */
int s = rand_int(0, count - 1);
int l = rand_len(count - s);
rand_fill(buf + s, l);
}
else if (m == 1)
{
/* Swap random bytes */
int a = rand_int(0, count - 1);
int b = rand_int(0, count - 1);
int x = buf[a];
buf[a] = buf[b];
buf[b] = x;
}
else if (m == 2)
{
/* Duplicate substring */
int s = rand_int(0, count - 2);
int l = rand_len((count - s) / 2);
memcpy(buf + s + l, buf + s, l);
}
else if (m == 3)
{
/* Add random protobuf noise */
int s = rand_int(0, count - 1);
int l = rand_len(count - s);
rand_fill_protobuf(buf + s, l, count - s, 1);
}
}
/* Append or prepend protobuf noise */
void rand_protobuf_noise(uint8_t *buffer, size_t bufsize, size_t *msglen)
{
int m = rand_int(0, 2);
size_t max_size = bufsize - 32 - *msglen;
if (m == 1)
{
/* Prepend */
uint8_t *tmp = malloc_with_check(bufsize);
size_t s = rand_fill_protobuf(tmp, rand_len(max_size), bufsize - *msglen, 1000);
memmove(buffer + s, buffer, *msglen);
memcpy(buffer, tmp, s);
free_with_check(tmp);
*msglen += s;
}
else if (m == 2)
{
/* Append */
size_t s = rand_fill_protobuf(buffer + *msglen, rand_len(max_size), bufsize - *msglen, 1000);
*msglen += s;
}
}
#endif

View file

@ -0,0 +1,44 @@
/* This module handles generating & modifying messages randomly for the fuzz test. */
#ifndef RANDOM_DATA_H
#define RANDOM_DATA_H
#include <pb.h>
void random_set_seed(uint32_t seed);
uint32_t random_get_seed();
/* Random 32-bit integer */
uint32_t rand_word();
/* Get a random integer in range, with approximately flat distribution. */
int rand_int(int min, int max);
/* Random boolean, equal probability */
bool rand_bool();
/* Get a random byte, with skewed distribution.
* Important corner cases like 0xFF, 0x00 and 0xFE occur more
* often than other values. */
uint8_t rand_byte();
/* Get a random length, with skewed distribution.
* Favors the shorter lengths, but always at least 1. */
size_t rand_len(size_t max);
/* Fills a buffer with random bytes with skewed distribution. */
void rand_fill(uint8_t *buf, size_t count);
/* Fill with random protobuf-like data */
size_t rand_fill_protobuf(uint8_t *buf, size_t min_bytes, size_t max_bytes, int min_tag);
/* Given a buffer of data, mess it up a bit by copying / swapping bytes */
void rand_mess(uint8_t *buf, size_t count);
/* Append or prepend protobuf noise, with tag values > 1000 */
void rand_protobuf_noise(uint8_t *buffer, size_t bufsize, size_t *msglen);
#endif

Binary file not shown.

Binary file not shown.

Binary file not shown.

View file

@ -0,0 +1,47 @@
#!/bin/bash
# This script is used to update corpus.zip and regressions.zip from the
# Google oss-fuzz project. To actually run this requires access rights to the
# nanopb oss-fuzz storage.
#
# The oss-fuzz project uses separate fuzzer test cases for better performance
# with coverage based fuzzers. This script merges the corpus to a single one
# for the combined fuzztest test case.
set -x
set -e
mkdir tmp-corpusupdate
cd tmp-corpusupdate
gsutil cp gs://nanopb-backup.clusterfuzz-external.appspot.com/corpus/libFuzzer/nanopb_fuzztest_proto2_static/latest.zip proto2_static.zip
gsutil cp gs://nanopb-backup.clusterfuzz-external.appspot.com/corpus/libFuzzer/nanopb_fuzztest_proto2_pointer/latest.zip proto2_pointer.zip
gsutil cp gs://nanopb-backup.clusterfuzz-external.appspot.com/corpus/libFuzzer/nanopb_fuzztest_proto3_static/latest.zip proto3_static.zip
gsutil cp gs://nanopb-backup.clusterfuzz-external.appspot.com/corpus/libFuzzer/nanopb_fuzztest_proto3_pointer/latest.zip proto3_pointer.zip
gsutil cp gs://nanopb-backup.clusterfuzz-external.appspot.com/corpus/libFuzzer/nanopb_fuzztest_io_errors/latest.zip io_errors.zip
unzip -nd corpus ../corpus.zip # Unzip old corpus
unzip -nd new_corpus proto2_static.zip
unzip -nd new_corpus proto2_pointer.zip
unzip -nd new_corpus proto3_static.zip
unzip -nd new_corpus proto3_pointer.zip
unzip -nd new_corpus io_errors.zip
# Build fuzztest with libfuzzer to merge corpuses
CCFLAGS="-DLLVMFUZZER -fsanitize=address -fsanitize-address-use-after-scope -fsanitize=fuzzer-no-link"
LINKFLAGS="-std=c++11 -O1 -fsanitize=address -fsanitize-address-use-after-scope -fsanitize=fuzzer-no-link -stdlib=libc++"
LINKLIBS="-fsanitize=fuzzer"
scons -u CC=clang LINK=clang++ \
BUILDDIR=fuzztest/tmp-corpusupdate/build build/fuzztest/fuzztest \
"CCFLAGS=$CCFLAGS" "LINKFLAGS=$LINKFLAGS" "LINKLIBS=$LINKLIBS"
# Copy any files with new features into corpus directory
build/fuzztest/fuzztest corpus new_corpus -merge=1
# Add files to end of the zips. This should work relatively efficiently
# with gits binary diff feature.
(cd corpus; zip -u ../../corpus.zip ./*)
(cd new_corpus/regressions; zip -u ../../regressions.zip *)
cd ..
rm -rf tmp-corpusupdate

View file

@ -0,0 +1,163 @@
#include "validation.h"
#include "malloc_wrappers.h"
#include <pb_common.h>
#include <assert.h>
void validate_static(pb_field_iter_t *iter)
{
pb_size_t count = 1;
pb_size_t i;
bool truebool = true;
bool falsebool = false;
if (PB_HTYPE(iter->type) == PB_HTYPE_REPEATED && iter->pSize)
{
/* Array count must be between 0 and statically allocated size */
count = *(pb_size_t*)iter->pSize;
assert(count <= iter->array_size);
}
else if (PB_HTYPE(iter->type) == PB_HTYPE_OPTIONAL && iter->pSize)
{
/* Boolean has_ field must have a valid value */
assert(memcmp(iter->pSize, &truebool, sizeof(bool)) == 0 ||
memcmp(iter->pSize, &falsebool, sizeof(bool)) == 0);
}
else if (PB_HTYPE(iter->type) == PB_HTYPE_ONEOF)
{
if (*(pb_size_t*)iter->pSize != iter->tag)
{
/* Some different field in oneof */
return;
}
}
for (i = 0; i < count; i++)
{
void *pData = (char*)iter->pData + iter->data_size * i;
if (PB_LTYPE(iter->type) == PB_LTYPE_STRING)
{
/* String length must be at most statically allocated size */
assert(strlen(pData) + 1 <= iter->data_size);
}
else if (PB_LTYPE(iter->type) == PB_LTYPE_BYTES)
{
/* Bytes length must be at most statically allocated size */
pb_bytes_array_t *bytes = pData;
assert(PB_BYTES_ARRAY_T_ALLOCSIZE(bytes->size) <= iter->data_size);
}
else if (PB_LTYPE(iter->type) == PB_LTYPE_BOOL)
{
/* Bool fields must have valid value */
assert(memcmp(pData, &truebool, sizeof(bool)) == 0 ||
memcmp(pData, &falsebool, sizeof(bool)) == 0);
}
else if (PB_LTYPE_IS_SUBMSG(iter->type))
{
validate_message(pData, 0, iter->submsg_desc);
}
}
}
void validate_pointer(pb_field_iter_t *iter)
{
pb_size_t count = 1;
pb_size_t i;
bool truebool = true;
bool falsebool = false;
if (PB_HTYPE(iter->type) == PB_HTYPE_ONEOF)
{
if (*(pb_size_t*)iter->pSize != iter->tag)
{
/* Some different field in oneof */
return;
}
}
else if (!iter->pData)
{
/* Nothing allocated */
if (PB_HTYPE(iter->type) == PB_HTYPE_REPEATED && iter->pSize != &iter->array_size)
{
assert(*(pb_size_t*)iter->pSize == 0);
}
return;
}
if (PB_HTYPE(iter->type) == PB_HTYPE_REPEATED)
{
/* Check that enough memory has been allocated for array */
size_t allocated_size = get_allocation_size(iter->pData);
count = *(pb_size_t*)iter->pSize;
assert(allocated_size >= count * iter->data_size);
}
else if (PB_LTYPE(iter->type) != PB_LTYPE_STRING && PB_LTYPE(iter->type) != PB_LTYPE_BYTES)
{
size_t allocated_size = get_allocation_size(iter->pData);
assert(allocated_size >= iter->data_size);
}
for (i = 0; i < count; i++)
{
void *pData = (char*)iter->pData + iter->data_size * i;
if (PB_LTYPE(iter->type) == PB_LTYPE_STRING)
{
/* Check that enough memory is allocated for string and that
the string is properly terminated. */
const char *str = pData;
if (PB_HTYPE(iter->type) == PB_HTYPE_REPEATED)
{
/* String arrays are stored as array of pointers */
str = ((const char**)iter->pData)[i];
}
assert(strlen(str) + 1 <= get_allocation_size(str));
}
else if (PB_LTYPE(iter->type) == PB_LTYPE_BYTES)
{
/* Bytes length must be at most statically allocated size */
const pb_bytes_array_t *bytes = pData;
if (PB_HTYPE(iter->type) == PB_HTYPE_REPEATED)
{
/* Bytes arrays are stored as array of pointers */
bytes = ((const pb_bytes_array_t**)iter->pData)[i];
}
assert(PB_BYTES_ARRAY_T_ALLOCSIZE(bytes->size) <= get_allocation_size(bytes));
}
else if (PB_LTYPE(iter->type) == PB_LTYPE_BOOL)
{
/* Bool fields must have valid value */
assert(memcmp(pData, &truebool, sizeof(bool)) == 0 ||
memcmp(pData, &falsebool, sizeof(bool)) == 0);
}
else if (PB_LTYPE_IS_SUBMSG(iter->type))
{
validate_message(pData, 0, iter->submsg_desc);
}
}
}
void validate_message(const void *msg, size_t structsize, const pb_msgdesc_t *msgtype)
{
pb_field_iter_t iter;
if (pb_field_iter_begin_const(&iter, msgtype, msg))
{
do
{
if (PB_ATYPE(iter.type) == PB_ATYPE_STATIC)
{
validate_static(&iter);
}
else if (PB_ATYPE(iter.type) == PB_ATYPE_POINTER)
{
validate_pointer(&iter);
}
} while (pb_field_iter_next(&iter));
}
}

View file

@ -0,0 +1,12 @@
/* This module validates that the message structures are in valid state
* after decoding the input data. */
#ifndef VALIDATION_H
#define VALIDATION_H
#include <pb.h>
void validate_message(const void *msg, size_t structsize, const pb_msgdesc_t *msgtype);
#endif