Skip to content

Instantly share code, notes, and snippets.

@RandyGaul

RandyGaul/serialize.h

Last active January 14, 2026 18:06
Show Gist options

  • Select an option

    Learn more about clone URLs
  • Save RandyGaul/b65cea57a8de7f18de970084a4480dd7 to your computer and use it in GitHub Desktop.

Select an option

Learn more about clone URLs
Save RandyGaul/b65cea57a8de7f18de970084a4480dd7 to your computer and use it in GitHub Desktop.
Download ZIP
SV - Save Version - Saving versioned binary data
Raw
serialize.h
// SV - Save Version
// This file has an API for saving binary data with versioning support for backwards compatibility.
// Original API design by Media Molecule.
// See: https://gist.githubusercontent.com/OswaldHurlem/4810ad510669097db872c6de305c9df0/raw/2fdf47eead527e954d29950aa41debf34547e5bd/mmalex_serialization_and_formats.log
//
// Design specs:
// + Very fast reads/writes
// + Backwards compat
// - Not self-describing (serializes opaque data)
// + The code itself describes the data, and versioning, all in one place
// - Not forward tolerant (cannot open newer data version with older application)
// - Uncompressed in serialized form (can do this with an external tool trivially)
// - Doesn't scale well to double digit team size (basically: merge conflicts)
// + Extremely simple (just ~500 loc here, including comments)
// - Slightly error prone with versioning (but simple to notice + fix)
//
// Steps to use:
// 1. Add the type you want to serialize to SV_TYPES table, or to SV_MEMCPY_SAFE_TYPES.
// - Note: SV_MEMCPY_SAFE_TYPES can only be used on types you *will not change*. There is *no*
// backwards compatibility for these types, but, as an optimization they will get memcpy'd
// to disk in a single call, even for arrays of this type. Recommended for vector, transform,
// or other fundamental types that will never change.
//
// 2. Increment the version enum. You will increment this enum every single time you modify a serialization
// routine. This is how the versioning system works -- with a global monotincally incrementing version id.
//
// 3. Create the serialization routine for your type.
// Example:
//
// typedef struct ExampleData
// {
// int a;
// float b;
// const char* c;
// };
//
// SV_SERIALIZE(ExampleData)
// {
// SV_ADD(SV_ADDED_EXAMPLE_DATA, a);
// SV_ADD(SV_ADDED_EXAMPLE_DATA, b);
// SV_ADD(SV_ADDED_EXAMPLE_DATA, c);
// }
//
// Where `SV_ADDED_EXAMPLE_DATA` is the new enum version when `ExampleData` was created. Increment it:
//
// enum
// {
// SV_INITIAL,
// SV_ADDED_EXAMPLE_DATA, // <-- Newly added.
// // --
// SV_LATEST_PLUS_ONE
// };
//
// Then, if we want to remove a member, we use `SV_REM`:
//
// typedef struct ExampleData
// {
// int a;
// float b;
// // const char* c; <-- Removed in SV_REMOVE_C_FROM_EXAMPLE_DATA.
// };
//
// SV_SERIALIZE(ExampleData)
// {
// SV_ADD(SV_ADDED_EXAMPLE_DATA, a);
// SV_ADD(SV_ADDED_EXAMPLE_DATA, b);
// //SV_ADD(SV_ADDED_EXAMPLE_DATA, c);
// SV_REM(SV_ADDED_EXAMPLE_DATA, SV_REMOVE_C_FROM_EXAMPLE_DATA, const char*, c, NULL);
// // If-needed you can use local variable c for migrating to new fields later (not needed here).
// }
//
// If you want to migrate an old value to something new, you can write whatever custom code you want to
// do so (you have access to the object `o` being serialized):
//
// typedef struct ExampleData
// {
// int a;
// float b;
// // const char* c; <-- Removed.
// const char* d;
// };
//
// SV_SERIALIZE(ExampleData)
// {
// SV_ADD(SV_ADDED_EXAMPLE_DATA, a);
// SV_ADD(SV_ADDED_EXAMPLE_DATA, b);
// //SV_ADD(SV_ADDED_EXAMPLE_DATA, c);
// SV_REM(SV_ADDED_EXAMPLE_DATA, SV_REMOVE_C_FROM_EXAMPLE_DATA, const char*, c, NULL);
// o->d = c; // Copy over the missing field (essentially just renaming c to d).
// SV_ADD(SV_ADDED_EXAMPLE_DATA_D, d);
// }
//
// Usually that's it! But, if you want to create a new kind of file:
//
// 4. Open a file with either `SV_SAVE_BEGIN` or `SV_LOAD_BEGIN`. Call `SV_ADD_LOCAL` to recursively serialize.
// When done, `call `SV_SAVE_END` or `SV_LOAD_END`.
// Example:
//
// void save(const char* path, ExampleData data)
// {
// SV_SAVE_BEGIN(path);
//
// SV_ADD_LOCAL(SV_ADDED_EXAMPLE_DATA, data);
//
// SV_SAVE_END();
// }
//
// Other rules:
// - All strings coming out of the serialization layer (when reading) are unique stable strings from
// `sintern` string interning.
// - If you use `SV_ADD_LIST` objects will be allocated with calloc (cleared to zero) and handed back
// to you. You should design your object for valid zero-initialization to play nicely here, or, run
// initialization after serialization. Otherwise, redesign your data layout to flattened arrays and
// simply call `SV_ADD_ARRAY`.
//--------------------------------------------------------------------------------------------------
// Public API.
// Global version id. Increment this each time you alter any serialiation routines.
// ...Add to the end, but make sure `SV_LATEST_PLUS_ONE` is last.
enum
{
SV_INITIAL,
SV_SIMPLIFIED_PROXY, // Removed pal_slot, sprite, bb from Proxy (now recreated from MAKE_PROXY on load).
SV_ADD_ROOM_PALETTES, // Added tile_palette to RoomFile.
SV_ADD_GEM_INVENTORY, // Added GemInventory for player save data.
SV_AUTOTILE, // Replaced Tile with SolidTile, added tilesets array to RoomFile.
// --
SV_LATEST_PLUS_ONE
};
#define SV_LATEST (SV_LATEST_PLUS_ONE - 1)
// @JANK - Forward typedefs for compilation simplicity. Would be nice to find another way.
typedef struct RoomFile RoomFile;
typedef struct Tile Tile;
typedef struct SolidTile SolidTile;
typedef struct GemInventory GemInventory;
// Extend this table whenever you add a new type to serialize.
// ...For any type that's memcpy safe add it below to the `SV_MEMCPY_SAFE_TYPES` table.
#define SV_TYPES(X) \
X(Var) \
X(Property) \
X(CF_Sprite) \
X(Proxy) \
X(RoomFile) \
X(Tile) \
X(SolidTile) \
X(GemInventory) \
// As an optimization we place memcpy-safe types here.
#define SV_MEMCPY_SAFE_TYPES(X) \
X(CF_Aabb) \
X(CF_V2) \
// Intended use pattern:
// void save(const char* path, ExampleData data)
// {
// SV_SAVE_BEGIN(path);
//
// SV_ADD_LOCAL(SV_ADDED_EXAMPLE_DATA, data);
//
// SV_SAVE_END();
// }
#define SV_SAVE_BEGIN(path) SV_Context ctx = sv_make(path, true), *S = &ctx;
#define SV_SAVE_END() sv_destroy(S)
#define SV_LOAD_BEGIN(path) SV_Context ctx = sv_make(path, false), *S = &ctx;
#define SV_LOAD_END() sv_destroy(S)
// Add a struct member.
#define SV_ADD(VERSION, MEMBER) \
do { \
if (S->saving) { \
SV_WRITE(o->MEMBER); \
} else if (S->loading && S->version >= VERSION) { \
SV_READ(o->MEMBER); \
} \
} while (0)
// Add a local variable.
// ...Does not support arrays.
#define SV_ADD_LOCAL(VERSION, LOCAL_VAR) \
do { \
if (S->saving) { \
SV_WRITE(LOCAL_VAR); \
} else if (S->loading && S->version >= VERSION) { \
SV_READ(LOCAL_VAR); \
} \
} while (0)
// Add an array (as a struct member).
#define SV_ADD_ARRAY(VERSION, ARRAY) \
do { \
if (S->saving || S->loading && S->version >= VERSION) { \
if (SV_IS_MEMCPY_SAFE(o->ARRAY)) { \
if (S->saving) { \
SV_WRITE(o->ARRAY); \
} else if (S->loading && S->version >= VERSION) { \
SV_READ(o->ARRAY); \
} \
} else { \
int n = asize(o->ARRAY); \
SV_ADD_LOCAL(VERSION, n); \
if (n) { \
if (S->loading) { \
afit(o->ARRAY, n); \
alen(o->ARRAY) = n; \
memset(o->ARRAY, 0, sizeof(o->ARRAY[0]) * n); \
} \
for (int i = 0; i < n; ++i) { \
if (S->saving) { \
SV_WRITE(o->ARRAY[i]); \
} else { \
SV_READ(o->ARRAY[i]); \
} \
} \
} \
} \
} \
} while (0)
// Add a local dynamic array.
#define SV_ADD_LOCAL_ARRAY(VERSION, ARRAY) \
do { \
if (S->saving || (S->loading && S->version >= (VERSION))) { \
if (SV_IS_MEMCPY_SAFE(ARRAY)) { \
if (S->saving) { \
SV_WRITE(ARRAY); \
} else { \
SV_READ(ARRAY); \
} \
} else { \
int n = asize(ARRAY); \
SV_ADD_LOCAL((VERSION), n); \
if (S->loading) { \
afit((ARRAY), n); \
alen((ARRAY)) = n; \
memset(o->ARRAY, 0, sizeof(o->ARRAY[0]) * n); \
} \
for (int i = 0; i < n; ++i) { \
if (S->saving) { \
SV_WRITE((ARRAY)[i]); \
} else { \
SV_READ((ARRAY)[i]); \
} \
} \
} \
} \
} while (0)
// Same as `SV_ADD_ARRAY` but works for arrays of pointers, such as: struct ObjectList { dyna Object** objects; };
#define SV_ADD_LIST(VERSION, ARRAY_OF_PTRS) \
do { \
int n = asize(o->ARRAY_OF_PTRS); \
SV_ADD_LOCAL(VERSION, n); \
afit(o->ARRAY_OF_PTRS, n); \
alen(o->ARRAY_OF_PTRS) = n; \
for (int i = 0; i < n; ++i) { \
if (S->loading) { \
void* v = CALLOC(o->ARRAY_OF_PTRS[0][0]); \
memcpy(o->ARRAY_OF_PTRS + i, &v, sizeof(void*)); \
} \
SV_ADD(VERSION, ARRAY_OF_PTRS[i][0]); \
} \
} while (0)
// Remove something from the serialization. This needs to increment the global version and passed in as `VERSION_REMOVED`.
// ...T is the type of the value removed.
// ...A local variable called `NAME` is created and data is read into it. You can then freely
// use this local variable to handle conversion to your newer format (if-needed).
// ...DEFAULT is a default value to use for the removed value in case it isn't present in the file version.
// ...Remove the prior `SV_ADD` for the removed data.
#define SV_REM(VERSION_ADDED, VERSION_REMOVED, T, NAME, DEFAULT) \
T NAME = (DEFAULT); \
if (S->loading && S->version >= VERSION_ADDED && S->version < VERSION_REMOVED) { \
SV_READ(NAME); \
}
// Define a serialization routine for a user struct.
#define SV_SERIALIZABLE(T) void serialize_##T(SV_Context* S, T* o)
// Optional sync check. Asserts if serialization is out of sync.
// ...Place this at the end of a `SV_SERIALIZABLE` routine to narrow down issues.
// ...Will assert when loading to catch (likely) bugs when saving.
// ...This adds an int counter to your struct, so you *must* bump global version enum.
// Common issues:
// - Missing ADD/REM
// - Wrong version specified
// - Field reordered/removed
// - Wrong array count/loop
// - Struct changed w/o bumping version
#define SV_SYNC() \
do { \
int e = S->sync_counter; \
if (S->saving) { SV_WRITE(e); } \
else { int g; SV_READ(g); assert(g == e); } \
S->sync_counter++; \
} while (0)
// Context struct passed through all serialization routines.
typedef struct SV_Context
{
int version;
bool saving;
bool loading;
const char* path;
File* file;
int sync_counter;
} SV_Context;
//--------------------------------------------------------------------------------------------------
// Private implementation details.
// Tells whether a type can be safe memcpy'd. Used for optimizing serialization for certain types.
// Arrays of these types can be dumped all at once.
#define SV_MEMCPY_TYPE(T) T: true,
#define SV_IS_MEMCPY_SAFE(T) \
_Generic(T, \
SV_MEMCPY_SAFE_TYPES(SV_MEMCPY_TYPE) \
default: false \
)
// Create declarations for all serializable types for compilation simplicity.
#define SV_SERIALIZE_DECL(T) void serialize_##T(struct SV_Context* S, T* o);
SV_TYPES(SV_SERIALIZE_DECL)
// Create definitions for functions to dump full arrays for the types in SV_MEMCPY_SAFE_TYPES.
#define SV_SERIALIZE_BY_MEMCPY(T) \
void serialize_##T(SV_Context* S, T* o) \
{ \
if (S->saving) write(S->file, o, sizeof(T)); \
else read(S->file, o, sizeof(T)); \
} \
void serialize_##T##_array(SV_Context* S, T** a) \
{ \
T* o = *a; \
if (S->saving) { \
int sz = asize(o) * sizeof(T); \
write(S->file, &sz, sizeof(sz)); \
write(S->file, o, sz); \
} else { \
int sz; \
read(S->file, &sz, sizeof(sz)); \
int n = sz / sizeof(T); \
afit(o, n); \
alen(o) = n; \
read(S->file, o, sz); \
} \
*a = o; \
}
SV_MEMCPY_SAFE_TYPES(SV_SERIALIZE_BY_MEMCPY)
// Internally used read/write function overloads for all types.
#define SV_TYPE(T) T: serialize_##T, T*: serialize_T_array_noop,
#define SV_MTYPE(T) T: serialize_##T, T*: serialize_##T##_array,
#define SV_READ(V) \
_Generic(V, \
SV_TYPES(SV_TYPE) \
SV_MEMCPY_SAFE_TYPES(SV_MTYPE) \
uint8_t: read_uint8, \
uint16_t: read_uint16, \
uint32_t: read_uint32, \
uint64_t: read_uint64, \
int8_t: read_int8, \
int16_t: read_int16, \
int32_t: read_int32, \
int64_t: read_int64, \
bool: read_bool, \
float: read_float, \
double: read_double, \
const char*: read_string, \
char*: read_string, \
const char**: read_string_noop, \
char**: read_string_noop \
)(S, &V)
#define SV_WRITE(V) \
_Generic(V, \
SV_TYPES(SV_TYPE) \
SV_MEMCPY_SAFE_TYPES(SV_MTYPE) \
uint8_t: write_uint8, \
uint16_t: write_uint16, \
uint32_t: write_uint32, \
uint64_t: write_uint64, \
int8_t: write_int8, \
int16_t: write_int16, \
int32_t: write_int32, \
int64_t: write_int64, \
bool: write_bool, \
float: write_float, \
double: write_double, \
const char*: write_string, \
char*: write_string, \
const char**: write_string_noop, \
char**: write_string_noop \
)(S, &V)
void serialize_T_array_noop(SV_Context* S, void* v) { UNUSED(S); UNUSED(v); assert(!"Here just to compile -- matches array semantics for user structs."); }
void read_uint8(SV_Context* S, uint8_t* v) { read(S->file, v, sizeof(*v)); }
void read_uint16(SV_Context* S, uint16_t* v) { read(S->file, v, sizeof(*v)); }
void read_uint32(SV_Context* S, uint32_t* v) { read(S->file, v, sizeof(*v)); }
void read_uint64(SV_Context* S, uint64_t* v) { read(S->file, v, sizeof(*v)); }
void read_int8(SV_Context* S, int8_t* v) { read(S->file, v, sizeof(*v)); }
void read_int16(SV_Context* S, int16_t* v) { read(S->file, v, sizeof(*v)); }
void read_int32(SV_Context* S, int32_t* v) { read(S->file, v, sizeof(*v)); }
void read_int64(SV_Context* S, int64_t* v) { read(S->file, v, sizeof(*v)); }
void read_bool(SV_Context* S, bool* v) { uint8_t t; read(S->file, &t, sizeof(t)); *v = t != 0; }
void read_float(SV_Context* S, float* v) { read(S->file, v, sizeof(*v)); }
void read_double(SV_Context* S, double* v) { read(S->file, v, sizeof(*v)); }
void read_string(SV_Context* S, const char** out) { uint32_t len; read_uint32(S, &len); dyna char* buf = atmp(char, len+1); read(S->file, buf, len); buf[len] = 0; *out = sintern(buf); afree(buf); }
void read_string_noop(SV_Context* S, const char*** out) { UNUSED(S); UNUSED(out); assert(!"SV_ADD_ARRAY performs array loop itself, this is just here to compile."); }
void write_uint8(SV_Context* S, const uint8_t* v) { write(S->file, v, sizeof(*v)); }
void write_uint16(SV_Context* S, const uint16_t* v) { write(S->file, v, sizeof(*v)); }
void write_uint32(SV_Context* S, const uint32_t* v) { write(S->file, v, sizeof(*v)); }
void write_uint64(SV_Context* S, const uint64_t* v) { write(S->file, v, sizeof(*v)); }
void write_int8(SV_Context* S, const int8_t* v) { write(S->file, v, sizeof(*v)); }
void write_int16(SV_Context* S, const int16_t* v) { write(S->file, v, sizeof(*v)); }
void write_int32(SV_Context* S, const int32_t* v) { write(S->file, v, sizeof(*v)); }
void write_int64(SV_Context* S, const int64_t* v) { write(S->file, v, sizeof(*v)); }
void write_bool(SV_Context* S, const bool* v) { uint8_t b = *v ? 1 : 0; write(S->file, &b, sizeof(b)); }
void write_float(SV_Context* S, const float* v) { write(S->file, v, sizeof(*v)); }
void write_double(SV_Context* S, const double* v) { write(S->file, v, sizeof(*v)); }
void write_string(SV_Context* S, const char** v) { uint32_t len = (uint32_t)strlen(*v); write_uint32(S, &len); write(S->file, *v, len); }
void write_string_noop(SV_Context* S, const char*** out) { UNUSED(S); UNUSED(out); assert(!"SV_ADD_ARRAY performs array loop itself, this is just here to compile."); }
SV_Context sv_make(const char* path, bool saving)
{
SV_Context ctx = { 0 };
SV_Context* S = &ctx;
ctx.saving = saving;
ctx.loading = !saving;
ctx.path = sintern(path);
if (saving) {
ctx.file = open_for_write(path);
assert(ctx.file);
ctx.version = SV_LATEST;
SV_WRITE(ctx.version);
} else {
ctx.file = open_for_read(path);
assert(ctx.file);
SV_READ(ctx.version);
}
return ctx;
}
void sv_destroy(SV_Context* S)
{
close(S->file);
}
//--------------------------------------------------------------------------------------------------
// Registration of some fundamental types.
SV_SERIALIZABLE(Var)
{
SV_ADD(SV_INITIAL, type);
switch (o->type) {
case VAR_TYPE_NONE: break;
case VAR_TYPE_INT: SV_ADD(SV_INITIAL, i); break;
case VAR_TYPE_FLOAT: SV_ADD(SV_INITIAL, f); break;
case VAR_TYPE_STRING: SV_ADD(SV_INITIAL, s); break;
case VAR_TYPE_BOOL: SV_ADD(SV_INITIAL, b); break;
}
}
SV_SERIALIZABLE(Property)
{
SV_ADD(SV_INITIAL, k);
SV_ADD(SV_INITIAL, v);
}
SV_SERIALIZABLE(CF_Sprite)
{
SV_ADD(SV_INITIAL, name);
if (S->loading) {
*o = cf_make_sprite(o->name);
}
}
SV_SERIALIZABLE(Proxy)
{
if (S->saving) {
if (!o->alive) return;
}
SV_ADD(SV_INITIAL, type);
SV_ADD(SV_INITIAL, position);
// These are now recreated from MAKE_PROXY on load.
SV_REM(SV_INITIAL, SV_SIMPLIFIED_PROXY, int, pal_slot, 0);
SV_REM(SV_INITIAL, SV_SIMPLIFIED_PROXY, CF_Sprite, sprite, (CF_Sprite){0});
SV_REM(SV_INITIAL, SV_SIMPLIFIED_PROXY, CF_Aabb, bb, (CF_Aabb){0});
SV_ADD_ARRAY(SV_INITIAL, properties);
}
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment