serialization.h

Go to the documentation of this file.

#pragma once

 
#include <algorithm>
#include <array>
#include <bit>
#include <concepts>
#include <cstddef>
#include <cstdint>
#include <limits>
#include <memory>
#include <span>
#include <stdexcept>
#include <string>
#include <type_traits>
#include <utility>
#include <vector>
 
namespace pixie {

enum class DeserializationValidation : std::uint8_t {
  kQuick,

  kFull,
};

class SerializationError : public std::invalid_argument {
 public:
  SerializationError(std::string message, std::size_t byte_offset)
      : std::invalid_argument(std::move(message) + " at byte " +
                              std::to_string(byte_offset)),
        byte_offset_(byte_offset) {}

  std::size_t byte_offset() const noexcept { return byte_offset_; }
 
 private:
  std::size_t byte_offset_;
};

template <class Sink>
concept SeekableBinaryOutputSink = requires(Sink& sink,
                                            std::size_t position,
                                            std::span<const std::byte> bytes) {
  { sink.write(bytes) } -> std::same_as<void>;
  { sink.write_at(position, bytes) } -> std::same_as<void>;
  { sink.finish() } -> std::same_as<void>;
};

class VectorOutputSink {
 public:
  bool empty() const noexcept { return data_.empty(); }

  std::size_t size_bytes() const noexcept { return data_.size(); }

  std::span<const std::byte> bytes() const noexcept { return data_; }

  void reserve_bytes(std::size_t size_bytes) { data_.reserve(size_bytes); }

  void write(std::span<const std::byte> bytes) {
    if (bytes.size() > data_.max_size() - data_.size()) {
      throw std::length_error("Binary output is too large");
    }
    data_.insert(data_.end(), bytes.begin(), bytes.end());
  }

  void write_at(std::size_t position, std::span<const std::byte> bytes) {
    if (position > data_.size() || bytes.size() > data_.size() - position) {
      throw std::out_of_range("Binary output patch is outside the output");
    }
    std::ranges::copy(bytes, data_.begin() + position);
  }

  void finish() noexcept {}

  std::vector<std::byte> take() { return std::exchange(data_, {}); }
 
 private:
  std::vector<std::byte> data_;
};
 
static_assert(SeekableBinaryOutputSink<VectorOutputSink>);

class SpanOutputSink {
 public:
  explicit SpanOutputSink(std::span<std::byte> storage) : storage_(storage) {}

  bool empty() const noexcept { return size_bytes_ == 0; }

  std::size_t size_bytes() const noexcept { return size_bytes_; }

  std::size_t capacity_bytes() const noexcept { return storage_.size(); }

  std::span<const std::byte> bytes() const noexcept {
    return storage_.first(size_bytes_);
  }

  void write(std::span<const std::byte> bytes) {
    if (bytes.size() > storage_.size() - size_bytes_) {
      throw std::length_error("Fixed binary output capacity exceeded");
    }
    std::ranges::copy(bytes, storage_.begin() + size_bytes_);
    size_bytes_ += bytes.size();
  }

  void write_at(std::size_t position, std::span<const std::byte> bytes) {
    if (position > size_bytes_ || bytes.size() > size_bytes_ - position) {
      throw std::out_of_range("Binary output patch is outside the output");
    }
    std::ranges::copy(bytes, storage_.begin() + position);
  }

  void finish() noexcept {}
 
 private:
  std::span<std::byte> storage_;
  std::size_t size_bytes_ = 0;
};
 
static_assert(SeekableBinaryOutputSink<SpanOutputSink>);

class BinaryWriter {
 public:
  static constexpr std::size_t kDefaultBufferBytes = 64 * 1024;

  template <SeekableBinaryOutputSink Sink>
  explicit BinaryWriter(Sink& sink,
                        std::size_t buffer_bytes = kDefaultBufferBytes)
      : owned_buffer_(allocate_buffer(buffer_bytes)),
        buffer_(owned_buffer_.get(), buffer_bytes) {
    bind(sink);
  }

  template <SeekableBinaryOutputSink Sink>
  BinaryWriter(Sink& sink, std::span<std::byte> buffer) : buffer_(buffer) {
    if (buffer.empty()) {
      throw std::invalid_argument(
          "BinaryWriter staging buffer must be non-empty");
    }
    bind(sink);
  }
 
  BinaryWriter(const BinaryWriter&) = delete;
  BinaryWriter& operator=(const BinaryWriter&) = delete;
  BinaryWriter(BinaryWriter&&) = delete;
  BinaryWriter& operator=(BinaryWriter&&) = delete;

  bool empty() const noexcept { return size_bytes() == 0; }

  std::size_t size_bytes() const noexcept {
    return flushed_bytes_ + buffered_bytes_;
  }

  std::size_t buffer_size_bytes() const noexcept { return buffer_.size(); }

  void write_u8(std::uint8_t value) { write_unsigned(value); }

  void write_u16(std::uint16_t value) { write_unsigned(value); }

  void write_u32(std::uint32_t value) { write_unsigned(value); }

  void write_u64(std::uint64_t value) { write_unsigned(value); }

  void write_i8(std::int8_t value) {
    write_unsigned(std::bit_cast<std::uint8_t>(value));
  }

  void write_i16(std::int16_t value) {
    write_unsigned(std::bit_cast<std::uint16_t>(value));
  }

  void write_i32(std::int32_t value) {
    write_unsigned(std::bit_cast<std::uint32_t>(value));
  }

  void write_i64(std::int64_t value) {
    write_unsigned(std::bit_cast<std::uint64_t>(value));
  }

  void write_size(std::size_t value) {
    if constexpr (sizeof(std::size_t) > sizeof(std::uint64_t)) {
      if (value > std::numeric_limits<std::uint64_t>::max()) {
        throw std::length_error("Size does not fit in the wire format");
      }
    }
    write_u64(static_cast<std::uint64_t>(value));
  }

  void write_bytes(std::span<const std::byte> bytes) {
    require_open();
    require_output_size(bytes.size());
    while (!bytes.empty()) {
      if (buffered_bytes_ != 0) {
        const std::size_t count =
            std::min(bytes.size(), buffer_.size() - buffered_bytes_);
        std::ranges::copy(bytes.first(count),
                          buffer_.begin() + buffered_bytes_);
        buffered_bytes_ += count;
        bytes = bytes.subspan(count);
        if (buffered_bytes_ == buffer_.size()) {
          flush_buffer();
        }
        continue;
      }
 
      if (bytes.size() >= buffer_.size()) {
        const std::size_t count = bytes.size();
        write_to_sink(bytes);
        flushed_bytes_ += count;
        return;
      }
 
      std::ranges::copy(bytes, buffer_.begin());
      buffered_bytes_ = bytes.size();
      return;
    }
  }

  void write_zeros(std::size_t count) {
    require_open();
    require_output_size(count);
    while (count != 0) {
      const std::size_t writable = buffer_.size() - buffered_bytes_;
      const std::size_t chunk = std::min(count, writable);
      std::fill_n(buffer_.begin() + buffered_bytes_, chunk, std::byte{0});
      buffered_bytes_ += chunk;
      count -= chunk;
      if (buffered_bytes_ == buffer_.size()) {
        flush_buffer();
      }
    }
  }

  void align_to(std::size_t alignment) {
    require_open();
    if (alignment == 0) {
      throw std::invalid_argument("Serialization alignment must be non-zero");
    }
    const std::size_t remainder = size_bytes() % alignment;
    if (remainder != 0) {
      write_zeros(alignment - remainder);
    }
  }

  std::size_t write_u64_placeholder() {
    const std::size_t position = size_bytes();
    write_u64(0);
    return position;
  }

  void patch_u64(std::size_t position, std::uint64_t value) {
    require_open();
    if (position > size_bytes() || size_bytes() - position < sizeof(value)) {
      throw std::out_of_range("Serialization patch is outside the output");
    }
 
    std::array<std::byte, sizeof(value)> encoded{};
    for (std::size_t byte = 0; byte < sizeof(value); ++byte) {
      encoded[byte] = static_cast<std::byte>((value >> (byte * 8)) & 0xffu);
    }
 
    if (position >= flushed_bytes_) {
      const std::size_t buffered_position = position - flushed_bytes_;
      std::ranges::copy(encoded, buffer_.begin() + buffered_position);
      return;
    }
 
    if (position + encoded.size() > flushed_bytes_) {
      flush_buffer();
    }
    write_at_sink(position, encoded);
  }

  void flush() {
    require_open();
    flush_buffer();
  }

  void finish() {
    if (state_ == State::kFinished) {
      return;
    }
    require_open();
    flush_buffer();
    try {
      finish_sink_(sink_);
      state_ = State::kFinished;
    } catch (...) {
      state_ = State::kFailed;
      throw;
    }
  }
 
 private:
  enum class State { kOpen, kFinished, kFailed };
 
  using WriteSink = void (*)(void*, std::span<const std::byte>);
  using WriteAtSink = void (*)(void*, std::size_t, std::span<const std::byte>);
  using FinishSink = void (*)(void*);
 
  static std::unique_ptr<std::byte[]> allocate_buffer(
      std::size_t buffer_bytes) {
    if (buffer_bytes == 0) {
      throw std::invalid_argument(
          "BinaryWriter staging buffer must be non-empty");
    }
    return std::make_unique<std::byte[]>(buffer_bytes);
  }
 
  template <SeekableBinaryOutputSink Sink>
  void bind(Sink& sink) noexcept {
    sink_ = std::addressof(sink);
    write_sink_ = [](void* output, std::span<const std::byte> bytes) {
      static_cast<Sink*>(output)->write(bytes);
    };
    write_at_sink_ = [](void* output, std::size_t position,
                        std::span<const std::byte> bytes) {
      static_cast<Sink*>(output)->write_at(position, bytes);
    };
    finish_sink_ = [](void* output) { static_cast<Sink*>(output)->finish(); };
  }
 
  void require_open() const {
    if (state_ == State::kFinished) {
      throw std::logic_error("BinaryWriter is already finished");
    }
    if (state_ == State::kFailed) {
      throw std::logic_error("BinaryWriter is unusable after sink failure");
    }
  }
 
  void require_output_size(std::size_t count) const {
    if (count > std::numeric_limits<std::size_t>::max() - size_bytes()) {
      throw std::length_error("Serialized output is too large");
    }
  }
 
  void flush_buffer() {
    if (buffered_bytes_ == 0) {
      return;
    }
    const std::size_t count = buffered_bytes_;
    write_to_sink(buffer_.first(count));
    flushed_bytes_ += count;
    buffered_bytes_ = 0;
  }
 
  void write_to_sink(std::span<const std::byte> bytes) {
    try {
      write_sink_(sink_, bytes);
    } catch (...) {
      state_ = State::kFailed;
      throw;
    }
  }
 
  void write_at_sink(std::size_t position, std::span<const std::byte> bytes) {
    try {
      write_at_sink_(sink_, position, bytes);
    } catch (...) {
      state_ = State::kFailed;
      throw;
    }
  }
 
  template <std::unsigned_integral T>
  void write_unsigned(T value) {
    std::array<std::byte, sizeof(T)> encoded{};
    for (std::size_t byte = 0; byte < sizeof(T); ++byte) {
      encoded[byte] = static_cast<std::byte>((value >> (byte * 8)) & T{0xff});
    }
    write_bytes(encoded);
  }
 
  void* sink_ = nullptr;
  WriteSink write_sink_ = nullptr;
  WriteAtSink write_at_sink_ = nullptr;
  FinishSink finish_sink_ = nullptr;
  std::unique_ptr<std::byte[]> owned_buffer_;
  std::span<std::byte> buffer_;
  std::size_t flushed_bytes_ = 0;
  std::size_t buffered_bytes_ = 0;
  State state_ = State::kOpen;
};

class BinaryReader {
 public:
  explicit BinaryReader(std::span<const std::byte> data) : data_(data) {}

  bool empty() const noexcept { return remaining() == 0; }

  std::size_t size_bytes() const noexcept { return data_.size(); }

  std::size_t position() const noexcept { return offset_; }

  std::size_t byte_offset() const noexcept { return absolute_position(); }

  std::size_t remaining() const noexcept { return data_.size() - offset_; }

  std::span<const std::byte> remaining_bytes() const noexcept {
    return data_.subspan(offset_);
  }

  std::uint8_t read_u8() { return read_unsigned<std::uint8_t>(); }

  std::uint16_t read_u16() { return read_unsigned<std::uint16_t>(); }

  std::uint32_t read_u32() { return read_unsigned<std::uint32_t>(); }

  std::uint64_t read_u64() { return read_unsigned<std::uint64_t>(); }

  std::int8_t read_i8() {
    return std::bit_cast<std::int8_t>(read_unsigned<std::uint8_t>());
  }

  std::int16_t read_i16() {
    return std::bit_cast<std::int16_t>(read_unsigned<std::uint16_t>());
  }

  std::int32_t read_i32() {
    return std::bit_cast<std::int32_t>(read_unsigned<std::uint32_t>());
  }

  std::int64_t read_i64() {
    return std::bit_cast<std::int64_t>(read_unsigned<std::uint64_t>());
  }

  std::size_t read_size() {
    BinaryReader candidate = *this;
    const std::size_t field_offset = candidate.absolute_position();
    const std::uint64_t encoded = candidate.read_u64();
    if (encoded > std::numeric_limits<std::size_t>::max()) {
      throw std::length_error("Serialized size at byte " +
                              std::to_string(field_offset) +
                              " does not fit in size_t");
    }
    *this = candidate;
    return static_cast<std::size_t>(encoded);
  }

  std::span<const std::byte> read_bytes(std::size_t count) {
    require(count);
    const std::span<const std::byte> result = data_.subspan(offset_, count);
    offset_ += count;
    return result;
  }

  BinaryReader read_subreader(std::size_t count) {
    const std::size_t origin = absolute_position();
    return BinaryReader(read_bytes(count), origin);
  }

  void skip(std::size_t count) {
    require(count);
    offset_ += count;
  }

  void require_zero_padding(std::size_t maximum) {
    if (remaining() > maximum) {
      throw SerializationError("Unexpected serialized payload bytes",
                               absolute_position());
    }
    BinaryReader candidate = *this;
    while (!candidate.empty()) {
      const std::size_t byte_offset = candidate.absolute_position();
      if (candidate.read_u8() != 0) {
        throw SerializationError("Non-zero serialized payload padding",
                                 byte_offset);
      }
    }
    *this = candidate;
  }
 
 private:
  BinaryReader(std::span<const std::byte> data, std::size_t origin)
      : data_(data), origin_(origin) {}
 
  std::size_t absolute_position() const noexcept { return origin_ + offset_; }
 
  void require(std::size_t count) const {
    if (count > remaining()) {
      throw SerializationError("Truncated serialized input",
                               absolute_position());
    }
  }
 
  template <std::unsigned_integral T>
  T read_unsigned() {
    require(sizeof(T));
    T value = 0;
    for (std::size_t byte = 0; byte < sizeof(T); ++byte) {
      value |=
          static_cast<T>(std::to_integer<std::uint8_t>(data_[offset_ + byte]))
          << (byte * 8);
    }
    offset_ += sizeof(T);
    return value;
  }
 
  std::span<const std::byte> data_;
  std::size_t offset_ = 0;
  std::size_t origin_ = 0;
};

template <class Derived>
class SerializationBase {
 public:
  void serialize(BinaryWriter& writer) const
    requires requires(const Derived& value, BinaryWriter& output) {
      { value.serialize_impl(output) } -> std::same_as<void>;
    }
  {
    derived().serialize_impl(writer);
  }

  template <class... Context>
  static Derived deserialize(BinaryReader& reader, Context&&... context)
    requires requires(BinaryReader& input, Context&&... arguments) {
      {
        Derived::deserialize_impl(input, std::forward<Context>(arguments)...)
      } -> std::same_as<Derived>;
    }
  {
    BinaryReader candidate = reader;
    Derived result =
        Derived::deserialize_impl(candidate, std::forward<Context>(context)...);
    reader = candidate;
    return result;
  }

  template <class... Context>
  static Derived deserialize(std::span<const std::byte>& data,
                             Context&&... context)
    requires requires(BinaryReader& input, Context&&... arguments) {
      {
        Derived::deserialize_impl(input, std::forward<Context>(arguments)...)
      } -> std::same_as<Derived>;
    }
  {
    BinaryReader reader(data);
    Derived result = deserialize(reader, std::forward<Context>(context)...);
    data = data.subspan(reader.position());
    return result;
  }
 
 private:
  const Derived& derived() const { return static_cast<const Derived&>(*this); }
};

template <class T>
concept Serializable = requires(const T& value, BinaryWriter& writer) {
  { value.serialize(writer) } -> std::same_as<void>;
};

template <class T, class... Context>
concept Deserializable = requires(BinaryReader& reader, Context&&... context) {
  {
    T::deserialize(reader, std::forward<Context>(context)...)
  } -> std::same_as<T>;
};
 
}  // namespace pixie