Add the SBE runner

Works for the test dataset, now time to make sure everyone agrees.

marketdata.xml

@@ -59,23 +59,26 @@
             -->
             <field name="timestamp" id="3" type="int64"/>
             <!--
-            SBE specifically doesn't have "union" types, so we include both a `trade` and `quote`
+            SBE specifically doesn't have "union" types, so we include a type tag, and both
-            here and the tag to identify.
+            `trade` and `quote` as "optional". This style was chosen to approximate how
+            Cap'n Proto and Flatbuffers do it:
             https://github.com/real-logic/simple-binary-encoding/issues/232
 
-            In the future, there are a couple options:
+            However, space is actually reserved for *both* `trade` and `quote` in the message;
-                1. Create a "payload header" type and promote "trade" and "quote" to <sbe:message>,
+            that is, the payload size is the same if neither, one, or both are filled.
-                   since the SBE message header is already able to distinguish between message types
+            Other ways you can try to emulate unions:
-                2. Create a "group" for each message type, and just include no elements of that type.
+                1. Use a "payload header" composite type and promote "trade" and "quote" to <sbe:message>;
-                   Uses a `u16` for each message type, instead of padding bytes for the message size.
+                   SBE can distinguish message types based on the SBE header.
-                3. Split up the message components and use "session types" to chain things together,
+                2. Create a "group" for each message type; adds an extra `u16` per type, but overall payload
-                   as in https://polysync.io/blog/session-types-for-hearty-codecs
+                   size goes down because we no longer reserve space that is potentially unused.
-
+                3. Split up the message components into individual <composite/> blocks, and write
-            For now, this message format was chosen to approximate the schemas of Cap'n Proto and Flatbuffers
+                   a state machine (by hand) to chain the blocks. For a better explanation,
+                   see "session types" in:
+                   https://polysync.io/blog/session-types-for-hearty-codecs
             -->
             <field name="msg_type" id="4" type="MsgType"/>
-            <field name="trade" id="5" type="Trade"/>
+            <field name="trade" id="5" type="Trade" presence="optional"/>
-            <field name="quote" id="6" type="Quote"/>
+            <field name="quote" id="6" type="Quote" presence="optional"/>
             <data name="symbol" id="100" type="varAsciiEncoding"/>
         </group>
     </sbe:message>

src/lib.rs

@@ -1,2 +0,0 @@
-// This file is needed for tests outside the main source tree to find the project files
-pub mod marketdata_capnp;

src/main.rs

@@ -17,9 +17,11 @@ use crate::iex::IexParser;
 pub mod marketdata_capnp;
 #[allow(unused_imports)]
 pub mod marketdata_generated; // Flatbuffers
+pub mod marketdata_sbe;
 
 mod capnp_runner;
 mod flatbuffers_runner;
+mod sbe_runner;
 mod iex;
 mod parsers;
 
@@ -81,6 +83,7 @@ fn main() {
     }
     */
 
+    /*
     let mut capnp_writer = capnp_runner::CapnpWriter::new();
     for iex_payload in parser {
         //let iex_payload = parser.next().unwrap();
@@ -93,6 +96,20 @@ fn main() {
     while let Ok(_) = capnp_reader.deserialize_unpacked(&mut read_buf, &mut summarizer) {
         parsed_msgs += 1;
     }
+    */
+
+    let mut sbe_writer = sbe_runner::SBEWriter::new();
+    for iex_payload in parser {
+        //let iex_payload = parser.next().unwrap();
+        sbe_writer.serialize(&iex_payload, &mut output_buf);
+    }
+
+    let sbe_reader = sbe_runner::SBEReader::new();
+    let mut read_buf = StreamVec::new(output_buf);
+    let mut parsed_msgs: u64 = 0;
+    while let Ok(_) = sbe_reader.deserialize(&mut read_buf, &mut summarizer) {
+        parsed_msgs += 1;
+    }
 
     dbg!(parsed_msgs);
     dbg!(summarizer);

src/marketdata_capnp.rs

@@ -144,17 +144,17 @@ pub mod multi_message {
   impl Pipeline  {
   }
   mod _private {
-      use capnp::private::layout;
+    use capnp::private::layout;
 
-      pub const STRUCT_SIZE: layout::StructSize = layout::StructSize { data: 1, pointers: 1 };
+    pub const STRUCT_SIZE: layout::StructSize = layout::StructSize { data: 1, pointers: 1 };
     pub const TYPE_ID: u64 = 0xd13b_1bd4_36e1_ca9f;
   }
 }
 
 pub mod message {
-    pub use self::Which::{Quote, Trade};
+  pub use self::Which::{Quote, Trade};
 
-    #[derive(Copy, Clone)]
+  #[derive(Copy, Clone)]
   pub struct Owned;
   impl <'a> ::capnp::traits::Owned<'a> for Owned { type Reader = Reader<'a>; type Builder = Builder<'a>; }
   impl <'a> ::capnp::traits::OwnedStruct<'a> for Owned { type Reader = Reader<'a>; type Builder = Builder<'a>; }
@@ -362,9 +362,9 @@ pub mod message {
   impl Pipeline  {
   }
   mod _private {
-      use capnp::private::layout;
+    use capnp::private::layout;
 
-      pub const STRUCT_SIZE: layout::StructSize = layout::StructSize { data: 2, pointers: 2 };
+    pub const STRUCT_SIZE: layout::StructSize = layout::StructSize { data: 2, pointers: 2 };
     pub const TYPE_ID: u64 = 0x91d7_2965_3a3d_4be4;
   }
   pub enum Which<A0,A1> {
@@ -506,9 +506,9 @@ pub mod trade {
   impl Pipeline  {
   }
   mod _private {
-      use capnp::private::layout;
+    use capnp::private::layout;
 
-      pub const STRUCT_SIZE: layout::StructSize = layout::StructSize { data: 2, pointers: 0 };
+    pub const STRUCT_SIZE: layout::StructSize = layout::StructSize { data: 2, pointers: 0 };
     pub const TYPE_ID: u64 = 0xd29e_10bd_4e5f_c241;
   }
 }
@@ -668,9 +668,9 @@ pub mod level_update {
   impl Pipeline  {
   }
   mod _private {
-      use capnp::private::layout;
+    use capnp::private::layout;
 
-      pub const STRUCT_SIZE: layout::StructSize = layout::StructSize { data: 2, pointers: 0 };
+    pub const STRUCT_SIZE: layout::StructSize = layout::StructSize { data: 2, pointers: 0 };
     pub const TYPE_ID: u64 = 0xe664_c3b5_6628_c453;
   }
 }

src/marketdata_sbe.rs

@@ -9,13 +9,13 @@ extern crate core;
 /// Errors that may occur during the course of encoding or decoding.
 #[derive(Debug)]
 pub enum CodecErr {
-    /// Too few bytes in the byte-slice to read or write the data structure relevant
+  /// Too few bytes in the byte-slice to read or write the data structure relevant
-    /// to the current state of the codec
+  /// to the current state of the codec
-    NotEnoughBytes,
+  NotEnoughBytes,
 
-    /// Groups and vardata are constrained by the numeric type chosen to represent their
+  /// Groups and vardata are constrained by the numeric type chosen to represent their
-    /// length as well as optional maxima imposed by the schema
+  /// length as well as optional maxima imposed by the schema
-    SliceIsLongerThanAllowedBySchema,
+  SliceIsLongerThanAllowedBySchema,
 }
 
 pub type CodecResult<T> = core::result::Result<T, CodecErr>;
@@ -23,208 +23,206 @@ pub type CodecResult<T> = core::result::Result<T, CodecErr>;
 /// Scratch Decoder Data Wrapper - codec internal use only
 #[derive(Debug)]
 pub struct ScratchDecoderData<'d> {
-    data: &'d [u8],
+  data: &'d [u8],
-    pos: usize,
+  pos: usize,
 }
 
 impl<'d> ScratchDecoderData<'d> {
-    /// Create a struct reference overlaid atop the data buffer
+  /// Create a struct reference overlaid atop the data buffer
-    /// such that the struct's contents directly reflect the buffer.
+  /// such that the struct's contents directly reflect the buffer. 
-    /// Advances the `pos` index by the size of the struct in bytes.
+  /// Advances the `pos` index by the size of the struct in bytes.
-    #[inline]
+  #[inline]
-    fn read_type<T>(&mut self, num_bytes: usize) -> CodecResult<&'d T> {
+  fn read_type<T>(&mut self, num_bytes: usize) -> CodecResult<&'d T> {
-        let end = self.pos + num_bytes;
+    let end = self.pos + num_bytes;
-        if end <= self.data.len() {
+    if end <= self.data.len() {
-            let s = self.data[self.pos..end].as_ptr() as *mut T;
+      let s = self.data[self.pos..end].as_ptr() as *mut T;
-            let v: &'d T = unsafe { &*s };
+      let v: &'d T = unsafe { &*s };
-            self.pos = end;
+      self.pos = end;
-            Ok(v)
+      Ok(v)
-        } else {
+    } else {
-            Err(CodecErr::NotEnoughBytes)
+      Err(CodecErr::NotEnoughBytes)
-        }
     }
+  }
 
-    /// Advances the `pos` index by a set number of bytes.
+  /// Advances the `pos` index by a set number of bytes.
-    #[inline]
+  #[inline]
-    fn skip_bytes(&mut self, num_bytes: usize) -> CodecResult<()> {
+  fn skip_bytes(&mut self, num_bytes: usize) -> CodecResult<()> {
-        let end = self.pos + num_bytes;
+    let end = self.pos + num_bytes;
-        if end <= self.data.len() {
+    if end <= self.data.len() {
-            self.pos = end;
+      self.pos = end;
-            Ok(())
+      Ok(())
-        } else {
+    } else {
-            Err(CodecErr::NotEnoughBytes)
+      Err(CodecErr::NotEnoughBytes)
-        }
     }
+  }
 
-    /// Create a slice reference overlaid atop the data buffer
+  /// Create a slice reference overlaid atop the data buffer
-    /// such that the slice's members' contents directly reflect the buffer.
+  /// such that the slice's members' contents directly reflect the buffer.
-    /// Advances the `pos` index by the size of the slice contents in bytes.
+  /// Advances the `pos` index by the size of the slice contents in bytes.
-    #[inline]
+  #[inline]
-    fn read_slice<T>(&mut self, count: usize, bytes_per_item: usize) -> CodecResult<&'d [T]> {
+  fn read_slice<T>(&mut self, count: usize, bytes_per_item: usize) -> CodecResult<&'d [T]> {
-        let num_bytes = bytes_per_item * count;
+    let num_bytes = bytes_per_item * count;
-        let end = self.pos + num_bytes;
+    let end = self.pos + num_bytes;
-        if end <= self.data.len() {
+    if end <= self.data.len() {
-            let v: &'d [T] = unsafe {
+      let v: &'d [T] = unsafe {
-                core::slice::from_raw_parts(self.data[self.pos..end].as_ptr() as *const T, count)
+        core::slice::from_raw_parts(self.data[self.pos..end].as_ptr() as *const T, count)
-            };
+      };
-            self.pos = end;
+      self.pos = end;
-            Ok(v)
+      Ok(v)
-        } else {
+    } else {
-            Err(CodecErr::NotEnoughBytes)
+      Err(CodecErr::NotEnoughBytes)
-        }
     }
+  }
 }
 
 /// Scratch Encoder Data Wrapper - codec internal use only
 #[derive(Debug)]
 pub struct ScratchEncoderData<'d> {
-    data: &'d mut [u8],
+  data: &'d mut [u8],
-    pos: usize,
+  pos: usize,
 }
 
 impl<'d> ScratchEncoderData<'d> {
-    /// Copy the bytes of a value into the data buffer
+  /// Copy the bytes of a value into the data buffer
-    /// Advances the `pos` index to after the newly-written bytes.
+  /// Advances the `pos` index to after the newly-written bytes.
-    #[inline]
+  #[inline]
-    fn write_type<T>(&mut self, t: &T, num_bytes: usize) -> CodecResult<()> {
+  fn write_type<T>(&mut self, t: &T, num_bytes: usize) -> CodecResult<()> {
-        let end = self.pos + num_bytes;
+    let end = self.pos + num_bytes;
-        if end <= self.data.len() {
+    if end <= self.data.len() {
-            let source_bytes: &[u8] = unsafe {
+      let source_bytes: &[u8] = unsafe {
-                core::slice::from_raw_parts(t as *const T as *const u8, num_bytes)
+        core::slice::from_raw_parts(t as *const T as *const u8, num_bytes)
-            };
+      };
-            (&mut self.data[self.pos..end]).copy_from_slice(source_bytes);
+      (&mut self.data[self.pos..end]).copy_from_slice(source_bytes);
-            self.pos = end;
+      self.pos = end;
-            Ok(())
+      Ok(())
-        } else {
+    } else {
-            Err(CodecErr::NotEnoughBytes)
+      Err(CodecErr::NotEnoughBytes)
-        }
     }
+  }
 
-    /// Advances the `pos` index by a set number of bytes.
+  /// Advances the `pos` index by a set number of bytes.
-    #[inline]
+  #[inline]
-    fn skip_bytes(&mut self, num_bytes: usize) -> CodecResult<()> {
+  fn skip_bytes(&mut self, num_bytes: usize) -> CodecResult<()> {
-        let end = self.pos + num_bytes;
+    let end = self.pos + num_bytes;
-        if end <= self.data.len() {
+    if end <= self.data.len() {
-            self.pos = end;
+      self.pos = end;
-            Ok(())
+      Ok(())
-        } else {
+    } else {
-            Err(CodecErr::NotEnoughBytes)
+      Err(CodecErr::NotEnoughBytes)
-        }
     }
+  }
 
-    /// Create a struct reference overlaid atop the data buffer
+  /// Create a struct reference overlaid atop the data buffer
-    /// such that changes to the struct directly edit the buffer.
+  /// such that changes to the struct directly edit the buffer. 
-    /// Note that the initial content of the struct's fields may be garbage.
+  /// Note that the initial content of the struct's fields may be garbage.
-    /// Advances the `pos` index to after the newly-written bytes.
+  /// Advances the `pos` index to after the newly-written bytes.
-    #[inline]
+  #[inline]
-    fn writable_overlay<T>(&mut self, num_bytes: usize) -> CodecResult<&'d mut T> {
+  fn writable_overlay<T>(&mut self, num_bytes: usize) -> CodecResult<&'d mut T> {
-        let end = self.pos + num_bytes;
+    let end = self.pos + num_bytes;
-        if end <= self.data.len() {
+    if end <= self.data.len() {
-            let v: &'d mut T = unsafe {
+      let v: &'d mut T = unsafe {
-                let s = self.data.as_ptr().offset(self.pos as isize) as *mut T;
+        let s = self.data.as_ptr().offset(self.pos as isize) as *mut T;
-                &mut *s
+        &mut *s
-            };
+      };
-            self.pos = end;
+      self.pos = end;
-            Ok(v)
+      Ok(v)
-        } else {
+    } else {
-            Err(CodecErr::NotEnoughBytes)
+      Err(CodecErr::NotEnoughBytes)
-        }
     }
+  }
 
-    /// Copy the bytes of a value into the data buffer at a specific position
+  /// Copy the bytes of a value into the data buffer at a specific position
-    /// Does **not** alter the `pos` index.
+  /// Does **not** alter the `pos` index.
-    #[inline]
+  #[inline]
-    fn write_at_position<T>(&mut self, position: usize, t: &T, num_bytes: usize) -> CodecResult<()> {
+  fn write_at_position<T>(&mut self, position: usize, t: &T, num_bytes: usize) -> CodecResult<()> {
-        let end = position + num_bytes;
+    let end = position + num_bytes;
-        if end <= self.data.len() {
+    if end <= self.data.len() {
-            let source_bytes: &[u8] = unsafe {
+      let source_bytes: &[u8] = unsafe {
-                core::slice::from_raw_parts(t as *const T as *const u8, num_bytes)
+        core::slice::from_raw_parts(t as *const T as *const u8, num_bytes)
-            };
+      };
-            (&mut self.data[position..end]).copy_from_slice(source_bytes);
+      (&mut self.data[position..end]).copy_from_slice(source_bytes);
-            Ok(())
+      Ok(())
-        } else {
+    } else {
-            Err(CodecErr::NotEnoughBytes)
+      Err(CodecErr::NotEnoughBytes)
-        }
     }
-    /// Create a mutable slice overlaid atop the data buffer directly
+  }
-    /// such that changes to the slice contents directly edit the buffer
+  /// Create a mutable slice overlaid atop the data buffer directly
-    /// Note that the initial content of the slice's members' fields may be garbage.
+  /// such that changes to the slice contents directly edit the buffer
-    /// Advances the `pos` index to after the region representing the slice.
+  /// Note that the initial content of the slice's members' fields may be garbage.
-    #[inline]
+  /// Advances the `pos` index to after the region representing the slice.
-    fn writable_slice<T>(&mut self, count: usize, bytes_per_item: usize) -> CodecResult<&'d mut [T]> {
+  #[inline]
-        let end = self.pos + (count * bytes_per_item);
+  fn writable_slice<T>(&mut self, count: usize, bytes_per_item: usize) -> CodecResult<&'d mut [T]> {
-        if end <= self.data.len() {
+    let end = self.pos + (count * bytes_per_item);
-            let v: &'d mut [T] = unsafe {
+    if end <= self.data.len() {
-                core::slice::from_raw_parts_mut(self.data[self.pos..end].as_mut_ptr() as *mut T, count)
+      let v: &'d mut [T] = unsafe {
-            };
+        core::slice::from_raw_parts_mut(self.data[self.pos..end].as_mut_ptr() as *mut T, count)
-            self.pos = end;
+      };
-            Ok(v)
+      self.pos = end;
-        } else {
+      Ok(v)
-            Err(CodecErr::NotEnoughBytes)
+    } else {
-        }
+      Err(CodecErr::NotEnoughBytes)
     }
+  }
 
-    /// Copy the raw bytes of a slice's contents into the data buffer
+  /// Copy the raw bytes of a slice's contents into the data buffer
-    /// Does **not** encode the length of the slice explicitly into the buffer.
+  /// Does **not** encode the length of the slice explicitly into the buffer.
-    /// Advances the `pos` index to after the newly-written slice bytes.
+  /// Advances the `pos` index to after the newly-written slice bytes.
-    #[inline]
+  #[inline]
-    fn write_slice_without_count<T>(&mut self, t: &[T], bytes_per_item: usize) -> CodecResult<()> {
+  fn write_slice_without_count<T>(&mut self, t: &[T], bytes_per_item: usize) -> CodecResult<()> {
-        let content_bytes_size = bytes_per_item * t.len();
+    let content_bytes_size = bytes_per_item * t.len();
-        let end = self.pos + content_bytes_size;
+    let end = self.pos + content_bytes_size;
-        if end <= self.data.len() {
+    if end <= self.data.len() {
-            let source_bytes: &[u8] = unsafe {
+      let source_bytes: &[u8] = unsafe {
-                core::slice::from_raw_parts(t.as_ptr() as *const u8, content_bytes_size)
+        core::slice::from_raw_parts(t.as_ptr() as *const u8, content_bytes_size)
-            };
+      };
-            (&mut self.data[self.pos..end]).copy_from_slice(source_bytes);
+      (&mut self.data[self.pos..end]).copy_from_slice(source_bytes);
-            self.pos = end;
+      self.pos = end;
-            Ok(())
+      Ok(())
-        } else {
+    } else {
-            Err(CodecErr::NotEnoughBytes)
+      Err(CodecErr::NotEnoughBytes)
-        }
     }
+  }
 }
 
 /// Convenience Either enum
 #[derive(Copy, Clone, PartialEq, Eq, PartialOrd, Ord, Hash, Debug)]
 pub enum Either<L, R> {
-    Left(L),
+  Left(L),
-    Right(R),
+  Right(R)
 }
 
 /// Enum Side
 #[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord, Hash)]
 #[repr(u8)]
 pub enum Side {
-    Buy = 0u8,
+  Buy = 0u8,
-    Sell = 1u8,
+  Sell = 1u8,
-    NullVal = 255u8,
+  NullVal = 255u8,
 }
-
 impl Default for Side {
-    fn default() -> Self { Side::NullVal }
+  fn default() -> Self { Side::NullVal }
 }
 
 /// Enum MsgType
 #[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord, Hash)]
 #[repr(u8)]
 pub enum MsgType {
-    Trade = 0u8,
+  Trade = 0u8,
-    Quote = 1u8,
+  Quote = 1u8,
-    NullVal = 255u8,
+  NullVal = 255u8,
 }
-
 impl Default for MsgType {
-    fn default() -> Self { MsgType::NullVal }
+  fn default() -> Self { MsgType::NullVal }
 }
 
 /// Quote
 #[repr(C, packed)]
 #[derive(Default)]
 pub struct Quote {
-    pub price: u64,
+  pub price: u64,
-    pub size: u32,
+  pub size: u32,
-    pub flags: u8,
+  pub flags: u8,
-    pub side: Side,
+  pub side: Side,
 }
 
 impl Quote {}
@@ -233,8 +231,8 @@ impl Quote {}
 #[repr(C, packed)]
 #[derive(Default)]
 pub struct Trade {
-    pub price: u64,
+  pub price: u64,
-    pub size: u32,
+  pub size: u32,
 }
 
 impl Trade {}
@@ -243,10 +241,10 @@ impl Trade {}
 #[repr(C, packed)]
 #[derive(Default)]
 pub struct MessageHeader {
-    pub block_length: u16,
+  pub block_length: u16,
-    pub template_id: u16,
+  pub template_id: u16,
-    pub schema_id: u16,
+  pub schema_id: u16,
-    pub version: u16,
+  pub version: u16,
 }
 
 impl MessageHeader {}
@@ -255,8 +253,8 @@ impl MessageHeader {}
 #[repr(C, packed)]
 #[derive(Default)]
 pub struct GroupSizeEncoding {
-    pub block_length: u16,
+  pub block_length: u16,
-    pub num_in_group: u16,
+  pub num_in_group: u16,
 }
 
 impl GroupSizeEncoding {}
@@ -265,24 +263,24 @@ impl GroupSizeEncoding {}
 #[repr(C, packed)]
 #[derive(Default)]
 pub struct VarAsciiEncoding {
-    pub length: u32,
+  pub length: u32,
-    pub var_data: u8,
+  pub var_data: u8,
 }
 
 impl VarAsciiEncoding {}
 
 /// MessageHeader Decoder entry point
 pub fn start_decoding_message_header<'d>(data: &'d [u8]) -> CodecResult<(&'d MessageHeader, ScratchDecoderData<'d>)> {
-    let mut scratch = ScratchDecoderData { data: data, pos: 0 };
+  let mut scratch = ScratchDecoderData { data: data, pos: 0 };
-    let v = scratch.read_type::<MessageHeader>(8)?;
+  let v = scratch.read_type::<MessageHeader>(8)?;
-    Ok((v, scratch))
+  Ok((v, scratch))
 }
 
 /// MultiMessage Fixed-size Fields (8 bytes)
 #[repr(C, packed)]
 #[derive(Default)]
 pub struct MultiMessageFields {
-    pub sequence_number: u64,
+  pub sequence_number: u64,
 }
 
 
@@ -291,293 +289,279 @@ impl MultiMessageFields {}
 /// MultiMessage specific Message Header 
 #[repr(C, packed)]
 pub struct MultiMessageMessageHeader {
-    pub message_header: MessageHeader
+  pub message_header: MessageHeader
 }
-
 impl MultiMessageMessageHeader {
-    pub const BLOCK_LENGTH: u16 = 8;
+  pub const BLOCK_LENGTH: u16 = 8;
-    pub const TEMPLATE_ID: u16 = 1;
+  pub const TEMPLATE_ID: u16 = 1;
-    pub const SCHEMA_ID: u16 = 1;
+  pub const SCHEMA_ID: u16 = 1;
-    pub const VERSION: u16 = 0;
+  pub const VERSION: u16 = 0;
 }
-
 impl Default for MultiMessageMessageHeader {
-    fn default() -> MultiMessageMessageHeader {
+  fn default() -> MultiMessageMessageHeader {
-        MultiMessageMessageHeader {
+    MultiMessageMessageHeader {
-            message_header: MessageHeader {
+      message_header: MessageHeader {
-                block_length: 8u16,
+        block_length: 8u16,
-                template_id: 1u16,
+        template_id: 1u16,
-                schema_id: 1u16,
+        schema_id: 1u16,
-                version: 0u16,
+        version: 0u16,
-            }
+      }
-        }
     }
+  }
 }
 
 /// Group fixed-field member representations
 #[repr(C, packed)]
 #[derive(Default)]
 pub struct MultiMessageMessagesMember {
-    pub timestamp: i64,
+  pub timestamp: i64,
-    pub msg_type: MsgType,
+  pub msg_type: MsgType,
-    pub trade: Trade,
+  pub trade: Trade,
-    pub quote: Quote,
+  pub quote: Quote,
 }
 
 impl MultiMessageMessagesMember {}
 
 /// MultiMessageDecoderDone
 pub struct MultiMessageDecoderDone<'d> {
-    scratch: ScratchDecoderData<'d>,
+  scratch: ScratchDecoderData<'d>,
 }
-
 impl<'d> MultiMessageDecoderDone<'d> {
-    /// Returns the number of bytes decoded
+  /// Returns the number of bytes decoded
-    pub fn unwrap(self) -> usize {
+  pub fn unwrap(self) -> usize {
-        self.scratch.pos
+    self.scratch.pos
-    }
+  }
 
-    pub fn wrap(scratch: ScratchDecoderData<'d>) -> MultiMessageDecoderDone<'d> {
+  pub fn wrap(scratch: ScratchDecoderData<'d>) -> MultiMessageDecoderDone<'d> {
-        MultiMessageDecoderDone { scratch: scratch }
+    MultiMessageDecoderDone { scratch: scratch }
-    }
+  }
 }
 
 /// symbol variable-length data
 pub struct MultiMessageMessagesSymbolDecoder<'d> {
-    parent: MultiMessageMessagesMemberDecoder<'d>,
+  parent: MultiMessageMessagesMemberDecoder<'d>,
 }
-
 impl<'d> MultiMessageMessagesSymbolDecoder<'d> {
-    fn wrap(parent: MultiMessageMessagesMemberDecoder<'d>) -> Self {
+  fn wrap(parent: MultiMessageMessagesMemberDecoder<'d>) -> Self {
-        MultiMessageMessagesSymbolDecoder { parent: parent }
+    MultiMessageMessagesSymbolDecoder { parent: parent }
-    }
+  }
-    pub fn symbol(mut self) -> CodecResult<(&'d [u8], Either<MultiMessageMessagesMemberDecoder<'d>, MultiMessageDecoderDone<'d>>)> {
+  pub fn symbol(mut self) -> CodecResult<(&'d [u8], Either<MultiMessageMessagesMemberDecoder<'d>, MultiMessageDecoderDone<'d>>)> {
-        let count = *self.parent.scratch.read_type::<u32>(4)?;
+    let count = *self.parent.scratch.read_type::<u32>(4)?;
-        Ok((self.parent.scratch.read_slice::<u8>(count as usize, 1)?, self.parent.after_member()))
+    Ok((self.parent.scratch.read_slice::<u8>(count as usize, 1)?, self.parent.after_member()))
-    }
+  }
 }
 
 /// MultiMessageMessages Decoder for fields and header
 pub struct MultiMessageMessagesMemberDecoder<'d> {
-    scratch: ScratchDecoderData<'d>,
+  scratch: ScratchDecoderData<'d>,
-    max_index: u16,
+  max_index: u16,
-    index: u16,
+  index: u16,
 }
 
 impl<'d> MultiMessageMessagesMemberDecoder<'d> {
-    fn new(scratch: ScratchDecoderData<'d>, count: u16) -> Self {
+  fn new(scratch: ScratchDecoderData<'d>, count: u16) -> Self {
-        assert!(count > 0u16);
+    assert!(count > 0u16);
-        MultiMessageMessagesMemberDecoder {
+    MultiMessageMessagesMemberDecoder {
-            scratch: scratch,
+      scratch: scratch,
-            max_index: count - 1,
+      max_index: count - 1,
-            index: 0,
+      index: 0,
-        }
     }
+  }
 
-    pub fn next_messages_member(mut self) -> CodecResult<(&'d MultiMessageMessagesMember, MultiMessageMessagesSymbolDecoder<'d>)> {
+  pub fn next_messages_member(mut self) -> CodecResult<(&'d MultiMessageMessagesMember, MultiMessageMessagesSymbolDecoder<'d>)> {
-        let v = self.scratch.read_type::<MultiMessageMessagesMember>(35)?;
+    let v = self.scratch.read_type::<MultiMessageMessagesMember>(35)?;
-        self.index += 1;
+    self.index += 1;
-        Ok((v, MultiMessageMessagesSymbolDecoder::wrap(self)))
+    Ok((v, MultiMessageMessagesSymbolDecoder::wrap(self)))
-    }
+  }
-    #[inline]
+  #[inline]
-    fn after_member(self) -> Either<MultiMessageMessagesMemberDecoder<'d>, MultiMessageDecoderDone<'d>> {
+  fn after_member(self) -> Either<MultiMessageMessagesMemberDecoder<'d>, MultiMessageDecoderDone<'d>> {
-        if self.index <= self.max_index {
+    if self.index <= self.max_index {
-            Either::Left(self)
+      Either::Left(self)
-        } else {
+    } else {
-            Either::Right(MultiMessageDecoderDone::wrap(self.scratch))
+      Either::Right(MultiMessageDecoderDone::wrap(self.scratch))
-        }
     }
+  }
 }
-
 pub struct MultiMessageMessagesHeaderDecoder<'d> {
-    scratch: ScratchDecoderData<'d>,
+  scratch: ScratchDecoderData<'d>,
 }
-
 impl<'d> MultiMessageMessagesHeaderDecoder<'d> {
-    fn wrap(scratch: ScratchDecoderData<'d>) -> Self {
+  fn wrap(scratch: ScratchDecoderData<'d>) -> Self {
-        MultiMessageMessagesHeaderDecoder { scratch: scratch }
+    MultiMessageMessagesHeaderDecoder { scratch: scratch }
-    }
+  }
-    pub fn messages_individually(mut self) -> CodecResult<Either<MultiMessageMessagesMemberDecoder<'d>, MultiMessageDecoderDone<'d>>> {
+  pub fn messages_individually(mut self) -> CodecResult<Either<MultiMessageMessagesMemberDecoder<'d>, MultiMessageDecoderDone<'d>>> {
-        let dim = self.scratch.read_type::<GroupSizeEncoding>(4)?;
+    let dim = self.scratch.read_type::<GroupSizeEncoding>(4)?;
-        if dim.num_in_group > 0 {
+    if dim.num_in_group > 0 {
-            Ok(Either::Left(MultiMessageMessagesMemberDecoder::new(self.scratch, dim.num_in_group)))
+      Ok(Either::Left(MultiMessageMessagesMemberDecoder::new(self.scratch, dim.num_in_group)))
-        } else {
+    } else {
-            Ok(Either::Right(MultiMessageDecoderDone::wrap(self.scratch)))
+      Ok(Either::Right(MultiMessageDecoderDone::wrap(self.scratch)))
-        }
     }
+  }
 }
 
 /// MultiMessage Fixed fields Decoder
 pub struct MultiMessageFieldsDecoder<'d> {
-    scratch: ScratchDecoderData<'d>,
+  scratch: ScratchDecoderData<'d>,
 }
-
 impl<'d> MultiMessageFieldsDecoder<'d> {
-    pub fn wrap(scratch: ScratchDecoderData<'d>) -> MultiMessageFieldsDecoder<'d> {
+  pub fn wrap(scratch: ScratchDecoderData<'d>) -> MultiMessageFieldsDecoder<'d> {
-        MultiMessageFieldsDecoder { scratch: scratch }
+    MultiMessageFieldsDecoder { scratch: scratch }
-    }
+  }
-    pub fn multi_message_fields(mut self) -> CodecResult<(&'d MultiMessageFields, MultiMessageMessagesHeaderDecoder<'d>)> {
+  pub fn multi_message_fields(mut self) -> CodecResult<(&'d MultiMessageFields, MultiMessageMessagesHeaderDecoder<'d>)> {
-        let v = self.scratch.read_type::<MultiMessageFields>(8)?;
+    let v = self.scratch.read_type::<MultiMessageFields>(8)?;
-        Ok((v, MultiMessageMessagesHeaderDecoder::wrap(self.scratch)))
+    Ok((v, MultiMessageMessagesHeaderDecoder::wrap(self.scratch)))
-    }
+  }
 }
 
 /// MultiMessageMessageHeaderDecoder
 pub struct MultiMessageMessageHeaderDecoder<'d> {
-    scratch: ScratchDecoderData<'d>,
+  scratch: ScratchDecoderData<'d>,
 }
-
 impl<'d> MultiMessageMessageHeaderDecoder<'d> {
-    pub fn wrap(scratch: ScratchDecoderData<'d>) -> MultiMessageMessageHeaderDecoder<'d> {
+  pub fn wrap(scratch: ScratchDecoderData<'d>) -> MultiMessageMessageHeaderDecoder<'d> {
-        MultiMessageMessageHeaderDecoder { scratch: scratch }
+    MultiMessageMessageHeaderDecoder { scratch: scratch }
-    }
+  }
-    pub fn header(mut self) -> CodecResult<(&'d MessageHeader, MultiMessageFieldsDecoder<'d>)> {
+  pub fn header(mut self) -> CodecResult<(&'d MessageHeader, MultiMessageFieldsDecoder<'d>)> {
-        let v = self.scratch.read_type::<MessageHeader>(8)?;
+    let v = self.scratch.read_type::<MessageHeader>(8)?;
-        Ok((v, MultiMessageFieldsDecoder::wrap(self.scratch)))
+    Ok((v, MultiMessageFieldsDecoder::wrap(self.scratch)))
-    }
+  }
 }
 
 /// MultiMessage Decoder entry point
 pub fn start_decoding_multi_message<'d>(data: &'d [u8]) -> MultiMessageMessageHeaderDecoder<'d> {
-    MultiMessageMessageHeaderDecoder::wrap(ScratchDecoderData { data: data, pos: 0 })
+  MultiMessageMessageHeaderDecoder::wrap(ScratchDecoderData { data: data, pos: 0 })
 }
 
 /// MultiMessageEncoderDone
 pub struct MultiMessageEncoderDone<'d> {
-    scratch: ScratchEncoderData<'d>,
+  scratch: ScratchEncoderData<'d>,
 }
-
 impl<'d> MultiMessageEncoderDone<'d> {
-    /// Returns the number of bytes encoded
+  /// Returns the number of bytes encoded
-    pub fn unwrap(self) -> usize {
+  pub fn unwrap(self) -> usize {
-        self.scratch.pos
+    self.scratch.pos
-    }
+  }
 
-    pub fn wrap(scratch: ScratchEncoderData<'d>) -> MultiMessageEncoderDone<'d> {
+  pub fn wrap(scratch: ScratchEncoderData<'d>) -> MultiMessageEncoderDone<'d> {
-        MultiMessageEncoderDone { scratch: scratch }
+    MultiMessageEncoderDone { scratch: scratch }
-    }
+  }
 }
 
 /// symbol variable-length data
 pub struct MultiMessageMessagesSymbolEncoder<'d> {
-    parent: MultiMessageMessagesMemberEncoder<'d>,
+  parent: MultiMessageMessagesMemberEncoder<'d>,
 }
-
 impl<'d> MultiMessageMessagesSymbolEncoder<'d> {
-    fn wrap(parent: MultiMessageMessagesMemberEncoder<'d>) -> Self {
+  fn wrap(parent: MultiMessageMessagesMemberEncoder<'d>) -> Self {
-        MultiMessageMessagesSymbolEncoder { parent: parent }
+    MultiMessageMessagesSymbolEncoder { parent: parent }
-    }
+  }
-    pub fn symbol(mut self, s: &'d [u8]) -> CodecResult<MultiMessageMessagesMemberEncoder> {
+  pub fn symbol(mut self, s: &'d [u8]) -> CodecResult<MultiMessageMessagesMemberEncoder> {
-        let l = s.len();
+    let l = s.len();
-        if l > 4294967294 {
+    if l > 4294967294 {
-            return Err(CodecErr::SliceIsLongerThanAllowedBySchema);
+      return Err(CodecErr::SliceIsLongerThanAllowedBySchema)
-        }
-        // Write data length
-        self.parent.scratch.write_type::<u32>(&(l as u32), 4)?; // group length
-        self.parent.scratch.write_slice_without_count::<u8>(s, 1)?;
-        Ok(self.parent)
     }
+    // Write data length
+    self.parent.scratch.write_type::<u32>(&(l as u32), 4)?; // group length
+    self.parent.scratch.write_slice_without_count::<u8>(s, 1)?;
+    Ok(self.parent)
+  }
 }
 
 /// MultiMessageMessages Encoder for fields and header
 pub struct MultiMessageMessagesMemberEncoder<'d> {
-    scratch: ScratchEncoderData<'d>,
+  scratch: ScratchEncoderData<'d>,
-    count_write_pos: usize,
+  count_write_pos: usize,
-    count: u16,
+  count: u16,
 }
 
 impl<'d> MultiMessageMessagesMemberEncoder<'d> {
-    #[inline]
+  #[inline]
-    fn new(scratch: ScratchEncoderData<'d>, count_write_pos: usize) -> Self {
+  fn new(scratch: ScratchEncoderData<'d>, count_write_pos: usize) -> Self {
-        MultiMessageMessagesMemberEncoder {
+    MultiMessageMessagesMemberEncoder {
-            scratch: scratch,
+      scratch: scratch,
-            count_write_pos: count_write_pos,
+      count_write_pos: count_write_pos,
-            count: 0,
+      count: 0,
-        }
     }
+  }
 
-    #[inline]
+  #[inline]
-    pub fn next_messages_member(mut self, fields: &MultiMessageMessagesMember) -> CodecResult<MultiMessageMessagesSymbolEncoder<'d>> {
+  pub fn next_messages_member(mut self, fields: &MultiMessageMessagesMember) -> CodecResult<MultiMessageMessagesSymbolEncoder<'d>> {
-        self.scratch.write_type::<MultiMessageMessagesMember>(fields, 35)?; // block length
+    self.scratch.write_type::<MultiMessageMessagesMember>(fields, 35)?; // block length
-        self.count += 1;
+    self.count += 1;
-        Ok(MultiMessageMessagesSymbolEncoder::wrap(self))
+    Ok(MultiMessageMessagesSymbolEncoder::wrap(self))
-    }
+  }
-    #[inline]
+  #[inline]
-    pub fn done_with_messages(mut self) -> CodecResult<MultiMessageEncoderDone<'d>> {
+  pub fn done_with_messages(mut self) -> CodecResult<MultiMessageEncoderDone<'d>> {
-        self.scratch.write_at_position::<u16>(self.count_write_pos, &self.count, 2)?;
+    self.scratch.write_at_position::<u16>(self.count_write_pos, &self.count, 2)?;
-        Ok(MultiMessageEncoderDone::wrap(self.scratch))
+    Ok(MultiMessageEncoderDone::wrap(self.scratch))
-    }
+  }
 }
-
 pub struct MultiMessageMessagesHeaderEncoder<'d> {
-    scratch: ScratchEncoderData<'d>,
+  scratch: ScratchEncoderData<'d>,
 }
-
 impl<'d> MultiMessageMessagesHeaderEncoder<'d> {
-    #[inline]
+  #[inline]
-    fn wrap(scratch: ScratchEncoderData<'d>) -> Self {
+  fn wrap(scratch: ScratchEncoderData<'d>) -> Self {
-        MultiMessageMessagesHeaderEncoder { scratch: scratch }
+    MultiMessageMessagesHeaderEncoder { scratch: scratch }
-    }
+  }
-    #[inline]
+  #[inline]
-    pub fn messages_individually(mut self) -> CodecResult<MultiMessageMessagesMemberEncoder<'d>> {
+  pub fn messages_individually(mut self) -> CodecResult<MultiMessageMessagesMemberEncoder<'d>> {
-        self.scratch.write_type::<u16>(&35u16, 2)?; // block length
+    self.scratch.write_type::<u16>(&35u16, 2)?; // block length
-        let count_pos = self.scratch.pos;
+    let count_pos = self.scratch.pos;
-        self.scratch.write_type::<u16>(&0, 2)?; // preliminary group member count
+    self.scratch.write_type::<u16>(&0, 2)?; // preliminary group member count
-        Ok(MultiMessageMessagesMemberEncoder::new(self.scratch, count_pos))
+    Ok(MultiMessageMessagesMemberEncoder::new(self.scratch, count_pos))
-    }
+  }
 }
 
 /// MultiMessage Fixed fields Encoder
 pub struct MultiMessageFieldsEncoder<'d> {
-    scratch: ScratchEncoderData<'d>,
+  scratch: ScratchEncoderData<'d>,
 }
-
 impl<'d> MultiMessageFieldsEncoder<'d> {
-    pub fn wrap(scratch: ScratchEncoderData<'d>) -> MultiMessageFieldsEncoder<'d> {
+  pub fn wrap(scratch: ScratchEncoderData<'d>) -> MultiMessageFieldsEncoder<'d> {
-        MultiMessageFieldsEncoder { scratch: scratch }
+    MultiMessageFieldsEncoder { scratch: scratch }
-    }
+  }
 
-    /// Create a mutable struct reference overlaid atop the data buffer
+  /// Create a mutable struct reference overlaid atop the data buffer
-      /// such that changes to the struct directly edit the buffer.
+    /// such that changes to the struct directly edit the buffer.
-      /// Note that the initial content of the struct's fields may be garbage.
+    /// Note that the initial content of the struct's fields may be garbage.
-    pub fn multi_message_fields(mut self) -> CodecResult<(&'d mut MultiMessageFields, MultiMessageMessagesHeaderEncoder<'d>)> {
+  pub fn multi_message_fields(mut self) -> CodecResult<(&'d mut MultiMessageFields, MultiMessageMessagesHeaderEncoder<'d>)> {
-        let v = self.scratch.writable_overlay::<MultiMessageFields>(8 + 0)?;
+    let v = self.scratch.writable_overlay::<MultiMessageFields>(8 + 0)?;
-        Ok((v, MultiMessageMessagesHeaderEncoder::wrap(self.scratch)))
+    Ok((v, MultiMessageMessagesHeaderEncoder::wrap(self.scratch)))
-    }
+  }
 
-    /// Copy the bytes of a value into the data buffer
+  /// Copy the bytes of a value into the data buffer
-    pub fn multi_message_fields_copy(mut self, t: &MultiMessageFields) -> CodecResult<MultiMessageMessagesHeaderEncoder<'d>> {
+  pub fn multi_message_fields_copy(mut self, t: &MultiMessageFields) -> CodecResult<MultiMessageMessagesHeaderEncoder<'d>> {
-        self.scratch.write_type::<MultiMessageFields>(t, 8)?;
+    self.scratch.write_type::<MultiMessageFields>(t, 8)?;
-        Ok(MultiMessageMessagesHeaderEncoder::wrap(self.scratch))
+    Ok(MultiMessageMessagesHeaderEncoder::wrap(self.scratch))
-    }
+  }
 }
 
 /// MultiMessageMessageHeaderEncoder
 pub struct MultiMessageMessageHeaderEncoder<'d> {
-    scratch: ScratchEncoderData<'d>,
+  scratch: ScratchEncoderData<'d>,
 }
-
 impl<'d> MultiMessageMessageHeaderEncoder<'d> {
-    pub fn wrap(scratch: ScratchEncoderData<'d>) -> MultiMessageMessageHeaderEncoder<'d> {
+  pub fn wrap(scratch: ScratchEncoderData<'d>) -> MultiMessageMessageHeaderEncoder<'d> {
-        MultiMessageMessageHeaderEncoder { scratch: scratch }
+    MultiMessageMessageHeaderEncoder { scratch: scratch }
-    }
+  }
 
-    /// Create a mutable struct reference overlaid atop the data buffer
+  /// Create a mutable struct reference overlaid atop the data buffer
-      /// such that changes to the struct directly edit the buffer.
+    /// such that changes to the struct directly edit the buffer.
-      /// Note that the initial content of the struct's fields may be garbage.
+    /// Note that the initial content of the struct's fields may be garbage.
-    pub fn header(mut self) -> CodecResult<(&'d mut MessageHeader, MultiMessageFieldsEncoder<'d>)> {
+  pub fn header(mut self) -> CodecResult<(&'d mut MessageHeader, MultiMessageFieldsEncoder<'d>)> {
-        let v = self.scratch.writable_overlay::<MessageHeader>(8 + 0)?;
+    let v = self.scratch.writable_overlay::<MessageHeader>(8 + 0)?;
-        Ok((v, MultiMessageFieldsEncoder::wrap(self.scratch)))
+    Ok((v, MultiMessageFieldsEncoder::wrap(self.scratch)))
-    }
+  }
 
-    /// Copy the bytes of a value into the data buffer
+  /// Copy the bytes of a value into the data buffer
-    pub fn header_copy(mut self, t: &MessageHeader) -> CodecResult<MultiMessageFieldsEncoder<'d>> {
+  pub fn header_copy(mut self, t: &MessageHeader) -> CodecResult<MultiMessageFieldsEncoder<'d>> {
-        self.scratch.write_type::<MessageHeader>(t, 8)?;
+    self.scratch.write_type::<MessageHeader>(t, 8)?;
-        Ok(MultiMessageFieldsEncoder::wrap(self.scratch))
+    Ok(MultiMessageFieldsEncoder::wrap(self.scratch))
-    }
+  }
 }
 
 /// MultiMessage Encoder entry point
 pub fn start_encoding_multi_message<'d>(data: &'d mut [u8]) -> MultiMessageMessageHeaderEncoder<'d> {
-    MultiMessageMessageHeaderEncoder::wrap(ScratchEncoderData { data: data, pos: 0 })
+  MultiMessageMessageHeaderEncoder::wrap(ScratchEncoderData { data: data, pos: 0 })
 }

src/sbe_runner.rs

@@ -0,0 +1,144 @@
+use std::io::{BufRead, Write};
+use std::str::from_utf8_unchecked;
+
+use nom::bytes::complete::take_until;
+use nom::IResult;
+
+use crate::{marketdata_sbe, StreamVec, Summarizer};
+use crate::iex::{IexMessage, IexPayload};
+use crate::marketdata_sbe::{Either, MultiMessageFields, MultiMessageMessageHeader, MultiMessageMessagesMember, MultiMessageMessagesMemberEncoder, MultiMessageMessagesSymbolEncoder, Side, start_decoding_multi_message, start_encoding_multi_message};
+
+fn __take_until<'a>(tag: &'static str, input: &'a [u8]) -> IResult<&'a [u8], &'a [u8]> {
+    take_until(tag)(input)
+}
+
+fn parse_symbol(sym: &[u8; 8]) -> &str {
+    // TODO: Use the `jetscii` library for all that SIMD goodness
+    // IEX guarantees ASCII, so we're fine using an unsafe conversion
+    let (_, sym_bytes) = __take_until(" ", &sym[..]).unwrap();
+    unsafe { from_utf8_unchecked(sym_bytes) }
+}
+
+pub struct SBEWriter {
+    /// Buffer to construct messages before copying. While SBE benefits
+    /// from easily being able to create messages directly in output buffer,
+    /// we'll construct in a scratch buffer and then copy to more fairly
+    /// benchmark against Cap'n Proto and Flatbuffers.
+    scratch_buffer: Vec<u8>,
+    default_header: MultiMessageMessageHeader,
+}
+
+impl SBEWriter {
+    pub fn new() -> SBEWriter {
+        SBEWriter {
+            // 8K scratch buffer is *way* more than necessary,
+            // but we don't want to run into issues with not enough
+            // data to encode messages
+            scratch_buffer: vec![0; 1024 * 8],
+            default_header: MultiMessageMessageHeader::default(),
+        }
+    }
+
+    pub fn serialize(&mut self, payload: &IexPayload, output: &mut Vec<u8>) {
+        let (fields, encoder) = start_encoding_multi_message(&mut self.scratch_buffer[..])
+            .header_copy(&self.default_header.message_header).unwrap()
+            .multi_message_fields().unwrap();
+        fields.sequence_number = payload.first_seq_no;
+
+        let mut encoder = encoder.messages_individually().unwrap();
+        let mut encoder: MultiMessageMessagesMemberEncoder = payload.messages.iter().fold(encoder, |enc, m| {
+            match m {
+                IexMessage::TradeReport(tr) => {
+                    let fields = MultiMessageMessagesMember {
+                        msg_type: marketdata_sbe::MsgType::Trade,
+                        timestamp: tr.timestamp,
+                        trade: marketdata_sbe::Trade {
+                            size: tr.size,
+                            price: tr.price,
+                        },
+                        ..Default::default()
+                    };
+                    let sym_enc: MultiMessageMessagesSymbolEncoder = enc.next_messages_member(&fields).unwrap();
+                    sym_enc.symbol(parse_symbol(&tr.symbol).as_bytes()).unwrap()
+                }
+                IexMessage::PriceLevelUpdate(plu) => {
+                    let fields = MultiMessageMessagesMember {
+                        msg_type: marketdata_sbe::MsgType::Quote,
+                        timestamp: plu.timestamp,
+                        quote: marketdata_sbe::Quote {
+                            price: plu.price,
+                            size: plu.size,
+                            flags: plu.event_flags,
+                            side: if plu.msg_type == 0x38 { Side::Buy } else { Side::Sell },
+                        },
+                        ..Default::default()
+                    };
+                    let sym_enc: MultiMessageMessagesSymbolEncoder = enc.next_messages_member(&fields).unwrap();
+                    sym_enc.symbol(parse_symbol(&plu.symbol).as_bytes()).unwrap()
+                }
+                _ => enc
+            }
+        });
+
+        let finished = encoder.done_with_messages().unwrap();
+        let data_len = finished.unwrap();
+
+        output.write(&self.scratch_buffer[..data_len]).unwrap();
+    }
+}
+
+pub struct SBEReader;
+
+impl SBEReader {
+    pub fn new() -> SBEReader {
+        SBEReader {}
+    }
+
+    pub fn deserialize<'a>(&self, buf: &'a mut StreamVec, stats: &mut Summarizer) -> Result<(), ()> {
+        let data = buf.fill_buf().unwrap();
+        if data.len() == 0 {
+            return Err(());
+        }
+
+        let (header, decoder) = start_decoding_multi_message(data)
+            .header().unwrap();
+
+        let (fields, decoder) = decoder.multi_message_fields().unwrap();
+        let mut msg_decoder = decoder.messages_individually().unwrap();
+        while let Either::Left(msg) = msg_decoder {
+            let (member, sym_dec) = msg.next_messages_member().unwrap();
+            let (sym, next_msg_dec) = sym_dec.symbol().unwrap();
+            match member.msg_type {
+                marketdata_sbe::MsgType::Trade => stats.append_trade_volume(
+                    unsafe { from_utf8_unchecked(sym) },
+                    member.trade.size as u64,
+                ),
+                marketdata_sbe::MsgType::Quote => stats.update_quote_prices(
+                    unsafe { from_utf8_unchecked(sym) },
+                    member.quote.price,
+                    match member.quote.side {
+                        Side::Buy => true,
+                        _ => false
+                    },
+                ),
+                _ => ()
+            }
+            msg_decoder = next_msg_dec;
+        }
+
+        // We now have a `Right`, which is a finished messages block
+        let msg_decoder = match msg_decoder {
+            Either::Right(r) => r,
+            _ => panic!("Didn't parse all messages")
+        };
+
+        // Interestingly enough, `buf.consume(msg_decoder.unwrap())` isn't OK,
+        // presumably something to do with when *precisely* the drop of `self`
+        // happens for `msg_decoder`. Leave it as two statments so that
+        // Rust is able to prove our immutable borrow of `data` ends in time
+        // to consume the buffer
+        let msg_len = msg_decoder.unwrap();
+        buf.consume(msg_len);
+        Ok(())
+    }
+}

tests/capnp.rs

@@ -1,50 +0,0 @@
-use alloc_counter::{AllocCounterSystem, count_alloc, deny_alloc};
-
-use md_shootout::marketdata_capnp::multi_message;
-
-#[global_allocator]
-static A: AllocCounterSystem = AllocCounterSystem;
-
-#[test]
-fn reinit_memory_check() {
-    // Setting up the builder doesn't reserve any heap memory
-    let mut msg_block = deny_alloc(|| {
-        capnp::message::Builder::new_default()
-    });
-
-    // Setting up the root object, however, does reserve a first segment
-    let (stats, result) = count_alloc(|| {
-        let multimsg = msg_block.init_root::<multi_message::Builder>();
-        multimsg.init_messages(32);
-    });
-
-    assert_eq!(stats.0, 4);
-    assert_eq!(stats.1, 0);
-    assert_eq!(stats.2, 0);
-
-    // If we reinitialize an object on that original builder, we re-use memory
-    deny_alloc(|| {
-        let multimsg = msg_block.init_root::<multi_message::Builder>();
-        multimsg.init_messages(32);
-
-        // Even if we down-size and up-size the message list size, we don't need
-        // to re-allocate
-        let multimsg = msg_block.init_root::<multi_message::Builder>();
-        multimsg.init_messages(16);
-
-        let multimsg = msg_block.init_root::<multi_message::Builder>();
-        multimsg.init_messages(32);
-    });
-
-    // It's only when we init a larger message count that a fresh allocation occurs
-    let (stats, _) = count_alloc(|| {
-        let multimsg = msg_block.init_root::<multi_message::Builder>();
-        // Note: calling `init_messages(33)` doesn't force allocation because
-        // the Capnproto builder reserved extra memory the first time around
-        multimsg.init_messages(256);
-    });
-
-    assert_eq!(stats.0, 1);
-    assert_eq!(stats.1, 3);
-    assert_eq!(stats.2, 0);
-}