Add run analysis code, and clean up most warnings
Cap'n Proto has a bug in deserialization; both in packed and unpacked, it gets to 43116 of 62253 messages.master
parent
f151d86604
commit
369613843d
|
|
@ -1,5 +1,10 @@
|
|||
# This file is automatically @generated by Cargo.
|
||||
# It is not intended for manual editing.
|
||||
[[package]]
|
||||
name = "adler32"
|
||||
version = "1.0.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
|
||||
[[package]]
|
||||
name = "alloc_counter"
|
||||
version = "0.0.2"
|
||||
|
|
@ -35,11 +40,29 @@ dependencies = [
|
|||
"winapi 0.3.7 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "autocfg"
|
||||
version = "0.1.6"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
|
||||
[[package]]
|
||||
name = "base64"
|
||||
version = "0.10.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
dependencies = [
|
||||
"byteorder 1.3.2 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "bitflags"
|
||||
version = "1.1.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
|
||||
[[package]]
|
||||
name = "byteorder"
|
||||
version = "1.3.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
|
||||
[[package]]
|
||||
name = "capnp"
|
||||
version = "0.10.1"
|
||||
|
|
@ -53,6 +76,11 @@ dependencies = [
|
|||
"capnp 0.10.1 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "cc"
|
||||
version = "1.0.42"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
|
||||
[[package]]
|
||||
name = "cfg-if"
|
||||
version = "0.1.9"
|
||||
|
|
@ -72,6 +100,31 @@ dependencies = [
|
|||
"vec_map 0.8.1 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "crc32fast"
|
||||
version = "1.2.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
dependencies = [
|
||||
"cfg-if 0.1.9 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "crossbeam-channel"
|
||||
version = "0.3.9"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
dependencies = [
|
||||
"crossbeam-utils 0.6.6 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "crossbeam-utils"
|
||||
version = "0.6.6"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
dependencies = [
|
||||
"cfg-if 0.1.9 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"lazy_static 1.4.0 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "flatbuffers"
|
||||
version = "0.6.0"
|
||||
|
|
@ -88,6 +141,35 @@ dependencies = [
|
|||
"log 0.4.8 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "flate2"
|
||||
version = "1.0.11"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
dependencies = [
|
||||
"crc32fast 1.2.0 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"libc 0.2.62 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"miniz-sys 0.1.12 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"miniz_oxide 0.3.2 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "hdrhistogram"
|
||||
version = "6.3.4"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
dependencies = [
|
||||
"base64 0.10.1 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"byteorder 1.3.2 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"crossbeam-channel 0.3.9 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"flate2 1.0.11 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"nom 4.2.3 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"num-traits 0.2.8 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "lazy_static"
|
||||
version = "1.4.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
|
||||
[[package]]
|
||||
name = "lexical-core"
|
||||
version = "0.4.3"
|
||||
|
|
@ -123,6 +205,7 @@ dependencies = [
|
|||
"clap 2.33.0 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"flatbuffers 0.6.0 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"flatc-rust 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"hdrhistogram 6.3.4 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"nom 5.0.0 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"smallvec 0.6.10 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
]
|
||||
|
|
@ -132,6 +215,32 @@ name = "memchr"
|
|||
version = "2.2.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
|
||||
[[package]]
|
||||
name = "miniz-sys"
|
||||
version = "0.1.12"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
dependencies = [
|
||||
"cc 1.0.42 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"libc 0.2.62 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "miniz_oxide"
|
||||
version = "0.3.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
dependencies = [
|
||||
"adler32 1.0.3 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "nom"
|
||||
version = "4.2.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
dependencies = [
|
||||
"memchr 2.2.1 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"version_check 0.1.5 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "nom"
|
||||
version = "5.0.0"
|
||||
|
|
@ -142,6 +251,14 @@ dependencies = [
|
|||
"version_check 0.1.5 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "num-traits"
|
||||
version = "0.2.8"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
dependencies = [
|
||||
"autocfg 0.1.6 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "proc-macro2"
|
||||
version = "0.4.30"
|
||||
|
|
@ -279,22 +396,37 @@ version = "0.4.0"
|
|||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
|
||||
[metadata]
|
||||
"checksum adler32 1.0.3 (registry+https://github.com/rust-lang/crates.io-index)" = "7e522997b529f05601e05166c07ed17789691f562762c7f3b987263d2dedee5c"
|
||||
"checksum alloc_counter 0.0.2 (registry+https://github.com/rust-lang/crates.io-index)" = "a169230586814a38a47b9764bb5e5310120df93952df01ee3ea9d832aef10e2f"
|
||||
"checksum alloc_counter_macro 0.0.1 (registry+https://github.com/rust-lang/crates.io-index)" = "c37d9ddd812e5223e8de74a2152fa79dce52ca9f6af38a54c6bcd1ae8b26a05d"
|
||||
"checksum ansi_term 0.11.0 (registry+https://github.com/rust-lang/crates.io-index)" = "ee49baf6cb617b853aa8d93bf420db2383fab46d314482ca2803b40d5fde979b"
|
||||
"checksum atty 0.2.13 (registry+https://github.com/rust-lang/crates.io-index)" = "1803c647a3ec87095e7ae7acfca019e98de5ec9a7d01343f611cf3152ed71a90"
|
||||
"checksum autocfg 0.1.6 (registry+https://github.com/rust-lang/crates.io-index)" = "b671c8fb71b457dd4ae18c4ba1e59aa81793daacc361d82fcd410cef0d491875"
|
||||
"checksum base64 0.10.1 (registry+https://github.com/rust-lang/crates.io-index)" = "0b25d992356d2eb0ed82172f5248873db5560c4721f564b13cb5193bda5e668e"
|
||||
"checksum bitflags 1.1.0 (registry+https://github.com/rust-lang/crates.io-index)" = "3d155346769a6855b86399e9bc3814ab343cd3d62c7e985113d46a0ec3c281fd"
|
||||
"checksum byteorder 1.3.2 (registry+https://github.com/rust-lang/crates.io-index)" = "a7c3dd8985a7111efc5c80b44e23ecdd8c007de8ade3b96595387e812b957cf5"
|
||||
"checksum capnp 0.10.1 (registry+https://github.com/rust-lang/crates.io-index)" = "3a31dc984081bce3531cb7e1c69b5b926904095baf90be9da42df07af54e123a"
|
||||
"checksum capnpc 0.10.1 (registry+https://github.com/rust-lang/crates.io-index)" = "6c1a2eead5ee094e8adb30edd6ae55a139b42b8dbe27cc7e4007553ea2b5eb01"
|
||||
"checksum cc 1.0.42 (registry+https://github.com/rust-lang/crates.io-index)" = "a61c7bce55cd2fae6ec8cb935ebd76256c2959a1f95790f6118a441c2cd5b406"
|
||||
"checksum cfg-if 0.1.9 (registry+https://github.com/rust-lang/crates.io-index)" = "b486ce3ccf7ffd79fdeb678eac06a9e6c09fc88d33836340becb8fffe87c5e33"
|
||||
"checksum clap 2.33.0 (registry+https://github.com/rust-lang/crates.io-index)" = "5067f5bb2d80ef5d68b4c87db81601f0b75bca627bc2ef76b141d7b846a3c6d9"
|
||||
"checksum crc32fast 1.2.0 (registry+https://github.com/rust-lang/crates.io-index)" = "ba125de2af0df55319f41944744ad91c71113bf74a4646efff39afe1f6842db1"
|
||||
"checksum crossbeam-channel 0.3.9 (registry+https://github.com/rust-lang/crates.io-index)" = "c8ec7fcd21571dc78f96cc96243cab8d8f035247c3efd16c687be154c3fa9efa"
|
||||
"checksum crossbeam-utils 0.6.6 (registry+https://github.com/rust-lang/crates.io-index)" = "04973fa96e96579258a5091af6003abde64af786b860f18622b82e026cca60e6"
|
||||
"checksum flatbuffers 0.6.0 (registry+https://github.com/rust-lang/crates.io-index)" = "2fc1af59fd8248b59beb048d614a869ce211315c195f5412334e47f5b7e22726"
|
||||
"checksum flatc-rust 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)" = "b37a2ed85bee7b6aa0d5305b4765bf4cc0f0cfbc25b86d524126a1ab755f6aed"
|
||||
"checksum flate2 1.0.11 (registry+https://github.com/rust-lang/crates.io-index)" = "2adaffba6388640136149e18ed080b77a78611c1e1d6de75aedcdf78df5d4682"
|
||||
"checksum hdrhistogram 6.3.4 (registry+https://github.com/rust-lang/crates.io-index)" = "08d331ebcdbca4acbefe5da8c3299b2e246f198a8294cc5163354e743398b89d"
|
||||
"checksum lazy_static 1.4.0 (registry+https://github.com/rust-lang/crates.io-index)" = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646"
|
||||
"checksum lexical-core 0.4.3 (registry+https://github.com/rust-lang/crates.io-index)" = "b8b0f90c979adde96d19eb10eb6431ba0c441e2f9e9bdff868b2f6f5114ff519"
|
||||
"checksum libc 0.2.62 (registry+https://github.com/rust-lang/crates.io-index)" = "34fcd2c08d2f832f376f4173a231990fa5aef4e99fb569867318a227ef4c06ba"
|
||||
"checksum log 0.4.8 (registry+https://github.com/rust-lang/crates.io-index)" = "14b6052be84e6b71ab17edffc2eeabf5c2c3ae1fdb464aae35ac50c67a44e1f7"
|
||||
"checksum memchr 2.2.1 (registry+https://github.com/rust-lang/crates.io-index)" = "88579771288728879b57485cc7d6b07d648c9f0141eb955f8ab7f9d45394468e"
|
||||
"checksum miniz-sys 0.1.12 (registry+https://github.com/rust-lang/crates.io-index)" = "1e9e3ae51cea1576ceba0dde3d484d30e6e5b86dee0b2d412fe3a16a15c98202"
|
||||
"checksum miniz_oxide 0.3.2 (registry+https://github.com/rust-lang/crates.io-index)" = "7108aff85b876d06f22503dcce091e29f76733b2bfdd91eebce81f5e68203a10"
|
||||
"checksum nom 4.2.3 (registry+https://github.com/rust-lang/crates.io-index)" = "2ad2a91a8e869eeb30b9cb3119ae87773a8f4ae617f41b1eb9c154b2905f7bd6"
|
||||
"checksum nom 5.0.0 (registry+https://github.com/rust-lang/crates.io-index)" = "e9761d859320e381010a4f7f8ed425f2c924de33ad121ace447367c713ad561b"
|
||||
"checksum num-traits 0.2.8 (registry+https://github.com/rust-lang/crates.io-index)" = "6ba9a427cfca2be13aa6f6403b0b7e7368fe982bfa16fccc450ce74c46cd9b32"
|
||||
"checksum proc-macro2 0.4.30 (registry+https://github.com/rust-lang/crates.io-index)" = "cf3d2011ab5c909338f7887f4fc896d35932e29146c12c8d01da6b22a80ba759"
|
||||
"checksum quote 0.6.13 (registry+https://github.com/rust-lang/crates.io-index)" = "6ce23b6b870e8f94f81fb0a363d65d86675884b34a09043c81e5562f11c1f8e1"
|
||||
"checksum rustc_version 0.2.3 (registry+https://github.com/rust-lang/crates.io-index)" = "138e3e0acb6c9fb258b19b67cb8abd63c00679d2851805ea151465464fe9030a"
|
||||
|
|
|
|||
|
|
@ -8,6 +8,7 @@ edition = "2018"
|
|||
capnp = "0.10.1"
|
||||
clap = "2.33.0"
|
||||
flatbuffers = "0.6.0"
|
||||
hdrhistogram = "6.3.4"
|
||||
nom = "5.0.0"
|
||||
smallvec = "0.6.10"
|
||||
|
||||
|
|
|
|||
6
build.rs
6
build.rs
|
|
@ -15,7 +15,8 @@ fn main() {
|
|||
inputs: &[Path::new("marketdata.fbs")],
|
||||
out_dir: Path::new("src/"),
|
||||
..Default::default()
|
||||
}).expect("Unable to compile flatc");
|
||||
})
|
||||
.expect("Unable to compile flatc");
|
||||
|
||||
// There's no Rust-style builder crate for SBE,
|
||||
// so we need to run the command by hand.
|
||||
|
|
@ -25,7 +26,8 @@ fn main() {
|
|||
.arg("-Dsbe.xinclude.aware=true")
|
||||
.arg("-Dsbe.target.language=uk.co.real_logic.sbe.generation.rust.Rust")
|
||||
.arg("-Dsbe.target.namespace=marketdata_sbe")
|
||||
.arg("-jar").arg("sbe-all-1.13.2-all.jar")
|
||||
.arg("-jar")
|
||||
.arg("sbe-all-1.13.2-all.jar")
|
||||
.arg("marketdata.xml")
|
||||
.output()
|
||||
.expect("Unable to execute SBE compiler");
|
||||
|
|
|
|||
|
|
@ -1,26 +1,25 @@
|
|||
use std::convert::TryInto;
|
||||
use std::io::{BufRead, Read};
|
||||
use std::mem::size_of;
|
||||
use std::str::from_utf8_unchecked;
|
||||
|
||||
use capnp::Error;
|
||||
use capnp::message::{Builder, ReaderOptions, ScratchSpace, ScratchSpaceHeapAllocator};
|
||||
use capnp::message::{ReaderOptions, ScratchSpace, ScratchSpaceHeapAllocator};
|
||||
use capnp::serialize::write_message;
|
||||
use capnp::serialize_packed::{read_message as read_message_packed, write_message as write_message_packed};
|
||||
use nom::bytes::complete::take_until;
|
||||
use nom::IResult;
|
||||
use capnp::serialize_packed::{
|
||||
read_message as read_message_packed, write_message as write_message_packed,
|
||||
};
|
||||
use capnp::Error;
|
||||
|
||||
use crate::{RunnerDeserialize, RunnerSerialize, StreamVec, Summarizer};
|
||||
use crate::iex::{IexMessage, IexPayload};
|
||||
use crate::marketdata_capnp::{multi_message, Side};
|
||||
use crate::marketdata_capnp::message;
|
||||
use crate::marketdata_capnp::{multi_message, Side};
|
||||
use crate::{RunnerDeserialize, RunnerSerialize, StreamVec, Summarizer};
|
||||
|
||||
pub struct CapnpWriter<'a> {
|
||||
// We have to be very careful with how messages are built, as running
|
||||
// `init_root` and rebuilding will still accumulate garbage if using
|
||||
// the standard HeapAllocator.
|
||||
// https://github.com/capnproto/capnproto-rust/issues/111
|
||||
words: Vec<capnp::Word>,
|
||||
_words: Vec<capnp::Word>,
|
||||
scratch: ScratchSpace<'a>,
|
||||
packed: bool,
|
||||
}
|
||||
|
|
@ -31,14 +30,12 @@ impl<'a> CapnpWriter<'a> {
|
|||
// In practice, let's just make sure everything fits.
|
||||
let mut words = capnp::Word::allocate_zeroed_vec(1024);
|
||||
|
||||
let mut scratch = ScratchSpace::new(unsafe {
|
||||
std::mem::transmute(&mut words[..])
|
||||
});
|
||||
let scratch = ScratchSpace::new(unsafe { std::mem::transmute(&mut words[..]) });
|
||||
|
||||
CapnpWriter {
|
||||
words,
|
||||
_words: words,
|
||||
scratch,
|
||||
packed
|
||||
packed,
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -56,12 +53,14 @@ impl<'a> CapnpWriter<'a> {
|
|||
impl<'a> RunnerSerialize for CapnpWriter<'a> {
|
||||
fn serialize(&mut self, payload: &IexPayload, mut output: &mut Vec<u8>) {
|
||||
// First, count the messages we actually care about.
|
||||
let num_msgs = payload.messages.iter().map(|m| {
|
||||
match m {
|
||||
let num_msgs = payload
|
||||
.messages
|
||||
.iter()
|
||||
.map(|m| match m {
|
||||
IexMessage::TradeReport(_) | IexMessage::PriceLevelUpdate(_) => 1,
|
||||
_ => 0
|
||||
}
|
||||
}).fold(0, |sum, i| sum + i);
|
||||
_ => 0,
|
||||
})
|
||||
.fold(0, |sum, i| sum + i);
|
||||
|
||||
if num_msgs == 0 {
|
||||
return;
|
||||
|
|
@ -109,13 +108,21 @@ impl<'a> RunnerSerialize for CapnpWriter<'a> {
|
|||
msg_plu.set_price(plu.price);
|
||||
msg_plu.set_size(plu.size);
|
||||
msg_plu.set_flags(plu.event_flags);
|
||||
msg_plu.set_side(if plu.msg_type == 0x38 { Side::Buy } else { Side::Sell });
|
||||
msg_plu.set_side(if plu.msg_type == 0x38 {
|
||||
Side::Buy
|
||||
} else {
|
||||
Side::Sell
|
||||
});
|
||||
}
|
||||
_ => ()
|
||||
_ => (),
|
||||
}
|
||||
}
|
||||
|
||||
let write_fn = if self.packed { write_message_packed } else { write_message };
|
||||
let write_fn = if self.packed {
|
||||
write_message_packed
|
||||
} else {
|
||||
write_message
|
||||
};
|
||||
|
||||
write_fn(&mut output, &builder).unwrap();
|
||||
}
|
||||
|
|
@ -123,25 +130,28 @@ impl<'a> RunnerSerialize for CapnpWriter<'a> {
|
|||
|
||||
pub struct CapnpReader {
|
||||
read_opts: ReaderOptions,
|
||||
packed: bool
|
||||
packed: bool,
|
||||
}
|
||||
|
||||
impl CapnpReader {
|
||||
pub fn new(packed: bool) -> CapnpReader {
|
||||
CapnpReader {
|
||||
read_opts: ReaderOptions::new(),
|
||||
packed
|
||||
packed,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl CapnpReader {
|
||||
fn deserialize_packed<'a>(&self, buf: &'a mut StreamVec, stats: &mut Summarizer) -> Result<(), ()> {
|
||||
fn deserialize_packed<'a>(
|
||||
&self,
|
||||
buf: &'a mut StreamVec,
|
||||
stats: &mut Summarizer,
|
||||
) -> Result<(), ()> {
|
||||
// Because `capnp::serialize_packed::PackedRead` is hidden from us, packed reads
|
||||
// *have* to both allocate new segments every read, and copy the buffer into
|
||||
// those same segments, no ability to re-use allocated memory.
|
||||
let reader = read_message_packed(buf, self.read_opts)
|
||||
.map_err(|_| ())?;
|
||||
let reader = read_message_packed(buf, self.read_opts).map_err(|_| ())?;
|
||||
|
||||
let multimsg = reader.get_root::<multi_message::Reader>().unwrap();
|
||||
for msg in multimsg.get_messages().unwrap().iter() {
|
||||
|
|
@ -149,18 +159,18 @@ impl CapnpReader {
|
|||
Ok(message::Trade(tr)) => {
|
||||
let tr = tr.unwrap();
|
||||
stats.append_trade_volume(msg.get_symbol().unwrap(), tr.get_size() as u64);
|
||||
},
|
||||
}
|
||||
Ok(message::Quote(q)) => {
|
||||
let q = q.unwrap();
|
||||
let is_bid = match q.get_side().unwrap() {
|
||||
Side::Buy => true,
|
||||
_ => false
|
||||
_ => false,
|
||||
};
|
||||
stats.update_quote_prices(msg.get_symbol().unwrap(), q.get_price(), is_bid);
|
||||
},
|
||||
_ => panic!("Unrecognized message type!")
|
||||
}
|
||||
};
|
||||
_ => panic!("Unrecognized message type!"),
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
|
|
@ -192,8 +202,7 @@ impl CapnpReader {
|
|||
There is no documentation on how to calculate `bytes_consumed` when parsing by hand
|
||||
that I could find, you just have to guess and check until you figure this one out.
|
||||
*/
|
||||
let (num_words, offsets) = read_segment_table(&mut data, reader_opts)
|
||||
.map_err(|_| ())?;
|
||||
let (num_words, offsets) = read_segment_table(&mut data, reader_opts).map_err(|_| ())?;
|
||||
let words = unsafe { capnp::Word::bytes_to_words(data) };
|
||||
let reader = capnp::message::Reader::new(
|
||||
SliceSegments {
|
||||
|
|
@ -206,8 +215,7 @@ impl CapnpReader {
|
|||
let msg_bytes = num_words * size_of::<capnp::Word>();
|
||||
let bytes_consumed = segment_table_bytes + msg_bytes;
|
||||
|
||||
let multimsg = reader.get_root::<multi_message::Reader>()
|
||||
.map_err(|_| ())?;
|
||||
let multimsg = reader.get_root::<multi_message::Reader>().map_err(|_| ())?;
|
||||
for msg in multimsg.get_messages().map_err(|_| ())?.iter() {
|
||||
let sym = msg.get_symbol().map_err(|_| ())?;
|
||||
|
||||
|
|
@ -215,15 +223,15 @@ impl CapnpReader {
|
|||
message::Trade(trade) => {
|
||||
let trade = trade.unwrap();
|
||||
stats.append_trade_volume(sym, trade.get_size().into());
|
||||
},
|
||||
}
|
||||
message::Quote(quote) => {
|
||||
let quote = quote.unwrap();
|
||||
let is_buy = match quote.get_side().unwrap() {
|
||||
Side::Buy => true,
|
||||
_ => false
|
||||
_ => false,
|
||||
};
|
||||
stats.update_quote_prices(sym, quote.get_price(), is_buy);
|
||||
},
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -244,7 +252,6 @@ impl RunnerDeserialize for CapnpReader {
|
|||
}
|
||||
}
|
||||
|
||||
|
||||
pub struct SliceSegments<'a> {
|
||||
words: &'a [capnp::Word],
|
||||
segment_slices: Vec<(usize, usize)>,
|
||||
|
|
@ -265,10 +272,12 @@ impl<'a> capnp::message::ReaderSegments for SliceSegments<'a> {
|
|||
}
|
||||
}
|
||||
|
||||
fn read_segment_table<R>(read: &mut R,
|
||||
options: capnp::message::ReaderOptions)
|
||||
-> capnp::Result<(usize, Vec<(usize, usize)>)>
|
||||
where R: Read
|
||||
fn read_segment_table<R>(
|
||||
read: &mut R,
|
||||
options: capnp::message::ReaderOptions,
|
||||
) -> capnp::Result<(usize, Vec<(usize, usize)>)>
|
||||
where
|
||||
R: Read,
|
||||
{
|
||||
let mut buf: [u8; 8] = [0; 8];
|
||||
|
||||
|
|
@ -277,9 +286,15 @@ fn read_segment_table<R>(read: &mut R,
|
|||
let segment_count = u32::from_le_bytes(buf[0..4].try_into().unwrap()).wrapping_add(1) as usize;
|
||||
|
||||
if segment_count >= 512 {
|
||||
return Err(Error::failed(format!("Too many segments: {}", segment_count)))
|
||||
return Err(Error::failed(format!(
|
||||
"Too many segments: {}",
|
||||
segment_count
|
||||
)));
|
||||
} else if segment_count == 0 {
|
||||
return Err(Error::failed(format!("Too few segments: {}", segment_count)))
|
||||
return Err(Error::failed(format!(
|
||||
"Too few segments: {}",
|
||||
segment_count
|
||||
)));
|
||||
}
|
||||
|
||||
let mut segment_slices = Vec::with_capacity(segment_count);
|
||||
|
|
@ -301,7 +316,8 @@ fn read_segment_table<R>(read: &mut R,
|
|||
read.read_exact(&mut segment_sizes[..])?;
|
||||
for idx in 0..(segment_count - 1) {
|
||||
let segment_len =
|
||||
u32::from_le_bytes(segment_sizes[(idx * 4)..(idx + 1) * 4].try_into().unwrap()) as usize;
|
||||
u32::from_le_bytes(segment_sizes[(idx * 4)..(idx + 1) * 4].try_into().unwrap())
|
||||
as usize;
|
||||
|
||||
segment_slices.push((total_words, total_words + segment_len));
|
||||
total_words += segment_len;
|
||||
|
|
@ -313,9 +329,11 @@ fn read_segment_table<R>(read: &mut R,
|
|||
// traversal limit. Without this check, a malicious client could transmit a very large segment
|
||||
// size to make the receiver allocate excessive space and possibly crash.
|
||||
if total_words as u64 > options.traversal_limit_in_words {
|
||||
return Err(Error::failed(
|
||||
format!("Message has {} words, which is too large. To increase the limit on the \
|
||||
receiving end, see capnp::message::ReaderOptions.", total_words)))
|
||||
return Err(Error::failed(format!(
|
||||
"Message has {} words, which is too large. To increase the limit on the \
|
||||
receiving end, see capnp::message::ReaderOptions.",
|
||||
total_words
|
||||
)));
|
||||
}
|
||||
|
||||
Ok((total_words, segment_slices))
|
||||
|
|
|
|||
|
|
@ -1,15 +1,10 @@
|
|||
use std::convert::TryInto;
|
||||
use std::io::{BufRead, Error, Write};
|
||||
use std::io::{BufRead, Write};
|
||||
use std::mem::size_of;
|
||||
use std::str::from_utf8_unchecked;
|
||||
|
||||
use capnp::data::new_builder;
|
||||
use flatbuffers::buffer_has_identifier;
|
||||
use nom::{bytes::complete::take_until, IResult};
|
||||
|
||||
use crate::{RunnerDeserialize, RunnerSerialize, StreamVec, Summarizer};
|
||||
use crate::iex::{IexMessage, IexPayload};
|
||||
use crate::marketdata_generated::md_shootout;
|
||||
use crate::{RunnerDeserialize, RunnerSerialize, StreamVec, Summarizer};
|
||||
|
||||
pub struct FlatbuffersWriter<'a> {
|
||||
builder: flatbuffers::FlatBufferBuilder<'a>,
|
||||
|
|
@ -27,7 +22,6 @@ impl<'a> FlatbuffersWriter<'a> {
|
|||
|
||||
impl<'a> RunnerSerialize for FlatbuffersWriter<'a> {
|
||||
fn serialize(&mut self, payload: &IexPayload, output: &mut Vec<u8>) {
|
||||
|
||||
// Because FlatBuffers can't handle nested vectors (specifically, we can't track
|
||||
// both the variable-length vector of messages, and the variable-length strings
|
||||
// within those messages), we have to cache the messages as they get built
|
||||
|
|
@ -61,7 +55,11 @@ impl<'a> RunnerSerialize for FlatbuffersWriter<'a> {
|
|||
price: plu.price,
|
||||
size_: plu.size,
|
||||
flags: plu.event_flags,
|
||||
side: if plu.msg_type == 0x38 { md_shootout::Side::Buy } else { md_shootout::Side::Sell },
|
||||
side: if plu.msg_type == 0x38 {
|
||||
md_shootout::Side::Buy
|
||||
} else {
|
||||
md_shootout::Side::Sell
|
||||
},
|
||||
},
|
||||
);
|
||||
|
||||
|
|
@ -73,7 +71,7 @@ impl<'a> RunnerSerialize for FlatbuffersWriter<'a> {
|
|||
body: Some(level_update.as_union_value()),
|
||||
})
|
||||
}
|
||||
_ => None
|
||||
_ => None,
|
||||
};
|
||||
|
||||
msg_args.map(|a| {
|
||||
|
|
@ -118,7 +116,7 @@ impl RunnerDeserialize for FlatbuffersReader {
|
|||
// a view over the underlying buffer.
|
||||
let data = buf.fill_buf().unwrap();
|
||||
if data.len() == 0 {
|
||||
return Err(())
|
||||
return Err(());
|
||||
}
|
||||
|
||||
let msg_len_buf: [u8; 4] = data[..size_of::<u32>()].try_into().unwrap();
|
||||
|
|
@ -127,7 +125,7 @@ impl RunnerDeserialize for FlatbuffersReader {
|
|||
let multimsg = flatbuffers::get_size_prefixed_root::<md_shootout::MultiMessage>(data);
|
||||
let msg_vec = match multimsg.messages() {
|
||||
Some(m) => m,
|
||||
None => panic!("Couldn't find messages")
|
||||
None => panic!("Couldn't find messages"),
|
||||
};
|
||||
|
||||
for i in 0..msg_vec.len() {
|
||||
|
|
@ -136,16 +134,16 @@ impl RunnerDeserialize for FlatbuffersReader {
|
|||
md_shootout::MessageBody::Trade => {
|
||||
let trade = msg.body_as_trade().unwrap();
|
||||
stats.append_trade_volume(msg.symbol().unwrap(), trade.size_().into());
|
||||
},
|
||||
}
|
||||
md_shootout::MessageBody::LevelUpdate => {
|
||||
let lu = msg.body_as_level_update().unwrap();
|
||||
let is_bid = match lu.side() {
|
||||
md_shootout::Side::Buy => true,
|
||||
_ => false
|
||||
_ => false,
|
||||
};
|
||||
stats.update_quote_prices(msg.symbol().unwrap(), lu.price(), is_bid);
|
||||
},
|
||||
md_shootout::MessageBody::NONE => panic!("Unrecognized message type")
|
||||
}
|
||||
md_shootout::MessageBody::NONE => panic!("Unrecognized message type"),
|
||||
}
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -1,8 +1,8 @@
|
|||
use std::convert::TryInto;
|
||||
|
||||
use nom::{bytes::complete::take, IResult, number::complete::*, sequence::tuple};
|
||||
use nom::{bytes::complete::take, number::complete::*, sequence::tuple, IResult};
|
||||
|
||||
use crate::parsers::{Block, extract_iex_data, read_block};
|
||||
use crate::parsers::{extract_iex_data, read_block, Block};
|
||||
|
||||
pub struct IexParser<'a> {
|
||||
pcap_buffer: &'a [u8],
|
||||
|
|
|
|||
194
src/main.rs
194
src/main.rs
|
|
@ -2,13 +2,14 @@ use std::cmp::{max, min};
|
|||
use std::collections::hash_map::{DefaultHasher, HashMap};
|
||||
use std::fs::File;
|
||||
use std::hash::Hasher;
|
||||
use std::io::{BufRead, Read};
|
||||
use std::io::Error;
|
||||
use std::io::{BufRead, Read};
|
||||
use std::path::Path;
|
||||
use std::str::from_utf8_unchecked;
|
||||
use std::time::{Instant, SystemTime};
|
||||
use std::time::Instant;
|
||||
|
||||
use clap::{App, Arg};
|
||||
use hdrhistogram::Histogram;
|
||||
use nom::{bytes::complete::take_until, IResult};
|
||||
|
||||
use crate::iex::{IexParser, IexPayload};
|
||||
|
|
@ -19,13 +20,14 @@ use crate::iex::{IexParser, IexPayload};
|
|||
pub mod marketdata_capnp;
|
||||
#[allow(unused_imports)]
|
||||
pub mod marketdata_generated; // Flatbuffers
|
||||
#[allow(dead_code)]
|
||||
pub mod marketdata_sbe;
|
||||
|
||||
mod capnp_runner;
|
||||
mod flatbuffers_runner;
|
||||
mod sbe_runner;
|
||||
mod iex;
|
||||
mod parsers;
|
||||
mod sbe_runner;
|
||||
|
||||
fn main() {
|
||||
let matches = App::new("Marketdata Shootout")
|
||||
|
|
@ -48,84 +50,32 @@ fn main() {
|
|||
file.read_to_end(&mut buf)
|
||||
.expect(&format!("Unable to read file={}", path.display()));
|
||||
|
||||
let _start = SystemTime::now();
|
||||
let mut summarizer = Summarizer::default();
|
||||
let mut parser = IexParser::new(&buf[..]);
|
||||
let _capnp_unpacked = run_analysis(
|
||||
&buf,
|
||||
&mut capnp_runner::CapnpWriter::new(false),
|
||||
&mut capnp_runner::CapnpReader::new(false),
|
||||
);
|
||||
|
||||
// Pre-allocate the same size as the backing file. Will be way more than
|
||||
// necessary, but makes sure there's no re-allocation not related to
|
||||
// actual parsing/serialization code
|
||||
let mut output_buf: Vec<u8> = Vec::with_capacity(buf.capacity());
|
||||
let _capnp_packed = run_analysis(
|
||||
&buf,
|
||||
&mut capnp_runner::CapnpWriter::new(true),
|
||||
&mut capnp_runner::CapnpReader::new(true),
|
||||
);
|
||||
|
||||
/*
|
||||
let mut capnp_writer = capnp_runner::CapnpWriter::new();
|
||||
for iex_payload in parser {
|
||||
//let iex_payload = parser.next().unwrap();
|
||||
capnp_writer.serialize(&iex_payload, &mut output_buf, true);
|
||||
let _flatbuffers = run_analysis(
|
||||
&buf,
|
||||
&mut flatbuffers_runner::FlatbuffersWriter::new(),
|
||||
&mut flatbuffers_runner::FlatbuffersReader::new(),
|
||||
);
|
||||
|
||||
let _sbe = run_analysis(
|
||||
&buf,
|
||||
&mut sbe_runner::SBEWriter::new(),
|
||||
&mut sbe_runner::SBEReader::new(),
|
||||
);
|
||||
}
|
||||
|
||||
let capnp_reader = capnp_runner::CapnpReader::new();
|
||||
let mut read_buf = StreamVec::new(output_buf);
|
||||
let mut parsed_msgs: u64 = 0;
|
||||
while let Ok(_) = capnp_reader.deserialize_packed(&mut read_buf, &mut summarizer) {
|
||||
parsed_msgs += 1;
|
||||
}
|
||||
*/
|
||||
|
||||
let mut fb_writer = flatbuffers_runner::FlatbuffersWriter::new();
|
||||
for iex_payload in parser {
|
||||
let now = Instant::now();
|
||||
fb_writer.serialize(&iex_payload, &mut output_buf);
|
||||
let serialize_nanos = Instant::now().duration_since(now).as_nanos();
|
||||
dbg!(serialize_nanos);
|
||||
}
|
||||
|
||||
let mut read_buf = StreamVec::new(output_buf);
|
||||
|
||||
let fb_reader = flatbuffers_runner::FlatbuffersReader::new();
|
||||
let mut parsed_msgs = 0;
|
||||
while let Ok(_) = fb_reader.deserialize(&mut read_buf, &mut summarizer) {
|
||||
parsed_msgs += 1;
|
||||
}
|
||||
|
||||
/*
|
||||
let mut capnp_writer = capnp_runner::CapnpWriter::new();
|
||||
for iex_payload in parser {
|
||||
//let iex_payload = parser.next().unwrap();
|
||||
let now = Instant::now();
|
||||
capnp_writer.serialize(&iex_payload, &mut output_buf, false);
|
||||
let serialize_nanos = Instant::now().duration_since(now).as_nanos();
|
||||
dbg!(serialize_nanos);
|
||||
}
|
||||
|
||||
let capnp_reader = capnp_runner::CapnpReader::new();
|
||||
let mut read_buf = StreamVec::new(output_buf);
|
||||
let mut parsed_msgs: u64 = 0;
|
||||
while let Ok(_) = capnp_reader.deserialize_unpacked(&mut read_buf, &mut summarizer) {
|
||||
parsed_msgs += 1;
|
||||
}
|
||||
*/
|
||||
|
||||
/*
|
||||
let mut sbe_writer = sbe_runner::SBEWriter::new();
|
||||
for iex_payload in parser {
|
||||
//let iex_payload = parser.next().unwrap();
|
||||
sbe_writer.serialize(&iex_payload, &mut output_buf);
|
||||
}
|
||||
|
||||
let sbe_reader = sbe_runner::SBEReader::new();
|
||||
let mut read_buf = StreamVec::new(output_buf);
|
||||
let mut parsed_msgs: u64 = 0;
|
||||
while let Ok(_) = sbe_reader.deserialize(&mut read_buf, &mut summarizer) {
|
||||
parsed_msgs += 1;
|
||||
}
|
||||
*/
|
||||
|
||||
dbg!(parsed_msgs);
|
||||
dbg!(summarizer);
|
||||
}
|
||||
|
||||
#[derive(Debug)]
|
||||
#[derive(Debug, PartialEq)]
|
||||
pub struct SummaryStats {
|
||||
symbol: String,
|
||||
trade_volume: u64,
|
||||
|
|
@ -135,17 +85,16 @@ pub struct SummaryStats {
|
|||
ask_low: u64,
|
||||
}
|
||||
|
||||
#[derive(Default, Debug)]
|
||||
#[derive(Default, Debug, PartialEq)]
|
||||
pub struct Summarizer {
|
||||
data: HashMap<u64, SummaryStats>
|
||||
data: HashMap<u64, SummaryStats>,
|
||||
}
|
||||
|
||||
impl Summarizer {
|
||||
fn entry(&mut self, sym: &str) -> &mut SummaryStats {
|
||||
let mut hasher = DefaultHasher::new();
|
||||
hasher.write(sym.as_bytes());
|
||||
self.data.entry(hasher.finish())
|
||||
.or_insert(SummaryStats {
|
||||
self.data.entry(hasher.finish()).or_insert(SummaryStats {
|
||||
symbol: sym.to_string(),
|
||||
trade_volume: 0,
|
||||
bid_high: 0,
|
||||
|
|
@ -178,10 +127,7 @@ pub struct StreamVec {
|
|||
|
||||
impl StreamVec {
|
||||
pub fn new(buf: Vec<u8>) -> StreamVec {
|
||||
StreamVec {
|
||||
pos: 0,
|
||||
inner: buf,
|
||||
}
|
||||
StreamVec { pos: 0, inner: buf }
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -189,7 +135,11 @@ impl Read for StreamVec {
|
|||
fn read(&mut self, buf: &mut [u8]) -> Result<usize, Error> {
|
||||
// TODO: There's *got* to be a better way to handle this
|
||||
let end = self.pos + buf.len();
|
||||
let end = if end > self.inner.len() { self.inner.len() } else { end };
|
||||
let end = if end > self.inner.len() {
|
||||
self.inner.len()
|
||||
} else {
|
||||
end
|
||||
};
|
||||
let read_size = end - self.pos;
|
||||
buf[..read_size].copy_from_slice(&self.inner[self.pos..end]);
|
||||
self.pos = end;
|
||||
|
|
@ -226,3 +176,75 @@ fn parse_symbol(sym: &[u8; 8]) -> &str {
|
|||
let (_, sym_bytes) = __take_until(" ", &sym[..]).unwrap();
|
||||
unsafe { from_utf8_unchecked(sym_bytes) }
|
||||
}
|
||||
|
||||
struct RunAnalysis {
|
||||
serialize_hist: Histogram<u64>,
|
||||
deserialize_hist: Histogram<u64>,
|
||||
summary_stats: Summarizer,
|
||||
serialize_total_nanos: u128,
|
||||
deserialize_total_nanos: u128,
|
||||
buf_len: usize,
|
||||
}
|
||||
|
||||
fn run_analysis<S, D>(iex_data: &Vec<u8>, serializer: &mut S, deserializer: &mut D) -> RunAnalysis
|
||||
where
|
||||
S: RunnerSerialize,
|
||||
D: RunnerDeserialize,
|
||||
{
|
||||
let upper = if cfg!(debug_assertions) {
|
||||
1_000_000
|
||||
} else {
|
||||
100_000
|
||||
};
|
||||
let iex_parser = IexParser::new(iex_data);
|
||||
|
||||
let mut output_buf = Vec::with_capacity(iex_data.len());
|
||||
let mut serialize_hist = Histogram::<u64>::new_with_bounds(1, upper, 2).unwrap();
|
||||
let mut serialize_nanos_total = 0u128;
|
||||
let mut serialize_msgs = 0;
|
||||
|
||||
for iex_payload in iex_parser {
|
||||
let serialize_start = Instant::now();
|
||||
|
||||
serializer.serialize(&iex_payload, &mut output_buf);
|
||||
|
||||
let serialize_end = Instant::now().duration_since(serialize_start).as_nanos();
|
||||
serialize_hist.record(serialize_end as u64).unwrap();
|
||||
serialize_nanos_total += serialize_end;
|
||||
serialize_msgs += 1;
|
||||
}
|
||||
let output_len = output_buf.len();
|
||||
|
||||
let mut read_buf = StreamVec::new(output_buf);
|
||||
let mut summarizer = Summarizer::default();
|
||||
let mut deserialize_hist = Histogram::<u64>::new_with_bounds(1, upper, 2).unwrap();
|
||||
let mut parsed_msgs: u64 = 0;
|
||||
let mut deserialize_nanos_total = 0u128;
|
||||
|
||||
loop {
|
||||
let deserialize_start = Instant::now();
|
||||
|
||||
let res = deserializer.deserialize(&mut read_buf, &mut summarizer);
|
||||
|
||||
let deserialize_end = Instant::now().duration_since(deserialize_start).as_nanos();
|
||||
|
||||
if res.is_ok() {
|
||||
deserialize_hist.record(deserialize_end as u64).unwrap();
|
||||
deserialize_nanos_total += deserialize_end;
|
||||
parsed_msgs += 1;
|
||||
} else {
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
dbg!(serialize_msgs, parsed_msgs);
|
||||
|
||||
RunAnalysis {
|
||||
serialize_hist,
|
||||
deserialize_hist,
|
||||
summary_stats: summarizer,
|
||||
serialize_total_nanos: serialize_nanos_total,
|
||||
deserialize_total_nanos: deserialize_nanos_total,
|
||||
buf_len: output_len,
|
||||
}
|
||||
}
|
||||
|
|
|
|||
File diff suppressed because it is too large
Load Diff
|
|
@ -1,7 +1,5 @@
|
|||
// automatically generated by the FlatBuffers compiler, do not modify
|
||||
|
||||
|
||||
|
||||
extern crate flatbuffers;
|
||||
|
||||
use std::cmp::Ordering;
|
||||
|
|
@ -25,7 +23,6 @@ pub enum MessageBody {
|
|||
NONE = 0,
|
||||
Trade = 1,
|
||||
LevelUpdate = 2,
|
||||
|
||||
}
|
||||
|
||||
const ENUM_MIN_MESSAGE_BODY: u8 = 0;
|
||||
|
|
@ -66,15 +63,11 @@ impl flatbuffers::Push for MessageBody {
|
|||
const ENUM_VALUES_MESSAGE_BODY: [MessageBody; 3] = [
|
||||
MessageBody::NONE,
|
||||
MessageBody::Trade,
|
||||
MessageBody::LevelUpdate
|
||||
MessageBody::LevelUpdate,
|
||||
];
|
||||
|
||||
#[allow(non_camel_case_types)]
|
||||
const ENUM_NAMES_MESSAGE_BODY:[&'static str; 3] = [
|
||||
"NONE",
|
||||
"Trade",
|
||||
"LevelUpdate"
|
||||
];
|
||||
const ENUM_NAMES_MESSAGE_BODY: [&'static str; 3] = ["NONE", "Trade", "LevelUpdate"];
|
||||
|
||||
pub fn enum_name_message_body(e: MessageBody) -> &'static str {
|
||||
let index = e as u8;
|
||||
|
|
@ -88,7 +81,6 @@ pub struct MessageBodyUnionTableOffset {}
|
|||
pub enum Side {
|
||||
Buy = 0,
|
||||
Sell = 1,
|
||||
|
||||
}
|
||||
|
||||
const ENUM_MIN_SIDE: u8 = 0;
|
||||
|
|
@ -126,16 +118,10 @@ impl flatbuffers::Push for Side {
|
|||
}
|
||||
|
||||
#[allow(non_camel_case_types)]
|
||||
const ENUM_VALUES_SIDE:[Side; 2] = [
|
||||
Side::Buy,
|
||||
Side::Sell
|
||||
];
|
||||
const ENUM_VALUES_SIDE: [Side; 2] = [Side::Buy, Side::Sell];
|
||||
|
||||
#[allow(non_camel_case_types)]
|
||||
const ENUM_NAMES_SIDE:[&'static str; 2] = [
|
||||
"Buy",
|
||||
"Sell"
|
||||
];
|
||||
const ENUM_NAMES_SIDE: [&'static str; 2] = ["Buy", "Sell"];
|
||||
|
||||
pub fn enum_name_side(e: Side) -> &'static str {
|
||||
let index = e as u8;
|
||||
|
|
@ -162,14 +148,13 @@ impl<'a> flatbuffers::Follow<'a> for Trade<'a> {
|
|||
impl<'a> Trade<'a> {
|
||||
#[inline]
|
||||
pub fn init_from_table(table: flatbuffers::Table<'a>) -> Self {
|
||||
Trade {
|
||||
_tab: table,
|
||||
}
|
||||
Trade { _tab: table }
|
||||
}
|
||||
#[allow(unused_mut)]
|
||||
pub fn create<'bldr: 'args, 'args: 'mut_bldr, 'mut_bldr>(
|
||||
_fbb: &'mut_bldr mut flatbuffers::FlatBufferBuilder<'bldr>,
|
||||
args: &'args TradeArgs) -> flatbuffers::WIPOffset<Trade<'bldr>> {
|
||||
args: &'args TradeArgs,
|
||||
) -> flatbuffers::WIPOffset<Trade<'bldr>> {
|
||||
let mut builder = TradeBuilder::new(_fbb);
|
||||
builder.add_price(args.price);
|
||||
builder.add_size_(args.size_);
|
||||
|
|
@ -196,10 +181,7 @@ pub struct TradeArgs {
|
|||
impl<'a> Default for TradeArgs {
|
||||
#[inline]
|
||||
fn default() -> Self {
|
||||
TradeArgs {
|
||||
price: 0,
|
||||
size_: 0,
|
||||
}
|
||||
TradeArgs { price: 0, size_: 0 }
|
||||
}
|
||||
}
|
||||
pub struct TradeBuilder<'a: 'b, 'b> {
|
||||
|
|
@ -250,14 +232,13 @@ impl<'a> flatbuffers::Follow<'a> for LevelUpdate<'a> {
|
|||
impl<'a> LevelUpdate<'a> {
|
||||
#[inline]
|
||||
pub fn init_from_table(table: flatbuffers::Table<'a>) -> Self {
|
||||
LevelUpdate {
|
||||
_tab: table,
|
||||
}
|
||||
LevelUpdate { _tab: table }
|
||||
}
|
||||
#[allow(unused_mut)]
|
||||
pub fn create<'bldr: 'args, 'args: 'mut_bldr, 'mut_bldr>(
|
||||
_fbb: &'mut_bldr mut flatbuffers::FlatBufferBuilder<'bldr>,
|
||||
args: &'args LevelUpdateArgs) -> flatbuffers::WIPOffset<LevelUpdate<'bldr>> {
|
||||
args: &'args LevelUpdateArgs,
|
||||
) -> flatbuffers::WIPOffset<LevelUpdate<'bldr>> {
|
||||
let mut builder = LevelUpdateBuilder::new(_fbb);
|
||||
builder.add_price(args.price);
|
||||
builder.add_size_(args.size_);
|
||||
|
|
@ -273,11 +254,15 @@ impl<'a> LevelUpdate<'a> {
|
|||
|
||||
#[inline]
|
||||
pub fn price(&self) -> u64 {
|
||||
self._tab.get::<u64>(LevelUpdate::VT_PRICE, Some(0)).unwrap()
|
||||
self._tab
|
||||
.get::<u64>(LevelUpdate::VT_PRICE, Some(0))
|
||||
.unwrap()
|
||||
}
|
||||
#[inline]
|
||||
pub fn size_(&self) -> u32 {
|
||||
self._tab.get::<u32>(LevelUpdate::VT_SIZE_, Some(0)).unwrap()
|
||||
self._tab
|
||||
.get::<u32>(LevelUpdate::VT_SIZE_, Some(0))
|
||||
.unwrap()
|
||||
}
|
||||
#[inline]
|
||||
pub fn flags(&self) -> u8 {
|
||||
|
|
@ -285,7 +270,9 @@ impl<'a> LevelUpdate<'a> {
|
|||
}
|
||||
#[inline]
|
||||
pub fn side(&self) -> Side {
|
||||
self._tab.get::<Side>(LevelUpdate::VT_SIDE, Some(Side::Buy)).unwrap()
|
||||
self._tab
|
||||
.get::<Side>(LevelUpdate::VT_SIDE, Some(Side::Buy))
|
||||
.unwrap()
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -325,7 +312,8 @@ impl<'a: 'b, 'b> LevelUpdateBuilder<'a, 'b> {
|
|||
}
|
||||
#[inline]
|
||||
pub fn add_side(&mut self, side: Side) {
|
||||
self.fbb_.push_slot::<Side>(LevelUpdate::VT_SIDE, side, Side::Buy);
|
||||
self.fbb_
|
||||
.push_slot::<Side>(LevelUpdate::VT_SIDE, side, Side::Buy);
|
||||
}
|
||||
#[inline]
|
||||
pub fn new(_fbb: &'b mut flatbuffers::FlatBufferBuilder<'a>) -> LevelUpdateBuilder<'a, 'b> {
|
||||
|
|
@ -362,18 +350,21 @@ impl<'a> flatbuffers::Follow<'a> for Message<'a> {
|
|||
impl<'a> Message<'a> {
|
||||
#[inline]
|
||||
pub fn init_from_table(table: flatbuffers::Table<'a>) -> Self {
|
||||
Message {
|
||||
_tab: table,
|
||||
}
|
||||
Message { _tab: table }
|
||||
}
|
||||
#[allow(unused_mut)]
|
||||
pub fn create<'bldr: 'args, 'args: 'mut_bldr, 'mut_bldr>(
|
||||
_fbb: &'mut_bldr mut flatbuffers::FlatBufferBuilder<'bldr>,
|
||||
args: &'args MessageArgs<'args>) -> flatbuffers::WIPOffset<Message<'bldr>> {
|
||||
args: &'args MessageArgs<'args>,
|
||||
) -> flatbuffers::WIPOffset<Message<'bldr>> {
|
||||
let mut builder = MessageBuilder::new(_fbb);
|
||||
builder.add_ts_nanos(args.ts_nanos);
|
||||
if let Some(x) = args.body { builder.add_body(x); }
|
||||
if let Some(x) = args.symbol { builder.add_symbol(x); }
|
||||
if let Some(x) = args.body {
|
||||
builder.add_body(x);
|
||||
}
|
||||
if let Some(x) = args.symbol {
|
||||
builder.add_symbol(x);
|
||||
}
|
||||
builder.add_body_type(args.body_type);
|
||||
builder.finish()
|
||||
}
|
||||
|
|
@ -389,15 +380,19 @@ impl<'a> Message<'a> {
|
|||
}
|
||||
#[inline]
|
||||
pub fn symbol(&self) -> Option<&'a str> {
|
||||
self._tab.get::<flatbuffers::ForwardsUOffset<&str>>(Message::VT_SYMBOL, None)
|
||||
self._tab
|
||||
.get::<flatbuffers::ForwardsUOffset<&str>>(Message::VT_SYMBOL, None)
|
||||
}
|
||||
#[inline]
|
||||
pub fn body_type(&self) -> MessageBody {
|
||||
self._tab.get::<MessageBody>(Message::VT_BODY_TYPE, Some(MessageBody::NONE)).unwrap()
|
||||
self._tab
|
||||
.get::<MessageBody>(Message::VT_BODY_TYPE, Some(MessageBody::NONE))
|
||||
.unwrap()
|
||||
}
|
||||
#[inline]
|
||||
pub fn body(&self) -> Option<flatbuffers::Table<'a>> {
|
||||
self._tab.get::<flatbuffers::ForwardsUOffset<flatbuffers::Table<'a>>>(Message::VT_BODY, None)
|
||||
self._tab
|
||||
.get::<flatbuffers::ForwardsUOffset<flatbuffers::Table<'a>>>(Message::VT_BODY, None)
|
||||
}
|
||||
#[inline]
|
||||
#[allow(non_snake_case)]
|
||||
|
|
@ -418,7 +413,6 @@ impl<'a> Message<'a> {
|
|||
None
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
pub struct MessageArgs<'a> {
|
||||
|
|
@ -445,19 +439,23 @@ pub struct MessageBuilder<'a: 'b, 'b> {
|
|||
impl<'a: 'b, 'b> MessageBuilder<'a, 'b> {
|
||||
#[inline]
|
||||
pub fn add_ts_nanos(&mut self, ts_nanos: i64) {
|
||||
self.fbb_.push_slot::<i64>(Message::VT_TS_NANOS, ts_nanos, 0);
|
||||
self.fbb_
|
||||
.push_slot::<i64>(Message::VT_TS_NANOS, ts_nanos, 0);
|
||||
}
|
||||
#[inline]
|
||||
pub fn add_symbol(&mut self, symbol: flatbuffers::WIPOffset<&'b str>) {
|
||||
self.fbb_.push_slot_always::<flatbuffers::WIPOffset<_>>(Message::VT_SYMBOL, symbol);
|
||||
self.fbb_
|
||||
.push_slot_always::<flatbuffers::WIPOffset<_>>(Message::VT_SYMBOL, symbol);
|
||||
}
|
||||
#[inline]
|
||||
pub fn add_body_type(&mut self, body_type: MessageBody) {
|
||||
self.fbb_.push_slot::<MessageBody>(Message::VT_BODY_TYPE, body_type, MessageBody::NONE);
|
||||
self.fbb_
|
||||
.push_slot::<MessageBody>(Message::VT_BODY_TYPE, body_type, MessageBody::NONE);
|
||||
}
|
||||
#[inline]
|
||||
pub fn add_body(&mut self, body: flatbuffers::WIPOffset<flatbuffers::UnionWIPOffset>) {
|
||||
self.fbb_.push_slot_always::<flatbuffers::WIPOffset<_>>(Message::VT_BODY, body);
|
||||
self.fbb_
|
||||
.push_slot_always::<flatbuffers::WIPOffset<_>>(Message::VT_BODY, body);
|
||||
}
|
||||
#[inline]
|
||||
pub fn new(_fbb: &'b mut flatbuffers::FlatBufferBuilder<'a>) -> MessageBuilder<'a, 'b> {
|
||||
|
|
@ -494,17 +492,18 @@ impl<'a> flatbuffers::Follow<'a> for MultiMessage<'a> {
|
|||
impl<'a> MultiMessage<'a> {
|
||||
#[inline]
|
||||
pub fn init_from_table(table: flatbuffers::Table<'a>) -> Self {
|
||||
MultiMessage {
|
||||
_tab: table,
|
||||
}
|
||||
MultiMessage { _tab: table }
|
||||
}
|
||||
#[allow(unused_mut)]
|
||||
pub fn create<'bldr: 'args, 'args: 'mut_bldr, 'mut_bldr>(
|
||||
_fbb: &'mut_bldr mut flatbuffers::FlatBufferBuilder<'bldr>,
|
||||
args: &'args MultiMessageArgs<'args>) -> flatbuffers::WIPOffset<MultiMessage<'bldr>> {
|
||||
args: &'args MultiMessageArgs<'args>,
|
||||
) -> flatbuffers::WIPOffset<MultiMessage<'bldr>> {
|
||||
let mut builder = MultiMessageBuilder::new(_fbb);
|
||||
builder.add_seq_no(args.seq_no);
|
||||
if let Some(x) = args.messages { builder.add_messages(x); }
|
||||
if let Some(x) = args.messages {
|
||||
builder.add_messages(x);
|
||||
}
|
||||
builder.finish()
|
||||
}
|
||||
|
||||
|
|
@ -513,17 +512,27 @@ impl<'a> MultiMessage<'a> {
|
|||
|
||||
#[inline]
|
||||
pub fn seq_no(&self) -> u64 {
|
||||
self._tab.get::<u64>(MultiMessage::VT_SEQ_NO, Some(0)).unwrap()
|
||||
self._tab
|
||||
.get::<u64>(MultiMessage::VT_SEQ_NO, Some(0))
|
||||
.unwrap()
|
||||
}
|
||||
#[inline]
|
||||
pub fn messages(&self) -> Option<flatbuffers::Vector<'a, flatbuffers::ForwardsUOffset<Message<'a>>>> {
|
||||
self._tab.get::<flatbuffers::ForwardsUOffset<flatbuffers::Vector<flatbuffers::ForwardsUOffset<Message<'a>>>>>(MultiMessage::VT_MESSAGES, None)
|
||||
pub fn messages(
|
||||
&self,
|
||||
) -> Option<flatbuffers::Vector<'a, flatbuffers::ForwardsUOffset<Message<'a>>>> {
|
||||
self._tab.get::<flatbuffers::ForwardsUOffset<
|
||||
flatbuffers::Vector<flatbuffers::ForwardsUOffset<Message<'a>>>,
|
||||
>>(MultiMessage::VT_MESSAGES, None)
|
||||
}
|
||||
}
|
||||
|
||||
pub struct MultiMessageArgs<'a> {
|
||||
pub seq_no: u64,
|
||||
pub messages: Option<flatbuffers::WIPOffset<flatbuffers::Vector<'a , flatbuffers::ForwardsUOffset<Message<'a >>>>>,
|
||||
pub messages: Option<
|
||||
flatbuffers::WIPOffset<
|
||||
flatbuffers::Vector<'a, flatbuffers::ForwardsUOffset<Message<'a>>>,
|
||||
>,
|
||||
>,
|
||||
}
|
||||
impl<'a> Default for MultiMessageArgs<'a> {
|
||||
#[inline]
|
||||
|
|
@ -541,14 +550,23 @@ pub struct MultiMessageBuilder<'a: 'b, 'b> {
|
|||
impl<'a: 'b, 'b> MultiMessageBuilder<'a, 'b> {
|
||||
#[inline]
|
||||
pub fn add_seq_no(&mut self, seq_no: u64) {
|
||||
self.fbb_.push_slot::<u64>(MultiMessage::VT_SEQ_NO, seq_no, 0);
|
||||
self.fbb_
|
||||
.push_slot::<u64>(MultiMessage::VT_SEQ_NO, seq_no, 0);
|
||||
}
|
||||
#[inline]
|
||||
pub fn add_messages(&mut self, messages: flatbuffers::WIPOffset<flatbuffers::Vector<'b , flatbuffers::ForwardsUOffset<Message<'b >>>>) {
|
||||
self.fbb_.push_slot_always::<flatbuffers::WIPOffset<_>>(MultiMessage::VT_MESSAGES, messages);
|
||||
pub fn add_messages(
|
||||
&mut self,
|
||||
messages: flatbuffers::WIPOffset<
|
||||
flatbuffers::Vector<'b, flatbuffers::ForwardsUOffset<Message<'b>>>,
|
||||
>,
|
||||
) {
|
||||
self.fbb_
|
||||
.push_slot_always::<flatbuffers::WIPOffset<_>>(MultiMessage::VT_MESSAGES, messages);
|
||||
}
|
||||
#[inline]
|
||||
pub fn new(_fbb: &'b mut flatbuffers::FlatBufferBuilder<'a>) -> MultiMessageBuilder<'a, 'b> {
|
||||
pub fn new(
|
||||
_fbb: &'b mut flatbuffers::FlatBufferBuilder<'a>,
|
||||
) -> MultiMessageBuilder<'a, 'b> {
|
||||
let start = _fbb.start_table();
|
||||
MultiMessageBuilder {
|
||||
fbb_: _fbb,
|
||||
|
|
@ -563,4 +581,3 @@ impl<'a: 'b, 'b> MultiMessageBuilder<'a, 'b> {
|
|||
}
|
||||
|
||||
} // pub mod MdShootout
|
||||
|
||||
|
|
|
|||
|
|
@ -1,6 +1,5 @@
|
|||
/// Generated code for SBE package marketdata_sbe
|
||||
|
||||
|
||||
/// Imports core rather than std to broaden usable environments.
|
||||
extern crate core;
|
||||
|
||||
|
|
@ -89,9 +88,8 @@ impl<'d> ScratchEncoderData<'d> {
|
|||
fn write_type<T>(&mut self, t: &T, num_bytes: usize) -> CodecResult<()> {
|
||||
let end = self.pos + num_bytes;
|
||||
if end <= self.data.len() {
|
||||
let source_bytes: &[u8] = unsafe {
|
||||
core::slice::from_raw_parts(t as *const T as *const u8, num_bytes)
|
||||
};
|
||||
let source_bytes: &[u8] =
|
||||
unsafe { core::slice::from_raw_parts(t as *const T as *const u8, num_bytes) };
|
||||
(&mut self.data[self.pos..end]).copy_from_slice(source_bytes);
|
||||
self.pos = end;
|
||||
Ok(())
|
||||
|
|
@ -134,12 +132,16 @@ impl<'d> ScratchEncoderData<'d> {
|
|||
/// Copy the bytes of a value into the data buffer at a specific position
|
||||
/// Does **not** alter the `pos` index.
|
||||
#[inline]
|
||||
fn write_at_position<T>(&mut self, position: usize, t: &T, num_bytes: usize) -> CodecResult<()> {
|
||||
fn write_at_position<T>(
|
||||
&mut self,
|
||||
position: usize,
|
||||
t: &T,
|
||||
num_bytes: usize,
|
||||
) -> CodecResult<()> {
|
||||
let end = position + num_bytes;
|
||||
if end <= self.data.len() {
|
||||
let source_bytes: &[u8] = unsafe {
|
||||
core::slice::from_raw_parts(t as *const T as *const u8, num_bytes)
|
||||
};
|
||||
let source_bytes: &[u8] =
|
||||
unsafe { core::slice::from_raw_parts(t as *const T as *const u8, num_bytes) };
|
||||
(&mut self.data[position..end]).copy_from_slice(source_bytes);
|
||||
Ok(())
|
||||
} else {
|
||||
|
|
@ -151,11 +153,18 @@ impl<'d> ScratchEncoderData<'d> {
|
|||
/// Note that the initial content of the slice's members' fields may be garbage.
|
||||
/// Advances the `pos` index to after the region representing the slice.
|
||||
#[inline]
|
||||
fn writable_slice<T>(&mut self, count: usize, bytes_per_item: usize) -> CodecResult<&'d mut [T]> {
|
||||
fn writable_slice<T>(
|
||||
&mut self,
|
||||
count: usize,
|
||||
bytes_per_item: usize,
|
||||
) -> CodecResult<&'d mut [T]> {
|
||||
let end = self.pos + (count * bytes_per_item);
|
||||
if end <= self.data.len() {
|
||||
let v: &'d mut [T] = unsafe {
|
||||
core::slice::from_raw_parts_mut(self.data[self.pos..end].as_mut_ptr() as *mut T, count)
|
||||
core::slice::from_raw_parts_mut(
|
||||
self.data[self.pos..end].as_mut_ptr() as *mut T,
|
||||
count,
|
||||
)
|
||||
};
|
||||
self.pos = end;
|
||||
Ok(v)
|
||||
|
|
@ -172,9 +181,8 @@ impl<'d> ScratchEncoderData<'d> {
|
|||
let content_bytes_size = bytes_per_item * t.len();
|
||||
let end = self.pos + content_bytes_size;
|
||||
if end <= self.data.len() {
|
||||
let source_bytes: &[u8] = unsafe {
|
||||
core::slice::from_raw_parts(t.as_ptr() as *const u8, content_bytes_size)
|
||||
};
|
||||
let source_bytes: &[u8] =
|
||||
unsafe { core::slice::from_raw_parts(t.as_ptr() as *const u8, content_bytes_size) };
|
||||
(&mut self.data[self.pos..end]).copy_from_slice(source_bytes);
|
||||
self.pos = end;
|
||||
Ok(())
|
||||
|
|
@ -188,7 +196,7 @@ impl<'d> ScratchEncoderData<'d> {
|
|||
#[derive(Copy, Clone, PartialEq, Eq, PartialOrd, Ord, Hash, Debug)]
|
||||
pub enum Either<L, R> {
|
||||
Left(L),
|
||||
Right(R)
|
||||
Right(R),
|
||||
}
|
||||
|
||||
/// Enum Side
|
||||
|
|
@ -200,7 +208,9 @@ pub enum Side {
|
|||
NullVal = 255u8,
|
||||
}
|
||||
impl Default for Side {
|
||||
fn default() -> Self { Side::NullVal }
|
||||
fn default() -> Self {
|
||||
Side::NullVal
|
||||
}
|
||||
}
|
||||
|
||||
/// Enum MsgType
|
||||
|
|
@ -212,7 +222,9 @@ pub enum MsgType {
|
|||
NullVal = 255u8,
|
||||
}
|
||||
impl Default for MsgType {
|
||||
fn default() -> Self { MsgType::NullVal }
|
||||
fn default() -> Self {
|
||||
MsgType::NullVal
|
||||
}
|
||||
}
|
||||
|
||||
/// Quote
|
||||
|
|
@ -270,7 +282,9 @@ pub struct VarAsciiEncoding {
|
|||
impl VarAsciiEncoding {}
|
||||
|
||||
/// MessageHeader Decoder entry point
|
||||
pub fn start_decoding_message_header<'d>(data: &'d [u8]) -> CodecResult<(&'d MessageHeader, ScratchDecoderData<'d>)> {
|
||||
pub fn start_decoding_message_header<'d>(
|
||||
data: &'d [u8],
|
||||
) -> CodecResult<(&'d MessageHeader, ScratchDecoderData<'d>)> {
|
||||
let mut scratch = ScratchDecoderData { data: data, pos: 0 };
|
||||
let v = scratch.read_type::<MessageHeader>(8)?;
|
||||
Ok((v, scratch))
|
||||
|
|
@ -283,13 +297,12 @@ pub struct MultiMessageFields {
|
|||
pub sequence_number: u64,
|
||||
}
|
||||
|
||||
|
||||
impl MultiMessageFields {}
|
||||
|
||||
/// MultiMessage specific Message Header
|
||||
#[repr(C, packed)]
|
||||
pub struct MultiMessageMessageHeader {
|
||||
pub message_header: MessageHeader
|
||||
pub message_header: MessageHeader,
|
||||
}
|
||||
impl MultiMessageMessageHeader {
|
||||
pub const BLOCK_LENGTH: u16 = 8;
|
||||
|
|
@ -305,7 +318,7 @@ impl Default for MultiMessageMessageHeader {
|
|||
template_id: 1u16,
|
||||
schema_id: 1u16,
|
||||
version: 0u16,
|
||||
}
|
||||
},
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -345,9 +358,17 @@ impl<'d> MultiMessageMessagesSymbolDecoder<'d> {
|
|||
fn wrap(parent: MultiMessageMessagesMemberDecoder<'d>) -> Self {
|
||||
MultiMessageMessagesSymbolDecoder { parent: parent }
|
||||
}
|
||||
pub fn symbol(mut self) -> CodecResult<(&'d [u8], Either<MultiMessageMessagesMemberDecoder<'d>, MultiMessageDecoderDone<'d>>)> {
|
||||
pub fn symbol(
|
||||
mut self,
|
||||
) -> CodecResult<(
|
||||
&'d [u8],
|
||||
Either<MultiMessageMessagesMemberDecoder<'d>, MultiMessageDecoderDone<'d>>,
|
||||
)> {
|
||||
let count = *self.parent.scratch.read_type::<u32>(4)?;
|
||||
Ok((self.parent.scratch.read_slice::<u8>(count as usize, 1)?, self.parent.after_member()))
|
||||
Ok((
|
||||
self.parent.scratch.read_slice::<u8>(count as usize, 1)?,
|
||||
self.parent.after_member(),
|
||||
))
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -368,13 +389,20 @@ impl<'d> MultiMessageMessagesMemberDecoder<'d> {
|
|||
}
|
||||
}
|
||||
|
||||
pub fn next_messages_member(mut self) -> CodecResult<(&'d MultiMessageMessagesMember, MultiMessageMessagesSymbolDecoder<'d>)> {
|
||||
pub fn next_messages_member(
|
||||
mut self,
|
||||
) -> CodecResult<(
|
||||
&'d MultiMessageMessagesMember,
|
||||
MultiMessageMessagesSymbolDecoder<'d>,
|
||||
)> {
|
||||
let v = self.scratch.read_type::<MultiMessageMessagesMember>(35)?;
|
||||
self.index += 1;
|
||||
Ok((v, MultiMessageMessagesSymbolDecoder::wrap(self)))
|
||||
}
|
||||
#[inline]
|
||||
fn after_member(self) -> Either<MultiMessageMessagesMemberDecoder<'d>, MultiMessageDecoderDone<'d>> {
|
||||
fn after_member(
|
||||
self,
|
||||
) -> Either<MultiMessageMessagesMemberDecoder<'d>, MultiMessageDecoderDone<'d>> {
|
||||
if self.index <= self.max_index {
|
||||
Either::Left(self)
|
||||
} else {
|
||||
|
|
@ -389,10 +417,16 @@ impl<'d> MultiMessageMessagesHeaderDecoder<'d> {
|
|||
fn wrap(scratch: ScratchDecoderData<'d>) -> Self {
|
||||
MultiMessageMessagesHeaderDecoder { scratch: scratch }
|
||||
}
|
||||
pub fn messages_individually(mut self) -> CodecResult<Either<MultiMessageMessagesMemberDecoder<'d>, MultiMessageDecoderDone<'d>>> {
|
||||
pub fn messages_individually(
|
||||
mut self,
|
||||
) -> CodecResult<Either<MultiMessageMessagesMemberDecoder<'d>, MultiMessageDecoderDone<'d>>>
|
||||
{
|
||||
let dim = self.scratch.read_type::<GroupSizeEncoding>(4)?;
|
||||
if dim.num_in_group > 0 {
|
||||
Ok(Either::Left(MultiMessageMessagesMemberDecoder::new(self.scratch, dim.num_in_group)))
|
||||
Ok(Either::Left(MultiMessageMessagesMemberDecoder::new(
|
||||
self.scratch,
|
||||
dim.num_in_group,
|
||||
)))
|
||||
} else {
|
||||
Ok(Either::Right(MultiMessageDecoderDone::wrap(self.scratch)))
|
||||
}
|
||||
|
|
@ -404,11 +438,15 @@ pub struct MultiMessageFieldsDecoder<'d> {
|
|||
scratch: ScratchDecoderData<'d>,
|
||||
}
|
||||
impl<'d> MultiMessageFieldsDecoder<'d> {
|
||||
|
||||
pub fn wrap(scratch: ScratchDecoderData<'d>) -> MultiMessageFieldsDecoder<'d> {
|
||||
MultiMessageFieldsDecoder { scratch: scratch }
|
||||
}
|
||||
pub fn multi_message_fields(mut self) -> CodecResult<(&'d MultiMessageFields, MultiMessageMessagesHeaderDecoder<'d>)> {
|
||||
pub fn multi_message_fields(
|
||||
mut self,
|
||||
) -> CodecResult<(
|
||||
&'d MultiMessageFields,
|
||||
MultiMessageMessagesHeaderDecoder<'d>,
|
||||
)> {
|
||||
let v = self.scratch.read_type::<MultiMessageFields>(8)?;
|
||||
Ok((v, MultiMessageMessagesHeaderDecoder::wrap(self.scratch)))
|
||||
}
|
||||
|
|
@ -419,7 +457,6 @@ pub struct MultiMessageMessageHeaderDecoder<'d> {
|
|||
scratch: ScratchDecoderData<'d>,
|
||||
}
|
||||
impl<'d> MultiMessageMessageHeaderDecoder<'d> {
|
||||
|
||||
pub fn wrap(scratch: ScratchDecoderData<'d>) -> MultiMessageMessageHeaderDecoder<'d> {
|
||||
MultiMessageMessageHeaderDecoder { scratch: scratch }
|
||||
}
|
||||
|
|
@ -460,7 +497,7 @@ impl<'d> MultiMessageMessagesSymbolEncoder<'d> {
|
|||
pub fn symbol(mut self, s: &'d [u8]) -> CodecResult<MultiMessageMessagesMemberEncoder> {
|
||||
let l = s.len();
|
||||
if l > 4294967294 {
|
||||
return Err(CodecErr::SliceIsLongerThanAllowedBySchema)
|
||||
return Err(CodecErr::SliceIsLongerThanAllowedBySchema);
|
||||
}
|
||||
// Write data length
|
||||
self.parent.scratch.write_type::<u32>(&(l as u32), 4)?; // group length
|
||||
|
|
@ -487,14 +524,19 @@ impl<'d> MultiMessageMessagesMemberEncoder<'d> {
|
|||
}
|
||||
|
||||
#[inline]
|
||||
pub fn next_messages_member(mut self, fields: &MultiMessageMessagesMember) -> CodecResult<MultiMessageMessagesSymbolEncoder<'d>> {
|
||||
self.scratch.write_type::<MultiMessageMessagesMember>(fields, 35)?; // block length
|
||||
pub fn next_messages_member(
|
||||
mut self,
|
||||
fields: &MultiMessageMessagesMember,
|
||||
) -> CodecResult<MultiMessageMessagesSymbolEncoder<'d>> {
|
||||
self.scratch
|
||||
.write_type::<MultiMessageMessagesMember>(fields, 35)?; // block length
|
||||
self.count += 1;
|
||||
Ok(MultiMessageMessagesSymbolEncoder::wrap(self))
|
||||
}
|
||||
#[inline]
|
||||
pub fn done_with_messages(mut self) -> CodecResult<MultiMessageEncoderDone<'d>> {
|
||||
self.scratch.write_at_position::<u16>(self.count_write_pos, &self.count, 2)?;
|
||||
self.scratch
|
||||
.write_at_position::<u16>(self.count_write_pos, &self.count, 2)?;
|
||||
Ok(MultiMessageEncoderDone::wrap(self.scratch))
|
||||
}
|
||||
}
|
||||
|
|
@ -511,7 +553,10 @@ impl<'d> MultiMessageMessagesHeaderEncoder<'d> {
|
|||
self.scratch.write_type::<u16>(&35u16, 2)?; // block length
|
||||
let count_pos = self.scratch.pos;
|
||||
self.scratch.write_type::<u16>(&0, 2)?; // preliminary group member count
|
||||
Ok(MultiMessageMessagesMemberEncoder::new(self.scratch, count_pos))
|
||||
Ok(MultiMessageMessagesMemberEncoder::new(
|
||||
self.scratch,
|
||||
count_pos,
|
||||
))
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -520,7 +565,6 @@ pub struct MultiMessageFieldsEncoder<'d> {
|
|||
scratch: ScratchEncoderData<'d>,
|
||||
}
|
||||
impl<'d> MultiMessageFieldsEncoder<'d> {
|
||||
|
||||
pub fn wrap(scratch: ScratchEncoderData<'d>) -> MultiMessageFieldsEncoder<'d> {
|
||||
MultiMessageFieldsEncoder { scratch: scratch }
|
||||
}
|
||||
|
|
@ -528,13 +572,21 @@ impl<'d> MultiMessageFieldsEncoder<'d> {
|
|||
/// Create a mutable struct reference overlaid atop the data buffer
|
||||
/// such that changes to the struct directly edit the buffer.
|
||||
/// Note that the initial content of the struct's fields may be garbage.
|
||||
pub fn multi_message_fields(mut self) -> CodecResult<(&'d mut MultiMessageFields, MultiMessageMessagesHeaderEncoder<'d>)> {
|
||||
pub fn multi_message_fields(
|
||||
mut self,
|
||||
) -> CodecResult<(
|
||||
&'d mut MultiMessageFields,
|
||||
MultiMessageMessagesHeaderEncoder<'d>,
|
||||
)> {
|
||||
let v = self.scratch.writable_overlay::<MultiMessageFields>(8 + 0)?;
|
||||
Ok((v, MultiMessageMessagesHeaderEncoder::wrap(self.scratch)))
|
||||
}
|
||||
|
||||
/// Copy the bytes of a value into the data buffer
|
||||
pub fn multi_message_fields_copy(mut self, t: &MultiMessageFields) -> CodecResult<MultiMessageMessagesHeaderEncoder<'d>> {
|
||||
pub fn multi_message_fields_copy(
|
||||
mut self,
|
||||
t: &MultiMessageFields,
|
||||
) -> CodecResult<MultiMessageMessagesHeaderEncoder<'d>> {
|
||||
self.scratch.write_type::<MultiMessageFields>(t, 8)?;
|
||||
Ok(MultiMessageMessagesHeaderEncoder::wrap(self.scratch))
|
||||
}
|
||||
|
|
@ -565,6 +617,8 @@ impl<'d> MultiMessageMessageHeaderEncoder<'d> {
|
|||
}
|
||||
|
||||
/// MultiMessage Encoder entry point
|
||||
pub fn start_encoding_multi_message<'d>(data: &'d mut [u8]) -> MultiMessageMessageHeaderEncoder<'d> {
|
||||
pub fn start_encoding_multi_message<'d>(
|
||||
data: &'d mut [u8],
|
||||
) -> MultiMessageMessageHeaderEncoder<'d> {
|
||||
MultiMessageMessageHeaderEncoder::wrap(ScratchEncoderData { data: data, pos: 0 })
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,6 +1,6 @@
|
|||
use nom::{
|
||||
branch::alt, bytes::complete::tag, bytes::complete::take, IResult, number::complete::*,
|
||||
sequence::tuple,
|
||||
branch::alt, bytes::complete::tag, bytes::complete::take, number::complete::*, sequence::tuple,
|
||||
IResult,
|
||||
};
|
||||
|
||||
pub enum Block<'a> {
|
||||
|
|
|
|||
|
|
@ -1,12 +1,13 @@
|
|||
use std::io::{BufRead, Write};
|
||||
use std::str::from_utf8_unchecked;
|
||||
|
||||
use nom::bytes::complete::take_until;
|
||||
use nom::IResult;
|
||||
|
||||
use crate::{marketdata_sbe, RunnerDeserialize, RunnerSerialize, StreamVec, Summarizer};
|
||||
use crate::iex::{IexMessage, IexPayload};
|
||||
use crate::marketdata_sbe::{Either, MultiMessageFields, MultiMessageMessageHeader, MultiMessageMessagesMember, MultiMessageMessagesMemberEncoder, MultiMessageMessagesSymbolEncoder, Side, start_decoding_multi_message, start_encoding_multi_message};
|
||||
use crate::marketdata_sbe::{
|
||||
start_decoding_multi_message, start_encoding_multi_message, Either, MultiMessageMessageHeader,
|
||||
MultiMessageMessagesMember, MultiMessageMessagesMemberEncoder,
|
||||
MultiMessageMessagesSymbolEncoder, Side,
|
||||
};
|
||||
use crate::{marketdata_sbe, RunnerDeserialize, RunnerSerialize, StreamVec, Summarizer};
|
||||
|
||||
pub struct SBEWriter {
|
||||
/// Buffer to construct messages before copying. While SBE benefits
|
||||
|
|
@ -32,13 +33,15 @@ impl SBEWriter {
|
|||
impl RunnerSerialize for SBEWriter {
|
||||
fn serialize(&mut self, payload: &IexPayload, output: &mut Vec<u8>) {
|
||||
let (fields, encoder) = start_encoding_multi_message(&mut self.scratch_buffer[..])
|
||||
.header_copy(&self.default_header.message_header).unwrap()
|
||||
.multi_message_fields().unwrap();
|
||||
.header_copy(&self.default_header.message_header)
|
||||
.unwrap()
|
||||
.multi_message_fields()
|
||||
.unwrap();
|
||||
fields.sequence_number = payload.first_seq_no;
|
||||
|
||||
let mut encoder = encoder.messages_individually().unwrap();
|
||||
let mut encoder: MultiMessageMessagesMemberEncoder = payload.messages.iter().fold(encoder, |enc, m| {
|
||||
match m {
|
||||
let encoder = encoder.messages_individually().unwrap();
|
||||
let encoder: MultiMessageMessagesMemberEncoder =
|
||||
payload.messages.iter().fold(encoder, |enc, m| match m {
|
||||
IexMessage::TradeReport(tr) => {
|
||||
let fields = MultiMessageMessagesMember {
|
||||
msg_type: marketdata_sbe::MsgType::Trade,
|
||||
|
|
@ -49,8 +52,11 @@ impl RunnerSerialize for SBEWriter {
|
|||
},
|
||||
..Default::default()
|
||||
};
|
||||
let sym_enc: MultiMessageMessagesSymbolEncoder = enc.next_messages_member(&fields).unwrap();
|
||||
sym_enc.symbol(crate::parse_symbol(&tr.symbol).as_bytes()).unwrap()
|
||||
let sym_enc: MultiMessageMessagesSymbolEncoder =
|
||||
enc.next_messages_member(&fields).unwrap();
|
||||
sym_enc
|
||||
.symbol(crate::parse_symbol(&tr.symbol).as_bytes())
|
||||
.unwrap()
|
||||
}
|
||||
IexMessage::PriceLevelUpdate(plu) => {
|
||||
let fields = MultiMessageMessagesMember {
|
||||
|
|
@ -60,15 +66,21 @@ impl RunnerSerialize for SBEWriter {
|
|||
price: plu.price,
|
||||
size: plu.size,
|
||||
flags: plu.event_flags,
|
||||
side: if plu.msg_type == 0x38 { Side::Buy } else { Side::Sell },
|
||||
side: if plu.msg_type == 0x38 {
|
||||
Side::Buy
|
||||
} else {
|
||||
Side::Sell
|
||||
},
|
||||
},
|
||||
..Default::default()
|
||||
};
|
||||
let sym_enc: MultiMessageMessagesSymbolEncoder = enc.next_messages_member(&fields).unwrap();
|
||||
sym_enc.symbol(crate::parse_symbol(&plu.symbol).as_bytes()).unwrap()
|
||||
}
|
||||
_ => enc
|
||||
let sym_enc: MultiMessageMessagesSymbolEncoder =
|
||||
enc.next_messages_member(&fields).unwrap();
|
||||
sym_enc
|
||||
.symbol(crate::parse_symbol(&plu.symbol).as_bytes())
|
||||
.unwrap()
|
||||
}
|
||||
_ => enc,
|
||||
});
|
||||
|
||||
let finished = encoder.done_with_messages().unwrap();
|
||||
|
|
@ -93,10 +105,9 @@ impl RunnerDeserialize for SBEReader {
|
|||
return Err(());
|
||||
}
|
||||
|
||||
let (header, decoder) = start_decoding_multi_message(data)
|
||||
.header().unwrap();
|
||||
let (_header, decoder) = start_decoding_multi_message(data).header().unwrap();
|
||||
|
||||
let (fields, decoder) = decoder.multi_message_fields().unwrap();
|
||||
let (_fields, decoder) = decoder.multi_message_fields().unwrap();
|
||||
let mut msg_decoder = decoder.messages_individually().unwrap();
|
||||
while let Either::Left(msg) = msg_decoder {
|
||||
let (member, sym_dec) = msg.next_messages_member().unwrap();
|
||||
|
|
@ -111,10 +122,10 @@ impl RunnerDeserialize for SBEReader {
|
|||
member.quote.price,
|
||||
match member.quote.side {
|
||||
Side::Buy => true,
|
||||
_ => false
|
||||
_ => false,
|
||||
},
|
||||
),
|
||||
_ => ()
|
||||
_ => (),
|
||||
}
|
||||
msg_decoder = next_msg_dec;
|
||||
}
|
||||
|
|
@ -122,7 +133,7 @@ impl RunnerDeserialize for SBEReader {
|
|||
// We now have a `Right`, which is a finished messages block
|
||||
let msg_decoder = match msg_decoder {
|
||||
Either::Right(r) => r,
|
||||
_ => panic!("Didn't parse all messages")
|
||||
_ => panic!("Didn't parse all messages"),
|
||||
};
|
||||
|
||||
// Interestingly enough, `buf.consume(msg_decoder.unwrap())` isn't OK,
|
||||
|
|
|
|||
Loading…
Reference in New Issue