mirror-github-speiceio/marketdata-shootout
1
0
Fork 0
mirror of https://github.com/speice-io/marketdata-shootout synced 2024-11-23 08:28:32 -05:00
Code Issues Releases Wiki Activity

Add run analysis code, and clean up most warnings

Browse Source 
Cap'n Proto has a bug in deserialization; both in packed and unpacked, it gets to 43116 of 62253 messages.
This commit is contained in:
Bradlee Speice 2019-09-06 22:23:46 -04:00
parent f151d86604
commit 369613843d
12 changed files with 2160 additions and 1653 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

132
Cargo.lock generated
View File

@ -1,5 +1,10 @@
# This file is automatically @generated by Cargo. # This file is automatically @generated by Cargo.
# It is not intended for manual editing. # It is not intended for manual editing.
[[package]]
name = "adler32"
version = "1.0.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
[[package]] [[package]]
name = "alloc_counter" name = "alloc_counter"
version = "0.0.2" version = "0.0.2"
@ -35,11 +40,29 @@ dependencies = [
"winapi 0.3.7 (registry+https://github.com/rust-lang/crates.io-index)", "winapi 0.3.7 (registry+https://github.com/rust-lang/crates.io-index)",
] ]
[[package]]
name = "autocfg"
version = "0.1.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
[[package]]
name = "base64"
version = "0.10.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
dependencies = [
"byteorder 1.3.2 (registry+https://github.com/rust-lang/crates.io-index)",
]
[[package]] [[package]]
name = "bitflags" name = "bitflags"
version = "1.1.0" version = "1.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
[[package]]
name = "byteorder"
version = "1.3.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
[[package]] [[package]]
name = "capnp" name = "capnp"
version = "0.10.1" version = "0.10.1"
@ -53,6 +76,11 @@ dependencies = [
"capnp 0.10.1 (registry+https://github.com/rust-lang/crates.io-index)", "capnp 0.10.1 (registry+https://github.com/rust-lang/crates.io-index)",
] ]
[[package]]
name = "cc"
version = "1.0.42"
source = "registry+https://github.com/rust-lang/crates.io-index"
[[package]] [[package]]
name = "cfg-if" name = "cfg-if"
version = "0.1.9" version = "0.1.9"
@ -72,6 +100,31 @@ dependencies = [
"vec_map 0.8.1 (registry+https://github.com/rust-lang/crates.io-index)", "vec_map 0.8.1 (registry+https://github.com/rust-lang/crates.io-index)",
] ]
[[package]]
name = "crc32fast"
version = "1.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
dependencies = [
"cfg-if 0.1.9 (registry+https://github.com/rust-lang/crates.io-index)",
]
[[package]]
name = "crossbeam-channel"
version = "0.3.9"
source = "registry+https://github.com/rust-lang/crates.io-index"
dependencies = [
"crossbeam-utils 0.6.6 (registry+https://github.com/rust-lang/crates.io-index)",
]
[[package]]
name = "crossbeam-utils"
version = "0.6.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
dependencies = [
"cfg-if 0.1.9 (registry+https://github.com/rust-lang/crates.io-index)",
"lazy_static 1.4.0 (registry+https://github.com/rust-lang/crates.io-index)",
]
[[package]] [[package]]
name = "flatbuffers" name = "flatbuffers"
version = "0.6.0" version = "0.6.0"
@ -88,6 +141,35 @@ dependencies = [
"log 0.4.8 (registry+https://github.com/rust-lang/crates.io-index)", "log 0.4.8 (registry+https://github.com/rust-lang/crates.io-index)",
] ]
[[package]]
name = "flate2"
version = "1.0.11"
source = "registry+https://github.com/rust-lang/crates.io-index"
dependencies = [
"crc32fast 1.2.0 (registry+https://github.com/rust-lang/crates.io-index)",
"libc 0.2.62 (registry+https://github.com/rust-lang/crates.io-index)",
"miniz-sys 0.1.12 (registry+https://github.com/rust-lang/crates.io-index)",
"miniz_oxide 0.3.2 (registry+https://github.com/rust-lang/crates.io-index)",
]
[[package]]
name = "hdrhistogram"
version = "6.3.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
dependencies = [
"base64 0.10.1 (registry+https://github.com/rust-lang/crates.io-index)",
"byteorder 1.3.2 (registry+https://github.com/rust-lang/crates.io-index)",
"crossbeam-channel 0.3.9 (registry+https://github.com/rust-lang/crates.io-index)",
"flate2 1.0.11 (registry+https://github.com/rust-lang/crates.io-index)",
"nom 4.2.3 (registry+https://github.com/rust-lang/crates.io-index)",
"num-traits 0.2.8 (registry+https://github.com/rust-lang/crates.io-index)",
]
[[package]]
name = "lazy_static"
version = "1.4.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
[[package]] [[package]]
name = "lexical-core" name = "lexical-core"
version = "0.4.3" version = "0.4.3"
@ -123,6 +205,7 @@ dependencies = [
"clap 2.33.0 (registry+https://github.com/rust-lang/crates.io-index)", "clap 2.33.0 (registry+https://github.com/rust-lang/crates.io-index)",
"flatbuffers 0.6.0 (registry+https://github.com/rust-lang/crates.io-index)", "flatbuffers 0.6.0 (registry+https://github.com/rust-lang/crates.io-index)",
"flatc-rust 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)", "flatc-rust 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)",
"hdrhistogram 6.3.4 (registry+https://github.com/rust-lang/crates.io-index)",
"nom 5.0.0 (registry+https://github.com/rust-lang/crates.io-index)", "nom 5.0.0 (registry+https://github.com/rust-lang/crates.io-index)",
"smallvec 0.6.10 (registry+https://github.com/rust-lang/crates.io-index)", "smallvec 0.6.10 (registry+https://github.com/rust-lang/crates.io-index)",
] ]
@ -132,6 +215,32 @@ name = "memchr"
version = "2.2.1" version = "2.2.1"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
[[package]]
name = "miniz-sys"
version = "0.1.12"
source = "registry+https://github.com/rust-lang/crates.io-index"
dependencies = [
"cc 1.0.42 (registry+https://github.com/rust-lang/crates.io-index)",
"libc 0.2.62 (registry+https://github.com/rust-lang/crates.io-index)",
]
[[package]]
name = "miniz_oxide"
version = "0.3.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
dependencies = [
"adler32 1.0.3 (registry+https://github.com/rust-lang/crates.io-index)",
]
[[package]]
name = "nom"
version = "4.2.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
dependencies = [
"memchr 2.2.1 (registry+https://github.com/rust-lang/crates.io-index)",
"version_check 0.1.5 (registry+https://github.com/rust-lang/crates.io-index)",
]
[[package]] [[package]]
name = "nom" name = "nom"
version = "5.0.0" version = "5.0.0"
@ -142,6 +251,14 @@ dependencies = [
"version_check 0.1.5 (registry+https://github.com/rust-lang/crates.io-index)", "version_check 0.1.5 (registry+https://github.com/rust-lang/crates.io-index)",
] ]
[[package]]
name = "num-traits"
version = "0.2.8"
source = "registry+https://github.com/rust-lang/crates.io-index"
dependencies = [
"autocfg 0.1.6 (registry+https://github.com/rust-lang/crates.io-index)",
]
[[package]] [[package]]
name = "proc-macro2" name = "proc-macro2"
version = "0.4.30" version = "0.4.30"
@ -279,22 +396,37 @@ version = "0.4.0"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
[metadata] [metadata]
"checksum adler32 1.0.3 (registry+https://github.com/rust-lang/crates.io-index)" = "7e522997b529f05601e05166c07ed17789691f562762c7f3b987263d2dedee5c"
"checksum alloc_counter 0.0.2 (registry+https://github.com/rust-lang/crates.io-index)" = "a169230586814a38a47b9764bb5e5310120df93952df01ee3ea9d832aef10e2f" "checksum alloc_counter 0.0.2 (registry+https://github.com/rust-lang/crates.io-index)" = "a169230586814a38a47b9764bb5e5310120df93952df01ee3ea9d832aef10e2f"
"checksum alloc_counter_macro 0.0.1 (registry+https://github.com/rust-lang/crates.io-index)" = "c37d9ddd812e5223e8de74a2152fa79dce52ca9f6af38a54c6bcd1ae8b26a05d" "checksum alloc_counter_macro 0.0.1 (registry+https://github.com/rust-lang/crates.io-index)" = "c37d9ddd812e5223e8de74a2152fa79dce52ca9f6af38a54c6bcd1ae8b26a05d"
"checksum ansi_term 0.11.0 (registry+https://github.com/rust-lang/crates.io-index)" = "ee49baf6cb617b853aa8d93bf420db2383fab46d314482ca2803b40d5fde979b" "checksum ansi_term 0.11.0 (registry+https://github.com/rust-lang/crates.io-index)" = "ee49baf6cb617b853aa8d93bf420db2383fab46d314482ca2803b40d5fde979b"
"checksum atty 0.2.13 (registry+https://github.com/rust-lang/crates.io-index)" = "1803c647a3ec87095e7ae7acfca019e98de5ec9a7d01343f611cf3152ed71a90" "checksum atty 0.2.13 (registry+https://github.com/rust-lang/crates.io-index)" = "1803c647a3ec87095e7ae7acfca019e98de5ec9a7d01343f611cf3152ed71a90"
"checksum autocfg 0.1.6 (registry+https://github.com/rust-lang/crates.io-index)" = "b671c8fb71b457dd4ae18c4ba1e59aa81793daacc361d82fcd410cef0d491875"
"checksum base64 0.10.1 (registry+https://github.com/rust-lang/crates.io-index)" = "0b25d992356d2eb0ed82172f5248873db5560c4721f564b13cb5193bda5e668e"
"checksum bitflags 1.1.0 (registry+https://github.com/rust-lang/crates.io-index)" = "3d155346769a6855b86399e9bc3814ab343cd3d62c7e985113d46a0ec3c281fd" "checksum bitflags 1.1.0 (registry+https://github.com/rust-lang/crates.io-index)" = "3d155346769a6855b86399e9bc3814ab343cd3d62c7e985113d46a0ec3c281fd"
"checksum byteorder 1.3.2 (registry+https://github.com/rust-lang/crates.io-index)" = "a7c3dd8985a7111efc5c80b44e23ecdd8c007de8ade3b96595387e812b957cf5"
"checksum capnp 0.10.1 (registry+https://github.com/rust-lang/crates.io-index)" = "3a31dc984081bce3531cb7e1c69b5b926904095baf90be9da42df07af54e123a" "checksum capnp 0.10.1 (registry+https://github.com/rust-lang/crates.io-index)" = "3a31dc984081bce3531cb7e1c69b5b926904095baf90be9da42df07af54e123a"
"checksum capnpc 0.10.1 (registry+https://github.com/rust-lang/crates.io-index)" = "6c1a2eead5ee094e8adb30edd6ae55a139b42b8dbe27cc7e4007553ea2b5eb01" "checksum capnpc 0.10.1 (registry+https://github.com/rust-lang/crates.io-index)" = "6c1a2eead5ee094e8adb30edd6ae55a139b42b8dbe27cc7e4007553ea2b5eb01"
"checksum cc 1.0.42 (registry+https://github.com/rust-lang/crates.io-index)" = "a61c7bce55cd2fae6ec8cb935ebd76256c2959a1f95790f6118a441c2cd5b406"
"checksum cfg-if 0.1.9 (registry+https://github.com/rust-lang/crates.io-index)" = "b486ce3ccf7ffd79fdeb678eac06a9e6c09fc88d33836340becb8fffe87c5e33" "checksum cfg-if 0.1.9 (registry+https://github.com/rust-lang/crates.io-index)" = "b486ce3ccf7ffd79fdeb678eac06a9e6c09fc88d33836340becb8fffe87c5e33"
"checksum clap 2.33.0 (registry+https://github.com/rust-lang/crates.io-index)" = "5067f5bb2d80ef5d68b4c87db81601f0b75bca627bc2ef76b141d7b846a3c6d9" "checksum clap 2.33.0 (registry+https://github.com/rust-lang/crates.io-index)" = "5067f5bb2d80ef5d68b4c87db81601f0b75bca627bc2ef76b141d7b846a3c6d9"
"checksum crc32fast 1.2.0 (registry+https://github.com/rust-lang/crates.io-index)" = "ba125de2af0df55319f41944744ad91c71113bf74a4646efff39afe1f6842db1"
"checksum crossbeam-channel 0.3.9 (registry+https://github.com/rust-lang/crates.io-index)" = "c8ec7fcd21571dc78f96cc96243cab8d8f035247c3efd16c687be154c3fa9efa"
"checksum crossbeam-utils 0.6.6 (registry+https://github.com/rust-lang/crates.io-index)" = "04973fa96e96579258a5091af6003abde64af786b860f18622b82e026cca60e6"
"checksum flatbuffers 0.6.0 (registry+https://github.com/rust-lang/crates.io-index)" = "2fc1af59fd8248b59beb048d614a869ce211315c195f5412334e47f5b7e22726" "checksum flatbuffers 0.6.0 (registry+https://github.com/rust-lang/crates.io-index)" = "2fc1af59fd8248b59beb048d614a869ce211315c195f5412334e47f5b7e22726"
"checksum flatc-rust 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)" = "b37a2ed85bee7b6aa0d5305b4765bf4cc0f0cfbc25b86d524126a1ab755f6aed" "checksum flatc-rust 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)" = "b37a2ed85bee7b6aa0d5305b4765bf4cc0f0cfbc25b86d524126a1ab755f6aed"
"checksum flate2 1.0.11 (registry+https://github.com/rust-lang/crates.io-index)" = "2adaffba6388640136149e18ed080b77a78611c1e1d6de75aedcdf78df5d4682"
"checksum hdrhistogram 6.3.4 (registry+https://github.com/rust-lang/crates.io-index)" = "08d331ebcdbca4acbefe5da8c3299b2e246f198a8294cc5163354e743398b89d"
"checksum lazy_static 1.4.0 (registry+https://github.com/rust-lang/crates.io-index)" = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646"
"checksum lexical-core 0.4.3 (registry+https://github.com/rust-lang/crates.io-index)" = "b8b0f90c979adde96d19eb10eb6431ba0c441e2f9e9bdff868b2f6f5114ff519" "checksum lexical-core 0.4.3 (registry+https://github.com/rust-lang/crates.io-index)" = "b8b0f90c979adde96d19eb10eb6431ba0c441e2f9e9bdff868b2f6f5114ff519"
"checksum libc 0.2.62 (registry+https://github.com/rust-lang/crates.io-index)" = "34fcd2c08d2f832f376f4173a231990fa5aef4e99fb569867318a227ef4c06ba" "checksum libc 0.2.62 (registry+https://github.com/rust-lang/crates.io-index)" = "34fcd2c08d2f832f376f4173a231990fa5aef4e99fb569867318a227ef4c06ba"
"checksum log 0.4.8 (registry+https://github.com/rust-lang/crates.io-index)" = "14b6052be84e6b71ab17edffc2eeabf5c2c3ae1fdb464aae35ac50c67a44e1f7" "checksum log 0.4.8 (registry+https://github.com/rust-lang/crates.io-index)" = "14b6052be84e6b71ab17edffc2eeabf5c2c3ae1fdb464aae35ac50c67a44e1f7"
"checksum memchr 2.2.1 (registry+https://github.com/rust-lang/crates.io-index)" = "88579771288728879b57485cc7d6b07d648c9f0141eb955f8ab7f9d45394468e" "checksum memchr 2.2.1 (registry+https://github.com/rust-lang/crates.io-index)" = "88579771288728879b57485cc7d6b07d648c9f0141eb955f8ab7f9d45394468e"
"checksum miniz-sys 0.1.12 (registry+https://github.com/rust-lang/crates.io-index)" = "1e9e3ae51cea1576ceba0dde3d484d30e6e5b86dee0b2d412fe3a16a15c98202"
"checksum miniz_oxide 0.3.2 (registry+https://github.com/rust-lang/crates.io-index)" = "7108aff85b876d06f22503dcce091e29f76733b2bfdd91eebce81f5e68203a10"
"checksum nom 4.2.3 (registry+https://github.com/rust-lang/crates.io-index)" = "2ad2a91a8e869eeb30b9cb3119ae87773a8f4ae617f41b1eb9c154b2905f7bd6"
"checksum nom 5.0.0 (registry+https://github.com/rust-lang/crates.io-index)" = "e9761d859320e381010a4f7f8ed425f2c924de33ad121ace447367c713ad561b" "checksum nom 5.0.0 (registry+https://github.com/rust-lang/crates.io-index)" = "e9761d859320e381010a4f7f8ed425f2c924de33ad121ace447367c713ad561b"
"checksum num-traits 0.2.8 (registry+https://github.com/rust-lang/crates.io-index)" = "6ba9a427cfca2be13aa6f6403b0b7e7368fe982bfa16fccc450ce74c46cd9b32"
"checksum proc-macro2 0.4.30 (registry+https://github.com/rust-lang/crates.io-index)" = "cf3d2011ab5c909338f7887f4fc896d35932e29146c12c8d01da6b22a80ba759" "checksum proc-macro2 0.4.30 (registry+https://github.com/rust-lang/crates.io-index)" = "cf3d2011ab5c909338f7887f4fc896d35932e29146c12c8d01da6b22a80ba759"
"checksum quote 0.6.13 (registry+https://github.com/rust-lang/crates.io-index)" = "6ce23b6b870e8f94f81fb0a363d65d86675884b34a09043c81e5562f11c1f8e1" "checksum quote 0.6.13 (registry+https://github.com/rust-lang/crates.io-index)" = "6ce23b6b870e8f94f81fb0a363d65d86675884b34a09043c81e5562f11c1f8e1"
"checksum rustc_version 0.2.3 (registry+https://github.com/rust-lang/crates.io-index)" = "138e3e0acb6c9fb258b19b67cb8abd63c00679d2851805ea151465464fe9030a" "checksum rustc_version 0.2.3 (registry+https://github.com/rust-lang/crates.io-index)" = "138e3e0acb6c9fb258b19b67cb8abd63c00679d2851805ea151465464fe9030a"

1
Cargo.toml
View File

@ -8,6 +8,7 @@ edition = "2018"
capnp = "0.10.1" capnp = "0.10.1"
clap = "2.33.0" clap = "2.33.0"
flatbuffers = "0.6.0" flatbuffers = "0.6.0"
hdrhistogram = "6.3.4"
nom = "5.0.0" nom = "5.0.0"
smallvec = "0.6.10" smallvec = "0.6.10"

6
build.rs
View File

@ -15,7 +15,8 @@ fn main() {
inputs: &[Path::new("marketdata.fbs")], inputs: &[Path::new("marketdata.fbs")],
out_dir: Path::new("src/"), out_dir: Path::new("src/"),
..Default::default() ..Default::default()
}).expect("Unable to compile flatc"); })
.expect("Unable to compile flatc");
// There's no Rust-style builder crate for SBE, // There's no Rust-style builder crate for SBE,
// so we need to run the command by hand. // so we need to run the command by hand.
@ -25,7 +26,8 @@ fn main() {
.arg("-Dsbe.xinclude.aware=true") .arg("-Dsbe.xinclude.aware=true")
.arg("-Dsbe.target.language=uk.co.real_logic.sbe.generation.rust.Rust") .arg("-Dsbe.target.language=uk.co.real_logic.sbe.generation.rust.Rust")
.arg("-Dsbe.target.namespace=marketdata_sbe") .arg("-Dsbe.target.namespace=marketdata_sbe")
.arg("-jar").arg("sbe-all-1.13.2-all.jar") .arg("-jar")
.arg("sbe-all-1.13.2-all.jar")
.arg("marketdata.xml") .arg("marketdata.xml")
.output() .output()
.expect("Unable to execute SBE compiler"); .expect("Unable to execute SBE compiler");

118
src/capnp_runner.rs
View File

@ -1,26 +1,25 @@
use std::convert::TryInto; use std::convert::TryInto;
use std::io::{BufRead, Read}; use std::io::{BufRead, Read};
use std::mem::size_of; use std::mem::size_of;
use std::str::from_utf8_unchecked;
use capnp::Error; use capnp::message::{ReaderOptions, ScratchSpace, ScratchSpaceHeapAllocator};
use capnp::message::{Builder, ReaderOptions, ScratchSpace, ScratchSpaceHeapAllocator};
use capnp::serialize::write_message; use capnp::serialize::write_message;
use capnp::serialize_packed::{read_message as read_message_packed, write_message as write_message_packed}; use capnp::serialize_packed::{
use nom::bytes::complete::take_until; read_message as read_message_packed, write_message as write_message_packed,
use nom::IResult; };
use capnp::Error;
use crate::{RunnerDeserialize, RunnerSerialize, StreamVec, Summarizer};
use crate::iex::{IexMessage, IexPayload}; use crate::iex::{IexMessage, IexPayload};
use crate::marketdata_capnp::{multi_message, Side};
use crate::marketdata_capnp::message; use crate::marketdata_capnp::message;
use crate::marketdata_capnp::{multi_message, Side};
use crate::{RunnerDeserialize, RunnerSerialize, StreamVec, Summarizer};
pub struct CapnpWriter<'a> { pub struct CapnpWriter<'a> {
// We have to be very careful with how messages are built, as running // We have to be very careful with how messages are built, as running
// `init_root` and rebuilding will still accumulate garbage if using // `init_root` and rebuilding will still accumulate garbage if using
// the standard HeapAllocator. // the standard HeapAllocator.
// https://github.com/capnproto/capnproto-rust/issues/111 // https://github.com/capnproto/capnproto-rust/issues/111
words: Vec<capnp::Word>, _words: Vec<capnp::Word>,
scratch: ScratchSpace<'a>, scratch: ScratchSpace<'a>,
packed: bool, packed: bool,
} }
@ -31,14 +30,12 @@ impl<'a> CapnpWriter<'a> {
// In practice, let's just make sure everything fits. // In practice, let's just make sure everything fits.
let mut words = capnp::Word::allocate_zeroed_vec(1024); let mut words = capnp::Word::allocate_zeroed_vec(1024);
let mut scratch = ScratchSpace::new(unsafe { let scratch = ScratchSpace::new(unsafe { std::mem::transmute(&mut words[..]) });
std::mem::transmute(&mut words[..])
});
CapnpWriter { CapnpWriter {
words, _words: words,
scratch, scratch,
packed packed,
} }
} }
@ -56,12 +53,14 @@ impl<'a> CapnpWriter<'a> {
impl<'a> RunnerSerialize for CapnpWriter<'a> { impl<'a> RunnerSerialize for CapnpWriter<'a> {
fn serialize(&mut self, payload: &IexPayload, mut output: &mut Vec<u8>) { fn serialize(&mut self, payload: &IexPayload, mut output: &mut Vec<u8>) {
// First, count the messages we actually care about. // First, count the messages we actually care about.
let num_msgs = payload.messages.iter().map(|m| { let num_msgs = payload
match m { .messages
.iter()
.map(|m| match m {
IexMessage::TradeReport(_) | IexMessage::PriceLevelUpdate(_) => 1, IexMessage::TradeReport(_) | IexMessage::PriceLevelUpdate(_) => 1,
_ => 0 _ => 0,
} })
}).fold(0, |sum, i| sum + i); .fold(0, |sum, i| sum + i);
if num_msgs == 0 { if num_msgs == 0 {
return; return;
@ -109,13 +108,21 @@ impl<'a> RunnerSerialize for CapnpWriter<'a> {
msg_plu.set_price(plu.price); msg_plu.set_price(plu.price);
msg_plu.set_size(plu.size); msg_plu.set_size(plu.size);
msg_plu.set_flags(plu.event_flags); msg_plu.set_flags(plu.event_flags);
msg_plu.set_side(if plu.msg_type == 0x38 { Side::Buy } else { Side::Sell }); msg_plu.set_side(if plu.msg_type == 0x38 {
Side::Buy
} else {
Side::Sell
});
} }
_ => () _ => (),
} }
} }
let write_fn = if self.packed { write_message_packed } else { write_message }; let write_fn = if self.packed {
write_message_packed
} else {
write_message
};
write_fn(&mut output, &builder).unwrap(); write_fn(&mut output, &builder).unwrap();
} }
@ -123,25 +130,28 @@ impl<'a> RunnerSerialize for CapnpWriter<'a> {
pub struct CapnpReader { pub struct CapnpReader {
read_opts: ReaderOptions, read_opts: ReaderOptions,
packed: bool packed: bool,
} }
impl CapnpReader { impl CapnpReader {
pub fn new(packed: bool) -> CapnpReader { pub fn new(packed: bool) -> CapnpReader {
CapnpReader { CapnpReader {
read_opts: ReaderOptions::new(), read_opts: ReaderOptions::new(),
packed packed,
} }
} }
} }
impl CapnpReader { impl CapnpReader {
fn deserialize_packed<'a>(&self, buf: &'a mut StreamVec, stats: &mut Summarizer) -> Result<(), ()> { fn deserialize_packed<'a>(
&self,
buf: &'a mut StreamVec,
stats: &mut Summarizer,
) -> Result<(), ()> {
// Because `capnp::serialize_packed::PackedRead` is hidden from us, packed reads // Because `capnp::serialize_packed::PackedRead` is hidden from us, packed reads
// *have* to both allocate new segments every read, and copy the buffer into // *have* to both allocate new segments every read, and copy the buffer into
// those same segments, no ability to re-use allocated memory. // those same segments, no ability to re-use allocated memory.
let reader = read_message_packed(buf, self.read_opts) let reader = read_message_packed(buf, self.read_opts).map_err(|_| ())?;
.map_err(|_| ())?;
let multimsg = reader.get_root::<multi_message::Reader>().unwrap(); let multimsg = reader.get_root::<multi_message::Reader>().unwrap();
for msg in multimsg.get_messages().unwrap().iter() { for msg in multimsg.get_messages().unwrap().iter() {
@ -149,18 +159,18 @@ impl CapnpReader {
Ok(message::Trade(tr)) => { Ok(message::Trade(tr)) => {
let tr = tr.unwrap(); let tr = tr.unwrap();
stats.append_trade_volume(msg.get_symbol().unwrap(), tr.get_size() as u64); stats.append_trade_volume(msg.get_symbol().unwrap(), tr.get_size() as u64);
}, }
Ok(message::Quote(q)) => { Ok(message::Quote(q)) => {
let q = q.unwrap(); let q = q.unwrap();
let is_bid = match q.get_side().unwrap() { let is_bid = match q.get_side().unwrap() {
Side::Buy => true, Side::Buy => true,
_ => false _ => false,
}; };
stats.update_quote_prices(msg.get_symbol().unwrap(), q.get_price(), is_bid); stats.update_quote_prices(msg.get_symbol().unwrap(), q.get_price(), is_bid);
},
_ => panic!("Unrecognized message type!")
} }
}; _ => panic!("Unrecognized message type!"),
}
}
Ok(()) Ok(())
} }
@ -192,8 +202,7 @@ impl CapnpReader {
There is no documentation on how to calculate `bytes_consumed` when parsing by hand There is no documentation on how to calculate `bytes_consumed` when parsing by hand
that I could find, you just have to guess and check until you figure this one out. that I could find, you just have to guess and check until you figure this one out.
*/ */
let (num_words, offsets) = read_segment_table(&mut data, reader_opts) let (num_words, offsets) = read_segment_table(&mut data, reader_opts).map_err(|_| ())?;
.map_err(|_| ())?;
let words = unsafe { capnp::Word::bytes_to_words(data) }; let words = unsafe { capnp::Word::bytes_to_words(data) };
let reader = capnp::message::Reader::new( let reader = capnp::message::Reader::new(
SliceSegments { SliceSegments {
@ -206,8 +215,7 @@ impl CapnpReader {
let msg_bytes = num_words * size_of::<capnp::Word>(); let msg_bytes = num_words * size_of::<capnp::Word>();
let bytes_consumed = segment_table_bytes + msg_bytes; let bytes_consumed = segment_table_bytes + msg_bytes;
let multimsg = reader.get_root::<multi_message::Reader>() let multimsg = reader.get_root::<multi_message::Reader>().map_err(|_| ())?;
.map_err(|_| ())?;
for msg in multimsg.get_messages().map_err(|_| ())?.iter() { for msg in multimsg.get_messages().map_err(|_| ())?.iter() {
let sym = msg.get_symbol().map_err(|_| ())?; let sym = msg.get_symbol().map_err(|_| ())?;
@ -215,15 +223,15 @@ impl CapnpReader {
message::Trade(trade) => { message::Trade(trade) => {
let trade = trade.unwrap(); let trade = trade.unwrap();
stats.append_trade_volume(sym, trade.get_size().into()); stats.append_trade_volume(sym, trade.get_size().into());
}, }
message::Quote(quote) => { message::Quote(quote) => {
let quote = quote.unwrap(); let quote = quote.unwrap();
let is_buy = match quote.get_side().unwrap() { let is_buy = match quote.get_side().unwrap() {
Side::Buy => true, Side::Buy => true,
_ => false _ => false,
}; };
stats.update_quote_prices(sym, quote.get_price(), is_buy); stats.update_quote_prices(sym, quote.get_price(), is_buy);
}, }
} }
} }
@ -244,7 +252,6 @@ impl RunnerDeserialize for CapnpReader {
} }
} }
pub struct SliceSegments<'a> { pub struct SliceSegments<'a> {
words: &'a [capnp::Word], words: &'a [capnp::Word],
segment_slices: Vec<(usize, usize)>, segment_slices: Vec<(usize, usize)>,
@ -265,10 +272,12 @@ impl<'a> capnp::message::ReaderSegments for SliceSegments<'a> {
} }
} }
fn read_segment_table<R>(read: &mut R, fn read_segment_table<R>(
options: capnp::message::ReaderOptions) read: &mut R,
-> capnp::Result<(usize, Vec<(usize, usize)>)> options: capnp::message::ReaderOptions,
where R: Read ) -> capnp::Result<(usize, Vec<(usize, usize)>)>
where
R: Read,
{ {
let mut buf: [u8; 8] = [0; 8]; let mut buf: [u8; 8] = [0; 8];
@ -277,9 +286,15 @@ fn read_segment_table<R>(read: &mut R,
let segment_count = u32::from_le_bytes(buf[0..4].try_into().unwrap()).wrapping_add(1) as usize; let segment_count = u32::from_le_bytes(buf[0..4].try_into().unwrap()).wrapping_add(1) as usize;
if segment_count >= 512 { if segment_count >= 512 {
return Err(Error::failed(format!("Too many segments: {}", segment_count))) return Err(Error::failed(format!(
"Too many segments: {}",
segment_count
)));
} else if segment_count == 0 { } else if segment_count == 0 {
return Err(Error::failed(format!("Too few segments: {}", segment_count))) return Err(Error::failed(format!(
"Too few segments: {}",
segment_count
)));
} }
let mut segment_slices = Vec::with_capacity(segment_count); let mut segment_slices = Vec::with_capacity(segment_count);
@ -301,7 +316,8 @@ fn read_segment_table<R>(read: &mut R,
read.read_exact(&mut segment_sizes[..])?; read.read_exact(&mut segment_sizes[..])?;
for idx in 0..(segment_count - 1) { for idx in 0..(segment_count - 1) {
let segment_len = let segment_len =
u32::from_le_bytes(segment_sizes[(idx * 4)..(idx + 1) * 4].try_into().unwrap()) as usize; u32::from_le_bytes(segment_sizes[(idx * 4)..(idx + 1) * 4].try_into().unwrap())
as usize;
segment_slices.push((total_words, total_words + segment_len)); segment_slices.push((total_words, total_words + segment_len));
total_words += segment_len; total_words += segment_len;
@ -313,9 +329,11 @@ fn read_segment_table<R>(read: &mut R,
// traversal limit. Without this check, a malicious client could transmit a very large segment // traversal limit. Without this check, a malicious client could transmit a very large segment
// size to make the receiver allocate excessive space and possibly crash. // size to make the receiver allocate excessive space and possibly crash.
if total_words as u64 > options.traversal_limit_in_words { if total_words as u64 > options.traversal_limit_in_words {
return Err(Error::failed( return Err(Error::failed(format!(
format!("Message has {} words, which is too large. To increase the limit on the \ "Message has {} words, which is too large. To increase the limit on the \
receiving end, see capnp::message::ReaderOptions.", total_words))) receiving end, see capnp::message::ReaderOptions.",
total_words
)));
} }
Ok((total_words, segment_slices)) Ok((total_words, segment_slices))

30
src/flatbuffers_runner.rs
View File

@ -1,15 +1,10 @@
use std::convert::TryInto; use std::convert::TryInto;
use std::io::{BufRead, Error, Write}; use std::io::{BufRead, Write};
use std::mem::size_of; use std::mem::size_of;
use std::str::from_utf8_unchecked;
use capnp::data::new_builder;
use flatbuffers::buffer_has_identifier;
use nom::{bytes::complete::take_until, IResult};
use crate::{RunnerDeserialize, RunnerSerialize, StreamVec, Summarizer};
use crate::iex::{IexMessage, IexPayload}; use crate::iex::{IexMessage, IexPayload};
use crate::marketdata_generated::md_shootout; use crate::marketdata_generated::md_shootout;
use crate::{RunnerDeserialize, RunnerSerialize, StreamVec, Summarizer};
pub struct FlatbuffersWriter<'a> { pub struct FlatbuffersWriter<'a> {
builder: flatbuffers::FlatBufferBuilder<'a>, builder: flatbuffers::FlatBufferBuilder<'a>,
@ -27,7 +22,6 @@ impl<'a> FlatbuffersWriter<'a> {
impl<'a> RunnerSerialize for FlatbuffersWriter<'a> { impl<'a> RunnerSerialize for FlatbuffersWriter<'a> {
fn serialize(&mut self, payload: &IexPayload, output: &mut Vec<u8>) { fn serialize(&mut self, payload: &IexPayload, output: &mut Vec<u8>) {
// Because FlatBuffers can't handle nested vectors (specifically, we can't track // Because FlatBuffers can't handle nested vectors (specifically, we can't track
// both the variable-length vector of messages, and the variable-length strings // both the variable-length vector of messages, and the variable-length strings
// within those messages), we have to cache the messages as they get built // within those messages), we have to cache the messages as they get built
@ -61,7 +55,11 @@ impl<'a> RunnerSerialize for FlatbuffersWriter<'a> {
price: plu.price, price: plu.price,
size_: plu.size, size_: plu.size,
flags: plu.event_flags, flags: plu.event_flags,
side: if plu.msg_type == 0x38 { md_shootout::Side::Buy } else { md_shootout::Side::Sell }, side: if plu.msg_type == 0x38 {
md_shootout::Side::Buy
} else {
md_shootout::Side::Sell
},
}, },
); );
@ -73,7 +71,7 @@ impl<'a> RunnerSerialize for FlatbuffersWriter<'a> {
body: Some(level_update.as_union_value()), body: Some(level_update.as_union_value()),
}) })
} }
_ => None _ => None,
}; };
msg_args.map(|a| { msg_args.map(|a| {
@ -118,7 +116,7 @@ impl RunnerDeserialize for FlatbuffersReader {
// a view over the underlying buffer. // a view over the underlying buffer.
let data = buf.fill_buf().unwrap(); let data = buf.fill_buf().unwrap();
if data.len() == 0 { if data.len() == 0 {
return Err(()) return Err(());
} }
let msg_len_buf: [u8; 4] = data[..size_of::<u32>()].try_into().unwrap(); let msg_len_buf: [u8; 4] = data[..size_of::<u32>()].try_into().unwrap();
@ -127,7 +125,7 @@ impl RunnerDeserialize for FlatbuffersReader {
let multimsg = flatbuffers::get_size_prefixed_root::<md_shootout::MultiMessage>(data); let multimsg = flatbuffers::get_size_prefixed_root::<md_shootout::MultiMessage>(data);
let msg_vec = match multimsg.messages() { let msg_vec = match multimsg.messages() {
Some(m) => m, Some(m) => m,
None => panic!("Couldn't find messages") None => panic!("Couldn't find messages"),
}; };
for i in 0..msg_vec.len() { for i in 0..msg_vec.len() {
@ -136,16 +134,16 @@ impl RunnerDeserialize for FlatbuffersReader {
md_shootout::MessageBody::Trade => { md_shootout::MessageBody::Trade => {
let trade = msg.body_as_trade().unwrap(); let trade = msg.body_as_trade().unwrap();
stats.append_trade_volume(msg.symbol().unwrap(), trade.size_().into()); stats.append_trade_volume(msg.symbol().unwrap(), trade.size_().into());
}, }
md_shootout::MessageBody::LevelUpdate => { md_shootout::MessageBody::LevelUpdate => {
let lu = msg.body_as_level_update().unwrap(); let lu = msg.body_as_level_update().unwrap();
let is_bid = match lu.side() { let is_bid = match lu.side() {
md_shootout::Side::Buy => true, md_shootout::Side::Buy => true,
_ => false _ => false,
}; };
stats.update_quote_prices(msg.symbol().unwrap(), lu.price(), is_bid); stats.update_quote_prices(msg.symbol().unwrap(), lu.price(), is_bid);
}, }
md_shootout::MessageBody::NONE => panic!("Unrecognized message type") md_shootout::MessageBody::NONE => panic!("Unrecognized message type"),
} }
} }

4
src/iex.rs
View File

@ -1,8 +1,8 @@
use std::convert::TryInto; use std::convert::TryInto;
use nom::{bytes::complete::take, IResult, number::complete::*, sequence::tuple}; use nom::{bytes::complete::take, number::complete::*, sequence::tuple, IResult};
use crate::parsers::{Block, extract_iex_data, read_block}; use crate::parsers::{extract_iex_data, read_block, Block};
pub struct IexParser<'a> { pub struct IexParser<'a> {
pcap_buffer: &'a [u8], pcap_buffer: &'a [u8],

194
src/main.rs
View File

@ -2,13 +2,14 @@ use std::cmp::{max, min};
use std::collections::hash_map::{DefaultHasher, HashMap}; use std::collections::hash_map::{DefaultHasher, HashMap};
use std::fs::File; use std::fs::File;
use std::hash::Hasher; use std::hash::Hasher;
use std::io::{BufRead, Read};
use std::io::Error; use std::io::Error;
use std::io::{BufRead, Read};
use std::path::Path; use std::path::Path;
use std::str::from_utf8_unchecked; use std::str::from_utf8_unchecked;
use std::time::{Instant, SystemTime}; use std::time::Instant;
use clap::{App, Arg}; use clap::{App, Arg};
use hdrhistogram::Histogram;
use nom::{bytes::complete::take_until, IResult}; use nom::{bytes::complete::take_until, IResult};
use crate::iex::{IexParser, IexPayload}; use crate::iex::{IexParser, IexPayload};
@ -19,13 +20,14 @@ use crate::iex::{IexParser, IexPayload};
pub mod marketdata_capnp; pub mod marketdata_capnp;
#[allow(unused_imports)] #[allow(unused_imports)]
pub mod marketdata_generated; // Flatbuffers pub mod marketdata_generated; // Flatbuffers
#[allow(dead_code)]
pub mod marketdata_sbe; pub mod marketdata_sbe;
mod capnp_runner; mod capnp_runner;
mod flatbuffers_runner; mod flatbuffers_runner;
mod sbe_runner;
mod iex; mod iex;
mod parsers; mod parsers;
mod sbe_runner;
fn main() { fn main() {
let matches = App::new("Marketdata Shootout") let matches = App::new("Marketdata Shootout")
@ -48,84 +50,32 @@ fn main() {
file.read_to_end(&mut buf) file.read_to_end(&mut buf)
.expect(&format!("Unable to read file={}", path.display())); .expect(&format!("Unable to read file={}", path.display()));
let _start = SystemTime::now(); let _capnp_unpacked = run_analysis(
let mut summarizer = Summarizer::default(); &buf,
let mut parser = IexParser::new(&buf[..]); &mut capnp_runner::CapnpWriter::new(false),
&mut capnp_runner::CapnpReader::new(false),
);
// Pre-allocate the same size as the backing file. Will be way more than let _capnp_packed = run_analysis(
// necessary, but makes sure there's no re-allocation not related to &buf,
// actual parsing/serialization code &mut capnp_runner::CapnpWriter::new(true),
let mut output_buf: Vec<u8> = Vec::with_capacity(buf.capacity()); &mut capnp_runner::CapnpReader::new(true),
);
/* let _flatbuffers = run_analysis(
let mut capnp_writer = capnp_runner::CapnpWriter::new(); &buf,
for iex_payload in parser { &mut flatbuffers_runner::FlatbuffersWriter::new(),
//let iex_payload = parser.next().unwrap(); &mut flatbuffers_runner::FlatbuffersReader::new(),
capnp_writer.serialize(&iex_payload, &mut output_buf, true); );
let _sbe = run_analysis(
&buf,
&mut sbe_runner::SBEWriter::new(),
&mut sbe_runner::SBEReader::new(),
);
} }
let capnp_reader = capnp_runner::CapnpReader::new(); #[derive(Debug, PartialEq)]
let mut read_buf = StreamVec::new(output_buf);
let mut parsed_msgs: u64 = 0;
while let Ok(_) = capnp_reader.deserialize_packed(&mut read_buf, &mut summarizer) {
parsed_msgs += 1;
}
*/
let mut fb_writer = flatbuffers_runner::FlatbuffersWriter::new();
for iex_payload in parser {
let now = Instant::now();
fb_writer.serialize(&iex_payload, &mut output_buf);
let serialize_nanos = Instant::now().duration_since(now).as_nanos();
dbg!(serialize_nanos);
}
let mut read_buf = StreamVec::new(output_buf);
let fb_reader = flatbuffers_runner::FlatbuffersReader::new();
let mut parsed_msgs = 0;
while let Ok(_) = fb_reader.deserialize(&mut read_buf, &mut summarizer) {
parsed_msgs += 1;
}
/*
let mut capnp_writer = capnp_runner::CapnpWriter::new();
for iex_payload in parser {
//let iex_payload = parser.next().unwrap();
let now = Instant::now();
capnp_writer.serialize(&iex_payload, &mut output_buf, false);
let serialize_nanos = Instant::now().duration_since(now).as_nanos();
dbg!(serialize_nanos);
}
let capnp_reader = capnp_runner::CapnpReader::new();
let mut read_buf = StreamVec::new(output_buf);
let mut parsed_msgs: u64 = 0;
while let Ok(_) = capnp_reader.deserialize_unpacked(&mut read_buf, &mut summarizer) {
parsed_msgs += 1;
}
*/
/*
let mut sbe_writer = sbe_runner::SBEWriter::new();
for iex_payload in parser {
//let iex_payload = parser.next().unwrap();
sbe_writer.serialize(&iex_payload, &mut output_buf);
}
let sbe_reader = sbe_runner::SBEReader::new();
let mut read_buf = StreamVec::new(output_buf);
let mut parsed_msgs: u64 = 0;
while let Ok(_) = sbe_reader.deserialize(&mut read_buf, &mut summarizer) {
parsed_msgs += 1;
}
*/
dbg!(parsed_msgs);
dbg!(summarizer);
}
#[derive(Debug)]
pub struct SummaryStats { pub struct SummaryStats {
symbol: String, symbol: String,
trade_volume: u64, trade_volume: u64,
@ -135,17 +85,16 @@ pub struct SummaryStats {
ask_low: u64, ask_low: u64,
} }
#[derive(Default, Debug)] #[derive(Default, Debug, PartialEq)]
pub struct Summarizer { pub struct Summarizer {
data: HashMap<u64, SummaryStats> data: HashMap<u64, SummaryStats>,
} }
impl Summarizer { impl Summarizer {
fn entry(&mut self, sym: &str) -> &mut SummaryStats { fn entry(&mut self, sym: &str) -> &mut SummaryStats {
let mut hasher = DefaultHasher::new(); let mut hasher = DefaultHasher::new();
hasher.write(sym.as_bytes()); hasher.write(sym.as_bytes());
self.data.entry(hasher.finish()) self.data.entry(hasher.finish()).or_insert(SummaryStats {
.or_insert(SummaryStats {
symbol: sym.to_string(), symbol: sym.to_string(),
trade_volume: 0, trade_volume: 0,
bid_high: 0, bid_high: 0,
@ -178,10 +127,7 @@ pub struct StreamVec {
impl StreamVec { impl StreamVec {
pub fn new(buf: Vec<u8>) -> StreamVec { pub fn new(buf: Vec<u8>) -> StreamVec {
StreamVec { StreamVec { pos: 0, inner: buf }
pos: 0,
inner: buf,
}
} }
} }
@ -189,7 +135,11 @@ impl Read for StreamVec {
fn read(&mut self, buf: &mut [u8]) -> Result<usize, Error> { fn read(&mut self, buf: &mut [u8]) -> Result<usize, Error> {
// TODO: There's *got* to be a better way to handle this // TODO: There's *got* to be a better way to handle this
let end = self.pos + buf.len(); let end = self.pos + buf.len();
let end = if end > self.inner.len() { self.inner.len() } else { end }; let end = if end > self.inner.len() {
self.inner.len()
} else {
end
};
let read_size = end - self.pos; let read_size = end - self.pos;
buf[..read_size].copy_from_slice(&self.inner[self.pos..end]); buf[..read_size].copy_from_slice(&self.inner[self.pos..end]);
self.pos = end; self.pos = end;
@ -226,3 +176,75 @@ fn parse_symbol(sym: &[u8; 8]) -> &str {
let (_, sym_bytes) = __take_until(" ", &sym[..]).unwrap(); let (_, sym_bytes) = __take_until(" ", &sym[..]).unwrap();
unsafe { from_utf8_unchecked(sym_bytes) } unsafe { from_utf8_unchecked(sym_bytes) }
} }
struct RunAnalysis {
serialize_hist: Histogram<u64>,
deserialize_hist: Histogram<u64>,
summary_stats: Summarizer,
serialize_total_nanos: u128,
deserialize_total_nanos: u128,
buf_len: usize,
}
fn run_analysis<S, D>(iex_data: &Vec<u8>, serializer: &mut S, deserializer: &mut D) -> RunAnalysis
where
S: RunnerSerialize,
D: RunnerDeserialize,
{
let upper = if cfg!(debug_assertions) {
1_000_000
} else {
100_000
};
let iex_parser = IexParser::new(iex_data);
let mut output_buf = Vec::with_capacity(iex_data.len());
let mut serialize_hist = Histogram::<u64>::new_with_bounds(1, upper, 2).unwrap();
let mut serialize_nanos_total = 0u128;
let mut serialize_msgs = 0;
for iex_payload in iex_parser {
let serialize_start = Instant::now();
serializer.serialize(&iex_payload, &mut output_buf);
let serialize_end = Instant::now().duration_since(serialize_start).as_nanos();
serialize_hist.record(serialize_end as u64).unwrap();
serialize_nanos_total += serialize_end;
serialize_msgs += 1;
}
let output_len = output_buf.len();
let mut read_buf = StreamVec::new(output_buf);
let mut summarizer = Summarizer::default();
let mut deserialize_hist = Histogram::<u64>::new_with_bounds(1, upper, 2).unwrap();
let mut parsed_msgs: u64 = 0;
let mut deserialize_nanos_total = 0u128;
loop {
let deserialize_start = Instant::now();
let res = deserializer.deserialize(&mut read_buf, &mut summarizer);
let deserialize_end = Instant::now().duration_since(deserialize_start).as_nanos();
if res.is_ok() {
deserialize_hist.record(deserialize_end as u64).unwrap();
deserialize_nanos_total += deserialize_end;
parsed_msgs += 1;
} else {
break;
}
}
dbg!(serialize_msgs, parsed_msgs);
RunAnalysis {
serialize_hist,
deserialize_hist,
summary_stats: summarizer,
serialize_total_nanos: serialize_nanos_total,
deserialize_total_nanos: deserialize_nanos_total,
buf_len: output_len,
}
}

690
src/marketdata_capnp.rs
View File

File diff suppressed because it is too large Load Diff

141
src/marketdata_generated.rs
View File

@ -1,7 +1,5 @@
// automatically generated by the FlatBuffers compiler, do not modify // automatically generated by the FlatBuffers compiler, do not modify
extern crate flatbuffers; extern crate flatbuffers;
use std::cmp::Ordering; use std::cmp::Ordering;
@ -25,7 +23,6 @@ pub enum MessageBody {
NONE = 0, NONE = 0,
Trade = 1, Trade = 1,
LevelUpdate = 2, LevelUpdate = 2,
} }
const ENUM_MIN_MESSAGE_BODY: u8 = 0; const ENUM_MIN_MESSAGE_BODY: u8 = 0;
@ -66,15 +63,11 @@ impl flatbuffers::Push for MessageBody {
const ENUM_VALUES_MESSAGE_BODY: [MessageBody; 3] = [ const ENUM_VALUES_MESSAGE_BODY: [MessageBody; 3] = [
MessageBody::NONE, MessageBody::NONE,
MessageBody::Trade, MessageBody::Trade,
MessageBody::LevelUpdate MessageBody::LevelUpdate,
]; ];
#[allow(non_camel_case_types)] #[allow(non_camel_case_types)]
const ENUM_NAMES_MESSAGE_BODY:[&'static str; 3] = [ const ENUM_NAMES_MESSAGE_BODY: [&'static str; 3] = ["NONE", "Trade", "LevelUpdate"];
"NONE",
"Trade",
"LevelUpdate"
];
pub fn enum_name_message_body(e: MessageBody) -> &'static str { pub fn enum_name_message_body(e: MessageBody) -> &'static str {
let index = e as u8; let index = e as u8;
@ -88,7 +81,6 @@ pub struct MessageBodyUnionTableOffset {}
pub enum Side { pub enum Side {
Buy = 0, Buy = 0,
Sell = 1, Sell = 1,
} }
const ENUM_MIN_SIDE: u8 = 0; const ENUM_MIN_SIDE: u8 = 0;
@ -126,16 +118,10 @@ impl flatbuffers::Push for Side {
} }
#[allow(non_camel_case_types)] #[allow(non_camel_case_types)]
const ENUM_VALUES_SIDE:[Side; 2] = [ const ENUM_VALUES_SIDE: [Side; 2] = [Side::Buy, Side::Sell];
Side::Buy,
Side::Sell
];
#[allow(non_camel_case_types)] #[allow(non_camel_case_types)]
const ENUM_NAMES_SIDE:[&'static str; 2] = [ const ENUM_NAMES_SIDE: [&'static str; 2] = ["Buy", "Sell"];
"Buy",
"Sell"
];
pub fn enum_name_side(e: Side) -> &'static str { pub fn enum_name_side(e: Side) -> &'static str {
let index = e as u8; let index = e as u8;
@ -162,14 +148,13 @@ impl<'a> flatbuffers::Follow<'a> for Trade<'a> {
impl<'a> Trade<'a> { impl<'a> Trade<'a> {
#[inline] #[inline]
pub fn init_from_table(table: flatbuffers::Table<'a>) -> Self { pub fn init_from_table(table: flatbuffers::Table<'a>) -> Self {
Trade { Trade { _tab: table }
_tab: table,
}
} }
#[allow(unused_mut)] #[allow(unused_mut)]
pub fn create<'bldr: 'args, 'args: 'mut_bldr, 'mut_bldr>( pub fn create<'bldr: 'args, 'args: 'mut_bldr, 'mut_bldr>(
_fbb: &'mut_bldr mut flatbuffers::FlatBufferBuilder<'bldr>, _fbb: &'mut_bldr mut flatbuffers::FlatBufferBuilder<'bldr>,
args: &'args TradeArgs) -> flatbuffers::WIPOffset<Trade<'bldr>> { args: &'args TradeArgs,
) -> flatbuffers::WIPOffset<Trade<'bldr>> {
let mut builder = TradeBuilder::new(_fbb); let mut builder = TradeBuilder::new(_fbb);
builder.add_price(args.price); builder.add_price(args.price);
builder.add_size_(args.size_); builder.add_size_(args.size_);
@ -196,10 +181,7 @@ pub struct TradeArgs {
impl<'a> Default for TradeArgs { impl<'a> Default for TradeArgs {
#[inline] #[inline]
fn default() -> Self { fn default() -> Self {
TradeArgs { TradeArgs { price: 0, size_: 0 }
price: 0,
size_: 0,
}
} }
} }
pub struct TradeBuilder<'a: 'b, 'b> { pub struct TradeBuilder<'a: 'b, 'b> {
@ -250,14 +232,13 @@ impl<'a> flatbuffers::Follow<'a> for LevelUpdate<'a> {
impl<'a> LevelUpdate<'a> { impl<'a> LevelUpdate<'a> {
#[inline] #[inline]
pub fn init_from_table(table: flatbuffers::Table<'a>) -> Self { pub fn init_from_table(table: flatbuffers::Table<'a>) -> Self {
LevelUpdate { LevelUpdate { _tab: table }
_tab: table,
}
} }
#[allow(unused_mut)] #[allow(unused_mut)]
pub fn create<'bldr: 'args, 'args: 'mut_bldr, 'mut_bldr>( pub fn create<'bldr: 'args, 'args: 'mut_bldr, 'mut_bldr>(
_fbb: &'mut_bldr mut flatbuffers::FlatBufferBuilder<'bldr>, _fbb: &'mut_bldr mut flatbuffers::FlatBufferBuilder<'bldr>,
args: &'args LevelUpdateArgs) -> flatbuffers::WIPOffset<LevelUpdate<'bldr>> { args: &'args LevelUpdateArgs,
) -> flatbuffers::WIPOffset<LevelUpdate<'bldr>> {
let mut builder = LevelUpdateBuilder::new(_fbb); let mut builder = LevelUpdateBuilder::new(_fbb);
builder.add_price(args.price); builder.add_price(args.price);
builder.add_size_(args.size_); builder.add_size_(args.size_);
@ -273,11 +254,15 @@ impl<'a> LevelUpdate<'a> {
#[inline] #[inline]
pub fn price(&self) -> u64 { pub fn price(&self) -> u64 {
self._tab.get::<u64>(LevelUpdate::VT_PRICE, Some(0)).unwrap() self._tab
.get::<u64>(LevelUpdate::VT_PRICE, Some(0))
.unwrap()
} }
#[inline] #[inline]
pub fn size_(&self) -> u32 { pub fn size_(&self) -> u32 {
self._tab.get::<u32>(LevelUpdate::VT_SIZE_, Some(0)).unwrap() self._tab
.get::<u32>(LevelUpdate::VT_SIZE_, Some(0))
.unwrap()
} }
#[inline] #[inline]
pub fn flags(&self) -> u8 { pub fn flags(&self) -> u8 {
@ -285,7 +270,9 @@ impl<'a> LevelUpdate<'a> {
} }
#[inline] #[inline]
pub fn side(&self) -> Side { pub fn side(&self) -> Side {
self._tab.get::<Side>(LevelUpdate::VT_SIDE, Some(Side::Buy)).unwrap() self._tab
.get::<Side>(LevelUpdate::VT_SIDE, Some(Side::Buy))
.unwrap()
} }
} }
@ -325,7 +312,8 @@ impl<'a: 'b, 'b> LevelUpdateBuilder<'a, 'b> {
} }
#[inline] #[inline]
pub fn add_side(&mut self, side: Side) { pub fn add_side(&mut self, side: Side) {
self.fbb_.push_slot::<Side>(LevelUpdate::VT_SIDE, side, Side::Buy); self.fbb_
.push_slot::<Side>(LevelUpdate::VT_SIDE, side, Side::Buy);
} }
#[inline] #[inline]
pub fn new(_fbb: &'b mut flatbuffers::FlatBufferBuilder<'a>) -> LevelUpdateBuilder<'a, 'b> { pub fn new(_fbb: &'b mut flatbuffers::FlatBufferBuilder<'a>) -> LevelUpdateBuilder<'a, 'b> {
@ -362,18 +350,21 @@ impl<'a> flatbuffers::Follow<'a> for Message<'a> {
impl<'a> Message<'a> { impl<'a> Message<'a> {
#[inline] #[inline]
pub fn init_from_table(table: flatbuffers::Table<'a>) -> Self { pub fn init_from_table(table: flatbuffers::Table<'a>) -> Self {
Message { Message { _tab: table }
_tab: table,
}
} }
#[allow(unused_mut)] #[allow(unused_mut)]
pub fn create<'bldr: 'args, 'args: 'mut_bldr, 'mut_bldr>( pub fn create<'bldr: 'args, 'args: 'mut_bldr, 'mut_bldr>(
_fbb: &'mut_bldr mut flatbuffers::FlatBufferBuilder<'bldr>, _fbb: &'mut_bldr mut flatbuffers::FlatBufferBuilder<'bldr>,
args: &'args MessageArgs<'args>) -> flatbuffers::WIPOffset<Message<'bldr>> { args: &'args MessageArgs<'args>,
) -> flatbuffers::WIPOffset<Message<'bldr>> {
let mut builder = MessageBuilder::new(_fbb); let mut builder = MessageBuilder::new(_fbb);
builder.add_ts_nanos(args.ts_nanos); builder.add_ts_nanos(args.ts_nanos);
if let Some(x) = args.body { builder.add_body(x); } if let Some(x) = args.body {
if let Some(x) = args.symbol { builder.add_symbol(x); } builder.add_body(x);
}
if let Some(x) = args.symbol {
builder.add_symbol(x);
}
builder.add_body_type(args.body_type); builder.add_body_type(args.body_type);
builder.finish() builder.finish()
} }
@ -389,15 +380,19 @@ impl<'a> Message<'a> {
} }
#[inline] #[inline]
pub fn symbol(&self) -> Option<&'a str> { pub fn symbol(&self) -> Option<&'a str> {
self._tab.get::<flatbuffers::ForwardsUOffset<&str>>(Message::VT_SYMBOL, None) self._tab
.get::<flatbuffers::ForwardsUOffset<&str>>(Message::VT_SYMBOL, None)
} }
#[inline] #[inline]
pub fn body_type(&self) -> MessageBody { pub fn body_type(&self) -> MessageBody {
self._tab.get::<MessageBody>(Message::VT_BODY_TYPE, Some(MessageBody::NONE)).unwrap() self._tab
.get::<MessageBody>(Message::VT_BODY_TYPE, Some(MessageBody::NONE))
.unwrap()
} }
#[inline] #[inline]
pub fn body(&self) -> Option<flatbuffers::Table<'a>> { pub fn body(&self) -> Option<flatbuffers::Table<'a>> {
self._tab.get::<flatbuffers::ForwardsUOffset<flatbuffers::Table<'a>>>(Message::VT_BODY, None) self._tab
.get::<flatbuffers::ForwardsUOffset<flatbuffers::Table<'a>>>(Message::VT_BODY, None)
} }
#[inline] #[inline]
#[allow(non_snake_case)] #[allow(non_snake_case)]
@ -418,7 +413,6 @@ impl<'a> Message<'a> {
None None
} }
} }
} }
pub struct MessageArgs<'a> { pub struct MessageArgs<'a> {
@ -445,19 +439,23 @@ pub struct MessageBuilder<'a: 'b, 'b> {
impl<'a: 'b, 'b> MessageBuilder<'a, 'b> { impl<'a: 'b, 'b> MessageBuilder<'a, 'b> {
#[inline] #[inline]
pub fn add_ts_nanos(&mut self, ts_nanos: i64) { pub fn add_ts_nanos(&mut self, ts_nanos: i64) {
self.fbb_.push_slot::<i64>(Message::VT_TS_NANOS, ts_nanos, 0); self.fbb_
.push_slot::<i64>(Message::VT_TS_NANOS, ts_nanos, 0);
} }
#[inline] #[inline]
pub fn add_symbol(&mut self, symbol: flatbuffers::WIPOffset<&'b str>) { pub fn add_symbol(&mut self, symbol: flatbuffers::WIPOffset<&'b str>) {
self.fbb_.push_slot_always::<flatbuffers::WIPOffset<_>>(Message::VT_SYMBOL, symbol); self.fbb_
.push_slot_always::<flatbuffers::WIPOffset<_>>(Message::VT_SYMBOL, symbol);
} }
#[inline] #[inline]
pub fn add_body_type(&mut self, body_type: MessageBody) { pub fn add_body_type(&mut self, body_type: MessageBody) {
self.fbb_.push_slot::<MessageBody>(Message::VT_BODY_TYPE, body_type, MessageBody::NONE); self.fbb_
.push_slot::<MessageBody>(Message::VT_BODY_TYPE, body_type, MessageBody::NONE);
} }
#[inline] #[inline]
pub fn add_body(&mut self, body: flatbuffers::WIPOffset<flatbuffers::UnionWIPOffset>) { pub fn add_body(&mut self, body: flatbuffers::WIPOffset<flatbuffers::UnionWIPOffset>) {
self.fbb_.push_slot_always::<flatbuffers::WIPOffset<_>>(Message::VT_BODY, body); self.fbb_
.push_slot_always::<flatbuffers::WIPOffset<_>>(Message::VT_BODY, body);
} }
#[inline] #[inline]
pub fn new(_fbb: &'b mut flatbuffers::FlatBufferBuilder<'a>) -> MessageBuilder<'a, 'b> { pub fn new(_fbb: &'b mut flatbuffers::FlatBufferBuilder<'a>) -> MessageBuilder<'a, 'b> {
@ -494,17 +492,18 @@ impl<'a> flatbuffers::Follow<'a> for MultiMessage<'a> {
impl<'a> MultiMessage<'a> { impl<'a> MultiMessage<'a> {
#[inline] #[inline]
pub fn init_from_table(table: flatbuffers::Table<'a>) -> Self { pub fn init_from_table(table: flatbuffers::Table<'a>) -> Self {
MultiMessage { MultiMessage { _tab: table }
_tab: table,
}
} }
#[allow(unused_mut)] #[allow(unused_mut)]
pub fn create<'bldr: 'args, 'args: 'mut_bldr, 'mut_bldr>( pub fn create<'bldr: 'args, 'args: 'mut_bldr, 'mut_bldr>(
_fbb: &'mut_bldr mut flatbuffers::FlatBufferBuilder<'bldr>, _fbb: &'mut_bldr mut flatbuffers::FlatBufferBuilder<'bldr>,
args: &'args MultiMessageArgs<'args>) -> flatbuffers::WIPOffset<MultiMessage<'bldr>> { args: &'args MultiMessageArgs<'args>,
) -> flatbuffers::WIPOffset<MultiMessage<'bldr>> {
let mut builder = MultiMessageBuilder::new(_fbb); let mut builder = MultiMessageBuilder::new(_fbb);
builder.add_seq_no(args.seq_no); builder.add_seq_no(args.seq_no);
if let Some(x) = args.messages { builder.add_messages(x); } if let Some(x) = args.messages {
builder.add_messages(x);
}
builder.finish() builder.finish()
} }
@ -513,17 +512,27 @@ impl<'a> MultiMessage<'a> {
#[inline] #[inline]
pub fn seq_no(&self) -> u64 { pub fn seq_no(&self) -> u64 {
self._tab.get::<u64>(MultiMessage::VT_SEQ_NO, Some(0)).unwrap() self._tab
.get::<u64>(MultiMessage::VT_SEQ_NO, Some(0))
.unwrap()
} }
#[inline] #[inline]
pub fn messages(&self) -> Option<flatbuffers::Vector<'a, flatbuffers::ForwardsUOffset<Message<'a>>>> { pub fn messages(
self._tab.get::<flatbuffers::ForwardsUOffset<flatbuffers::Vector<flatbuffers::ForwardsUOffset<Message<'a>>>>>(MultiMessage::VT_MESSAGES, None) &self,
) -> Option<flatbuffers::Vector<'a, flatbuffers::ForwardsUOffset<Message<'a>>>> {
self._tab.get::<flatbuffers::ForwardsUOffset<
flatbuffers::Vector<flatbuffers::ForwardsUOffset<Message<'a>>>,
>>(MultiMessage::VT_MESSAGES, None)
} }
} }
pub struct MultiMessageArgs<'a> { pub struct MultiMessageArgs<'a> {
pub seq_no: u64, pub seq_no: u64,
pub messages: Option<flatbuffers::WIPOffset<flatbuffers::Vector<'a , flatbuffers::ForwardsUOffset<Message<'a >>>>>, pub messages: Option<
flatbuffers::WIPOffset<
flatbuffers::Vector<'a, flatbuffers::ForwardsUOffset<Message<'a>>>,
>,
>,
} }
impl<'a> Default for MultiMessageArgs<'a> { impl<'a> Default for MultiMessageArgs<'a> {
#[inline] #[inline]
@ -541,14 +550,23 @@ pub struct MultiMessageBuilder<'a: 'b, 'b> {
impl<'a: 'b, 'b> MultiMessageBuilder<'a, 'b> { impl<'a: 'b, 'b> MultiMessageBuilder<'a, 'b> {
#[inline] #[inline]
pub fn add_seq_no(&mut self, seq_no: u64) { pub fn add_seq_no(&mut self, seq_no: u64) {
self.fbb_.push_slot::<u64>(MultiMessage::VT_SEQ_NO, seq_no, 0); self.fbb_
.push_slot::<u64>(MultiMessage::VT_SEQ_NO, seq_no, 0);
} }
#[inline] #[inline]
pub fn add_messages(&mut self, messages: flatbuffers::WIPOffset<flatbuffers::Vector<'b , flatbuffers::ForwardsUOffset<Message<'b >>>>) { pub fn add_messages(
self.fbb_.push_slot_always::<flatbuffers::WIPOffset<_>>(MultiMessage::VT_MESSAGES, messages); &mut self,
messages: flatbuffers::WIPOffset<
flatbuffers::Vector<'b, flatbuffers::ForwardsUOffset<Message<'b>>>,
>,
) {
self.fbb_
.push_slot_always::<flatbuffers::WIPOffset<_>>(MultiMessage::VT_MESSAGES, messages);
} }
#[inline] #[inline]
pub fn new(_fbb: &'b mut flatbuffers::FlatBufferBuilder<'a>) -> MultiMessageBuilder<'a, 'b> { pub fn new(
_fbb: &'b mut flatbuffers::FlatBufferBuilder<'a>,
) -> MultiMessageBuilder<'a, 'b> {
let start = _fbb.start_table(); let start = _fbb.start_table();
MultiMessageBuilder { MultiMessageBuilder {
fbb_: _fbb, fbb_: _fbb,
@ -563,4 +581,3 @@ impl<'a: 'b, 'b> MultiMessageBuilder<'a, 'b> {
} }
} // pub mod MdShootout } // pub mod MdShootout

130
src/marketdata_sbe.rs
View File

@ -1,6 +1,5 @@
/// Generated code for SBE package marketdata_sbe /// Generated code for SBE package marketdata_sbe
/// Imports core rather than std to broaden usable environments. /// Imports core rather than std to broaden usable environments.
extern crate core; extern crate core;
@ -89,9 +88,8 @@ impl<'d> ScratchEncoderData<'d> {
fn write_type<T>(&mut self, t: &T, num_bytes: usize) -> CodecResult<()> { fn write_type<T>(&mut self, t: &T, num_bytes: usize) -> CodecResult<()> {
let end = self.pos + num_bytes; let end = self.pos + num_bytes;
if end <= self.data.len() { if end <= self.data.len() {
let source_bytes: &[u8] = unsafe { let source_bytes: &[u8] =
core::slice::from_raw_parts(t as *const T as *const u8, num_bytes) unsafe { core::slice::from_raw_parts(t as *const T as *const u8, num_bytes) };
};
(&mut self.data[self.pos..end]).copy_from_slice(source_bytes); (&mut self.data[self.pos..end]).copy_from_slice(source_bytes);
self.pos = end; self.pos = end;
Ok(()) Ok(())
@ -134,12 +132,16 @@ impl<'d> ScratchEncoderData<'d> {
/// Copy the bytes of a value into the data buffer at a specific position /// Copy the bytes of a value into the data buffer at a specific position
/// Does **not** alter the `pos` index. /// Does **not** alter the `pos` index.
#[inline] #[inline]
fn write_at_position<T>(&mut self, position: usize, t: &T, num_bytes: usize) -> CodecResult<()> { fn write_at_position<T>(
&mut self,
position: usize,
t: &T,
num_bytes: usize,
) -> CodecResult<()> {
let end = position + num_bytes; let end = position + num_bytes;
if end <= self.data.len() { if end <= self.data.len() {
let source_bytes: &[u8] = unsafe { let source_bytes: &[u8] =
core::slice::from_raw_parts(t as *const T as *const u8, num_bytes) unsafe { core::slice::from_raw_parts(t as *const T as *const u8, num_bytes) };
};
(&mut self.data[position..end]).copy_from_slice(source_bytes); (&mut self.data[position..end]).copy_from_slice(source_bytes);
Ok(()) Ok(())
} else { } else {
@ -151,11 +153,18 @@ impl<'d> ScratchEncoderData<'d> {
/// Note that the initial content of the slice's members' fields may be garbage. /// Note that the initial content of the slice's members' fields may be garbage.
/// Advances the `pos` index to after the region representing the slice. /// Advances the `pos` index to after the region representing the slice.
#[inline] #[inline]
fn writable_slice<T>(&mut self, count: usize, bytes_per_item: usize) -> CodecResult<&'d mut [T]> { fn writable_slice<T>(
&mut self,
count: usize,
bytes_per_item: usize,
) -> CodecResult<&'d mut [T]> {
let end = self.pos + (count * bytes_per_item); let end = self.pos + (count * bytes_per_item);
if end <= self.data.len() { if end <= self.data.len() {
let v: &'d mut [T] = unsafe { let v: &'d mut [T] = unsafe {
core::slice::from_raw_parts_mut(self.data[self.pos..end].as_mut_ptr() as *mut T, count) core::slice::from_raw_parts_mut(
self.data[self.pos..end].as_mut_ptr() as *mut T,
count,
)
}; };
self.pos = end; self.pos = end;
Ok(v) Ok(v)
@ -172,9 +181,8 @@ impl<'d> ScratchEncoderData<'d> {
let content_bytes_size = bytes_per_item * t.len(); let content_bytes_size = bytes_per_item * t.len();
let end = self.pos + content_bytes_size; let end = self.pos + content_bytes_size;
if end <= self.data.len() { if end <= self.data.len() {
let source_bytes: &[u8] = unsafe { let source_bytes: &[u8] =
core::slice::from_raw_parts(t.as_ptr() as *const u8, content_bytes_size) unsafe { core::slice::from_raw_parts(t.as_ptr() as *const u8, content_bytes_size) };
};
(&mut self.data[self.pos..end]).copy_from_slice(source_bytes); (&mut self.data[self.pos..end]).copy_from_slice(source_bytes);
self.pos = end; self.pos = end;
Ok(()) Ok(())
@ -188,7 +196,7 @@ impl<'d> ScratchEncoderData<'d> {
#[derive(Copy, Clone, PartialEq, Eq, PartialOrd, Ord, Hash, Debug)] #[derive(Copy, Clone, PartialEq, Eq, PartialOrd, Ord, Hash, Debug)]
pub enum Either<L, R> { pub enum Either<L, R> {
Left(L), Left(L),
Right(R) Right(R),
} }
/// Enum Side /// Enum Side
@ -200,7 +208,9 @@ pub enum Side {
NullVal = 255u8, NullVal = 255u8,
} }
impl Default for Side { impl Default for Side {
fn default() -> Self { Side::NullVal } fn default() -> Self {
Side::NullVal
}
} }
/// Enum MsgType /// Enum MsgType
@ -212,7 +222,9 @@ pub enum MsgType {
NullVal = 255u8, NullVal = 255u8,
} }
impl Default for MsgType { impl Default for MsgType {
fn default() -> Self { MsgType::NullVal } fn default() -> Self {
MsgType::NullVal
}
} }
/// Quote /// Quote
@ -270,7 +282,9 @@ pub struct VarAsciiEncoding {
impl VarAsciiEncoding {} impl VarAsciiEncoding {}
/// MessageHeader Decoder entry point /// MessageHeader Decoder entry point
pub fn start_decoding_message_header<'d>(data: &'d [u8]) -> CodecResult<(&'d MessageHeader, ScratchDecoderData<'d>)> { pub fn start_decoding_message_header<'d>(
data: &'d [u8],
) -> CodecResult<(&'d MessageHeader, ScratchDecoderData<'d>)> {
let mut scratch = ScratchDecoderData { data: data, pos: 0 }; let mut scratch = ScratchDecoderData { data: data, pos: 0 };
let v = scratch.read_type::<MessageHeader>(8)?; let v = scratch.read_type::<MessageHeader>(8)?;
Ok((v, scratch)) Ok((v, scratch))
@ -283,13 +297,12 @@ pub struct MultiMessageFields {
pub sequence_number: u64, pub sequence_number: u64,
} }
impl MultiMessageFields {} impl MultiMessageFields {}
/// MultiMessage specific Message Header /// MultiMessage specific Message Header
#[repr(C, packed)] #[repr(C, packed)]
pub struct MultiMessageMessageHeader { pub struct MultiMessageMessageHeader {
pub message_header: MessageHeader pub message_header: MessageHeader,
} }
impl MultiMessageMessageHeader { impl MultiMessageMessageHeader {
pub const BLOCK_LENGTH: u16 = 8; pub const BLOCK_LENGTH: u16 = 8;
@ -305,7 +318,7 @@ impl Default for MultiMessageMessageHeader {
template_id: 1u16, template_id: 1u16,
schema_id: 1u16, schema_id: 1u16,
version: 0u16, version: 0u16,
} },
} }
} }
} }
@ -345,9 +358,17 @@ impl<'d> MultiMessageMessagesSymbolDecoder<'d> {
fn wrap(parent: MultiMessageMessagesMemberDecoder<'d>) -> Self { fn wrap(parent: MultiMessageMessagesMemberDecoder<'d>) -> Self {
MultiMessageMessagesSymbolDecoder { parent: parent } MultiMessageMessagesSymbolDecoder { parent: parent }
} }
pub fn symbol(mut self) -> CodecResult<(&'d [u8], Either<MultiMessageMessagesMemberDecoder<'d>, MultiMessageDecoderDone<'d>>)> { pub fn symbol(
mut self,
) -> CodecResult<(
&'d [u8],
Either<MultiMessageMessagesMemberDecoder<'d>, MultiMessageDecoderDone<'d>>,
)> {
let count = *self.parent.scratch.read_type::<u32>(4)?; let count = *self.parent.scratch.read_type::<u32>(4)?;
Ok((self.parent.scratch.read_slice::<u8>(count as usize, 1)?, self.parent.after_member())) Ok((
self.parent.scratch.read_slice::<u8>(count as usize, 1)?,
self.parent.after_member(),
))
} }
} }
@ -368,13 +389,20 @@ impl<'d> MultiMessageMessagesMemberDecoder<'d> {
} }
} }
pub fn next_messages_member(mut self) -> CodecResult<(&'d MultiMessageMessagesMember, MultiMessageMessagesSymbolDecoder<'d>)> { pub fn next_messages_member(
mut self,
) -> CodecResult<(
&'d MultiMessageMessagesMember,
MultiMessageMessagesSymbolDecoder<'d>,
)> {
let v = self.scratch.read_type::<MultiMessageMessagesMember>(35)?; let v = self.scratch.read_type::<MultiMessageMessagesMember>(35)?;
self.index += 1; self.index += 1;
Ok((v, MultiMessageMessagesSymbolDecoder::wrap(self))) Ok((v, MultiMessageMessagesSymbolDecoder::wrap(self)))
} }
#[inline] #[inline]
fn after_member(self) -> Either<MultiMessageMessagesMemberDecoder<'d>, MultiMessageDecoderDone<'d>> { fn after_member(
self,
) -> Either<MultiMessageMessagesMemberDecoder<'d>, MultiMessageDecoderDone<'d>> {
if self.index <= self.max_index { if self.index <= self.max_index {
Either::Left(self) Either::Left(self)
} else { } else {
@ -389,10 +417,16 @@ impl<'d> MultiMessageMessagesHeaderDecoder<'d> {
fn wrap(scratch: ScratchDecoderData<'d>) -> Self { fn wrap(scratch: ScratchDecoderData<'d>) -> Self {
MultiMessageMessagesHeaderDecoder { scratch: scratch } MultiMessageMessagesHeaderDecoder { scratch: scratch }
} }
pub fn messages_individually(mut self) -> CodecResult<Either<MultiMessageMessagesMemberDecoder<'d>, MultiMessageDecoderDone<'d>>> { pub fn messages_individually(
mut self,
) -> CodecResult<Either<MultiMessageMessagesMemberDecoder<'d>, MultiMessageDecoderDone<'d>>>
{
let dim = self.scratch.read_type::<GroupSizeEncoding>(4)?; let dim = self.scratch.read_type::<GroupSizeEncoding>(4)?;
if dim.num_in_group > 0 { if dim.num_in_group > 0 {
Ok(Either::Left(MultiMessageMessagesMemberDecoder::new(self.scratch, dim.num_in_group))) Ok(Either::Left(MultiMessageMessagesMemberDecoder::new(
self.scratch,
dim.num_in_group,
)))
} else { } else {
Ok(Either::Right(MultiMessageDecoderDone::wrap(self.scratch))) Ok(Either::Right(MultiMessageDecoderDone::wrap(self.scratch)))
} }
@ -404,11 +438,15 @@ pub struct MultiMessageFieldsDecoder<'d> {
scratch: ScratchDecoderData<'d>, scratch: ScratchDecoderData<'d>,
} }
impl<'d> MultiMessageFieldsDecoder<'d> { impl<'d> MultiMessageFieldsDecoder<'d> {
pub fn wrap(scratch: ScratchDecoderData<'d>) -> MultiMessageFieldsDecoder<'d> { pub fn wrap(scratch: ScratchDecoderData<'d>) -> MultiMessageFieldsDecoder<'d> {
MultiMessageFieldsDecoder { scratch: scratch } MultiMessageFieldsDecoder { scratch: scratch }
} }
pub fn multi_message_fields(mut self) -> CodecResult<(&'d MultiMessageFields, MultiMessageMessagesHeaderDecoder<'d>)> { pub fn multi_message_fields(
mut self,
) -> CodecResult<(
&'d MultiMessageFields,
MultiMessageMessagesHeaderDecoder<'d>,
)> {
let v = self.scratch.read_type::<MultiMessageFields>(8)?; let v = self.scratch.read_type::<MultiMessageFields>(8)?;
Ok((v, MultiMessageMessagesHeaderDecoder::wrap(self.scratch))) Ok((v, MultiMessageMessagesHeaderDecoder::wrap(self.scratch)))
} }
@ -419,7 +457,6 @@ pub struct MultiMessageMessageHeaderDecoder<'d> {
scratch: ScratchDecoderData<'d>, scratch: ScratchDecoderData<'d>,
} }
impl<'d> MultiMessageMessageHeaderDecoder<'d> { impl<'d> MultiMessageMessageHeaderDecoder<'d> {
pub fn wrap(scratch: ScratchDecoderData<'d>) -> MultiMessageMessageHeaderDecoder<'d> { pub fn wrap(scratch: ScratchDecoderData<'d>) -> MultiMessageMessageHeaderDecoder<'d> {
MultiMessageMessageHeaderDecoder { scratch: scratch } MultiMessageMessageHeaderDecoder { scratch: scratch }
} }
@ -460,7 +497,7 @@ impl<'d> MultiMessageMessagesSymbolEncoder<'d> {
pub fn symbol(mut self, s: &'d [u8]) -> CodecResult<MultiMessageMessagesMemberEncoder> { pub fn symbol(mut self, s: &'d [u8]) -> CodecResult<MultiMessageMessagesMemberEncoder> {
let l = s.len(); let l = s.len();
if l > 4294967294 { if l > 4294967294 {
return Err(CodecErr::SliceIsLongerThanAllowedBySchema) return Err(CodecErr::SliceIsLongerThanAllowedBySchema);
} }
// Write data length // Write data length
self.parent.scratch.write_type::<u32>(&(l as u32), 4)?; // group length self.parent.scratch.write_type::<u32>(&(l as u32), 4)?; // group length
@ -487,14 +524,19 @@ impl<'d> MultiMessageMessagesMemberEncoder<'d> {
} }
#[inline] #[inline]
pub fn next_messages_member(mut self, fields: &MultiMessageMessagesMember) -> CodecResult<MultiMessageMessagesSymbolEncoder<'d>> { pub fn next_messages_member(
self.scratch.write_type::<MultiMessageMessagesMember>(fields, 35)?; // block length mut self,
fields: &MultiMessageMessagesMember,
) -> CodecResult<MultiMessageMessagesSymbolEncoder<'d>> {
self.scratch
.write_type::<MultiMessageMessagesMember>(fields, 35)?; // block length
self.count += 1; self.count += 1;
Ok(MultiMessageMessagesSymbolEncoder::wrap(self)) Ok(MultiMessageMessagesSymbolEncoder::wrap(self))
} }
#[inline] #[inline]
pub fn done_with_messages(mut self) -> CodecResult<MultiMessageEncoderDone<'d>> { pub fn done_with_messages(mut self) -> CodecResult<MultiMessageEncoderDone<'d>> {
self.scratch.write_at_position::<u16>(self.count_write_pos, &self.count, 2)?; self.scratch
.write_at_position::<u16>(self.count_write_pos, &self.count, 2)?;
Ok(MultiMessageEncoderDone::wrap(self.scratch)) Ok(MultiMessageEncoderDone::wrap(self.scratch))
} }
} }
@ -511,7 +553,10 @@ impl<'d> MultiMessageMessagesHeaderEncoder<'d> {
self.scratch.write_type::<u16>(&35u16, 2)?; // block length self.scratch.write_type::<u16>(&35u16, 2)?; // block length
let count_pos = self.scratch.pos; let count_pos = self.scratch.pos;
self.scratch.write_type::<u16>(&0, 2)?; // preliminary group member count self.scratch.write_type::<u16>(&0, 2)?; // preliminary group member count
Ok(MultiMessageMessagesMemberEncoder::new(self.scratch, count_pos)) Ok(MultiMessageMessagesMemberEncoder::new(
self.scratch,
count_pos,
))
} }
} }
@ -520,7 +565,6 @@ pub struct MultiMessageFieldsEncoder<'d> {
scratch: ScratchEncoderData<'d>, scratch: ScratchEncoderData<'d>,
} }
impl<'d> MultiMessageFieldsEncoder<'d> { impl<'d> MultiMessageFieldsEncoder<'d> {
pub fn wrap(scratch: ScratchEncoderData<'d>) -> MultiMessageFieldsEncoder<'d> { pub fn wrap(scratch: ScratchEncoderData<'d>) -> MultiMessageFieldsEncoder<'d> {
MultiMessageFieldsEncoder { scratch: scratch } MultiMessageFieldsEncoder { scratch: scratch }
} }
@ -528,13 +572,21 @@ impl<'d> MultiMessageFieldsEncoder<'d> {
/// Create a mutable struct reference overlaid atop the data buffer /// Create a mutable struct reference overlaid atop the data buffer
/// such that changes to the struct directly edit the buffer. /// such that changes to the struct directly edit the buffer.
/// Note that the initial content of the struct's fields may be garbage. /// Note that the initial content of the struct's fields may be garbage.
pub fn multi_message_fields(mut self) -> CodecResult<(&'d mut MultiMessageFields, MultiMessageMessagesHeaderEncoder<'d>)> { pub fn multi_message_fields(
mut self,
) -> CodecResult<(
&'d mut MultiMessageFields,
MultiMessageMessagesHeaderEncoder<'d>,
)> {
let v = self.scratch.writable_overlay::<MultiMessageFields>(8 + 0)?; let v = self.scratch.writable_overlay::<MultiMessageFields>(8 + 0)?;
Ok((v, MultiMessageMessagesHeaderEncoder::wrap(self.scratch))) Ok((v, MultiMessageMessagesHeaderEncoder::wrap(self.scratch)))
} }
/// Copy the bytes of a value into the data buffer /// Copy the bytes of a value into the data buffer
pub fn multi_message_fields_copy(mut self, t: &MultiMessageFields) -> CodecResult<MultiMessageMessagesHeaderEncoder<'d>> { pub fn multi_message_fields_copy(
mut self,
t: &MultiMessageFields,
) -> CodecResult<MultiMessageMessagesHeaderEncoder<'d>> {
self.scratch.write_type::<MultiMessageFields>(t, 8)?; self.scratch.write_type::<MultiMessageFields>(t, 8)?;
Ok(MultiMessageMessagesHeaderEncoder::wrap(self.scratch)) Ok(MultiMessageMessagesHeaderEncoder::wrap(self.scratch))
} }
@ -565,6 +617,8 @@ impl<'d> MultiMessageMessageHeaderEncoder<'d> {
} }
/// MultiMessage Encoder entry point /// MultiMessage Encoder entry point
pub fn start_encoding_multi_message<'d>(data: &'d mut [u8]) -> MultiMessageMessageHeaderEncoder<'d> { pub fn start_encoding_multi_message<'d>(
data: &'d mut [u8],
) -> MultiMessageMessageHeaderEncoder<'d> {
MultiMessageMessageHeaderEncoder::wrap(ScratchEncoderData { data: data, pos: 0 }) MultiMessageMessageHeaderEncoder::wrap(ScratchEncoderData { data: data, pos: 0 })
} }

4
src/parsers.rs
View File

@ -1,6 +1,6 @@
use nom::{ use nom::{
branch::alt, bytes::complete::tag, bytes::complete::take, IResult, number::complete::*, branch::alt, bytes::complete::tag, bytes::complete::take, number::complete::*, sequence::tuple,
sequence::tuple, IResult,
}; };
pub enum Block<'a> { pub enum Block<'a> {

57
src/sbe_runner.rs
View File

@ -1,12 +1,13 @@
use std::io::{BufRead, Write}; use std::io::{BufRead, Write};
use std::str::from_utf8_unchecked; use std::str::from_utf8_unchecked;
use nom::bytes::complete::take_until;
use nom::IResult;
use crate::{marketdata_sbe, RunnerDeserialize, RunnerSerialize, StreamVec, Summarizer};
use crate::iex::{IexMessage, IexPayload}; use crate::iex::{IexMessage, IexPayload};
use crate::marketdata_sbe::{Either, MultiMessageFields, MultiMessageMessageHeader, MultiMessageMessagesMember, MultiMessageMessagesMemberEncoder, MultiMessageMessagesSymbolEncoder, Side, start_decoding_multi_message, start_encoding_multi_message}; use crate::marketdata_sbe::{
start_decoding_multi_message, start_encoding_multi_message, Either, MultiMessageMessageHeader,
MultiMessageMessagesMember, MultiMessageMessagesMemberEncoder,
MultiMessageMessagesSymbolEncoder, Side,
};
use crate::{marketdata_sbe, RunnerDeserialize, RunnerSerialize, StreamVec, Summarizer};
pub struct SBEWriter { pub struct SBEWriter {
/// Buffer to construct messages before copying. While SBE benefits /// Buffer to construct messages before copying. While SBE benefits
@ -32,13 +33,15 @@ impl SBEWriter {
impl RunnerSerialize for SBEWriter { impl RunnerSerialize for SBEWriter {
fn serialize(&mut self, payload: &IexPayload, output: &mut Vec<u8>) { fn serialize(&mut self, payload: &IexPayload, output: &mut Vec<u8>) {
let (fields, encoder) = start_encoding_multi_message(&mut self.scratch_buffer[..]) let (fields, encoder) = start_encoding_multi_message(&mut self.scratch_buffer[..])
.header_copy(&self.default_header.message_header).unwrap() .header_copy(&self.default_header.message_header)
.multi_message_fields().unwrap(); .unwrap()
.multi_message_fields()
.unwrap();
fields.sequence_number = payload.first_seq_no; fields.sequence_number = payload.first_seq_no;
let mut encoder = encoder.messages_individually().unwrap(); let encoder = encoder.messages_individually().unwrap();
let mut encoder: MultiMessageMessagesMemberEncoder = payload.messages.iter().fold(encoder, |enc, m| { let encoder: MultiMessageMessagesMemberEncoder =
match m { payload.messages.iter().fold(encoder, |enc, m| match m {
IexMessage::TradeReport(tr) => { IexMessage::TradeReport(tr) => {
let fields = MultiMessageMessagesMember { let fields = MultiMessageMessagesMember {
msg_type: marketdata_sbe::MsgType::Trade, msg_type: marketdata_sbe::MsgType::Trade,
@ -49,8 +52,11 @@ impl RunnerSerialize for SBEWriter {
}, },
..Default::default() ..Default::default()
}; };
let sym_enc: MultiMessageMessagesSymbolEncoder = enc.next_messages_member(&fields).unwrap(); let sym_enc: MultiMessageMessagesSymbolEncoder =
sym_enc.symbol(crate::parse_symbol(&tr.symbol).as_bytes()).unwrap() enc.next_messages_member(&fields).unwrap();
sym_enc
.symbol(crate::parse_symbol(&tr.symbol).as_bytes())
.unwrap()
} }
IexMessage::PriceLevelUpdate(plu) => { IexMessage::PriceLevelUpdate(plu) => {
let fields = MultiMessageMessagesMember { let fields = MultiMessageMessagesMember {
@ -60,15 +66,21 @@ impl RunnerSerialize for SBEWriter {
price: plu.price, price: plu.price,
size: plu.size, size: plu.size,
flags: plu.event_flags, flags: plu.event_flags,
side: if plu.msg_type == 0x38 { Side::Buy } else { Side::Sell }, side: if plu.msg_type == 0x38 {
Side::Buy
} else {
Side::Sell
},
}, },
..Default::default() ..Default::default()
}; };
let sym_enc: MultiMessageMessagesSymbolEncoder = enc.next_messages_member(&fields).unwrap(); let sym_enc: MultiMessageMessagesSymbolEncoder =
sym_enc.symbol(crate::parse_symbol(&plu.symbol).as_bytes()).unwrap() enc.next_messages_member(&fields).unwrap();
} sym_enc
_ => enc .symbol(crate::parse_symbol(&plu.symbol).as_bytes())
.unwrap()
} }
_ => enc,
}); });
let finished = encoder.done_with_messages().unwrap(); let finished = encoder.done_with_messages().unwrap();
@ -93,10 +105,9 @@ impl RunnerDeserialize for SBEReader {
return Err(()); return Err(());
} }
let (header, decoder) = start_decoding_multi_message(data) let (_header, decoder) = start_decoding_multi_message(data).header().unwrap();
.header().unwrap();
let (fields, decoder) = decoder.multi_message_fields().unwrap(); let (_fields, decoder) = decoder.multi_message_fields().unwrap();
let mut msg_decoder = decoder.messages_individually().unwrap(); let mut msg_decoder = decoder.messages_individually().unwrap();
while let Either::Left(msg) = msg_decoder { while let Either::Left(msg) = msg_decoder {
let (member, sym_dec) = msg.next_messages_member().unwrap(); let (member, sym_dec) = msg.next_messages_member().unwrap();
@ -111,10 +122,10 @@ impl RunnerDeserialize for SBEReader {
member.quote.price, member.quote.price,
match member.quote.side { match member.quote.side {
Side::Buy => true, Side::Buy => true,
_ => false _ => false,
}, },
), ),
_ => () _ => (),
} }
msg_decoder = next_msg_dec; msg_decoder = next_msg_dec;
} }
@ -122,7 +133,7 @@ impl RunnerDeserialize for SBEReader {
// We now have a `Right`, which is a finished messages block // We now have a `Right`, which is a finished messages block
let msg_decoder = match msg_decoder { let msg_decoder = match msg_decoder {
Either::Right(r) => r, Either::Right(r) => r,
_ => panic!("Didn't parse all messages") _ => panic!("Didn't parse all messages"),
}; };
// Interestingly enough, `buf.consume(msg_decoder.unwrap())` isn't OK, // Interestingly enough, `buf.consume(msg_decoder.unwrap())` isn't OK,