mirror-github-speiceio/marketdata-shootout
1
0
Fork 0
mirror of https://github.com/speice-io/marketdata-shootout synced 2024-11-23 00:18:22 -05:00
Code Issues Releases Wiki Activity

Add run analysis code, and clean up most warnings

Browse Source 
Cap'n Proto has a bug in deserialization; both in packed and unpacked, it gets to 43116 of 62253 messages.
This commit is contained in:
Bradlee Speice 2019-09-06 22:23:46 -04:00
parent f151d86604
commit 369613843d
12 changed files with 2160 additions and 1653 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

132
Cargo.lock generated
View File

@ -1,5 +1,10 @@
# This file is automatically @generated by Cargo. # This file is automatically @generated by Cargo.
# It is not intended for manual editing. # It is not intended for manual editing.
[[package]]
name = "adler32"
version = "1.0.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
[[package]] [[package]]
name = "alloc_counter" name = "alloc_counter"
version = "0.0.2" version = "0.0.2"
@ -35,11 +40,29 @@ dependencies = [
"winapi 0.3.7 (registry+https://github.com/rust-lang/crates.io-index)", "winapi 0.3.7 (registry+https://github.com/rust-lang/crates.io-index)",
] ]
[[package]]
name = "autocfg"
version = "0.1.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
[[package]]
name = "base64"
version = "0.10.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
dependencies = [
"byteorder 1.3.2 (registry+https://github.com/rust-lang/crates.io-index)",
]
[[package]] [[package]]
name = "bitflags" name = "bitflags"
version = "1.1.0" version = "1.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
[[package]]
name = "byteorder"
version = "1.3.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
[[package]] [[package]]
name = "capnp" name = "capnp"
version = "0.10.1" version = "0.10.1"
@ -53,6 +76,11 @@ dependencies = [
"capnp 0.10.1 (registry+https://github.com/rust-lang/crates.io-index)", "capnp 0.10.1 (registry+https://github.com/rust-lang/crates.io-index)",
] ]
[[package]]
name = "cc"
version = "1.0.42"
source = "registry+https://github.com/rust-lang/crates.io-index"
[[package]] [[package]]
name = "cfg-if" name = "cfg-if"
version = "0.1.9" version = "0.1.9"
@ -72,6 +100,31 @@ dependencies = [
"vec_map 0.8.1 (registry+https://github.com/rust-lang/crates.io-index)", "vec_map 0.8.1 (registry+https://github.com/rust-lang/crates.io-index)",
] ]
[[package]]
name = "crc32fast"
version = "1.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
dependencies = [
"cfg-if 0.1.9 (registry+https://github.com/rust-lang/crates.io-index)",
]
[[package]]
name = "crossbeam-channel"
version = "0.3.9"
source = "registry+https://github.com/rust-lang/crates.io-index"
dependencies = [
"crossbeam-utils 0.6.6 (registry+https://github.com/rust-lang/crates.io-index)",
]
[[package]]
name = "crossbeam-utils"
version = "0.6.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
dependencies = [
"cfg-if 0.1.9 (registry+https://github.com/rust-lang/crates.io-index)",
"lazy_static 1.4.0 (registry+https://github.com/rust-lang/crates.io-index)",
]
[[package]] [[package]]
name = "flatbuffers" name = "flatbuffers"
version = "0.6.0" version = "0.6.0"
@ -88,6 +141,35 @@ dependencies = [
"log 0.4.8 (registry+https://github.com/rust-lang/crates.io-index)", "log 0.4.8 (registry+https://github.com/rust-lang/crates.io-index)",
] ]
[[package]]
name = "flate2"
version = "1.0.11"
source = "registry+https://github.com/rust-lang/crates.io-index"
dependencies = [
"crc32fast 1.2.0 (registry+https://github.com/rust-lang/crates.io-index)",
"libc 0.2.62 (registry+https://github.com/rust-lang/crates.io-index)",
"miniz-sys 0.1.12 (registry+https://github.com/rust-lang/crates.io-index)",
"miniz_oxide 0.3.2 (registry+https://github.com/rust-lang/crates.io-index)",
]
[[package]]
name = "hdrhistogram"
version = "6.3.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
dependencies = [
"base64 0.10.1 (registry+https://github.com/rust-lang/crates.io-index)",
"byteorder 1.3.2 (registry+https://github.com/rust-lang/crates.io-index)",
"crossbeam-channel 0.3.9 (registry+https://github.com/rust-lang/crates.io-index)",
"flate2 1.0.11 (registry+https://github.com/rust-lang/crates.io-index)",
"nom 4.2.3 (registry+https://github.com/rust-lang/crates.io-index)",
"num-traits 0.2.8 (registry+https://github.com/rust-lang/crates.io-index)",
]
[[package]]
name = "lazy_static"
version = "1.4.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
[[package]] [[package]]
name = "lexical-core" name = "lexical-core"
version = "0.4.3" version = "0.4.3"
@ -123,6 +205,7 @@ dependencies = [
"clap 2.33.0 (registry+https://github.com/rust-lang/crates.io-index)", "clap 2.33.0 (registry+https://github.com/rust-lang/crates.io-index)",
"flatbuffers 0.6.0 (registry+https://github.com/rust-lang/crates.io-index)", "flatbuffers 0.6.0 (registry+https://github.com/rust-lang/crates.io-index)",
"flatc-rust 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)", "flatc-rust 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)",
"hdrhistogram 6.3.4 (registry+https://github.com/rust-lang/crates.io-index)",
"nom 5.0.0 (registry+https://github.com/rust-lang/crates.io-index)", "nom 5.0.0 (registry+https://github.com/rust-lang/crates.io-index)",
"smallvec 0.6.10 (registry+https://github.com/rust-lang/crates.io-index)", "smallvec 0.6.10 (registry+https://github.com/rust-lang/crates.io-index)",
] ]
@ -132,6 +215,32 @@ name = "memchr"
version = "2.2.1" version = "2.2.1"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
[[package]]
name = "miniz-sys"
version = "0.1.12"
source = "registry+https://github.com/rust-lang/crates.io-index"
dependencies = [
"cc 1.0.42 (registry+https://github.com/rust-lang/crates.io-index)",
"libc 0.2.62 (registry+https://github.com/rust-lang/crates.io-index)",
]
[[package]]
name = "miniz_oxide"
version = "0.3.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
dependencies = [
"adler32 1.0.3 (registry+https://github.com/rust-lang/crates.io-index)",
]
[[package]]
name = "nom"
version = "4.2.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
dependencies = [
"memchr 2.2.1 (registry+https://github.com/rust-lang/crates.io-index)",
"version_check 0.1.5 (registry+https://github.com/rust-lang/crates.io-index)",
]
[[package]] [[package]]
name = "nom" name = "nom"
version = "5.0.0" version = "5.0.0"
@ -142,6 +251,14 @@ dependencies = [
"version_check 0.1.5 (registry+https://github.com/rust-lang/crates.io-index)", "version_check 0.1.5 (registry+https://github.com/rust-lang/crates.io-index)",
] ]
[[package]]
name = "num-traits"
version = "0.2.8"
source = "registry+https://github.com/rust-lang/crates.io-index"
dependencies = [
"autocfg 0.1.6 (registry+https://github.com/rust-lang/crates.io-index)",
]
[[package]] [[package]]
name = "proc-macro2" name = "proc-macro2"
version = "0.4.30" version = "0.4.30"
@ -279,22 +396,37 @@ version = "0.4.0"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
[metadata] [metadata]
"checksum adler32 1.0.3 (registry+https://github.com/rust-lang/crates.io-index)" = "7e522997b529f05601e05166c07ed17789691f562762c7f3b987263d2dedee5c"
"checksum alloc_counter 0.0.2 (registry+https://github.com/rust-lang/crates.io-index)" = "a169230586814a38a47b9764bb5e5310120df93952df01ee3ea9d832aef10e2f" "checksum alloc_counter 0.0.2 (registry+https://github.com/rust-lang/crates.io-index)" = "a169230586814a38a47b9764bb5e5310120df93952df01ee3ea9d832aef10e2f"
"checksum alloc_counter_macro 0.0.1 (registry+https://github.com/rust-lang/crates.io-index)" = "c37d9ddd812e5223e8de74a2152fa79dce52ca9f6af38a54c6bcd1ae8b26a05d" "checksum alloc_counter_macro 0.0.1 (registry+https://github.com/rust-lang/crates.io-index)" = "c37d9ddd812e5223e8de74a2152fa79dce52ca9f6af38a54c6bcd1ae8b26a05d"
"checksum ansi_term 0.11.0 (registry+https://github.com/rust-lang/crates.io-index)" = "ee49baf6cb617b853aa8d93bf420db2383fab46d314482ca2803b40d5fde979b" "checksum ansi_term 0.11.0 (registry+https://github.com/rust-lang/crates.io-index)" = "ee49baf6cb617b853aa8d93bf420db2383fab46d314482ca2803b40d5fde979b"
"checksum atty 0.2.13 (registry+https://github.com/rust-lang/crates.io-index)" = "1803c647a3ec87095e7ae7acfca019e98de5ec9a7d01343f611cf3152ed71a90" "checksum atty 0.2.13 (registry+https://github.com/rust-lang/crates.io-index)" = "1803c647a3ec87095e7ae7acfca019e98de5ec9a7d01343f611cf3152ed71a90"
"checksum autocfg 0.1.6 (registry+https://github.com/rust-lang/crates.io-index)" = "b671c8fb71b457dd4ae18c4ba1e59aa81793daacc361d82fcd410cef0d491875"
"checksum base64 0.10.1 (registry+https://github.com/rust-lang/crates.io-index)" = "0b25d992356d2eb0ed82172f5248873db5560c4721f564b13cb5193bda5e668e"
"checksum bitflags 1.1.0 (registry+https://github.com/rust-lang/crates.io-index)" = "3d155346769a6855b86399e9bc3814ab343cd3d62c7e985113d46a0ec3c281fd" "checksum bitflags 1.1.0 (registry+https://github.com/rust-lang/crates.io-index)" = "3d155346769a6855b86399e9bc3814ab343cd3d62c7e985113d46a0ec3c281fd"
"checksum byteorder 1.3.2 (registry+https://github.com/rust-lang/crates.io-index)" = "a7c3dd8985a7111efc5c80b44e23ecdd8c007de8ade3b96595387e812b957cf5"
"checksum capnp 0.10.1 (registry+https://github.com/rust-lang/crates.io-index)" = "3a31dc984081bce3531cb7e1c69b5b926904095baf90be9da42df07af54e123a" "checksum capnp 0.10.1 (registry+https://github.com/rust-lang/crates.io-index)" = "3a31dc984081bce3531cb7e1c69b5b926904095baf90be9da42df07af54e123a"
"checksum capnpc 0.10.1 (registry+https://github.com/rust-lang/crates.io-index)" = "6c1a2eead5ee094e8adb30edd6ae55a139b42b8dbe27cc7e4007553ea2b5eb01" "checksum capnpc 0.10.1 (registry+https://github.com/rust-lang/crates.io-index)" = "6c1a2eead5ee094e8adb30edd6ae55a139b42b8dbe27cc7e4007553ea2b5eb01"
"checksum cc 1.0.42 (registry+https://github.com/rust-lang/crates.io-index)" = "a61c7bce55cd2fae6ec8cb935ebd76256c2959a1f95790f6118a441c2cd5b406"
"checksum cfg-if 0.1.9 (registry+https://github.com/rust-lang/crates.io-index)" = "b486ce3ccf7ffd79fdeb678eac06a9e6c09fc88d33836340becb8fffe87c5e33" "checksum cfg-if 0.1.9 (registry+https://github.com/rust-lang/crates.io-index)" = "b486ce3ccf7ffd79fdeb678eac06a9e6c09fc88d33836340becb8fffe87c5e33"
"checksum clap 2.33.0 (registry+https://github.com/rust-lang/crates.io-index)" = "5067f5bb2d80ef5d68b4c87db81601f0b75bca627bc2ef76b141d7b846a3c6d9" "checksum clap 2.33.0 (registry+https://github.com/rust-lang/crates.io-index)" = "5067f5bb2d80ef5d68b4c87db81601f0b75bca627bc2ef76b141d7b846a3c6d9"
"checksum crc32fast 1.2.0 (registry+https://github.com/rust-lang/crates.io-index)" = "ba125de2af0df55319f41944744ad91c71113bf74a4646efff39afe1f6842db1"
"checksum crossbeam-channel 0.3.9 (registry+https://github.com/rust-lang/crates.io-index)" = "c8ec7fcd21571dc78f96cc96243cab8d8f035247c3efd16c687be154c3fa9efa"
"checksum crossbeam-utils 0.6.6 (registry+https://github.com/rust-lang/crates.io-index)" = "04973fa96e96579258a5091af6003abde64af786b860f18622b82e026cca60e6"
"checksum flatbuffers 0.6.0 (registry+https://github.com/rust-lang/crates.io-index)" = "2fc1af59fd8248b59beb048d614a869ce211315c195f5412334e47f5b7e22726" "checksum flatbuffers 0.6.0 (registry+https://github.com/rust-lang/crates.io-index)" = "2fc1af59fd8248b59beb048d614a869ce211315c195f5412334e47f5b7e22726"
"checksum flatc-rust 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)" = "b37a2ed85bee7b6aa0d5305b4765bf4cc0f0cfbc25b86d524126a1ab755f6aed" "checksum flatc-rust 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)" = "b37a2ed85bee7b6aa0d5305b4765bf4cc0f0cfbc25b86d524126a1ab755f6aed"
"checksum flate2 1.0.11 (registry+https://github.com/rust-lang/crates.io-index)" = "2adaffba6388640136149e18ed080b77a78611c1e1d6de75aedcdf78df5d4682"
"checksum hdrhistogram 6.3.4 (registry+https://github.com/rust-lang/crates.io-index)" = "08d331ebcdbca4acbefe5da8c3299b2e246f198a8294cc5163354e743398b89d"
"checksum lazy_static 1.4.0 (registry+https://github.com/rust-lang/crates.io-index)" = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646"
"checksum lexical-core 0.4.3 (registry+https://github.com/rust-lang/crates.io-index)" = "b8b0f90c979adde96d19eb10eb6431ba0c441e2f9e9bdff868b2f6f5114ff519" "checksum lexical-core 0.4.3 (registry+https://github.com/rust-lang/crates.io-index)" = "b8b0f90c979adde96d19eb10eb6431ba0c441e2f9e9bdff868b2f6f5114ff519"
"checksum libc 0.2.62 (registry+https://github.com/rust-lang/crates.io-index)" = "34fcd2c08d2f832f376f4173a231990fa5aef4e99fb569867318a227ef4c06ba" "checksum libc 0.2.62 (registry+https://github.com/rust-lang/crates.io-index)" = "34fcd2c08d2f832f376f4173a231990fa5aef4e99fb569867318a227ef4c06ba"
"checksum log 0.4.8 (registry+https://github.com/rust-lang/crates.io-index)" = "14b6052be84e6b71ab17edffc2eeabf5c2c3ae1fdb464aae35ac50c67a44e1f7" "checksum log 0.4.8 (registry+https://github.com/rust-lang/crates.io-index)" = "14b6052be84e6b71ab17edffc2eeabf5c2c3ae1fdb464aae35ac50c67a44e1f7"
"checksum memchr 2.2.1 (registry+https://github.com/rust-lang/crates.io-index)" = "88579771288728879b57485cc7d6b07d648c9f0141eb955f8ab7f9d45394468e" "checksum memchr 2.2.1 (registry+https://github.com/rust-lang/crates.io-index)" = "88579771288728879b57485cc7d6b07d648c9f0141eb955f8ab7f9d45394468e"
"checksum miniz-sys 0.1.12 (registry+https://github.com/rust-lang/crates.io-index)" = "1e9e3ae51cea1576ceba0dde3d484d30e6e5b86dee0b2d412fe3a16a15c98202"
"checksum miniz_oxide 0.3.2 (registry+https://github.com/rust-lang/crates.io-index)" = "7108aff85b876d06f22503dcce091e29f76733b2bfdd91eebce81f5e68203a10"
"checksum nom 4.2.3 (registry+https://github.com/rust-lang/crates.io-index)" = "2ad2a91a8e869eeb30b9cb3119ae87773a8f4ae617f41b1eb9c154b2905f7bd6"
"checksum nom 5.0.0 (registry+https://github.com/rust-lang/crates.io-index)" = "e9761d859320e381010a4f7f8ed425f2c924de33ad121ace447367c713ad561b" "checksum nom 5.0.0 (registry+https://github.com/rust-lang/crates.io-index)" = "e9761d859320e381010a4f7f8ed425f2c924de33ad121ace447367c713ad561b"
"checksum num-traits 0.2.8 (registry+https://github.com/rust-lang/crates.io-index)" = "6ba9a427cfca2be13aa6f6403b0b7e7368fe982bfa16fccc450ce74c46cd9b32"
"checksum proc-macro2 0.4.30 (registry+https://github.com/rust-lang/crates.io-index)" = "cf3d2011ab5c909338f7887f4fc896d35932e29146c12c8d01da6b22a80ba759" "checksum proc-macro2 0.4.30 (registry+https://github.com/rust-lang/crates.io-index)" = "cf3d2011ab5c909338f7887f4fc896d35932e29146c12c8d01da6b22a80ba759"
"checksum quote 0.6.13 (registry+https://github.com/rust-lang/crates.io-index)" = "6ce23b6b870e8f94f81fb0a363d65d86675884b34a09043c81e5562f11c1f8e1" "checksum quote 0.6.13 (registry+https://github.com/rust-lang/crates.io-index)" = "6ce23b6b870e8f94f81fb0a363d65d86675884b34a09043c81e5562f11c1f8e1"
"checksum rustc_version 0.2.3 (registry+https://github.com/rust-lang/crates.io-index)" = "138e3e0acb6c9fb258b19b67cb8abd63c00679d2851805ea151465464fe9030a" "checksum rustc_version 0.2.3 (registry+https://github.com/rust-lang/crates.io-index)" = "138e3e0acb6c9fb258b19b67cb8abd63c00679d2851805ea151465464fe9030a"

1
Cargo.toml
View File

@ -8,6 +8,7 @@ edition = "2018"
capnp = "0.10.1" capnp = "0.10.1"
clap = "2.33.0" clap = "2.33.0"
flatbuffers = "0.6.0" flatbuffers = "0.6.0"
hdrhistogram = "6.3.4"
nom = "5.0.0" nom = "5.0.0"
smallvec = "0.6.10" smallvec = "0.6.10"

6
build.rs
View File

@ -15,7 +15,8 @@ fn main() {
inputs: &[Path::new("marketdata.fbs")], inputs: &[Path::new("marketdata.fbs")],
out_dir: Path::new("src/"), out_dir: Path::new("src/"),
..Default::default() ..Default::default()
}).expect("Unable to compile flatc"); })
.expect("Unable to compile flatc");
// There's no Rust-style builder crate for SBE, // There's no Rust-style builder crate for SBE,
// so we need to run the command by hand. // so we need to run the command by hand.
@ -25,7 +26,8 @@ fn main() {
.arg("-Dsbe.xinclude.aware=true") .arg("-Dsbe.xinclude.aware=true")
.arg("-Dsbe.target.language=uk.co.real_logic.sbe.generation.rust.Rust") .arg("-Dsbe.target.language=uk.co.real_logic.sbe.generation.rust.Rust")
.arg("-Dsbe.target.namespace=marketdata_sbe") .arg("-Dsbe.target.namespace=marketdata_sbe")
.arg("-jar").arg("sbe-all-1.13.2-all.jar") .arg("-jar")
.arg("sbe-all-1.13.2-all.jar")
.arg("marketdata.xml") .arg("marketdata.xml")
.output() .output()
.expect("Unable to execute SBE compiler"); .expect("Unable to execute SBE compiler");

118
src/capnp_runner.rs
View File

@ -1,26 +1,25 @@
use std::convert::TryInto; use std::convert::TryInto;
use std::io::{BufRead, Read}; use std::io::{BufRead, Read};
use std::mem::size_of; use std::mem::size_of;
use std::str::from_utf8_unchecked;
use capnp::Error; use capnp::message::{ReaderOptions, ScratchSpace, ScratchSpaceHeapAllocator};
use capnp::message::{Builder, ReaderOptions, ScratchSpace, ScratchSpaceHeapAllocator};
use capnp::serialize::write_message; use capnp::serialize::write_message;
use capnp::serialize_packed::{read_message as read_message_packed, write_message as write_message_packed}; use capnp::serialize_packed::{
use nom::bytes::complete::take_until; read_message as read_message_packed, write_message as write_message_packed,
use nom::IResult; };
use capnp::Error;
use crate::{RunnerDeserialize, RunnerSerialize, StreamVec, Summarizer};
use crate::iex::{IexMessage, IexPayload}; use crate::iex::{IexMessage, IexPayload};
use crate::marketdata_capnp::{multi_message, Side};
use crate::marketdata_capnp::message; use crate::marketdata_capnp::message;
use crate::marketdata_capnp::{multi_message, Side};
use crate::{RunnerDeserialize, RunnerSerialize, StreamVec, Summarizer};
pub struct CapnpWriter<'a> { pub struct CapnpWriter<'a> {
// We have to be very careful with how messages are built, as running // We have to be very careful with how messages are built, as running
// `init_root` and rebuilding will still accumulate garbage if using // `init_root` and rebuilding will still accumulate garbage if using
// the standard HeapAllocator. // the standard HeapAllocator.
// https://github.com/capnproto/capnproto-rust/issues/111 // https://github.com/capnproto/capnproto-rust/issues/111
words: Vec<capnp::Word>, _words: Vec<capnp::Word>,
scratch: ScratchSpace<'a>, scratch: ScratchSpace<'a>,
packed: bool, packed: bool,
} }
@ -31,14 +30,12 @@ impl<'a> CapnpWriter<'a> {
// In practice, let's just make sure everything fits. // In practice, let's just make sure everything fits.
let mut words = capnp::Word::allocate_zeroed_vec(1024); let mut words = capnp::Word::allocate_zeroed_vec(1024);
let mut scratch = ScratchSpace::new(unsafe { let scratch = ScratchSpace::new(unsafe { std::mem::transmute(&mut words[..]) });
std::mem::transmute(&mut words[..])
});
CapnpWriter { CapnpWriter {
words, _words: words,
scratch, scratch,
packed packed,
} }
} }
@ -56,12 +53,14 @@ impl<'a> CapnpWriter<'a> {
impl<'a> RunnerSerialize for CapnpWriter<'a> { impl<'a> RunnerSerialize for CapnpWriter<'a> {
fn serialize(&mut self, payload: &IexPayload, mut output: &mut Vec<u8>) { fn serialize(&mut self, payload: &IexPayload, mut output: &mut Vec<u8>) {
// First, count the messages we actually care about. // First, count the messages we actually care about.
let num_msgs = payload.messages.iter().map(|m| { let num_msgs = payload
match m { .messages
.iter()
.map(|m| match m {
IexMessage::TradeReport(_) | IexMessage::PriceLevelUpdate(_) => 1, IexMessage::TradeReport(_) | IexMessage::PriceLevelUpdate(_) => 1,
_ => 0 _ => 0,
} })
}).fold(0, |sum, i| sum + i); .fold(0, |sum, i| sum + i);
if num_msgs == 0 { if num_msgs == 0 {
return; return;
@ -109,13 +108,21 @@ impl<'a> RunnerSerialize for CapnpWriter<'a> {
msg_plu.set_price(plu.price); msg_plu.set_price(plu.price);
msg_plu.set_size(plu.size); msg_plu.set_size(plu.size);
msg_plu.set_flags(plu.event_flags); msg_plu.set_flags(plu.event_flags);
msg_plu.set_side(if plu.msg_type == 0x38 { Side::Buy } else { Side::Sell }); msg_plu.set_side(if plu.msg_type == 0x38 {
Side::Buy
} else {
Side::Sell
});
} }
_ => () _ => (),
} }
} }
let write_fn = if self.packed { write_message_packed } else { write_message }; let write_fn = if self.packed {
write_message_packed
} else {
write_message
};
write_fn(&mut output, &builder).unwrap(); write_fn(&mut output, &builder).unwrap();
} }
@ -123,25 +130,28 @@ impl<'a> RunnerSerialize for CapnpWriter<'a> {
pub struct CapnpReader { pub struct CapnpReader {
read_opts: ReaderOptions, read_opts: ReaderOptions,
packed: bool packed: bool,
} }
impl CapnpReader { impl CapnpReader {
pub fn new(packed: bool) -> CapnpReader { pub fn new(packed: bool) -> CapnpReader {
CapnpReader { CapnpReader {
read_opts: ReaderOptions::new(), read_opts: ReaderOptions::new(),
packed packed,
} }
} }
} }
impl CapnpReader { impl CapnpReader {
fn deserialize_packed<'a>(&self, buf: &'a mut StreamVec, stats: &mut Summarizer) -> Result<(), ()> { fn deserialize_packed<'a>(
&self,
buf: &'a mut StreamVec,
stats: &mut Summarizer,
) -> Result<(), ()> {
// Because `capnp::serialize_packed::PackedRead` is hidden from us, packed reads // Because `capnp::serialize_packed::PackedRead` is hidden from us, packed reads
// *have* to both allocate new segments every read, and copy the buffer into // *have* to both allocate new segments every read, and copy the buffer into
// those same segments, no ability to re-use allocated memory. // those same segments, no ability to re-use allocated memory.
let reader = read_message_packed(buf, self.read_opts) let reader = read_message_packed(buf, self.read_opts).map_err(|_| ())?;
.map_err(|_| ())?;
let multimsg = reader.get_root::<multi_message::Reader>().unwrap(); let multimsg = reader.get_root::<multi_message::Reader>().unwrap();
for msg in multimsg.get_messages().unwrap().iter() { for msg in multimsg.get_messages().unwrap().iter() {
@ -149,18 +159,18 @@ impl CapnpReader {
Ok(message::Trade(tr)) => { Ok(message::Trade(tr)) => {
let tr = tr.unwrap(); let tr = tr.unwrap();
stats.append_trade_volume(msg.get_symbol().unwrap(), tr.get_size() as u64); stats.append_trade_volume(msg.get_symbol().unwrap(), tr.get_size() as u64);
}, }
Ok(message::Quote(q)) => { Ok(message::Quote(q)) => {
let q = q.unwrap(); let q = q.unwrap();
let is_bid = match q.get_side().unwrap() { let is_bid = match q.get_side().unwrap() {
Side::Buy => true, Side::Buy => true,
_ => false _ => false,
}; };
stats.update_quote_prices(msg.get_symbol().unwrap(), q.get_price(), is_bid); stats.update_quote_prices(msg.get_symbol().unwrap(), q.get_price(), is_bid);
}, }
_ => panic!("Unrecognized message type!") _ => panic!("Unrecognized message type!"),
} }
}; }
Ok(()) Ok(())
} }
@ -192,8 +202,7 @@ impl CapnpReader {
There is no documentation on how to calculate `bytes_consumed` when parsing by hand There is no documentation on how to calculate `bytes_consumed` when parsing by hand
that I could find, you just have to guess and check until you figure this one out. that I could find, you just have to guess and check until you figure this one out.
*/ */
let (num_words, offsets) = read_segment_table(&mut data, reader_opts) let (num_words, offsets) = read_segment_table(&mut data, reader_opts).map_err(|_| ())?;
.map_err(|_| ())?;
let words = unsafe { capnp::Word::bytes_to_words(data) }; let words = unsafe { capnp::Word::bytes_to_words(data) };
let reader = capnp::message::Reader::new( let reader = capnp::message::Reader::new(
SliceSegments { SliceSegments {
@ -206,8 +215,7 @@ impl CapnpReader {
let msg_bytes = num_words * size_of::<capnp::Word>(); let msg_bytes = num_words * size_of::<capnp::Word>();
let bytes_consumed = segment_table_bytes + msg_bytes; let bytes_consumed = segment_table_bytes + msg_bytes;
let multimsg = reader.get_root::<multi_message::Reader>() let multimsg = reader.get_root::<multi_message::Reader>().map_err(|_| ())?;
.map_err(|_| ())?;
for msg in multimsg.get_messages().map_err(|_| ())?.iter() { for msg in multimsg.get_messages().map_err(|_| ())?.iter() {
let sym = msg.get_symbol().map_err(|_| ())?; let sym = msg.get_symbol().map_err(|_| ())?;
@ -215,15 +223,15 @@ impl CapnpReader {
message::Trade(trade) => { message::Trade(trade) => {
let trade = trade.unwrap(); let trade = trade.unwrap();
stats.append_trade_volume(sym, trade.get_size().into()); stats.append_trade_volume(sym, trade.get_size().into());
}, }
message::Quote(quote) => { message::Quote(quote) => {
let quote = quote.unwrap(); let quote = quote.unwrap();
let is_buy = match quote.get_side().unwrap() { let is_buy = match quote.get_side().unwrap() {
Side::Buy => true, Side::Buy => true,
_ => false _ => false,
}; };
stats.update_quote_prices(sym, quote.get_price(), is_buy); stats.update_quote_prices(sym, quote.get_price(), is_buy);
}, }
} }
} }
@ -244,7 +252,6 @@ impl RunnerDeserialize for CapnpReader {
} }
} }
pub struct SliceSegments<'a> { pub struct SliceSegments<'a> {
words: &'a [capnp::Word], words: &'a [capnp::Word],
segment_slices: Vec<(usize, usize)>, segment_slices: Vec<(usize, usize)>,
@ -265,10 +272,12 @@ impl<'a> capnp::message::ReaderSegments for SliceSegments<'a> {
} }
} }
fn read_segment_table<R>(read: &mut R, fn read_segment_table<R>(
options: capnp::message::ReaderOptions) read: &mut R,
-> capnp::Result<(usize, Vec<(usize, usize)>)> options: capnp::message::ReaderOptions,
where R: Read ) -> capnp::Result<(usize, Vec<(usize, usize)>)>
where
R: Read,
{ {
let mut buf: [u8; 8] = [0; 8]; let mut buf: [u8; 8] = [0; 8];
@ -277,9 +286,15 @@ fn read_segment_table<R>(read: &mut R,
let segment_count = u32::from_le_bytes(buf[0..4].try_into().unwrap()).wrapping_add(1) as usize; let segment_count = u32::from_le_bytes(buf[0..4].try_into().unwrap()).wrapping_add(1) as usize;
if segment_count >= 512 { if segment_count >= 512 {
return Err(Error::failed(format!("Too many segments: {}", segment_count))) return Err(Error::failed(format!(
"Too many segments: {}",
segment_count
)));
} else if segment_count == 0 { } else if segment_count == 0 {
return Err(Error::failed(format!("Too few segments: {}", segment_count))) return Err(Error::failed(format!(
"Too few segments: {}",
segment_count
)));
} }
let mut segment_slices = Vec::with_capacity(segment_count); let mut segment_slices = Vec::with_capacity(segment_count);
@ -301,7 +316,8 @@ fn read_segment_table<R>(read: &mut R,
read.read_exact(&mut segment_sizes[..])?; read.read_exact(&mut segment_sizes[..])?;
for idx in 0..(segment_count - 1) { for idx in 0..(segment_count - 1) {
let segment_len = let segment_len =
u32::from_le_bytes(segment_sizes[(idx * 4)..(idx + 1) * 4].try_into().unwrap()) as usize; u32::from_le_bytes(segment_sizes[(idx * 4)..(idx + 1) * 4].try_into().unwrap())
as usize;
segment_slices.push((total_words, total_words + segment_len)); segment_slices.push((total_words, total_words + segment_len));
total_words += segment_len; total_words += segment_len;
@ -313,9 +329,11 @@ fn read_segment_table<R>(read: &mut R,
// traversal limit. Without this check, a malicious client could transmit a very large segment // traversal limit. Without this check, a malicious client could transmit a very large segment
// size to make the receiver allocate excessive space and possibly crash. // size to make the receiver allocate excessive space and possibly crash.
if total_words as u64 > options.traversal_limit_in_words { if total_words as u64 > options.traversal_limit_in_words {
return Err(Error::failed( return Err(Error::failed(format!(
format!("Message has {} words, which is too large. To increase the limit on the \ "Message has {} words, which is too large. To increase the limit on the \
receiving end, see capnp::message::ReaderOptions.", total_words))) receiving end, see capnp::message::ReaderOptions.",
total_words
)));
} }
Ok((total_words, segment_slices)) Ok((total_words, segment_slices))

30
src/flatbuffers_runner.rs
View File

@ -1,15 +1,10 @@
use std::convert::TryInto; use std::convert::TryInto;
use std::io::{BufRead, Error, Write}; use std::io::{BufRead, Write};
use std::mem::size_of; use std::mem::size_of;
use std::str::from_utf8_unchecked;
use capnp::data::new_builder;
use flatbuffers::buffer_has_identifier;
use nom::{bytes::complete::take_until, IResult};
use crate::{RunnerDeserialize, RunnerSerialize, StreamVec, Summarizer};
use crate::iex::{IexMessage, IexPayload}; use crate::iex::{IexMessage, IexPayload};
use crate::marketdata_generated::md_shootout; use crate::marketdata_generated::md_shootout;
use crate::{RunnerDeserialize, RunnerSerialize, StreamVec, Summarizer};
pub struct FlatbuffersWriter<'a> { pub struct FlatbuffersWriter<'a> {
builder: flatbuffers::FlatBufferBuilder<'a>, builder: flatbuffers::FlatBufferBuilder<'a>,
@ -27,7 +22,6 @@ impl<'a> FlatbuffersWriter<'a> {
impl<'a> RunnerSerialize for FlatbuffersWriter<'a> { impl<'a> RunnerSerialize for FlatbuffersWriter<'a> {
fn serialize(&mut self, payload: &IexPayload, output: &mut Vec<u8>) { fn serialize(&mut self, payload: &IexPayload, output: &mut Vec<u8>) {
// Because FlatBuffers can't handle nested vectors (specifically, we can't track // Because FlatBuffers can't handle nested vectors (specifically, we can't track
// both the variable-length vector of messages, and the variable-length strings // both the variable-length vector of messages, and the variable-length strings
// within those messages), we have to cache the messages as they get built // within those messages), we have to cache the messages as they get built
@ -61,7 +55,11 @@ impl<'a> RunnerSerialize for FlatbuffersWriter<'a> {
price: plu.price, price: plu.price,
size_: plu.size, size_: plu.size,
flags: plu.event_flags, flags: plu.event_flags,
side: if plu.msg_type == 0x38 { md_shootout::Side::Buy } else { md_shootout::Side::Sell }, side: if plu.msg_type == 0x38 {
md_shootout::Side::Buy
} else {
md_shootout::Side::Sell
},
}, },
); );
@ -73,7 +71,7 @@ impl<'a> RunnerSerialize for FlatbuffersWriter<'a> {
body: Some(level_update.as_union_value()), body: Some(level_update.as_union_value()),
}) })
} }
_ => None _ => None,
}; };
msg_args.map(|a| { msg_args.map(|a| {
@ -118,7 +116,7 @@ impl RunnerDeserialize for FlatbuffersReader {
// a view over the underlying buffer. // a view over the underlying buffer.
let data = buf.fill_buf().unwrap(); let data = buf.fill_buf().unwrap();
if data.len() == 0 { if data.len() == 0 {
return Err(()) return Err(());
} }
let msg_len_buf: [u8; 4] = data[..size_of::<u32>()].try_into().unwrap(); let msg_len_buf: [u8; 4] = data[..size_of::<u32>()].try_into().unwrap();
@ -127,7 +125,7 @@ impl RunnerDeserialize for FlatbuffersReader {
let multimsg = flatbuffers::get_size_prefixed_root::<md_shootout::MultiMessage>(data); let multimsg = flatbuffers::get_size_prefixed_root::<md_shootout::MultiMessage>(data);
let msg_vec = match multimsg.messages() { let msg_vec = match multimsg.messages() {
Some(m) => m, Some(m) => m,
None => panic!("Couldn't find messages") None => panic!("Couldn't find messages"),
}; };
for i in 0..msg_vec.len() { for i in 0..msg_vec.len() {
@ -136,16 +134,16 @@ impl RunnerDeserialize for FlatbuffersReader {
md_shootout::MessageBody::Trade => { md_shootout::MessageBody::Trade => {
let trade = msg.body_as_trade().unwrap(); let trade = msg.body_as_trade().unwrap();
stats.append_trade_volume(msg.symbol().unwrap(), trade.size_().into()); stats.append_trade_volume(msg.symbol().unwrap(), trade.size_().into());
}, }
md_shootout::MessageBody::LevelUpdate => { md_shootout::MessageBody::LevelUpdate => {
let lu = msg.body_as_level_update().unwrap(); let lu = msg.body_as_level_update().unwrap();
let is_bid = match lu.side() { let is_bid = match lu.side() {
md_shootout::Side::Buy => true, md_shootout::Side::Buy => true,
_ => false _ => false,
}; };
stats.update_quote_prices(msg.symbol().unwrap(), lu.price(), is_bid); stats.update_quote_prices(msg.symbol().unwrap(), lu.price(), is_bid);
}, }
md_shootout::MessageBody::NONE => panic!("Unrecognized message type") md_shootout::MessageBody::NONE => panic!("Unrecognized message type"),
} }
} }

4
src/iex.rs
View File

@ -1,8 +1,8 @@
use std::convert::TryInto; use std::convert::TryInto;
use nom::{bytes::complete::take, IResult, number::complete::*, sequence::tuple}; use nom::{bytes::complete::take, number::complete::*, sequence::tuple, IResult};
use crate::parsers::{Block, extract_iex_data, read_block}; use crate::parsers::{extract_iex_data, read_block, Block};
pub struct IexParser<'a> { pub struct IexParser<'a> {
pcap_buffer: &'a [u8], pcap_buffer: &'a [u8],

206
src/main.rs
View File

@ -2,13 +2,14 @@ use std::cmp::{max, min};
use std::collections::hash_map::{DefaultHasher, HashMap}; use std::collections::hash_map::{DefaultHasher, HashMap};
use std::fs::File; use std::fs::File;
use std::hash::Hasher; use std::hash::Hasher;
use std::io::{BufRead, Read};
use std::io::Error; use std::io::Error;
use std::io::{BufRead, Read};
use std::path::Path; use std::path::Path;
use std::str::from_utf8_unchecked; use std::str::from_utf8_unchecked;
use std::time::{Instant, SystemTime}; use std::time::Instant;
use clap::{App, Arg}; use clap::{App, Arg};
use hdrhistogram::Histogram;
use nom::{bytes::complete::take_until, IResult}; use nom::{bytes::complete::take_until, IResult};
use crate::iex::{IexParser, IexPayload}; use crate::iex::{IexParser, IexPayload};
@ -19,13 +20,14 @@ use crate::iex::{IexParser, IexPayload};
pub mod marketdata_capnp; pub mod marketdata_capnp;
#[allow(unused_imports)] #[allow(unused_imports)]
pub mod marketdata_generated; // Flatbuffers pub mod marketdata_generated; // Flatbuffers
#[allow(dead_code)]
pub mod marketdata_sbe; pub mod marketdata_sbe;
mod capnp_runner; mod capnp_runner;
mod flatbuffers_runner; mod flatbuffers_runner;
mod sbe_runner;
mod iex; mod iex;
mod parsers; mod parsers;
mod sbe_runner;
fn main() { fn main() {
let matches = App::new("Marketdata Shootout") let matches = App::new("Marketdata Shootout")
@ -48,84 +50,32 @@ fn main() {
file.read_to_end(&mut buf) file.read_to_end(&mut buf)
.expect(&format!("Unable to read file={}", path.display())); .expect(&format!("Unable to read file={}", path.display()));
let _start = SystemTime::now(); let _capnp_unpacked = run_analysis(
let mut summarizer = Summarizer::default(); &buf,
let mut parser = IexParser::new(&buf[..]); &mut capnp_runner::CapnpWriter::new(false),
&mut capnp_runner::CapnpReader::new(false),
);
// Pre-allocate the same size as the backing file. Will be way more than let _capnp_packed = run_analysis(
// necessary, but makes sure there's no re-allocation not related to &buf,
// actual parsing/serialization code &mut capnp_runner::CapnpWriter::new(true),
let mut output_buf: Vec<u8> = Vec::with_capacity(buf.capacity()); &mut capnp_runner::CapnpReader::new(true),
);
/* let _flatbuffers = run_analysis(
let mut capnp_writer = capnp_runner::CapnpWriter::new(); &buf,
for iex_payload in parser { &mut flatbuffers_runner::FlatbuffersWriter::new(),
//let iex_payload = parser.next().unwrap(); &mut flatbuffers_runner::FlatbuffersReader::new(),
capnp_writer.serialize(&iex_payload, &mut output_buf, true); );
}
let capnp_reader = capnp_runner::CapnpReader::new(); let _sbe = run_analysis(
let mut read_buf = StreamVec::new(output_buf); &buf,
let mut parsed_msgs: u64 = 0; &mut sbe_runner::SBEWriter::new(),
while let Ok(_) = capnp_reader.deserialize_packed(&mut read_buf, &mut summarizer) { &mut sbe_runner::SBEReader::new(),
parsed_msgs += 1; );
}
*/
let mut fb_writer = flatbuffers_runner::FlatbuffersWriter::new();
for iex_payload in parser {
let now = Instant::now();
fb_writer.serialize(&iex_payload, &mut output_buf);
let serialize_nanos = Instant::now().duration_since(now).as_nanos();
dbg!(serialize_nanos);
}
let mut read_buf = StreamVec::new(output_buf);
let fb_reader = flatbuffers_runner::FlatbuffersReader::new();
let mut parsed_msgs = 0;
while let Ok(_) = fb_reader.deserialize(&mut read_buf, &mut summarizer) {
parsed_msgs += 1;
}
/*
let mut capnp_writer = capnp_runner::CapnpWriter::new();
for iex_payload in parser {
//let iex_payload = parser.next().unwrap();
let now = Instant::now();
capnp_writer.serialize(&iex_payload, &mut output_buf, false);
let serialize_nanos = Instant::now().duration_since(now).as_nanos();
dbg!(serialize_nanos);
}
let capnp_reader = capnp_runner::CapnpReader::new();
let mut read_buf = StreamVec::new(output_buf);
let mut parsed_msgs: u64 = 0;
while let Ok(_) = capnp_reader.deserialize_unpacked(&mut read_buf, &mut summarizer) {
parsed_msgs += 1;
}
*/
/*
let mut sbe_writer = sbe_runner::SBEWriter::new();
for iex_payload in parser {
//let iex_payload = parser.next().unwrap();
sbe_writer.serialize(&iex_payload, &mut output_buf);
}
let sbe_reader = sbe_runner::SBEReader::new();
let mut read_buf = StreamVec::new(output_buf);
let mut parsed_msgs: u64 = 0;
while let Ok(_) = sbe_reader.deserialize(&mut read_buf, &mut summarizer) {
parsed_msgs += 1;
}
*/
dbg!(parsed_msgs);
dbg!(summarizer);
} }
#[derive(Debug)] #[derive(Debug, PartialEq)]
pub struct SummaryStats { pub struct SummaryStats {
symbol: String, symbol: String,
trade_volume: u64, trade_volume: u64,
@ -135,24 +85,23 @@ pub struct SummaryStats {
ask_low: u64, ask_low: u64,
} }
#[derive(Default, Debug)] #[derive(Default, Debug, PartialEq)]
pub struct Summarizer { pub struct Summarizer {
data: HashMap<u64, SummaryStats> data: HashMap<u64, SummaryStats>,
} }
impl Summarizer { impl Summarizer {
fn entry(&mut self, sym: &str) -> &mut SummaryStats { fn entry(&mut self, sym: &str) -> &mut SummaryStats {
let mut hasher = DefaultHasher::new(); let mut hasher = DefaultHasher::new();
hasher.write(sym.as_bytes()); hasher.write(sym.as_bytes());
self.data.entry(hasher.finish()) self.data.entry(hasher.finish()).or_insert(SummaryStats {
.or_insert(SummaryStats { symbol: sym.to_string(),
symbol: sym.to_string(), trade_volume: 0,
trade_volume: 0, bid_high: 0,
bid_high: 0, bid_low: u64::max_value(),
bid_low: u64::max_value(), ask_high: 0,
ask_high: 0, ask_low: u64::max_value(),
ask_low: u64::max_value(), })
})
} }
pub fn append_trade_volume(&mut self, sym: &str, volume: u64) { pub fn append_trade_volume(&mut self, sym: &str, volume: u64) {
@ -178,10 +127,7 @@ pub struct StreamVec {
impl StreamVec { impl StreamVec {
pub fn new(buf: Vec<u8>) -> StreamVec { pub fn new(buf: Vec<u8>) -> StreamVec {
StreamVec { StreamVec { pos: 0, inner: buf }
pos: 0,
inner: buf,
}
} }
} }
@ -189,7 +135,11 @@ impl Read for StreamVec {
fn read(&mut self, buf: &mut [u8]) -> Result<usize, Error> { fn read(&mut self, buf: &mut [u8]) -> Result<usize, Error> {
// TODO: There's *got* to be a better way to handle this // TODO: There's *got* to be a better way to handle this
let end = self.pos + buf.len(); let end = self.pos + buf.len();
let end = if end > self.inner.len() { self.inner.len() } else { end }; let end = if end > self.inner.len() {
self.inner.len()
} else {
end
};
let read_size = end - self.pos; let read_size = end - self.pos;
buf[..read_size].copy_from_slice(&self.inner[self.pos..end]); buf[..read_size].copy_from_slice(&self.inner[self.pos..end]);
self.pos = end; self.pos = end;
@ -226,3 +176,75 @@ fn parse_symbol(sym: &[u8; 8]) -> &str {
let (_, sym_bytes) = __take_until(" ", &sym[..]).unwrap(); let (_, sym_bytes) = __take_until(" ", &sym[..]).unwrap();
unsafe { from_utf8_unchecked(sym_bytes) } unsafe { from_utf8_unchecked(sym_bytes) }
} }
struct RunAnalysis {
serialize_hist: Histogram<u64>,
deserialize_hist: Histogram<u64>,
summary_stats: Summarizer,
serialize_total_nanos: u128,
deserialize_total_nanos: u128,
buf_len: usize,
}
fn run_analysis<S, D>(iex_data: &Vec<u8>, serializer: &mut S, deserializer: &mut D) -> RunAnalysis
where
S: RunnerSerialize,
D: RunnerDeserialize,
{
let upper = if cfg!(debug_assertions) {
1_000_000
} else {
100_000
};
let iex_parser = IexParser::new(iex_data);
let mut output_buf = Vec::with_capacity(iex_data.len());
let mut serialize_hist = Histogram::<u64>::new_with_bounds(1, upper, 2).unwrap();
let mut serialize_nanos_total = 0u128;
let mut serialize_msgs = 0;
for iex_payload in iex_parser {
let serialize_start = Instant::now();
serializer.serialize(&iex_payload, &mut output_buf);
let serialize_end = Instant::now().duration_since(serialize_start).as_nanos();
serialize_hist.record(serialize_end as u64).unwrap();
serialize_nanos_total += serialize_end;
serialize_msgs += 1;
}
let output_len = output_buf.len();
let mut read_buf = StreamVec::new(output_buf);
let mut summarizer = Summarizer::default();
let mut deserialize_hist = Histogram::<u64>::new_with_bounds(1, upper, 2).unwrap();
let mut parsed_msgs: u64 = 0;
let mut deserialize_nanos_total = 0u128;
loop {
let deserialize_start = Instant::now();
let res = deserializer.deserialize(&mut read_buf, &mut summarizer);
let deserialize_end = Instant::now().duration_since(deserialize_start).as_nanos();
if res.is_ok() {
deserialize_hist.record(deserialize_end as u64).unwrap();
deserialize_nanos_total += deserialize_end;
parsed_msgs += 1;
} else {
break;
}
}
dbg!(serialize_msgs, parsed_msgs);
RunAnalysis {
serialize_hist,
deserialize_hist,
summary_stats: summarizer,
serialize_total_nanos: serialize_nanos_total,
deserialize_total_nanos: deserialize_nanos_total,
buf_len: output_len,
}
}

1502
src/marketdata_capnp.rs
View File

File diff suppressed because it is too large Load Diff

1013
src/marketdata_generated.rs
View File

File diff suppressed because it is too large Load Diff

736
src/marketdata_sbe.rs
View File

@ -1,6 +1,5 @@
/// Generated code for SBE package marketdata_sbe /// Generated code for SBE package marketdata_sbe
/// Imports core rather than std to broaden usable environments. /// Imports core rather than std to broaden usable environments.
extern crate core; extern crate core;
@ -9,13 +8,13 @@ extern crate core;
/// Errors that may occur during the course of encoding or decoding. /// Errors that may occur during the course of encoding or decoding.
#[derive(Debug)] #[derive(Debug)]
pub enum CodecErr { pub enum CodecErr {
/// Too few bytes in the byte-slice to read or write the data structure relevant /// Too few bytes in the byte-slice to read or write the data structure relevant
/// to the current state of the codec /// to the current state of the codec
NotEnoughBytes, NotEnoughBytes,
/// Groups and vardata are constrained by the numeric type chosen to represent their /// Groups and vardata are constrained by the numeric type chosen to represent their
/// length as well as optional maxima imposed by the schema /// length as well as optional maxima imposed by the schema
SliceIsLongerThanAllowedBySchema, SliceIsLongerThanAllowedBySchema,
} }
pub type CodecResult<T> = core::result::Result<T, CodecErr>; pub type CodecResult<T> = core::result::Result<T, CodecErr>;
@ -23,206 +22,219 @@ pub type CodecResult<T> = core::result::Result<T, CodecErr>;
/// Scratch Decoder Data Wrapper - codec internal use only /// Scratch Decoder Data Wrapper - codec internal use only
#[derive(Debug)] #[derive(Debug)]
pub struct ScratchDecoderData<'d> { pub struct ScratchDecoderData<'d> {
data: &'d [u8], data: &'d [u8],
pos: usize, pos: usize,
} }
impl<'d> ScratchDecoderData<'d> { impl<'d> ScratchDecoderData<'d> {
/// Create a struct reference overlaid atop the data buffer /// Create a struct reference overlaid atop the data buffer
/// such that the struct's contents directly reflect the buffer. /// such that the struct's contents directly reflect the buffer.
/// Advances the `pos` index by the size of the struct in bytes. /// Advances the `pos` index by the size of the struct in bytes.
#[inline] #[inline]
fn read_type<T>(&mut self, num_bytes: usize) -> CodecResult<&'d T> { fn read_type<T>(&mut self, num_bytes: usize) -> CodecResult<&'d T> {
let end = self.pos + num_bytes; let end = self.pos + num_bytes;
if end <= self.data.len() { if end <= self.data.len() {
let s = self.data[self.pos..end].as_ptr() as *mut T; let s = self.data[self.pos..end].as_ptr() as *mut T;
let v: &'d T = unsafe { &*s }; let v: &'d T = unsafe { &*s };
self.pos = end; self.pos = end;
Ok(v) Ok(v)
} else { } else {
Err(CodecErr::NotEnoughBytes) Err(CodecErr::NotEnoughBytes)
}
} }
}
/// Advances the `pos` index by a set number of bytes. /// Advances the `pos` index by a set number of bytes.
#[inline] #[inline]
fn skip_bytes(&mut self, num_bytes: usize) -> CodecResult<()> { fn skip_bytes(&mut self, num_bytes: usize) -> CodecResult<()> {
let end = self.pos + num_bytes; let end = self.pos + num_bytes;
if end <= self.data.len() { if end <= self.data.len() {
self.pos = end; self.pos = end;
Ok(()) Ok(())
} else { } else {
Err(CodecErr::NotEnoughBytes) Err(CodecErr::NotEnoughBytes)
}
} }
}
/// Create a slice reference overlaid atop the data buffer /// Create a slice reference overlaid atop the data buffer
/// such that the slice's members' contents directly reflect the buffer. /// such that the slice's members' contents directly reflect the buffer.
/// Advances the `pos` index by the size of the slice contents in bytes. /// Advances the `pos` index by the size of the slice contents in bytes.
#[inline] #[inline]
fn read_slice<T>(&mut self, count: usize, bytes_per_item: usize) -> CodecResult<&'d [T]> { fn read_slice<T>(&mut self, count: usize, bytes_per_item: usize) -> CodecResult<&'d [T]> {
let num_bytes = bytes_per_item * count; let num_bytes = bytes_per_item * count;
let end = self.pos + num_bytes; let end = self.pos + num_bytes;
if end <= self.data.len() { if end <= self.data.len() {
let v: &'d [T] = unsafe { let v: &'d [T] = unsafe {
core::slice::from_raw_parts(self.data[self.pos..end].as_ptr() as *const T, count) core::slice::from_raw_parts(self.data[self.pos..end].as_ptr() as *const T, count)
}; };
self.pos = end; self.pos = end;
Ok(v) Ok(v)
} else { } else {
Err(CodecErr::NotEnoughBytes) Err(CodecErr::NotEnoughBytes)
}
} }
}
} }
/// Scratch Encoder Data Wrapper - codec internal use only /// Scratch Encoder Data Wrapper - codec internal use only
#[derive(Debug)] #[derive(Debug)]
pub struct ScratchEncoderData<'d> { pub struct ScratchEncoderData<'d> {
data: &'d mut [u8], data: &'d mut [u8],
pos: usize, pos: usize,
} }
impl<'d> ScratchEncoderData<'d> { impl<'d> ScratchEncoderData<'d> {
/// Copy the bytes of a value into the data buffer /// Copy the bytes of a value into the data buffer
/// Advances the `pos` index to after the newly-written bytes. /// Advances the `pos` index to after the newly-written bytes.
#[inline] #[inline]
fn write_type<T>(&mut self, t: &T, num_bytes: usize) -> CodecResult<()> { fn write_type<T>(&mut self, t: &T, num_bytes: usize) -> CodecResult<()> {
let end = self.pos + num_bytes; let end = self.pos + num_bytes;
if end <= self.data.len() { if end <= self.data.len() {
let source_bytes: &[u8] = unsafe { let source_bytes: &[u8] =
core::slice::from_raw_parts(t as *const T as *const u8, num_bytes) unsafe { core::slice::from_raw_parts(t as *const T as *const u8, num_bytes) };
}; (&mut self.data[self.pos..end]).copy_from_slice(source_bytes);
(&mut self.data[self.pos..end]).copy_from_slice(source_bytes); self.pos = end;
self.pos = end; Ok(())
Ok(()) } else {
} else { Err(CodecErr::NotEnoughBytes)
Err(CodecErr::NotEnoughBytes) }
} }
}
/// Advances the `pos` index by a set number of bytes. /// Advances the `pos` index by a set number of bytes.
#[inline] #[inline]
fn skip_bytes(&mut self, num_bytes: usize) -> CodecResult<()> { fn skip_bytes(&mut self, num_bytes: usize) -> CodecResult<()> {
let end = self.pos + num_bytes; let end = self.pos + num_bytes;
if end <= self.data.len() { if end <= self.data.len() {
self.pos = end; self.pos = end;
Ok(()) Ok(())
} else { } else {
Err(CodecErr::NotEnoughBytes) Err(CodecErr::NotEnoughBytes)
}
} }
}
/// Create a struct reference overlaid atop the data buffer /// Create a struct reference overlaid atop the data buffer
/// such that changes to the struct directly edit the buffer. /// such that changes to the struct directly edit the buffer.
/// Note that the initial content of the struct's fields may be garbage. /// Note that the initial content of the struct's fields may be garbage.
/// Advances the `pos` index to after the newly-written bytes. /// Advances the `pos` index to after the newly-written bytes.
#[inline] #[inline]
fn writable_overlay<T>(&mut self, num_bytes: usize) -> CodecResult<&'d mut T> { fn writable_overlay<T>(&mut self, num_bytes: usize) -> CodecResult<&'d mut T> {
let end = self.pos + num_bytes; let end = self.pos + num_bytes;
if end <= self.data.len() { if end <= self.data.len() {
let v: &'d mut T = unsafe { let v: &'d mut T = unsafe {
let s = self.data.as_ptr().offset(self.pos as isize) as *mut T; let s = self.data.as_ptr().offset(self.pos as isize) as *mut T;
&mut *s &mut *s
}; };
self.pos = end; self.pos = end;
Ok(v) Ok(v)
} else { } else {
Err(CodecErr::NotEnoughBytes) Err(CodecErr::NotEnoughBytes)
}
} }
}
/// Copy the bytes of a value into the data buffer at a specific position /// Copy the bytes of a value into the data buffer at a specific position
/// Does **not** alter the `pos` index. /// Does **not** alter the `pos` index.
#[inline] #[inline]
fn write_at_position<T>(&mut self, position: usize, t: &T, num_bytes: usize) -> CodecResult<()> { fn write_at_position<T>(
let end = position + num_bytes; &mut self,
if end <= self.data.len() { position: usize,
let source_bytes: &[u8] = unsafe { t: &T,
core::slice::from_raw_parts(t as *const T as *const u8, num_bytes) num_bytes: usize,
}; ) -> CodecResult<()> {
(&mut self.data[position..end]).copy_from_slice(source_bytes); let end = position + num_bytes;
Ok(()) if end <= self.data.len() {
} else { let source_bytes: &[u8] =
Err(CodecErr::NotEnoughBytes) unsafe { core::slice::from_raw_parts(t as *const T as *const u8, num_bytes) };
(&mut self.data[position..end]).copy_from_slice(source_bytes);
Ok(())
} else {
Err(CodecErr::NotEnoughBytes)
}
} }
} /// Create a mutable slice overlaid atop the data buffer directly
/// Create a mutable slice overlaid atop the data buffer directly /// such that changes to the slice contents directly edit the buffer
/// such that changes to the slice contents directly edit the buffer /// Note that the initial content of the slice's members' fields may be garbage.
/// Note that the initial content of the slice's members' fields may be garbage. /// Advances the `pos` index to after the region representing the slice.
/// Advances the `pos` index to after the region representing the slice. #[inline]
#[inline] fn writable_slice<T>(
fn writable_slice<T>(&mut self, count: usize, bytes_per_item: usize) -> CodecResult<&'d mut [T]> { &mut self,
let end = self.pos + (count * bytes_per_item); count: usize,
if end <= self.data.len() { bytes_per_item: usize,
let v: &'d mut [T] = unsafe { ) -> CodecResult<&'d mut [T]> {
core::slice::from_raw_parts_mut(self.data[self.pos..end].as_mut_ptr() as *mut T, count) let end = self.pos + (count * bytes_per_item);
}; if end <= self.data.len() {
self.pos = end; let v: &'d mut [T] = unsafe {
Ok(v) core::slice::from_raw_parts_mut(
} else { self.data[self.pos..end].as_mut_ptr() as *mut T,
Err(CodecErr::NotEnoughBytes) count,
)
};
self.pos = end;
Ok(v)
} else {
Err(CodecErr::NotEnoughBytes)
}
} }
}
/// Copy the raw bytes of a slice's contents into the data buffer /// Copy the raw bytes of a slice's contents into the data buffer
/// Does **not** encode the length of the slice explicitly into the buffer. /// Does **not** encode the length of the slice explicitly into the buffer.
/// Advances the `pos` index to after the newly-written slice bytes. /// Advances the `pos` index to after the newly-written slice bytes.
#[inline] #[inline]
fn write_slice_without_count<T>(&mut self, t: &[T], bytes_per_item: usize) -> CodecResult<()> { fn write_slice_without_count<T>(&mut self, t: &[T], bytes_per_item: usize) -> CodecResult<()> {
let content_bytes_size = bytes_per_item * t.len(); let content_bytes_size = bytes_per_item * t.len();
let end = self.pos + content_bytes_size; let end = self.pos + content_bytes_size;
if end <= self.data.len() { if end <= self.data.len() {
let source_bytes: &[u8] = unsafe { let source_bytes: &[u8] =
core::slice::from_raw_parts(t.as_ptr() as *const u8, content_bytes_size) unsafe { core::slice::from_raw_parts(t.as_ptr() as *const u8, content_bytes_size) };
}; (&mut self.data[self.pos..end]).copy_from_slice(source_bytes);
(&mut self.data[self.pos..end]).copy_from_slice(source_bytes); self.pos = end;
self.pos = end; Ok(())
Ok(()) } else {
} else { Err(CodecErr::NotEnoughBytes)
Err(CodecErr::NotEnoughBytes) }
} }
}
} }
/// Convenience Either enum /// Convenience Either enum
#[derive(Copy, Clone, PartialEq, Eq, PartialOrd, Ord, Hash, Debug)] #[derive(Copy, Clone, PartialEq, Eq, PartialOrd, Ord, Hash, Debug)]
pub enum Either<L, R> { pub enum Either<L, R> {
Left(L), Left(L),
Right(R) Right(R),
} }
/// Enum Side /// Enum Side
#[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord, Hash)] #[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord, Hash)]
#[repr(u8)] #[repr(u8)]
pub enum Side { pub enum Side {
Buy = 0u8, Buy = 0u8,
Sell = 1u8, Sell = 1u8,
NullVal = 255u8, NullVal = 255u8,
} }
impl Default for Side { impl Default for Side {
fn default() -> Self { Side::NullVal } fn default() -> Self {
Side::NullVal
}
} }
/// Enum MsgType /// Enum MsgType
#[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord, Hash)] #[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord, Hash)]
#[repr(u8)] #[repr(u8)]
pub enum MsgType { pub enum MsgType {
Trade = 0u8, Trade = 0u8,
Quote = 1u8, Quote = 1u8,
NullVal = 255u8, NullVal = 255u8,
} }
impl Default for MsgType { impl Default for MsgType {
fn default() -> Self { MsgType::NullVal } fn default() -> Self {
MsgType::NullVal
}
} }
/// Quote /// Quote
#[repr(C, packed)] #[repr(C, packed)]
#[derive(Default)] #[derive(Default)]
pub struct Quote { pub struct Quote {
pub price: u64, pub price: u64,
pub size: u32, pub size: u32,
pub flags: u8, pub flags: u8,
pub side: Side, pub side: Side,
} }
impl Quote {} impl Quote {}
@ -231,8 +243,8 @@ impl Quote {}
#[repr(C, packed)] #[repr(C, packed)]
#[derive(Default)] #[derive(Default)]
pub struct Trade { pub struct Trade {
pub price: u64, pub price: u64,
pub size: u32, pub size: u32,
} }
impl Trade {} impl Trade {}
@ -241,10 +253,10 @@ impl Trade {}
#[repr(C, packed)] #[repr(C, packed)]
#[derive(Default)] #[derive(Default)]
pub struct MessageHeader { pub struct MessageHeader {
pub block_length: u16, pub block_length: u16,
pub template_id: u16, pub template_id: u16,
pub schema_id: u16, pub schema_id: u16,
pub version: u16, pub version: u16,
} }
impl MessageHeader {} impl MessageHeader {}
@ -253,8 +265,8 @@ impl MessageHeader {}
#[repr(C, packed)] #[repr(C, packed)]
#[derive(Default)] #[derive(Default)]
pub struct GroupSizeEncoding { pub struct GroupSizeEncoding {
pub block_length: u16, pub block_length: u16,
pub num_in_group: u16, pub num_in_group: u16,
} }
impl GroupSizeEncoding {} impl GroupSizeEncoding {}
@ -263,308 +275,350 @@ impl GroupSizeEncoding {}
#[repr(C, packed)] #[repr(C, packed)]
#[derive(Default)] #[derive(Default)]
pub struct VarAsciiEncoding { pub struct VarAsciiEncoding {
pub length: u32, pub length: u32,
pub var_data: u8, pub var_data: u8,
} }
impl VarAsciiEncoding {} impl VarAsciiEncoding {}
/// MessageHeader Decoder entry point /// MessageHeader Decoder entry point
pub fn start_decoding_message_header<'d>(data: &'d [u8]) -> CodecResult<(&'d MessageHeader, ScratchDecoderData<'d>)> { pub fn start_decoding_message_header<'d>(
let mut scratch = ScratchDecoderData { data: data, pos: 0 }; data: &'d [u8],
let v = scratch.read_type::<MessageHeader>(8)?; ) -> CodecResult<(&'d MessageHeader, ScratchDecoderData<'d>)> {
Ok((v, scratch)) let mut scratch = ScratchDecoderData { data: data, pos: 0 };
let v = scratch.read_type::<MessageHeader>(8)?;
Ok((v, scratch))
} }
/// MultiMessage Fixed-size Fields (8 bytes) /// MultiMessage Fixed-size Fields (8 bytes)
#[repr(C, packed)] #[repr(C, packed)]
#[derive(Default)] #[derive(Default)]
pub struct MultiMessageFields { pub struct MultiMessageFields {
pub sequence_number: u64, pub sequence_number: u64,
} }
impl MultiMessageFields {} impl MultiMessageFields {}
/// MultiMessage specific Message Header /// MultiMessage specific Message Header
#[repr(C, packed)] #[repr(C, packed)]
pub struct MultiMessageMessageHeader { pub struct MultiMessageMessageHeader {
pub message_header: MessageHeader pub message_header: MessageHeader,
} }
impl MultiMessageMessageHeader { impl MultiMessageMessageHeader {
pub const BLOCK_LENGTH: u16 = 8; pub const BLOCK_LENGTH: u16 = 8;
pub const TEMPLATE_ID: u16 = 1; pub const TEMPLATE_ID: u16 = 1;
pub const SCHEMA_ID: u16 = 1; pub const SCHEMA_ID: u16 = 1;
pub const VERSION: u16 = 0; pub const VERSION: u16 = 0;
} }
impl Default for MultiMessageMessageHeader { impl Default for MultiMessageMessageHeader {
fn default() -> MultiMessageMessageHeader { fn default() -> MultiMessageMessageHeader {
MultiMessageMessageHeader { MultiMessageMessageHeader {
message_header: MessageHeader { message_header: MessageHeader {
block_length: 8u16, block_length: 8u16,
template_id: 1u16, template_id: 1u16,
schema_id: 1u16, schema_id: 1u16,
version: 0u16, version: 0u16,
} },
}
} }
}
} }
/// Group fixed-field member representations /// Group fixed-field member representations
#[repr(C, packed)] #[repr(C, packed)]
#[derive(Default)] #[derive(Default)]
pub struct MultiMessageMessagesMember { pub struct MultiMessageMessagesMember {
pub timestamp: i64, pub timestamp: i64,
pub msg_type: MsgType, pub msg_type: MsgType,
pub trade: Trade, pub trade: Trade,
pub quote: Quote, pub quote: Quote,
} }
impl MultiMessageMessagesMember {} impl MultiMessageMessagesMember {}
/// MultiMessageDecoderDone /// MultiMessageDecoderDone
pub struct MultiMessageDecoderDone<'d> { pub struct MultiMessageDecoderDone<'d> {
scratch: ScratchDecoderData<'d>, scratch: ScratchDecoderData<'d>,
} }
impl<'d> MultiMessageDecoderDone<'d> { impl<'d> MultiMessageDecoderDone<'d> {
/// Returns the number of bytes decoded /// Returns the number of bytes decoded
pub fn unwrap(self) -> usize { pub fn unwrap(self) -> usize {
self.scratch.pos self.scratch.pos
} }
pub fn wrap(scratch: ScratchDecoderData<'d>) -> MultiMessageDecoderDone<'d> { pub fn wrap(scratch: ScratchDecoderData<'d>) -> MultiMessageDecoderDone<'d> {
MultiMessageDecoderDone { scratch: scratch } MultiMessageDecoderDone { scratch: scratch }
} }
} }
/// symbol variable-length data /// symbol variable-length data
pub struct MultiMessageMessagesSymbolDecoder<'d> { pub struct MultiMessageMessagesSymbolDecoder<'d> {
parent: MultiMessageMessagesMemberDecoder<'d>, parent: MultiMessageMessagesMemberDecoder<'d>,
} }
impl<'d> MultiMessageMessagesSymbolDecoder<'d> { impl<'d> MultiMessageMessagesSymbolDecoder<'d> {
fn wrap(parent: MultiMessageMessagesMemberDecoder<'d>) -> Self { fn wrap(parent: MultiMessageMessagesMemberDecoder<'d>) -> Self {
MultiMessageMessagesSymbolDecoder { parent: parent } MultiMessageMessagesSymbolDecoder { parent: parent }
} }
pub fn symbol(mut self) -> CodecResult<(&'d [u8], Either<MultiMessageMessagesMemberDecoder<'d>, MultiMessageDecoderDone<'d>>)> { pub fn symbol(
let count = *self.parent.scratch.read_type::<u32>(4)?; mut self,
Ok((self.parent.scratch.read_slice::<u8>(count as usize, 1)?, self.parent.after_member())) ) -> CodecResult<(
} &'d [u8],
Either<MultiMessageMessagesMemberDecoder<'d>, MultiMessageDecoderDone<'d>>,
)> {
let count = *self.parent.scratch.read_type::<u32>(4)?;
Ok((
self.parent.scratch.read_slice::<u8>(count as usize, 1)?,
self.parent.after_member(),
))
}
} }
/// MultiMessageMessages Decoder for fields and header /// MultiMessageMessages Decoder for fields and header
pub struct MultiMessageMessagesMemberDecoder<'d> { pub struct MultiMessageMessagesMemberDecoder<'d> {
scratch: ScratchDecoderData<'d>, scratch: ScratchDecoderData<'d>,
max_index: u16, max_index: u16,
index: u16, index: u16,
} }
impl<'d> MultiMessageMessagesMemberDecoder<'d> { impl<'d> MultiMessageMessagesMemberDecoder<'d> {
fn new(scratch: ScratchDecoderData<'d>, count: u16) -> Self { fn new(scratch: ScratchDecoderData<'d>, count: u16) -> Self {
assert!(count > 0u16); assert!(count > 0u16);
MultiMessageMessagesMemberDecoder { MultiMessageMessagesMemberDecoder {
scratch: scratch, scratch: scratch,
max_index: count - 1, max_index: count - 1,
index: 0, index: 0,
}
} }
}
pub fn next_messages_member(mut self) -> CodecResult<(&'d MultiMessageMessagesMember, MultiMessageMessagesSymbolDecoder<'d>)> { pub fn next_messages_member(
let v = self.scratch.read_type::<MultiMessageMessagesMember>(35)?; mut self,
self.index += 1; ) -> CodecResult<(
Ok((v, MultiMessageMessagesSymbolDecoder::wrap(self))) &'d MultiMessageMessagesMember,
} MultiMessageMessagesSymbolDecoder<'d>,
#[inline] )> {
fn after_member(self) -> Either<MultiMessageMessagesMemberDecoder<'d>, MultiMessageDecoderDone<'d>> { let v = self.scratch.read_type::<MultiMessageMessagesMember>(35)?;
if self.index <= self.max_index { self.index += 1;
Either::Left(self) Ok((v, MultiMessageMessagesSymbolDecoder::wrap(self)))
} else { }
Either::Right(MultiMessageDecoderDone::wrap(self.scratch)) #[inline]
fn after_member(
self,
) -> Either<MultiMessageMessagesMemberDecoder<'d>, MultiMessageDecoderDone<'d>> {
if self.index <= self.max_index {
Either::Left(self)
} else {
Either::Right(MultiMessageDecoderDone::wrap(self.scratch))
}
} }
}
} }
pub struct MultiMessageMessagesHeaderDecoder<'d> { pub struct MultiMessageMessagesHeaderDecoder<'d> {
scratch: ScratchDecoderData<'d>, scratch: ScratchDecoderData<'d>,
} }
impl<'d> MultiMessageMessagesHeaderDecoder<'d> { impl<'d> MultiMessageMessagesHeaderDecoder<'d> {
fn wrap(scratch: ScratchDecoderData<'d>) -> Self { fn wrap(scratch: ScratchDecoderData<'d>) -> Self {
MultiMessageMessagesHeaderDecoder { scratch: scratch } MultiMessageMessagesHeaderDecoder { scratch: scratch }
} }
pub fn messages_individually(mut self) -> CodecResult<Either<MultiMessageMessagesMemberDecoder<'d>, MultiMessageDecoderDone<'d>>> { pub fn messages_individually(
let dim = self.scratch.read_type::<GroupSizeEncoding>(4)?; mut self,
if dim.num_in_group > 0 { ) -> CodecResult<Either<MultiMessageMessagesMemberDecoder<'d>, MultiMessageDecoderDone<'d>>>
Ok(Either::Left(MultiMessageMessagesMemberDecoder::new(self.scratch, dim.num_in_group))) {
} else { let dim = self.scratch.read_type::<GroupSizeEncoding>(4)?;
Ok(Either::Right(MultiMessageDecoderDone::wrap(self.scratch))) if dim.num_in_group > 0 {
Ok(Either::Left(MultiMessageMessagesMemberDecoder::new(
self.scratch,
dim.num_in_group,
)))
} else {
Ok(Either::Right(MultiMessageDecoderDone::wrap(self.scratch)))
}
} }
}
} }
/// MultiMessage Fixed fields Decoder /// MultiMessage Fixed fields Decoder
pub struct MultiMessageFieldsDecoder<'d> { pub struct MultiMessageFieldsDecoder<'d> {
scratch: ScratchDecoderData<'d>, scratch: ScratchDecoderData<'d>,
} }
impl<'d> MultiMessageFieldsDecoder<'d> { impl<'d> MultiMessageFieldsDecoder<'d> {
pub fn wrap(scratch: ScratchDecoderData<'d>) -> MultiMessageFieldsDecoder<'d> {
pub fn wrap(scratch: ScratchDecoderData<'d>) -> MultiMessageFieldsDecoder<'d> { MultiMessageFieldsDecoder { scratch: scratch }
MultiMessageFieldsDecoder { scratch: scratch } }
} pub fn multi_message_fields(
pub fn multi_message_fields(mut self) -> CodecResult<(&'d MultiMessageFields, MultiMessageMessagesHeaderDecoder<'d>)> { mut self,
let v = self.scratch.read_type::<MultiMessageFields>(8)?; ) -> CodecResult<(
Ok((v, MultiMessageMessagesHeaderDecoder::wrap(self.scratch))) &'d MultiMessageFields,
} MultiMessageMessagesHeaderDecoder<'d>,
)> {
let v = self.scratch.read_type::<MultiMessageFields>(8)?;
Ok((v, MultiMessageMessagesHeaderDecoder::wrap(self.scratch)))
}
} }
/// MultiMessageMessageHeaderDecoder /// MultiMessageMessageHeaderDecoder
pub struct MultiMessageMessageHeaderDecoder<'d> { pub struct MultiMessageMessageHeaderDecoder<'d> {
scratch: ScratchDecoderData<'d>, scratch: ScratchDecoderData<'d>,
} }
impl<'d> MultiMessageMessageHeaderDecoder<'d> { impl<'d> MultiMessageMessageHeaderDecoder<'d> {
pub fn wrap(scratch: ScratchDecoderData<'d>) -> MultiMessageMessageHeaderDecoder<'d> {
pub fn wrap(scratch: ScratchDecoderData<'d>) -> MultiMessageMessageHeaderDecoder<'d> { MultiMessageMessageHeaderDecoder { scratch: scratch }
MultiMessageMessageHeaderDecoder { scratch: scratch } }
} pub fn header(mut self) -> CodecResult<(&'d MessageHeader, MultiMessageFieldsDecoder<'d>)> {
pub fn header(mut self) -> CodecResult<(&'d MessageHeader, MultiMessageFieldsDecoder<'d>)> { let v = self.scratch.read_type::<MessageHeader>(8)?;
let v = self.scratch.read_type::<MessageHeader>(8)?; Ok((v, MultiMessageFieldsDecoder::wrap(self.scratch)))
Ok((v, MultiMessageFieldsDecoder::wrap(self.scratch))) }
}
} }
/// MultiMessage Decoder entry point /// MultiMessage Decoder entry point
pub fn start_decoding_multi_message<'d>(data: &'d [u8]) -> MultiMessageMessageHeaderDecoder<'d> { pub fn start_decoding_multi_message<'d>(data: &'d [u8]) -> MultiMessageMessageHeaderDecoder<'d> {
MultiMessageMessageHeaderDecoder::wrap(ScratchDecoderData { data: data, pos: 0 }) MultiMessageMessageHeaderDecoder::wrap(ScratchDecoderData { data: data, pos: 0 })
} }
/// MultiMessageEncoderDone /// MultiMessageEncoderDone
pub struct MultiMessageEncoderDone<'d> { pub struct MultiMessageEncoderDone<'d> {
scratch: ScratchEncoderData<'d>, scratch: ScratchEncoderData<'d>,
} }
impl<'d> MultiMessageEncoderDone<'d> { impl<'d> MultiMessageEncoderDone<'d> {
/// Returns the number of bytes encoded /// Returns the number of bytes encoded
pub fn unwrap(self) -> usize { pub fn unwrap(self) -> usize {
self.scratch.pos self.scratch.pos
} }
pub fn wrap(scratch: ScratchEncoderData<'d>) -> MultiMessageEncoderDone<'d> { pub fn wrap(scratch: ScratchEncoderData<'d>) -> MultiMessageEncoderDone<'d> {
MultiMessageEncoderDone { scratch: scratch } MultiMessageEncoderDone { scratch: scratch }
} }
} }
/// symbol variable-length data /// symbol variable-length data
pub struct MultiMessageMessagesSymbolEncoder<'d> { pub struct MultiMessageMessagesSymbolEncoder<'d> {
parent: MultiMessageMessagesMemberEncoder<'d>, parent: MultiMessageMessagesMemberEncoder<'d>,
} }
impl<'d> MultiMessageMessagesSymbolEncoder<'d> { impl<'d> MultiMessageMessagesSymbolEncoder<'d> {
fn wrap(parent: MultiMessageMessagesMemberEncoder<'d>) -> Self { fn wrap(parent: MultiMessageMessagesMemberEncoder<'d>) -> Self {
MultiMessageMessagesSymbolEncoder { parent: parent } MultiMessageMessagesSymbolEncoder { parent: parent }
} }
pub fn symbol(mut self, s: &'d [u8]) -> CodecResult<MultiMessageMessagesMemberEncoder> { pub fn symbol(mut self, s: &'d [u8]) -> CodecResult<MultiMessageMessagesMemberEncoder> {
let l = s.len(); let l = s.len();
if l > 4294967294 { if l > 4294967294 {
return Err(CodecErr::SliceIsLongerThanAllowedBySchema) return Err(CodecErr::SliceIsLongerThanAllowedBySchema);
}
// Write data length
self.parent.scratch.write_type::<u32>(&(l as u32), 4)?; // group length
self.parent.scratch.write_slice_without_count::<u8>(s, 1)?;
Ok(self.parent)
} }
// Write data length
self.parent.scratch.write_type::<u32>(&(l as u32), 4)?; // group length
self.parent.scratch.write_slice_without_count::<u8>(s, 1)?;
Ok(self.parent)
}
} }
/// MultiMessageMessages Encoder for fields and header /// MultiMessageMessages Encoder for fields and header
pub struct MultiMessageMessagesMemberEncoder<'d> { pub struct MultiMessageMessagesMemberEncoder<'d> {
scratch: ScratchEncoderData<'d>, scratch: ScratchEncoderData<'d>,
count_write_pos: usize, count_write_pos: usize,
count: u16, count: u16,
} }
impl<'d> MultiMessageMessagesMemberEncoder<'d> { impl<'d> MultiMessageMessagesMemberEncoder<'d> {
#[inline] #[inline]
fn new(scratch: ScratchEncoderData<'d>, count_write_pos: usize) -> Self { fn new(scratch: ScratchEncoderData<'d>, count_write_pos: usize) -> Self {
MultiMessageMessagesMemberEncoder { MultiMessageMessagesMemberEncoder {
scratch: scratch, scratch: scratch,
count_write_pos: count_write_pos, count_write_pos: count_write_pos,
count: 0, count: 0,
}
} }
}
#[inline] #[inline]
pub fn next_messages_member(mut self, fields: &MultiMessageMessagesMember) -> CodecResult<MultiMessageMessagesSymbolEncoder<'d>> { pub fn next_messages_member(
self.scratch.write_type::<MultiMessageMessagesMember>(fields, 35)?; // block length mut self,
self.count += 1; fields: &MultiMessageMessagesMember,
Ok(MultiMessageMessagesSymbolEncoder::wrap(self)) ) -> CodecResult<MultiMessageMessagesSymbolEncoder<'d>> {
} self.scratch
#[inline] .write_type::<MultiMessageMessagesMember>(fields, 35)?; // block length
pub fn done_with_messages(mut self) -> CodecResult<MultiMessageEncoderDone<'d>> { self.count += 1;
self.scratch.write_at_position::<u16>(self.count_write_pos, &self.count, 2)?; Ok(MultiMessageMessagesSymbolEncoder::wrap(self))
Ok(MultiMessageEncoderDone::wrap(self.scratch)) }
} #[inline]
pub fn done_with_messages(mut self) -> CodecResult<MultiMessageEncoderDone<'d>> {
self.scratch
.write_at_position::<u16>(self.count_write_pos, &self.count, 2)?;
Ok(MultiMessageEncoderDone::wrap(self.scratch))
}
} }
pub struct MultiMessageMessagesHeaderEncoder<'d> { pub struct MultiMessageMessagesHeaderEncoder<'d> {
scratch: ScratchEncoderData<'d>, scratch: ScratchEncoderData<'d>,
} }
impl<'d> MultiMessageMessagesHeaderEncoder<'d> { impl<'d> MultiMessageMessagesHeaderEncoder<'d> {
#[inline] #[inline]
fn wrap(scratch: ScratchEncoderData<'d>) -> Self { fn wrap(scratch: ScratchEncoderData<'d>) -> Self {
MultiMessageMessagesHeaderEncoder { scratch: scratch } MultiMessageMessagesHeaderEncoder { scratch: scratch }
} }
#[inline] #[inline]
pub fn messages_individually(mut self) -> CodecResult<MultiMessageMessagesMemberEncoder<'d>> { pub fn messages_individually(mut self) -> CodecResult<MultiMessageMessagesMemberEncoder<'d>> {
self.scratch.write_type::<u16>(&35u16, 2)?; // block length self.scratch.write_type::<u16>(&35u16, 2)?; // block length
let count_pos = self.scratch.pos; let count_pos = self.scratch.pos;
self.scratch.write_type::<u16>(&0, 2)?; // preliminary group member count self.scratch.write_type::<u16>(&0, 2)?; // preliminary group member count
Ok(MultiMessageMessagesMemberEncoder::new(self.scratch, count_pos)) Ok(MultiMessageMessagesMemberEncoder::new(
} self.scratch,
count_pos,
))
}
} }
/// MultiMessage Fixed fields Encoder /// MultiMessage Fixed fields Encoder
pub struct MultiMessageFieldsEncoder<'d> { pub struct MultiMessageFieldsEncoder<'d> {
scratch: ScratchEncoderData<'d>, scratch: ScratchEncoderData<'d>,
} }
impl<'d> MultiMessageFieldsEncoder<'d> { impl<'d> MultiMessageFieldsEncoder<'d> {
pub fn wrap(scratch: ScratchEncoderData<'d>) -> MultiMessageFieldsEncoder<'d> {
MultiMessageFieldsEncoder { scratch: scratch }
}
pub fn wrap(scratch: ScratchEncoderData<'d>) -> MultiMessageFieldsEncoder<'d> { /// Create a mutable struct reference overlaid atop the data buffer
MultiMessageFieldsEncoder { scratch: scratch }
}
/// Create a mutable struct reference overlaid atop the data buffer
/// such that changes to the struct directly edit the buffer. /// such that changes to the struct directly edit the buffer.
/// Note that the initial content of the struct's fields may be garbage. /// Note that the initial content of the struct's fields may be garbage.
pub fn multi_message_fields(mut self) -> CodecResult<(&'d mut MultiMessageFields, MultiMessageMessagesHeaderEncoder<'d>)> { pub fn multi_message_fields(
let v = self.scratch.writable_overlay::<MultiMessageFields>(8 + 0)?; mut self,
Ok((v, MultiMessageMessagesHeaderEncoder::wrap(self.scratch))) ) -> CodecResult<(
} &'d mut MultiMessageFields,
MultiMessageMessagesHeaderEncoder<'d>,
)> {
let v = self.scratch.writable_overlay::<MultiMessageFields>(8 + 0)?;
Ok((v, MultiMessageMessagesHeaderEncoder::wrap(self.scratch)))
}
/// Copy the bytes of a value into the data buffer /// Copy the bytes of a value into the data buffer
pub fn multi_message_fields_copy(mut self, t: &MultiMessageFields) -> CodecResult<MultiMessageMessagesHeaderEncoder<'d>> { pub fn multi_message_fields_copy(
self.scratch.write_type::<MultiMessageFields>(t, 8)?; mut self,
Ok(MultiMessageMessagesHeaderEncoder::wrap(self.scratch)) t: &MultiMessageFields,
} ) -> CodecResult<MultiMessageMessagesHeaderEncoder<'d>> {
self.scratch.write_type::<MultiMessageFields>(t, 8)?;
Ok(MultiMessageMessagesHeaderEncoder::wrap(self.scratch))
}
} }
/// MultiMessageMessageHeaderEncoder /// MultiMessageMessageHeaderEncoder
pub struct MultiMessageMessageHeaderEncoder<'d> { pub struct MultiMessageMessageHeaderEncoder<'d> {
scratch: ScratchEncoderData<'d>, scratch: ScratchEncoderData<'d>,
} }
impl<'d> MultiMessageMessageHeaderEncoder<'d> { impl<'d> MultiMessageMessageHeaderEncoder<'d> {
pub fn wrap(scratch: ScratchEncoderData<'d>) -> MultiMessageMessageHeaderEncoder<'d> { pub fn wrap(scratch: ScratchEncoderData<'d>) -> MultiMessageMessageHeaderEncoder<'d> {
MultiMessageMessageHeaderEncoder { scratch: scratch } MultiMessageMessageHeaderEncoder { scratch: scratch }
} }
/// Create a mutable struct reference overlaid atop the data buffer /// Create a mutable struct reference overlaid atop the data buffer
/// such that changes to the struct directly edit the buffer. /// such that changes to the struct directly edit the buffer.
/// Note that the initial content of the struct's fields may be garbage. /// Note that the initial content of the struct's fields may be garbage.
pub fn header(mut self) -> CodecResult<(&'d mut MessageHeader, MultiMessageFieldsEncoder<'d>)> { pub fn header(mut self) -> CodecResult<(&'d mut MessageHeader, MultiMessageFieldsEncoder<'d>)> {
let v = self.scratch.writable_overlay::<MessageHeader>(8 + 0)?; let v = self.scratch.writable_overlay::<MessageHeader>(8 + 0)?;
Ok((v, MultiMessageFieldsEncoder::wrap(self.scratch))) Ok((v, MultiMessageFieldsEncoder::wrap(self.scratch)))
} }
/// Copy the bytes of a value into the data buffer /// Copy the bytes of a value into the data buffer
pub fn header_copy(mut self, t: &MessageHeader) -> CodecResult<MultiMessageFieldsEncoder<'d>> { pub fn header_copy(mut self, t: &MessageHeader) -> CodecResult<MultiMessageFieldsEncoder<'d>> {
self.scratch.write_type::<MessageHeader>(t, 8)?; self.scratch.write_type::<MessageHeader>(t, 8)?;
Ok(MultiMessageFieldsEncoder::wrap(self.scratch)) Ok(MultiMessageFieldsEncoder::wrap(self.scratch))
} }
} }
/// MultiMessage Encoder entry point /// MultiMessage Encoder entry point
pub fn start_encoding_multi_message<'d>(data: &'d mut [u8]) -> MultiMessageMessageHeaderEncoder<'d> { pub fn start_encoding_multi_message<'d>(
MultiMessageMessageHeaderEncoder::wrap(ScratchEncoderData { data: data, pos: 0 }) data: &'d mut [u8],
) -> MultiMessageMessageHeaderEncoder<'d> {
MultiMessageMessageHeaderEncoder::wrap(ScratchEncoderData { data: data, pos: 0 })
} }

4
src/parsers.rs
View File

@ -1,6 +1,6 @@
use nom::{ use nom::{
branch::alt, bytes::complete::tag, bytes::complete::take, IResult, number::complete::*, branch::alt, bytes::complete::tag, bytes::complete::take, number::complete::*, sequence::tuple,
sequence::tuple, IResult,
}; };
pub enum Block<'a> { pub enum Block<'a> {

59
src/sbe_runner.rs
View File

@ -1,12 +1,13 @@
use std::io::{BufRead, Write}; use std::io::{BufRead, Write};
use std::str::from_utf8_unchecked; use std::str::from_utf8_unchecked;
use nom::bytes::complete::take_until;
use nom::IResult;
use crate::{marketdata_sbe, RunnerDeserialize, RunnerSerialize, StreamVec, Summarizer};
use crate::iex::{IexMessage, IexPayload}; use crate::iex::{IexMessage, IexPayload};
use crate::marketdata_sbe::{Either, MultiMessageFields, MultiMessageMessageHeader, MultiMessageMessagesMember, MultiMessageMessagesMemberEncoder, MultiMessageMessagesSymbolEncoder, Side, start_decoding_multi_message, start_encoding_multi_message}; use crate::marketdata_sbe::{
start_decoding_multi_message, start_encoding_multi_message, Either, MultiMessageMessageHeader,
MultiMessageMessagesMember, MultiMessageMessagesMemberEncoder,
MultiMessageMessagesSymbolEncoder, Side,
};
use crate::{marketdata_sbe, RunnerDeserialize, RunnerSerialize, StreamVec, Summarizer};
pub struct SBEWriter { pub struct SBEWriter {
/// Buffer to construct messages before copying. While SBE benefits /// Buffer to construct messages before copying. While SBE benefits
@ -32,13 +33,15 @@ impl SBEWriter {
impl RunnerSerialize for SBEWriter { impl RunnerSerialize for SBEWriter {
fn serialize(&mut self, payload: &IexPayload, output: &mut Vec<u8>) { fn serialize(&mut self, payload: &IexPayload, output: &mut Vec<u8>) {
let (fields, encoder) = start_encoding_multi_message(&mut self.scratch_buffer[..]) let (fields, encoder) = start_encoding_multi_message(&mut self.scratch_buffer[..])
.header_copy(&self.default_header.message_header).unwrap() .header_copy(&self.default_header.message_header)
.multi_message_fields().unwrap(); .unwrap()
.multi_message_fields()
.unwrap();
fields.sequence_number = payload.first_seq_no; fields.sequence_number = payload.first_seq_no;
let mut encoder = encoder.messages_individually().unwrap(); let encoder = encoder.messages_individually().unwrap();
let mut encoder: MultiMessageMessagesMemberEncoder = payload.messages.iter().fold(encoder, |enc, m| { let encoder: MultiMessageMessagesMemberEncoder =
match m { payload.messages.iter().fold(encoder, |enc, m| match m {
IexMessage::TradeReport(tr) => { IexMessage::TradeReport(tr) => {
let fields = MultiMessageMessagesMember { let fields = MultiMessageMessagesMember {
msg_type: marketdata_sbe::MsgType::Trade, msg_type: marketdata_sbe::MsgType::Trade,
@ -49,8 +52,11 @@ impl RunnerSerialize for SBEWriter {
}, },
..Default::default() ..Default::default()
}; };
let sym_enc: MultiMessageMessagesSymbolEncoder = enc.next_messages_member(&fields).unwrap(); let sym_enc: MultiMessageMessagesSymbolEncoder =
sym_enc.symbol(crate::parse_symbol(&tr.symbol).as_bytes()).unwrap() enc.next_messages_member(&fields).unwrap();
sym_enc
.symbol(crate::parse_symbol(&tr.symbol).as_bytes())
.unwrap()
} }
IexMessage::PriceLevelUpdate(plu) => { IexMessage::PriceLevelUpdate(plu) => {
let fields = MultiMessageMessagesMember { let fields = MultiMessageMessagesMember {
@ -60,16 +66,22 @@ impl RunnerSerialize for SBEWriter {
price: plu.price, price: plu.price,
size: plu.size, size: plu.size,
flags: plu.event_flags, flags: plu.event_flags,
side: if plu.msg_type == 0x38 { Side::Buy } else { Side::Sell }, side: if plu.msg_type == 0x38 {
Side::Buy
} else {
Side::Sell
},
}, },
..Default::default() ..Default::default()
}; };
let sym_enc: MultiMessageMessagesSymbolEncoder = enc.next_messages_member(&fields).unwrap(); let sym_enc: MultiMessageMessagesSymbolEncoder =
sym_enc.symbol(crate::parse_symbol(&plu.symbol).as_bytes()).unwrap() enc.next_messages_member(&fields).unwrap();
sym_enc
.symbol(crate::parse_symbol(&plu.symbol).as_bytes())
.unwrap()
} }
_ => enc _ => enc,
} });
});
let finished = encoder.done_with_messages().unwrap(); let finished = encoder.done_with_messages().unwrap();
let data_len = finished.unwrap(); let data_len = finished.unwrap();
@ -93,10 +105,9 @@ impl RunnerDeserialize for SBEReader {
return Err(()); return Err(());
} }
let (header, decoder) = start_decoding_multi_message(data) let (_header, decoder) = start_decoding_multi_message(data).header().unwrap();
.header().unwrap();
let (fields, decoder) = decoder.multi_message_fields().unwrap(); let (_fields, decoder) = decoder.multi_message_fields().unwrap();
let mut msg_decoder = decoder.messages_individually().unwrap(); let mut msg_decoder = decoder.messages_individually().unwrap();
while let Either::Left(msg) = msg_decoder { while let Either::Left(msg) = msg_decoder {
let (member, sym_dec) = msg.next_messages_member().unwrap(); let (member, sym_dec) = msg.next_messages_member().unwrap();
@ -111,10 +122,10 @@ impl RunnerDeserialize for SBEReader {
member.quote.price, member.quote.price,
match member.quote.side { match member.quote.side {
Side::Buy => true, Side::Buy => true,
_ => false _ => false,
}, },
), ),
_ => () _ => (),
} }
msg_decoder = next_msg_dec; msg_decoder = next_msg_dec;
} }
@ -122,7 +133,7 @@ impl RunnerDeserialize for SBEReader {
// We now have a `Right`, which is a finished messages block // We now have a `Right`, which is a finished messages block
let msg_decoder = match msg_decoder { let msg_decoder = match msg_decoder {
Either::Right(r) => r, Either::Right(r) => r,
_ => panic!("Didn't parse all messages") _ => panic!("Didn't parse all messages"),
}; };
// Interestingly enough, `buf.consume(msg_decoder.unwrap())` isn't OK, // Interestingly enough, `buf.consume(msg_decoder.unwrap())` isn't OK,