mirror-github-bspeice/dtparse
1
0
Fork 0
mirror of https://github.com/bspeice/dtparse synced 2024-12-22 12:28:08 -05:00
Code Issues Releases Wiki Activity

Remove the Token enum

Browse Source 
Ended up not being that helpful
This commit is contained in:
Bradlee Speice 2018-05-24 22:24:28 -04:00
parent c3abcd4a55
commit 205f75995e
3 changed files with 38 additions and 59 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

17
build_tests.py
View File

@ -23,23 +23,10 @@ S12 = ' ' * 12
def rust_tokenize(time_string): def rust_tokenize(time_string):
split_array = _timelex.split(time_string) split_array = _timelex.split(time_string)
return ['"{}".to_owned()'.format(token) for token in split_array]
def translate_token(token):
if token[0].isalpha():
return 'Token::Alpha("{}".to_owned())'.format(token)
elif token[0].isnumeric():
return 'Token::Numeric("{}".to_owned())'.format(token)
elif len(token) == 1:
return 'Token::Separator("{}".to_owned())'.format(token)
else:
raise Exception("Invalid token during parsing of dateutil "
"split: {}".format(token))
return [translate_token(t) for t in split_array]
def build_split_string_tests(): def build_split_string_tests():
header = '''use Token; header = '''use ::tokenize;
use tokenize;
#[test] #[test]
fn test_python_compat() {\n''' fn test_python_compat() {\n'''

43
src/lib.rs
View File

@ -29,15 +29,8 @@ enum ParseInternalError {
type ParseResult<I> = Result<I, ParseError>; type ParseResult<I> = Result<I, ParseError>;
type ParseIResult<I> = Result<I, ParseInternalError>; type ParseIResult<I> = Result<I, ParseInternalError>;
#[derive(Debug, PartialEq)]
pub enum Token {
Alpha(String),
Numeric(String),
Separator(String),
}
pub struct Tokenizer { pub struct Tokenizer {
token_stack: Vec<Token>, token_stack: Vec<String>,
parse_string: String, parse_string: String,
} }
@ -60,7 +53,7 @@ impl Tokenizer {
} }
impl Iterator for Tokenizer { impl Iterator for Tokenizer {
type Item = Token; type Item = String;
fn next(&mut self) -> Option<Self::Item> { fn next(&mut self) -> Option<Self::Item> {
if !self.token_stack.is_empty() { if !self.token_stack.is_empty() {
@ -151,24 +144,24 @@ impl Iterator for Tokenizer {
}); });
let needs_split = seen_letters || dot_count > 1 || char_stack.last().unwrap() == &'.' let needs_split = seen_letters || dot_count > 1 || char_stack.last().unwrap() == &'.'
|| char_stack.last().unwrap() == &','; || char_stack.last().unwrap() == &',';
let final_string = char_stack.into_iter().collect(); let final_string: String = char_stack.into_iter().collect();
let mut tokens = match state { let mut tokens = match state {
ParseState::Empty => vec![Token::Separator(final_string)], ParseState::Empty => vec![final_string],
ParseState::Alpha => vec![Token::Alpha(final_string)], ParseState::Alpha => vec![final_string],
ParseState::Numeric => vec![Token::Numeric(final_string)], ParseState::Numeric => vec![final_string],
ParseState::AlphaDecimal => { ParseState::AlphaDecimal => {
if needs_split { if needs_split {
decimal_split(&final_string, false) decimal_split(&final_string, false)
} else { } else {
vec![Token::Alpha(final_string)] vec![final_string]
} }
} }
ParseState::NumericDecimal => { ParseState::NumericDecimal => {
if needs_split { if needs_split {
decimal_split(&final_string, dot_count == 0) decimal_split(&final_string, dot_count == 0)
} else { } else {
vec![Token::Numeric(final_string)] vec![final_string]
} }
} }
}.into_iter() }.into_iter()
@ -181,8 +174,8 @@ impl Iterator for Tokenizer {
} }
} }
fn decimal_split(characters: &str, cast_period: bool) -> Vec<Token> { fn decimal_split(characters: &str, cast_period: bool) -> Vec<String> {
let mut token_stack: Vec<Token> = Vec::new(); let mut token_stack: Vec<String> = Vec::new();
let mut char_stack: Vec<char> = Vec::new(); let mut char_stack: Vec<char> = Vec::new();
let mut state = ParseState::Empty; let mut state = ParseState::Empty;
@ -197,17 +190,17 @@ fn decimal_split(characters: &str, cast_period: bool) -> Vec<Token> {
state = ParseState::Numeric; state = ParseState::Numeric;
} else { } else {
let character = if cast_period { '.' } else { c }; let character = if cast_period { '.' } else { c };
token_stack.push(Token::Separator(character.to_string())); token_stack.push(character.to_string());
} }
} }
ParseState::Alpha => { ParseState::Alpha => {
if c.is_alphabetic() { if c.is_alphabetic() {
char_stack.push(c); char_stack.push(c);
} else { } else {
token_stack.push(Token::Alpha(char_stack.iter().collect())); token_stack.push(char_stack.iter().collect());
char_stack.clear(); char_stack.clear();
let character = if cast_period { '.' } else { c }; let character = if cast_period { '.' } else { c };
token_stack.push(Token::Separator(character.to_string())); token_stack.push(character.to_string());
state = ParseState::Empty; state = ParseState::Empty;
} }
} }
@ -215,10 +208,10 @@ fn decimal_split(characters: &str, cast_period: bool) -> Vec<Token> {
if c.is_numeric() { if c.is_numeric() {
char_stack.push(c); char_stack.push(c);
} else { } else {
token_stack.push(Token::Numeric(char_stack.iter().collect())); token_stack.push(char_stack.iter().collect());
char_stack.clear(); char_stack.clear();
let character = if cast_period { '.' } else { c }; let character = if cast_period { '.' } else { c };
token_stack.push(Token::Separator(character.to_string())); token_stack.push(character.to_string());
state = ParseState::Empty; state = ParseState::Empty;
} }
} }
@ -227,8 +220,8 @@ fn decimal_split(characters: &str, cast_period: bool) -> Vec<Token> {
} }
match state { match state {
ParseState::Alpha => token_stack.push(Token::Alpha(char_stack.iter().collect())), ParseState::Alpha => token_stack.push(char_stack.iter().collect()),
ParseState::Numeric => token_stack.push(Token::Numeric(char_stack.iter().collect())), ParseState::Numeric => token_stack.push(char_stack.iter().collect()),
ParseState::Empty => (), ParseState::Empty => (),
_ => panic!("Invalid parse state during decimal_split()"), _ => panic!("Invalid parse state during decimal_split()"),
} }
@ -236,7 +229,7 @@ fn decimal_split(characters: &str, cast_period: bool) -> Vec<Token> {
token_stack token_stack
} }
pub fn tokenize(parse_string: &str) -> Vec<Token> { pub fn tokenize(parse_string: &str) -> Vec<String> {
let tokenizer = Tokenizer::new(parse_string.to_owned()); let tokenizer = Tokenizer::new(parse_string.to_owned());
tokenizer.collect() tokenizer.collect()
} }

37
src/tests/compat_split_string.rs
View File

@ -2,7 +2,6 @@
// This file was auto-generated using the `build_tests.py` script. // This file was auto-generated using the `build_tests.py` script.
// Please do not edit it manually. // Please do not edit it manually.
use Token;
use tokenize; use tokenize;
#[test] #[test]
@ -10,34 +9,34 @@ fn test_python_compat() {
assert_eq!( assert_eq!(
tokenize("2018.5.15"), tokenize("2018.5.15"),
vec![ vec![
Token::Numeric("2018".to_owned()), "2018".to_owned(),
Token::Separator(".".to_owned()), ".".to_owned(),
Token::Numeric("5".to_owned()), "5".to_owned(),
Token::Separator(".".to_owned()), ".".to_owned(),
Token::Numeric("15".to_owned()), "15".to_owned(),
] ]
); );
assert_eq!( assert_eq!(
tokenize("May 5, 2018"), tokenize("May 5, 2018"),
vec![ vec![
Token::Alpha("May".to_owned()), "May".to_owned(),
Token::Separator(" ".to_owned()), " ".to_owned(),
Token::Numeric("5".to_owned()), "5".to_owned(),
Token::Separator(",".to_owned()), ",".to_owned(),
Token::Separator(" ".to_owned()), " ".to_owned(),
Token::Numeric("2018".to_owned()), "2018".to_owned(),
] ]
); );
assert_eq!( assert_eq!(
tokenize("Mar. 5, 2018"), tokenize("Mar. 5, 2018"),
vec![ vec![
Token::Alpha("Mar".to_owned()), "Mar".to_owned(),
Token::Separator(".".to_owned()), ".".to_owned(),
Token::Separator(" ".to_owned()), " ".to_owned(),
Token::Numeric("5".to_owned()), "5".to_owned(),
Token::Separator(",".to_owned()), ",".to_owned(),
Token::Separator(" ".to_owned()), " ".to_owned(),
Token::Numeric("2018".to_owned()), "2018".to_owned(),
] ]
); );
} }