mirror-github-bspeice/dtparse
1
0
Fork 0
mirror of https://github.com/bspeice/dtparse synced 2024-12-22 04:18:09 -05:00
Code Issues Releases Wiki Activity

Merge pull request #8 from bspeice/tokenize_split

Browse Source 
Move tokenize code
This commit is contained in:
bspeice 2018-06-29 23:10:10 -04:00 committed by GitHub
commit f0c3d8998d
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
2 changed files with 206 additions and 204 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

207
src/lib.rs
View File

@ -27,11 +27,14 @@ use std::num::ParseIntError;
use std::str::FromStr;
use std::vec::Vec;
mod tokenize;
mod weekday;
#[cfg(test)]
mod tests;
use tokenize::ParseState;
use tokenize::Tokenizer;
use weekday::day_of_week;
use weekday::DayOfWeek;
@ -89,210 +92,6 @@ impl From<ParseInternalError> for ParseError {
type ParseResult<I> = Result<I, ParseError>;
type ParseIResult<I> = Result<I, ParseInternalError>;
pub struct Tokenizer {
token_stack: Vec<String>,
parse_string: String,
}
#[derive(Debug, PartialEq)]
enum ParseState {
Empty,
Alpha,
AlphaDecimal,
Numeric,
NumericDecimal,
}
impl Tokenizer {
fn new(parse_string: String) -> Self {
Tokenizer {
token_stack: Vec::new(),
parse_string: parse_string.chars().rev().collect(),
}
}
}
impl Iterator for Tokenizer {
type Item = String;
fn next(&mut self) -> Option<Self::Item> {
if !self.token_stack.is_empty() {
return Some(self.token_stack.pop().unwrap());
};
if self.parse_string.is_empty() {
return None;
};
let mut char_stack: Vec<char> = Vec::new();
let mut seen_letters = false;
let mut state = ParseState::Empty;
while let Some(next) = self.parse_string.pop() {
match state {
ParseState::Empty => {
if next.is_numeric() {
state = ParseState::Numeric;
char_stack.push(next);
} else if next.is_alphabetic() {
state = ParseState::Alpha;
seen_letters = true;
char_stack.push(next);
} else if next.is_whitespace() {
char_stack.push(' ');
break;
} else {
char_stack.push(next);
break;
}
}
ParseState::Alpha => {
if next.is_alphabetic() {
char_stack.push(next);
} else if next == '.' {
state = ParseState::AlphaDecimal;
char_stack.push(next);
} else {
// We don't recognize the character, so push it back
// to be handled later.
self.parse_string.push(next);
break;
}
}
ParseState::AlphaDecimal => {
if next == '.' || next.is_alphabetic() {
char_stack.push(next);
} else if next.is_numeric() && char_stack.last().unwrap().clone() == '.' {
char_stack.push(next);
state = ParseState::NumericDecimal;
} else {
self.parse_string.push(next);
break;
}
}
ParseState::Numeric => {
if next.is_numeric() {
char_stack.push(next);
} else if next == '.' || (next == ',' && char_stack.len() >= 2) {
char_stack.push(next);
state = ParseState::NumericDecimal;
} else {
// We don't recognize the character, so push it back
// to be handled later
self.parse_string.push(next);
break;
}
}
ParseState::NumericDecimal => {
if next == '.' || next.is_numeric() {
char_stack.push(next);
} else if next.is_alphabetic() && char_stack.last().unwrap().clone() == '.' {
char_stack.push(next);
state = ParseState::AlphaDecimal;
} else {
self.parse_string.push(next);
break;
}
}
}
}
// I like Python's version of this much better:
// needs_split = seen_letters or char_stack.count('.') > 1 or char_stack[-1] in '.,'
let dot_count = char_stack.iter().fold(0, |count, character| {
count + (if character == &'.' { 1 } else { 0 })
});
let needs_split = seen_letters || dot_count > 1 || char_stack.last().unwrap() == &'.'
|| char_stack.last().unwrap() == &',';
let final_string: String = char_stack.into_iter().collect();
let mut tokens = match state {
ParseState::Empty => vec![final_string],
ParseState::Alpha => vec![final_string],
ParseState::Numeric => vec![final_string],
ParseState::AlphaDecimal => {
if needs_split {
decimal_split(&final_string, false)
} else {
vec![final_string]
}
}
ParseState::NumericDecimal => {
if needs_split {
decimal_split(&final_string, dot_count == 0)
} else {
vec![final_string]
}
}
}.into_iter()
.rev()
.collect();
self.token_stack.append(&mut tokens);
// UNWRAP: Previous match guaranteed that at least one token was added
let token = self.token_stack.pop().unwrap();
if state == ParseState::NumericDecimal && !token.contains(".") {
Some(token.replace(",", "."))
} else {
Some(token)
}
}
}
fn decimal_split(characters: &str, cast_period: bool) -> Vec<String> {
let mut token_stack: Vec<String> = Vec::new();
let mut char_stack: Vec<char> = Vec::new();
let mut state = ParseState::Empty;
for c in characters.chars() {
match state {
ParseState::Empty => {
if c.is_alphabetic() {
char_stack.push(c);
state = ParseState::Alpha;
} else if c.is_numeric() {
char_stack.push(c);
state = ParseState::Numeric;
} else {
let character = if cast_period { '.' } else { c };
token_stack.push(character.to_string());
}
}
ParseState::Alpha => {
if c.is_alphabetic() {
char_stack.push(c);
} else {
token_stack.push(char_stack.iter().collect());
char_stack.clear();
let character = if cast_period { '.' } else { c };
token_stack.push(character.to_string());
state = ParseState::Empty;
}
}
ParseState::Numeric => {
if c.is_numeric() {
char_stack.push(c);
} else {
token_stack.push(char_stack.iter().collect());
char_stack.clear();
let character = if cast_period { '.' } else { c };
token_stack.push(character.to_string());
state = ParseState::Empty;
}
}
_ => panic!("Invalid parse state during decimal_split()"),
}
}
match state {
ParseState::Alpha => token_stack.push(char_stack.iter().collect()),
ParseState::Numeric => token_stack.push(char_stack.iter().collect()),
ParseState::Empty => (),
_ => panic!("Invalid parse state during decimal_split()"),
}
token_stack
}
pub fn tokenize(parse_string: &str) -> Vec<String> {
let tokenizer = Tokenizer::new(parse_string.to_owned());
tokenizer.collect()

203
src/tokenize.rs Normal file
View File

@ -0,0 +1,203 @@
pub(crate) struct Tokenizer {
token_stack: Vec<String>,
parse_string: String,
}
#[derive(Debug, PartialEq)]
pub(crate) enum ParseState {
Empty,
Alpha,
AlphaDecimal,
Numeric,
NumericDecimal,
}
impl Tokenizer {
pub(crate) fn new(parse_string: String) -> Self {
Tokenizer {
token_stack: Vec::new(),
parse_string: parse_string.chars().rev().collect(),
}
}
}
impl Iterator for Tokenizer {
type Item = String;
fn next(&mut self) -> Option<Self::Item> {
if !self.token_stack.is_empty() {
return Some(self.token_stack.pop().unwrap());
};
if self.parse_string.is_empty() {
return None;
};
let mut char_stack: Vec<char> = Vec::new();
let mut seen_letters = false;
let mut state = ParseState::Empty;
while let Some(next) = self.parse_string.pop() {
match state {
ParseState::Empty => {
if next.is_numeric() {
state = ParseState::Numeric;
char_stack.push(next);
} else if next.is_alphabetic() {
state = ParseState::Alpha;
seen_letters = true;
char_stack.push(next);
} else if next.is_whitespace() {
char_stack.push(' ');
break;
} else {
char_stack.push(next);
break;
}
}
ParseState::Alpha => {
if next.is_alphabetic() {
char_stack.push(next);
} else if next == '.' {
state = ParseState::AlphaDecimal;
char_stack.push(next);
} else {
// We don't recognize the character, so push it back
// to be handled later.
self.parse_string.push(next);
break;
}
}
ParseState::AlphaDecimal => {
if next == '.' || next.is_alphabetic() {
char_stack.push(next);
} else if next.is_numeric() && char_stack.last().unwrap().clone() == '.' {
char_stack.push(next);
state = ParseState::NumericDecimal;
} else {
self.parse_string.push(next);
break;
}
}
ParseState::Numeric => {
if next.is_numeric() {
char_stack.push(next);
} else if next == '.' || (next == ',' && char_stack.len() >= 2) {
char_stack.push(next);
state = ParseState::NumericDecimal;
} else {
// We don't recognize the character, so push it back
// to be handled later
self.parse_string.push(next);
break;
}
}
ParseState::NumericDecimal => {
if next == '.' || next.is_numeric() {
char_stack.push(next);
} else if next.is_alphabetic() && char_stack.last().unwrap().clone() == '.' {
char_stack.push(next);
state = ParseState::AlphaDecimal;
} else {
self.parse_string.push(next);
break;
}
}
}
}
// I like Python's version of this much better:
// needs_split = seen_letters or char_stack.count('.') > 1 or char_stack[-1] in '.,'
let dot_count = char_stack.iter().fold(0, |count, character| {
count + (if character == &'.' { 1 } else { 0 })
});
let needs_split = seen_letters || dot_count > 1 || char_stack.last().unwrap() == &'.'
|| char_stack.last().unwrap() == &',';
let final_string: String = char_stack.into_iter().collect();
let mut tokens = match state {
ParseState::Empty => vec![final_string],
ParseState::Alpha => vec![final_string],
ParseState::Numeric => vec![final_string],
ParseState::AlphaDecimal => {
if needs_split {
decimal_split(&final_string, false)
} else {
vec![final_string]
}
}
ParseState::NumericDecimal => {
if needs_split {
decimal_split(&final_string, dot_count == 0)
} else {
vec![final_string]
}
}
}.into_iter()
.rev()
.collect();
self.token_stack.append(&mut tokens);
// UNWRAP: Previous match guaranteed that at least one token was added
let token = self.token_stack.pop().unwrap();
if state == ParseState::NumericDecimal && !token.contains(".") {
Some(token.replace(",", "."))
} else {
Some(token)
}
}
}
fn decimal_split(characters: &str, cast_period: bool) -> Vec<String> {
let mut token_stack: Vec<String> = Vec::new();
let mut char_stack: Vec<char> = Vec::new();
let mut state = ParseState::Empty;
for c in characters.chars() {
match state {
ParseState::Empty => {
if c.is_alphabetic() {
char_stack.push(c);
state = ParseState::Alpha;
} else if c.is_numeric() {
char_stack.push(c);
state = ParseState::Numeric;
} else {
let character = if cast_period { '.' } else { c };
token_stack.push(character.to_string());
}
}
ParseState::Alpha => {
if c.is_alphabetic() {
char_stack.push(c);
} else {
token_stack.push(char_stack.iter().collect());
char_stack.clear();
let character = if cast_period { '.' } else { c };
token_stack.push(character.to_string());
state = ParseState::Empty;
}
}
ParseState::Numeric => {
if c.is_numeric() {
char_stack.push(c);
} else {
token_stack.push(char_stack.iter().collect());
char_stack.clear();
let character = if cast_period { '.' } else { c };
token_stack.push(character.to_string());
state = ParseState::Empty;
}
}
_ => panic!("Invalid parse state during decimal_split()"),
}
}
match state {
ParseState::Alpha => token_stack.push(char_stack.iter().collect()),
ParseState::Numeric => token_stack.push(char_stack.iter().collect()),
ParseState::Empty => (),
_ => panic!("Invalid parse state during decimal_split()"),
}
token_stack
}