Merge pull request #12 from bspeice/fuzzy_dates

Fuzzy dates

.gitignore

@@ -3,3 +3,4 @@
 **/*.rs.bk
 Cargo.lock
 .vscode
+*.pyc

README.md

@@ -17,8 +17,8 @@ Supported in v0.8
 theoretically would provide support, but I'd also like some helper things available (e.g. "EST" is not a named zone in `chrono-tz`).
 Explicit time zones (i.e. "00:00:00 -0300") are working as expected.
 
-3. "Fuzzy" and "Fuzzy with tokens" modes haven't been tested. The code should work, but I need to get the
+3. ~~"Fuzzy" and "Fuzzy with tokens" modes haven't been tested. The code should work, but I need to get the
-test cases added to the auto-generation suite
+test cases added to the auto-generation suite~~
 
 **Non-functional**: This library is intended to be a direct port from Python, and thus the code
 looks a lot more like Python than it does Rust. There are a ton of `TODO` comments in the code

build_pycompat.py

@@ -80,12 +80,26 @@ tests = {
         '1994-11-05T08:15:30Z', '1976-07-04T00:01:02Z',
         'Tue Apr 4 00:22:12 PDT 1995'
     ],
+    'test_fuzzy_tzinfo': [
+        'Today is 25 of September of 2003, exactly at 10:49:41 with timezone -03:00.'
+    ],
+    'test_fuzzy_tokens_tzinfo': [
+        'Today is 25 of September of 2003, exactly at 10:49:41 with timezone -03:00.'
+    ],
+    'test_fuzzy_simple': [
+        'I have a meeting on March 1, 1974', # testFuzzyAMPMProblem
+        'On June 8th, 2020, I am going to be the first man on Mars', # testFuzzyAMPMProblem
+        'Meet me at the AM/PM on Sunset at 3:00 AM on December 3rd, 2003', # testFuzzyAMPMProblem
+        'Meet me at 3:00 AM on December 3rd, 2003 at the AM/PM on Sunset', # testFuzzyAMPMProblem
+        'Jan 29, 1945 14:45 AM I going to see you there?', # testFuzzyIgnoreAMPM
+        '2017-07-17 06:15:', # test_idx_check
+    ],
     'test_parse_default_ignore': [
     ],
 }
 
 def main():
-    with open('tests/pycompat.rs', 'w+') as handle:
+    with open('src/tests/pycompat_parser.rs', 'w+') as handle:
         handle.write(TEST_HEADER)
 
         for test_name, test_strings in tests.items():
@@ -149,6 +163,30 @@ def test_parse_default_ignore(i, s):
 
     return TEST_PARSE_DEFAULT_IGNORE.format(i=i, d=d, s=s)
 
+
+def test_fuzzy_tzinfo(i, s):
+    d = parse(s, fuzzy=True)
+
+    return TEST_FUZZY_TZINFO.format(i=i, d=d, s=s, offset=int(d.tzinfo._offset.total_seconds()))
+
+
+def test_fuzzy_tokens_tzinfo(i, s):
+    d, tokens = parse(s, fuzzy_with_tokens=True)
+
+    r_tokens = ", ".join(list(map(lambda s: f'"{s}".to_owned()', tokens)))
+
+    return TEST_FUZZY_TOKENS_TZINFO.format(
+        i=i, d=d, s=s, offset=int(d.tzinfo._offset.total_seconds()),
+        tokens=r_tokens
+    )
+
+
+def test_fuzzy_simple(i, s):
+    d = parse(s, fuzzy=True)
+
+    return TEST_FUZZY_SIMPLE.format(i=i, d=d, s=s)
+
+
 # Here lies all the ugly junk.
 TEST_HEADER = '''
 extern crate chrono;
@@ -159,10 +197,9 @@ use chrono::NaiveDateTime;
 use chrono::Timelike;
 use std::collections::HashMap;
 
-extern crate dtparse;
+use Parser;
-
+use ParserInfo;
-use dtparse::Parser;
+use parse;
-use dtparse::ParserInfo;
 
 struct PyDateTime {
     year: i32,
@@ -205,23 +242,59 @@ fn parse_and_assert(
     assert_eq!(pdt.hour, rs_parsed.0.hour(), "Hour mismatch for '{}'", s);
     assert_eq!(pdt.minute, rs_parsed.0.minute(), "Minute mismatch f'or' {}", s);
     assert_eq!(pdt.second, rs_parsed.0.second(), "Second mismatch for '{}'", s);
-    assert_eq!(pdt.micros, rs_parsed.0.timestamp_subsec_micros(), "Microsecond mismatch for {}", s);
+    assert_eq!(pdt.micros, rs_parsed.0.timestamp_subsec_micros(), "Microsecond mismatch for '{}'", s);
-    assert_eq!(pdt.tzo, rs_parsed.1.map(|u| u.local_minus_utc()), "Timezone Offset mismatch for {}", s);
+    assert_eq!(pdt.tzo, rs_parsed.1.map(|u| u.local_minus_utc()), "Timezone Offset mismatch for '{}'", s);
 }
 
 fn parse_and_assert_simple(
     pdt: PyDateTime,
     s: &str,
 ) {
-    let rs_parsed = dtparse::parse(s).expect(&format!("Unable to parse date in Rust '{}'", s));
+    let rs_parsed = parse(s).expect(&format!("Unable to parse date in Rust '{}'", s));
-    assert_eq!(pdt.year, rs_parsed.0.year(), "Year mismatch for {}", s);
+    assert_eq!(pdt.year, rs_parsed.0.year(), "Year mismatch for '{}'", s);
-    assert_eq!(pdt.month, rs_parsed.0.month(), "Month mismatch for {}", s);
+    assert_eq!(pdt.month, rs_parsed.0.month(), "Month mismatch for '{}'", s);
-    assert_eq!(pdt.day, rs_parsed.0.day(), "Day mismatch for {}", s);
+    assert_eq!(pdt.day, rs_parsed.0.day(), "Day mismatch for '{}'", s);
-    assert_eq!(pdt.hour, rs_parsed.0.hour(), "Hour mismatch for {}", s);
+    assert_eq!(pdt.hour, rs_parsed.0.hour(), "Hour mismatch for '{}'", s);
-    assert_eq!(pdt.minute, rs_parsed.0.minute(), "Minute mismatch for {}", s);
+    assert_eq!(pdt.minute, rs_parsed.0.minute(), "Minute mismatch for '{}'", s);
-    assert_eq!(pdt.second, rs_parsed.0.second(), "Second mismatch for {}", s);
+    assert_eq!(pdt.second, rs_parsed.0.second(), "Second mismatch for '{}'", s);
-    assert_eq!(pdt.micros, rs_parsed.0.timestamp_subsec_micros(), "Microsecond mismatch for {}", s);
+    assert_eq!(pdt.micros, rs_parsed.0.timestamp_subsec_micros(), "Microsecond mismatch for '{}'", s);
-    assert_eq!(pdt.tzo, rs_parsed.1.map(|u| u.local_minus_utc()), "Timezone Offset mismatch for {}", s);
+    assert_eq!(pdt.tzo, rs_parsed.1.map(|u| u.local_minus_utc()), "Timezone Offset mismatch for '{}'", s);
+}
+
+fn parse_fuzzy_and_assert(
+    pdt: PyDateTime,
+    ptokens: Option<Vec<String>>,
+    info: ParserInfo,
+    s: &str,
+    dayfirst: Option<bool>,
+    yearfirst: Option<bool>,
+    fuzzy: bool,
+    fuzzy_with_tokens: bool,
+    default: Option<&NaiveDateTime>,
+    ignoretz: bool,
+    tzinfos: HashMap<String, i32>,
+) {
+
+    let mut parser = Parser::new(info);
+    let rs_parsed = parser.parse(
+        s,
+        dayfirst,
+        yearfirst,
+        fuzzy,
+        fuzzy_with_tokens,
+        default,
+        ignoretz,
+        tzinfos).expect(&format!("Unable to parse date in Rust '{}'", s));
+
+    assert_eq!(pdt.year, rs_parsed.0.year(), "Year mismatch for '{}'", s);
+    assert_eq!(pdt.month, rs_parsed.0.month(), "Month mismatch for '{}'", s);
+    assert_eq!(pdt.day, rs_parsed.0.day(), "Day mismatch for '{}'", s);
+    assert_eq!(pdt.hour, rs_parsed.0.hour(), "Hour mismatch for '{}'", s);
+    assert_eq!(pdt.minute, rs_parsed.0.minute(), "Minute mismatch f'or' {}", s);
+    assert_eq!(pdt.second, rs_parsed.0.second(), "Second mismatch for '{}'", s);
+    assert_eq!(pdt.micros, rs_parsed.0.timestamp_subsec_micros(), "Microsecond mismatch for '{}'", s);
+    assert_eq!(pdt.tzo, rs_parsed.1.map(|u| u.local_minus_utc()), "Timezone Offset mismatch for '{}'", s);
+    assert_eq!(ptokens, rs_parsed.2, "Tokens mismatch for '{}'", s);
 }
 
 macro_rules! rs_tzinfo_map {
@@ -364,6 +437,46 @@ fn test_parse_default_ignore{i}() {{
                      Some(default_rsdate), false, HashMap::new());
 }}\n'''
 
+TEST_FUZZY_TZINFO = '''
+#[test]
+fn test_fuzzy_tzinfo{i}() {{
+    let info = ParserInfo::default();
+    let pdt = PyDateTime {{
+        year: {d.year}, month: {d.month}, day: {d.day},
+        hour: {d.hour}, minute: {d.minute}, second: {d.second},
+        micros: {d.microsecond}, tzo: Some({offset})
+    }};
+    parse_fuzzy_and_assert(pdt, None, info, "{s}", None, None, true, false,
+                           None, false, HashMap::new());
+}}\n'''
+
+TEST_FUZZY_TOKENS_TZINFO = '''
+#[test]
+fn test_fuzzy_tokens_tzinfo{i}() {{
+    let info = ParserInfo::default();
+    let pdt = PyDateTime {{
+        year: {d.year}, month: {d.month}, day: {d.day},
+        hour: {d.hour}, minute: {d.minute}, second: {d.second},
+        micros: {d.microsecond}, tzo: Some({offset})
+    }};
+    let tokens = vec![{tokens}];
+    parse_fuzzy_and_assert(pdt, Some(tokens), info, "{s}", None, None, true, true,
+                           None, false, HashMap::new());
+}}\n'''
+
+TEST_FUZZY_SIMPLE = '''
+#[test]
+fn test_fuzzy_simple{i}() {{
+    let info = ParserInfo::default();
+    let pdt = PyDateTime {{
+        year: {d.year}, month: {d.month}, day: {d.day},
+        hour: {d.hour}, minute: {d.minute}, second: {d.second},
+        micros: {d.microsecond}, tzo: None
+    }};
+    parse_fuzzy_and_assert(pdt, None, info, "{s}", None, None, true, false,
+                           None, false, HashMap::new());
+}}\n'''
+
 
 if __name__ == '__main__':
     main()

build_pycompat_tokenizer.py

@@ -0,0 +1,35 @@
+from dateutil.parser import _timelex
+
+from build_pycompat import tests
+
+def main():
+    with open('src/tests/pycompat_tokenizer.rs', 'w+') as handle:
+        handle.write(TEST_HEADER)
+
+        counter = 0
+        for _, test_strings in tests.items():
+            for s in test_strings:
+                handle.write(build_test(counter, s))
+                counter += 1
+
+def build_test(i, test_string):
+    python_tokens = list(_timelex(test_string))
+    formatted_tokens = 'vec!["' + '", "'.join(python_tokens) + '"]'
+    return f'''
+#[test]
+fn test_tokenize{i}() {{
+    let comp = {formatted_tokens};
+    tokenize_assert("{test_string}", comp);
+}}\n'''
+
+
+TEST_HEADER = '''
+use tokenize::Tokenizer;
+
+fn tokenize_assert(test_str: &str, comparison: Vec<&str>) {
+    let tokens: Vec<String> = Tokenizer::new(test_str).collect();
+    assert_eq!(tokens, comparison, "Tokenizing mismatch for `{}`", test_str);
+}\n'''
+
+if __name__ == '__main__':
+    main()

src/lib.rs

@@ -1,6 +1,3 @@
-#![allow(dead_code)]
-#![allow(unused)]
-
 #[macro_use]
 extern crate lazy_static;
 
@@ -8,7 +5,6 @@ extern crate chrono;
 extern crate num_traits;
 extern crate rust_decimal;
 
-use chrono::DateTime;
 use chrono::Datelike;
 use chrono::Duration;
 use chrono::FixedOffset;
@@ -17,7 +13,6 @@ use chrono::NaiveDate;
 use chrono::NaiveDateTime;
 use chrono::NaiveTime;
 use chrono::Timelike;
-use chrono::Utc;
 use num_traits::cast::ToPrimitive;
 use rust_decimal::Decimal;
 use rust_decimal::Error as DecimalError;
@@ -33,7 +28,6 @@ mod weekday;
 #[cfg(test)]
 mod tests;
 
-use tokenize::ParseState;
 use tokenize::Tokenizer;
 use weekday::day_of_week;
 use weekday::DayOfWeek;
@@ -59,13 +53,13 @@ pub enum ParseInternalError {
 }
 
 impl From<DecimalError> for ParseInternalError {
-    fn from(err: DecimalError) -> Self {
+    fn from(_err: DecimalError) -> Self {
         ParseInternalError::InvalidDecimal
     }
 }
 
 impl From<ParseIntError> for ParseInternalError {
-    fn from(err: ParseIntError) -> Self {
+    fn from(_err: ParseIntError) -> Self {
         ParseInternalError::InvalidInteger
     }
 }
@@ -92,7 +86,7 @@ type ParseResult<I> = Result<I, ParseError>;
 type ParseIResult<I> = Result<I, ParseInternalError>;
 
 pub fn tokenize(parse_string: &str) -> Vec<String> {
-    let tokenizer = Tokenizer::new(parse_string.to_owned());
+    let tokenizer = Tokenizer::new(parse_string);
     tokenizer.collect()
 }
 
@@ -201,7 +195,9 @@ impl ParserInfo {
 
     fn get_ampm(&self, name: &str) -> Option<bool> {
         if let Some(v) = self.ampm.get(&name.to_lowercase()) {
-            Some(v.to_owned() == 1)
+            // Python technically uses numbers here, but given that the numbers are
+            // only 0 and 1, it's easier to use booleans
+            Some(*v == 1)
         } else {
             None
         }
@@ -293,11 +289,6 @@ struct YMD {
     ystridx: Option<usize>,
 }
 
-enum YMDAppendEither {
-    Number(i32),
-    Stringy(String),
-}
-
 impl YMD {
     fn len(&self) -> usize {
         self._ymd.len()
@@ -387,7 +378,7 @@ impl YMD {
                     Ok(())
                 }
             }
-            None => Err(ParseInternalError::ValueError("Missing label.".to_owned())),
+            None => Ok(()),
         }
     }
 
@@ -565,7 +556,6 @@ impl Parser {
 
         let default_ts = NaiveDateTime::new(default_date, NaiveTime::from_hms(0, 0, 0));
 
-        // TODO: What should be done with the tokens?
         let (res, tokens) =
             self.parse_with_tokens(timestr, dayfirst, yearfirst, fuzzy, fuzzy_with_tokens)?;
 
@@ -613,34 +603,34 @@ impl Parser {
         while i < len_l {
             let value_repr = l[i].clone();
 
-            if let Ok(v) = Decimal::from_str(&value_repr) {
+            if let Ok(_v) = Decimal::from_str(&value_repr) {
                 i = self.parse_numeric_token(&l, i, &self.info, &mut ymd, &mut res, fuzzy)?;
             } else if let Some(value) = self.info.get_weekday(&l[i]) {
                 res.weekday = Some(value);
             } else if let Some(value) = self.info.get_month(&l[i]) {
-                ymd.append(value as i32, &l[i], Some(YMDLabel::Month));
+                ymd.append(value as i32, &l[i], Some(YMDLabel::Month))?;
 
                 if i + 1 < len_l {
                     if l[i + 1] == "-" || l[i + 1] == "/" {
                         // Jan-01[-99]
                         let sep = &l[i + 1];
                         // TODO: This seems like a very unsafe unwrap
-                        ymd.append(l[i + 2].parse::<i32>().unwrap(), &l[i + 2], None);
+                        ymd.append(l[i + 2].parse::<i32>().unwrap(), &l[i + 2], None)?;
 
                         if i + 3 < len_l && &l[i + 3] == sep {
                             // Jan-01-99
-                            ymd.append(l[i + 4].parse::<i32>().unwrap(), &l[i + 4], None);
+                            ymd.append(l[i + 4].parse::<i32>().unwrap(), &l[i + 4], None)?;
                             i += 2;
                         }
 
                         i += 2;
-                    } else if (i + 4 < len_l && l[i + 1] == l[i + 3] && l[i + 3] == " "
+                    } else if i + 4 < len_l && l[i + 1] == l[i + 3] && l[i + 3] == " "
-                        && self.info.get_pertain(&l[i + 2]))
+                        && self.info.get_pertain(&l[i + 2])
                     {
                         // Jan of 01
                         if let Some(value) = l[i + 4].parse::<i32>().ok() {
                             let year = self.info.convertyear(value, false);
-                            ymd.append(year, &l[i + 4], Some(YMDLabel::Year));
+                            ymd.append(year, &l[i + 4], Some(YMDLabel::Year))?;
                         }
 
                         i += 4;
@@ -649,8 +639,8 @@ impl Parser {
             } else if let Some(value) = self.info.get_ampm(&l[i]) {
                 let is_ampm = self.ampm_valid(res.hour, res.ampm, fuzzy);
 
-                if is_ampm.is_ok() {
+                if is_ampm == Ok(true) {
-                    res.hour = Some(self.adjust_ampm(res.hour.unwrap(), value));
+                    res.hour = res.hour.map(|h| self.adjust_ampm(h, value));
                     res.ampm = Some(value);
                 } else if fuzzy {
                     skipped_idxs.push(i);
@@ -718,7 +708,7 @@ impl Parser {
                 }
 
                 i += 1;
-            } else if !self.info.get_jump(&l[i]) || fuzzy {
+            } else if !(self.info.get_jump(&l[i]) || fuzzy) {
                 return Err(ParseError::UnrecognizedToken(l[i].clone()));
             } else {
                 skipped_idxs.push(i);
@@ -737,7 +727,7 @@ impl Parser {
         if !self.info.validate(&mut res) {
             Err(ParseError::InvalidParseResult(res))
         } else if fuzzy_with_tokens {
-            let skipped_tokens = skipped_idxs.into_iter().map(|i| l[i].clone()).collect();
+            let skipped_tokens = self.recombine_skipped(skipped_idxs, l);
             Ok((res, Some(skipped_tokens)))
         } else {
             Ok((res, None))
@@ -759,25 +749,27 @@ impl Parser {
     }
 
     fn ampm_valid(&self, hour: Option<i32>, ampm: Option<bool>, fuzzy: bool) -> ParseResult<bool> {
-        if fuzzy && ampm == Some(true) {
+        let mut val_is_ampm = true;
-            return Ok(false);
+
+        if fuzzy && ampm.is_some() {
+            val_is_ampm = false;
         }
 
         if hour.is_none() {
             if fuzzy {
-                Ok(false)
+                val_is_ampm = false;
             } else {
-                Err(ParseError::AmPmWithoutHour)
+                return Err(ParseError::AmPmWithoutHour);
             }
         } else if !(0 <= hour.unwrap() && hour.unwrap() <= 12) {
             if fuzzy {
-                Ok(false)
+                val_is_ampm = false;
             } else {
-                Err(ParseError::ImpossibleTimestamp("Invalid hour"))
+                return Err(ParseError::ImpossibleTimestamp("Invalid hour"));
             }
-        } else {
-            Ok(false)
         }
+
+        Ok(val_is_ampm)
     }
 
     fn build_naive(&self, res: &ParsingResult, default: &NaiveDateTime) -> ParseResult<NaiveDateTime> {
@@ -797,7 +789,7 @@ impl Parser {
         };
 
         // TODO: Change month/day to u32
-        let mut d = NaiveDate::from_ymd(
+        let d = NaiveDate::from_ymd(
             y,
             m,
             min(res.day.unwrap_or(default.day() as i32) as u32, days_in_month(y, m as i32)?)
@@ -829,7 +821,7 @@ impl Parser {
 
     fn build_tzaware(
         &self,
-        dt: &NaiveDateTime,
+        _dt: &NaiveDateTime,
         res: &ParsingResult,
         tzinfos: HashMap<String, i32>,
     ) -> ParseResult<Option<FixedOffset>> {
@@ -888,9 +880,9 @@ impl Parser {
             let s = &tokens[idx];
 
             if ymd.len() == 0 && tokens[idx].find(".") == None {
-                ymd.append(s[0..2].parse::<i32>().unwrap(), &s[0..2], None);
+                ymd.append(s[0..2].parse::<i32>().unwrap(), &s[0..2], None)?;
-                ymd.append(s[2..4].parse::<i32>().unwrap(), &s[2..4], None);
+                ymd.append(s[2..4].parse::<i32>().unwrap(), &s[2..4], None)?;
-                ymd.append(s[4..6].parse::<i32>().unwrap(), &s[4..6], None);
+                ymd.append(s[4..6].parse::<i32>().unwrap(), &s[4..6], None)?;
             } else {
                 // 19990101T235959[.59]
                 res.hour = s[0..2].parse::<i32>().ok();
@@ -903,9 +895,9 @@ impl Parser {
         } else if vec![8, 12, 14].contains(&len_li) {
             // YYMMDD
             let s = &tokens[idx];
-            ymd.append(s[..4].parse::<i32>().unwrap(), &s[..4], Some(YMDLabel::Year));
+            ymd.append(s[..4].parse::<i32>().unwrap(), &s[..4], Some(YMDLabel::Year))?;
-            ymd.append(s[4..6].parse::<i32>().unwrap(), &s[4..6], None);
+            ymd.append(s[4..6].parse::<i32>().unwrap(), &s[4..6], None)?;
-            ymd.append(s[6..8].parse::<i32>().unwrap(), &s[6..8], None);
+            ymd.append(s[6..8].parse::<i32>().unwrap(), &s[6..8], None)?;
 
             if len_li > 8 {
                 res.hour = Some(s[8..10].parse::<i32>()?);
@@ -947,20 +939,20 @@ impl Parser {
         {
             // TODO: There's got to be a better way of handling the condition above
             let sep = &tokens[idx + 1];
-            ymd.append(value_repr.parse::<i32>().unwrap(), &value_repr, None);
+            ymd.append(value_repr.parse::<i32>().unwrap(), &value_repr, None)?;
 
             if idx + 2 < len_l && !info.get_jump(&tokens[idx + 2]) {
                 if let Ok(val) = tokens[idx + 2].parse::<i32>() {
-                    ymd.append(val, &tokens[idx + 2], None);
+                    ymd.append(val, &tokens[idx + 2], None)?;
                 } else if let Some(val) = info.get_month(&tokens[idx + 2]) {
-                    ymd.append(val as i32, &tokens[idx + 2], Some(YMDLabel::Month));
+                    ymd.append(val as i32, &tokens[idx + 2], Some(YMDLabel::Month))?;
                 }
 
                 if idx + 3 < len_l && &tokens[idx + 3] == sep {
                     if let Some(value) = info.get_month(&tokens[idx + 4]) {
-                        ymd.append(value as i32, &tokens[idx + 4], Some(YMDLabel::Month));
+                        ymd.append(value as i32, &tokens[idx + 4], Some(YMDLabel::Month))?;
                     } else {
-                        ymd.append(tokens[idx + 4].parse::<i32>().unwrap(), &tokens[idx + 4], None);
+                        ymd.append(tokens[idx + 4].parse::<i32>().unwrap(), &tokens[idx + 4], None)?;
                     }
 
                     idx += 2;
@@ -975,9 +967,12 @@ impl Parser {
                 let hour = value.to_i64().unwrap() as i32;
                 let ampm = info.get_ampm(&tokens[idx + 2]).unwrap();
                 res.hour = Some(self.adjust_ampm(hour, ampm));
+                idx += 1;
             } else {
-                ymd.append(value.floor().to_i64().unwrap() as i32, &value_repr, None);
+                ymd.append(value.floor().to_i64().unwrap() as i32, &value_repr, None)?;
             }
+
+            idx += 1;
         } else if info.get_ampm(&tokens[idx + 1]).is_some()
             && (*ZERO <= value && value < *TWENTY_FOUR)
         {
@@ -986,7 +981,7 @@ impl Parser {
             res.hour = Some(self.adjust_ampm(hour, info.get_ampm(&tokens[idx + 1]).unwrap()));
             idx += 1;
         } else if ymd.could_be_day(value.to_i64().unwrap() as i32) {
-            ymd.append(value.to_i64().unwrap() as i32, &value_repr, None);
+            ymd.append(value.to_i64().unwrap() as i32, &value_repr, None)?;
         } else if !fuzzy {
             return Err(ParseInternalError::ValueError("".to_owned()));
         }
@@ -1087,7 +1082,6 @@ impl Parser {
         if hms == 0 {
             res.hour = Some(value.to_i64().unwrap() as i32);
             if !close_to_integer(&value) {
-                // TODO: High probability of issues with rounding here.
                 res.minute = Some((*SIXTY * (value % *ONE)).to_i64().unwrap() as i32);
             }
         } else if hms == 1 {
@@ -1107,6 +1101,7 @@ impl Parser {
     }
 
     fn parse_min_sec(&self, value: Decimal) -> (i32, Option<i32>) {
+        // UNWRAP: i64 guaranteed to be fine because of preceding floor
         let minute = value.floor().to_i64().unwrap() as i32;
         let mut second = None;
 
@@ -1117,6 +1112,26 @@ impl Parser {
 
         (minute, second)
     }
+
+    fn recombine_skipped(&self, skipped_idxs: Vec<usize>, tokens: Vec<String>) -> Vec<String> {
+        let mut skipped_tokens: Vec<String> = vec![];
+
+        let mut sorted_idxs = skipped_idxs.clone();
+        sorted_idxs.sort();
+
+        for (i, idx) in sorted_idxs.iter().enumerate() {
+            if i > 0 && idx - 1 == skipped_idxs[i - 1] {
+                // UNWRAP: Having an initial value and unconditional push at end guarantees value
+                let mut t = skipped_tokens.pop().unwrap();
+                t.push_str(tokens[idx.clone()].as_ref());
+                skipped_tokens.push(t);
+            } else {
+                skipped_tokens.push(tokens[idx.clone()].to_owned());
+            }
+        }
+
+        skipped_tokens
+    }
 }
 
 fn close_to_integer(value: &Decimal) -> bool {

src/tests/mod.rs

@@ -0,0 +1,3 @@
+mod fuzzing;
+mod pycompat_parser;
+mod pycompat_tokenizer;

src/tests/pycompat_parser.rs

@@ -7,10 +7,9 @@ use chrono::NaiveDateTime;
 use chrono::Timelike;
 use std::collections::HashMap;
 
-extern crate dtparse;
+use Parser;
-
+use ParserInfo;
-use dtparse::Parser;
+use parse;
-use dtparse::ParserInfo;
 
 struct PyDateTime {
     year: i32,
@@ -53,23 +52,59 @@ fn parse_and_assert(
     assert_eq!(pdt.hour, rs_parsed.0.hour(), "Hour mismatch for '{}'", s);
     assert_eq!(pdt.minute, rs_parsed.0.minute(), "Minute mismatch f'or' {}", s);
     assert_eq!(pdt.second, rs_parsed.0.second(), "Second mismatch for '{}'", s);
-    assert_eq!(pdt.micros, rs_parsed.0.timestamp_subsec_micros(), "Microsecond mismatch for {}", s);
+    assert_eq!(pdt.micros, rs_parsed.0.timestamp_subsec_micros(), "Microsecond mismatch for '{}'", s);
-    assert_eq!(pdt.tzo, rs_parsed.1.map(|u| u.local_minus_utc()), "Timezone Offset mismatch for {}", s);
+    assert_eq!(pdt.tzo, rs_parsed.1.map(|u| u.local_minus_utc()), "Timezone Offset mismatch for '{}'", s);
 }
 
 fn parse_and_assert_simple(
     pdt: PyDateTime,
     s: &str,
 ) {
-    let rs_parsed = dtparse::parse(s).expect(&format!("Unable to parse date in Rust '{}'", s));
+    let rs_parsed = parse(s).expect(&format!("Unable to parse date in Rust '{}'", s));
-    assert_eq!(pdt.year, rs_parsed.0.year(), "Year mismatch for {}", s);
+    assert_eq!(pdt.year, rs_parsed.0.year(), "Year mismatch for '{}'", s);
-    assert_eq!(pdt.month, rs_parsed.0.month(), "Month mismatch for {}", s);
+    assert_eq!(pdt.month, rs_parsed.0.month(), "Month mismatch for '{}'", s);
-    assert_eq!(pdt.day, rs_parsed.0.day(), "Day mismatch for {}", s);
+    assert_eq!(pdt.day, rs_parsed.0.day(), "Day mismatch for '{}'", s);
-    assert_eq!(pdt.hour, rs_parsed.0.hour(), "Hour mismatch for {}", s);
+    assert_eq!(pdt.hour, rs_parsed.0.hour(), "Hour mismatch for '{}'", s);
-    assert_eq!(pdt.minute, rs_parsed.0.minute(), "Minute mismatch for {}", s);
+    assert_eq!(pdt.minute, rs_parsed.0.minute(), "Minute mismatch for '{}'", s);
-    assert_eq!(pdt.second, rs_parsed.0.second(), "Second mismatch for {}", s);
+    assert_eq!(pdt.second, rs_parsed.0.second(), "Second mismatch for '{}'", s);
-    assert_eq!(pdt.micros, rs_parsed.0.timestamp_subsec_micros(), "Microsecond mismatch for {}", s);
+    assert_eq!(pdt.micros, rs_parsed.0.timestamp_subsec_micros(), "Microsecond mismatch for '{}'", s);
-    assert_eq!(pdt.tzo, rs_parsed.1.map(|u| u.local_minus_utc()), "Timezone Offset mismatch for {}", s);
+    assert_eq!(pdt.tzo, rs_parsed.1.map(|u| u.local_minus_utc()), "Timezone Offset mismatch for '{}'", s);
+}
+
+fn parse_fuzzy_and_assert(
+    pdt: PyDateTime,
+    ptokens: Option<Vec<String>>,
+    info: ParserInfo,
+    s: &str,
+    dayfirst: Option<bool>,
+    yearfirst: Option<bool>,
+    fuzzy: bool,
+    fuzzy_with_tokens: bool,
+    default: Option<&NaiveDateTime>,
+    ignoretz: bool,
+    tzinfos: HashMap<String, i32>,
+) {
+
+    let mut parser = Parser::new(info);
+    let rs_parsed = parser.parse(
+        s,
+        dayfirst,
+        yearfirst,
+        fuzzy,
+        fuzzy_with_tokens,
+        default,
+        ignoretz,
+        tzinfos).expect(&format!("Unable to parse date in Rust '{}'", s));
+
+    assert_eq!(pdt.year, rs_parsed.0.year(), "Year mismatch for '{}'", s);
+    assert_eq!(pdt.month, rs_parsed.0.month(), "Month mismatch for '{}'", s);
+    assert_eq!(pdt.day, rs_parsed.0.day(), "Day mismatch for '{}'", s);
+    assert_eq!(pdt.hour, rs_parsed.0.hour(), "Hour mismatch for '{}'", s);
+    assert_eq!(pdt.minute, rs_parsed.0.minute(), "Minute mismatch f'or' {}", s);
+    assert_eq!(pdt.second, rs_parsed.0.second(), "Second mismatch for '{}'", s);
+    assert_eq!(pdt.micros, rs_parsed.0.timestamp_subsec_micros(), "Microsecond mismatch for '{}'", s);
+    assert_eq!(pdt.tzo, rs_parsed.1.map(|u| u.local_minus_utc()), "Timezone Offset mismatch for '{}'", s);
+    assert_eq!(ptokens, rs_parsed.2, "Tokens mismatch for '{}'", s);
 }
 
 macro_rules! rs_tzinfo_map {
@@ -1697,3 +1732,100 @@ fn test_parse_ignoretz7() {
     parse_and_assert(pdt, info, "Tue Apr 4 00:22:12 PDT 1995", None, None, false, false,
                      None, true, HashMap::new());
 }
+
+#[test]
+fn test_fuzzy_tzinfo0() {
+    let info = ParserInfo::default();
+    let pdt = PyDateTime {
+        year: 2003, month: 9, day: 25,
+        hour: 10, minute: 49, second: 41,
+        micros: 0, tzo: Some(-10800)
+    };
+    parse_fuzzy_and_assert(pdt, None, info, "Today is 25 of September of 2003, exactly at 10:49:41 with timezone -03:00.", None, None, true, false,
+                           None, false, HashMap::new());
+}
+
+#[test]
+fn test_fuzzy_tokens_tzinfo0() {
+    let info = ParserInfo::default();
+    let pdt = PyDateTime {
+        year: 2003, month: 9, day: 25,
+        hour: 10, minute: 49, second: 41,
+        micros: 0, tzo: Some(-10800)
+    };
+    let tokens = vec!["Today is ".to_owned(), "of ".to_owned(), ", exactly at ".to_owned(), " with timezone ".to_owned(), ".".to_owned()];
+    parse_fuzzy_and_assert(pdt, Some(tokens), info, "Today is 25 of September of 2003, exactly at 10:49:41 with timezone -03:00.", None, None, true, true,
+                           None, false, HashMap::new());
+}
+
+#[test]
+fn test_fuzzy_simple0() {
+    let info = ParserInfo::default();
+    let pdt = PyDateTime {
+        year: 1974, month: 3, day: 1,
+        hour: 0, minute: 0, second: 0,
+        micros: 0, tzo: None
+    };
+    parse_fuzzy_and_assert(pdt, None, info, "I have a meeting on March 1, 1974", None, None, true, false,
+                           None, false, HashMap::new());
+}
+
+#[test]
+fn test_fuzzy_simple1() {
+    let info = ParserInfo::default();
+    let pdt = PyDateTime {
+        year: 2020, month: 6, day: 8,
+        hour: 0, minute: 0, second: 0,
+        micros: 0, tzo: None
+    };
+    parse_fuzzy_and_assert(pdt, None, info, "On June 8th, 2020, I am going to be the first man on Mars", None, None, true, false,
+                           None, false, HashMap::new());
+}
+
+#[test]
+fn test_fuzzy_simple2() {
+    let info = ParserInfo::default();
+    let pdt = PyDateTime {
+        year: 2003, month: 12, day: 3,
+        hour: 3, minute: 0, second: 0,
+        micros: 0, tzo: None
+    };
+    parse_fuzzy_and_assert(pdt, None, info, "Meet me at the AM/PM on Sunset at 3:00 AM on December 3rd, 2003", None, None, true, false,
+                           None, false, HashMap::new());
+}
+
+#[test]
+fn test_fuzzy_simple3() {
+    let info = ParserInfo::default();
+    let pdt = PyDateTime {
+        year: 2003, month: 12, day: 3,
+        hour: 3, minute: 0, second: 0,
+        micros: 0, tzo: None
+    };
+    parse_fuzzy_and_assert(pdt, None, info, "Meet me at 3:00 AM on December 3rd, 2003 at the AM/PM on Sunset", None, None, true, false,
+                           None, false, HashMap::new());
+}
+
+#[test]
+fn test_fuzzy_simple4() {
+    let info = ParserInfo::default();
+    let pdt = PyDateTime {
+        year: 1945, month: 1, day: 29,
+        hour: 14, minute: 45, second: 0,
+        micros: 0, tzo: None
+    };
+    parse_fuzzy_and_assert(pdt, None, info, "Jan 29, 1945 14:45 AM I going to see you there?", None, None, true, false,
+                           None, false, HashMap::new());
+}
+
+#[test]
+fn test_fuzzy_simple5() {
+    let info = ParserInfo::default();
+    let pdt = PyDateTime {
+        year: 2017, month: 7, day: 17,
+        hour: 6, minute: 15, second: 0,
+        micros: 0, tzo: None
+    };
+    parse_fuzzy_and_assert(pdt, None, info, "2017-07-17 06:15:", None, None, true, false,
+                           None, false, HashMap::new());
+}

src/tests/pycompat_tokenizer.rs

@@ -0,0 +1,901 @@
+
+use tokenize::Tokenizer;
+
+fn tokenize_assert(test_str: &str, comparison: Vec<&str>) {
+    let tokens: Vec<String> = Tokenizer::new(test_str).collect();
+    assert_eq!(tokens, comparison, "Tokenizing mismatch for `{}`", test_str);
+}
+
+#[test]
+fn test_tokenize0() {
+    let comp = vec!["Thu", " ", "Sep", " ", "25", " ", "10", ":", "36", ":", "28"];
+    tokenize_assert("Thu Sep 25 10:36:28", comp);
+}
+
+#[test]
+fn test_tokenize1() {
+    let comp = vec!["Sep", " ", "10", ":", "36", ":", "28"];
+    tokenize_assert("Sep 10:36:28", comp);
+}
+
+#[test]
+fn test_tokenize2() {
+    let comp = vec!["10", ":", "36", ":", "28"];
+    tokenize_assert("10:36:28", comp);
+}
+
+#[test]
+fn test_tokenize3() {
+    let comp = vec!["10", ":", "36"];
+    tokenize_assert("10:36", comp);
+}
+
+#[test]
+fn test_tokenize4() {
+    let comp = vec!["Sep", " ", "2003"];
+    tokenize_assert("Sep 2003", comp);
+}
+
+#[test]
+fn test_tokenize5() {
+    let comp = vec!["Sep"];
+    tokenize_assert("Sep", comp);
+}
+
+#[test]
+fn test_tokenize6() {
+    let comp = vec!["2003"];
+    tokenize_assert("2003", comp);
+}
+
+#[test]
+fn test_tokenize7() {
+    let comp = vec!["10", "h", "36", "m", "28.5", "s"];
+    tokenize_assert("10h36m28.5s", comp);
+}
+
+#[test]
+fn test_tokenize8() {
+    let comp = vec!["10", "h", "36", "m", "28", "s"];
+    tokenize_assert("10h36m28s", comp);
+}
+
+#[test]
+fn test_tokenize9() {
+    let comp = vec!["10", "h", "36", "m"];
+    tokenize_assert("10h36m", comp);
+}
+
+#[test]
+fn test_tokenize10() {
+    let comp = vec!["10", "h"];
+    tokenize_assert("10h", comp);
+}
+
+#[test]
+fn test_tokenize11() {
+    let comp = vec!["10", " ", "h", " ", "36"];
+    tokenize_assert("10 h 36", comp);
+}
+
+#[test]
+fn test_tokenize12() {
+    let comp = vec!["10", " ", "h", " ", "36.5"];
+    tokenize_assert("10 h 36.5", comp);
+}
+
+#[test]
+fn test_tokenize13() {
+    let comp = vec!["36", " ", "m", " ", "5"];
+    tokenize_assert("36 m 5", comp);
+}
+
+#[test]
+fn test_tokenize14() {
+    let comp = vec!["36", " ", "m", " ", "5", " ", "s"];
+    tokenize_assert("36 m 5 s", comp);
+}
+
+#[test]
+fn test_tokenize15() {
+    let comp = vec!["36", " ", "m", " ", "05"];
+    tokenize_assert("36 m 05", comp);
+}
+
+#[test]
+fn test_tokenize16() {
+    let comp = vec!["36", " ", "m", " ", "05", " ", "s"];
+    tokenize_assert("36 m 05 s", comp);
+}
+
+#[test]
+fn test_tokenize17() {
+    let comp = vec!["10", "h", " ", "am"];
+    tokenize_assert("10h am", comp);
+}
+
+#[test]
+fn test_tokenize18() {
+    let comp = vec!["10", "h", " ", "pm"];
+    tokenize_assert("10h pm", comp);
+}
+
+#[test]
+fn test_tokenize19() {
+    let comp = vec!["10", "am"];
+    tokenize_assert("10am", comp);
+}
+
+#[test]
+fn test_tokenize20() {
+    let comp = vec!["10", "pm"];
+    tokenize_assert("10pm", comp);
+}
+
+#[test]
+fn test_tokenize21() {
+    let comp = vec!["10", ":", "00", " ", "am"];
+    tokenize_assert("10:00 am", comp);
+}
+
+#[test]
+fn test_tokenize22() {
+    let comp = vec!["10", ":", "00", " ", "pm"];
+    tokenize_assert("10:00 pm", comp);
+}
+
+#[test]
+fn test_tokenize23() {
+    let comp = vec!["10", ":", "00", "am"];
+    tokenize_assert("10:00am", comp);
+}
+
+#[test]
+fn test_tokenize24() {
+    let comp = vec!["10", ":", "00", "pm"];
+    tokenize_assert("10:00pm", comp);
+}
+
+#[test]
+fn test_tokenize25() {
+    let comp = vec!["10", ":", "00", "a", ".", "m"];
+    tokenize_assert("10:00a.m", comp);
+}
+
+#[test]
+fn test_tokenize26() {
+    let comp = vec!["10", ":", "00", "p", ".", "m"];
+    tokenize_assert("10:00p.m", comp);
+}
+
+#[test]
+fn test_tokenize27() {
+    let comp = vec!["10", ":", "00", "a", ".", "m", "."];
+    tokenize_assert("10:00a.m.", comp);
+}
+
+#[test]
+fn test_tokenize28() {
+    let comp = vec!["10", ":", "00", "p", ".", "m", "."];
+    tokenize_assert("10:00p.m.", comp);
+}
+
+#[test]
+fn test_tokenize29() {
+    let comp = vec!["October"];
+    tokenize_assert("October", comp);
+}
+
+#[test]
+fn test_tokenize30() {
+    let comp = vec!["31", "-", "Dec", "-", "00"];
+    tokenize_assert("31-Dec-00", comp);
+}
+
+#[test]
+fn test_tokenize31() {
+    let comp = vec!["0", ":", "01", ":", "02"];
+    tokenize_assert("0:01:02", comp);
+}
+
+#[test]
+fn test_tokenize32() {
+    let comp = vec!["12", "h", " ", "01", "m", "02", "s", " ", "am"];
+    tokenize_assert("12h 01m02s am", comp);
+}
+
+#[test]
+fn test_tokenize33() {
+    let comp = vec!["12", ":", "08", " ", "PM"];
+    tokenize_assert("12:08 PM", comp);
+}
+
+#[test]
+fn test_tokenize34() {
+    let comp = vec!["01", "h", "02", "m", "03"];
+    tokenize_assert("01h02m03", comp);
+}
+
+#[test]
+fn test_tokenize35() {
+    let comp = vec!["01", "h", "02"];
+    tokenize_assert("01h02", comp);
+}
+
+#[test]
+fn test_tokenize36() {
+    let comp = vec!["01", "h", "02", "s"];
+    tokenize_assert("01h02s", comp);
+}
+
+#[test]
+fn test_tokenize37() {
+    let comp = vec!["01", "m", "02"];
+    tokenize_assert("01m02", comp);
+}
+
+#[test]
+fn test_tokenize38() {
+    let comp = vec!["01", "m", "02", "h"];
+    tokenize_assert("01m02h", comp);
+}
+
+#[test]
+fn test_tokenize39() {
+    let comp = vec!["2004", " ", "10", " ", "Apr", " ", "11", "h", "30", "m"];
+    tokenize_assert("2004 10 Apr 11h30m", comp);
+}
+
+#[test]
+fn test_tokenize40() {
+    let comp = vec!["Sep", " ", "03"];
+    tokenize_assert("Sep 03", comp);
+}
+
+#[test]
+fn test_tokenize41() {
+    let comp = vec!["Sep", " ", "of", " ", "03"];
+    tokenize_assert("Sep of 03", comp);
+}
+
+#[test]
+fn test_tokenize42() {
+    let comp = vec!["02", ":", "17", "NOV", "2017"];
+    tokenize_assert("02:17NOV2017", comp);
+}
+
+#[test]
+fn test_tokenize43() {
+    let comp = vec!["Thu", " ", "Sep", " ", "10", ":", "36", ":", "28"];
+    tokenize_assert("Thu Sep 10:36:28", comp);
+}
+
+#[test]
+fn test_tokenize44() {
+    let comp = vec!["Thu", " ", "10", ":", "36", ":", "28"];
+    tokenize_assert("Thu 10:36:28", comp);
+}
+
+#[test]
+fn test_tokenize45() {
+    let comp = vec!["Wed"];
+    tokenize_assert("Wed", comp);
+}
+
+#[test]
+fn test_tokenize46() {
+    let comp = vec!["Wednesday"];
+    tokenize_assert("Wednesday", comp);
+}
+
+#[test]
+fn test_tokenize47() {
+    let comp = vec!["Thu", " ", "Sep", " ", "25", " ", "10", ":", "36", ":", "28", " ", "2003"];
+    tokenize_assert("Thu Sep 25 10:36:28 2003", comp);
+}
+
+#[test]
+fn test_tokenize48() {
+    let comp = vec!["Thu", " ", "Sep", " ", "25", " ", "2003"];
+    tokenize_assert("Thu Sep 25 2003", comp);
+}
+
+#[test]
+fn test_tokenize49() {
+    let comp = vec!["2003", "-", "09", "-", "25", "T", "10", ":", "49", ":", "41"];
+    tokenize_assert("2003-09-25T10:49:41", comp);
+}
+
+#[test]
+fn test_tokenize50() {
+    let comp = vec!["2003", "-", "09", "-", "25", "T", "10", ":", "49"];
+    tokenize_assert("2003-09-25T10:49", comp);
+}
+
+#[test]
+fn test_tokenize51() {
+    let comp = vec!["2003", "-", "09", "-", "25", "T", "10"];
+    tokenize_assert("2003-09-25T10", comp);
+}
+
+#[test]
+fn test_tokenize52() {
+    let comp = vec!["2003", "-", "09", "-", "25"];
+    tokenize_assert("2003-09-25", comp);
+}
+
+#[test]
+fn test_tokenize53() {
+    let comp = vec!["20030925", "T", "104941"];
+    tokenize_assert("20030925T104941", comp);
+}
+
+#[test]
+fn test_tokenize54() {
+    let comp = vec!["20030925", "T", "1049"];
+    tokenize_assert("20030925T1049", comp);
+}
+
+#[test]
+fn test_tokenize55() {
+    let comp = vec!["20030925", "T", "10"];
+    tokenize_assert("20030925T10", comp);
+}
+
+#[test]
+fn test_tokenize56() {
+    let comp = vec!["20030925"];
+    tokenize_assert("20030925", comp);
+}
+
+#[test]
+fn test_tokenize57() {
+    let comp = vec!["2003", "-", "09", "-", "25", " ", "10", ":", "49", ":", "41.502"];
+    tokenize_assert("2003-09-25 10:49:41,502", comp);
+}
+
+#[test]
+fn test_tokenize58() {
+    let comp = vec!["199709020908"];
+    tokenize_assert("199709020908", comp);
+}
+
+#[test]
+fn test_tokenize59() {
+    let comp = vec!["19970902090807"];
+    tokenize_assert("19970902090807", comp);
+}
+
+#[test]
+fn test_tokenize60() {
+    let comp = vec!["2003", "-", "09", "-", "25"];
+    tokenize_assert("2003-09-25", comp);
+}
+
+#[test]
+fn test_tokenize61() {
+    let comp = vec!["09", "-", "25", "-", "2003"];
+    tokenize_assert("09-25-2003", comp);
+}
+
+#[test]
+fn test_tokenize62() {
+    let comp = vec!["25", "-", "09", "-", "2003"];
+    tokenize_assert("25-09-2003", comp);
+}
+
+#[test]
+fn test_tokenize63() {
+    let comp = vec!["10", "-", "09", "-", "2003"];
+    tokenize_assert("10-09-2003", comp);
+}
+
+#[test]
+fn test_tokenize64() {
+    let comp = vec!["10", "-", "09", "-", "03"];
+    tokenize_assert("10-09-03", comp);
+}
+
+#[test]
+fn test_tokenize65() {
+    let comp = vec!["2003", ".", "09", ".", "25"];
+    tokenize_assert("2003.09.25", comp);
+}
+
+#[test]
+fn test_tokenize66() {
+    let comp = vec!["09", ".", "25", ".", "2003"];
+    tokenize_assert("09.25.2003", comp);
+}
+
+#[test]
+fn test_tokenize67() {
+    let comp = vec!["25", ".", "09", ".", "2003"];
+    tokenize_assert("25.09.2003", comp);
+}
+
+#[test]
+fn test_tokenize68() {
+    let comp = vec!["10", ".", "09", ".", "2003"];
+    tokenize_assert("10.09.2003", comp);
+}
+
+#[test]
+fn test_tokenize69() {
+    let comp = vec!["10", ".", "09", ".", "03"];
+    tokenize_assert("10.09.03", comp);
+}
+
+#[test]
+fn test_tokenize70() {
+    let comp = vec!["2003", "/", "09", "/", "25"];
+    tokenize_assert("2003/09/25", comp);
+}
+
+#[test]
+fn test_tokenize71() {
+    let comp = vec!["09", "/", "25", "/", "2003"];
+    tokenize_assert("09/25/2003", comp);
+}
+
+#[test]
+fn test_tokenize72() {
+    let comp = vec!["25", "/", "09", "/", "2003"];
+    tokenize_assert("25/09/2003", comp);
+}
+
+#[test]
+fn test_tokenize73() {
+    let comp = vec!["10", "/", "09", "/", "2003"];
+    tokenize_assert("10/09/2003", comp);
+}
+
+#[test]
+fn test_tokenize74() {
+    let comp = vec!["10", "/", "09", "/", "03"];
+    tokenize_assert("10/09/03", comp);
+}
+
+#[test]
+fn test_tokenize75() {
+    let comp = vec!["2003", " ", "09", " ", "25"];
+    tokenize_assert("2003 09 25", comp);
+}
+
+#[test]
+fn test_tokenize76() {
+    let comp = vec!["09", " ", "25", " ", "2003"];
+    tokenize_assert("09 25 2003", comp);
+}
+
+#[test]
+fn test_tokenize77() {
+    let comp = vec!["25", " ", "09", " ", "2003"];
+    tokenize_assert("25 09 2003", comp);
+}
+
+#[test]
+fn test_tokenize78() {
+    let comp = vec!["10", " ", "09", " ", "2003"];
+    tokenize_assert("10 09 2003", comp);
+}
+
+#[test]
+fn test_tokenize79() {
+    let comp = vec!["10", " ", "09", " ", "03"];
+    tokenize_assert("10 09 03", comp);
+}
+
+#[test]
+fn test_tokenize80() {
+    let comp = vec!["25", " ", "09", " ", "03"];
+    tokenize_assert("25 09 03", comp);
+}
+
+#[test]
+fn test_tokenize81() {
+    let comp = vec!["03", " ", "25", " ", "Sep"];
+    tokenize_assert("03 25 Sep", comp);
+}
+
+#[test]
+fn test_tokenize82() {
+    let comp = vec!["25", " ", "03", " ", "Sep"];
+    tokenize_assert("25 03 Sep", comp);
+}
+
+#[test]
+fn test_tokenize83() {
+    let comp = vec![" ", " ", "July", " ", " ", " ", "4", " ", ",", " ", " ", "1976", " ", " ", " ", "12", ":", "01", ":", "02", " ", " ", " ", "am", " ", " "];
+    tokenize_assert("  July   4 ,  1976   12:01:02   am  ", comp);
+}
+
+#[test]
+fn test_tokenize84() {
+    let comp = vec!["Wed", ",", " ", "July", " ", "10", ",", " ", "'", "96"];
+    tokenize_assert("Wed, July 10, '96", comp);
+}
+
+#[test]
+fn test_tokenize85() {
+    let comp = vec!["1996", ".", "July", ".", "10", " ", "AD", " ", "12", ":", "08", " ", "PM"];
+    tokenize_assert("1996.July.10 AD 12:08 PM", comp);
+}
+
+#[test]
+fn test_tokenize86() {
+    let comp = vec!["July", " ", "4", ",", " ", "1976"];
+    tokenize_assert("July 4, 1976", comp);
+}
+
+#[test]
+fn test_tokenize87() {
+    let comp = vec!["7", " ", "4", " ", "1976"];
+    tokenize_assert("7 4 1976", comp);
+}
+
+#[test]
+fn test_tokenize88() {
+    let comp = vec!["4", " ", "jul", " ", "1976"];
+    tokenize_assert("4 jul 1976", comp);
+}
+
+#[test]
+fn test_tokenize89() {
+    let comp = vec!["7", "-", "4", "-", "76"];
+    tokenize_assert("7-4-76", comp);
+}
+
+#[test]
+fn test_tokenize90() {
+    let comp = vec!["19760704"];
+    tokenize_assert("19760704", comp);
+}
+
+#[test]
+fn test_tokenize91() {
+    let comp = vec!["0", ":", "01", ":", "02", " ", "on", " ", "July", " ", "4", ",", " ", "1976"];
+    tokenize_assert("0:01:02 on July 4, 1976", comp);
+}
+
+#[test]
+fn test_tokenize92() {
+    let comp = vec!["0", ":", "01", ":", "02", " ", "on", " ", "July", " ", "4", ",", " ", "1976"];
+    tokenize_assert("0:01:02 on July 4, 1976", comp);
+}
+
+#[test]
+fn test_tokenize93() {
+    let comp = vec!["July", " ", "4", ",", " ", "1976", " ", "12", ":", "01", ":", "02", " ", "am"];
+    tokenize_assert("July 4, 1976 12:01:02 am", comp);
+}
+
+#[test]
+fn test_tokenize94() {
+    let comp = vec!["Mon", " ", "Jan", " ", " ", "2", " ", "04", ":", "24", ":", "27", " ", "1995"];
+    tokenize_assert("Mon Jan  2 04:24:27 1995", comp);
+}
+
+#[test]
+fn test_tokenize95() {
+    let comp = vec!["04", ".", "04", ".", "95", " ", "00", ":", "22"];
+    tokenize_assert("04.04.95 00:22", comp);
+}
+
+#[test]
+fn test_tokenize96() {
+    let comp = vec!["Jan", " ", "1", " ", "1999", " ", "11", ":", "23", ":", "34.578"];
+    tokenize_assert("Jan 1 1999 11:23:34.578", comp);
+}
+
+#[test]
+fn test_tokenize97() {
+    let comp = vec!["950404", " ", "122212"];
+    tokenize_assert("950404 122212", comp);
+}
+
+#[test]
+fn test_tokenize98() {
+    let comp = vec!["3", "rd", " ", "of", " ", "May", " ", "2001"];
+    tokenize_assert("3rd of May 2001", comp);
+}
+
+#[test]
+fn test_tokenize99() {
+    let comp = vec!["5", "th", " ", "of", " ", "March", " ", "2001"];
+    tokenize_assert("5th of March 2001", comp);
+}
+
+#[test]
+fn test_tokenize100() {
+    let comp = vec!["1", "st", " ", "of", " ", "May", " ", "2003"];
+    tokenize_assert("1st of May 2003", comp);
+}
+
+#[test]
+fn test_tokenize101() {
+    let comp = vec!["0099", "-", "01", "-", "01", "T", "00", ":", "00", ":", "00"];
+    tokenize_assert("0099-01-01T00:00:00", comp);
+}
+
+#[test]
+fn test_tokenize102() {
+    let comp = vec!["0031", "-", "01", "-", "01", "T", "00", ":", "00", ":", "00"];
+    tokenize_assert("0031-01-01T00:00:00", comp);
+}
+
+#[test]
+fn test_tokenize103() {
+    let comp = vec!["20080227", "T", "21", ":", "26", ":", "01.123456789"];
+    tokenize_assert("20080227T21:26:01.123456789", comp);
+}
+
+#[test]
+fn test_tokenize104() {
+    let comp = vec!["13", "NOV", "2017"];
+    tokenize_assert("13NOV2017", comp);
+}
+
+#[test]
+fn test_tokenize105() {
+    let comp = vec!["0003", "-", "03", "-", "04"];
+    tokenize_assert("0003-03-04", comp);
+}
+
+#[test]
+fn test_tokenize106() {
+    let comp = vec!["December", ".", "0031", ".", "30"];
+    tokenize_assert("December.0031.30", comp);
+}
+
+#[test]
+fn test_tokenize107() {
+    let comp = vec!["090107"];
+    tokenize_assert("090107", comp);
+}
+
+#[test]
+fn test_tokenize108() {
+    let comp = vec!["2015", "-", "15", "-", "May"];
+    tokenize_assert("2015-15-May", comp);
+}
+
+#[test]
+fn test_tokenize109() {
+    let comp = vec!["Thu", " ", "Sep", " ", "25", " ", "10", ":", "36", ":", "28", " ", "BRST", " ", "2003"];
+    tokenize_assert("Thu Sep 25 10:36:28 BRST 2003", comp);
+}
+
+#[test]
+fn test_tokenize110() {
+    let comp = vec!["2003", " ", "10", ":", "36", ":", "28", " ", "BRST", " ", "25", " ", "Sep", " ", "Thu"];
+    tokenize_assert("2003 10:36:28 BRST 25 Sep Thu", comp);
+}
+
+#[test]
+fn test_tokenize111() {
+    let comp = vec!["Thu", ",", " ", "25", " ", "Sep", " ", "2003", " ", "10", ":", "49", ":", "41", " ", "-", "0300"];
+    tokenize_assert("Thu, 25 Sep 2003 10:49:41 -0300", comp);
+}
+
+#[test]
+fn test_tokenize112() {
+    let comp = vec!["2003", "-", "09", "-", "25", "T", "10", ":", "49", ":", "41.5", "-", "03", ":", "00"];
+    tokenize_assert("2003-09-25T10:49:41.5-03:00", comp);
+}
+
+#[test]
+fn test_tokenize113() {
+    let comp = vec!["2003", "-", "09", "-", "25", "T", "10", ":", "49", ":", "41", "-", "03", ":", "00"];
+    tokenize_assert("2003-09-25T10:49:41-03:00", comp);
+}
+
+#[test]
+fn test_tokenize114() {
+    let comp = vec!["20030925", "T", "104941.5", "-", "0300"];
+    tokenize_assert("20030925T104941.5-0300", comp);
+}
+
+#[test]
+fn test_tokenize115() {
+    let comp = vec!["20030925", "T", "104941", "-", "0300"];
+    tokenize_assert("20030925T104941-0300", comp);
+}
+
+#[test]
+fn test_tokenize116() {
+    let comp = vec!["10", "-", "09", "-", "2003"];
+    tokenize_assert("10-09-2003", comp);
+}
+
+#[test]
+fn test_tokenize117() {
+    let comp = vec!["10", ".", "09", ".", "2003"];
+    tokenize_assert("10.09.2003", comp);
+}
+
+#[test]
+fn test_tokenize118() {
+    let comp = vec!["10", "/", "09", "/", "2003"];
+    tokenize_assert("10/09/2003", comp);
+}
+
+#[test]
+fn test_tokenize119() {
+    let comp = vec!["10", " ", "09", " ", "2003"];
+    tokenize_assert("10 09 2003", comp);
+}
+
+#[test]
+fn test_tokenize120() {
+    let comp = vec!["090107"];
+    tokenize_assert("090107", comp);
+}
+
+#[test]
+fn test_tokenize121() {
+    let comp = vec!["2015", " ", "09", " ", "25"];
+    tokenize_assert("2015 09 25", comp);
+}
+
+#[test]
+fn test_tokenize122() {
+    let comp = vec!["10", "-", "09", "-", "03"];
+    tokenize_assert("10-09-03", comp);
+}
+
+#[test]
+fn test_tokenize123() {
+    let comp = vec!["10", ".", "09", ".", "03"];
+    tokenize_assert("10.09.03", comp);
+}
+
+#[test]
+fn test_tokenize124() {
+    let comp = vec!["10", "/", "09", "/", "03"];
+    tokenize_assert("10/09/03", comp);
+}
+
+#[test]
+fn test_tokenize125() {
+    let comp = vec!["10", " ", "09", " ", "03"];
+    tokenize_assert("10 09 03", comp);
+}
+
+#[test]
+fn test_tokenize126() {
+    let comp = vec!["090107"];
+    tokenize_assert("090107", comp);
+}
+
+#[test]
+fn test_tokenize127() {
+    let comp = vec!["2015", " ", "09", " ", "25"];
+    tokenize_assert("2015 09 25", comp);
+}
+
+#[test]
+fn test_tokenize128() {
+    let comp = vec!["090107"];
+    tokenize_assert("090107", comp);
+}
+
+#[test]
+fn test_tokenize129() {
+    let comp = vec!["2015", " ", "09", " ", "25"];
+    tokenize_assert("2015 09 25", comp);
+}
+
+#[test]
+fn test_tokenize130() {
+    let comp = vec!["April", " ", "2009"];
+    tokenize_assert("April 2009", comp);
+}
+
+#[test]
+fn test_tokenize131() {
+    let comp = vec!["Feb", " ", "2007"];
+    tokenize_assert("Feb 2007", comp);
+}
+
+#[test]
+fn test_tokenize132() {
+    let comp = vec!["Feb", " ", "2008"];
+    tokenize_assert("Feb 2008", comp);
+}
+
+#[test]
+fn test_tokenize133() {
+    let comp = vec!["Thu", " ", "Sep", " ", "25", " ", "10", ":", "36", ":", "28", " ", "BRST", " ", "2003"];
+    tokenize_assert("Thu Sep 25 10:36:28 BRST 2003", comp);
+}
+
+#[test]
+fn test_tokenize134() {
+    let comp = vec!["1996", ".", "07", ".", "10", " ", "AD", " ", "at", " ", "15", ":", "08", ":", "56", " ", "PDT"];
+    tokenize_assert("1996.07.10 AD at 15:08:56 PDT", comp);
+}
+
+#[test]
+fn test_tokenize135() {
+    let comp = vec!["Tuesday", ",", " ", "April", " ", "12", ",", " ", "1952", " ", "AD", " ", "3", ":", "30", ":", "42", "pm", " ", "PST"];
+    tokenize_assert("Tuesday, April 12, 1952 AD 3:30:42pm PST", comp);
+}
+
+#[test]
+fn test_tokenize136() {
+    let comp = vec!["November", " ", "5", ",", " ", "1994", ",", " ", "8", ":", "15", ":", "30", " ", "am", " ", "EST"];
+    tokenize_assert("November 5, 1994, 8:15:30 am EST", comp);
+}
+
+#[test]
+fn test_tokenize137() {
+    let comp = vec!["1994", "-", "11", "-", "05", "T", "08", ":", "15", ":", "30", "-", "05", ":", "00"];
+    tokenize_assert("1994-11-05T08:15:30-05:00", comp);
+}
+
+#[test]
+fn test_tokenize138() {
+    let comp = vec!["1994", "-", "11", "-", "05", "T", "08", ":", "15", ":", "30", "Z"];
+    tokenize_assert("1994-11-05T08:15:30Z", comp);
+}
+
+#[test]
+fn test_tokenize139() {
+    let comp = vec!["1976", "-", "07", "-", "04", "T", "00", ":", "01", ":", "02", "Z"];
+    tokenize_assert("1976-07-04T00:01:02Z", comp);
+}
+
+#[test]
+fn test_tokenize140() {
+    let comp = vec!["Tue", " ", "Apr", " ", "4", " ", "00", ":", "22", ":", "12", " ", "PDT", " ", "1995"];
+    tokenize_assert("Tue Apr 4 00:22:12 PDT 1995", comp);
+}
+
+#[test]
+fn test_tokenize141() {
+    let comp = vec!["Today", " ", "is", " ", "25", " ", "of", " ", "September", " ", "of", " ", "2003", ",", " ", "exactly", " ", "at", " ", "10", ":", "49", ":", "41", " ", "with", " ", "timezone", " ", "-", "03", ":", "00", "."];
+    tokenize_assert("Today is 25 of September of 2003, exactly at 10:49:41 with timezone -03:00.", comp);
+}
+
+#[test]
+fn test_tokenize142() {
+    let comp = vec!["Today", " ", "is", " ", "25", " ", "of", " ", "September", " ", "of", " ", "2003", ",", " ", "exactly", " ", "at", " ", "10", ":", "49", ":", "41", " ", "with", " ", "timezone", " ", "-", "03", ":", "00", "."];
+    tokenize_assert("Today is 25 of September of 2003, exactly at 10:49:41 with timezone -03:00.", comp);
+}
+
+#[test]
+fn test_tokenize143() {
+    let comp = vec!["I", " ", "have", " ", "a", " ", "meeting", " ", "on", " ", "March", " ", "1", ",", " ", "1974"];
+    tokenize_assert("I have a meeting on March 1, 1974", comp);
+}
+
+#[test]
+fn test_tokenize144() {
+    let comp = vec!["On", " ", "June", " ", "8", "th", ",", " ", "2020", ",", " ", "I", " ", "am", " ", "going", " ", "to", " ", "be", " ", "the", " ", "first", " ", "man", " ", "on", " ", "Mars"];
+    tokenize_assert("On June 8th, 2020, I am going to be the first man on Mars", comp);
+}
+
+#[test]
+fn test_tokenize145() {
+    let comp = vec!["Meet", " ", "me", " ", "at", " ", "the", " ", "AM", "/", "PM", " ", "on", " ", "Sunset", " ", "at", " ", "3", ":", "00", " ", "AM", " ", "on", " ", "December", " ", "3", "rd", ",", " ", "2003"];
+    tokenize_assert("Meet me at the AM/PM on Sunset at 3:00 AM on December 3rd, 2003", comp);
+}
+
+#[test]
+fn test_tokenize146() {
+    let comp = vec!["Meet", " ", "me", " ", "at", " ", "3", ":", "00", " ", "AM", " ", "on", " ", "December", " ", "3", "rd", ",", " ", "2003", " ", "at", " ", "the", " ", "AM", "/", "PM", " ", "on", " ", "Sunset"];
+    tokenize_assert("Meet me at 3:00 AM on December 3rd, 2003 at the AM/PM on Sunset", comp);
+}
+
+#[test]
+fn test_tokenize147() {
+    let comp = vec!["Jan", " ", "29", ",", " ", "1945", " ", "14", ":", "45", " ", "AM", " ", "I", " ", "going", " ", "to", " ", "see", " ", "you", " ", "there", "?"];
+    tokenize_assert("Jan 29, 1945 14:45 AM I going to see you there?", comp);
+}
+
+#[test]
+fn test_tokenize148() {
+    let comp = vec!["2017", "-", "07", "-", "17", " ", "06", ":", "15", ":"];
+    tokenize_assert("2017-07-17 06:15:", comp);
+}

src/tokenize.rs

@@ -1,5 +1,6 @@
 pub(crate) struct Tokenizer {
     token_stack: Vec<String>,
+    // TODO: Should this be more generic? io::Read for example?
     parse_string: String,
 }
 
@@ -13,12 +14,49 @@ pub(crate) enum ParseState {
 }
 
 impl Tokenizer {
-    pub(crate) fn new(parse_string: String) -> Self {
+
+    pub(crate) fn new(parse_string: &str) -> Self {
         Tokenizer {
-            token_stack: Vec::new(),
+            token_stack: vec![],
             parse_string: parse_string.chars().rev().collect(),
         }
     }
+
+    fn isword(&self, c: char) -> bool {
+        c.is_alphabetic()
+    }
+
+    fn isnum(&self, c: char) -> bool {
+        c.is_numeric()
+    }
+
+    fn isspace(&self, c: char) -> bool {
+        c.is_whitespace()
+    }
+
+    fn decimal_split(&self, s: &str) -> Vec<String> {
+        // Handles the same thing as Python's re.split()
+        let mut tokens: Vec<String> = vec!["".to_owned()];
+
+        for c in s.chars() {
+            if c == '.' || c == ',' {
+                tokens.push(c.to_string());
+                tokens.push("".to_owned());
+            } else {
+                // UNWRAP: Initial setup guarantees we always have an item
+                let mut t = tokens.pop().unwrap();
+                t.push(c);
+                tokens.push(t);
+            }
+        }
+
+        // TODO: Do I really have to use &String instead of &str?
+        if tokens.last() == Some(&"".to_owned()) {
+            tokens.pop();
+        }
+
+        tokens
+    }
 }
 
 impl Iterator for Tokenizer {
@@ -26,178 +64,123 @@ impl Iterator for Tokenizer {
 
     fn next(&mut self) -> Option<Self::Item> {
         if !self.token_stack.is_empty() {
-            return Some(self.token_stack.pop().unwrap());
+            return Some(self.token_stack.remove(0));
-        };
+        }
-        if self.parse_string.is_empty() {
-            return None;
-        };
 
-        let mut char_stack: Vec<char> = Vec::new();
+        let mut seenletters = false;
-        let mut seen_letters = false;
+        let mut token: Option<String> = None;
         let mut state = ParseState::Empty;
 
-        while let Some(next) = self.parse_string.pop() {
+        while !self.parse_string.is_empty() {
+            // Dateutil uses a separate `charstack` to manage the incoming stream.
+            // Because parse_string can have things pushed back onto it, we skip
+            // a couple of steps related to the `charstack`.
+
+            // UNWRAP: Just checked that parse_string isn't empty
+            let nextchar = self.parse_string.pop().unwrap();
+
             match state {
                 ParseState::Empty => {
-                    if next.is_numeric() {
+                    token = Some(nextchar.to_string());
-                        state = ParseState::Numeric;
+                    if self.isword(nextchar) {
-                        char_stack.push(next);
-                    } else if next.is_alphabetic() {
                         state = ParseState::Alpha;
-                        seen_letters = true;
+                    } else if self.isnum(nextchar) {
-                        char_stack.push(next);
+                        state = ParseState::Numeric;
-                    } else if next.is_whitespace() {
+                    } else if self.isspace(nextchar) {
-                        char_stack.push(' ');
+                        token = Some(" ".to_owned());
                         break;
                     } else {
-                        char_stack.push(next);
                         break;
                     }
-                }
+                },
                 ParseState::Alpha => {
-                    if next.is_alphabetic() {
+                    seenletters = true;
-                        char_stack.push(next);
+                    if self.isword(nextchar) {
-                    } else if next == '.' {
+                        // UNWRAP: Because we're in non-empty parse state, we're guaranteed to have a token
-                        state = ParseState::AlphaDecimal;
+                        token.as_mut().unwrap().push(nextchar);
-                        char_stack.push(next);
+                    } else if nextchar == '.' {
-                    } else {
+                        token.as_mut().unwrap().push(nextchar);
-                        // We don't recognize the character, so push it back
-                        // to be handled later.
-                        self.parse_string.push(next);
-                        break;
-                    }
-                }
-                ParseState::AlphaDecimal => {
-                    if next == '.' || next.is_alphabetic() {
-                        char_stack.push(next);
-                    } else if next.is_numeric() && char_stack.last().unwrap().clone() == '.' {
-                        char_stack.push(next);
-                        state = ParseState::NumericDecimal;
-                    } else {
-                        self.parse_string.push(next);
-                        break;
-                    }
-                }
-                ParseState::Numeric => {
-                    if next.is_numeric() {
-                        char_stack.push(next);
-                    } else if next == '.' || (next == ',' && char_stack.len() >= 2) {
-                        char_stack.push(next);
-                        state = ParseState::NumericDecimal;
-                    } else {
-                        // We don't recognize the character, so push it back
-                        // to be handled later
-                        self.parse_string.push(next);
-                        break;
-                    }
-                }
-                ParseState::NumericDecimal => {
-                    if next == '.' || next.is_numeric() {
-                        char_stack.push(next);
-                    } else if next.is_alphabetic() && char_stack.last().unwrap().clone() == '.' {
-                        char_stack.push(next);
                         state = ParseState::AlphaDecimal;
                     } else {
-                        self.parse_string.push(next);
+                        self.parse_string.push(nextchar);
+                        break;
+                    }
+                },
+                ParseState::Numeric => {
+                    if self.isnum(nextchar) {
+                        // UNWRAP: Because we're in non-empty parse state, we're guaranteed to have a token
+                        token.as_mut().unwrap().push(nextchar);
+                    } else if nextchar == '.' || (nextchar == ',' && token.as_ref().unwrap().len() >= 2) {
+                        token.as_mut().unwrap().push(nextchar);
+                        state = ParseState::NumericDecimal;
+                    } else {
+                        self.parse_string.push(nextchar);
+                        break;
+                    }
+                },
+                ParseState::AlphaDecimal => {
+                    seenletters = true;
+                    if nextchar == '.' || self.isword(nextchar) {
+                        // UNWRAP: Because we're in non-empty parse state, we're guaranteed to have a token
+                        token.as_mut().unwrap().push(nextchar);
+                    } else if self.isnum(nextchar) && token.as_ref().unwrap().chars().last() == Some('.') {
+                        token.as_mut().unwrap().push(nextchar);
+                        state = ParseState::NumericDecimal;
+                    } else {
+                        self.parse_string.push(nextchar);
+                        break;
+                    }
+                },
+                ParseState::NumericDecimal => {
+                    if nextchar == '.' || self.isnum(nextchar) {
+                        // UNWRAP: Because we're in non-empty parse state, we're guaranteed to have a token
+                        token.as_mut().unwrap().push(nextchar);
+                    } else if self.isword(nextchar) && token.as_ref().unwrap().chars().last() == Some('.') {
+                        token.as_mut().unwrap().push(nextchar);
+                        state = ParseState::AlphaDecimal;
+                    } else {
+                        self.parse_string.push(nextchar);
                         break;
                     }
                 }
             }
         }
 
-        // I like Python's version of this much better:
+        // Python uses the state to short-circuit and make sure it doesn't run into issues with None
-        // needs_split = seen_letters or char_stack.count('.') > 1 or char_stack[-1] in '.,'
+        // We do something slightly different to express the same logic
-        let dot_count = char_stack.iter().fold(0, |count, character| {
+        if state == ParseState::AlphaDecimal || state == ParseState::NumericDecimal {
-            count + (if character == &'.' { 1 } else { 0 })
+            // UNWRAP: The state check guarantees that we have a value
-        });
+            let dot_count = token.as_ref().unwrap().chars().filter(|c| *c == '.').count();
-        let needs_split = seen_letters || dot_count > 1 || char_stack.last().unwrap() == &'.'
+            let last_char = token.as_ref().unwrap().chars().last();
-            || char_stack.last().unwrap() == &',';
+            let last_splittable = last_char == Some('.') || last_char == Some(',');
-        let final_string: String = char_stack.into_iter().collect();
     
-        let mut tokens = match state {
+            if seenletters || dot_count > 1 || last_splittable {
-            ParseState::Empty => vec![final_string],
+                let mut l = self.decimal_split(token.as_ref().unwrap());
-            ParseState::Alpha => vec![final_string],
+                let remaining = l.split_off(1);
-            ParseState::Numeric => vec![final_string],
-            ParseState::AlphaDecimal => {
-                if needs_split {
-                    decimal_split(&final_string, false)
-                } else {
-                    vec![final_string]
-                }
-            }
-            ParseState::NumericDecimal => {
-                if needs_split {
-                    decimal_split(&final_string, dot_count == 0)
-                } else {
-                    vec![final_string]
-                }
-            }
-        }.into_iter()
-            .rev()
-            .collect();
     
-        self.token_stack.append(&mut tokens);
+                token = Some(l[0].clone());
-        // UNWRAP: Previous match guaranteed that at least one token was added
+                for t in remaining {
-        let token = self.token_stack.pop().unwrap();
+                    self.token_stack.push(t);
-        if state == ParseState::NumericDecimal && !token.contains(".") {
-            Some(token.replace(",", "."))
-        } else {
-            Some(token)
-        }
                 }
             }
     
-fn decimal_split(characters: &str, cast_period: bool) -> Vec<String> {
+            if state == ParseState::NumericDecimal && dot_count == 0 {
-    let mut token_stack: Vec<String> = Vec::new();
+                token = Some(token.unwrap().replace(',', "."));
-    let mut char_stack: Vec<char> = Vec::new();
-    let mut state = ParseState::Empty;
-
-    for c in characters.chars() {
-        match state {
-            ParseState::Empty => {
-                if c.is_alphabetic() {
-                    char_stack.push(c);
-                    state = ParseState::Alpha;
-                } else if c.is_numeric() {
-                    char_stack.push(c);
-                    state = ParseState::Numeric;
-                } else {
-                    let character = if cast_period { '.' } else { c };
-                    token_stack.push(character.to_string());
-                }
-            }
-            ParseState::Alpha => {
-                if c.is_alphabetic() {
-                    char_stack.push(c);
-                } else {
-                    token_stack.push(char_stack.iter().collect());
-                    char_stack.clear();
-                    let character = if cast_period { '.' } else { c };
-                    token_stack.push(character.to_string());
-                    state = ParseState::Empty;
-                }
-            }
-            ParseState::Numeric => {
-                if c.is_numeric() {
-                    char_stack.push(c);
-                } else {
-                    token_stack.push(char_stack.iter().collect());
-                    char_stack.clear();
-                    let character = if cast_period { '.' } else { c };
-                    token_stack.push(character.to_string());
-                    state = ParseState::Empty;
-                }
-            }
-            _ => panic!("Invalid parse state during decimal_split()"),
             }
         }
 
-    match state {
+        token
-        ParseState::Alpha => token_stack.push(char_stack.iter().collect()),
+    }
-        ParseState::Numeric => token_stack.push(char_stack.iter().collect()),
-        ParseState::Empty => (),
-        _ => panic!("Invalid parse state during decimal_split()"),
 }
 
-    token_stack
+#[cfg(test)]
+mod tests {
+
+    use Tokenizer;
+
+    #[test]
+    fn test_basic() {
+        let tokens: Vec<String> = Tokenizer::new("September of 2003,").collect();
+        assert_eq!(tokens, vec!["September", " ", "of", " ", "2003", ","]);
+    }
 }

src/weekday.rs

@@ -1,5 +1,3 @@
-use std::cmp::max;
-
 use ParseResult;
 use ParseError;
 
@@ -101,6 +99,8 @@ pub fn day_of_week(year: u32, month: u32, day: u32) -> ParseResult<DayOfWeek> {
     }
 }
 
+// Rust warns about unused imports here, but they're definitely used.
+#[allow(unused_imports)]
 mod test {
 
     use weekday::day_of_week;