1091 lines
24 KiB
Rust
1091 lines
24 KiB
Rust
//! This code has been generated by running the `build_pycompat_tokenizer.py` script
|
|
//! in the repository root. Please do not edit it, as your edits will be destroyed
|
|
//! upon re-running code generation.
|
|
|
|
use tokenize::Tokenizer;
|
|
|
|
fn tokenize_assert(test_str: &str, comparison: Vec<&str>) {
|
|
let tokens: Vec<String> = Tokenizer::new(test_str).collect();
|
|
assert_eq!(tokens, comparison, "Tokenizing mismatch for `{}`", test_str);
|
|
}
|
|
|
|
#[test]
|
|
fn test_tokenize0() {
|
|
let comp = vec![
|
|
"Thu", " ", "Sep", " ", "25", " ", "10", ":", "36", ":", "28",
|
|
];
|
|
tokenize_assert("Thu Sep 25 10:36:28", comp);
|
|
}
|
|
|
|
#[test]
|
|
fn test_tokenize1() {
|
|
let comp = vec!["Sep", " ", "10", ":", "36", ":", "28"];
|
|
tokenize_assert("Sep 10:36:28", comp);
|
|
}
|
|
|
|
#[test]
|
|
fn test_tokenize2() {
|
|
let comp = vec!["10", ":", "36", ":", "28"];
|
|
tokenize_assert("10:36:28", comp);
|
|
}
|
|
|
|
#[test]
|
|
fn test_tokenize3() {
|
|
let comp = vec!["10", ":", "36"];
|
|
tokenize_assert("10:36", comp);
|
|
}
|
|
|
|
#[test]
|
|
fn test_tokenize4() {
|
|
let comp = vec!["Sep", " ", "2003"];
|
|
tokenize_assert("Sep 2003", comp);
|
|
}
|
|
|
|
#[test]
|
|
fn test_tokenize5() {
|
|
let comp = vec!["Sep"];
|
|
tokenize_assert("Sep", comp);
|
|
}
|
|
|
|
#[test]
|
|
fn test_tokenize6() {
|
|
let comp = vec!["2003"];
|
|
tokenize_assert("2003", comp);
|
|
}
|
|
|
|
#[test]
|
|
fn test_tokenize7() {
|
|
let comp = vec!["10", "h", "36", "m", "28.5", "s"];
|
|
tokenize_assert("10h36m28.5s", comp);
|
|
}
|
|
|
|
#[test]
|
|
fn test_tokenize8() {
|
|
let comp = vec!["10", "h", "36", "m", "28", "s"];
|
|
tokenize_assert("10h36m28s", comp);
|
|
}
|
|
|
|
#[test]
|
|
fn test_tokenize9() {
|
|
let comp = vec!["10", "h", "36", "m"];
|
|
tokenize_assert("10h36m", comp);
|
|
}
|
|
|
|
#[test]
|
|
fn test_tokenize10() {
|
|
let comp = vec!["10", "h"];
|
|
tokenize_assert("10h", comp);
|
|
}
|
|
|
|
#[test]
|
|
fn test_tokenize11() {
|
|
let comp = vec!["10", " ", "h", " ", "36"];
|
|
tokenize_assert("10 h 36", comp);
|
|
}
|
|
|
|
#[test]
|
|
fn test_tokenize12() {
|
|
let comp = vec!["10", " ", "h", " ", "36.5"];
|
|
tokenize_assert("10 h 36.5", comp);
|
|
}
|
|
|
|
#[test]
|
|
fn test_tokenize13() {
|
|
let comp = vec!["36", " ", "m", " ", "5"];
|
|
tokenize_assert("36 m 5", comp);
|
|
}
|
|
|
|
#[test]
|
|
fn test_tokenize14() {
|
|
let comp = vec!["36", " ", "m", " ", "5", " ", "s"];
|
|
tokenize_assert("36 m 5 s", comp);
|
|
}
|
|
|
|
#[test]
|
|
fn test_tokenize15() {
|
|
let comp = vec!["36", " ", "m", " ", "05"];
|
|
tokenize_assert("36 m 05", comp);
|
|
}
|
|
|
|
#[test]
|
|
fn test_tokenize16() {
|
|
let comp = vec!["36", " ", "m", " ", "05", " ", "s"];
|
|
tokenize_assert("36 m 05 s", comp);
|
|
}
|
|
|
|
#[test]
|
|
fn test_tokenize17() {
|
|
let comp = vec!["10", "h", " ", "am"];
|
|
tokenize_assert("10h am", comp);
|
|
}
|
|
|
|
#[test]
|
|
fn test_tokenize18() {
|
|
let comp = vec!["10", "h", " ", "pm"];
|
|
tokenize_assert("10h pm", comp);
|
|
}
|
|
|
|
#[test]
|
|
fn test_tokenize19() {
|
|
let comp = vec!["10", "am"];
|
|
tokenize_assert("10am", comp);
|
|
}
|
|
|
|
#[test]
|
|
fn test_tokenize20() {
|
|
let comp = vec!["10", "pm"];
|
|
tokenize_assert("10pm", comp);
|
|
}
|
|
|
|
#[test]
|
|
fn test_tokenize21() {
|
|
let comp = vec!["10", ":", "00", " ", "am"];
|
|
tokenize_assert("10:00 am", comp);
|
|
}
|
|
|
|
#[test]
|
|
fn test_tokenize22() {
|
|
let comp = vec!["10", ":", "00", " ", "pm"];
|
|
tokenize_assert("10:00 pm", comp);
|
|
}
|
|
|
|
#[test]
|
|
fn test_tokenize23() {
|
|
let comp = vec!["10", ":", "00", "am"];
|
|
tokenize_assert("10:00am", comp);
|
|
}
|
|
|
|
#[test]
|
|
fn test_tokenize24() {
|
|
let comp = vec!["10", ":", "00", "pm"];
|
|
tokenize_assert("10:00pm", comp);
|
|
}
|
|
|
|
#[test]
|
|
fn test_tokenize25() {
|
|
let comp = vec!["10", ":", "00", "a", ".", "m"];
|
|
tokenize_assert("10:00a.m", comp);
|
|
}
|
|
|
|
#[test]
|
|
fn test_tokenize26() {
|
|
let comp = vec!["10", ":", "00", "p", ".", "m"];
|
|
tokenize_assert("10:00p.m", comp);
|
|
}
|
|
|
|
#[test]
|
|
fn test_tokenize27() {
|
|
let comp = vec!["10", ":", "00", "a", ".", "m", "."];
|
|
tokenize_assert("10:00a.m.", comp);
|
|
}
|
|
|
|
#[test]
|
|
fn test_tokenize28() {
|
|
let comp = vec!["10", ":", "00", "p", ".", "m", "."];
|
|
tokenize_assert("10:00p.m.", comp);
|
|
}
|
|
|
|
#[test]
|
|
fn test_tokenize29() {
|
|
let comp = vec!["October"];
|
|
tokenize_assert("October", comp);
|
|
}
|
|
|
|
#[test]
|
|
fn test_tokenize30() {
|
|
let comp = vec!["31", "-", "Dec", "-", "00"];
|
|
tokenize_assert("31-Dec-00", comp);
|
|
}
|
|
|
|
#[test]
|
|
fn test_tokenize31() {
|
|
let comp = vec!["0", ":", "01", ":", "02"];
|
|
tokenize_assert("0:01:02", comp);
|
|
}
|
|
|
|
#[test]
|
|
fn test_tokenize32() {
|
|
let comp = vec!["12", "h", " ", "01", "m", "02", "s", " ", "am"];
|
|
tokenize_assert("12h 01m02s am", comp);
|
|
}
|
|
|
|
#[test]
|
|
fn test_tokenize33() {
|
|
let comp = vec!["12", ":", "08", " ", "PM"];
|
|
tokenize_assert("12:08 PM", comp);
|
|
}
|
|
|
|
#[test]
|
|
fn test_tokenize34() {
|
|
let comp = vec!["01", "h", "02", "m", "03"];
|
|
tokenize_assert("01h02m03", comp);
|
|
}
|
|
|
|
#[test]
|
|
fn test_tokenize35() {
|
|
let comp = vec!["01", "h", "02"];
|
|
tokenize_assert("01h02", comp);
|
|
}
|
|
|
|
#[test]
|
|
fn test_tokenize36() {
|
|
let comp = vec!["01", "h", "02", "s"];
|
|
tokenize_assert("01h02s", comp);
|
|
}
|
|
|
|
#[test]
|
|
fn test_tokenize37() {
|
|
let comp = vec!["01", "m", "02"];
|
|
tokenize_assert("01m02", comp);
|
|
}
|
|
|
|
#[test]
|
|
fn test_tokenize38() {
|
|
let comp = vec!["01", "m", "02", "h"];
|
|
tokenize_assert("01m02h", comp);
|
|
}
|
|
|
|
#[test]
|
|
fn test_tokenize39() {
|
|
let comp = vec!["2004", " ", "10", " ", "Apr", " ", "11", "h", "30", "m"];
|
|
tokenize_assert("2004 10 Apr 11h30m", comp);
|
|
}
|
|
|
|
#[test]
|
|
fn test_tokenize40() {
|
|
let comp = vec!["Sep", " ", "03"];
|
|
tokenize_assert("Sep 03", comp);
|
|
}
|
|
|
|
#[test]
|
|
fn test_tokenize41() {
|
|
let comp = vec!["Sep", " ", "of", " ", "03"];
|
|
tokenize_assert("Sep of 03", comp);
|
|
}
|
|
|
|
#[test]
|
|
fn test_tokenize42() {
|
|
let comp = vec!["02", ":", "17", "NOV", "2017"];
|
|
tokenize_assert("02:17NOV2017", comp);
|
|
}
|
|
|
|
#[test]
|
|
fn test_tokenize43() {
|
|
let comp = vec!["Thu", " ", "Sep", " ", "10", ":", "36", ":", "28"];
|
|
tokenize_assert("Thu Sep 10:36:28", comp);
|
|
}
|
|
|
|
#[test]
|
|
fn test_tokenize44() {
|
|
let comp = vec!["Thu", " ", "10", ":", "36", ":", "28"];
|
|
tokenize_assert("Thu 10:36:28", comp);
|
|
}
|
|
|
|
#[test]
|
|
fn test_tokenize45() {
|
|
let comp = vec!["Wed"];
|
|
tokenize_assert("Wed", comp);
|
|
}
|
|
|
|
#[test]
|
|
fn test_tokenize46() {
|
|
let comp = vec!["Wednesday"];
|
|
tokenize_assert("Wednesday", comp);
|
|
}
|
|
|
|
#[test]
|
|
fn test_tokenize47() {
|
|
let comp = vec![
|
|
"Thu", " ", "Sep", " ", "25", " ", "10", ":", "36", ":", "28", " ", "2003",
|
|
];
|
|
tokenize_assert("Thu Sep 25 10:36:28 2003", comp);
|
|
}
|
|
|
|
#[test]
|
|
fn test_tokenize48() {
|
|
let comp = vec!["Thu", " ", "Sep", " ", "25", " ", "2003"];
|
|
tokenize_assert("Thu Sep 25 2003", comp);
|
|
}
|
|
|
|
#[test]
|
|
fn test_tokenize49() {
|
|
let comp = vec![
|
|
"2003", "-", "09", "-", "25", "T", "10", ":", "49", ":", "41",
|
|
];
|
|
tokenize_assert("2003-09-25T10:49:41", comp);
|
|
}
|
|
|
|
#[test]
|
|
fn test_tokenize50() {
|
|
let comp = vec!["2003", "-", "09", "-", "25", "T", "10", ":", "49"];
|
|
tokenize_assert("2003-09-25T10:49", comp);
|
|
}
|
|
|
|
#[test]
|
|
fn test_tokenize51() {
|
|
let comp = vec!["2003", "-", "09", "-", "25", "T", "10"];
|
|
tokenize_assert("2003-09-25T10", comp);
|
|
}
|
|
|
|
#[test]
|
|
fn test_tokenize52() {
|
|
let comp = vec!["2003", "-", "09", "-", "25"];
|
|
tokenize_assert("2003-09-25", comp);
|
|
}
|
|
|
|
#[test]
|
|
fn test_tokenize53() {
|
|
let comp = vec!["20030925", "T", "104941"];
|
|
tokenize_assert("20030925T104941", comp);
|
|
}
|
|
|
|
#[test]
|
|
fn test_tokenize54() {
|
|
let comp = vec!["20030925", "T", "1049"];
|
|
tokenize_assert("20030925T1049", comp);
|
|
}
|
|
|
|
#[test]
|
|
fn test_tokenize55() {
|
|
let comp = vec!["20030925", "T", "10"];
|
|
tokenize_assert("20030925T10", comp);
|
|
}
|
|
|
|
#[test]
|
|
fn test_tokenize56() {
|
|
let comp = vec!["20030925"];
|
|
tokenize_assert("20030925", comp);
|
|
}
|
|
|
|
#[test]
|
|
fn test_tokenize57() {
|
|
let comp = vec![
|
|
"2003", "-", "09", "-", "25", " ", "10", ":", "49", ":", "41.502",
|
|
];
|
|
tokenize_assert("2003-09-25 10:49:41,502", comp);
|
|
}
|
|
|
|
#[test]
|
|
fn test_tokenize58() {
|
|
let comp = vec!["199709020908"];
|
|
tokenize_assert("199709020908", comp);
|
|
}
|
|
|
|
#[test]
|
|
fn test_tokenize59() {
|
|
let comp = vec!["19970902090807"];
|
|
tokenize_assert("19970902090807", comp);
|
|
}
|
|
|
|
#[test]
|
|
fn test_tokenize60() {
|
|
let comp = vec!["2003", "-", "09", "-", "25"];
|
|
tokenize_assert("2003-09-25", comp);
|
|
}
|
|
|
|
#[test]
|
|
fn test_tokenize61() {
|
|
let comp = vec!["09", "-", "25", "-", "2003"];
|
|
tokenize_assert("09-25-2003", comp);
|
|
}
|
|
|
|
#[test]
|
|
fn test_tokenize62() {
|
|
let comp = vec!["25", "-", "09", "-", "2003"];
|
|
tokenize_assert("25-09-2003", comp);
|
|
}
|
|
|
|
#[test]
|
|
fn test_tokenize63() {
|
|
let comp = vec!["10", "-", "09", "-", "2003"];
|
|
tokenize_assert("10-09-2003", comp);
|
|
}
|
|
|
|
#[test]
|
|
fn test_tokenize64() {
|
|
let comp = vec!["10", "-", "09", "-", "03"];
|
|
tokenize_assert("10-09-03", comp);
|
|
}
|
|
|
|
#[test]
|
|
fn test_tokenize65() {
|
|
let comp = vec!["2003", ".", "09", ".", "25"];
|
|
tokenize_assert("2003.09.25", comp);
|
|
}
|
|
|
|
#[test]
|
|
fn test_tokenize66() {
|
|
let comp = vec!["09", ".", "25", ".", "2003"];
|
|
tokenize_assert("09.25.2003", comp);
|
|
}
|
|
|
|
#[test]
|
|
fn test_tokenize67() {
|
|
let comp = vec!["25", ".", "09", ".", "2003"];
|
|
tokenize_assert("25.09.2003", comp);
|
|
}
|
|
|
|
#[test]
|
|
fn test_tokenize68() {
|
|
let comp = vec!["10", ".", "09", ".", "2003"];
|
|
tokenize_assert("10.09.2003", comp);
|
|
}
|
|
|
|
#[test]
|
|
fn test_tokenize69() {
|
|
let comp = vec!["10", ".", "09", ".", "03"];
|
|
tokenize_assert("10.09.03", comp);
|
|
}
|
|
|
|
#[test]
|
|
fn test_tokenize70() {
|
|
let comp = vec!["2003", "/", "09", "/", "25"];
|
|
tokenize_assert("2003/09/25", comp);
|
|
}
|
|
|
|
#[test]
|
|
fn test_tokenize71() {
|
|
let comp = vec!["09", "/", "25", "/", "2003"];
|
|
tokenize_assert("09/25/2003", comp);
|
|
}
|
|
|
|
#[test]
|
|
fn test_tokenize72() {
|
|
let comp = vec!["25", "/", "09", "/", "2003"];
|
|
tokenize_assert("25/09/2003", comp);
|
|
}
|
|
|
|
#[test]
|
|
fn test_tokenize73() {
|
|
let comp = vec!["10", "/", "09", "/", "2003"];
|
|
tokenize_assert("10/09/2003", comp);
|
|
}
|
|
|
|
#[test]
|
|
fn test_tokenize74() {
|
|
let comp = vec!["10", "/", "09", "/", "03"];
|
|
tokenize_assert("10/09/03", comp);
|
|
}
|
|
|
|
#[test]
|
|
fn test_tokenize75() {
|
|
let comp = vec!["2003", " ", "09", " ", "25"];
|
|
tokenize_assert("2003 09 25", comp);
|
|
}
|
|
|
|
#[test]
|
|
fn test_tokenize76() {
|
|
let comp = vec!["09", " ", "25", " ", "2003"];
|
|
tokenize_assert("09 25 2003", comp);
|
|
}
|
|
|
|
#[test]
|
|
fn test_tokenize77() {
|
|
let comp = vec!["25", " ", "09", " ", "2003"];
|
|
tokenize_assert("25 09 2003", comp);
|
|
}
|
|
|
|
#[test]
|
|
fn test_tokenize78() {
|
|
let comp = vec!["10", " ", "09", " ", "2003"];
|
|
tokenize_assert("10 09 2003", comp);
|
|
}
|
|
|
|
#[test]
|
|
fn test_tokenize79() {
|
|
let comp = vec!["10", " ", "09", " ", "03"];
|
|
tokenize_assert("10 09 03", comp);
|
|
}
|
|
|
|
#[test]
|
|
fn test_tokenize80() {
|
|
let comp = vec!["25", " ", "09", " ", "03"];
|
|
tokenize_assert("25 09 03", comp);
|
|
}
|
|
|
|
#[test]
|
|
fn test_tokenize81() {
|
|
let comp = vec!["03", " ", "25", " ", "Sep"];
|
|
tokenize_assert("03 25 Sep", comp);
|
|
}
|
|
|
|
#[test]
|
|
fn test_tokenize82() {
|
|
let comp = vec!["25", " ", "03", " ", "Sep"];
|
|
tokenize_assert("25 03 Sep", comp);
|
|
}
|
|
|
|
#[test]
|
|
fn test_tokenize83() {
|
|
let comp = vec![
|
|
" ", " ", "July", " ", " ", " ", "4", " ", ",", " ", " ", "1976", " ", " ", " ", "12", ":",
|
|
"01", ":", "02", " ", " ", " ", "am", " ", " ",
|
|
];
|
|
tokenize_assert(" July 4 , 1976 12:01:02 am ", comp);
|
|
}
|
|
|
|
#[test]
|
|
fn test_tokenize84() {
|
|
let comp = vec!["Wed", ",", " ", "July", " ", "10", ",", " ", "'", "96"];
|
|
tokenize_assert("Wed, July 10, '96", comp);
|
|
}
|
|
|
|
#[test]
|
|
fn test_tokenize85() {
|
|
let comp = vec![
|
|
"1996", ".", "July", ".", "10", " ", "AD", " ", "12", ":", "08", " ", "PM",
|
|
];
|
|
tokenize_assert("1996.July.10 AD 12:08 PM", comp);
|
|
}
|
|
|
|
#[test]
|
|
fn test_tokenize86() {
|
|
let comp = vec!["July", " ", "4", ",", " ", "1976"];
|
|
tokenize_assert("July 4, 1976", comp);
|
|
}
|
|
|
|
#[test]
|
|
fn test_tokenize87() {
|
|
let comp = vec!["7", " ", "4", " ", "1976"];
|
|
tokenize_assert("7 4 1976", comp);
|
|
}
|
|
|
|
#[test]
|
|
fn test_tokenize88() {
|
|
let comp = vec!["4", " ", "jul", " ", "1976"];
|
|
tokenize_assert("4 jul 1976", comp);
|
|
}
|
|
|
|
#[test]
|
|
fn test_tokenize89() {
|
|
let comp = vec!["7", "-", "4", "-", "76"];
|
|
tokenize_assert("7-4-76", comp);
|
|
}
|
|
|
|
#[test]
|
|
fn test_tokenize90() {
|
|
let comp = vec!["19760704"];
|
|
tokenize_assert("19760704", comp);
|
|
}
|
|
|
|
#[test]
|
|
fn test_tokenize91() {
|
|
let comp = vec![
|
|
"0", ":", "01", ":", "02", " ", "on", " ", "July", " ", "4", ",", " ", "1976",
|
|
];
|
|
tokenize_assert("0:01:02 on July 4, 1976", comp);
|
|
}
|
|
|
|
#[test]
|
|
fn test_tokenize92() {
|
|
let comp = vec![
|
|
"0", ":", "01", ":", "02", " ", "on", " ", "July", " ", "4", ",", " ", "1976",
|
|
];
|
|
tokenize_assert("0:01:02 on July 4, 1976", comp);
|
|
}
|
|
|
|
#[test]
|
|
fn test_tokenize93() {
|
|
let comp = vec![
|
|
"July", " ", "4", ",", " ", "1976", " ", "12", ":", "01", ":", "02", " ", "am",
|
|
];
|
|
tokenize_assert("July 4, 1976 12:01:02 am", comp);
|
|
}
|
|
|
|
#[test]
|
|
fn test_tokenize94() {
|
|
let comp = vec![
|
|
"Mon", " ", "Jan", " ", " ", "2", " ", "04", ":", "24", ":", "27", " ", "1995",
|
|
];
|
|
tokenize_assert("Mon Jan 2 04:24:27 1995", comp);
|
|
}
|
|
|
|
#[test]
|
|
fn test_tokenize95() {
|
|
let comp = vec!["04", ".", "04", ".", "95", " ", "00", ":", "22"];
|
|
tokenize_assert("04.04.95 00:22", comp);
|
|
}
|
|
|
|
#[test]
|
|
fn test_tokenize96() {
|
|
let comp = vec![
|
|
"Jan", " ", "1", " ", "1999", " ", "11", ":", "23", ":", "34.578",
|
|
];
|
|
tokenize_assert("Jan 1 1999 11:23:34.578", comp);
|
|
}
|
|
|
|
#[test]
|
|
fn test_tokenize97() {
|
|
let comp = vec!["950404", " ", "122212"];
|
|
tokenize_assert("950404 122212", comp);
|
|
}
|
|
|
|
#[test]
|
|
fn test_tokenize98() {
|
|
let comp = vec!["3", "rd", " ", "of", " ", "May", " ", "2001"];
|
|
tokenize_assert("3rd of May 2001", comp);
|
|
}
|
|
|
|
#[test]
|
|
fn test_tokenize99() {
|
|
let comp = vec!["5", "th", " ", "of", " ", "March", " ", "2001"];
|
|
tokenize_assert("5th of March 2001", comp);
|
|
}
|
|
|
|
#[test]
|
|
fn test_tokenize100() {
|
|
let comp = vec!["1", "st", " ", "of", " ", "May", " ", "2003"];
|
|
tokenize_assert("1st of May 2003", comp);
|
|
}
|
|
|
|
#[test]
|
|
fn test_tokenize101() {
|
|
let comp = vec![
|
|
"0099", "-", "01", "-", "01", "T", "00", ":", "00", ":", "00",
|
|
];
|
|
tokenize_assert("0099-01-01T00:00:00", comp);
|
|
}
|
|
|
|
#[test]
|
|
fn test_tokenize102() {
|
|
let comp = vec![
|
|
"0031", "-", "01", "-", "01", "T", "00", ":", "00", ":", "00",
|
|
];
|
|
tokenize_assert("0031-01-01T00:00:00", comp);
|
|
}
|
|
|
|
#[test]
|
|
fn test_tokenize103() {
|
|
let comp = vec!["20080227", "T", "21", ":", "26", ":", "01.123456789"];
|
|
tokenize_assert("20080227T21:26:01.123456789", comp);
|
|
}
|
|
|
|
#[test]
|
|
fn test_tokenize104() {
|
|
let comp = vec!["13", "NOV", "2017"];
|
|
tokenize_assert("13NOV2017", comp);
|
|
}
|
|
|
|
#[test]
|
|
fn test_tokenize105() {
|
|
let comp = vec!["0003", "-", "03", "-", "04"];
|
|
tokenize_assert("0003-03-04", comp);
|
|
}
|
|
|
|
#[test]
|
|
fn test_tokenize106() {
|
|
let comp = vec!["December", ".", "0031", ".", "30"];
|
|
tokenize_assert("December.0031.30", comp);
|
|
}
|
|
|
|
#[test]
|
|
fn test_tokenize107() {
|
|
let comp = vec!["090107"];
|
|
tokenize_assert("090107", comp);
|
|
}
|
|
|
|
#[test]
|
|
fn test_tokenize108() {
|
|
let comp = vec!["2015", "-", "15", "-", "May"];
|
|
tokenize_assert("2015-15-May", comp);
|
|
}
|
|
|
|
#[test]
|
|
fn test_tokenize109() {
|
|
let comp = vec![
|
|
"Thu", " ", "Sep", " ", "25", " ", "10", ":", "36", ":", "28", " ", "BRST", " ", "2003",
|
|
];
|
|
tokenize_assert("Thu Sep 25 10:36:28 BRST 2003", comp);
|
|
}
|
|
|
|
#[test]
|
|
fn test_tokenize110() {
|
|
let comp = vec![
|
|
"2003", " ", "10", ":", "36", ":", "28", " ", "BRST", " ", "25", " ", "Sep", " ", "Thu",
|
|
];
|
|
tokenize_assert("2003 10:36:28 BRST 25 Sep Thu", comp);
|
|
}
|
|
|
|
#[test]
|
|
fn test_tokenize111() {
|
|
let comp = vec![
|
|
"Thu", ",", " ", "25", " ", "Sep", " ", "2003", " ", "10", ":", "49", ":", "41", " ", "-",
|
|
"0300",
|
|
];
|
|
tokenize_assert("Thu, 25 Sep 2003 10:49:41 -0300", comp);
|
|
}
|
|
|
|
#[test]
|
|
fn test_tokenize112() {
|
|
let comp = vec![
|
|
"2003", "-", "09", "-", "25", "T", "10", ":", "49", ":", "41.5", "-", "03", ":", "00",
|
|
];
|
|
tokenize_assert("2003-09-25T10:49:41.5-03:00", comp);
|
|
}
|
|
|
|
#[test]
|
|
fn test_tokenize113() {
|
|
let comp = vec![
|
|
"2003", "-", "09", "-", "25", "T", "10", ":", "49", ":", "41", "-", "03", ":", "00",
|
|
];
|
|
tokenize_assert("2003-09-25T10:49:41-03:00", comp);
|
|
}
|
|
|
|
#[test]
|
|
fn test_tokenize114() {
|
|
let comp = vec!["20030925", "T", "104941.5", "-", "0300"];
|
|
tokenize_assert("20030925T104941.5-0300", comp);
|
|
}
|
|
|
|
#[test]
|
|
fn test_tokenize115() {
|
|
let comp = vec!["20030925", "T", "104941", "-", "0300"];
|
|
tokenize_assert("20030925T104941-0300", comp);
|
|
}
|
|
|
|
#[test]
|
|
fn test_tokenize116() {
|
|
let comp = vec![
|
|
"2018", "-", "08", "-", "10", " ", "10", ":", "00", ":", "00", " ", "UTC", "+", "3",
|
|
];
|
|
tokenize_assert("2018-08-10 10:00:00 UTC+3", comp);
|
|
}
|
|
|
|
#[test]
|
|
fn test_tokenize117() {
|
|
let comp = vec![
|
|
"2018", "-", "08", "-", "10", " ", "03", ":", "36", ":", "47", " ", "PM", " ", "GMT", "-",
|
|
"4",
|
|
];
|
|
tokenize_assert("2018-08-10 03:36:47 PM GMT-4", comp);
|
|
}
|
|
|
|
#[test]
|
|
fn test_tokenize118() {
|
|
let comp = vec![
|
|
"2018", "-", "08", "-", "10", " ", "04", ":", "15", ":", "00", " ", "AM", " ", "Z", "-",
|
|
"02", ":", "00",
|
|
];
|
|
tokenize_assert("2018-08-10 04:15:00 AM Z-02:00", comp);
|
|
}
|
|
|
|
#[test]
|
|
fn test_tokenize119() {
|
|
let comp = vec!["10", "-", "09", "-", "2003"];
|
|
tokenize_assert("10-09-2003", comp);
|
|
}
|
|
|
|
#[test]
|
|
fn test_tokenize120() {
|
|
let comp = vec!["10", ".", "09", ".", "2003"];
|
|
tokenize_assert("10.09.2003", comp);
|
|
}
|
|
|
|
#[test]
|
|
fn test_tokenize121() {
|
|
let comp = vec!["10", "/", "09", "/", "2003"];
|
|
tokenize_assert("10/09/2003", comp);
|
|
}
|
|
|
|
#[test]
|
|
fn test_tokenize122() {
|
|
let comp = vec!["10", " ", "09", " ", "2003"];
|
|
tokenize_assert("10 09 2003", comp);
|
|
}
|
|
|
|
#[test]
|
|
fn test_tokenize123() {
|
|
let comp = vec!["090107"];
|
|
tokenize_assert("090107", comp);
|
|
}
|
|
|
|
#[test]
|
|
fn test_tokenize124() {
|
|
let comp = vec!["2015", " ", "09", " ", "25"];
|
|
tokenize_assert("2015 09 25", comp);
|
|
}
|
|
|
|
#[test]
|
|
fn test_tokenize125() {
|
|
let comp = vec!["10", "-", "09", "-", "03"];
|
|
tokenize_assert("10-09-03", comp);
|
|
}
|
|
|
|
#[test]
|
|
fn test_tokenize126() {
|
|
let comp = vec!["10", ".", "09", ".", "03"];
|
|
tokenize_assert("10.09.03", comp);
|
|
}
|
|
|
|
#[test]
|
|
fn test_tokenize127() {
|
|
let comp = vec!["10", "/", "09", "/", "03"];
|
|
tokenize_assert("10/09/03", comp);
|
|
}
|
|
|
|
#[test]
|
|
fn test_tokenize128() {
|
|
let comp = vec!["10", " ", "09", " ", "03"];
|
|
tokenize_assert("10 09 03", comp);
|
|
}
|
|
|
|
#[test]
|
|
fn test_tokenize129() {
|
|
let comp = vec!["090107"];
|
|
tokenize_assert("090107", comp);
|
|
}
|
|
|
|
#[test]
|
|
fn test_tokenize130() {
|
|
let comp = vec!["2015", " ", "09", " ", "25"];
|
|
tokenize_assert("2015 09 25", comp);
|
|
}
|
|
|
|
#[test]
|
|
fn test_tokenize131() {
|
|
let comp = vec!["090107"];
|
|
tokenize_assert("090107", comp);
|
|
}
|
|
|
|
#[test]
|
|
fn test_tokenize132() {
|
|
let comp = vec!["2015", " ", "09", " ", "25"];
|
|
tokenize_assert("2015 09 25", comp);
|
|
}
|
|
|
|
#[test]
|
|
fn test_tokenize133() {
|
|
let comp = vec!["April", " ", "2009"];
|
|
tokenize_assert("April 2009", comp);
|
|
}
|
|
|
|
#[test]
|
|
fn test_tokenize134() {
|
|
let comp = vec!["Feb", " ", "2007"];
|
|
tokenize_assert("Feb 2007", comp);
|
|
}
|
|
|
|
#[test]
|
|
fn test_tokenize135() {
|
|
let comp = vec!["Feb", " ", "2008"];
|
|
tokenize_assert("Feb 2008", comp);
|
|
}
|
|
|
|
#[test]
|
|
fn test_tokenize136() {
|
|
let comp = vec![
|
|
"Thu", " ", "Sep", " ", "25", " ", "10", ":", "36", ":", "28", " ", "BRST", " ", "2003",
|
|
];
|
|
tokenize_assert("Thu Sep 25 10:36:28 BRST 2003", comp);
|
|
}
|
|
|
|
#[test]
|
|
fn test_tokenize137() {
|
|
let comp = vec![
|
|
"1996", ".", "07", ".", "10", " ", "AD", " ", "at", " ", "15", ":", "08", ":", "56", " ",
|
|
"PDT",
|
|
];
|
|
tokenize_assert("1996.07.10 AD at 15:08:56 PDT", comp);
|
|
}
|
|
|
|
#[test]
|
|
fn test_tokenize138() {
|
|
let comp = vec![
|
|
"Tuesday", ",", " ", "April", " ", "12", ",", " ", "1952", " ", "AD", " ", "3", ":", "30",
|
|
":", "42", "pm", " ", "PST",
|
|
];
|
|
tokenize_assert("Tuesday, April 12, 1952 AD 3:30:42pm PST", comp);
|
|
}
|
|
|
|
#[test]
|
|
fn test_tokenize139() {
|
|
let comp = vec![
|
|
"November", " ", "5", ",", " ", "1994", ",", " ", "8", ":", "15", ":", "30", " ", "am",
|
|
" ", "EST",
|
|
];
|
|
tokenize_assert("November 5, 1994, 8:15:30 am EST", comp);
|
|
}
|
|
|
|
#[test]
|
|
fn test_tokenize140() {
|
|
let comp = vec![
|
|
"1994", "-", "11", "-", "05", "T", "08", ":", "15", ":", "30", "-", "05", ":", "00",
|
|
];
|
|
tokenize_assert("1994-11-05T08:15:30-05:00", comp);
|
|
}
|
|
|
|
#[test]
|
|
fn test_tokenize141() {
|
|
let comp = vec![
|
|
"1994", "-", "11", "-", "05", "T", "08", ":", "15", ":", "30", "Z",
|
|
];
|
|
tokenize_assert("1994-11-05T08:15:30Z", comp);
|
|
}
|
|
|
|
#[test]
|
|
fn test_tokenize142() {
|
|
let comp = vec![
|
|
"1976", "-", "07", "-", "04", "T", "00", ":", "01", ":", "02", "Z",
|
|
];
|
|
tokenize_assert("1976-07-04T00:01:02Z", comp);
|
|
}
|
|
|
|
#[test]
|
|
fn test_tokenize143() {
|
|
let comp = vec![
|
|
"Tue", " ", "Apr", " ", "4", " ", "00", ":", "22", ":", "12", " ", "PDT", " ", "1995",
|
|
];
|
|
tokenize_assert("Tue Apr 4 00:22:12 PDT 1995", comp);
|
|
}
|
|
|
|
#[test]
|
|
fn test_tokenize144() {
|
|
let comp = vec![
|
|
"Today",
|
|
" ",
|
|
"is",
|
|
" ",
|
|
"25",
|
|
" ",
|
|
"of",
|
|
" ",
|
|
"September",
|
|
" ",
|
|
"of",
|
|
" ",
|
|
"2003",
|
|
",",
|
|
" ",
|
|
"exactly",
|
|
" ",
|
|
"at",
|
|
" ",
|
|
"10",
|
|
":",
|
|
"49",
|
|
":",
|
|
"41",
|
|
" ",
|
|
"with",
|
|
" ",
|
|
"timezone",
|
|
" ",
|
|
"-",
|
|
"03",
|
|
":",
|
|
"00",
|
|
".",
|
|
];
|
|
tokenize_assert(
|
|
"Today is 25 of September of 2003, exactly at 10:49:41 with timezone -03:00.",
|
|
comp,
|
|
);
|
|
}
|
|
|
|
#[test]
|
|
fn test_tokenize145() {
|
|
let comp = vec![
|
|
"Today",
|
|
" ",
|
|
"is",
|
|
" ",
|
|
"25",
|
|
" ",
|
|
"of",
|
|
" ",
|
|
"September",
|
|
" ",
|
|
"of",
|
|
" ",
|
|
"2003",
|
|
",",
|
|
" ",
|
|
"exactly",
|
|
" ",
|
|
"at",
|
|
" ",
|
|
"10",
|
|
":",
|
|
"49",
|
|
":",
|
|
"41",
|
|
" ",
|
|
"with",
|
|
" ",
|
|
"timezone",
|
|
" ",
|
|
"-",
|
|
"03",
|
|
":",
|
|
"00",
|
|
".",
|
|
];
|
|
tokenize_assert(
|
|
"Today is 25 of September of 2003, exactly at 10:49:41 with timezone -03:00.",
|
|
comp,
|
|
);
|
|
}
|
|
|
|
#[test]
|
|
fn test_tokenize146() {
|
|
let comp = vec![
|
|
"I", " ", "have", " ", "a", " ", "meeting", " ", "on", " ", "March", " ", "1", ",", " ",
|
|
"1974",
|
|
];
|
|
tokenize_assert("I have a meeting on March 1, 1974", comp);
|
|
}
|
|
|
|
#[test]
|
|
fn test_tokenize147() {
|
|
let comp = vec![
|
|
"On", " ", "June", " ", "8", "th", ",", " ", "2020", ",", " ", "I", " ", "am", " ",
|
|
"going", " ", "to", " ", "be", " ", "the", " ", "first", " ", "man", " ", "on", " ",
|
|
"Mars",
|
|
];
|
|
tokenize_assert(
|
|
"On June 8th, 2020, I am going to be the first man on Mars",
|
|
comp,
|
|
);
|
|
}
|
|
|
|
#[test]
|
|
fn test_tokenize148() {
|
|
let comp = vec![
|
|
"Meet", " ", "me", " ", "at", " ", "the", " ", "AM", "/", "PM", " ", "on", " ", "Sunset",
|
|
" ", "at", " ", "3", ":", "00", " ", "AM", " ", "on", " ", "December", " ", "3", "rd", ",",
|
|
" ", "2003",
|
|
];
|
|
tokenize_assert(
|
|
"Meet me at the AM/PM on Sunset at 3:00 AM on December 3rd, 2003",
|
|
comp,
|
|
);
|
|
}
|
|
|
|
#[test]
|
|
fn test_tokenize149() {
|
|
let comp = vec![
|
|
"Meet", " ", "me", " ", "at", " ", "3", ":", "00", " ", "AM", " ", "on", " ", "December",
|
|
" ", "3", "rd", ",", " ", "2003", " ", "at", " ", "the", " ", "AM", "/", "PM", " ", "on",
|
|
" ", "Sunset",
|
|
];
|
|
tokenize_assert(
|
|
"Meet me at 3:00 AM on December 3rd, 2003 at the AM/PM on Sunset",
|
|
comp,
|
|
);
|
|
}
|
|
|
|
#[test]
|
|
fn test_tokenize150() {
|
|
let comp = vec![
|
|
"Jan", " ", "29", ",", " ", "1945", " ", "14", ":", "45", " ", "AM", " ", "I", " ",
|
|
"going", " ", "to", " ", "see", " ", "you", " ", "there", "?",
|
|
];
|
|
tokenize_assert("Jan 29, 1945 14:45 AM I going to see you there?", comp);
|
|
}
|
|
|
|
#[test]
|
|
fn test_tokenize151() {
|
|
let comp = vec!["2017", "-", "07", "-", "17", " ", "06", ":", "15", ":"];
|
|
tokenize_assert("2017-07-17 06:15:", comp);
|
|
}
|