use talc_lang::parser::{Lexer, TokenKind}; use TokenKind as T; fn assert_tokens(src: &str, tokens: &[(TokenKind, &str)]) { let lexer = Lexer::new(src); for (i, tok) in lexer.enumerate() { let tok = if i >= tokens.len() { tok.expect("end of tokens") } else { tok.expect(&format!("token {} {}", tokens[i].0, tokens[i].1)) }; assert_eq!(tok.kind, tokens[i].0, "token kind did not match"); assert_eq!(tok.content, tokens[i].1, "token content did not match"); if tok.kind == TokenKind::Eof { break } } } fn assert_error(src: &str) { let lexer = Lexer::new(src); for tok in lexer { match tok { Err(_) => return, Ok(t) if t.kind == T::Eof => break, _ => (), } } panic!("expected error in source '{}'", src) } #[test] fn int_literals() { let src = "1 100 98_765_432 0x0 0x551 0x_12_34 0xabcABC 0o127 0b01100110 0s012345"; let tokens = vec![ (T::Integer, "1"), (T::Integer, "100"), (T::Integer, "98_765_432"), (T::Integer, "0x0"), (T::Integer, "0x551"), (T::Integer, "0x_12_34"), (T::Integer, "0xabcABC"), (T::Integer, "0o127"), (T::Integer, "0b01100110"), (T::Integer, "0s012345"), (T::Eof, ""), ]; assert_tokens(src, &tokens); assert_error("0m123"); assert_error("55p"); assert_error("0xabcdefg"); assert_error("0o178"); assert_error("0s156"); assert_error("0b012"); } #[test] fn float_literals() { let src = "1. 1.0 1e2 1.e2 1.0e2 1e+2 1.e+2 1.0e+2 1e-2 1___2_3_e+_5__"; let tokens = vec![ (T::Float, "1."), (T::Float, "1.0"), (T::Float, "1e2"), (T::Float, "1.e2"), (T::Float, "1.0e2"), (T::Float, "1e+2"), (T::Float, "1.e+2"), (T::Float, "1.0e+2"), (T::Float, "1e-2"), (T::Float, "1___2_3_e+_5__"), (T::Eof, ""), ]; assert_tokens(src, &tokens); }