3 releases
0.1.3 | Feb 26, 2023 |
---|---|
0.1.2 | Feb 26, 2023 |
0.1.1 | Jan 23, 2023 |
0.1.0 |
|
#791 in Development tools
26KB
566 lines
uscan
A universal source code scanner
/!\ work-in-progress
features
- configurable keywords, symbols and comments
- handles nested multi-line comments
- handles decimal (15), hexadecimal (0xf or 0xF) and binary (0b1111) literal numbers
usage
const LUA_CONFIG: ScannerConfig = ScannerConfig {
keywords: &[
"and", "break", "do", "else", "elseif", "end", "false", "for", "function", "if", "in",
"local", "nil", "not", "or", "repeat", "return", "then", "true", "until", "while",
],
symbols: &[
"...", "..", "==", "~=", "<=", ">=", "+", "-", "*", "/", "%", "^", "#", "<", ">", "=", "(",
")", "{", "}", "[", "]", ";", ":", ",", ".",
],
single_line_cmt: Some("--"),
multi_line_cmt_start: Some("--[["),
multi_line_cmt_end: Some("]]"),
};
let mut scanner_data = ScannerData::default();
let mut scanner = Scanner::default();
scanner.run(source_code, &LUA_CONFIG, &mut scanner_data)?;
=> you can now use the ScannerData struct in your parser to build your AST :
pub enum TokenType {
Symbol(String),
Identifier(String),
StringLiteral(String),
NumberLiteral(String, Number),
Keyword(String),
Comment(String),
// space
Ignore,
NewLine,
Eof,
Unknown,
}
pub struct ScannerData {
/// complete source code
pub source: Vec<char>,
/// resulting list of tokens
pub token_types: Vec<TokenType>,
/// token start line in the source code
pub token_lines: Vec<usize>,
/// token start offset from its line beginning
pub token_start: Vec<usize>,
/// token length in characters
/// not always = token value's length.
/// for example for TokenType::StringLiteral("aa") the value length is 2 but the token length including the quotes is 4
pub token_len: Vec<usize>,
}