生命周期：让 token 借用源码

1
[package]
2
name = "pebble-lexer"
3
description = "A zero-copy lexer for the Pebble language."
4
version.workspace = true
5
edition.workspace = true
6
rust-version.workspace = true
7
authors.workspace = true
8
license.workspace = true
9
repository.workspace = true
10

11
[dependencies]
12
pebble-span.workspace = true
13
pebble-diag.workspace = true
14

15
[lints]
16
workspace = true

1
struct Token<'src> {
2
    text: &'src str,
3
}
4

5
fn first_word(src: &str) -> Token<'_> {
6
    Token { text: src.split(' ').next().unwrap_or("") }
7
}
8

9
fn main() {
10
    let token;
11
    {
12
        let source = String::from("let answer = 42");
13
        token = first_word(&source);
14
    }
15
    println!("{}", token.text);
16
}

1
use pebble_diag::Diagnostic;
2
use pebble_span::{SourceId, Span};
3

4
#[derive(Clone, Debug, PartialEq)]
5
pub enum TokenKind<'src> {
6
    Ident(&'src str),
7
    Int(i64),
8
    LParen,
9
    RParen,
10
    Plus,
11
    Eof,
12
}
13

14
#[derive(Clone, Debug, PartialEq)]
15
pub struct Token<'src> {
16
    pub kind: TokenKind<'src>,
17
    pub span: Span,
18
}
19

20
pub struct Lexer<'src> {
21
    source: SourceId,
22
    src: &'src str,
23
    pos: usize,
24
}
25

26
impl<'src> Lexer<'src> {
27
    pub const fn new(source: SourceId, src: &'src str) -> Self {
28
        Self { source, src, pos: 0 }
29
    }
30

31
    fn rest(&self) -> &'src str {
32
        &self.src[self.pos..]
33
    }
34

35
    fn peek(&self) -> Option<char> {
36
        self.rest().chars().next()
37
    }
38

39
    fn bump(&mut self) -> Option<char> {
40
        let ch = self.peek()?;
41
        self.pos += ch.len_utf8();
42
        Some(ch)
43
    }
44

45
    fn span_from(&self, start: usize) -> Span {
46
        Span::new(self.source, start as u32, self.pos as u32)
47
    }
48

49
    pub fn next_token(&mut self) -> Option<Result<Token<'src>, Diagnostic>> {
50
        while self.peek().is_some_and(char::is_whitespace) {
51
            self.bump();
52
        }
53
        let start = self.pos;
54
        let ch = self.bump()?;
55
        let kind = match ch {
56
            '(' => TokenKind::LParen,
57
            ')' => TokenKind::RParen,
58
            '+' => TokenKind::Plus,
59
            '0'..='9' => {
60
                while self.peek().is_some_and(|c| c.is_ascii_digit()) {
61
                    self.bump();
62
                }
63
                TokenKind::Int(self.src[start..self.pos].parse().unwrap())
64
            }
65
            ch if ch == '_' || ch.is_alphabetic() => {
66
                while self.peek().is_some_and(|c| c == '_' || c.is_alphanumeric()) {
67
                    self.bump();
68
                }
69
                TokenKind::Ident(&self.src[start..self.pos])
70
            }
71
            other => {
72
                return Some(Err(Diagnostic::error(format!("unexpected character `{other}`"))
73
                    .with_span(self.span_from(start))));
74
            }
75
        };
76
        Some(Ok(Token { kind, span: self.span_from(start) }))
77
    }
78
}
79

80
#[cfg(test)]
81
mod tests {
82
    use super::*;
83

84
    #[test]
85
    fn identifiers_borrow_the_source() {
86
        let source = String::from("hello");
87
        let mut lexer = Lexer::new(SourceId(0), &source);
88
        let token = lexer.next_token().unwrap().unwrap();
89
        match token.kind {
90
            TokenKind::Ident(name) => {
91
                let base = source.as_ptr() as usize;
92
                let borrowed = name.as_ptr() as usize;
93
                assert!((base..base + source.len()).contains(&borrowed));
94
            }
95
            other => panic!("expected identifier, got {other:?}"),
96
        }
97
    }
98

99
    #[test]
100
    fn a_token_outlives_its_lexer() {
101
        let source = String::from("answer + 42");
102
        let first = {
103
            let mut lexer = Lexer::new(SourceId(0), &source);
104
            lexer.next_token().unwrap().unwrap()
105
        };
106
        assert_eq!(first.kind, TokenKind::Ident("answer"));
107
    }
108
}

1
#[cfg(test)]
2
mod tests {
3
    use super::*;
4

5
    #[test]
6
    fn identifiers_borrow_the_source() {
7
        let source = String::from("hello");
8
        let mut lexer = Lexer::new(SourceId(0), &source);
9
        let token = lexer.next_token().unwrap().unwrap();
10
        match token.kind {
11
            TokenKind::Ident(name) => {
12
                let base = source.as_ptr() as usize;
13
                let borrowed = name.as_ptr() as usize;
14
                assert!((base..base + source.len()).contains(&borrowed));
15
            }
16
            other => panic!("expected identifier, got {other:?}"),
17
        }
18
    }
19
}

1
use pebble_lexer::Lexer;
2
use pebble_span::SourceId;
3

4
fn main() {
5
    let mut lexer = Lexer::new(SourceId(0), "total + (x1 + 42)");
6
    while let Some(item) = lexer.next_token() {
7
        println!("{item:?}");
8
    }
9
    let mut lexer = Lexer::new(SourceId(0), "a @ b");
10
    while let Some(item) = lexer.next_token() {
11
        match item {
12
            Ok(token) => println!("ok  {:?}", token.kind),
13
            Err(diagnostic) => println!("err {}", diagnostic.message),
14
        }
15
    }
16
}

1
struct Lexer<'src> {
2
    src: &'src str,
3
    pos: usize,
4
}
5

6
impl<'src> Lexer<'src> {
7
    // Elided: the returned &str is tied to the &mut self borrow, not to 'src.
8
    fn next_word(&mut self) -> &str {
9
        let rest = &self.src[self.pos..];
10
        let end = rest.find(' ').unwrap_or(rest.len());
11
        self.pos += (end + 1).min(rest.len());
12
        &rest[..end]
13
    }
14
}
15

16
fn main() {
17
    let mut lexer = Lexer { src: "let answer", pos: 0 };
18
    let a = lexer.next_word();
19
    let b = lexer.next_word();
20
    println!("{a} {b}");
21
}

1
struct Lexer<'src> {
2
    src: &'src str,
3
    pos: usize,
4
}
5

6
impl<'src> Lexer<'src> {
7
    // Named: the result borrows the source, not the lexer.
8
    fn next_word(&mut self) -> &'src str {
9
        let rest = &self.src[self.pos..];
10
        let end = rest.find(' ').unwrap_or(rest.len());
11
        self.pos += (end + 1).min(rest.len());
12
        &rest[..end]
13
    }
14
}
15

16
fn main() {
17
    let mut lexer = Lexer { src: "let answer", pos: 0 };
18
    let a = lexer.next_word();
19
    let b = lexer.next_word();
20
    println!("{a} {b}");
21
}

1
error: lifetime may not live long enough
2
   --> crates/pebble-lexer/src/lib.rs:504:9
3
    |
4
500 | impl<'src> Iterator for Lexer<'src> {
5
    |      ---- lifetime `'src` defined here
6
...
7
503 |     fn next(&mut self) -> Option<Self::Item> {
8
    |             - let's call the lifetime of this reference `'1`
9
504 |         self.next_token()
10
    |         ^^^^^^^^^^^^^^^^^ method was supposed to return data with lifetime `'src` but it is returning data with lifetime `'1`
11

12
error[E0499]: cannot borrow `lexer` as mutable more than once at a time
13
   --> crates/pebble-lexer/src/lib.rs:517:28
14
    |
15
513 | pub fn tokenize(source: SourceId, src: &str) -> (Vec<Token<'_>>, Vec<Diagnostic>) {
16
    |                                        - let's call the lifetime of this reference `'1`
17
...
18
517 |     while let Some(item) = lexer.next_token() {
19
    |                            ^^^^^ `lexer` was mutably borrowed here in the previous iteration of the loop
20
...
21
529 |     (tokens, diagnostics)
22
    |     --------------------- returning this value requires that `lexer` is borrowed for `'1`
23

24
For more information about this error, try `rustc --explain E0499`.
25
error: could not compile `pebble-lexer` (lib) due to 2 previous errors

1
#![cfg_attr(not(test), no_std)]
2
#![forbid(unsafe_code)]
3
//! A hand-written, zero-copy lexer for Pebble.
4
//!
5
//! Two design decisions shape this crate:
6
//!
7
//! 1. **Zero-copy.** An identifier token does not own a `String`; it borrows a
8
//!    slice of the original source (`&'src str`). The token stream is therefore
9
//!    a set of views into the input, and lexing allocates nothing except in the
10
//!    rare case of a string literal that contains escape sequences.
11
//! 2. **It keeps going.** A bad character produces a [`Diagnostic`] and the
12
//!    lexer resumes at the next character, so one run can report every typo in
13
//!    a file instead of just the first.
14

15
extern crate alloc;
16

17
use alloc::borrow::Cow;
18
use alloc::format;
19
use alloc::string::String;
20
use alloc::vec::Vec;
21
use core::fmt;
22

23
use pebble_diag::Diagnostic;
24
use pebble_span::{SourceId, Span};

1
/// The kind of a token, plus any payload it carries.
2
///
3
/// The `&'src str` payloads tie a token to the source string it came from: a
4
/// [`Token`] may outlive the [`Lexer`] but never the input, which is exactly
5
/// the guarantee we want. `Clone` (but not `Copy`) because [`Cow`] may own data.
6
#[derive(Clone, Debug, PartialEq)]
7
pub enum TokenKind<'src> {
8
    // --- literals -----------------------------------------------------------
9
    /// `123`
10
    Int(i64),
11
    /// `1.5`, `0.25`
12
    Float(f64),
13
    /// `"..."`, with escapes already resolved.
14
    Str(Cow<'src, str>),
15
    /// `true`
16
    True,
17
    /// `false`
18
    False,
19
    /// `nil`
20
    Nil,
21

22
    // --- identifiers and keywords ------------------------------------------
23
    /// `foo`, `_bar`
24
    Ident(&'src str),
25
    /// `let`
26
    Let,
27
    /// `fn`
28
    Fn,
29
    /// `if`
30
    If,
31
    /// `else`
32
    Else,
33
    /// `while`
34
    While,
35
    /// `for`
36
    For,
37
    /// `in`
38
    In,
39
    /// `return`
40
    Return,
41
    /// `break`
42
    Break,
43
    /// `continue`
44
    Continue,
45
    /// `and` or `&&`
46
    And,
47
    /// `or` or `||`
48
    Or,
49
    /// `not` or `!`
50
    Not,
51

52
    // --- punctuation --------------------------------------------------------
53
    /// `(`
54
    LParen,
55
    /// `)`
56
    RParen,
57
    /// `{`
58
    LBrace,
59
    /// `}`
60
    RBrace,
61
    /// `[`
62
    LBracket,
63
    /// `]`
64
    RBracket,
65
    /// `#{`, the start of a map literal
66
    HashBrace,
67
    /// `,`
68
    Comma,
69
    /// `:`
70
    Colon,
71
    /// `;`
72
    Semicolon,
73
    /// `+`
74
    Plus,
75
    /// `-`
76
    Minus,
77
    /// `*`
78
    Star,
79
    /// `/`
80
    Slash,
81
    /// `%`
82
    Percent,
83
    /// `=`
84
    Assign,
85
    /// `==`
86
    Eq,
87
    /// `!=`
88
    NotEq,
89
    /// `<`
90
    Lt,
91
    /// `<=`
92
    LtEq,
93
    /// `>`
94
    Gt,
95
    /// `>=`
96
    GtEq,
97

98
    /// Synthetic token emitted once at the end of input.
99
    Eof,
100
}

1
/// A token: a kind plus where it came from.
2
#[derive(Clone, Debug, PartialEq)]
3
pub struct Token<'src> {
4
    pub kind: TokenKind<'src>,
5
    pub span: Span,
6
}
7

8
/// The zero-copy lexer.
9
///
10
/// It borrows the source for its whole lifetime, which is why the tokens it
11
/// produces can borrow from the same string.
12
pub struct Lexer<'src> {
13
    source: SourceId,
14
    src: &'src str,
15
    pos: usize,
16
    done: bool,
17
}

1
impl<'src> Lexer<'src> {
2
    /// Creates a lexer over `src`, tagging every span with `source`.
3
    #[must_use]
4
    pub const fn new(source: SourceId, src: &'src str) -> Self {
5
        Self {
6
            source,
7
            src,
8
            pos: 0,
9
            done: false,
10
        }
11
    }
12

13
    /// Byte offset the lexer has reached.
14
    #[must_use]
15
    pub const fn position(&self) -> usize {
16
        self.pos
17
    }
18

19
    /// Whether the underlying input has been exhausted.
20
    #[must_use]
21
    pub const fn is_at_end(&self) -> bool {
22
        self.pos >= self.src.len()
23
    }

1
    fn rest(&self) -> &'src str {
2
        &self.src[self.pos..]
3
    }
4

5
    fn peek(&self) -> Option<char> {
6
        self.rest().chars().next()
7
    }
8

9
    fn peek_second(&self) -> Option<char> {
10
        let mut chars = self.rest().chars();
11
        chars.next();
12
        chars.next()
13
    }
14

15
    fn bump(&mut self) -> Option<char> {
16
        let ch = self.peek()?;
17
        self.pos += ch.len_utf8();
18
        Some(ch)
19
    }
20

21
    fn span_from(&self, start: usize) -> Span {
22
        Span::new(self.source, start as u32, self.pos as u32)
23
    }