gearsco.de
An Erlang lexer and syntax highlighter in Gleam
Scaffold lexer
+86
-3
+1
gleam.toml
+1
gleam.toml
+2
manifest.toml
+2
manifest.toml
···
4
4
packages = [
5
5
{ name = "gleam_stdlib", version = "0.58.0", build_tools = ["gleam"], requirements = [], otp_app = "gleam_stdlib", source = "hex", outer_checksum = "091F2D2C4A3A4E2047986C47E2C2C9D728A4E068ABB31FDA17B0D347E6248467" },
6
6
{ name = "gleeunit", version = "1.3.0", build_tools = ["gleam"], requirements = ["gleam_stdlib"], otp_app = "gleeunit", source = "hex", outer_checksum = "0E6C83834BA65EDCAAF4FE4FB94AC697D9262D83E6F58A750D63C9F6C8A9D9FF" },
7
+
{ name = "splitter", version = "1.0.0", build_tools = ["gleam"], requirements = ["gleam_stdlib"], otp_app = "splitter", source = "hex", outer_checksum = "128FC521EE33B0012E3E64D5B55168586BC1B9C8D7B0D0CA223B68B0D770A547" },
7
8
]
8
9
9
10
[requirements]
10
11
gleam_stdlib = { version = ">= 0.44.0 and < 2.0.0" }
11
12
gleeunit = { version = ">= 1.0.0 and < 2.0.0" }
13
+
splitter = { version = ">= 1.0.0 and < 2.0.0" }
+77
-3
src/pearl.gleam
+77
-3
src/pearl.gleam
···
1
-
import gleam/io
1
+
import gleam/list
2
+
import gleam/string
3
+
import pearl/token.{type Token}
2
4
3
-
pub fn main() -> Nil {
4
-
io.println("Hello from pearl!")
5
+
pub opaque type Lexer {
6
+
Lexer(
7
+
source: String,
8
+
ignore_comments: Bool,
9
+
ignore_whitespace: Bool,
10
+
errors: List(Error),
11
+
splitters: Splitters,
12
+
)
13
+
}
14
+
15
+
type Splitters {
16
+
Splitters
17
+
}
18
+
19
+
pub type Error {
20
+
UnknownCharacter(character: String)
21
+
}
22
+
23
+
pub fn new(source: String) -> Lexer {
24
+
Lexer(
25
+
source:,
26
+
ignore_comments: False,
27
+
ignore_whitespace: False,
28
+
errors: [],
29
+
splitters: make_splitters(),
30
+
)
31
+
}
32
+
33
+
fn make_splitters() -> Splitters {
34
+
Splitters
35
+
}
36
+
37
+
pub fn ignore_comments(lexer: Lexer) -> Lexer {
38
+
Lexer(..lexer, ignore_comments: True)
39
+
}
40
+
41
+
pub fn ignore_whitespace(lexer: Lexer) -> Lexer {
42
+
Lexer(..lexer, ignore_whitespace: True)
43
+
}
44
+
45
+
pub fn tokenise(lexer: Lexer) -> #(List(Token), List(Error)) {
46
+
do_tokenise(lexer, [])
47
+
}
48
+
49
+
fn do_tokenise(lexer: Lexer, tokens: List(Token)) -> #(List(Token), List(Error)) {
50
+
case next(lexer) {
51
+
#(lexer, token.EndOfFile) -> #(
52
+
list.reverse([token.EndOfFile, ..tokens]),
53
+
list.reverse(lexer.errors),
54
+
)
55
+
#(lexer, token) -> do_tokenise(lexer, [token, ..tokens])
56
+
}
57
+
}
58
+
59
+
fn next(lexer: Lexer) -> #(Lexer, Token) {
60
+
case lexer.source {
61
+
"" -> #(lexer, token.EndOfFile)
62
+
_ ->
63
+
case string.pop_grapheme(lexer.source) {
64
+
Error(_) -> #(lexer, token.EndOfFile)
65
+
Ok(#(char, source)) -> #(
66
+
advance(error(lexer, UnknownCharacter(char)), source),
67
+
token.Unknown(char),
68
+
)
69
+
}
70
+
}
71
+
}
72
+
73
+
fn advance(lexer: Lexer, source: String) -> Lexer {
74
+
Lexer(..lexer, source:)
75
+
}
76
+
77
+
fn error(lexer: Lexer, error: Error) -> Lexer {
78
+
Lexer(..lexer, errors: [error, ..lexer.errors])
5
79
}
+6
src/pearl/token.gleam
+6
src/pearl/token.gleam
···
89
89
QuestionEqual
90
90
Bang
91
91
Equal
92
+
93
+
// Invalid tokens
94
+
Unknown(String)
92
95
}
93
96
94
97
pub fn to_source(token: Token) -> String {
···
185
188
QuestionEqual -> "?="
186
189
Bang -> "!"
187
190
Equal -> "="
191
+
192
+
// Invalid tokens
193
+
Unknown(char) -> char
188
194
}
189
195
}