src/rules/tokens.bend

bolt@1.27.0.0 · 104 lines · 13 definitions

1# src/rules/tokens: helpers the token rules share (put, escape, nat, strings,
2# twice, arms): significant tokens, literal kinds, Nat literals, case
3# patterns, and a name's uses under a node.
4import Base
5import bolt@1.27.0.0/src/lazy/lazy.bend as Lazy
6import bolt@1.27.0.0/src/syntax/lex.bend as Lex
7import bolt@1.27.0.0/src/syntax/tree.bend as Tree
8
9# the tokens without space, newlines and comments
10def sig(toks: List<&2, Lex.Tok>) -> List<&2, Lex.Tok>:
11  match toks:
12    case Nil{}:
13      Nil{}
14    case Con{Lex.Tok{+k, t, l, c}, rest}:
15      +more = sig(rest)
16      Bool.pick(List<&2, Lex.Tok>, Lex.significant(k), Lex.Tok{k, t, l, c} <> more, more)
17
18# a text cut to its first char
19def code.cut(tt: String) -> String:
20  match tt:
21    case SNil{}:
22      SNil{}
23    case SCon{h, _t}:
24      SCon{h, SNil{}}
25
26# a string literal cut to its opening quote; any other token as it is
27def code.tok(tk: Lex.Tok) -> Lex.Tok:
28  match tk:
29    case Lex.Tok{+k, +t, l, c}:
30      Lex.Tok{k, Bool.pick(String, Lex.is_str(k), code.cut(t), t), l, c}
31
32# the tokens, each string literal cut to its opening quote
33def code.toks(ts: List<&2, Lex.Tok>) -> List<&2, Lex.Tok>:
34  match ts:
35    case Nil{}:
36      Nil{}
37    case Con{h, t}:
38      code.tok(h) <> code.toks(t)
39
40# a text with each string literal cut to its opening quote: what of it is
41# code, as the lexer reads it
42def code(+tt: String) -> String:
43  Lex.text(code.toks(Lex.tokens(tt)))
44
45# a number, a char or a string literal?
46def is_lit(kk: Lex.TokKind) -> Bool:
47  match kk:
48    case Lex.TNum{}:
49      True{}
50    case Lex.TChar{}:
51      True{}
52    case Lex.TStr{}:
53      True{}
54    case other:
55      False{}
56
57# digits, then one final `n`: a Nat literal's chars
58def nat_ok(cs: List<&2, Char>) -> Bool:
59  match cs:
60    case Con{'n', Nil{}}:
61      True{}
62    case Con{+c, t}:
63      Lazy.and_then(Char.is_digit(c), _u => nat_ok(t))
64    case Nil{}:
65      False{}
66
67# digits alone, one or more: a U32 literal's chars
68def digits_ok(cs: List<&2, Char>) -> Bool:
69  match cs:
70    case Con{+c, Nil{}}:
71      Char.is_digit(c)
72    case Con{+c, t}:
73      Lazy.and_then(Char.is_digit(c), _u => digits_ok(t))
74    case Nil{}:
75      False{}
76
77# the text without its last char (a Nat literal's digits)
78def stem(+tt: String) -> String:
79  String.take(tt, Nat.sub(String.length(tt), 1n))
80
81# a Nat literal's value (`12n` is 12); none for anything else or past U32
82def nat_value(+tt: String) -> Maybe<&2, U32>:
83  Bool.pick(Maybe<&2, U32>, nat_ok(String.to_list(tt)), U32.read(stem(tt)), None{})
84
85# a text's width in chars
86def width(tt: String) -> U32:
87  U32.from_nat(String.length(tt))
88
89# is the first node the keyword?
90def keyword(nn: Tree.Node, +word: String) -> Bool:
91  match nn:
92    case Tree.NCons{Tree.Leaf{Lex.Tok{Lex.TKey{}, +t, l, c}}, rest}:
93      String.eq(t, word)
94    case other:
95      False{}
96
97# a case statement's pattern (its tokens after `case`, the colon and an
98# inline body included)
99def pattern(kids: Tree.Node) -> Tree.Node:
100  match kids:
101    case Tree.NCons{h, rest}:
102      rest
103    case other:
104      other