|
2
|
1 Import extendlib.rhope
|
|
|
2
|
|
|
3 Blueprint Token
|
|
|
4 {
|
|
|
5 Type
|
|
|
6 Raw Text
|
|
|
7 Text
|
|
|
8 }
|
|
|
9
|
|
|
10 Token[type,raw,text:out]
|
|
|
11 {
|
|
|
12 out <- [[[Build["Token"]]Type <<[type]]Raw Text <<[raw]]Text <<[text]
|
|
|
13 }
|
|
|
14
|
|
|
15 _Type Match[val, test type, type:out]
|
|
|
16 {
|
|
|
17 If[[test type]=[type]]
|
|
|
18 {
|
|
|
19 out <- Yes
|
|
|
20 }{
|
|
|
21 out <- Val[val]
|
|
|
22 }
|
|
|
23 }
|
|
|
24
|
|
|
25 Type Match@Token[token,type:match,nomatch]
|
|
|
26 {
|
|
|
27 match,nomatch <- If[Fold[["_Type Match"]Set Input[2, [token]Type >>], No, As List[type]]]
|
|
|
28 }
|
|
|
29
|
|
|
30 String Literal[string, raw string, escapes, text, simple tokens, token list:out]
|
|
|
31 {
|
|
|
32 first,rest <- [text]Slice[1]
|
|
|
33 If[[first] = ["\""]]
|
|
|
34 {
|
|
|
35 out <- _Lex[rest, [first]Slice[0], simple tokens, [token list]Append[Token["String Literal", raw string, string]]]
|
|
|
36 }{
|
|
|
37 next raw <- [raw string]Append[first]
|
|
|
38 If[[first] = ["\\"]]
|
|
|
39 {
|
|
|
40 second,next text <- [rest]Slice[1]
|
|
|
41 char <- [escapes]Index[To String[second]] {}
|
|
|
42 {
|
|
|
43 char <- Val[second]
|
|
|
44 }
|
|
|
45 next string <- [string]Append[char]
|
|
|
46 }{
|
|
|
47 next string <- [string]Append[first]
|
|
|
48 next text <- Val[rest]
|
|
|
49 }
|
|
|
50 out <- String Literal[next string, next raw, escapes, next text, simple tokens, token list]
|
|
|
51 }
|
|
|
52 }
|
|
|
53
|
|
|
54 Line Comment[start comment, text, simple tokens, token list:out]
|
|
|
55 {
|
|
|
56 next text, comment <- [text]Get DString["\n"] {} {} {}
|
|
|
57 {
|
|
|
58 next text <- ""
|
|
|
59 }
|
|
|
60 out <- _Lex[next text, [next text]Slice[0], simple tokens, [token list]Append[Token["Line Comment", [start comment]Append[comment], comment]]]
|
|
|
61
|
|
|
62 }
|
|
|
63
|
|
|
64 Block Comment[comment,raw comment, depth, text, simple tokens, token list:out]
|
|
|
65 {
|
|
|
66 Print[["Block Comment: Depth="]Append[depth]]
|
|
|
67 If[[depth] > [0]]
|
|
|
68 {
|
|
|
69 next text, chunk, delim <- [text]Get DString[("/*","*/")] {} {} {}
|
|
|
70 {
|
|
|
71 next text <- ""
|
|
|
72 delim <- ""
|
|
|
73 }
|
|
|
74 If[[delim] = ["/*"]]
|
|
|
75 {
|
|
|
76 next depth <- [depth] + [1]
|
|
|
77 }{
|
|
|
78 next depth <- [depth] - [1]
|
|
|
79 }
|
|
|
80 If[[next depth] = [0]]
|
|
|
81 {
|
|
|
82 next comment <- [comment]Append[chunk]
|
|
|
83 }{
|
|
|
84 next comment <- [[comment]Append[chunk]]Append[delim]
|
|
|
85 }
|
|
|
86 out <- Block Comment[next comment, [[raw comment]Append[chunk]]Append[delim], next depth, next text, simple tokens, token list]
|
|
|
87 }{
|
|
|
88 out <- _Lex[text, [raw comment]Slice[0], simple tokens, [token list]Append[Token["Block Comment", raw comment, comment]]]
|
|
|
89 }
|
|
|
90 }
|
|
|
91
|
|
|
92 Numeric Literal[literal, text, simple tokens, token list:out]
|
|
|
93 {
|
|
|
94 first,rest <- [text]Slice[1]
|
|
|
95 If[[first] In ["01234567890.x"]]
|
|
|
96 {
|
|
|
97 out <- Numeric Literal[[literal]Append[first], rest, simple tokens, token list]
|
|
|
98 }{
|
|
|
99 out <- _Lex[text, [first]Slice[0], simple tokens, [token list]Append[Token["Numeric Literal", literal, literal]]]
|
|
|
100 }
|
|
|
101 }
|
|
|
102
|
|
|
103 Add Token[token, text, simple tokens, token list:out]
|
|
|
104 {
|
|
|
105 out <- _Lex[text, [text]Slice[0], simple tokens, [token list]Append[token]]
|
|
|
106 }
|
|
|
107
|
|
|
108 _Lex[text, symbol, simple tokens,token list:out]
|
|
|
109 {
|
|
|
110 If[[[text]Length] > [0]]
|
|
|
111 {
|
|
|
112 first,rest <- [text]Slice[1]
|
|
|
113 [simple tokens]Index[To String[first]]
|
|
|
114 {
|
|
|
115 token worker <- [["Add Token"]Set Input[0, Token[~, first, ""]]]Set Input[1, rest]
|
|
|
116 }{
|
|
|
117 If[[first] = ["\""]]
|
|
|
118 {
|
|
|
119 escapes <- [[[New@Dictionary[]]Set["n","\n"]]Set["r","\r"]]Set["t","\t"]
|
|
|
120 token worker <- [[[["String Literal"]Set Input[0, [first]Slice[0]]]Set Input[1, first]]Set Input[2, escapes]]Set Input[3, rest]
|
|
|
121 //out <- String Literal["", first, rest, simple tokens, token list, escapes]
|
|
|
122 }{
|
|
|
123 second,second rest <- [rest]Slice[1]
|
|
|
124 If[[[first] = ["<"]] And [[second]=["-"]]]
|
|
|
125 {
|
|
|
126 [first]Append[second]
|
|
|
127 {
|
|
|
128 token worker <- [["Add Token"]Set Input[0, Token["Assignment", ~, ~]]]Set Input[1, second rest]
|
|
|
129 }
|
|
|
130 }{
|
|
|
131
|
|
|
132 If[[[first] = ["/"]] And [[second] = ["*"]]]
|
|
|
133 {
|
|
|
134 token worker <- [[[["Block Comment"]Set Input[0, [first]Slice[0]]]Set Input[1, [first]Append[second]]]Set Input[2, 1]]Set Input[3, second rest]
|
|
|
135 //out <- Block Comment[next text, simple tokens, token list, 1]
|
|
|
136 }{
|
|
|
137 If[[[first] = ["/"]] And [[second] = ["/"]]]
|
|
|
138 {
|
|
|
139 token worker <- [["Line Comment"]Set Input[0, [first]Append[second]]]Set Input[1, second rest]
|
|
|
140 //out <- Line Comment["", [first]Append[second], next text, simple tokens, token list]
|
|
|
141 }{
|
|
|
142 If[[[first]In["0123456789"]] Or [[[first] = ["-"]] And [[second]In["0123456789"]]]]
|
|
|
143 {
|
|
|
144 token worker <- [["Numeric Literal"]Set Input[0, first]]Set Input[1, rest]
|
|
|
145 //out <- Numeric Literal[text, simple tokens, token list]
|
|
|
146 }{
|
|
|
147 out <- _Lex[rest, [symbol]Append[first], simple tokens, token list]
|
|
|
148 }
|
|
|
149 }
|
|
|
150 }
|
|
|
151 }
|
|
|
152
|
|
|
153 }
|
|
|
154 }
|
|
|
155 Val[token worker]
|
|
|
156 {
|
|
|
157 trimmed <- Trim[symbol, " \t\r\n"]
|
|
|
158 If[[trimmed] = [""]]
|
|
|
159 {
|
|
|
160 next list <- Val[token list]
|
|
|
161 }{
|
|
|
162 next list <- [token list]Append[Token["Symbol", trimmed, trimmed]]
|
|
|
163 }
|
|
|
164 out <- [[token worker]Do[
|
|
|
165 [[()]Append[simple tokens]]Append[next list]]
|
|
|
166 ]Index[0]
|
|
|
167 }
|
|
|
168 }{
|
|
|
169 out <- token list
|
|
|
170 }
|
|
|
171 }
|
|
|
172
|
|
|
173 Lex[text:out]
|
|
|
174 {
|
|
|
175 simple tokens <- [[[[[[[[[[[New@Dictionary[]
|
|
|
176 ]Set["{", "Block Begin"]
|
|
|
177 ]Set["}", "Block End"]
|
|
|
178 ]Set["(", "List Begin"]
|
|
|
179 ]Set[")", "List End"]
|
|
|
180 ]Set["[", "Args Begin"]
|
|
|
181 ]Set["]", "Args End"]
|
|
|
182 ]Set[",", "List Separator"]
|
|
|
183 ]Set[":", "Name Separator"]
|
|
|
184 ]Set["@", "Method Separator"]
|
|
|
185 ]Set["`", "Binary Operation"]
|
|
|
186 ]Set["\n", "Newline"]
|
|
|
187 out <- _Lex[text, [text]Slice[0], simple tokens, ()]
|
|
|
188 }
|
