--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/lex.rhope	Wed Apr 29 02:58:03 2009 -0400
@@ -0,0 +1,188 @@
+Import extendlib.rhope
+
+Blueprint Token
+{
+	Type
+	Raw Text
+	Text
+}
+
+Token[type,raw,text:out]
+{
+	out <- [[[Build["Token"]]Type <<[type]]Raw Text <<[raw]]Text <<[text]
+}
+
+_Type Match[val, test type, type:out]
+{
+	If[[test type]=[type]]
+	{
+		out <- Yes
+	}{
+		out <- Val[val]
+	}
+}
+
+Type Match@Token[token,type:match,nomatch]
+{
+	match,nomatch <- If[Fold[["_Type Match"]Set Input[2, [token]Type >>], No, As List[type]]]
+}
+
+String Literal[string, raw string, escapes, text, simple tokens, token list:out]
+{
+	first,rest <- [text]Slice[1]
+	If[[first] = ["\""]]
+	{
+		out <- _Lex[rest, [first]Slice[0], simple tokens, [token list]Append[Token["String Literal", raw string, string]]]
+	}{
+		next raw <- [raw string]Append[first]
+		If[[first] = ["\\"]]
+		{
+			second,next text <- [rest]Slice[1]
+			char <- [escapes]Index[To String[second]] {} 
+			{
+				char <- Val[second]
+			}
+			next string <- [string]Append[char]
+		}{
+			next string <- [string]Append[first]
+			next text <- Val[rest]
+		}
+		out <- String Literal[next string, next raw, escapes, next text, simple tokens, token list]
+	}
+}
+
+Line Comment[start comment, text, simple tokens, token list:out]
+{
+	next text, comment <- [text]Get DString["\n"] {} {} {}
+	{
+		next text <- ""
+	}
+	out <- _Lex[next text, [next text]Slice[0], simple tokens, [token list]Append[Token["Line Comment", [start comment]Append[comment], comment]]]
+	
+}
+
+Block Comment[comment,raw comment, depth, text, simple tokens, token list:out]
+{
+	Print[["Block Comment: Depth="]Append[depth]]
+	If[[depth] > [0]]
+	{
+		next text, chunk, delim <- [text]Get DString[("/*","*/")] {} {} {}
+		{
+			next text <- ""
+			delim <- ""
+		}
+		If[[delim] = ["/*"]]
+		{
+			next depth <- [depth] + [1]
+		}{
+			next depth <- [depth] - [1]
+		}
+		If[[next depth] = [0]]
+		{
+			next comment <- [comment]Append[chunk]
+		}{
+			next comment <- [[comment]Append[chunk]]Append[delim]
+		}
+		out <- Block Comment[next comment, [[raw comment]Append[chunk]]Append[delim], next depth, next text, simple tokens, token list]
+	}{
+		out <- _Lex[text, [raw comment]Slice[0], simple tokens, [token list]Append[Token["Block Comment", raw comment, comment]]]
+	}
+}
+
+Numeric Literal[literal, text, simple tokens, token list:out]
+{
+	first,rest <- [text]Slice[1]
+	If[[first] In ["01234567890.x"]]
+	{
+		out <- Numeric Literal[[literal]Append[first], rest, simple tokens, token list]
+	}{
+		out <- _Lex[text, [first]Slice[0], simple tokens, [token list]Append[Token["Numeric Literal", literal, literal]]]
+	}
+}
+
+Add Token[token, text, simple tokens, token list:out]
+{
+	out <- _Lex[text, [text]Slice[0], simple tokens, [token list]Append[token]]
+}
+
+_Lex[text, symbol, simple tokens,token list:out]
+{
+	If[[[text]Length] > [0]]
+	{
+		first,rest <- [text]Slice[1]
+		[simple tokens]Index[To String[first]]
+		{
+			token worker <- [["Add Token"]Set Input[0, Token[~, first, ""]]]Set Input[1, rest]
+		}{
+			If[[first] = ["\""]]
+			{
+				escapes <- [[[New@Dictionary[]]Set["n","\n"]]Set["r","\r"]]Set["t","\t"]
+				token worker <- [[[["String Literal"]Set Input[0, [first]Slice[0]]]Set Input[1, first]]Set Input[2, escapes]]Set Input[3, rest]
+				//out <- String Literal["", first, rest, simple tokens, token list, escapes]
+			}{
+				second,second rest <- [rest]Slice[1]
+				If[[[first] = ["<"]] And [[second]=["-"]]]
+				{
+					[first]Append[second]
+					{
+						token worker <- [["Add Token"]Set Input[0, Token["Assignment", ~, ~]]]Set Input[1, second rest]
+					}
+				}{
+					
+					If[[[first] = ["/"]] And [[second] = ["*"]]]
+					{
+						token worker <- [[[["Block Comment"]Set Input[0, [first]Slice[0]]]Set Input[1, [first]Append[second]]]Set Input[2, 1]]Set Input[3, second rest]
+						//out <- Block Comment[next text, simple tokens, token list, 1]
+					}{
+						If[[[first] = ["/"]] And [[second] = ["/"]]]
+						{
+							token worker <- [["Line Comment"]Set Input[0, [first]Append[second]]]Set Input[1, second rest]
+							//out <- Line Comment["", [first]Append[second], next text, simple tokens, token list]
+						}{
+							If[[[first]In["0123456789"]] Or [[[first] = ["-"]] And [[second]In["0123456789"]]]]
+							{
+								token worker <- [["Numeric Literal"]Set Input[0, first]]Set Input[1, rest]
+								//out <- Numeric Literal[text, simple tokens, token list]
+							}{
+								out <- _Lex[rest, [symbol]Append[first], simple tokens, token list]
+							}
+						}
+					}
+				}
+				
+			}
+		}
+		Val[token worker]
+		{
+			trimmed <- Trim[symbol, " \t\r\n"]
+			If[[trimmed] = [""]]
+			{
+				next list <- Val[token list]
+			}{
+				next list <- [token list]Append[Token["Symbol", trimmed, trimmed]]
+			}
+			out <- [[token worker]Do[
+				[[()]Append[simple tokens]]Append[next list]]
+			]Index[0]
+		}
+	}{
+		out <- token list
+	}
+}
+
+Lex[text:out]
+{
+	simple tokens <- [[[[[[[[[[[New@Dictionary[]
+		]Set["{", "Block Begin"]
+		]Set["}", "Block End"]
+		]Set["(", "List Begin"]
+		]Set[")", "List End"]
+		]Set["[", "Args Begin"]
+		]Set["]", "Args End"]
+		]Set[",", "List Separator"]
+		]Set[":", "Name Separator"]
+		]Set["@", "Method Separator"]
+		]Set["`", "Binary Operation"]
+		]Set["\n", "Newline"]
+	out <- _Lex[text, [text]Slice[0], simple tokens, ()]
+}