# HG changeset patch
# User Mike Pavone <pavone@retrodev.com>
# Date 1288925014 0
# Node ID 0a4682be2db2dc995733075830f6e584414c3f71
# Parent  147dfc703161972f30c79c0c8a9f763ddb171c47
Modify lexer and new parser to work in compiler

diff -r 147dfc703161 -r 0a4682be2db2 compile.rhope
--- a/compile.rhope	Fri Nov 05 02:42:45 2010 +0000
+++ b/compile.rhope	Fri Nov 05 02:43:34 2010 +0000
@@ -7,11 +7,12 @@
 	[args]Index[1]
 	{
 		Print[["Parsing "]Append[~]]
-		file <- <String@File[~]
-		,data <- [file]Get FString[[file]Length]
+		file <- [File[~]]Open["r"]
+		data <- String[[file]Read[[file]Length]]
 		tokens <- Lex[Count String[data]]
-		Pretty Print[Parse[tokens], ""]
+		Print[Parse[tokens], ""]
 	}{
 		Print["Usage: rhope compile.rhope <filename>"]
 	}
 }
+
diff -r 147dfc703161 -r 0a4682be2db2 countstring.rhope
--- a/countstring.rhope	Fri Nov 05 02:42:45 2010 +0000
+++ b/countstring.rhope	Fri Nov 05 02:43:34 2010 +0000
@@ -13,7 +13,7 @@
 
 Count String At Pos[text,line,col:out]
 {
-	out <- [[[Build["Count String"]]Text <<[text]]Line <<[line]]Column <<[col]
+	out <- [[[Build[Count String()]]Text <<[text]]Line <<[line]]Column <<[col]
 }
 
 Slice@Count String[string,index:left,right]
@@ -45,11 +45,21 @@
 	out <- Length[[string]Text >>]
 }
 
+Byte Length@Count String[string:out]
+{
+	out <- Byte Length[[string]Text >>]
+}
+
+Byte@Count String[string,index:out,invalid]
+{
+	out,invalid <- Byte[[string]Text >>, index]
+}
+
 Append@Count String[left,right:out]
 {
 	If[[[left]Length] > [0]]
 	{
-		out <- [left]Text << [ [To String[left]]Append[To String[right]] ]
+		out <- [left]Text << [ [String[left]]Append[String[right]] ]
 	}{
 		out <- right
 	}
@@ -57,13 +67,44 @@
 
 =@Count String[left,right:out]
 {
-	out <- [To String[left]] = [To String[right]]
+	out <- [String[left]] = [String[right]]
+}
+
+_Slice=@Count String[left,right:out]
+{
+	out <- _Slice=[String[left], String[right]]
+}
+
+_Flat=@Count String[left,right:out]
+{
+	out <- _Flat=[String[left], String[right]]
+}
+
+_Flatten@Count String[string,dest,offset,count:out]
+{
+	out <- _Flatten[[string]Text >>, dest, offset, count]
 }
 
-Get DString@Count String[string, delims:after,before,delim,nomatch]
+/*Partition@Count String[string, delims:before,delim,after,nomatch]
 {
-	,,delim,nomatch <- [[string]Text >>]Get DString[delims]
+	,delim,,nomatch <- [[string]Text >>]Partition[delims]
 	{
+                before <- [string]Text <<[~]
+                bparts <- [~]Split["\n"]
+                preblines <- [[bparts]Length] - [1]
+                If[[preblines] > [-1]]
+                {
+                        blines <- Val[preblines]
+                        bcols <- Length[[bparts]Index[blines]]
+                }{
+                        bcols <- 0
+                        blines <- 0
+                }
+        }{
+		dparts <- [~]Split["\n"]
+		dlines <- [[dparts]Length]-[1]
+		dcols <- Length[[dparts]Index[dlines]]
+	}{
 		If[[dlines] > [0]]
 		{
 			cols <- Val[dcols]
@@ -76,26 +117,10 @@
 			}
 		}
 		after <- Count String At Pos[~, [[blines]+[dlines]] + [[string]Line >>], cols]
-	}{
-		before <- [string]Text <<[~]
-		bparts <- [~]Split["\n"]
-		preblines <- [[bparts]Length] - [1]
-		If[[preblines] > [-1]]
-		{
-			blines <- Val[preblines]
-			bcols <- Length[[bparts]Index[blines]]
-		}{
-			bcols <- 0
-			blines <- 0
-		}
-	}{
-		dparts <- [~]Split["\n"]
-		dlines <- [[dparts]Length] - [1]
-		dcols <- Length[[dparts]Index[dlines]]
 	}
-}
+}*/
 
-To String@Count String[string:out]
+String@Count String[string:out]
 {
 	out <- [string]Text >>
 }
@@ -104,3 +129,4 @@
 {
 	out <- Count String[""]
 }
+
diff -r 147dfc703161 -r 0a4682be2db2 lex.rhope
--- a/lex.rhope	Fri Nov 05 02:42:45 2010 +0000
+++ b/lex.rhope	Fri Nov 05 02:43:34 2010 +0000
@@ -1,4 +1,3 @@
-Import extendlib.rhope
 
 Blueprint Token
 {
@@ -9,7 +8,7 @@
 
 Token[type,raw,text:out]
 {
-	out <- [[[Build["Token"]]Type <<[type]]Raw Text <<[raw]]Text <<[text]
+	out <- [[[Build[Token()]]Type <<[type]]Raw Text <<[raw]]Text <<[text]
 }
 
 _Type Match[val, test type, type:out]
@@ -22,9 +21,19 @@
 	}
 }
 
+As List[val:out]
+{
+	[(List(),List Leaf())]Find[=[?, Blueprint Of[val]]]
+	{
+		out <- val
+	}{
+		out <- [()]Append[val]
+	}
+}
+
 Type Match@Token[token,type:match,nomatch]
 {
-	match,nomatch <- If[Fold[["_Type Match"]Set Input[2, [token]Type >>], No, As List[type]]]
+	match,nomatch <- If[Fold[_Type Match[?,?, [token]Type >>], No, As List[type]]]
 }
 
 String Literal[string, raw string, escapes, text, simple tokens, token list:out]
@@ -38,7 +47,7 @@
 		If[[first] = ["\\"]]
 		{
 			second,next text <- [rest]Slice[1]
-			char <- [escapes]Index[To String[second]] {} 
+			char <- [escapes]Index[String[second]] {} 
 			{
 				char <- Val[second]
 			}
@@ -53,9 +62,10 @@
 
 Line Comment[start comment, text, simple tokens, token list:out]
 {
-	next text, comment <- [text]Get DString["\n"] {} {} {}
+	comment,,next text <- [text]Partition["\n"] {} {} {}
 	{
 		next text <- ""
+		comment <- Val[text]
 	}
 	out <- _Lex[next text, [next text]Slice[0], simple tokens, [token list]Append[Token["Line Comment", [start comment]Append[comment], comment]]]
 	
@@ -63,13 +73,14 @@
 
 Block Comment[comment,raw comment, depth, text, simple tokens, token list:out]
 {
-	Print[["Block Comment: Depth="]Append[depth]]
+	Print[["Block Comment: Depth="]Append[String[depth]]]
 	If[[depth] > [0]]
 	{
-		next text, chunk, delim <- [text]Get DString[("/*","*/")] {} {} {}
+		chunk, delim, next text <- [text]Partition[("/*","*/")] {} {} {}
 		{
 			next text <- ""
 			delim <- ""
+			chunk <- Val[text]
 		}
 		If[[delim] = ["/*"]]
 		{
@@ -110,14 +121,14 @@
 	If[[[text]Length] > [0]]
 	{
 		first,rest <- [text]Slice[1]
-		[simple tokens]Index[To String[first]]
+		[simple tokens]Index[String[first]]
 		{
-			token worker <- [["Add Token"]Set Input[0, Token[~, first, ""]]]Set Input[1, rest]
+			token worker <- Val[Add Token[Token[~, first, ""], rest, ?]]
 		}{
 			If[[first] = ["\""]]
 			{
-				escapes <- [[[New@Dictionary[]]Set["n","\n"]]Set["r","\r"]]Set["t","\t"]
-				token worker <- [[[["String Literal"]Set Input[0, [first]Slice[0]]]Set Input[1, first]]Set Input[2, escapes]]Set Input[3, rest]
+				escapes <- [[[Dictionary[]]Set["n","\n"]]Set["r","\r"]]Set["t","\t"]
+				token worker <- Val[String Literal[[first]Slice[0], first, escapes, rest, ?]]
 				//out <- String Literal["", first, rest, simple tokens, token list, escapes]
 			}{
 				second,second rest <- [rest]Slice[1]
@@ -125,23 +136,23 @@
 				{
 					[first]Append[second]
 					{
-						token worker <- [["Add Token"]Set Input[0, Token["Assignment", ~, ~]]]Set Input[1, second rest]
+						token worker <- Val[Add Token[Token["Assignment", ~, ~], second rest, ?]]
 					}
 				}{
 					
 					If[[[first] = ["/"]] And [[second] = ["*"]]]
 					{
-						token worker <- [[[["Block Comment"]Set Input[0, [first]Slice[0]]]Set Input[1, [first]Append[second]]]Set Input[2, 1]]Set Input[3, second rest]
+						token worker <- Val[Block Comment[[first]Slice[0], [first]Append[second], 1, second rest, ?]]
 						//out <- Block Comment[next text, simple tokens, token list, 1]
 					}{
 						If[[[first] = ["/"]] And [[second] = ["/"]]]
 						{
-							token worker <- [["Line Comment"]Set Input[0, [first]Append[second]]]Set Input[1, second rest]
+							token worker <- Val[Line Comment[[first]Append[second], second rest, ?]]
 							//out <- Line Comment["", [first]Append[second], next text, simple tokens, token list]
 						}{
 							If[[[first]In["0123456789"]] Or [[[first] = ["-"]] And [[second]In["0123456789"]]]]
 							{
-								token worker <- [["Numeric Literal"]Set Input[0, first]]Set Input[1, rest]
+								token worker <- Val[Numeric Literal[first, rest, ?]]
 								//out <- Numeric Literal[text, simple tokens, token list]
 							}{
 								out <- _Lex[rest, [symbol]Append[first], simple tokens, token list]
@@ -161,9 +172,7 @@
 			}{
 				next list <- [token list]Append[Token["Symbol", trimmed, trimmed]]
 			}
-			out <- [[token worker]Do[
-				[[()]Append[simple tokens]]Append[next list]]
-			]Index[0]
+			out <- [token worker]Call[simple tokens, next list]
 		}
 	}{
 		out <- token list
@@ -172,7 +181,7 @@
 
 Lex[text:out]
 {
-	simple tokens <- [[[[[[[[[[[New@Dictionary[]
+	simple tokens <- [[[[[[[[[[[Dictionary[]
 		]Set["{", "Block Begin"]
 		]Set["}", "Block End"]
 		]Set["(", "List Begin"]
diff -r 147dfc703161 -r 0a4682be2db2 parse.rhope
--- a/parse.rhope	Fri Nov 05 02:42:45 2010 +0000
+++ b/parse.rhope	Fri Nov 05 02:43:34 2010 +0000
@@ -8,12 +8,12 @@
 
 Error[msg,text:out]
 {
-	out <- [[[Build["Error"]]Message <<[msg]]Line << [ [[text]Line >>]+[1] ]]Column << [ [[text]Column >>]+[1] ]
+	out <- [[[Build[Error()]]Message <<[msg]]Line << [ [[text]Line >>]+[1] ]]Column << [ [[text]Column >>]+[1] ]
 }
 
-To String@Error[error:out]
+String@Error[error:out]
 {
-	out <- [[[[[error]Message >>]Append[" on line "]]Append[[error]Line >>]]Append[" at column "]]Append[[error]Column >>]
+	out <- [[[[[error]Message >>]Append[" on line "]]Append[String[[error]Line >>]]]Append[" at column "]]Append[String[[error]Column >>]]
 }
 
 Blueprint PImport Node
@@ -23,7 +23,12 @@
 
 PImport Node[file:out]
 {
-	out <- [Build["PImport Node"]]File <<[file]
+	out <- [Build[PImport Node()]]File <<[file]
+}
+
+String@PImport Node[node:out]
+{
+	out <- ["Import: "]Append[[node]File >>]
 }
 
 Blueprint Worker Node
@@ -38,7 +43,7 @@
 Add Node Input@Worker Node[node,input:out]
 {
 	Print["Add Node Input"]
-	Print[["Input: "]Append[To String[input]]]
+	Print[["Input: "]Append[String[input]]]
 	out <- [node]Inputs <<[[[node]Inputs >>]Append[input]]
 }
 
@@ -47,6 +52,15 @@
 	out <- [node]Outputs <<[[[node]Outputs >>]Append[output]]
 }
 
+String@Worker Node[node:out]
+{
+	out <- [[[[["Worker: "]Append[[node]Name >>]
+		]Append[ ["\n\tBlueprint:\t"]Append[String[[node]Blueprint >>]] ]
+		]Append[ ["\n\tInputs: \t"]Append[[[node]Inputs >>]Join[", "]] ]
+		]Append[ ["\n\tOutputs:\t"]Append[[[node]Outputs >>]Join[", "]] ]
+		]Append[ ["\n\tTree:   \t"]Append[ [[String[[node]Tree >>]]Split["\n"]]Join["\n\t"] ] ]
+}
+
 Blueprint Statement
 {
 	Expression
@@ -56,7 +70,7 @@
 
 Statement[expression,assignments:out]
 {
-	out <- [[[Build["Statement"]]Expression <<[expression]]Assignments <<[assignments]]Blocks <<[()]
+	out <- [[[Build[Statement()]]Expression <<[expression]]Assignments <<[assignments]]Blocks <<[()]
 }
 
 Add Block@Statement[statement,block:out]
@@ -64,6 +78,14 @@
 	out <- [statement]Blocks <<[ [[statement]Blocks >>]Append[block] ]
 }
 
+String@Statement[statement:out]
+{
+	out <- [[["Statement\n\tExpression:\t"
+		]Append[ [[String[[statement]Expression >>]]Split["\n"]]Join["\n\t"] ]
+		]Append[ ["\n\tAssignments:\t"]Append[[[statement]Assignments >>]Join[", "]] ]
+		]Append[ ["\n\tBlocks:\t"]Append[ [[String[[statement]Blocks >>]]Split["\n"]]Join["\n\t"] ] ]
+}
+
 Blueprint Worker Expression
 {
 	Worker
@@ -71,6 +93,11 @@
 	Arguments
 }
 
+String@Worker Expression[exp:out]
+{
+	out <- ["Worker Expression\n\tWorker:\t"]Append[[exp]Worker >>]
+}
+
 Blueprint Global Expression
 {
 	Store
@@ -84,7 +111,12 @@
 
 Pipe Expression[name:out]
 {
-	out <- [Build["Pipe Expression"]]Name <<[name]
+	out <- [Build[Pipe Expression()]]Name <<[name]
+}
+
+String@Pipe Expression[pipe:out]
+{
+	out <- ["Pipe Expression\n\tName:\t"]Append[[pipe]Name >>]
 }
 
 Blueprint Block
@@ -94,7 +126,12 @@
 
 Block[:out]
 {
-	out <- [Build["Block"]]Tree <<[()]
+	out <- [Build[Block()]]Tree <<[()]
+}
+
+String@Block[block:out]
+{
+	out <- Fold[_String Seq[?], "Block", [block]Tree >>]
 }
 
 Blueprint Field Expression
@@ -105,12 +142,12 @@
 
 Field Expression[name,set?:out]
 {
-	out <- [[Build["Field Expression"]]Name <<[name]]Set? <<[set?]
+	out <- [[Build[Field Expression()]]Name <<[name]]Set? <<[set?]
 }
 
 Body[node,tokens,current,depth:out,out index, done]
 {
-	Print[["Body: Depth="]Append[depth]]
+	Print[["Body: Depth="]Append[String[depth]]]
 	If[[depth] > [0]]
 	{
 		token <- [tokens]Index[current]
@@ -147,7 +184,7 @@
 	}
 	Val[end stream]
 	{
-		Print[To String[Error[["Unexpected end of stream after token of type "]Append[[token]Type >>], [token]Raw Text >>]]]
+		Print[String[Error[["Unexpected end of stream after token of type "]Append[[token]Type >>], [token]Raw Text >>]]]
 	}
 }
 
@@ -166,7 +203,7 @@
 	}
 	Val[end stream]
 	{
-		Print[To String[Error[["Unexpected end of stream after token of type "]Append[[token]Type >>], [token]Raw Text >>]]]
+		Print[String[Error[["Unexpected end of stream after token of type "]Append[[token]Type >>], [token]Raw Text >>]]]
 	}
 }
 
@@ -188,14 +225,14 @@
 				{
 					out,out index, done <- Outputs[node, tokens, next]
 				}{
-					Print[To String[Error[["Unexpected token of type "]Append[[token]Type >>], [token]Raw Text >>]]]
+					Print[String[Error[["Unexpected token of type "]Append[[token]Type >>], [token]Raw Text >>]]]
 				}
 			}
 		}
 	}
 	Val[end stream]
 	{
-		Print[To String[Error[["Unexpected end of stream after token of type "]Append[[token]Type >>], [token]Raw Text >>]]]
+		Print[String[Error[["Unexpected end of stream after token of type "]Append[[token]Type >>], [token]Raw Text >>]]]
 	}
 }
 
@@ -223,7 +260,7 @@
 					{
 						out,out index,done <- Inputs[node, tokens, next]
 					}{
-						Print[To String[Error[["Unexpected token of type "]Append[[token]Type >>], [token]Raw Text >>]]]
+						Print[String[Error[["Unexpected token of type "]Append[[token]Type >>], [token]Raw Text >>]]]
 					}
 				}
 			}
@@ -231,7 +268,7 @@
 	}
 	Val[end stream]
 	{
-		Print[To String[Error[["Unexpected end of stream after token of type "]Append[[token]Type >>], [token]Raw Text >>]]]
+		Print[String[Error[["Unexpected end of stream after token of type "]Append[[token]Type >>], [token]Raw Text >>]]]
 	}
 }
 
@@ -248,11 +285,11 @@
 			{
 				out,out index,done <- Method[node, tokens, next]
 			}{
-				Print[To String[Error[["Unexpected token of type "]Append[[token]Type >>], [token]Raw Text >>]]]
+				Print[String[Error[["Unexpected token of type "]Append[[token]Type >>], [token]Raw Text >>]]]
 			}
 		}
 	}{
-		Print[To String[Error[["Unexpected end of stream after token of type "]Append[[token]Type >>], [token]Raw Text >>]]]
+		Print[String[Error[["Unexpected end of stream after token of type "]Append[[token]Type >>], [token]Raw Text >>]]]
 	}
 }
 
@@ -267,7 +304,7 @@
 		{
 			out,out index, done <- Inputs[node, tokens, ~]
 		}{
-			Print[To String[Error[["Unexpected end of stream after token of type "]Append[[token]Type >>], [token]Raw Text >>]]]
+			Print[String[Error[["Unexpected end of stream after token of type "]Append[[token]Type >>], [token]Raw Text >>]]]
 		}
 	}{
 		[token]Type Match["Method Separator"]
@@ -276,14 +313,14 @@
 			{
 				out,out index,done <- Method[node, tokens, ~]
 			}{
-				Print[To String[Error[["Unexpected end of stream after token of type "]Append[[token]Type >>], [token]Raw Text >>]]]
+				Print[String[Error[["Unexpected end of stream after token of type "]Append[[token]Type >>], [token]Raw Text >>]]]
 			}
 		}{
 			[token]Type Match[("Line Comment","Block Comment","Newline")]
 			{
 				continue <- Yes
 			}{
-				Print[To String[Error[["Unexpected token of type "]Append[[token]Type >>], [token]Raw Text >>]]]
+				Print[String[Error[["Unexpected token of type "]Append[[token]Type >>], [token]Raw Text >>]]]
 			}
 			Val[continue]
 			{
@@ -291,7 +328,7 @@
 				{
 					out,out index, done <- Before Inputs[node, tokens, ~]
 				}{
-					Print[To String[Error[["Unexpected end of stream after token of type "]Append[[token]Type >>], [token]Raw Text >>]]]
+					Print[String[Error[["Unexpected end of stream after token of type "]Append[[token]Type >>], [token]Raw Text >>]]]
 				}
 			}
 		}
@@ -300,8 +337,8 @@
 
 Worker Node[name, tokens, current:out,out index, done]
 {
-	Print[["Worker: "]Append[To String[name]]]
-	out,out index, done <- Before Inputs[[[[[Build["Worker Node"]]Name <<[name]]Inputs <<[()]]Outputs <<[()]]Tree <<[()], tokens, current]
+	Print[["Worker: "]Append[String[name]]]
+	out,out index, done <- Before Inputs[[[[[[Build[Worker Node()]]Name <<[name]]Inputs <<[()]]Outputs <<[()]]Blueprint <<[No]]Tree <<[()], tokens, current]
 }
 
 Skip Nodes[toskip, stop, tokens, current:out]
@@ -317,11 +354,11 @@
 			{
 				out <- Skip Nodes[toskip, stop, tokens, next]
 			}{
-				Print[To String[Error[["Unexpected end of stream after token of type "]Append[[token]Type >>], [token]Raw Text >>]]]
+				Print[String[Error[["Unexpected end of stream after token of type "]Append[[token]Type >>], [token]Raw Text >>]]]
 			}
 		}
 	}{
-		Print[To String[Error[["Unexpected end of stream after token of type "]Append[[token]Type >>], [token]Raw Text >>]]]
+		Print[String[Error[["Unexpected end of stream after token of type "]Append[[token]Type >>], [token]Raw Text >>]]]
 	}
 }
 
@@ -349,7 +386,7 @@
 				{
 					out,out index,done <- Fields[node, tokens, next]
 				}{
-					Print[To String[Error[["Unexpected end of stream after token of type "]Append[[token]Type >>], [token]Raw Text >>]]]
+					Print[String[Error[["Unexpected end of stream after token of type "]Append[[token]Type >>], [token]Raw Text >>]]]
 				}
 			}
 		}
@@ -359,14 +396,14 @@
 			out <- Val[node]
 			done <- Yes
 		}{
-			Print[To String[Error[["Unexpected end of stream after token of type "]Append[[token]Type >>], [token]Raw Text >>]]]
+			Print[String[Error[["Unexpected end of stream after token of type "]Append[[token]Type >>], [token]Raw Text >>]]]
 		}
 	}
 }
 
 PBlueprint Node[name, tokens, current:out,out index,done]
 {
-	node <- [[Build["PBlueprint Node"]]Name <<[name]]Fields <<[()]
+	node <- [[Build[PBlueprint Node()]]Name <<[name]]Fields <<[()]
 	next <- Skip Nodes[("Newline","Block Comment","Comment"), "Block Begin", tokens, current]
 	out,out index,done <- Fields[node, tokens, next]
 }
@@ -399,7 +436,7 @@
 						out <- Val[next nodes]
 					}
 				}{
-					Print[To String[Error[["Unexpected end of stream after symbol "]Append[[token]Text >>], [token]Raw Text >>]]]
+					Print[String[Error[["Unexpected end of stream after symbol "]Append[[token]Text >>], [token]Raw Text >>]]]
 				}
 			}{
 				[tokens]Next[current]
@@ -412,7 +449,7 @@
 						out <- Val[next nodes]
 					}
 				}{
-					Print[To String[Error[["Unexpected end of stream after symbol "]Append[[token]Text >>], [token]Raw Text >>]]]
+					Print[String[Error[["Unexpected end of stream after symbol "]Append[[token]Text >>], [token]Raw Text >>]]]
 				}
 			}
 		}
@@ -423,7 +460,7 @@
 			next nodes <- Val[nodes]
 			next index <- Val[current]
 		}{
-			Print[To String[Error[["Unexpected token of type "]Append[[token]Type >>], [token]Raw Text >>]]]
+			Print[String[Error[["Unexpected token of type "]Append[[token]Type >>], [token]Raw Text >>]]]
 		}
 	}
 	[tokens]Next[next index]
diff -r 147dfc703161 -r 0a4682be2db2 string.rhope
--- a/string.rhope	Fri Nov 05 02:42:45 2010 +0000
+++ b/string.rhope	Fri Nov 05 02:43:34 2010 +0000
@@ -1049,3 +1049,13 @@
 	}
 }
 
+After[text,prefix:after,not found]
+{
+	If[[text]Starts With[prefix]]
+	{
+		,after <- [text]Slice[[prefix]Length]
+	}{
+		not found <- text
+	}
+}
+