Mercurial > repos > rhope
changeset 131:0a4682be2db2
Modify lexer and new parser to work in compiler
| author | Mike Pavone <pavone@retrodev.com> |
|---|---|
| date | Fri, 05 Nov 2010 02:43:34 +0000 |
| parents | 147dfc703161 |
| children | 1f238280047f |
| files | compile.rhope countstring.rhope lex.rhope parse.rhope string.rhope |
| diffstat | 5 files changed, 160 insertions(+), 77 deletions(-) [+] |
line wrap: on
line diff
--- a/compile.rhope Fri Nov 05 02:42:45 2010 +0000 +++ b/compile.rhope Fri Nov 05 02:43:34 2010 +0000 @@ -7,11 +7,12 @@ [args]Index[1] { Print[["Parsing "]Append[~]] - file <- <String@File[~] - ,data <- [file]Get FString[[file]Length] + file <- [File[~]]Open["r"] + data <- String[[file]Read[[file]Length]] tokens <- Lex[Count String[data]] - Pretty Print[Parse[tokens], ""] + Print[Parse[tokens], ""] }{ Print["Usage: rhope compile.rhope <filename>"] } } +
--- a/countstring.rhope Fri Nov 05 02:42:45 2010 +0000 +++ b/countstring.rhope Fri Nov 05 02:43:34 2010 +0000 @@ -13,7 +13,7 @@ Count String At Pos[text,line,col:out] { - out <- [[[Build["Count String"]]Text <<[text]]Line <<[line]]Column <<[col] + out <- [[[Build[Count String()]]Text <<[text]]Line <<[line]]Column <<[col] } Slice@Count String[string,index:left,right] @@ -45,11 +45,21 @@ out <- Length[[string]Text >>] } +Byte Length@Count String[string:out] +{ + out <- Byte Length[[string]Text >>] +} + +Byte@Count String[string,index:out,invalid] +{ + out,invalid <- Byte[[string]Text >>, index] +} + Append@Count String[left,right:out] { If[[[left]Length] > [0]] { - out <- [left]Text << [ [To String[left]]Append[To String[right]] ] + out <- [left]Text << [ [String[left]]Append[String[right]] ] }{ out <- right } @@ -57,13 +67,44 @@ =@Count String[left,right:out] { - out <- [To String[left]] = [To String[right]] + out <- [String[left]] = [String[right]] +} + +_Slice=@Count String[left,right:out] +{ + out <- _Slice=[String[left], String[right]] +} + +_Flat=@Count String[left,right:out] +{ + out <- _Flat=[String[left], String[right]] +} + +_Flatten@Count String[string,dest,offset,count:out] +{ + out <- _Flatten[[string]Text >>, dest, offset, count] } -Get DString@Count String[string, delims:after,before,delim,nomatch] +/*Partition@Count String[string, delims:before,delim,after,nomatch] { - ,,delim,nomatch <- [[string]Text >>]Get DString[delims] + ,delim,,nomatch <- [[string]Text >>]Partition[delims] { + before <- [string]Text <<[~] + bparts <- [~]Split["\n"] + preblines <- [[bparts]Length] - [1] + If[[preblines] > [-1]] + { + blines <- Val[preblines] + bcols <- Length[[bparts]Index[blines]] + }{ + bcols <- 0 + blines <- 0 + } + }{ + dparts <- [~]Split["\n"] + dlines <- [[dparts]Length]-[1] + dcols <- Length[[dparts]Index[dlines]] + }{ If[[dlines] > [0]] { cols <- Val[dcols] @@ -76,26 +117,10 @@ } } after <- Count String At Pos[~, [[blines]+[dlines]] + [[string]Line >>], cols] - }{ - before <- [string]Text <<[~] - bparts <- [~]Split["\n"] - preblines <- [[bparts]Length] - [1] - If[[preblines] > [-1]] - { - blines <- Val[preblines] - bcols <- Length[[bparts]Index[blines]] - }{ - bcols <- 0 - blines <- 0 - } - }{ - dparts <- [~]Split["\n"] - dlines <- [[dparts]Length] - [1] - dcols <- Length[[dparts]Index[dlines]] } -} +}*/ -To String@Count String[string:out] +String@Count String[string:out] { out <- [string]Text >> } @@ -104,3 +129,4 @@ { out <- Count String[""] } +
--- a/lex.rhope Fri Nov 05 02:42:45 2010 +0000 +++ b/lex.rhope Fri Nov 05 02:43:34 2010 +0000 @@ -1,4 +1,3 @@ -Import extendlib.rhope Blueprint Token { @@ -9,7 +8,7 @@ Token[type,raw,text:out] { - out <- [[[Build["Token"]]Type <<[type]]Raw Text <<[raw]]Text <<[text] + out <- [[[Build[Token()]]Type <<[type]]Raw Text <<[raw]]Text <<[text] } _Type Match[val, test type, type:out] @@ -22,9 +21,19 @@ } } +As List[val:out] +{ + [(List(),List Leaf())]Find[=[?, Blueprint Of[val]]] + { + out <- val + }{ + out <- [()]Append[val] + } +} + Type Match@Token[token,type:match,nomatch] { - match,nomatch <- If[Fold[["_Type Match"]Set Input[2, [token]Type >>], No, As List[type]]] + match,nomatch <- If[Fold[_Type Match[?,?, [token]Type >>], No, As List[type]]] } String Literal[string, raw string, escapes, text, simple tokens, token list:out] @@ -38,7 +47,7 @@ If[[first] = ["\\"]] { second,next text <- [rest]Slice[1] - char <- [escapes]Index[To String[second]] {} + char <- [escapes]Index[String[second]] {} { char <- Val[second] } @@ -53,9 +62,10 @@ Line Comment[start comment, text, simple tokens, token list:out] { - next text, comment <- [text]Get DString["\n"] {} {} {} + comment,,next text <- [text]Partition["\n"] {} {} {} { next text <- "" + comment <- Val[text] } out <- _Lex[next text, [next text]Slice[0], simple tokens, [token list]Append[Token["Line Comment", [start comment]Append[comment], comment]]] @@ -63,13 +73,14 @@ Block Comment[comment,raw comment, depth, text, simple tokens, token list:out] { - Print[["Block Comment: Depth="]Append[depth]] + Print[["Block Comment: Depth="]Append[String[depth]]] If[[depth] > [0]] { - next text, chunk, delim <- [text]Get DString[("/*","*/")] {} {} {} + chunk, delim, next text <- [text]Partition[("/*","*/")] {} {} {} { next text <- "" delim <- "" + chunk <- Val[text] } If[[delim] = ["/*"]] { @@ -110,14 +121,14 @@ If[[[text]Length] > [0]] { first,rest <- [text]Slice[1] - [simple tokens]Index[To String[first]] + [simple tokens]Index[String[first]] { - token worker <- [["Add Token"]Set Input[0, Token[~, first, ""]]]Set Input[1, rest] + token worker <- Val[Add Token[Token[~, first, ""], rest, ?]] }{ If[[first] = ["\""]] { - escapes <- [[[New@Dictionary[]]Set["n","\n"]]Set["r","\r"]]Set["t","\t"] - token worker <- [[[["String Literal"]Set Input[0, [first]Slice[0]]]Set Input[1, first]]Set Input[2, escapes]]Set Input[3, rest] + escapes <- [[[Dictionary[]]Set["n","\n"]]Set["r","\r"]]Set["t","\t"] + token worker <- Val[String Literal[[first]Slice[0], first, escapes, rest, ?]] //out <- String Literal["", first, rest, simple tokens, token list, escapes] }{ second,second rest <- [rest]Slice[1] @@ -125,23 +136,23 @@ { [first]Append[second] { - token worker <- [["Add Token"]Set Input[0, Token["Assignment", ~, ~]]]Set Input[1, second rest] + token worker <- Val[Add Token[Token["Assignment", ~, ~], second rest, ?]] } }{ If[[[first] = ["/"]] And [[second] = ["*"]]] { - token worker <- [[[["Block Comment"]Set Input[0, [first]Slice[0]]]Set Input[1, [first]Append[second]]]Set Input[2, 1]]Set Input[3, second rest] + token worker <- Val[Block Comment[[first]Slice[0], [first]Append[second], 1, second rest, ?]] //out <- Block Comment[next text, simple tokens, token list, 1] }{ If[[[first] = ["/"]] And [[second] = ["/"]]] { - token worker <- [["Line Comment"]Set Input[0, [first]Append[second]]]Set Input[1, second rest] + token worker <- Val[Line Comment[[first]Append[second], second rest, ?]] //out <- Line Comment["", [first]Append[second], next text, simple tokens, token list] }{ If[[[first]In["0123456789"]] Or [[[first] = ["-"]] And [[second]In["0123456789"]]]] { - token worker <- [["Numeric Literal"]Set Input[0, first]]Set Input[1, rest] + token worker <- Val[Numeric Literal[first, rest, ?]] //out <- Numeric Literal[text, simple tokens, token list] }{ out <- _Lex[rest, [symbol]Append[first], simple tokens, token list] @@ -161,9 +172,7 @@ }{ next list <- [token list]Append[Token["Symbol", trimmed, trimmed]] } - out <- [[token worker]Do[ - [[()]Append[simple tokens]]Append[next list]] - ]Index[0] + out <- [token worker]Call[simple tokens, next list] } }{ out <- token list @@ -172,7 +181,7 @@ Lex[text:out] { - simple tokens <- [[[[[[[[[[[New@Dictionary[] + simple tokens <- [[[[[[[[[[[Dictionary[] ]Set["{", "Block Begin"] ]Set["}", "Block End"] ]Set["(", "List Begin"]
--- a/parse.rhope Fri Nov 05 02:42:45 2010 +0000 +++ b/parse.rhope Fri Nov 05 02:43:34 2010 +0000 @@ -8,12 +8,12 @@ Error[msg,text:out] { - out <- [[[Build["Error"]]Message <<[msg]]Line << [ [[text]Line >>]+[1] ]]Column << [ [[text]Column >>]+[1] ] + out <- [[[Build[Error()]]Message <<[msg]]Line << [ [[text]Line >>]+[1] ]]Column << [ [[text]Column >>]+[1] ] } -To String@Error[error:out] +String@Error[error:out] { - out <- [[[[[error]Message >>]Append[" on line "]]Append[[error]Line >>]]Append[" at column "]]Append[[error]Column >>] + out <- [[[[[error]Message >>]Append[" on line "]]Append[String[[error]Line >>]]]Append[" at column "]]Append[String[[error]Column >>]] } Blueprint PImport Node @@ -23,7 +23,12 @@ PImport Node[file:out] { - out <- [Build["PImport Node"]]File <<[file] + out <- [Build[PImport Node()]]File <<[file] +} + +String@PImport Node[node:out] +{ + out <- ["Import: "]Append[[node]File >>] } Blueprint Worker Node @@ -38,7 +43,7 @@ Add Node Input@Worker Node[node,input:out] { Print["Add Node Input"] - Print[["Input: "]Append[To String[input]]] + Print[["Input: "]Append[String[input]]] out <- [node]Inputs <<[[[node]Inputs >>]Append[input]] } @@ -47,6 +52,15 @@ out <- [node]Outputs <<[[[node]Outputs >>]Append[output]] } +String@Worker Node[node:out] +{ + out <- [[[[["Worker: "]Append[[node]Name >>] + ]Append[ ["\n\tBlueprint:\t"]Append[String[[node]Blueprint >>]] ] + ]Append[ ["\n\tInputs: \t"]Append[[[node]Inputs >>]Join[", "]] ] + ]Append[ ["\n\tOutputs:\t"]Append[[[node]Outputs >>]Join[", "]] ] + ]Append[ ["\n\tTree: \t"]Append[ [[String[[node]Tree >>]]Split["\n"]]Join["\n\t"] ] ] +} + Blueprint Statement { Expression @@ -56,7 +70,7 @@ Statement[expression,assignments:out] { - out <- [[[Build["Statement"]]Expression <<[expression]]Assignments <<[assignments]]Blocks <<[()] + out <- [[[Build[Statement()]]Expression <<[expression]]Assignments <<[assignments]]Blocks <<[()] } Add Block@Statement[statement,block:out] @@ -64,6 +78,14 @@ out <- [statement]Blocks <<[ [[statement]Blocks >>]Append[block] ] } +String@Statement[statement:out] +{ + out <- [[["Statement\n\tExpression:\t" + ]Append[ [[String[[statement]Expression >>]]Split["\n"]]Join["\n\t"] ] + ]Append[ ["\n\tAssignments:\t"]Append[[[statement]Assignments >>]Join[", "]] ] + ]Append[ ["\n\tBlocks:\t"]Append[ [[String[[statement]Blocks >>]]Split["\n"]]Join["\n\t"] ] ] +} + Blueprint Worker Expression { Worker @@ -71,6 +93,11 @@ Arguments } +String@Worker Expression[exp:out] +{ + out <- ["Worker Expression\n\tWorker:\t"]Append[[exp]Worker >>] +} + Blueprint Global Expression { Store @@ -84,7 +111,12 @@ Pipe Expression[name:out] { - out <- [Build["Pipe Expression"]]Name <<[name] + out <- [Build[Pipe Expression()]]Name <<[name] +} + +String@Pipe Expression[pipe:out] +{ + out <- ["Pipe Expression\n\tName:\t"]Append[[pipe]Name >>] } Blueprint Block @@ -94,7 +126,12 @@ Block[:out] { - out <- [Build["Block"]]Tree <<[()] + out <- [Build[Block()]]Tree <<[()] +} + +String@Block[block:out] +{ + out <- Fold[_String Seq[?], "Block", [block]Tree >>] } Blueprint Field Expression @@ -105,12 +142,12 @@ Field Expression[name,set?:out] { - out <- [[Build["Field Expression"]]Name <<[name]]Set? <<[set?] + out <- [[Build[Field Expression()]]Name <<[name]]Set? <<[set?] } Body[node,tokens,current,depth:out,out index, done] { - Print[["Body: Depth="]Append[depth]] + Print[["Body: Depth="]Append[String[depth]]] If[[depth] > [0]] { token <- [tokens]Index[current] @@ -147,7 +184,7 @@ } Val[end stream] { - Print[To String[Error[["Unexpected end of stream after token of type "]Append[[token]Type >>], [token]Raw Text >>]]] + Print[String[Error[["Unexpected end of stream after token of type "]Append[[token]Type >>], [token]Raw Text >>]]] } } @@ -166,7 +203,7 @@ } Val[end stream] { - Print[To String[Error[["Unexpected end of stream after token of type "]Append[[token]Type >>], [token]Raw Text >>]]] + Print[String[Error[["Unexpected end of stream after token of type "]Append[[token]Type >>], [token]Raw Text >>]]] } } @@ -188,14 +225,14 @@ { out,out index, done <- Outputs[node, tokens, next] }{ - Print[To String[Error[["Unexpected token of type "]Append[[token]Type >>], [token]Raw Text >>]]] + Print[String[Error[["Unexpected token of type "]Append[[token]Type >>], [token]Raw Text >>]]] } } } } Val[end stream] { - Print[To String[Error[["Unexpected end of stream after token of type "]Append[[token]Type >>], [token]Raw Text >>]]] + Print[String[Error[["Unexpected end of stream after token of type "]Append[[token]Type >>], [token]Raw Text >>]]] } } @@ -223,7 +260,7 @@ { out,out index,done <- Inputs[node, tokens, next] }{ - Print[To String[Error[["Unexpected token of type "]Append[[token]Type >>], [token]Raw Text >>]]] + Print[String[Error[["Unexpected token of type "]Append[[token]Type >>], [token]Raw Text >>]]] } } } @@ -231,7 +268,7 @@ } Val[end stream] { - Print[To String[Error[["Unexpected end of stream after token of type "]Append[[token]Type >>], [token]Raw Text >>]]] + Print[String[Error[["Unexpected end of stream after token of type "]Append[[token]Type >>], [token]Raw Text >>]]] } } @@ -248,11 +285,11 @@ { out,out index,done <- Method[node, tokens, next] }{ - Print[To String[Error[["Unexpected token of type "]Append[[token]Type >>], [token]Raw Text >>]]] + Print[String[Error[["Unexpected token of type "]Append[[token]Type >>], [token]Raw Text >>]]] } } }{ - Print[To String[Error[["Unexpected end of stream after token of type "]Append[[token]Type >>], [token]Raw Text >>]]] + Print[String[Error[["Unexpected end of stream after token of type "]Append[[token]Type >>], [token]Raw Text >>]]] } } @@ -267,7 +304,7 @@ { out,out index, done <- Inputs[node, tokens, ~] }{ - Print[To String[Error[["Unexpected end of stream after token of type "]Append[[token]Type >>], [token]Raw Text >>]]] + Print[String[Error[["Unexpected end of stream after token of type "]Append[[token]Type >>], [token]Raw Text >>]]] } }{ [token]Type Match["Method Separator"] @@ -276,14 +313,14 @@ { out,out index,done <- Method[node, tokens, ~] }{ - Print[To String[Error[["Unexpected end of stream after token of type "]Append[[token]Type >>], [token]Raw Text >>]]] + Print[String[Error[["Unexpected end of stream after token of type "]Append[[token]Type >>], [token]Raw Text >>]]] } }{ [token]Type Match[("Line Comment","Block Comment","Newline")] { continue <- Yes }{ - Print[To String[Error[["Unexpected token of type "]Append[[token]Type >>], [token]Raw Text >>]]] + Print[String[Error[["Unexpected token of type "]Append[[token]Type >>], [token]Raw Text >>]]] } Val[continue] { @@ -291,7 +328,7 @@ { out,out index, done <- Before Inputs[node, tokens, ~] }{ - Print[To String[Error[["Unexpected end of stream after token of type "]Append[[token]Type >>], [token]Raw Text >>]]] + Print[String[Error[["Unexpected end of stream after token of type "]Append[[token]Type >>], [token]Raw Text >>]]] } } } @@ -300,8 +337,8 @@ Worker Node[name, tokens, current:out,out index, done] { - Print[["Worker: "]Append[To String[name]]] - out,out index, done <- Before Inputs[[[[[Build["Worker Node"]]Name <<[name]]Inputs <<[()]]Outputs <<[()]]Tree <<[()], tokens, current] + Print[["Worker: "]Append[String[name]]] + out,out index, done <- Before Inputs[[[[[[Build[Worker Node()]]Name <<[name]]Inputs <<[()]]Outputs <<[()]]Blueprint <<[No]]Tree <<[()], tokens, current] } Skip Nodes[toskip, stop, tokens, current:out] @@ -317,11 +354,11 @@ { out <- Skip Nodes[toskip, stop, tokens, next] }{ - Print[To String[Error[["Unexpected end of stream after token of type "]Append[[token]Type >>], [token]Raw Text >>]]] + Print[String[Error[["Unexpected end of stream after token of type "]Append[[token]Type >>], [token]Raw Text >>]]] } } }{ - Print[To String[Error[["Unexpected end of stream after token of type "]Append[[token]Type >>], [token]Raw Text >>]]] + Print[String[Error[["Unexpected end of stream after token of type "]Append[[token]Type >>], [token]Raw Text >>]]] } } @@ -349,7 +386,7 @@ { out,out index,done <- Fields[node, tokens, next] }{ - Print[To String[Error[["Unexpected end of stream after token of type "]Append[[token]Type >>], [token]Raw Text >>]]] + Print[String[Error[["Unexpected end of stream after token of type "]Append[[token]Type >>], [token]Raw Text >>]]] } } } @@ -359,14 +396,14 @@ out <- Val[node] done <- Yes }{ - Print[To String[Error[["Unexpected end of stream after token of type "]Append[[token]Type >>], [token]Raw Text >>]]] + Print[String[Error[["Unexpected end of stream after token of type "]Append[[token]Type >>], [token]Raw Text >>]]] } } } PBlueprint Node[name, tokens, current:out,out index,done] { - node <- [[Build["PBlueprint Node"]]Name <<[name]]Fields <<[()] + node <- [[Build[PBlueprint Node()]]Name <<[name]]Fields <<[()] next <- Skip Nodes[("Newline","Block Comment","Comment"), "Block Begin", tokens, current] out,out index,done <- Fields[node, tokens, next] } @@ -399,7 +436,7 @@ out <- Val[next nodes] } }{ - Print[To String[Error[["Unexpected end of stream after symbol "]Append[[token]Text >>], [token]Raw Text >>]]] + Print[String[Error[["Unexpected end of stream after symbol "]Append[[token]Text >>], [token]Raw Text >>]]] } }{ [tokens]Next[current] @@ -412,7 +449,7 @@ out <- Val[next nodes] } }{ - Print[To String[Error[["Unexpected end of stream after symbol "]Append[[token]Text >>], [token]Raw Text >>]]] + Print[String[Error[["Unexpected end of stream after symbol "]Append[[token]Text >>], [token]Raw Text >>]]] } } } @@ -423,7 +460,7 @@ next nodes <- Val[nodes] next index <- Val[current] }{ - Print[To String[Error[["Unexpected token of type "]Append[[token]Type >>], [token]Raw Text >>]]] + Print[String[Error[["Unexpected token of type "]Append[[token]Type >>], [token]Raw Text >>]]] } } [tokens]Next[next index]