adding tokens to basic types

lib/parser/basic.citrus

@@ -42,19 +42,65 @@ grammar BasicTypes
 
   rule string_expression
     #"'" (/.*/ !"'") "'"
-     ('"' str:(!'"' .)* '"') {Ast::StringExpression.new capture(:str).to_str }
-  end
-
-  rule instance_variable
-    ('@' ivar:name_expression) { Ast::VariableExpression.new (capture(:ivar).value).name }
+     ('"' str:(!'"' .)* '"') {Ast::StringExpression.new(capture(:str).to_str) }
   end
 
   rule basic_expression
-    name_expression | integer_expression | instance_variable |
+    name_expression | integer_expression  |
     module_name_expression | string_expression
   end
 
 
+  # Tokens are single or double character combinations with "meaning"
+  # braces, comman, point, questionmark , quotes, that kind of thing
+  # operator symbols are separate in Opreators
 
+  rule left_parenthesis
+     '('  space?
+  end
+
+  rule right_parenthesis
+    ')' space?
+  end
+
+  rule left_brace
+    '{'  space?
+  end
+
+  rule right_brace
+    '}'  space?
+  end
+
+  rule left_bracket
+    '['  space?
+  end
+
+  rule right_bracket
+    ']'  space?
+  end
+
+  rule association
+    "=>"  space?
+  end
+
+  rule comma
+    ','  space?
+  end
+
+  rule colon
+    ':'  space?
+  end
+
+  rule semicolon
+    ';'  space?
+  end
+
+  rule question_mark
+    '?' space?
+  end
+
+  rule excamation_mark
+    '!' space?
+  end
 
 end

lib/parser/basic_types.rb

@@ -1,49 +0,0 @@
-module Parser
-  # Basic types are numbers and strings
-  module BasicTypes
-    include output
-    # space really is just space. ruby is newline sensitive, so there is more whitespace footwork
-    # rule of thumb is that anything eats space behind it, but only space, no newlines
-    rule(:space)  { (str('\t') | str(' ')).repeat(1) }
-    rule(:space?) { space.maybe }
-    rule(:linebreak){ str("\n") >> space? >> linebreak.repeat }
-    
-    rule(:quote)      { str('"') }
-    rule(:nonquote)   { str('"').absent? >> any }
-
-    rule(:comment){ match('#') >> (linebreak.absent? >> any).repeat >> linebreak }
-    rule(:newline) { linebreak | comment }
-    rule(:eol) { newline  | any.absent? }
-    
-    rule(:double_quote){ str('"') }
-    rule(:minus) { str('-') }
-    rule(:plus) { str('+') }
-
-    rule(:sign) { plus | minus }
-    rule(:dot) {  str('.') }
-    rule(:digit) { match('[0-9]') }
-    rule(:exponent) { (str('e')| str('E')) }
-     
-    # identifier must start with lower case
-    # TODO rule forbit names like if_true, because it starts with a keyword. a little looser please!
-    rule(:name)   { keyword.absent? >> (match['a-z_'] >> match['a-zA-Z0-9_'].repeat).as(:name)  >> space? }
-    # instance variables must have the @
-    rule(:instance_variable) { (str('@') >> name).as(:instance_variable) }
-    # and class/module names must start with capital 
-    # (admittatly the rule matches constants too, but one step at a time)
-    rule(:module_name) { keyword.absent? >> (match['A-Z'] >> match['a-zA-Z0-9_'].repeat).as(:module_name)  >> space? }
-    
-    rule(:escape)     { str('\\') >> any.as(:esc) }
-    rule(:string)     { quote >> (
-        escape | 
-        nonquote.as(:char)
-      ).repeat(1).as(:string) >> quote }
-    
-    rule(:integer)    { sign.maybe >> digit.repeat(1).as(:integer) >> space? }
-    
-    rule(:float) { integer >>  dot >> integer >> 
-                            (exponent >> sign.maybe >> digit.repeat(1,3)).maybe >> space?}
-    rule(:basic_type){ integer | name | string | float | instance_variable | module_name | 
-                       keyword_true | keyword_false | keyword_nil }
-  end
-end

lib/parser/tokens.rb

@@ -1,22 +0,0 @@
-module Parser
-  # Tokens are single or double character combinations with "meaning"
-  # braces, comman, point, questionmark , quotes, that kind of thing
-  # operator symbols are separate in Opreators
-  module Tokens
-    include output
-    rule(:left_parenthesis) { str('(') >> space? }
-    rule(:right_parenthesis) { str(')') >> space? }
-    rule(:left_brace)  { str('{')    >> space? }
-    rule(:right_brace)  { str('}')    >> space? }
-    rule(:left_bracket)  { str('[')    >> space? }
-    rule(:right_bracket)  { str(']')    >> space? }
-
-    rule(:association)  { str("=>") >> space? }
-    rule(:comma)  { str(',') >> space? }
-    rule(:colon)  { str(':') >> space? }
-    rule(:semicolon)  { str(';') >> space? }
-    rule(:question_mark)  { str('?') >> space? }
-    rule(:excamation_mark)  { str('!') >> space? }  
-
-  end
-end