use utf safe regex basics , some renaming

lib/parser/basic.citrus

@@ -1,22 +1,35 @@
 
 grammar BasicTypes
 
-  # space really is just space. ruby is newline sensitive, so there is more whitespace footwork
-  # rule of thumb is that anything eats space behind it, but only space, no newlines
-  rule space  [ \t]*  end
+  # unicode generalized categories , according to regex ruby page
+  rule lower  /[[:lower:]]/ end    # Lowercase alphabetical character
+  rule upper  /[[:upper:]]/ end    # Uppercase alphabetical
+  rule alnum  /[[:alnum:]]/ end    # Alphabetic and numeric character
+  rule alpha  /[[:alpha:]]/ end    # Alphabetic character
+  rule blank  /[[:blank:]]/ end    # Space or tab
+  rule space  /[[:space:]]/ end    # Whitespace character ([:blank:], newline, carriage return, etc.)
+  rule digit  /[[:digit:]]/ end    # Digit
+  rule graph  /[[:graph:]]/ end    # Non-blank character (excludes spaces, control characters, and similar)
+  rule print  /[[:print:]]/ end    # Like [:graph:], but includes the space character
+  rule xdigit /[[:xdigit:]]/ end   # Digit allowed in a hexadecimal number (i.e., 0-9a-fA-F)
 
-  rule linebreak "\n" end
+  #  ruby is newline sensitive, so there is more whitespace footwork
+  #  rule of thumb is that anything eats space behind it, but only blank, no newlines
+  #  comments are also deliminatord, but also don't include the newline
+  rule deliminator  blank*  comment? end
 
-  rule comment
-    "#" (/.*/ !linebreak) linebreak
+  rule linebreak (!blank space) end  #define in regex terms for utf
+
+  rule comment  #don't include the newline (which ends the comment)
+    "#" /.*/
   end
 
   rule name_expression
-     (name:([a-z_] [a-zA-Z0-9_]*) space?) { Ast::NameExpression.new(capture(:name).to_str)}
+     (name:([a-z_] [a-zA-Z0-9_]*) deliminator) { Ast::NameExpression.new(capture(:name).to_str)}
   end
 
   rule module_name_expression
-     (name:([A-Z] [a-zA-Z0-9_]*) space?) { Ast::ModuleName.new(capture(:name).to_str)}
+     (name:([A-Z] [a-zA-Z0-9_]*) deliminator) { Ast::ModuleName.new(capture(:name).to_str)}
   end
 
   rule digits
@@ -24,7 +37,7 @@ grammar BasicTypes
   end
 
   rule integer_expression
-    (digits space?) { Ast::IntegerExpression.new(to_str.to_i) }
+    (digits deliminator) { Ast::IntegerExpression.new(to_str.to_i) }
   end
 
   rule string_expression
@@ -40,4 +53,8 @@ grammar BasicTypes
     name_expression | integer_expression | instance_variable |
     module_name_expression | string_expression
   end
+
+
+
+
 end

lib/parser/keywords.citrus

@@ -1,37 +1,37 @@
 grammar Keywords
   include BasicTypes
 
-  rule keyword_begin    'begin'  space? end
-  rule keyword_class    'class'  space?  end
-  rule keyword_def      'def'    space?  end
-  rule keyword_do       'do'     space? end
-  rule keyword_else     'else'   space?  end
-  rule keyword_end      'end'    space?  end
-  rule keyword_if       'if'     space?  end
-  rule keyword_rescue   'rescue' space? end
-  rule keyword_return   'return' space? end
-  rule keyword_module   'module' space?  end
-  rule keyword_unless   'unless' space? end
-  rule keyword_until    'until'  space? end
-  rule keyword_while    'while'  space? end
+  rule keyword_begin    'begin'  deliminator end
+  rule keyword_class    'class'  deliminator  end
+  rule keyword_def      'def'    deliminator  end
+  rule keyword_do       'do'     deliminator end
+  rule keyword_else     'else'   deliminator  end
+  rule keyword_end      'end'    deliminator  end
+  rule keyword_if       'if'     deliminator  end
+  rule keyword_rescue   'rescue' deliminator end
+  rule keyword_return   'return' deliminator end
+  rule keyword_module   'module' deliminator  end
+  rule keyword_unless   'unless' deliminator end
+  rule keyword_until    'until'  deliminator end
+  rule keyword_while    'while'  deliminator end
 
   rule keyword_nil
-    ('nil'    space? ){ Ast::NilExpression.new }
+    ('nil'    deliminator ){ Ast::NilExpression.new }
   end
 
   rule keyword_false
-    ('false'  space?) { Ast::FalseExpression.new }
+    ('false'  deliminator) { Ast::FalseExpression.new }
   end
 
   rule keyword_true
-    ('true'   space?) { Ast::TrueExpression.new }
+    ('true'   deliminator) { Ast::TrueExpression.new }
   end
 
   # this rule is just to make sure identifiers can't be keywords. Kind of duplication here, but we need the
   # space in above rules, so just make sure to add any here too.
   rule keyword   ('begin'  |  'def'  |  'do'  |  'else'  |  'end'  |
                    'false' |  'if' |  'rescue' |  'true' |  'nil'  |
-                   'unless' |  'until' |  'while') space?
+                   'unless' |  'until' |  'while') deliminator
   end
 
   rule keyword_expression