added char, string, bool, null literals

Scanning.py

@@ -14,10 +14,11 @@ class Token():
 ##################### Joos Tokens Map ###############################
 
 # For tokens that are recognized as another name in the maximal munch scanner
-# e.g. all keywords are scanned as ID first
+# e.g. all keywords are scanned as ID first, as well as boolean literals, and null
 # Key: lexeme, Value: name, use lexeme to reassign those tokens correct names
 
-keywordDict = dict({
+idToTokenDict = dict({
+    # Keywords
     'abstract': 'ABSTRACT',
     'boolean': 'BOOLEAN',
     'byte': 'BYTE',
@@ -42,20 +43,25 @@ keywordDict = dict({
     'String': 'STRING',
     'this': 'THIS',
     'void': 'VOID',
-    'while': 'WHILE'
+    'while': 'WHILE',
+
+    # Literals
+    'true': 'LITERALBOOL',
+    'false': 'LITERALBOOL',
+    'null': 'NULL'
     })
 
-def tokenToKeywords(tokens):
+def idsToTokens(tokens):
     for t in tokens:
         if t.name == 'ID':
-            if t.lex in keywordDict:
-                t.name = keywordDict.get(t.lex)
+            if t.lex in idToTokenDict:
+                t.name = idToTokenDict.get(t.lex)
 
 ######################## DFA Stuff ###################################
 
 #################  Joos DFA Tokens ###################################
 JoosDFATokens = set([
-    # Identifiers or Keywords (keywords are mapped to their token name in keywordDict above)
+    # Identifiers or Keywords (keywords are mapped to their token name in idToTokenDict above)
     'ID',             
 
     # Literals and names (note: 'this' is considered as a keyword)
@@ -115,8 +121,6 @@ def JoosTransition(input, state):
             return 'WHITESPACE'
         else:
             return None
-    elif (state != 'START' and input in (' ', '\n')):
-        return None
 
     elif (state == 'START'):
         if (input.isalpha()):
@@ -183,13 +187,38 @@ def JoosTransition(input, state):
             return 'PERIOD'
         if (input == '@'):
             return 'AT'
-     
+        
+        # literals
+        if (input == '\"'):
+            return 'LSTRING'
+        if (input == '\''):
+            return 'LCHAR'
 
     elif (state == 'ID'):
         if (input.isalpha() or input.isdigit()):
             return 'ID'
         return None
 
+    elif (state == 'LSTRING'):
+        if (input == '\\'):
+            return 'STRINGESC'
+        if (input == '\"'):
+            return 'LITERALSTRING'
+        return 'LSTRING'
+    elif (state == 'STRINGESC'):
+        return 'LSTRING'
+
+    elif (state == 'LCHAR'):
+        if (input == '\\'):
+            return 'CHARESC'
+        return 'CHAREND'
+    elif (state == 'CHARESC'):
+        return 'CHAREND'
+    elif (state == 'CHAREND'):
+        if (input == '\''):
+            return 'LITERALCHAR'
+        return None
+
     else:
         return None
 
@@ -247,7 +276,7 @@ def scan(input):
     if (tokens):
         # TODO: handle edge cases (e.g. check int range)
 
-        tokenToKeywords(tokens)
+        idsToTokens(tokens)
 
         # remove whitespace and comments
         tokens = filter(lambda t: t.name not in ("WHITESPACE", "COMMENT"), tokens)

Tests/literals.joos

+public class Literals {
+    public static void main(String[] args) {
+        char i    = 'h';
+        char x    = '\'';
+        String s  = "hello";
+        String s2 = "hello \' and \"";
+        boolean b = true;
+        boolean a = false;
+        int x     = null;
+    }
+}
\ No newline at end of file