diff --git a/Scanning.py b/Scanning.py index 83bbc0d72ca029e70832960b046989fac332d107..e22086256feb5315f1dac43dfe3c65c0495ca08d 100644 --- a/Scanning.py +++ b/Scanning.py @@ -1,38 +1,30 @@ import re # regex import string - # class ScanDFA(): # scans the input line by line, and char by char for each line # explicitly recognize whitespace when scanning, but discard whitespace tokens at the end +##################### Token ########################################## +# A Token is a pair (name, lexeme) +class Token(){ + def __init__(self, name, lex): + self.name = name + self.lex = lex +} + ################# Joos Token Names in 5 categoeis ########################## JoosTokens = set([ - # Literals: 'INT', # Operants: '+', - - # Separators: 'L(', - - - # Keywords: 'ID', '', 'SQL', 'Git', 'Tableau', 'SAS', - - ]) -##################### Token ########################################## -# A Token is a pair (name, lexeme) -class Token(){ - def __init__(self, name, lex): - self.name = name - self.lex = lex -} ##################### Transition function ############################ @@ -50,6 +42,14 @@ def JoosTransition(input, state): ##################### Other DFA elements ############################## +JoosAccept = JoosTokens.union({'WHITESPACE', 'COMMENT'}) +JoosStates = JoosAccept.union({'START'}) #TODO: add intermediate states here +JoosAlphabet = set(string.ascii_lowercase) + .union(set(string.ascii_uppercase)) + .union(set(string.digits)) + .union(set(list(".;,{}()[]<>!+-*/=''\"\\"))), + #TODO: add operand and separator characters to alphabet + ######################### DFA ####################################### class DFA (): def __init__(self, states, alphabet, transition, start, accepting): @@ -69,10 +69,7 @@ class DFA (): JoosDFA = DFA( states = JoosTokens, - alphabet = set(string.ascii_lowercase) - .union(set(string.ascii_uppercase)) - .union(set(list(".;,{}()[]<>!+-*/=''\"\\"))), - #TODO: add operand and separator characters to alphabet + alphabet = JoosAlphabet, start = 'START', accept = JoosTokens, transition = JoosTransition @@ -104,5 +101,5 @@ def scan(input): # TODO: handle edge cases (e.g. check int range, error on ++ # remove whitespace and comments - tokens = filter(lambda t: t.name not in ("WHITESPACE", "COMMENT"), tokens) + tokens = filter(lambda t: t.name not in ("WHITESPACE", "COMMENT"), tokens) return tokens diff --git a/test.joos b/test.joos new file mode 100644 index 0000000000000000000000000000000000000000..2ce35b458fc6e5c28e07d517639e77d5f52606e0 --- /dev/null +++ b/test.joos @@ -0,0 +1,4 @@ +public class A { + public A() {} + protected boolean x; +}