From 91dbad2f2004819b3ed9301d39221fb5d3987b0b Mon Sep 17 00:00:00 2001
From: pycsham <shampuiyanchloe@gmail.com>
Date: Sun, 16 Feb 2020 23:07:44 -0500
Subject: [PATCH] fixed integer range checking in Scanning
---
ChloeTests/ErrorTest.py | 69 +++++++++++++++++++++++++++++++++++++++
ChloeTests/Test.py | 72 +++++++++++++++++++++++++++++++++++++++++
Scanning.py | 46 +++++++++++++++++---------
3 files changed, 171 insertions(+), 16 deletions(-)
create mode 100644 ChloeTests/ErrorTest.py
create mode 100644 ChloeTests/Test.py
diff --git a/ChloeTests/ErrorTest.py b/ChloeTests/ErrorTest.py
new file mode 100644
index 0000000..aba22c8
--- /dev/null
+++ b/ChloeTests/ErrorTest.py
@@ -0,0 +1,69 @@
+import sys
+from os import listdir
+from os.path import isfile, join
+
+from Scanning import scan
+from Parsing import parse
+import Weeding
+
+
+def allFiles(testDir):
+ return [testDir + f for f in listdir(testDir) if isfile(join(testDir, f)) and f.startswith('Je')]
+
+def main():
+
+ # All files in the test directory
+ testDirectory = "./Tests/"
+ testFiles = allFiles(testDirectory)
+ # testFiles = ['./Tests/Je_1_IntRange_MinusTooBigInt.java']
+ print("**********************************************************")
+
+ for f in testFiles:
+ # print(f)
+
+ # Scanning
+ content = open(f, "r").read()
+ (tokens, errorString) = scan(content)
+ # Error in Scanning
+ if tokens is None:
+ # print("ERROR in Scanning: " + errorString)
+ # print("**********************************************************")
+ continue
+
+ # s = "All Tokens: "
+ # for token in tokens:
+ # if (token.name and token.lex):
+ # s += '(' + token.name + ',' + token.lex + '), '
+ # print(s)
+
+ # No weeds if everything is good (weeds = None)
+ weeds = Weeding.fileNameCheck(tokens, f)
+ if weeds:
+ # print(weeds)
+ continue
+
+
+
+ # Parsing
+ # print("Parsing starts")
+
+ try:
+ (steps, errorString) = parse(tokens)
+ except:
+ print("Exception in Parsing")
+
+ # Error in Parsing
+ if steps is None:
+ # print("ERROR in Parsing: ", errorString)
+ # print("**********************************************************")
+ continue
+
+ print(f)
+ print("Succeeded")
+ # print("All Steps:")
+ # print(steps)
+
+ print("**********************************************************")
+
+main()
+
diff --git a/ChloeTests/Test.py b/ChloeTests/Test.py
new file mode 100644
index 0000000..02941d7
--- /dev/null
+++ b/ChloeTests/Test.py
@@ -0,0 +1,72 @@
+import sys
+from os import listdir
+from os.path import isfile, join
+
+from Scanning import scan
+from Parsing import parse
+import Weeding
+
+
+def allFiles(testDir):
+ return [testDir + f for f in listdir(testDir) if isfile(join(testDir, f)) and f.startswith('J1')]
+
+def main():
+
+ # All files in the test directory
+ testDirectory = "./Tests/"
+ testFiles = allFiles(testDirectory)
+ testFiles = ['./Tests/J1_IntRange_MinNegativeInt.java']
+ print("**********************************************************")
+
+ for f in testFiles:
+ # print(f)
+
+ # Scanning
+ content = open(f, "r").read()
+ (tokens, errorString) = scan(content)
+ # Error in Scanning
+ if tokens is None:
+ print(f)
+ print("ERROR in Scanning: " + errorString)
+ print("**********************************************************")
+ continue
+
+ s = "All Tokens: "
+ for token in tokens:
+ if (token.name and token.lex):
+ s += '(' + token.name + ',' + token.lex + '), '
+ print(s)
+
+ # No weeds if everything is good (weeds = None)
+ weeds = Weeding.fileNameCheck(tokens, f)
+ if weeds:
+ print(weeds)
+ continue
+
+
+
+ # Parsing
+ # print("Parsing starts")
+
+ try:
+ (steps, errorString) = parse(tokens)
+ except:
+ print("Exception in Parsing")
+
+ # Error in Parsing
+ if steps is None:
+ # print(f)
+ # # print("ERROR IN PARSING")
+ # print("ERROR in Parsing: ", errorString)
+ # print("**********************************************************")
+ continue
+
+ # print(f)
+ # print("Succeeded")
+ # print("All Steps:")
+ # print(steps)
+
+ print("**********************************************************")
+
+main()
+
diff --git a/Scanning.py b/Scanning.py
index ab4ced5..5d86346 100644
--- a/Scanning.py
+++ b/Scanning.py
@@ -106,6 +106,7 @@ JoosDFATokens = set([
# Literals and names (note: 'this' is considered as a keyword)
'NUM', # number (excludes 0)
'ZERO', # 0
+ 'NEGATIVENUM', # to do integer range checking, will be mapped to NUM after checking
'LITERALBOOL', # true or false
'LITERALCHAR', # character e.g. 'c', includes escape characters?
'LITERALSTRING', # string e.g. "hello", includes escape sequences
@@ -238,7 +239,7 @@ def JoosTransition(input, state):
if (input == '\''):
return 'LCHAR'
- # Handling all operators that are not allowed in Joos (some cases are handled elsewhere)
+ # Handling all operators that are not allowed in Joos (some cases are handled elsewhere) and negative numbers
elif state == 'ADD':
if input == '+':
return 'OPDISCARD'
@@ -248,6 +249,8 @@ def JoosTransition(input, state):
elif state == 'SUB':
if (input == '>'):
return 'OPDISCARD'
+ if input.isdigit():
+ return 'NEGATIVENUM'
if input == '-':
return 'OPDISCARD'
if input == '=':
@@ -293,6 +296,13 @@ def JoosTransition(input, state):
if input == '.':
return 'FLOAT' # not accepted
return None
+
+ elif state == 'NEGATIVENUM':
+ if input.isdigit():
+ return 'NEGATIVENUM'
+ if input == '.':
+ return 'FLOAT' # not accepted
+ return None
# string literal
elif (state == 'LSTRING'):
@@ -372,7 +382,7 @@ def JoosTransition(input, state):
elif input == '=':
return 'OPDISCARD'
return None
-
+
# Comments
elif(state == 'DIV'):
@@ -494,12 +504,9 @@ def scan(input):
if index < indexRange-1:
if tokens[index+1].name == 'NUM':
return (None, "wrong integer literal: starts with 0")
- # Checking integer range (does not cover all edge cases)
- # elif token.name == 'NUM' and index > 0 and tokens[index-1].name == 'SUB' and int(token.lex) > 2147483648:
- # return (None, "integer too small")
- # elif token.name == 'NUM' and int(token.lex) > 2147483647 and (index is 0 or tokens[index-1].name is not 'SUB'):
- # return (None, "interger too large")
-
+ # Checking integer range
+ elif token.name == 'NEGATIVENUM' and int(token.lex) < -2147483648:
+ return (None, "negative int too large")
# dealing with keywords in Java but not in Joos
elif token.name == 'ID' and token.lex in wrongJavaKeyWordDict:
@@ -536,13 +543,20 @@ def scan(input):
idsToTokens(tokens)
# remove whitespace, newline characters and comments
- tokens = list(filter(lambda t: t.name not in ("WHITESPACE", "COMMENT", 'LCOMMENT', 'RCOMMENT', 'NEWLINE', 'NEWLINEC', 'LCOM2', 'LCOM3'), tokens))
+ tokens = list(filter(lambda t: t.name not in ("COMMENT", 'LCOMMENT', 'RCOMMENT', 'NEWLINE', 'NEWLINEC', 'LCOM2', 'LCOM3'), tokens))
for index,token in enumerate(tokens):
- # Checking integer range (does not cover all edge cases)
- if token.name == 'NUM' and index > 0 and tokens[index-1].name == 'SUB' and int(token.lex) > 2147483648:
- return (None, "integer too small")
- elif token.name == 'NUM' and int(token.lex) > 2147483647 and (index is 0 or tokens[index-1].name is not 'SUB'):
- return (None, "interger too large")
-
- return (tokens, "success")
+ if token.name == 'NUM' and int(token.lex) > 2147483647 and (index is 0 or tokens[index-1].name is not 'SUB'):
+ return (None, "positive int too large")
+
+ tokens = list(filter(lambda t: t.name is not 'WHITESPACE', tokens))
+
+ tokensNew = []
+ for index, t in enumerate(tokens):
+ if t.name is 'NEGATIVENUM':
+ tokensNew.append(Token('SUB', '-'))
+ tokensNew.append(Token('NUM', t.lex[1:]))
+ else:
+ tokensNew.append(t)
+
+ return (tokensNew, "success")
--
GitLab