import tokenize
import cStringIO as StringIO
import functools
import operator
from odict import OrderedDict
import JassUtility
import StructManager
#------------
# TOKENIZER
#------------
clean_tokens = tokenize.INDENT, tokenize.DEDENT, tokenize.NEWLINE, tokenize.ENDMARKER
class MissingReferenceError(JassUtility.ParseError):
pass
TOKEN_IDENTIFIER = 1
TOKEN_PUNCTUATION = 51
SMARTSTRUCT_NAME = "attachstruct"
FUNCTION_TYPES = ("function", "callable", "callback")
RESERVED_KEYWORDS = ("set", "call", "if", "then", "else", "elseif", "endif", "then", "loop", "endloop", "exitwhen", "return", "debug",
"function", "endfunction", "null", "and", "or", "not", "false", "true")
INDENTING_KEYWORDS = ("if", "then", "else", "elseif", "loop")
DEDENTING_KEYWORDS = ("endif", "endloop", "else", "elseif")
def CleanToken(token):
return token[0] not in clean_tokens
def GenerateTokens(string):
buf = StringIO.StringIO(string)
return filter(CleanToken, [t[0:2] for t in tokenize.generate_tokens(buf.readline)])
def GenerateTokenLines(script):
tokenlines = []
for line in script:
if line.strip():
buf = StringIO.StringIO(line)
try:
tokenlines.append((filter(CleanToken, [t[0:2] for t in tokenize.generate_tokens(buf.readline)]), line))
except tokenize.TokenError, e:
raise JassUtility.ParseError("Could not tokenize script at line '%s': %s" % (line, str(e)))
return tokenlines
def GenerateTokenLinesString(string):
buf = StringIO.StringIO(string)
return GenerateTokenLines(buf)
#------------
# UTILITY
#------------
def Nth(list, i):
return [l[i] for l in list]
def OdictInsert(od, name, value):
od.insert(0, name, value)
def OdictAppend(od, name, value):
od[name] = value
def ListInsert(list, value):
list.insert(0, value)
def ListAppend(list, value):
list.append(value)
#def OdictInsertBefore(od, name, value, key):
# od.insert(od.index(key), name, value)
class OdictInsertBefore:
def __init__(self, key):
self.key = key
self.inserted_once = False
def __call__(self, od, name, value):
if self.inserted_once:
od.insert(od.index(self.key)+1, name, value)
else:
od.insert(od.index(self.key), name, value)
self.inserted_once = True
self.key = name
class OdictInsertAfter:
def __init__(self, key):
self.key = key
def __call__(self, od, name, value):
od.insert(od.index(self.key)+1, name, value)
self.key = name
#def OdictInsertAfter(od, name, value, key):
# od.insert(od.index(key)+1, name, value)
def GetPrintValue(type, value):
#if identifier.type.IsInstance("struct")
# return '"struct %s(" + I2S(%s) + ")"' % (identifier.type.name, identifier.name)
if type.IsInstance("integer"):
return 'I2S(%s)' % value
elif type.IsInstance("real"):
return 'R2S(%s)' % value
elif type.IsInstance("boolean"):
return 'Jlib_B2S(%s)' % value
elif type.IsInstance("string"):
return '%s' % value
elif type.IsInstance("unit"):
return 'GetUnitName(%s)' % value
elif type.IsInstance("item"):
return 'GetItemName(%s)' % value
elif type.IsInstance("player"):
return 'GetPlayerName(%s)' % value
elif type.IsInstance("destructible"):
return 'GetDestructableName(%s)' % value
elif type.IsInstance("code"):
return 'I2S(Jlib_C2Int(%s))' % value
elif type.IsInstance("handle"):
return 'I2S(Jlib_H2Int(%s))' % value
else:
return '<Unknown object>'
def GetNullValue(t):
base = t.Basetype()
return JassUtility.get_null_value_ex(base.name)
def GetGcFuncs(scope, type):
setconverter = "%s"
getconverter = "%s"
setter = "StoreInteger"
getter = "GetStoredInteger"
if type.IsInstance(scope.GetType("integer")):
pass
elif type.IsInstance(scope.GetType("real")):
setter = "StoreReal"
getter = "GetStoredReal"
elif type.IsInstance(scope.GetType("boolean")):
setter = "StoreBoolean"
getter = "GetStoredBoolean"
elif type.IsInstance(scope.GetType("string")):
setter = "StoreString"
getter = "GetStoredString"
elif type.IsInstance(scope.GetType("handle")):
setconverter = "Jlib_H2Int(%s)"
getconverter = "Jlib_I2%s(%s)" % (type.name.capitalize(), "%s")
elif type.IsInstance(scope.GetType("code")):
setconverter = "Jlib_C2Int(%s)"
getconverter = "Jlib_I2Code(%s)"
else:
raise JassUtility.ParseError("Unknown type %s" % type.name)
return setter, getter, setconverter, getconverter
def ParseFunctionHeader(tokens):
tokens = tokens[:]
if tokens[0][1] == "constant":
del tokens[0]
constant = True
else:
constant = False
# print tokens, len(tokens)
if len(tokens) < 6:
raise JassUtility.ParseError("Malformed function definition. Expected at least 6 tokens.")
start, name, takes = Nth(tokens[0:3], 1) #2nd element of tokens
del tokens[0:3]
ret, returntype = Nth(tokens[-2:], 1)#[t[1] for t in zip(tokens[-2:])[1]
del tokens[-2:]
if len(tokens) == 1:
if tokens[0][1] != "nothing":
raise JassUtility.ParseError("Malformed function definition: expected 'nothing' in argument list")
tokens = []
tokens = filter(lambda x: x[1] != ",", tokens)
if len(tokens) % 2:
#if not dividable with 2
raise JassUtility.ParseError("Uneven number of tokens in argument list")
args = []
tokens = list(tokens)
for a, b in zip(tokens[::2], tokens[1::2]):
args.append((a[1], b[1]))
#args = CommandParser.Function.parse_argdef(args) #tuples of (type, name)
# print args
return start, name, constant, args, returntype
def ParseArgumentList(scope, args):
out = OrderedDict()
for typename, name in args:
type = scope.GetType(typename)
argdef = ArgumentDefinition(type, name)
if name in out:
raise JassUtility.ParseError("Multiple use of same name in argument definition")
out[name] = argdef
return out
def ParseCommand(tokens, line):
if len(tokens) < 3:
raise JassUtility.ParseError("Misformed preprocessor command: %s" % line)
name, content = tokens[2][1], tokens[3:]
return PreprocessorNode(name, tokens)
unique_key_num = 2
def UniqueKey():
global unique_key_num
#Starts at 'b' for 0
str = "#"
num = unique_key_num
while num > 0:
str = str + chr((ord('(')) + num % 52)
num = num // 52
unique_key_num += 1
return str
#------------
# NODES
#------------
class Node:
def __init__(self, name):
#print "creating node:", name
# if name == '"#UNIQUE#"':
# self.name = UniqueKey()
self.name = name
def Render(self, renderflags):
#print "render default token: -%s-" % self.name
return self.name
def InferType(self):
return None
class PunctuationNode(Node):
pass
class LiteralNode(Node):
pass
class NumberLiteral(LiteralNode):
def InferType(self):
if "." in self.name:
return basetypes["real"]
return basetypes["integer"]
class StringLiteral(LiteralNode):
def InferType(self):
return basetypes["string"]
class BooleanLiteral(LiteralNode):
def InferType(self):
return basetypes["boolean"]
##
##def ConvertLiteralTokens(tokens):
## outtok = []
## for tok in tokens:
## if isinstance(tok, Node):
## outtok.append(tok)
## else:
## type, name = tok
## if type == tokenize.NUMBER:
## outtok.append(NumberLiteral(name))
## elif type == tokenize.STRING:
## outtok.append(StringLiteral(name))
## elif type == TOKEN_IDENTIFIER and name in ("true", "false", "TRUE", "FALSE"):
## outtok.append(BooleanLiteral(name.lower()))
## else:
## #otherwise dont modify
## outtok.append(tok)
## return outtok
##def ConvertKeywordTokens(tokens):
## outtok = []
## for tok in tokens:
## if isinstance(tok, Node):
## outtok.append(tok)
## else:
## type, name = tok
## if name in RESERVED_KEYWORDS:
## outtok.append(KeywordNode(tok[1]))
## else:
## #otherwise dont modify
## outtok.append(tok)
## return outtok
##
##def ConvertAssignments(tokens, scope):
## if not tokens:
## return tokens
## first = tokens[0]
## if isinstance(first, KeywordNode) and first.name == "set":
## tokens.pop(0) #discard 'set'
## found_equalsign = False
## before, after = [], []
## while tokens:
## tok = tokens.pop(0)
## if isinstance(tok, Node) or tok[1] != "=":
## if not found_equalsign:
## before.append(tok)
## else:
## after.append(tok)
## else:
## #it's not a node and it is "="
## found_equalsign = True
## if not found_equalsign:
## raise JassUtility.ParseError("Expected assignment after 'set' keyword")
## return [AssignmentNode(scope, before, after)]
## #Only touch set keyword
## return tokens
##
##def ConvertLiteralFunctionTokens(tokens, scope):
## outtok = []
## prevtok = None
## for tok in tokens:
## if isinstance(tok, Node):
## outtok.append(tok)
## else:
## if isinstance(prevtok, KeywordNode) and prevtok.name == "function":
## if tok[0] != TOKEN_IDENTIFIER:
## raise JassUtility.ParseError("Syntax error after 'function' keyword: expected identifier")
## outtok.append(FunctionIdentifier(scope.GetFunction(tok[1])))
## else:
## outtok.append(tok)
## prevtok = tok
## return outtok
##
##def ConvertFunctionTokens(tokens, scope):
## outtok = []
## for x in range(len(tokens)):
## tok = tokens[x]
## try:
## nexttok = tokens[x+1]
## except IndexError:
## nexttok = None
##
## if isinstance(tok, Node):
## outtok.append(tok)
## else:
## type, name = tok
## if nexttok and not isinstance(nexttok, Node):
## nexttype, nextname = nexttok
## else:
## nexttype, nextname = None, None
## #if this is an identifier and next is an opening bracket, it must be a function call
## if type == TOKEN_IDENTIFIER and nexttype == TOKEN_PUNCTUATION and nextname == "(":
## outtok.append(FunctionIdentifier(scope.GetFunction(name)))
## else:
## outtok.append(tok)
## return outtok
##
##
def PopBracketedTokens(tokens, openbracket = "(", closebracket = ")"):
if not tokens:
return []
out = []
level = 1
while tokens:
tok = tokens.pop(0)
if not isinstance(tok, Node):
if tok[1] == openbracket:
level += 1
if tok[1] == closebracket:
level -= 1
if level == 0:
return out
out.append(tok)
raise JassUtility.ParseError("Missing ending bracket '%s'" % closebracket)
##
##def ConvertBracketAccessors(tokens, scope):
## outtok = []
## while tokens:
## tok = tokens.pop(0)
## if isinstance(tok, Node) or tok[1] != "[":
## outtok.append(tok)
## else:
## #it's not a node and it is "["
## #pop prevtok
## try:
## prevtok = outtok.pop()
## except IndexError:
## raise JassUtility.ParseError("Misplaced bracket index syntax")
## outtok.append(BracketAccessor(scope, prevtok, PopBracketedTokens(tokens, openbracket="[", closebracket="]")))
## return outtok
##
##def IsGcAccess(scope, bracket_accessor):
## if isinstance(bracket_accessor, BracketAccessor):
## return (bracket_accessor.key.InferType() == scope.GetType("string") and
## isinstance(bracket_accessor.target.type, StructDefinition))
##
##def ConvertGcSetters(tokens, scope):
## if not tokens:
## return tokens
## # o_O
## if len(tokens) == 1 and isinstance(tokens[0], AssignmentNode):
## assignment_node = tokens[0]
## assign_to = assignment_node.assign_to
## if len(assign_to.code) == 1:
## bracket_accessor = assign_to.code[0]
## if IsGcAccess(scope, bracket_accessor):
### if isinstance(bracket_accessor, BracketAccessor):
### inner_code = bracket_accessor.codenode
## #if inner_code.InferType() == scope.GetType("string"):
## return [GcAssignmentNode(scope, bracket_accessor, assignment_node.assignment)]
## return tokens
##
##def ConvertGcGetters(tokens, scope):
## if not tokens:
## return tokens
## # o_O
## if len(tokens) == 1 and isinstance(tokens[0], AssignmentNode):
## assignment_node = tokens[0]
## assignment = assignment_node.assignment
## if len(assignment.code) == 1:
## bracket_accessor = assignment.code[0]
## if IsGcAccess(scope, bracket_accessor):
### if isinstance(bracket_accessor, BracketAccessor):
### inner_code = bracket_accessor.codenode
## #if inner_code.InferType() == scope.GetType("string"):
## gc_getter = GcGetterNode(scope, bracket_accessor, assignment_node.assign_to.InferType())
## assignment_node.assignment = gc_getter
## return tokens
##
##
##def ParseGcGetterInFunctionCall(scope, tokens, receiving_function):
## #With function names like this...
## #Anyway, convert each token into a GcGetter by supplying it with an expected type.
## #Tokens have already been made into bracket accessors, so just find them
## outtok = []
## index = 0
## while tokens:
## tok = tokens.pop(0)
## if IsGcAccess(scope, tok):
## outtok.append(GcGetterNode(scope, tok, receiving_function.GetArgumentType(index)))
## else:
## if not isinstance(tok, Node) and tok[1] == ",":
## index += 1
## outtok.append(tok)
## return outtok
##
##
##def ConvertFunctionCalls(tokens, scope):
## outtok = []
## prev_function_keyword = False
## while tokens:
## tok = tokens.pop(0)
## if isinstance(tok, FunctionIdentifier) and not prev_function_keyword:
## #discard the opener
## tokens.pop(0)
## arg_tokens = PopBracketedTokens(tokens)
## arg_tokens = ParseGcGetterInFunctionCall(scope, arg_tokens, tok.definition)
## outtok.append(FunctionCallIdentifier(scope, tok, arg_tokens))
## else:
## outtok.append(tok)
## prev_function_keyword = isinstance(tok, KeywordNode) and tok.name == "function"
## return outtok
##
##def IsPropertyAccess(token):
## return token and not isinstance(token, Node) and token[0] == TOKEN_PUNCTUATION and token[1] == "."
##
##def ConvertIdentifierTokens(tokens, scope):
## outtok = []
## prevtok = None
## for tok in tokens:
## if isinstance(tok, Node):
## #already parsed
## outtok.append(tok)
## continue
## type, name = tok
## #function openers are already taken care of
## #so are keywords
## #still we must skip property definitions
## if type == TOKEN_IDENTIFIER and not IsPropertyAccess(prevtok):
## outtok.append(VariableIdentifier(scope.GetIdentifier(name)))
## else:
## #leave unmodified
## outtok.append(tok)
## prevtok = tok
## return outtok
##
##def ConvertPropertyTokens(tokens, scope):
## outtok = []
## targettok = None
## prevtok = None
## for tok in tokens:
## if isinstance(tok, Node):
## outtok.append(tok)
## else:
## #Members are unparsed tokens preceeded by a punctuation, also in the form of an unparsed token
## if IsPropertyAccess(prevtok):
## #Can only member-access identifiers
## #print targettok, prevtok, tok
## if not isinstance(targettok, VariableIdentifier):
## raise JassUtility.ParseError("Invalid member access of non-identifier token")
## #remove the previous 2 tokens from outtok
## del outtok[-2:]
## tok = StructPropertyIdentifier(targettok, tok[1])
## outtok.append(tok)
## else:
## outtok.append(tok)
## targettok = prevtok
## prevtok = tok
## return outtok
##
##
##def ConvertLeftoverTokens(tokens):
## outtok = []
## for tok in tokens:
## if isinstance(tok, Node):
## outtok.append(tok)
## else:
## outtok.append(PunctuationNode(tok[1]))
## return outtok
##
def printtok(tokens):
buf = ""
for t in tokens:
if isinstance(t, Node):
buf += t.Render() + "__"
else:
buf += "[%s-%s]" % t
return buf
##def TokenizeCode(scope, tokens):
## tokens = ConvertLiteralTokens(tokens)
## tokens = ConvertKeywordTokens(tokens)
## tokens = ConvertAssignments(tokens, scope)
## tokens = ConvertLiteralFunctionTokens(tokens, scope)
## tokens = ConvertFunctionTokens(tokens, scope)
## tokens = ConvertBracketAccessors(tokens, scope)
## tokens = ConvertGcSetters(tokens, scope)
## tokens = ConvertGcGetters(tokens, scope)
## tokens = ConvertFunctionCalls(tokens, scope)
## tokens = ConvertIdentifierTokens(tokens, scope)
## tokens = ConvertPropertyTokens(tokens, scope)
## tokens = ConvertLeftoverTokens(tokens)
##
## return tokens
def ConvertLiteralToken(scope, t, tokens):
type, name = t
if type == tokenize.NUMBER:
return NumberLiteral(name)
elif type == tokenize.STRING:
return StringLiteral(name)
elif type == TOKEN_IDENTIFIER and name in ("true", "false", "TRUE", "FALSE"):
return BooleanLiteral(name.lower())
return t
def MakeAssignment(scope, before, after):
assign_to = CodeNode(scope, before)
#This is where we fix possible null values
if len(after) == 1:
t, name = node = after[0]
if name == JassUtility.NULL_VALUE_TOKEN:
type = assign_to.InferType()
if type == None:
raise JassUtility.ParseError("Cannot infer type of assignment: %s" % assign_to.Render(()))
after[0] = GenerateTokens(GetNullValue(type))[0]
assignment = CodeNode(scope, after)
code = assign_to.code
if len(code) == 1 and isinstance(code[0], BracketAccessor):
return GcAssignmentNode(scope, code[0], assignment)
return AssignmentNode(scope, assign_to, assignment)
def ConvertAssignment(scope, t, tokens):
found_equalsign = False
before, after = [], []
while tokens:
type, name = tok = tokens.pop(0)
if name == "=" and not found_equalsign:
found_equalsign = True
else:
if not found_equalsign:
before.append(tok)
else:
after.append(tok)
if not found_equalsign:
raise JassUtility.ParseError("Expected assignment after 'set' keyword")
#print "make assign, before", printtok(before), "after", printtok(after)
return MakeAssignment(scope, before, after)
def ConvertLiteralFunction(scope, t, tokens):
try:
type, name = id = tokens.pop(0)
except (IndexError):
raise JassUtility.ParseError("Expected identifier after 'function' keyword, got nothing")
if type != TOKEN_IDENTIFIER:
raise JassUtility.ParseError("Expected identifier after 'function' keyword, got %s" % name)
return FunctionLiteralIdentifier(scope.GetFunction(name))
def ConvertKeywordToken(scope, t, tokens):
type, name = t
if name in RESERVED_KEYWORDS:
t = KeywordNode(name)
if t.name == "set":
return ConvertAssignment(scope, t, tokens)
elif t.name == "function":
return ConvertLiteralFunction(scope, t, tokens)
return t
def ConvertFunctionToken(scope, t, tokens):
#discard the opener
tokens.pop(0)
arg_tokens = PopBracketedTokens(tokens)
return FunctionCallIdentifier(scope, t, arg_tokens)
def ConvertIdentifierToken(scope, t, tokens):
type, name = t
if type == TOKEN_IDENTIFIER:
if len(tokens) == 0:
#must be a normal id
return VariableIdentifier(scope.GetIdentifier(name))
#otherwise lets peek at the next
ntype, nname = next = TokenPeek(tokens, 0)
if nname == "(":
return ConvertFunctionToken(scope, scope.GetFunction(name), tokens)
#otherwise its normal
return VariableIdentifier(scope.GetIdentifier(name))
return t
def ConvertBracketAccessor(scope, t, tokens):
return BracketAccessor(scope, t, PopBracketedTokens(tokens, openbracket="[", closebracket="]"))
def ConvertMemberAccess(scope, target, tokens):
if not tokens:
raise JassUtility.ParseError("Expected a token after . operator")
type, name = tokens.pop(0)
if not type == TOKEN_IDENTIFIER:
raise JassUtility.ParseError("Epected an identifier after . operator")
if len(tokens) >= 2:
if TokenPeek(tokens, 1)[1] == "(":
#function
pass
return StructPropertyIdentifier(target, name)
def ConvertLeftoverToken(scope, t, tokens):
return PunctuationNode(t[1])
def TokenPeek(tokens, index):
return tokens[index]
tokenizer_parsers = [ConvertLiteralToken,
ConvertKeywordToken,
ConvertIdentifierToken,
ConvertLeftoverToken]
def TokenParse(scope, t, tokens):
for parser in tokenizer_parsers:
parsed = parser(scope, t, tokens)
#If the token was changed it is considered finished and not passed to other parsers
if parsed != t:
return parsed
assert(False)
def TokenizeCode(scope, tokens):
out = []
#print printtok(tokens)
while tokens:
type, name = t = tokens.pop(0)
if name == ".":
if not out:
raise JassUtility.ParseError("Invalid member access at start of block")
out.append(ConvertMemberAccess(scope, out.pop(), tokens))
continue
if name == "[":
if tokens:
if not out:
raise JassUtility.ParseError("Invalid [] access at start of block")
ntype, nname = next = TokenPeek(tokens, 0)
if ntype == tokenize.STRING:
out.append(ConvertBracketAccessor(scope, out.pop(), tokens))
continue
out.append(TokenParse(scope, t, tokens))
return out
class CodeNode(Node):
def __init__(self, scope, tokens):
self.scope = scope
self.Modify(tokens)
def Modify(self, tokens):
self.code = TokenizeCode(self.scope, tokens)
def FixNullValue(self, type):
if len(self.code) == 1:
node = self.code[0]
if isinstance(node, StringLiteral) and node.name == JassUtility.NULL_VALUE_TOKEN:
self.Modify(GenerateTokens(GetNullValue(type)))
def RenderNode(self, node, prevnode, renderflags):
if isinstance(node, PunctuationNode):
#punctuation must be spaced from any keyword
if isinstance(prevnode, KeywordNode):
return " " + node.Render(renderflags)
return node.Render(renderflags)
#default action is to space from anything non-punctuation
if prevnode == None or isinstance(prevnode, PunctuationNode):
return node.Render(renderflags)
return " " + node.Render(renderflags)
def Render(self, renderflags):
#print "rendering code-node:", "-".join([n.Render() for n in self.code])
buf = ""
prevnode = None
for node in self.code:
#print "render single %s:" % node, node.Render()
buf += self.RenderNode(node, prevnode, renderflags)
prevnode = node
#print "result: ", buf
return buf
def InferType(self):
#The inferred type of a code node is the first node that doesn't return None
t = None
for node in self.code:
t = node.InferType()
if t != None:
break
return t
class AssignmentNode(Node):
def __init__(self, scope, assign_to, assignment):
self.assign_to = assign_to
self.assignment = assignment
code = self.assignment.code
if len(code) == 1 and isinstance(code[0], BracketAccessor):
code[0] = GcGetterNode(scope, code[0], self.assign_to.InferType())
def Render(self, renderflags):
return "set %s = %s" % (self.assign_to.Render(renderflags), self.assignment.Render(renderflags))
class GcAssignmentNode(Node):
def __init__(self, scope, bracket_accessor, assignment):
self.target = bracket_accessor.target
self.key = bracket_accessor.key
self.assignment = assignment
self.scope = scope
def Render(self, renderflags):
struct = self.target.type
type = self.assignment.InferType()
if not type:
raise JassUtility.ParseError("Cannot infer the type of %s" % self.assignment.Render(renderflags))
setter, getter, converter, getconverter = GetGcFuncs(self.scope, type)
converter = converter % self.assignment.Render(renderflags)
return 'call %s(%s, %s[%s], %s, %s)' % (setter, StructManager.Manager.gc_varname,
struct.gc_string_global.name, self.target.Render(renderflags),
self.key.Render(renderflags), converter)
class GcGetterNode(Node):
def __init__(self, scope, bracket_accessor, expected_type):
self.scope = scope
self.target = bracket_accessor.target
self.key = bracket_accessor.key
self.returntype = expected_type
def Render(self, renderflags):
struct = self.target.type
setter, getter, setconverter, converter = GetGcFuncs(self.scope, self.returntype)
str = '%s(%s, %s[%s], %s)' % (getter, StructManager.Manager.gc_varname,
struct.gc_string_global.name, self.target.Render(renderflags),
self.key.Render([]))
return converter % str
class BracketAccessor(Node):
def __init__(self, scope, target, tokens):
self.target = CodeNode(scope, [target]).code[0]
self.key = CodeNode(scope, tokens)
def Render(self, renderflags):
return "%s[%s]" % (self.target.Render(renderflags), self.key.Render(renderflags))
def InferType(self):
return self.target.InferType()
class KeywordNode(Node):
pass
##class FunctionArgumentNode(CodeNode):
## def Render(self):
## return "(%s)" % CodeNode.Render(self)
class IdentifierNode(Node):
"Locals, globals, functions - everything except keywords, types and punctuation"
class VariableIdentifier(IdentifierNode):
def GetName(self):
return self.definition.name
name = property(GetName)
def GetType(self):
return self.definition.type
type = property(GetType)
def __init__(self, definition):
self.definition = definition
def InferType(self):
return self.GetType()
class PropertyIdentifier(VariableIdentifier):
pass
class StructPropertyIdentifier(PropertyIdentifier):
#def GetName(self):
# return "%s.%s" % (self.target_node.type.Render(), self.member.name)
#name = property(GetName)
def __init__(self, target_node, name):
self.target_node = target_node
struct = target_node.type
if not isinstance(struct, StructDefinition):
raise JassUtility.ParseError("Attempt to access a member of non-struct %s of type %s" % (self.target_node.name, struct.name))
self.member = struct.GetIdentifier(name)
def GetType(self):
return self.member.type
type = property(GetType)
def GetName(self):
return "%s.%s" % (self.target_node.name, self.member.name)
name = property(GetName)
def Render(self, renderflags):
return "%s[%s]" % (self.member.glob.name, self.target_node.Render(renderflags))
class FunctionIdentifier(VariableIdentifier):
pass
class FunctionLiteralIdentifier(FunctionIdentifier):
def Render(self, renderflags):
return "function %s" % FunctionIdentifier.Render(self, renderflags)
class FunctionCallIdentifier(FunctionIdentifier):
def __init__(self, scope, functionid, argtokens):
FunctionIdentifier.__init__(self, functionid)
self.argumentcode = CodeNode(scope, argtokens)
def Render(self, renderflags):
return "%s(%s)" % (FunctionIdentifier.Render(self, renderflags), self.argumentcode.Render(renderflags))
class DefinitionNode(Node):
pass
def IsLocalDefinition(tokens):
return len(tokens) > 0 and tokens[0][1] == "local"
def IsEndfunction(tokens):
return len(tokens) == 1 and tokens[0][1] == "endfunction"
class FunctionDefinitionBase(DefinitionNode):
def __init__(self, scope, deftokens, defline):
self.GetType = scope.GetType
try:
self.functiontype, self.name, self.constant, args, self.type = ParseFunctionHeader(deftokens)
self.type = self.GetType(self.type)
self.argument_locals = ParseArgumentList(self, args)
except JassUtility.ParseError, e:
raise JassUtility.ParseError("Error parsing function definition %s:\n%s" % (defline, str(e)))
def GetArgumentType(self, index):
try:
return self.argument_locals.values()[index].type
except IndexError:
raise JassUtility.ParseError("Invalid argument index accessing argument %d in %s" % (index, self.name))
class NativeDefinition(FunctionDefinitionBase):
pass
## def __init__(self, scope, tokens, defline):
## try:
## if tokens[0][1] == "constant":
## del tokens[0]
## self.name = tokens[1][1]
## self.type = scope.GetType(tokens[-1][1])
## except StandardError, e:
## raise JassUtility.ParseError("Could not parse native definition %s (%s)" % (line, str(e)))
class ArgumentDefinition(DefinitionNode):
def __init__(self, type, name):
self.type, self.name = type, name
class FunctionDefinition(FunctionDefinitionBase):
def RenderDefinition(self, renderflags):
str = ""
if self.constant:
str += "constant "
str += "function %s takes " % self.name
if not self.argument_locals:
str += "nothing"
else:
prevarg = False
for arg in self.argument_locals.values():
if prevarg:
str += ", "
prevarg = True
str += "%s %s" % (arg.type.Render(renderflags), arg.name)
str += " returns %s\n" % self.type.Render(renderflags)
return str
def RenderLocals(self, renderflags):
return "".join(map(lambda x: x.Render(renderflags), self.locals.values()))
def RenderCode(self, renderflags):
buf = ""
add_indent = False
indent = 1
for line in self.code:
firstword = line.code[0] if len(line.code) > 0 else None
if firstword and isinstance(firstword, KeywordNode):
if firstword.name in INDENTING_KEYWORDS:
add_indent = True
if firstword.name in DEDENTING_KEYWORDS:
indent -= 1
if indent < 1:
raise JassUtility.ParseError("Misplaced keyword: %s" + firstword.name)
buf += " " * indent + line.Render(renderflags) + "\n"
if add_indent:
indent += 1
add_indent = False
if indent != 1:
raise JassUtility.ParseError("Missing endloop/endif in %s" % self.name)
return buf
def Render(self, renderflags):
return self.RenderDefinition(renderflags) + self.RenderLocals(renderflags) + self.RenderCode(renderflags) + "endfunction\n"
def ParseCode(self, lineiter, inserter):
try:
for tokens, line in lineiter:
if IsLocalDefinition(tokens):
self.ParseLocalDefinition(tokens, line)
continue
if IsPreprocessorCommand(tokens):
self.commands.append(ParseCommand(tokens, line))
continue
if IsEndfunction(tokens):
break
#otherwise parse it as a code
inserter(self.code, (CodeNode(self, tokens)))
except JassUtility.ParseError, e:
raise JassUtility.ParseError("Error parsing function definition: %sat: %s-->\n%s" % (self.defline, line, str(e)))
def __init__(self, scope, lineiter, deftokens, defline):
self.commands = []
self.locals = OrderedDict()
self.code = []
self.scope = scope
FunctionDefinitionBase.__init__(self, scope, deftokens, defline)
self.defline = defline
self.ParseCode(lineiter, ListAppend)
def ParseLocalDefinition(self, tokens, line):
try:
local = LocalDefinition(self, tokens)
except JassUtility.ParseError, e:
raise JassUtility.ParseError("Error parsing local definition '%s'\n%s" % (line, str(e)))
if local.name in self.locals:
raise JassUtility.ParseError("Multiple definition of local named %s at %s" % (local.name, line))
self.locals[local.name] = local
def GetFunction(self, name):
#allow recursion or refer to the parent scope
if name == self.name:
return self
return self.scope.GetFunction(name)
def GetIdentifier(self, name):
if name in self.argument_locals:
return self.argument_locals[name]
if name in self.locals:
return self.locals[name]
#otherwise refer to globals
return self.scope.GetIdentifier(name)
class LocalDefinition(DefinitionNode):
@staticmethod
def ParseHeader(tokens):
if len(tokens) >= 3 and tokens[2][1] == "array":
array = True
del tokens[2]
else:
array = False
if len(tokens) >= 4 and tokens[3][1] == "=":
if len(tokens) < 5:
raise JassUtility.ParseError("Missing assignment after '='")
assignment = tokens[4:]
del tokens[3:]
else:
assignment = []
try:
local, type, name = Nth(tokens, 1)
except ValueError:
raise JassUtility.ParseError("Syntax error in local definition")
return array, type, name, assignment
def __init__(self, scope, tokens):
self.array, self.type, self.name, self.assignment = self.ParseHeader(tokens)
self.type = scope.GetType(self.type)
self.assignment = CodeNode(scope, self.assignment)
def Render(self, renderflags):
str = " local %s " % self.type.Render(renderflags)
if self.array:
str += "array "
str += self.name
if self.assignment.code:
str = "%-30s = %s" % (str, self.assignment.Render(renderflags))
return str.rstrip() + "\n"
class MemberDefinition(DefinitionNode):
def __init__(self, scope, tokens):
#The typeorder of members will later be modified for each type so that
#if a struct has integer, real, integer, string, they will be ordered 0, 0, 1, 0
self.typeorder = -1
if len(tokens) >= 3 and tokens[2][1] == "=":
if len(tokens) < 4:
raise JassUtility.ParseError("Missing assignment after '=' in member definition")
assignment = tokens[3:]
del tokens[2:]
else:
assignment = []
try:
type, self.name = Nth(tokens, 1)
except ValueError:
raise JassUtility.ParseError("Syntax error in member definition")
self.type = scope.GetType(type)
self.assignment = CodeNode(scope, assignment)
def HasAssignment(self):
return self.assignment.code
def RenderAssignment(self):
if self.HasAssignment:
return self.assignment.Render(set())
return GetNullValue(self.type)
class GlobalDefinition(DefinitionNode):
@staticmethod
def ParseHeader(tokens):
try:
if tokens[0][1] == "constant":
constant = True
del tokens[0]
else:
constant = False
if tokens[1][1] == "array":
array = True
del tokens[1]
else:
array = False
#print tokens
#print [t[1] for t in tokens]
type, name = [t[1] for t in tokens[0:2]] #2nd element
except StandardError, e:
raise JassUtility.ParseError("Could not parse global declaration (%s)" % str(e))
return constant, array, type, name
def __init__(self, scope, tokens):
#
#FIXME: Assignment needs to be turned into nodes! (or do they?)
#
self.constant, self.array, self.type, self.name = self.ParseHeader(tokens)
self.type = scope.GetType(self.type)
del tokens[0:2]
self.assignment = CodeNode(scope, tokens[1:])
self.assignment.FixNullValue(self.type)
#if tokens:
# self.assignment = tokens[1:]
#else:
# self.assignment = []
def Render(self, renderflags):
str = " "
if self.constant:
str += "constant "
str += self.type.Render(renderflags) + " "
if self.array:
str += "array "
str = "%-25s%s" % (str, self.name)
if self.assignment.code:
str = "%-60s = %s" % (str, self.assignment.Render(renderflags))
return str + "\n"
class TypeDefinition(DefinitionNode):
def __init__(self, name, extends = None):
self.name, self.extends = name, extends
def RenderDefinition(self, renderflags):
return "type %20s extends %20s\n" % (self.name, self.extends.Render(renderflags))
def IsInstance(self, type):
if isinstance(type, TypeDefinition):
type = type.name
if type == self.name:
return True
if self.extends:
return self.extends.IsInstance(type)
return False
def Basetype(self):
if self.extends:
return self.extends.Basetype()
return self
def IsEndstruct(tokens):
return len(tokens) == 1 and tokens[0][1] == "endstruct"
class StructDefinition(TypeDefinition):
@staticmethod
def GetName(deftokens):
return deftokens[1][1]
def AddLocal(self, local):
if local in self.locals:
raise JassUtility.ParseError("Multiple definition of property %s" % local.name)
self.locals[local.name] = local
def GetParents(self, include_self = True):
all = OrderedDict()
if include_self:
all[self.name] = self
for p in self.parents.itervalues():
all.update(p.GetParents())
all.update(self.parents)
return all
def GetAllMembers(self):
members = OrderedDict()
for p in self.parents.itervalues():
members.update(p.GetAllMembers())
members.update(self.locals)
return members
def ParseDefinition(self, deftokens, line):
try:
parents = OrderedDict()
if len(deftokens) > 2:
if deftokens[2][1] != "extends":
raise ValueError("expected 'extends' or nothing after struct name")
parent_names = Nth(filter(lambda x: x[1] != ",", deftokens[3:]), 1)
for n in parent_names:
parents[n] = self.scope.GetType(n)
deftokens = deftokens[0:2]
struct, name = deftokens
smartstruct = struct[1] == SMARTSTRUCT_NAME
except ValueError:
raise JassUtility.ParseError("Error parsing struct definition: '%s'" % line)
return smartstruct, name[1], parents
def GetIdentifier(self, name):
#Try own locals first
try:
return self.locals[name]
except KeyError:
pass
#Try each parent in order
for p in self.parents.itervalues():
try:
return p.GetIdentifier(name)
except MissingReferenceError:
pass
#not found?
raise MissingReferenceError("Referring to non-existing member %s of struct %s" % (name, self.name))
# def RenderMember(self, member, target):
# self.scope.struct_manager.RenderMember(member, target)
def Parse(self, lineiter):
for tokens, line in lineiter:
if IsFunctionSection(tokens):
self.ParseFunctionSection(tokens, line, lineiter)
continue
if IsEndstruct(tokens):
break
if tokens:
try:
l = MemberDefinition(self.scope, tokens)
except JassUtility.ParseError, e:
raise JassUtility.ParseError("Error parsing member definition %s in struct %s: %s" % (self.name, line, str(e)))
self.AddLocal(l)
#print self.function_data
def FixFunctionHeader(self, deftokens, line):
out = []
while deftokens:
type, name = t = deftokens.pop(0)
if name == "takes":
if not deftokens:
raise JassUtility.ParseError("Expected a token after 'takes'")
ntype, nname = next = deftokens.pop(0)
if nname == "nothing":
return out + GenerateTokens("takes integer self_id") + deftokens
else:
return out + GenerateTokens("takes integer self_id, %s" % nname) + deftokens
out.append(t)
raise JassUtility.ParseError("Expected 'takes' in function definition %s" % line)
def ParseFunctionSection(self, deftokens, line, lineiter):
out = self.function_data
out += [(self.FixFunctionHeader(deftokens, line), "")]
out += GenerateTokenLines(["local %s self = self_id" % self.name])
for tokens, line in lineiter:
out += [(tokens, line)]
if IsEndfunction(tokens):
return
raise JassUtility.ParseError("Expected endfunction while parsing method for %s" % self.name)
def __init__(self, scope, deftokens, line):
self.scope = scope
self.locals = OrderedDict()
self.extends = scope.GetType("integer")
self.function_data = []
self.functions = OrderedDict()
self.smartstruct, self.name, self.parents = self.ParseDefinition(deftokens, line)
def Render(self, renderflags):
#print "render struct type:", self.name, ScriptNode.RENDER_STRUCTS_AS_INTEGERS in renderflags
if ScriptNode.RENDER_STRUCTS_AS_INTEGERS in renderflags:
return "integer"
return self.name
class PreprocessorNode(Node):
def __init__(self, name, tokens):
self.name, self.tokens = name, tokens
def IsGlobalSection(tokens):
return len(tokens) == 1 and tokens[0][1] == "globals"
def IsStructSection(tokens):
return len(tokens) >= 2 and tokens[0][1] in ("struct", SMARTSTRUCT_NAME)
def IsTypeDefinition(tokens):
return tokens[0][1] == "type"
def IsEndGlobalSection(tokens):
return len(tokens) == 1 and tokens[0][1] == "endglobals"
def IsPreprocessorCommand(tokens):
return len(tokens) >= 2 and tokens[0][1] == "//" and tokens[1][1] == "!"
def IsFunctionSection(tokens):
if tokens[0][1] == "constant":
return tokens[1][1] in FUNCTION_TYPES
return tokens[0][1] in FUNCTION_TYPES
def IsNativeDefinition(tokens):
if tokens[0][1] == "constant":
return tokens[1][1] == "native"
return tokens[0][1] == "native"
def IsNamespaceSection(tokens):
return len(tokens) == 2 and tokens[0][1] == "namespace"
def IsEndNamespaceSection(tokens):
return len(tokens) == 2 and tokens[0][1] == "endnamespace"
integer_basetype = TypeDefinition("integer")
basetypes = {"boolean": TypeDefinition("boolean"),
"integer": integer_basetype,
"real": TypeDefinition("real"),
"string": TypeDefinition("string"),
"handle": TypeDefinition("handle"),
"nothing": TypeDefinition("nothing"),
"code": TypeDefinition("code")}
class ScriptNode(Node):
RENDER_TYPES = 0
RENDER_STRUCTS_AS_INTEGERS = 1
def RenderGlobals(self, renderflags):
buf = "globals\n"
for g in self.globals.values():
buf += g.Render(renderflags)
buf += "endglobals\n"
return buf
def RenderFunctions(self, renderflags):
return "".join(map(lambda x: x.Render(renderflags), self.functions.values()))
def RenderTypes(self, renderflags):
return "".join(map(lambda x: x.RenderDefinition(renderflags), self.types.values()))
def Render(self, renderflags = set()):
if self.RENDER_TYPES in renderflags:
buf = self.RenderTypes(renderflags) + "\n"
else:
buf = ""
return buf + self.RenderGlobals(renderflags) + self.RenderFunctions(renderflags)
def Finalize(self):
self.struct_manager.Finalize()
def ParseTokenlines(self, tokenlines, name, inserter):
print "Parsing %s" % name or "<unnamed>"
lineiter = iter(tokenlines)
for tokens, original_line in lineiter:
if IsPreprocessorCommand(tokens):
self.ParsePreprocessorCommand(tokens, original_line)
continue
if IsTypeDefinition(tokens):
self.ParseTypeDefinition(tokens, original_line, inserter)
continue
if IsGlobalSection(tokens):
self.ParseGlobalsSection(lineiter, inserter)
continue
if IsFunctionSection(tokens):
self.ParseFunctionSection(lineiter, tokens, original_line, inserter)
continue
if IsNativeDefinition(tokens):
self.ParseNativeDefinition(tokens, original_line, inserter)
continue
if IsStructSection(tokens):
self.ParseStructSection(lineiter, tokens, original_line, inserter)
continue
if IsNamespaceSection(tokens) or IsEndNamespaceSection(tokens):
continue
raise JassUtility.ParseError("Unknown section of script found: %s" % original_line)
def PreParseStructTypes(self, tokenlines, inserter):
for tokens, original_line in tokenlines:
if IsStructSection(tokens):
self.PreParseStructSection(tokens, original_line, inserter)
self.struct_manager.CalculateSharedGlobals()
def __init__(self, tokenlines, reference_script = None, name = None, main_script = False):
self.globals = OrderedDict()
self.functions = OrderedDict()
# self.structs = {} #goes in under types
self.types = OrderedDict()
self.natives = OrderedDict()
self.commands = []
self.reference_script = reference_script
self.struct_manager = StructManager.Manager(self, main_script, reference_script.struct_manager if reference_script else None)
self.PreParseStructTypes(tokenlines, OdictAppend)
self.ParseTokenlines(tokenlines, name, OdictAppend)
def ParseNativeDefinition(self, tokens, original_line, inserter):
native = NativeDefinition(self, tokens, original_line)
inserter(self.natives, native.name, native)
def ParseStructSection(self, lineiter, tokens, original_line, inserter):
struct = self.GetType(StructDefinition.GetName(tokens))
struct.Parse(lineiter)
self.struct_manager.Update(struct)
def PreParseStructSection(self, tokens, original_line, inserter):
name = StructDefinition.GetName(tokens)
try:
self.GetType(name)
raise JassUtility.ParseError("Multiple definition of type/struct %s" % struct.name)
except MissingReferenceError:
pass
struct = StructDefinition(self, tokens, original_line)
inserter(self.types, name, struct)
self.struct_manager.PreUpdate(struct)
def ParseTypeDefinition(self, tokens, line, inserter):
if len(tokens) != 4 or tokens[0][1] != "type" or tokens[2][1] != "extends":
raise JassUtility.ParseError("Malformed type definition: %s" % line)
name = tokens[1][1]
extends = tokens[3][1]
t = TypeDefinition(name, self.GetType(extends))
try:
self.GetType(t.name)
raise JassUtility.ParseError("Multiple definition of type/struct %s" % t.name)
except MissingReferenceError:
pass
inserter(self.types, t.name, t)
def ParseFunctionSection(self, lineiter, deftokens, defline, inserter):
#try:
# start, name, constant, args, returntype = ParseFunctionHeader(deftokens)
#except JassUtility.ParseError, e:
# raise JassUtility.ParseError("Error parsing function definition %s:\n%s" % (defline, str(e)))
func = FunctionDefinition(self, lineiter, deftokens, defline)
try:
self.GetFunction(func.name)
raise JassUtility.ParseError("Multiple definition of function %s" % func.name)
except MissingReferenceError:
pass
inserter(self.functions, func.name, func)
def ParsePreprocessorCommand(self, tokens, line):
self.commands.append(ParseCommand(tokens, line))
def AddGlobal(self, glob, inserter):
try:
self.GetIdentifier(glob.name)
raise JassUtility.ParseError("Multiple definition of type %s" % glob.name)
except MissingReferenceError:
pass
inserter(self.globals, glob.name, glob)
def ParseGlobalsSection(self, lineiter, inserter):
#Any preprocessor commands found here goes to top level
for tokens, line in lineiter:
try:
if IsPreprocessorCommand(tokens):
self.ParsePreprocessorCommand(tokens, line)
else:
if IsEndGlobalSection(tokens):
return
glob = GlobalDefinition(self, tokens)
self.AddGlobal(glob, inserter)
except JassUtility.ParseError, e:
raise JassUtility.ParseError("Error parsing global at line: %s (%s)" % (line, str(e)))
def GetType(self, name):
if name in basetypes:
return basetypes[name]
try:
return self.reference_script.GetType(name)
except (JassUtility.ParseError, KeyError, AttributeError):
if not name in self.types:
raise MissingReferenceError("Request for non-existant 'type'-node %s" % name)
return self.types[name]
def RenameType(self, old, new):
if old in self.types:
pos = self.types.index(old)
t = self.types.pop(old)
t.name = new
self.types.insert(pos, new, t)
else:
raise JassUtility.ParseError("Cannot rename type %s to %s, %s does not exist in the script" % (old, new, old))
def GetIdentifier(self, name):
try:
return self.reference_script.GetIdentifier(name)
except (JassUtility.ParseError, KeyError, AttributeError):
if not name in self.globals:
raise MissingReferenceError("Request for non-existant 'identifier'-node %s" % name)
return self.globals[name]
def GetFunction(self, name):
#ask reference script, then own natives, then own functions
#print "asking for %s" % name
try:
#print "try reference..."
return self.reference_script.GetFunction(name)
except (JassUtility.ParseError, KeyError, AttributeError):
pass
try:
#print "try own natives...", self.natives.keys()
return self.natives[name]
except KeyError:
pass
try:
#print "try own functions..."
return self.functions[name]
except KeyError:
raise MissingReferenceError("Request for non-existant 'function'-node %s" % name)
def ParseScript(script, reference_script = None, name = None, main = False):
try:
name = script.name
except AttributeError:
pass
# script = JassUtility.StripComments(script)
return ScriptNode(GenerateTokenLines(script), reference_script, name, main)
def Parse(script, reference_scripts):
ref = None
#print reference_scripts
for s in reference_scripts:
#print s.name
ref = ParseScript(s, ref)
return ParseScript(script, ref, main=True)
