| # a glorified C pre-processor parser |
| |
| import sys, re, string |
| from utils import * |
| from defaults import * |
| |
| debugTokens = False |
| debugDirectiveTokenizer = False |
| debugLineParsing = False |
| debugCppExpr = False |
| debugOptimIf01 = False |
| |
| ##################################################################################### |
| ##################################################################################### |
| ##### ##### |
| ##### C P P T O K E N S ##### |
| ##### ##### |
| ##################################################################################### |
| ##################################################################################### |
| |
| # the list of supported C-preprocessor tokens |
| # plus a couple of C tokens as well |
| tokEOF = "\0" |
| tokLN = "\n" |
| tokSTRINGIFY = "#" |
| tokCONCAT = "##" |
| tokLOGICAND = "&&" |
| tokLOGICOR = "||" |
| tokSHL = "<<" |
| tokSHR = ">>" |
| tokEQUAL = "==" |
| tokNEQUAL = "!=" |
| tokLT = "<" |
| tokLTE = "<=" |
| tokGT = ">" |
| tokGTE = ">=" |
| tokELLIPSIS = "..." |
| tokSPACE = " " |
| tokDEFINED = "defined" |
| tokLPAREN = "(" |
| tokRPAREN = ")" |
| tokNOT = "!" |
| tokPLUS = "+" |
| tokMINUS = "-" |
| tokMULTIPLY = "*" |
| tokDIVIDE = "/" |
| tokMODULUS = "%" |
| tokBINAND = "&" |
| tokBINOR = "|" |
| tokBINXOR = "^" |
| tokCOMMA = "," |
| tokLBRACE = "{" |
| tokRBRACE = "}" |
| tokARROW = "->" |
| tokINCREMENT = "++" |
| tokDECREMENT = "--" |
| tokNUMBER = "<number>" |
| tokIDENT = "<ident>" |
| tokSTRING = "<string>" |
| |
| class Token: |
| """a simple class to hold information about a given token. |
| each token has a position in the source code, as well as |
| an 'id' and a 'value'. the id is a string that identifies |
| the token's class, while the value is the string of the |
| original token itself. |
| |
| for example, the tokenizer concatenates a series of spaces |
| and tabs as a single tokSPACE id, whose value if the original |
| spaces+tabs sequence.""" |
| |
| def __init__(self): |
| self.id = None |
| self.value = None |
| self.lineno = 0 |
| self.colno = 0 |
| |
| def set(self,id,val=None): |
| self.id = id |
| if val: |
| self.value = val |
| else: |
| self.value = id |
| return None |
| |
| def copyFrom(self,src): |
| self.id = src.id |
| self.value = src.value |
| self.lineno = src.lineno |
| self.colno = src.colno |
| |
| def __repr__(self): |
| if self.id == tokIDENT: |
| return "(ident %s)" % self.value |
| if self.id == tokNUMBER: |
| return "(number %s)" % self.value |
| if self.id == tokSTRING: |
| return "(string '%s')" % self.value |
| if self.id == tokLN: |
| return "<LN>" |
| if self.id == tokEOF: |
| return "<EOF>" |
| if self.id == tokSPACE and self.value == "\\": |
| # this corresponds to a trailing \ that was transformed into a tokSPACE |
| return "<\\>" |
| |
| return self.id |
| |
| def __str__(self): |
| if self.id == tokIDENT: |
| return self.value |
| if self.id == tokNUMBER: |
| return self.value |
| if self.id == tokSTRING: |
| return self.value |
| if self.id == tokEOF: |
| return "<EOF>" |
| if self.id == tokSPACE: |
| if self.value == "\\": # trailing \ |
| return "\\\n" |
| else: |
| return self.value |
| |
| return self.id |
| |
| class BadExpectedToken(Exception): |
| def __init__(self,msg): |
| print msg |
| |
| ##################################################################################### |
| ##################################################################################### |
| ##### ##### |
| ##### C P P T O K E N C U R S O R ##### |
| ##### ##### |
| ##################################################################################### |
| ##################################################################################### |
| |
| class TokenCursor: |
| """a small class to iterate over a list of Token objects""" |
| def __init__(self,tokens): |
| self.tokens = tokens |
| self.n = 0 |
| self.count = len(tokens) |
| |
| def set(self,n): |
| """set the current position""" |
| if n < 0: |
| n = 0 |
| if n > self.count: |
| n = self.count |
| self.n = n |
| |
| def peekId(self): |
| """retrieve the id of the current token""" |
| if (self.n >= self.count): |
| return None |
| return self.tokens[self.n].id |
| |
| def peek(self): |
| """retrieve the current token. does not change position""" |
| if (self.n >= self.count): |
| return None |
| return self.tokens[self.n] |
| |
| def skip(self): |
| """increase current token position""" |
| if (self.n < self.count): |
| self.n += 1 |
| |
| def skipSpaces(self): |
| """skip over all space tokens, this includes tokSPACE and tokLN""" |
| while 1: |
| tok = self.peekId() |
| if tok != tokSPACE and tok != tokLN: |
| break |
| self.skip() |
| |
| def skipIfId(self,id): |
| """skip an optional token""" |
| if self.peekId() == id: |
| self.skip() |
| |
| def expectId(self,id): |
| """raise an exception if the current token hasn't a given id. |
| otherwise skip over it""" |
| tok = self.peek() |
| if tok.id != id: |
| raise BadExpectedToken, "%d:%d: '%s' expected, received '%s'" % (tok.lineno, tok.colno, id, tok.id) |
| self.skip() |
| |
| def remain(self): |
| """return the list of remaining tokens""" |
| return self.tokens[self.n:] |
| |
| |
| ##################################################################################### |
| ##################################################################################### |
| ##### ##### |
| ##### C P P T O K E N I Z E R ##### |
| ##### ##### |
| ##################################################################################### |
| ##################################################################################### |
| |
| # list of long symbols, i.e. those that take more than one characters |
| cppLongSymbols = [ tokCONCAT, tokLOGICAND, tokLOGICOR, tokSHL, tokSHR, tokELLIPSIS, tokEQUAL,\ |
| tokNEQUAL, tokLTE, tokGTE, tokARROW, tokINCREMENT, tokDECREMENT ] |
| |
| class CppTokenizer: |
| """an abstract class used to convert some input text into a list |
| of tokens. real implementations follow and differ in the format |
| of the input text only""" |
| |
| def __init__(self): |
| """initialize a new CppTokenizer object""" |
| self.eof = False # end of file reached ? |
| self.text = None # content of current line, with final \n stripped |
| self.line = 0 # number of current line |
| self.pos = 0 # current character position in current line |
| self.len = 0 # length of current line text |
| self.held = Token() |
| |
| def setLineText(self,line): |
| """set the content of the (next) current line. should be called |
| by fillLineText() in derived classes""" |
| self.text = line |
| self.len = len(line) |
| self.pos = 0 |
| |
| def fillLineText(self): |
| """refresh the content of 'line' with a new line of input""" |
| # to be overriden |
| self.eof = True |
| |
| def markPos(self,tok): |
| """mark the position of the current token in the source file""" |
| if self.eof or self.pos > self.len: |
| tok.lineno = self.line + 1 |
| tok.colno = 0 |
| else: |
| tok.lineno = self.line |
| tok.colno = self.pos |
| |
| def peekChar(self): |
| """return the current token under the cursor without moving it""" |
| if self.eof: |
| return tokEOF |
| |
| if self.pos > self.len: |
| self.pos = 0 |
| self.line += 1 |
| self.fillLineText() |
| if self.eof: |
| return tokEOF |
| |
| if self.pos == self.len: |
| return tokLN |
| else: |
| return self.text[self.pos] |
| |
| def peekNChar(self,n): |
| """try to peek the next n chars on the same line""" |
| if self.pos + n > self.len: |
| return None |
| return self.text[self.pos:self.pos+n] |
| |
| def skipChar(self): |
| """increment the token cursor position""" |
| if not self.eof: |
| self.pos += 1 |
| |
| def skipNChars(self,n): |
| if self.pos + n <= self.len: |
| self.pos += n |
| else: |
| while n > 0: |
| self.skipChar() |
| n -= 1 |
| |
| def nextChar(self): |
| """retrieve the token at the current cursor position, then skip it""" |
| result = self.peekChar() |
| self.skipChar() |
| return result |
| |
| def getEscape(self): |
| # try to get all characters after a backslash (\) |
| result = self.nextChar() |
| if result == "0": |
| # octal number ? |
| num = self.peekNChar(3) |
| if num != None: |
| isOctal = True |
| for d in num: |
| if not d in "01234567": |
| isOctal = False |
| break |
| if isOctal: |
| result += num |
| self.skipNChars(3) |
| elif result == "x" or result == "X": |
| # hex number ? |
| num = self.peekNChar(2) |
| if num != None: |
| isHex = True |
| for d in num: |
| if not d in "012345678abcdefABCDEF": |
| isHex = False |
| break |
| if isHex: |
| result += num |
| self.skipNChars(2) |
| elif result == "u" or result == "U": |
| # unicode char ? |
| num = self.peekNChar(4) |
| if num != None: |
| isHex = True |
| for d in num: |
| if not d in "012345678abcdefABCDEF": |
| isHex = False |
| break |
| if isHex: |
| result += num |
| self.skipNChars(4) |
| |
| return result |
| |
| def nextRealToken(self,tok): |
| """return next CPP token, used internally by nextToken()""" |
| c = self.nextChar() |
| if c == tokEOF or c == tokLN: |
| return tok.set(c) |
| |
| if c == '/': |
| c = self.peekChar() |
| if c == '/': # C++ comment line |
| self.skipChar() |
| while 1: |
| c = self.nextChar() |
| if c == tokEOF or c == tokLN: |
| break |
| return tok.set(tokLN) |
| if c == '*': # C comment start |
| self.skipChar() |
| value = "/*" |
| prev_c = None |
| while 1: |
| c = self.nextChar() |
| if c == tokEOF: |
| #print "## EOF after '%s'" % value |
| return tok.set(tokEOF,value) |
| if c == '/' and prev_c == '*': |
| break |
| prev_c = c |
| value += c |
| |
| value += "/" |
| #print "## COMMENT: '%s'" % value |
| return tok.set(tokSPACE,value) |
| c = '/' |
| |
| if c.isspace(): |
| while 1: |
| c2 = self.peekChar() |
| if c2 == tokLN or not c2.isspace(): |
| break |
| c += c2 |
| self.skipChar() |
| return tok.set(tokSPACE,c) |
| |
| if c == '\\': |
| if debugTokens: |
| print "nextRealToken: \\ found, next token is '%s'" % repr(self.peekChar()) |
| if self.peekChar() == tokLN: # trailing \ |
| # eat the tokLN |
| self.skipChar() |
| # we replace a trailing \ by a tokSPACE whose value is |
| # simply "\\". this allows us to detect them later when |
| # needed. |
| return tok.set(tokSPACE,"\\") |
| else: |
| # treat as a single token here ? |
| c +=self.getEscape() |
| return tok.set(c) |
| |
| if c == "'": # chars |
| c2 = self.nextChar() |
| c += c2 |
| if c2 == '\\': |
| c += self.getEscape() |
| |
| while 1: |
| c2 = self.nextChar() |
| if c2 == tokEOF: |
| break |
| c += c2 |
| if c2 == "'": |
| break |
| |
| return tok.set(tokSTRING, c) |
| |
| if c == '"': # strings |
| quote = 0 |
| while 1: |
| c2 = self.nextChar() |
| if c2 == tokEOF: |
| return tok.set(tokSTRING,c) |
| |
| c += c2 |
| if not quote: |
| if c2 == '"': |
| return tok.set(tokSTRING,c) |
| if c2 == "\\": |
| quote = 1 |
| else: |
| quote = 0 |
| |
| if c >= "0" and c <= "9": # integers ? |
| while 1: |
| c2 = self.peekChar() |
| if c2 == tokLN or (not c2.isalnum() and c2 != "_"): |
| break |
| c += c2 |
| self.skipChar() |
| return tok.set(tokNUMBER,c) |
| |
| if c.isalnum() or c == "_": # identifiers ? |
| while 1: |
| c2 = self.peekChar() |
| if c2 == tokLN or (not c2.isalnum() and c2 != "_"): |
| break |
| c += c2 |
| self.skipChar() |
| if c == tokDEFINED: |
| return tok.set(tokDEFINED) |
| else: |
| return tok.set(tokIDENT,c) |
| |
| # check special symbols |
| for sk in cppLongSymbols: |
| if c == sk[0]: |
| sklen = len(sk[1:]) |
| if self.pos + sklen <= self.len and \ |
| self.text[self.pos:self.pos+sklen] == sk[1:]: |
| self.pos += sklen |
| return tok.set(sk) |
| |
| return tok.set(c) |
| |
| def nextToken(self,tok): |
| """return the next token from the input text. this function |
| really updates 'tok', and does not return a new one""" |
| self.markPos(tok) |
| self.nextRealToken(tok) |
| |
| def getToken(self): |
| tok = Token() |
| self.nextToken(tok) |
| if debugTokens: |
| print "getTokens: %s" % repr(tok) |
| return tok |
| |
| def toTokenList(self): |
| """convert the input text of a CppTokenizer into a direct |
| list of token objects. tokEOF is stripped from the result""" |
| result = [] |
| while 1: |
| tok = Token() |
| self.nextToken(tok) |
| if tok.id == tokEOF: |
| break |
| result.append(tok) |
| return result |
| |
| class CppLineTokenizer(CppTokenizer): |
| """a CppTokenizer derived class that accepts a single line of text as input""" |
| def __init__(self,line,lineno=1): |
| CppTokenizer.__init__(self) |
| self.line = lineno |
| self.setLineText(line) |
| |
| |
| class CppLinesTokenizer(CppTokenizer): |
| """a CppTokenizer derived class that accepts a list of texdt lines as input. |
| the lines must not have a trailing \n""" |
| def __init__(self,lines=[],lineno=1): |
| """initialize a CppLinesTokenizer. you can later add lines using addLines()""" |
| CppTokenizer.__init__(self) |
| self.line = lineno |
| self.lines = lines |
| self.index = 0 |
| self.count = len(lines) |
| |
| if self.count > 0: |
| self.fillLineText() |
| else: |
| self.eof = True |
| |
| def addLine(self,line): |
| """add a line to a CppLinesTokenizer. this can be done after tokenization |
| happens""" |
| if self.count == 0: |
| self.setLineText(line) |
| self.index = 1 |
| self.lines.append(line) |
| self.count += 1 |
| self.eof = False |
| |
| def fillLineText(self): |
| if self.index < self.count: |
| self.setLineText(self.lines[self.index]) |
| self.index += 1 |
| else: |
| self.eof = True |
| |
| |
| class CppFileTokenizer(CppTokenizer): |
| def __init__(self,file,lineno=1): |
| CppTokenizer.__init__(self) |
| self.file = file |
| self.line = lineno |
| |
| def fillLineText(self): |
| line = self.file.readline() |
| if len(line) > 0: |
| if line[-1] == '\n': |
| line = line[:-1] |
| if len(line) > 0 and line[-1] == "\r": |
| line = line[:-1] |
| self.setLineText(line) |
| else: |
| self.eof = True |
| |
| # Unit testing |
| # |
| class CppTokenizerTester: |
| """a class used to test CppTokenizer classes""" |
| def __init__(self,tokenizer=None): |
| self.tokenizer = tokenizer |
| self.token = Token() |
| |
| def setTokenizer(self,tokenizer): |
| self.tokenizer = tokenizer |
| |
| def expect(self,id): |
| self.tokenizer.nextToken(self.token) |
| tokid = self.token.id |
| if tokid == id: |
| return |
| if self.token.value == id and (tokid == tokIDENT or tokid == tokNUMBER): |
| return |
| raise BadExpectedToken, "### BAD TOKEN: '%s' expecting '%s'" % (self.token.id,id) |
| |
| def expectToken(self,id,line,col): |
| self.expect(id) |
| if self.token.lineno != line: |
| raise BadExpectedToken, "### BAD LINENO: token '%s' got '%d' expecting '%d'" % (id,self.token.lineno,line) |
| if self.token.colno != col: |
| raise BadExpectedToken, "### BAD COLNO: '%d' expecting '%d'" % (self.token.colno,col) |
| |
| def expectTokenVal(self,id,value,line,col): |
| self.expectToken(id,line,col) |
| if self.token.value != value: |
| raise BadExpectedToken, "### BAD VALUE: '%s' expecting '%s'" % (self.token.value,value) |
| |
| def expectList(self,list): |
| for item in list: |
| self.expect(item) |
| |
| def test_CppTokenizer(): |
| print "running CppTokenizer tests" |
| tester = CppTokenizerTester() |
| |
| tester.setTokenizer( CppLineTokenizer("#an/example && (01923_xy)") ) |
| tester.expectList( ["#", "an", "/", "example", tokSPACE, tokLOGICAND, tokSPACE, tokLPAREN, "01923_xy", \ |
| tokRPAREN, tokLN, tokEOF] ) |
| |
| tester.setTokenizer( CppLineTokenizer("FOO(BAR) && defined(BAZ)") ) |
| tester.expectList( ["FOO", tokLPAREN, "BAR", tokRPAREN, tokSPACE, tokLOGICAND, tokSPACE, |
| tokDEFINED, tokLPAREN, "BAZ", tokRPAREN, tokLN, tokEOF] ) |
| |
| tester.setTokenizer( CppLinesTokenizer( ["/*", "#", "*/"] ) ) |
| tester.expectList( [ tokSPACE, tokLN, tokEOF ] ) |
| |
| tester.setTokenizer( CppLinesTokenizer( ["first", "second"] ) ) |
| tester.expectList( [ "first", tokLN, "second", tokLN, tokEOF ] ) |
| |
| tester.setTokenizer( CppLinesTokenizer( ["first second", " third"] ) ) |
| tester.expectToken( "first", 1, 0 ) |
| tester.expectToken( tokSPACE, 1, 5 ) |
| tester.expectToken( "second", 1, 6 ) |
| tester.expectToken( tokLN, 1, 12 ) |
| tester.expectToken( tokSPACE, 2, 0 ) |
| tester.expectToken( "third", 2, 2 ) |
| |
| tester.setTokenizer( CppLinesTokenizer( [ "boo /* what the", "hell */" ] ) ) |
| tester.expectList( [ "boo", tokSPACE ] ) |
| tester.expectTokenVal( tokSPACE, "/* what the\nhell */", 1, 4 ) |
| tester.expectList( [ tokLN, tokEOF ] ) |
| |
| tester.setTokenizer( CppLinesTokenizer( [ "an \\", " example" ] ) ) |
| tester.expectToken( "an", 1, 0 ) |
| tester.expectToken( tokSPACE, 1, 2 ) |
| tester.expectTokenVal( tokSPACE, "\\", 1, 3 ) |
| tester.expectToken( tokSPACE, 2, 0 ) |
| tester.expectToken( "example", 2, 1 ) |
| tester.expectToken( tokLN, 2, 8 ) |
| |
| return True |
| |
| |
| ##################################################################################### |
| ##################################################################################### |
| ##### ##### |
| ##### C P P E X P R E S S I O N S ##### |
| ##### ##### |
| ##################################################################################### |
| ##################################################################################### |
| |
| # Cpp expressions are modeled by tuples of the form (op,arg) or (op,arg1,arg2), etc.. |
| # op is an "operator" string |
| |
| class Expr: |
| """a class used to model a CPP expression""" |
| opInteger = "int" |
| opIdent = "ident" |
| opCall = "call" |
| opDefined = "defined" |
| opTest = "?" |
| opLogicNot = "!" |
| opNot = "~" |
| opNeg = "[-]" |
| opUnaryPlus = "[+]" |
| opAdd = "+" |
| opSub = "-" |
| opMul = "*" |
| opDiv = "/" |
| opMod = "%" |
| opAnd = "&" |
| opOr = "|" |
| opXor = "^" |
| opLogicAnd = "&&" |
| opLogicOr = "||" |
| opEqual = "==" |
| opNotEqual = "!=" |
| opLess = "<" |
| opLessEq = "<=" |
| opGreater = ">" |
| opGreaterEq = ">=" |
| opShl = "<<" |
| opShr = ">>" |
| |
| unaries = [ opLogicNot, opNot, opNeg, opUnaryPlus ] |
| binaries = [ opAdd, opSub, opMul, opDiv, opMod, opAnd, opOr, opXor, opLogicAnd, opLogicOr, |
| opEqual, opNotEqual, opLess, opLessEq, opGreater, opGreaterEq ] |
| |
| precedences = { |
| opTest: 0, |
| opLogicOr: 1, |
| opLogicNot: 2, |
| opOr : 3, |
| opXor: 4, |
| opAnd: 5, |
| opEqual: 6, opNotEqual: 6, |
| opLess:7, opLessEq:7, opGreater:7, opGreaterEq:7, |
| opShl:8, opShr:8, |
| opAdd:9, opSub:9, |
| opMul:10, opDiv:10, opMod:10, |
| opLogicNot:11, |
| opNot: 12, |
| } |
| |
| def __init__(self,op): |
| self.op = op |
| |
| def __repr__(self): |
| return "(%s)" % self.op |
| |
| def __str__(self): |
| return "operator(%s)" % self.op |
| |
| def precedence(self): |
| """return the precedence of a given operator""" |
| return Expr.precedences.get(self.op, 1000) |
| |
| def isUnary(self): |
| return self.op in Expr.unaries |
| |
| def isBinary(self): |
| return self.op in Expr.binaries |
| |
| def isDefined(self): |
| return self.op is opDefined |
| |
| def toInt(self): |
| """return the integer value of a given expression. only valid for integer expressions |
| will return None otherwise""" |
| return None |
| |
| class IntExpr(Expr): |
| def __init__(self,value): |
| Expr.__init__(self,opInteger) |
| self.arg = value |
| |
| def __repr__(self): |
| return "(int %s)" % self.arg |
| |
| def __str__(self): |
| return self.arg |
| |
| def toInt(self): |
| s = self.arg # string value |
| # get rid of U or L suffixes |
| while len(s) > 0 and s[-1] in "LUlu": |
| s = s[:-1] |
| return string.atoi(s) |
| |
| class IdentExpr(Expr): |
| def __init__(self,name): |
| Expr.__init__(self,opIdent) |
| self.name = name |
| |
| def __repr__(self): |
| return "(ident %s)" % self.name |
| |
| def __str__(self): |
| return self.name |
| |
| class CallExpr(Expr): |
| def __init__(self,funcname,params): |
| Expr.__init__(self,opCall) |
| self.funcname = funcname |
| self.params = params |
| |
| def __repr__(self): |
| result = "(call %s [" % self.funcname |
| comma = "" |
| for param in self.params: |
| result += "%s%s" % (comma, repr(param)) |
| comma = "," |
| result += "])" |
| return result |
| |
| def __str__(self): |
| result = "%s(" % self.funcname |
| comma = "" |
| for param in self.params: |
| result += "%s%s" % (comma, str(param)) |
| comma = "," |
| |
| result += ")" |
| return result |
| |
| class TestExpr(Expr): |
| def __init__(self,cond,iftrue,iffalse): |
| Expr.__init__(self,opTest) |
| self.cond = cond |
| self.iftrue = iftrue |
| self.iffalse = iffalse |
| |
| def __repr__(self): |
| return "(?: %s %s %s)" % (repr(self.cond),repr(self.iftrue),repr(self.iffalse)) |
| |
| def __str__(self): |
| return "(%s) ? (%s) : (%s)" % (self.cond, self.iftrue, self.iffalse) |
| |
| class SingleArgExpr(Expr): |
| def __init__(self,op,arg): |
| Expr.__init__(self,op) |
| self.arg = arg |
| |
| def __repr__(self): |
| return "(%s %s)" % (self.op, repr(self.arg)) |
| |
| class DefinedExpr(SingleArgExpr): |
| def __init__(self,op,macroname): |
| SingleArgExpr.__init__(self.opDefined,macroname) |
| |
| def __str__(self): |
| return "defined(%s)" % self.arg |
| |
| |
| class UnaryExpr(SingleArgExpr): |
| def __init__(self,op,arg,opstr=None): |
| SingleArgExpr.__init__(self,op,arg) |
| if not opstr: |
| opstr = op |
| self.opstr = opstr |
| |
| def __str__(self): |
| arg_s = str(self.arg) |
| arg_prec = self.arg.precedence() |
| self_prec = self.precedence() |
| if arg_prec < self_prec: |
| return "%s(%s)" % (self.opstr,arg_s) |
| else: |
| return "%s%s" % (self.opstr, arg_s) |
| |
| class TwoArgExpr(Expr): |
| def __init__(self,op,arg1,arg2): |
| Expr.__init__(self,op) |
| self.arg1 = arg1 |
| self.arg2 = arg2 |
| |
| def __repr__(self): |
| return "(%s %s %s)" % (self.op, repr(self.arg1), repr(self.arg2)) |
| |
| class BinaryExpr(TwoArgExpr): |
| def __init__(self,op,arg1,arg2,opstr=None): |
| TwoArgExpr.__init__(self,op,arg1,arg2) |
| if not opstr: |
| opstr = op |
| self.opstr = opstr |
| |
| def __str__(self): |
| arg1_s = str(self.arg1) |
| arg2_s = str(self.arg2) |
| arg1_prec = self.arg1.precedence() |
| arg2_prec = self.arg2.precedence() |
| self_prec = self.precedence() |
| |
| result = "" |
| if arg1_prec < self_prec: |
| result += "(%s)" % arg1_s |
| else: |
| result += arg1_s |
| |
| result += " %s " % self.opstr |
| |
| if arg2_prec < self_prec: |
| result += "(%s)" % arg2_s |
| else: |
| result += arg2_s |
| |
| return result |
| |
| ##################################################################################### |
| ##################################################################################### |
| ##### ##### |
| ##### C P P E X P R E S S I O N P A R S E R ##### |
| ##### ##### |
| ##################################################################################### |
| ##################################################################################### |
| |
| |
| class ExprParser: |
| """a class used to convert a list of tokens into a cpp Expr object""" |
| |
| re_octal = re.compile(r"\s*\(0[0-7]+\).*") |
| re_decimal = re.compile(r"\s*\(\d+[ulUL]*\).*") |
| re_hexadecimal = re.compile(r"\s*\(0[xX][0-9a-fA-F]*\).*") |
| |
| def __init__(self,tokens): |
| self.tok = tokens |
| self.n = len(self.tok) |
| self.i = 0 |
| |
| def mark(self): |
| return self.i |
| |
| def release(self,pos): |
| self.i = pos |
| |
| def peekId(self): |
| if self.i < self.n: |
| return self.tok[self.i].id |
| return None |
| |
| def peek(self): |
| if self.i < self.n: |
| return self.tok[self.i] |
| return None |
| |
| def skip(self): |
| if self.i < self.n: |
| self.i += 1 |
| |
| def skipOptional(self,id): |
| if self.i < self.n and self.tok[self.i].id == id: |
| self.i += 1 |
| |
| def skipSpaces(self): |
| i = self.i |
| n = self.n |
| tok = self.tok |
| while i < n and (tok[i] == tokSPACE or tok[i] == tokLN): |
| i += 1 |
| self.i = i |
| |
| # all the isXXX functions returns a (expr,nextpos) pair if a match is found |
| # or None if not |
| |
| def is_integer(self): |
| id = self.tok[self.i].id |
| c = id[0] |
| if c < '0' or c > '9': |
| return None |
| |
| m = ExprParser.re_octal.match(id) |
| if m: |
| return (IntExpr(id), m.end(1)) |
| |
| m = ExprParser.re_decimal.match(id) |
| if m: |
| return (IntExpr(id), m.end(1)) |
| |
| m = ExprParser.re_hexadecimal(id) |
| if m: |
| return (IntExpr(id), m.end(1)) |
| |
| return None |
| |
| def is_defined(self): |
| id = self.tok[self.i].id |
| if id != "defined": |
| return None |
| |
| pos = self.mark() |
| |
| use_paren = 0 |
| if self.peekId() == tokLPAREN: |
| self.skip() |
| use_paren = 1 |
| |
| if self.peekId() != tokIDENT: |
| self.throw( BadExpectedToken, "identifier expected") |
| |
| macroname = self.peek().value |
| self.skip() |
| if use_paren: |
| self.skipSpaces() |
| if self.peekId() != tokRPAREN: |
| self.throw( BadExpectedToken, "missing right-paren after 'defined' directive") |
| self.skip() |
| |
| i = self.i |
| return (DefinedExpr(macroname),i+1) |
| |
| def is_call_or_ident(self): |
| pass |
| |
| def parse(self, i): |
| return None |
| |
| ##################################################################################### |
| ##################################################################################### |
| ##### ##### |
| ##### C P P E X P R E S S I O N S ##### |
| ##### ##### |
| ##################################################################################### |
| ##################################################################################### |
| |
| class CppInvalidExpression(Exception): |
| """an exception raised when an invalid/unsupported cpp expression is detected""" |
| pass |
| |
| class CppExpr: |
| """a class that models the condition of #if directives into |
| an expression tree. each node in the tree is of the form (op,arg) or (op,arg1,arg2) |
| where "op" is a string describing the operation""" |
| |
| unaries = [ "!", "~" ] |
| binaries = [ "+", "-", "<", "<=", ">=", ">", "&&", "||", "*", "/", "%", "&", "|", "^", "<<", ">>", "==", "!=" ] |
| precedences = { "||": 1, |
| "&&": 2, |
| "|": 3, |
| "^": 4, |
| "&": 5, |
| "==":6, "!=":6, |
| "<":7, "<=":7, ">":7, ">=":7, |
| "<<":8, ">>":8, |
| "+":9, "-":9, |
| "*":10, "/":10, "%":10, |
| "!":11, "~":12 |
| } |
| |
| def __init__(self, tokens): |
| """initialize a CppExpr. 'tokens' must be a CppToken list""" |
| self.tok = tokens |
| self.n = len(tokens) |
| if debugCppExpr: |
| print "CppExpr: trying to parse %s" % repr(tokens) |
| expr = self.is_expr(0) |
| if debugCppExpr: |
| print "CppExpr: got " + repr(expr) |
| self.expr = expr[0] |
| |
| re_cpp_constant = re.compile(r"((\d|\w|_)+)") |
| |
| def throw(self,exception,i,msg): |
| if i < self.n: |
| tok = self.tok[i] |
| print "%d:%d: %s" % (tok.lineno,tok.colno,msg) |
| else: |
| print "EOF: %s" % msg |
| raise exception |
| |
| def skip_spaces(self,i): |
| """skip spaces in input token list""" |
| while i < self.n: |
| t = self.tok[i] |
| if t.id != tokSPACE and t.id != tokLN: |
| break |
| i += 1 |
| return i |
| |
| def expectId(self,i,id): |
| """check that a given token id is at the current position, then skip over it""" |
| i = self.skip_spaces(i) |
| if i >= self.n or self.tok[i].id != id: |
| self.throw(BadExpectedToken,i,"### expecting '%s' in expression, got '%s'" % (id, self.tok[i].id)) |
| return i+1 |
| |
| def expectIdent(self,i): |
| i = self.skip_spaces(i) |
| if i >= self.n or self.tok[i].id != tokIDENT: |
| self.throw(BadExpectedToken,i,"### expecting identifier in expression, got '%s'" % (id, self.tok[i].id)) |
| return i+1 |
| |
| # the is_xxxxx function returns either None or a pair (e,nextpos) |
| # where 'e' is an expression tuple (e.g. (op,arg)) and 'nextpos' is |
| # the corresponding next position in the input token list |
| # |
| |
| def is_decimal(self,i): |
| v = self.tok[i].value[:] |
| while len(v) > 0 and v[-1] in "ULul": |
| v = v[:-1] |
| for digit in v: |
| if not digit.isdigit(): |
| return None |
| |
| # for an integer expression tuple, the argument |
| # is simply the value as an integer |
| val = string.atoi(v) |
| return ("int", val), i+1 |
| |
| def is_hexadecimal(self,i): |
| v = self.tok[i].value[:] |
| while len(v) > 0 and v[-1] in "ULul": |
| v = v[:-1] |
| if len(v) > 2 and (v[0:2] == "0x" or v[0:2] == "0X"): |
| for digit in v[2:]: |
| if not digit in "0123456789abcdefABCDEF": |
| return None |
| |
| # for an hex expression tuple, the argument |
| # is the value as an integer |
| val = int(v[2:], 16) |
| return ("hex", val), i+1 |
| |
| return None |
| |
| def is_integer(self,i): |
| if self.tok[i].id != tokNUMBER: |
| return None |
| |
| c = self.is_decimal(i) |
| if c: return c |
| |
| c = self.is_hexadecimal(i) |
| if c: return c |
| |
| return None |
| |
| def is_number(self,i): |
| t = self.tok[i] |
| if t.id == tokMINUS and i+1 < self.n: |
| c = self.is_integer(i+1) |
| if c: |
| e, i2 = c |
| op, val = e |
| return (op, -val), i2 |
| if t.id == tokPLUS and i+1 < self.n: |
| c = self.is_integer(i+1) |
| if c: return c |
| |
| return self.is_integer(i) |
| |
| |
| def is_alnum(self,i): |
| """test wether a given token is alpha-numeric""" |
| i = self.skip_spaces(i) |
| if i >= self.n: |
| return None |
| t = self.tok[i] |
| m = CppExpr.re_cpp_constant.match(t.id) |
| if m: |
| #print "... alnum '%s'" % m.group(1) |
| r = m.group(1) |
| return ("ident", r), i+1 |
| return None |
| |
| def is_defined(self,i): |
| t = self.tok[i] |
| if t.id != tokDEFINED: |
| return None |
| |
| # we have the defined keyword, check the rest |
| i = self.skip_spaces(i+1) |
| use_parens = 0 |
| if i < self.n and self.tok[i].id == tokLPAREN: |
| use_parens = 1 |
| i = self.skip_spaces(i+1) |
| |
| if i >= self.n: |
| self.throw(CppConstantExpected,i,"### 'defined' must be followed by macro name or left paren") |
| |
| t = self.tok[i] |
| if t.id != tokIDENT: |
| self.throw(CppConstantExpected,i,"### 'defined' must be followed by macro name") |
| |
| i += 1 |
| if use_parens: |
| i = self.expectId(i,tokRPAREN) |
| |
| return ("defined",t.value), i |
| |
| |
| def is_call_or_ident(self,i): |
| i = self.skip_spaces(i) |
| if i >= self.n: |
| return None |
| |
| t = self.tok[i] |
| if t.id != tokIDENT: |
| return None |
| |
| name = t.value |
| |
| i = self.skip_spaces(i+1) |
| if i >= self.n or self.tok[i].id != tokLPAREN: |
| return ("ident", name), i |
| |
| params = [] |
| depth = 1 |
| i += 1 |
| j = i |
| while i < self.n: |
| id = self.tok[i].id |
| if id == tokLPAREN: |
| depth += 1 |
| elif depth == 1 and (id == tokCOMMA or id == tokRPAREN): |
| while j < i and self.tok[j].id == tokSPACE: |
| j += 1 |
| k = i |
| while k > j and self.tok[k-1].id == tokSPACE: |
| k -= 1 |
| param = self.tok[j:k] |
| params.append( param ) |
| if id == tokRPAREN: |
| break |
| j = i+1 |
| elif id == tokRPAREN: |
| depth -= 1 |
| i += 1 |
| |
| if i >= self.n: |
| return None |
| |
| return ("call", (name, params)), i+1 |
| |
| def is_token(self,i,token): |
| i = self.skip_spaces(i) |
| if i >= self.n or self.tok[i].id != token: |
| return None |
| return token, i+1 |
| |
| |
| def is_value(self,i): |
| t = self.tok[i] |
| if t.id == tokSTRING: |
| return ("string", t.value), i+1 |
| |
| c = self.is_number(i) |
| if c: return c |
| |
| c = self.is_defined(i) |
| if c: return c |
| |
| c = self.is_call_or_ident(i) |
| if c: return c |
| |
| i = self.skip_spaces(i) |
| if i >= self.n or self.tok[i].id != tokLPAREN: |
| return None |
| |
| popcount = 1 |
| i2 = i+1 |
| while i2 < self.n: |
| t = self.tok[i2] |
| if t.id == tokLPAREN: |
| popcount += 1 |
| elif t.id == tokRPAREN: |
| popcount -= 1 |
| if popcount == 0: |
| break |
| i2 += 1 |
| |
| if popcount != 0: |
| self.throw(CppInvalidExpression, i, "expression missing closing parenthesis") |
| |
| if debugCppExpr: |
| print "CppExpr: trying to parse sub-expression %s" % repr(self.tok[i+1:i2]) |
| oldcount = self.n |
| self.n = i2 |
| c = self.is_expr(i+1) |
| self.n = oldcount |
| if not c: |
| self.throw(CppInvalidExpression, i, "invalid expression within parenthesis") |
| |
| e, i = c |
| return e, i2+1 |
| |
| def is_unary(self,i): |
| i = self.skip_spaces(i) |
| if i >= self.n: |
| return None |
| |
| t = self.tok[i] |
| if t.id in CppExpr.unaries: |
| c = self.is_unary(i+1) |
| if not c: |
| self.throw(CppInvalidExpression, i, "%s operator must be followed by value" % t.id) |
| e, i = c |
| return (t.id, e), i |
| |
| return self.is_value(i) |
| |
| def is_binary(self,i): |
| i = self.skip_spaces(i) |
| if i >= self.n: |
| return None |
| |
| c = self.is_unary(i) |
| if not c: |
| return None |
| |
| e1, i2 = c |
| i2 = self.skip_spaces(i2) |
| if i2 >= self.n: |
| return c |
| |
| t = self.tok[i2] |
| if t.id in CppExpr.binaries: |
| c = self.is_binary(i2+1) |
| if not c: |
| self.throw(CppInvalidExpression, i,"### %s operator must be followed by value" % t.id ) |
| e2, i3 = c |
| return (t.id, e1, e2), i3 |
| |
| return None |
| |
| def is_expr(self,i): |
| return self.is_binary(i) |
| |
| def dump_node(self,e): |
| op = e[0] |
| line = "(" + op |
| if op == "int": |
| line += " %d)" % e[1] |
| elif op == "hex": |
| line += " 0x%x)" % e[1] |
| elif op == "ident": |
| line += " %s)" % e[1] |
| elif op == "defined": |
| line += " %s)" % e[1] |
| elif op == "call": |
| arg = e[1] |
| line += " %s [" % arg[0] |
| prefix = "" |
| for param in arg[1]: |
| par = "" |
| for tok in param: |
| par += str(tok) |
| line += "%s%s" % (prefix, par) |
| prefix = "," |
| line += "])" |
| elif op in CppExpr.unaries: |
| line += " %s)" % self.dump_node(e[1]) |
| elif op in CppExpr.binaries: |
| line += " %s %s)" % (self.dump_node(e[1]), self.dump_node(e[2])) |
| else: |
| line += " ?%s)" % repr(e[1]) |
| |
| return line |
| |
| def __repr__(self): |
| return self.dump_node(self.expr) |
| |
| def source_node(self,e): |
| op = e[0] |
| if op == "int": |
| return "%d" % e[1] |
| if op == "hex": |
| return "0x%x" % e[1] |
| if op == "ident": |
| # XXX: should try to expand |
| return e[1] |
| if op == "defined": |
| return "defined(%s)" % e[1] |
| |
| prec = CppExpr.precedences.get(op,1000) |
| arg = e[1] |
| if op in CppExpr.unaries: |
| arg_src = self.source_node(arg) |
| arg_op = arg[0] |
| arg_prec = CppExpr.precedences.get(arg[0],1000) |
| if arg_prec < prec: |
| return "!(" + arg_src + ")" |
| else: |
| return "!" + arg_src |
| if op in CppExpr.binaries: |
| arg2 = e[2] |
| arg1_op = arg[0] |
| arg2_op = arg2[0] |
| arg1_src = self.source_node(arg) |
| arg2_src = self.source_node(arg2) |
| if CppExpr.precedences.get(arg1_op,1000) < prec: |
| arg1_src = "(%s)" % arg1_src |
| if CppExpr.precedences.get(arg2_op,1000) < prec: |
| arg2_src = "(%s)" % arg2_src |
| |
| return "%s %s %s" % (arg1_src, op, arg2_src) |
| return "???" |
| |
| def __str__(self): |
| return self.source_node(self.expr) |
| |
| def int_node(self,e): |
| if e[0] == "int": |
| return e[1] |
| elif e[1] == "hex": |
| return int(e[1],16) |
| else: |
| return None |
| |
| def toInt(self): |
| return self.int_node(self.expr) |
| |
| def optimize_node(self,e,macros={}): |
| op = e[0] |
| if op == "defined": |
| name = e[1] |
| if macros.has_key(name): |
| if macros[name] == kCppUndefinedMacro: |
| return ("int", 0) |
| else: |
| return ("int", 1) |
| |
| if kernel_remove_config_macros and name.startswith("CONFIG_"): |
| return ("int", 0) |
| |
| elif op == "!": |
| op, v = e |
| v = self.optimize_node(v, macros) |
| if v[0] == "int": |
| if v[1] == 0: |
| return ("int", 1) |
| else: |
| return ("int", 0) |
| |
| elif op == "&&": |
| op, l, r = e |
| l = self.optimize_node(l, macros) |
| r = self.optimize_node(r, macros) |
| li = self.int_node(l) |
| ri = self.int_node(r) |
| if li != None: |
| if li == 0: |
| return ("int", 0) |
| else: |
| return r |
| |
| elif op == "||": |
| op, l, r = e |
| l = self.optimize_node(l, macros) |
| r = self.optimize_node(r, macros) |
| li = self.int_node(l) |
| ri = self.int_node(r) |
| if li != None: |
| if li == 0: |
| return r |
| else: |
| return ("int", 1) |
| elif ri != None: |
| if ri == 0: |
| return l |
| else: |
| return ("int", 1) |
| return e |
| |
| def optimize(self,macros={}): |
| self.expr = self.optimize_node(self.expr,macros) |
| |
| def removePrefixedNode(self,e,prefix,names): |
| op = e[0] |
| if op == "defined": |
| name = e[1] |
| if name.startswith(prefix): |
| if names.has_key[name] and names[name] == "y": |
| return ("int", 1) |
| else: |
| return ("int", 0) |
| |
| elif op in CppExpr.unaries: |
| op, v = e |
| v = self.removePrefixedNode(v,prefix,names) |
| return (op, v) |
| elif op in CppExpr.binaries: |
| op, v1, v2 = e |
| v1 = self.removePrefixedNode(v1,prefix,names) |
| v2 = self.removePrefixedNode(v2,prefix,names) |
| return (op, v1, v2) |
| elif op == "call": |
| func, params = e[1] |
| params2 = [] |
| for param in params: |
| params2.append( self.removePrefixedNode(param,prefix,names) ) |
| return (op, (func, params2)) |
| |
| return e |
| |
| def removePrefixed(self,prefix,names={}): |
| self.expr = self.removePrefixedNode(self.expr,prefix,names) |
| |
| def is_equal_node(self,e1,e2): |
| if e1[0] != e2[0] or len(e1) != len(e2): |
| return False |
| |
| op = e1[0] |
| if op == "int" or op == "hex" or op == "!" or op == "defined": |
| return e1[0] == e2[0] |
| |
| return self.is_equal_node(e1[1],e2[1]) and self.is_equal_node(e1[2],e2[2]) |
| |
| def is_equal(self,other): |
| return self.is_equal_node(self.expr,other.expr) |
| |
| def test_cpp_expr(expr, expected): |
| e = CppExpr( CppLineTokenizer( expr ).toTokenList() ) |
| #print repr(e.expr) |
| s1 = repr(e) |
| if s1 != expected: |
| print "KO: expression '%s' generates '%s', should be '%s'" % (expr, s1, expected) |
| else: |
| #print "OK: expression '%s'" % expr |
| pass |
| |
| def test_cpp_expr_optim(expr, expected, macros={}): |
| e = CppExpr( CppLineTokenizer( expr ).toTokenList() ) |
| e.optimize(macros) |
| |
| s1 = repr(e) |
| if s1 != expected: |
| print "KO: optimized expression '%s' generates '%s', should be '%s'" % (expr, s1, expected) |
| else: |
| #print "OK: optmized expression '%s'" % expr |
| pass |
| |
| def test_cpp_expr_source(expr, expected): |
| e = CppExpr( CppLineTokenizer( expr ).toTokenList() ) |
| s1 = str(e) |
| if s1 != expected: |
| print "KO: source expression '%s' generates '%s', should be '%s'" % (expr, s1, expected) |
| else: |
| #print "OK: source expression '%s'" % expr |
| pass |
| |
| def test_CppExpr(): |
| print "testing CppExpr" |
| test_cpp_expr( "0", "(int 0)" ) |
| test_cpp_expr( "1", "(int 1)" ) |
| test_cpp_expr( "1 && 1", "(&& (int 1) (int 1))" ) |
| test_cpp_expr( "1 && 0", "(&& (int 1) (int 0))" ) |
| test_cpp_expr( "EXAMPLE", "(ident EXAMPLE)" ) |
| test_cpp_expr( "EXAMPLE - 3", "(- (ident EXAMPLE) (int 3))" ) |
| test_cpp_expr( "defined(EXAMPLE)", "(defined EXAMPLE)" ) |
| test_cpp_expr( "!defined(EXAMPLE)", "(! (defined EXAMPLE))" ) |
| test_cpp_expr( "defined(ABC) || defined(BINGO)", "(|| (defined ABC) (defined BINGO))" ) |
| test_cpp_expr( "FOO(BAR)", "(call FOO [BAR])" ) |
| |
| test_cpp_expr_optim( "0", "(int 0)" ) |
| test_cpp_expr_optim( "1", "(int 1)" ) |
| test_cpp_expr_optim( "1 && 1", "(int 1)" ) |
| test_cpp_expr_optim( "1 && 0", "(int 0)" ) |
| test_cpp_expr_optim( "0 && 1", "(int 0)" ) |
| test_cpp_expr_optim( "0 && 0", "(int 0)" ) |
| test_cpp_expr_optim( "1 || 1", "(int 1)" ) |
| test_cpp_expr_optim( "1 || 0", "(int 1)" ) |
| test_cpp_expr_optim( "0 || 1", "(int 1)" ) |
| test_cpp_expr_optim( "0 || 0", "(int 0)" ) |
| test_cpp_expr_optim( "EXAMPLE", "(ident EXAMPLE)" ) |
| test_cpp_expr_optim( "EXAMPLE - 3", "(- (ident EXAMPLE) (int 3))" ) |
| test_cpp_expr_optim( "defined(EXAMPLE)", "(defined EXAMPLE)" ) |
| test_cpp_expr_optim( "defined(EXAMPLE)", "(int 1)", { "EXAMPLE": "XOWOE" } ) |
| test_cpp_expr_optim( "defined(EXAMPLE)", "(int 0)", { "EXAMPLE": kCppUndefinedMacro} ) |
| test_cpp_expr_optim( "!defined(EXAMPLE)", "(! (defined EXAMPLE))" ) |
| test_cpp_expr_optim( "!defined(EXAMPLE)", "(int 0)", { "EXAMPLE" : "XOWOE" } ) |
| test_cpp_expr_optim( "!defined(EXAMPLE)", "(int 1)", { "EXAMPLE" : kCppUndefinedMacro } ) |
| test_cpp_expr_optim( "defined(ABC) || defined(BINGO)", "(|| (defined ABC) (defined BINGO))" ) |
| test_cpp_expr_optim( "defined(ABC) || defined(BINGO)", "(int 1)", { "ABC" : "1" } ) |
| test_cpp_expr_optim( "defined(ABC) || defined(BINGO)", "(int 1)", { "BINGO" : "1" } ) |
| test_cpp_expr_optim( "defined(ABC) || defined(BINGO)", "(defined ABC)", { "BINGO" : kCppUndefinedMacro } ) |
| test_cpp_expr_optim( "defined(ABC) || defined(BINGO)", "(int 0)", { "ABC" : kCppUndefinedMacro, "BINGO" : kCppUndefinedMacro } ) |
| |
| test_cpp_expr_source( "0", "0" ) |
| test_cpp_expr_source( "1", "1" ) |
| test_cpp_expr_source( "1 && 1", "1 && 1" ) |
| test_cpp_expr_source( "1 && 0", "1 && 0" ) |
| test_cpp_expr_source( "0 && 1", "0 && 1" ) |
| test_cpp_expr_source( "0 && 0", "0 && 0" ) |
| test_cpp_expr_source( "1 || 1", "1 || 1" ) |
| test_cpp_expr_source( "1 || 0", "1 || 0" ) |
| test_cpp_expr_source( "0 || 1", "0 || 1" ) |
| test_cpp_expr_source( "0 || 0", "0 || 0" ) |
| test_cpp_expr_source( "EXAMPLE", "EXAMPLE" ) |
| test_cpp_expr_source( "EXAMPLE - 3", "EXAMPLE - 3" ) |
| test_cpp_expr_source( "defined(EXAMPLE)", "defined(EXAMPLE)" ) |
| test_cpp_expr_source( "defined EXAMPLE", "defined(EXAMPLE)" ) |
| |
| |
| ##################################################################################### |
| ##################################################################################### |
| ##### ##### |
| ##### C P P B L O C K ##### |
| ##### ##### |
| ##################################################################################### |
| ##################################################################################### |
| |
| class Block: |
| """a class used to model a block of input source text. there are two block types: |
| - directive blocks: contain the tokens of a single pre-processor directive (e.g. #if) |
| - text blocks, contain the tokens of non-directive blocks |
| |
| the cpp parser class below will transform an input source file into a list of Block |
| objects (grouped in a BlockList object for convenience)""" |
| |
| def __init__(self,tokens,directive=None,lineno=0): |
| """initialize a new block, if 'directive' is None, this is a text block |
| NOTE: this automatically converts '#ifdef MACRO' into '#if defined(MACRO)' |
| and '#ifndef MACRO' into '#if !defined(MACRO)'""" |
| if directive == "ifdef": |
| tok = Token() |
| tok.set(tokDEFINED) |
| tokens = [ tok ] + tokens |
| directive = "if" |
| |
| elif directive == "ifndef": |
| tok1 = Token() |
| tok2 = Token() |
| tok1.set(tokNOT) |
| tok2.set(tokDEFINED) |
| tokens = [ tok1, tok2 ] + tokens |
| directive = "if" |
| |
| self.tokens = tokens |
| self.directive = directive |
| if lineno > 0: |
| self.lineno = lineno |
| else: |
| self.lineno = self.tokens[0].lineno |
| |
| if self.isIf(): |
| self.expr = CppExpr( self.tokens ) |
| |
| def isDirective(self): |
| """returns True iff this is a directive block""" |
| return self.directive != None |
| |
| def isConditional(self): |
| """returns True iff this is a conditional directive block""" |
| return self.directive in ["if","ifdef","ifndef","else","elif","endif"] |
| |
| def isDefine(self): |
| """returns the macro name in a #define directive, or None otherwise""" |
| if self.directive != "define": |
| return None |
| |
| return self.tokens[0].value |
| |
| def isIf(self): |
| """returns True iff this is an #if-like directive block""" |
| return self.directive in ["if","ifdef","ifndef","elif"] |
| |
| def isInclude(self): |
| """checks wether this is a #include directive. if true, then returns the |
| corresponding file name (with brackets or double-qoutes). None otherwise""" |
| if self.directive != "include": |
| return None |
| |
| #print "iii " + repr(self.tokens) |
| if self.tokens[0].id == tokSTRING: |
| # a double-quote include, that's easy |
| return self.tokens[0].value |
| |
| # we only want the bracket part, not any comments or junk after it |
| if self.tokens[0].id == "<": |
| i = 0 |
| tok = self.tokens |
| n = len(tok) |
| while i < n and tok[i].id != ">": |
| i += 1 |
| |
| if i >= n: |
| return None |
| |
| return string.join([ str(x) for x in tok[:i+1] ],"") |
| |
| else: |
| return None |
| |
| def removeWhiteSpace(self): |
| # Remove trailing whitespace and empty lines |
| # All whitespace is also contracted to a single space |
| if self.directive != None: |
| return |
| |
| tokens = [] |
| line = 0 # index of line start |
| space = -1 # index of first space, or -1 |
| ii = 0 |
| nn = len(self.tokens) |
| while ii < nn: |
| tok = self.tokens[ii] |
| |
| # If we find a space, record its position if this is the first |
| # one the line start or the previous character. Don't append |
| # anything to tokens array yet though. |
| if tok.id == tokSPACE: |
| if space < 0: |
| space = ii |
| ii += 1 |
| continue |
| |
| # If this is a line space, ignore the spaces we found previously |
| # on the line, and remove empty lines. |
| if tok.id == tokLN: |
| old_line = line |
| old_space = space |
| #print "N line=%d space=%d ii=%d" % (line, space, ii) |
| ii += 1 |
| line = ii |
| space = -1 |
| if old_space == old_line: # line only contains spaces |
| #print "-s" |
| continue |
| if ii-1 == old_line: # line is empty |
| #print "-e" |
| continue |
| tokens.append(tok) |
| continue |
| |
| # Other token, append any space range if any, converting each |
| # one to a single space character, then append the token. |
| if space >= 0: |
| jj = space |
| space = -1 |
| while jj < ii: |
| tok2 = self.tokens[jj] |
| tok2.value = " " |
| tokens.append(tok2) |
| jj += 1 |
| |
| tokens.append(tok) |
| ii += 1 |
| |
| self.tokens = tokens |
| |
| def writeWithWarning(self,out,warning,left_count,repeat_count): |
| # removeWhiteSpace() will sometimes creates non-directive blocks |
| # without any tokens. These come from blocks that only contained |
| # empty lines and spaces. They should not be printed in the final |
| # output, and then should not be counted for this operation. |
| # |
| if not self.directive and self.tokens == []: |
| return left_count |
| |
| if self.directive: |
| out.write(str(self).rstrip() + "\n") |
| left_count -= 1 |
| if left_count == 0: |
| out.write(warning) |
| left_count = repeat_count |
| |
| else: |
| for tok in self.tokens: |
| out.write(str(tok)) |
| if tok.id == tokLN: |
| left_count -= 1 |
| if left_count == 0: |
| out.write(warning) |
| left_count = repeat_count |
| |
| return left_count |
| |
| |
| def __repr__(self): |
| """generate the representation of a given block""" |
| if self.directive: |
| result = "#%s " % self.directive |
| if self.isIf(): |
| result += repr(self.expr) |
| else: |
| for tok in self.tokens: |
| result += repr(tok) |
| else: |
| result = "" |
| for tok in self.tokens: |
| result += repr(tok) |
| |
| return result |
| |
| def __str__(self): |
| """generate the string representation of a given block""" |
| if self.directive: |
| if self.directive == "if": |
| # small optimization to re-generate #ifdef and #ifndef |
| e = self.expr.expr |
| op = e[0] |
| if op == "defined": |
| result = "#ifdef %s" % e[1] |
| elif op == "!" and e[1][0] == "defined": |
| result = "#ifndef %s" % e[1][1] |
| else: |
| result = "#if " + str(self.expr) |
| else: |
| result = "#%s" % self.directive |
| if len(self.tokens): |
| result += " " |
| for tok in self.tokens: |
| result += str(tok) |
| else: |
| result = "" |
| for tok in self.tokens: |
| result += str(tok) |
| |
| return result |
| |
| class BlockList: |
| """a convenience class used to hold and process a list of blocks returned by |
| the cpp parser""" |
| def __init__(self,blocks): |
| self.blocks = blocks |
| |
| def __len__(self): |
| return len(self.blocks) |
| |
| def __getitem__(self,n): |
| return self.blocks[n] |
| |
| def __repr__(self): |
| return repr(self.blocks) |
| |
| def __str__(self): |
| result = "" |
| for b in self.blocks: |
| result += str(b) |
| if b.isDirective(): |
| result = result.rstrip() + '\n' |
| return result |
| |
| def optimizeIf01(self): |
| """remove the code between #if 0 .. #endif in a BlockList""" |
| self.blocks = optimize_if01(self.blocks) |
| |
| def optimizeMacros(self, macros): |
| """remove known defined and undefined macros from a BlockList""" |
| for b in self.blocks: |
| if b.isIf(): |
| b.expr.optimize(macros) |
| |
| def removeMacroDefines(self,macros): |
| """remove known macro definitions from a BlockList""" |
| self.blocks = remove_macro_defines(self.blocks,macros) |
| |
| def removePrefixed(self,prefix,names): |
| for b in self.blocks: |
| if b.isIf(): |
| b.expr.removePrefixed(prefix,names) |
| |
| def removeWhiteSpace(self): |
| for b in self.blocks: |
| b.removeWhiteSpace() |
| |
| def optimizeAll(self,macros): |
| self.optimizeMacros(macros) |
| self.optimizeIf01() |
| return |
| |
| def findIncludes(self): |
| """return the list of included files in a BlockList""" |
| result = [] |
| for b in self.blocks: |
| i = b.isInclude() |
| if i: |
| result.append(i) |
| |
| return result |
| |
| |
| def write(self,out): |
| out.write(str(self)) |
| |
| def writeWithWarning(self,out,warning,repeat_count): |
| left_count = repeat_count |
| for b in self.blocks: |
| left_count = b.writeWithWarning(out,warning,left_count,repeat_count) |
| |
| def removeComments(self): |
| for b in self.blocks: |
| for tok in b.tokens: |
| if tok.id == tokSPACE: |
| tok.value = " " |
| |
| def removeVarsAndFuncs(self,knownStatics=set()): |
| """remove all extern and static declarations corresponding |
| to variable and function declarations. we only accept typedefs |
| and enum/structs/union declarations. |
| |
| however, we keep the definitions corresponding to the set |
| of known static inline functions in the set 'knownStatics', |
| which is useful for optimized byteorder swap functions and |
| stuff like that. |
| """ |
| # state = 0 => normal (i.e. LN + spaces) |
| # state = 1 => typedef/struct encountered, ends with ";" |
| # state = 2 => var declaration encountered, ends with ";" |
| # state = 3 => func declaration encountered, ends with "}" |
| state = 0 |
| depth = 0 |
| blocks2 = [] |
| skipTokens = False |
| for b in self.blocks: |
| if b.isDirective(): |
| blocks2.append(b) |
| else: |
| n = len(b.tokens) |
| i = 0 |
| if skipTokens: |
| first = n |
| else: |
| first = 0 |
| while i < n: |
| tok = b.tokens[i] |
| tokid = tok.id |
| # If we are not looking for the start of a new |
| # type/var/func, then skip over tokens until |
| # we find our terminator, managing the depth of |
| # accolades as we go. |
| if state > 0: |
| terminator = False |
| if tokid == '{': |
| depth += 1 |
| elif tokid == '}': |
| if depth > 0: |
| depth -= 1 |
| if (depth == 0) and (state == 3): |
| terminator = True |
| elif tokid == ';' and depth == 0: |
| terminator = True |
| |
| if terminator: |
| # we found the terminator |
| state = 0 |
| if skipTokens: |
| skipTokens = False |
| first = i+1 |
| |
| i = i+1 |
| continue |
| |
| # We are looking for the start of a new type/func/var |
| # ignore whitespace |
| if tokid in [tokLN, tokSPACE]: |
| i = i+1 |
| continue |
| |
| # Is it a new type definition, then start recording it |
| if tok.value in [ 'struct', 'typedef', 'enum', 'union', '__extension__' ]: |
| #print "$$$ keep type declr" + repr(b.tokens[i:]) |
| state = 1 |
| i = i+1 |
| continue |
| |
| # Is it a variable or function definition. If so, first |
| # try to determine which type it is, and also extract |
| # its name. |
| # |
| # We're going to parse the next tokens of the same block |
| # until we find a semi-column or a left parenthesis. |
| # |
| # The semi-column corresponds to a variable definition, |
| # the left-parenthesis to a function definition. |
| # |
| # We also assume that the var/func name is the last |
| # identifier before the terminator. |
| # |
| j = i+1 |
| ident = "" |
| while j < n: |
| tokid = b.tokens[j].id |
| if tokid == '(': # a function declaration |
| state = 3 |
| break |
| elif tokid == ';': # a variable declaration |
| state = 2 |
| break |
| if tokid == tokIDENT: |
| ident = b.tokens[j].value |
| j += 1 |
| |
| if j >= n: |
| # This can only happen when the declaration |
| # does not end on the current block (e.g. with |
| # a directive mixed inside it. |
| # |
| # We will treat it as malformed because |
| # it's very hard to recover from this case |
| # without making our parser much more |
| # complex. |
| # |
| #print "### skip unterminated static '%s'" % ident |
| break |
| |
| if ident in knownStatics: |
| #print "### keep var/func '%s': %s" % (ident,repr(b.tokens[i:j])) |
| pass |
| else: |
| # We're going to skip the tokens for this declaration |
| #print "### skip variable /func'%s': %s" % (ident,repr(b.tokens[i:j])) |
| if i > first: |
| blocks2.append( Block(b.tokens[first:i])) |
| skipTokens = True |
| first = n |
| |
| i = i+1 |
| |
| if i > first: |
| #print "### final '%s'" % repr(b.tokens[first:i]) |
| blocks2.append( Block(b.tokens[first:i]) ) |
| |
| self.blocks = blocks2 |
| |
| def insertDisclaimer(self,disclaimer="/* auto-generated file, DO NOT EDIT */"): |
| """insert your standard issue disclaimer that this is an |
| auto-generated file, etc..""" |
| tokens = CppLineTokenizer( disclaimer ).toTokenList() |
| tokens = tokens[:-1] # remove trailing tokLN |
| self.blocks = [ Block(tokens) ] + self.blocks |
| |
| def replaceTokens(self,replacements=dict()): |
| """replace tokens according to the given dict |
| """ |
| for b in self.blocks: |
| if not b.isDirective(): |
| for tok in b.tokens: |
| if tok.id == tokIDENT: |
| if tok.value in replacements: |
| tok.value = replacements[tok.value] |
| |
| class BlockParser: |
| """a class used to convert an input source file into a BlockList object""" |
| |
| def __init__(self,tokzer=None): |
| """initialize a block parser. the input source is provided through a Tokenizer |
| object""" |
| self.reset(tokzer) |
| |
| def reset(self,tokzer): |
| self.state = 1 |
| self.tokzer = tokzer |
| |
| def getBlocks(self,tokzer=None): |
| """tokenize and parse the input source, return a BlockList object |
| NOTE: empty and line-numbering directives are ignored and removed |
| from the result. as a consequence, it is possible to have |
| two successive text blocks in the result""" |
| # state 0 => in source code |
| # state 1 => in source code, after a LN |
| # state 2 => in source code, after LN then some space |
| state = 1 |
| lastLN = 0 |
| current = [] |
| blocks = [] |
| |
| if tokzer == None: |
| tokzer = self.tokzer |
| |
| while 1: |
| tok = tokzer.getToken() |
| if tok.id == tokEOF: |
| break |
| |
| if tok.id == tokLN: |
| state = 1 |
| current.append(tok) |
| lastLN = len(current) |
| |
| elif tok.id == tokSPACE: |
| if state == 1: |
| state = 2 |
| current.append(tok) |
| |
| elif tok.id == "#": |
| if state > 0: |
| # this is the start of a directive |
| |
| if lastLN > 0: |
| # record previous tokens as text block |
| block = Block(current[:lastLN]) |
| blocks.append(block) |
| lastLN = 0 |
| |
| current = [] |
| |
| # skip spaces after the # |
| while 1: |
| tok = tokzer.getToken() |
| if tok.id != tokSPACE: |
| break |
| |
| if tok.id != tokIDENT: |
| # empty or line-numbering, ignore it |
| if tok.id != tokLN and tok.id != tokEOF: |
| while 1: |
| tok = tokzer.getToken() |
| if tok.id == tokLN or tok.id == tokEOF: |
| break |
| continue |
| |
| directive = tok.value |
| lineno = tok.lineno |
| |
| # skip spaces |
| tok = tokzer.getToken() |
| while tok.id == tokSPACE: |
| tok = tokzer.getToken() |
| |
| # then record tokens until LN |
| dirtokens = [] |
| while tok.id != tokLN and tok.id != tokEOF: |
| dirtokens.append(tok) |
| tok = tokzer.getToken() |
| |
| block = Block(dirtokens,directive,lineno) |
| blocks.append(block) |
| state = 1 |
| |
| else: |
| state = 0 |
| current.append(tok) |
| |
| if len(current) > 0: |
| block = Block(current) |
| blocks.append(block) |
| |
| return BlockList(blocks) |
| |
| def parse(self,tokzer): |
| return self.getBlocks( tokzer ) |
| |
| def parseLines(self,lines): |
| """parse a list of text lines into a BlockList object""" |
| return self.getBlocks( CppLinesTokenizer(lines) ) |
| |
| def parseFile(self,path): |
| """parse a file into a BlockList object""" |
| file = open(path, "rt") |
| result = self.getBlocks( CppFileTokenizer(file) ) |
| file.close() |
| return result |
| |
| |
| def test_block_parsing(lines,expected): |
| blocks = BlockParser().parse( CppLinesTokenizer(lines) ) |
| if len(blocks) != len(expected): |
| raise BadExpectedToken, "parser.buildBlocks returned '%s' expecting '%s'" \ |
| % (str(blocks), repr(expected)) |
| for n in range(len(blocks)): |
| if str(blocks[n]) != expected[n]: |
| raise BadExpectedToken, "parser.buildBlocks()[%d] is '%s', expecting '%s'" \ |
| % (n, str(blocks[n]), expected[n]) |
| #for block in blocks: |
| # print block |
| |
| def test_BlockParser(): |
| test_block_parsing(["#error hello"],["#error hello"]) |
| test_block_parsing([ "foo", "", "bar" ], [ "foo\n\nbar\n" ]) |
| test_block_parsing([ "foo", " # ", "bar" ], [ "foo\n","bar\n" ]) |
| test_block_parsing(\ |
| [ "foo", " # ", " # /* ahah */ if defined(__KERNEL__) ", "bar", "#endif" ], |
| [ "foo\n", "#ifdef __KERNEL__", "bar\n", "#endif" ] ) |
| |
| |
| ##################################################################################### |
| ##################################################################################### |
| ##### ##### |
| ##### B L O C K L I S T O P T I M I Z A T I O N ##### |
| ##### ##### |
| ##################################################################################### |
| ##################################################################################### |
| |
| def remove_macro_defines( blocks, excludedMacros=set() ): |
| """remove macro definitions like #define <macroName> ....""" |
| result = [] |
| for b in blocks: |
| macroName = b.isDefine() |
| if macroName == None or not macroName in excludedMacros: |
| result.append(b) |
| |
| return result |
| |
| def find_matching_endif( blocks, i ): |
| n = len(blocks) |
| depth = 1 |
| while i < n: |
| if blocks[i].isDirective(): |
| dir = blocks[i].directive |
| if dir in [ "if", "ifndef", "ifdef" ]: |
| depth += 1 |
| elif depth == 1 and dir in [ "else", "elif" ]: |
| return i |
| elif dir == "endif": |
| depth -= 1 |
| if depth == 0: |
| return i |
| i += 1 |
| return i |
| |
| def optimize_if01( blocks ): |
| """remove the code between #if 0 .. #endif in a list of CppBlocks""" |
| i = 0 |
| n = len(blocks) |
| result = [] |
| while i < n: |
| j = i |
| while j < n and not blocks[j].isIf(): |
| j += 1 |
| if j > i: |
| D2("appending lines %d to %d" % (blocks[i].lineno, blocks[j-1].lineno)) |
| result += blocks[i:j] |
| if j >= n: |
| break |
| expr = blocks[j].expr |
| r = expr.toInt() |
| if r == None: |
| result.append(blocks[j]) |
| i = j + 1 |
| continue |
| |
| if r == 0: |
| # if 0 => skip everything until the corresponding #endif |
| j = find_matching_endif( blocks, j+1 ) |
| if j >= n: |
| # unterminated #if 0, finish here |
| break |
| dir = blocks[j].directive |
| if dir == "endif": |
| D2("remove 'if 0' .. 'endif' (lines %d to %d)" % (blocks[i].lineno, blocks[j].lineno)) |
| i = j + 1 |
| elif dir == "else": |
| # convert 'else' into 'if 1' |
| D2("convert 'if 0' .. 'else' into 'if 1' (lines %d to %d)" % (blocks[i].lineno, blocks[j-1].lineno)) |
| blocks[j].directive = "if" |
| blocks[j].expr = CppExpr( CppLineTokenizer("1").toTokenList() ) |
| i = j |
| elif dir == "elif": |
| # convert 'elif' into 'if' |
| D2("convert 'if 0' .. 'elif' into 'if'") |
| blocks[j].directive = "if" |
| i = j |
| continue |
| |
| # if 1 => find corresponding endif and remove/transform them |
| k = find_matching_endif( blocks, j+1 ) |
| if k >= n: |
| # unterminated #if 1, finish here |
| D2("unterminated 'if 1'") |
| result += blocks[j+1:k] |
| break |
| |
| dir = blocks[k].directive |
| if dir == "endif": |
| D2("convert 'if 1' .. 'endif' (lines %d to %d)" % (blocks[j].lineno, blocks[k].lineno)) |
| result += optimize_if01(blocks[j+1:k]) |
| i = k+1 |
| elif dir == "else": |
| # convert 'else' into 'if 0' |
| D2("convert 'if 1' .. 'else' (lines %d to %d)" % (blocks[j].lineno, blocks[k].lineno)) |
| result += optimize_if01(blocks[j+1:k]) |
| blocks[k].directive = "if" |
| blocks[k].expr = CppExpr( CppLineTokenizer("0").toTokenList() ) |
| i = k |
| elif dir == "elif": |
| # convert 'elif' into 'if 0' |
| D2("convert 'if 1' .. 'elif' (lines %d to %d)" % (blocks[j].lineno, blocks[k].lineno)) |
| result += optimize_if01(blocks[j+1:k]) |
| blocks[k].expr = CppExpr( CppLineTokenizer("0").toTokenList() ) |
| i = k |
| return result |
| |
| def test_optimizeAll(): |
| text = """\ |
| #if 1 |
| #define GOOD_1 |
| #endif |
| #if 0 |
| #define BAD_2 |
| #define BAD_3 |
| #endif |
| |
| #if 1 |
| #define GOOD_2 |
| #else |
| #define BAD_4 |
| #endif |
| |
| #if 0 |
| #define BAD_5 |
| #else |
| #define GOOD_3 |
| #endif |
| |
| #if 0 |
| #if 1 |
| #define BAD_6 |
| #endif |
| #endif\ |
| """ |
| |
| expected = """\ |
| #define GOOD_1 |
| |
| #define GOOD_2 |
| |
| #define GOOD_3 |
| |
| """ |
| |
| print "running test_BlockList.optimizeAll" |
| out = StringOutput() |
| lines = string.split(text, '\n') |
| list = BlockParser().parse( CppLinesTokenizer(lines) ) |
| #D_setlevel(2) |
| list.optimizeAll( {"__KERNEL__":kCppUndefinedMacro} ) |
| #print repr(list) |
| list.write(out) |
| if out.get() != expected: |
| print "KO: macro optimization failed\n" |
| print "<<<< expecting '", |
| print expected, |
| print "'\n>>>> result '" |
| print out.get(), |
| print "'\n----" |
| |
| |
| ##################################################################################### |
| ##################################################################################### |
| ##### ##### |
| ##### ##### |
| ##### ##### |
| ##################################################################################### |
| ##################################################################################### |
| |
| def runUnitTests(): |
| """run all unit tests for this program""" |
| print "running unit tests" |
| test_CppTokenizer() |
| test_CppExpr() |
| test_optimizeAll() |
| test_BlockParser() |
| print "OK" |
| |
| if __name__ == "__main__": |
| runUnitTests() |
