现在的位置: 主页 > 在线留言 > 文章列表

python实现词法分析

作者:北京东方雨泽科技有限公司 来源:www.dfyz6789.com 发布时间:2017-09-03 12:40:29
 

python实现词法分析 #请先安装Ply # -*- coding: utf-8 -*- #-------------------------------------------------------------------------- #Author:Jmdebugger #email: pengkailb@gmail.com #date: 2013-9-17 #-------------------------------------------------------------------------- import ply.lex as lex tokens = [ "TOKEN_IDENT", "TOKEN_INT", "TOKEN_FLOAT", "TOKEN_STRING", "TOKEN_OP", "TOKEN_DELIM_COMMA", #, "TOKEN_DELIM_OPEN_PAREN", #( "TOKEN_DELIM_CLOSE_PAREN", #) "TOKEN_DELIM_OPEN_BRACKET", #[ "TOKEN_DELIM_CLOSE_BRACKET",#] "TOKEN_DELIM_OPEN_BRACE", #{ "TOKEN_DELIM_CLOSE_BRACE", #} "TOKEN_DELIM_SEMICOLON" #; ] reserved = { 'if' : 'TOKEN_RSRVD_IF', 'else' : 'TOKEN_RSRVD_ELSE', 'true' : 'TOKEN_RSRVD_TRUE', 'false' : 'TOKEN_RSRVD_FALSE', 'while' : 'TOKEN_RSRVD_WHILE', 'break' : 'TOKEN_RSRVD_BREAK', 'continue': 'TOKEN_RSRVD_CONTINUE', 'goto' : 'TOKEN_RSRVD_GOTO', 'func' : 'TOKEN_RSRVD_FUNC', 'var' : 'TOKEN_RSRVD_VAR', 'for' : 'TOKEN_RSRVD_FOR', 'return' : 'TOKEN_RSRVD_RETURN' } tokens += reserved .values() t_ignore = r' \t\r' def t_COMMENT(t): r'(/\*(.|\n)*?\*/)|(\/\/.*)' pass def t_newline(t): r'\n+' t.lexer.lineno += len(t.value) def t_error(t): print "LaunchScript error: "+repr(t.value) def t_TOKEN_IDENT(t): r'[a-zA-Z_][a-zA-Z_0-9]*' #标识符 t.type = reserved.get(t.value , 'TOKEN_IDENT') return t def t_TOKEN_INT(t): r'(0x[a-fA-F0-9]+)|([0-9]+)' return t t_TOKEN_FLOAT = r'[0-9]*\.[0-9]+' t_TOKEN_STRING = r'(\"([^\\\r]|(\\.))*?\")' #|(\"([^\\\n]|(\\.))*?\")' only for windows t_TOKEN_DELIM_COMMA = r'\,' t_TOKEN_DELIM_OPEN_PAREN = r'\(' t_TOKEN_DELIM_CLOSE_PAREN = r'\)' t_TOKEN_DELIM_OPEN_BRACKET = r'\[' t_TOKEN_DELIM_CLOSE_BRACKET = r'\]' t_TOKEN_DELIM_OPEN_BRACE = r'\{' t_TOKEN_DELIM_CLOSE_BRACE = r'\}' t_TOKEN_DELIM_SEMICOLON = r'\;' def t_TOKEN_OP(t): r'(\<\<\=)|(\>\>\=)|([\+\-\*\/\%\&\|\^\=\!\>\<]\=)|(\|\|)|(\&\&)|(\+\+)|(\-\-)|[\+\-\*\/\%\^\=\&\|\>\<\!\~]' return t if __name__ == "__main__": lexer = lex.lex() f = open("./test.txt" , 'rb') data = f.read() f.close() lexer.input(data) while True: tok = lexer.token() if not tok: break # No more input print tok.value+"\t---->\t"+tok.type

,专题系统,采集软件

企业建站2800元起,携手武汉肥猫科技,做一个有见地的颜值派!更多优惠请戳:湖北网站建设 http://hubei.45qun.com

  • 上一篇:python 基本内容
  • 下一篇:最后一页
  •