用Python实现词法分析器（LexicalAnalyzer）

from __future__ import print_function

成都创新互联公司主营石林网站建设的网络公司,主营网站建设方案,APP应用开发,石林h5成都小程序开发搭建,石林网站营销推广欢迎石林等地区企业咨询

import sys

# following two must remain in the same order

tk_EOI, tk_Mul, tk_Div, tk_Mod, tk_Add, tk_Sub, tk_Negate, tk_Not, tk_Lss, tk_Leq, tk_Gtr, \

tk_Geq, tk_Eq, tk_Neq, tk_Assign, tk_And, tk_Or, tk_If, tk_Else, tk_While, tk_Print, \

tk_Putc, tk_Lparen, tk_Rparen, tk_Lbrace, tk_Rbrace, tk_Semi, tk_Comma, tk_Ident, \

tk_Integer, tk_String = range(31)

all_syms = ["End_of_input", "Op_multiply", "Op_divide", "Op_mod", "Op_add", "Op_subtract",

"Op_negate", "Op_not", "Op_less", "Op_lessequal", "Op_greater", "Op_greaterequal",

"Op_equal", "Op_notequal", "Op_assign", "Op_and", "Op_or", "Keyword_if",

"Keyword_else", "Keyword_while", "Keyword_print", "Keyword_putc", "LeftParen",

"RightParen", "LeftBrace", "RightBrace", "Semicolon", "Comma", "Identifier",

"Integer", "String"]

# single character only symbols

symbols = { '{': tk_Lbrace, '}': tk_Rbrace, '(': tk_Lparen, ')': tk_Rparen, '+': tk_Add, '-': tk_Sub,

'*': tk_Mul, '%': tk_Mod, ';': tk_Semi, ',': tk_Comma }

key_words = {'if': tk_If, 'else': tk_Else, 'print': tk_Print, 'putc': tk_Putc, 'while': tk_While}

the_ch = " " # dummy first char - but it must be a space

the_col = 0

the_line = 1

input_file = None

#*** show error and exit

def error(line, col, msg):

print(line, col, msg)

exit(1)

#*** get the next character from the input

def next_ch():

global the_ch, the_col, the_line

the_ch = input_file.read(1)

the_col += 1

if the_ch == '\n':

the_line += 1

the_col = 0

return the_ch

#*** 'x' - character constants

def char_lit(err_line, err_col):

n = ord(next_ch()) # skip opening quote

if the_ch == '\'':

error(err_line, err_col, "empty character constant")

elif the_ch == '\\':

next_ch()

if the_ch == 'n':

n = 10

elif the_ch == '\\':

n = ord('\\')

else:

error(err_line, err_col, "unknown escape sequence \\%c" % (the_ch))

if next_ch() != '\'':

error(err_line, err_col, "multi-character constant")

next_ch()

return tk_Integer, err_line, err_col, n

#*** process divide or comments

def div_or_cmt(err_line, err_col):

if next_ch() != '*':

return tk_Div, err_line, err_col

# comment found

next_ch()

while True:

if the_ch == '*':

if next_ch() == '/':

next_ch()

return gettok()

elif len(the_ch) == 0:

error(err_line, err_col, "EOF in comment")

else:

next_ch()

#*** "string"

def string_lit(start, err_line, err_col):

text = ""

while next_ch() != start:

if len(the_ch) == 0:

error(err_line, err_col, "EOF while scanning string literal")

if the_ch == '\n':

error(err_line, err_col, "EOL while scanning string literal")

text += the_ch

next_ch()

return tk_String, err_line, err_col, text

#*** handle identifiers and integers

def ident_or_int(err_line, err_col):

is_number = True

text = ""

while the_ch.isalnum() or the_ch == '_':

text += the_ch

if not the_ch.isdigit():

is_number = False

next_ch()

if len(text) == 0:

error(err_line, err_col, "ident_or_int: unrecognized character: (%d) '%c'" % (ord(the_ch), the_ch))

if text[0].isdigit():

if not is_number:

error(err_line, err_col, "invalid number: %s" % (text))

n = int(text)

return tk_Integer, err_line, err_col, n

if text in key_words:

return key_words[text], err_line, err_col

return tk_Ident, err_line, err_col, text

#*** look ahead for '>=', etc.

def follow(expect, ifyes, ifno, err_line, err_col):

if next_ch() == expect:

next_ch()

return ifyes, err_line, err_col

if ifno == tk_EOI: 郑州人流医院哪家好 http://mobile.zhongyuan120.com/

error(err_line, err_col, "follow: unrecognized character: (%d) '%c'" % (ord(the_ch), the_ch))

return ifno, err_line, err_col

#*** return the next token type

def gettok():

while the_ch.isspace():

next_ch()

err_line = the_line

err_col = the_col

if len(the_ch) == 0: return tk_EOI, err_line, err_col

elif the_ch == '/': return div_or_cmt(err_line, err_col)

elif the_ch == '\'': return char_lit(err_line, err_col)

elif the_ch == '<': return follow('=', tk_Leq, tk_Lss, err_line, err_col)

elif the_ch == '>': return follow('=', tk_Geq, tk_Gtr, err_line, err_col)

elif the_ch == '=': return follow('=', tk_Eq, tk_Assign, err_line, err_col)

elif the_ch == '!': return follow('=', tk_Neq, tk_Not, err_line, err_col)

elif the_ch == '&': return follow('&', tk_And, tk_EOI, err_line, err_col)

elif the_ch == '|': return follow('|', tk_Or, tk_EOI, err_line, err_col)

elif the_ch == '"': return string_lit(the_ch, err_line, err_col)

elif the_ch in symbols:

sym = symbols[the_ch]

next_ch()

return sym, err_line, err_col

else: return ident_or_int(err_line, err_col)

#*** main driver

input_file = sys.stdin

if len(sys.argv) > 1:

try:

input_file = open(sys.argv[1], "r", 4096)

except IOError as e:

error(0, 0, "Can't open %s" % sys.argv[1])

while True:

t = gettok()

tok = t[0]

line = t[1]

col = t[2]

print("%5d %5d %-14s" % (line, col, all_syms[tok]), end='')

if tok == tk_Integer: print(" %5d" % (t[3]))

elif tok == tk_Ident: print(" %s" % (t[3]))

elif tok == tk_String: print(' "%s"' % (t[3]))

else: print("")

if tok == tk_EOI:

break

输出(测试用例三)

5 16 Keyword_print

5 40 Op_subtract

6 16 Keyword_putc

6 40 Op_less

7 16 Keyword_if

7 40 Op_greater

8 16 Keyword_else

8 40 Op_lessequal

9 16 Keyword_while

9 40 Op_greaterequal

10 16 LeftBrace

10 40 Op_equal

11 16 RightBrace

11 40 Op_notequal

12 16 LeftParen

12 40 Op_and

13 16 RightParen

13 40 Op_or

14 16 Op_subtract

14 40 Semicolon

15 16 Op_not

15 40 Comma

16 16 Op_multiply

16 40 Op_assign

17 16 Op_divide

17 40 Integer 42

18 16 Op_mod

18 40 String "String literal"

19 16 Op_add

19 40 Identifier variable_name

20 26 Integer 10

21 26 Integer 92

22 26 Integer 32

23 1 End_of_input