先删空白再切,会切出几个
有人图省事,先把源码里所有空白删光,再照正确规则切。输出这样切那段源码会得到几个 Token。
let rate = 12; let msg = "hi there"; total = rate >= 3;
KEYWORDS = ("let", "if", "else", "while")
TWO = ("==", "!=", ">=", "<=")
ONE = "+-*/=<>;()"
SRC = 'let rate = 12;\nlet msg = "hi there";\ntotal = rate >= 3;'
import re
PAT = re.compile(r'[A-Za-z_]\w*|\d+|"[^"]*"|==|!=|>=|<=|[+\-*/=<>;()]')
squeezed = "".join(ch for ch in SRC if ch not in " \t\n")
print(len([m for m in PAT.finditer(squeezed)]))
全部评论