词法分析器JavaCC
程序输入:从test.txt文件中读取内容
void main()
{
while(a>45)
{
int x,x1;
x=x1+1;
}
}
程序输出:识别出的单词序列,格式为:(种别码,属性值)
一、支持类型
能识别关键字、标识符、常量(整常数、浮常数、字符串)、运算符、分隔符。能处理单行和多行注释。支持识别的各单词种别码及宏定义如下表
单词 |
种别码 |
宏 |
单词 |
种别码 |
宏 |
单词 |
种别码 |
宏 |
and |
1 |
AND |
Or |
22 |
OR |
/ |
43 |
RDIV |
array |
2 |
ARRAY |
Packed |
23 |
PACKED |
= |
44 |
EQ |
begin |
3 |
BEGIN |
procedure |
24 |
PROC |
< |
45 |
LT |
case |
4 |
CASE |
Program |
25 |
PROG |
> |
46 |
GT |
const |
5 |
CONST |
Record |
26 |
RECORD |
<= |
47 |
LE |
div |
6 |
DIV |
Repeat |
27 |
REPEAT |
>= |
48 |
GE |
do |
7 |
DO |
Set |
28 |
SET |
<> |
49 |
NE |
downto |
8 |
DOWNTO |
Then |
29 |
THEN |
( |
50 |
LA_BRAC |
else |
9 |
ELSE |
To |
30 |
TO |
) |
51 |
BR_BRAC |
end |
10 |
END |
Type |
31 |
TYPE |
, |
52 |
COMMA |
file |
11 |
FILE |
Until |
32 |
UNTIL |
、 |
53 |
P_MARK |
for |
12 |
FOR |
Car |
33 |
CAR |
. |
54 |
F_STOP |
function |
13 |
FUNC |
While |
34 |
WHILE |
.. |
55 |
RANGE |
goto |
14 |
GOTO |
With |
35 |
WITH |
: |
56 |
COLON |
if |
15 |
IF |
标识符 |
36 |
ID |
:= |
57 |
ASSIGN |
in |
16 |
IN |
整常数 |
37 |
INT |
; |
58 |
SEMIC |
label |
17 |
LABEL |
实常数 |
38 |
REAL |
^ |
59 |
CAP |
mod |
18 |
MOD |
字符串 |
39 |
STRING |
** |
60 |
EXP |
nil |
19 |
NIL |
+ |
40 |
PLUS |
[ |
61 |
LS_BRAC |
not |
20 |
NOT |
- |
41 |
MINUS |
] |
62 |
RS_BRAC |
of |
21 |
OF |
* |
|
MULTI |
‘ |
63 |
Q_MARK |
main方法
public class lexical
{
public static void main(String args [])
{
System.out.println("Reading from standard input...");
System.out.print("Enter an expression like \"1+(2+3)*var;\" :");
try {
lexical a;
String file = "test.txt";
FileInputStream fin=new FileInputStream(file);
SimpleCharStream scs = new SimpleCharStream(fin);
lexicalTokenManager tm=new lexicalTokenManager(scs);
Token t;
int i=0;
while(true)
{
t=tm.getNextToken();
if(t.kind==0)break;
System.out.println(t.kind+ "\t" +t.toString());
}
} catch (FileNotFoundException e) {
// TODO 自动生成的 catch 块
e.printStackTrace();
}
}
}
TOKEN
TOKEN : /* LITERALS */
{
< INTEGER_LITERAL :
< DECIMAL_LITERAL > ([ "l", "L" ])?
| < HEX_LITERAL > ([ "l", "L" ])?
| < OCTAL_LITERAL > ([ "l", "L" ])?
>
| < #DECIMAL_LITERAL : [ "1"-"9" ] ([ "0"-"9" ])* >
| < #HEX_LITERAL : "0" [ "x", "X" ] ([ "0"-"9", "a"-"f", "A"-"F" ])+ >
| < #OCTAL_LITERAL : "0" ([ "0"-"7" ])* >
}
TOKEN : /*keywords*/
{
< WHILE : "while" >
| < IF : "if" >
}
TOKEN : /*OPERATORS*/
{
< ADD : "+" >
| < MUL : "*" >
}
TOKEN : /*separators*/
{
< COMMA : "," >
| < SEMI : ";" >
}
TOKEN : /*kuohao*/
{
< LPAREN : "(" >
| < RPAREN : ")" >
| < LCURLY : "{" >
| < RCURLY : "}" >
| < GT : ">" >
| < EQ : "=" >
| < LT : "<" >
}
TOKEN : /* IDENTIFIERS */
{
< IDENTIFIER :
< LETTER >
(
< LETTER >
| < DIGIT >
)* >
| < #LETTER : [ "_", "a"-"z", "A"-"Z" ] >
| < #DIGIT : [ "0"-"9" ] >
}
lexical.java源代码
/* Generated By:JJTree&JavaCC: Do not edit this line. lexical.java */
package lexicalll;
import java.io.FileInputStream;
import java.io.FileNotFoundException;
public class lexical/*@bgen(jjtree)*/implements lexicalTreeConstants, lexicalConstants {/*@bgen(jjtree)*/
protected JJTlexicalState jjtree = new JJTlexicalState();public static void main(String args [])
{
System.out.println("Reading from standard input...");
System.out.print("Enter an expression like \u005c"1+(2+3)*var;\u005c" :\n");
// lexical parser = new lexical(System.in);
// try
// {
// SimpleNode n = parser.Start();
// n.dump("");
// System.out.println("Thank you.");
// }
// catch (Exception e)
// {
// System.out.println("Oops.");
// System.out.println(e.getMessage());
// }
try {
lexical a;
String file = "test.txt";
FileInputStream fin=new FileInputStream(file);
SimpleCharStream scs = new SimpleCharStream(fin);
lexicalTokenManager tm=new lexicalTokenManager(scs);
Token t;
int i=0;
while(true)
{
t=tm.getNextToken();
if(t.kind==0)break;
System.out.println(t.kind+ "\u005ct" +t.toString());
}
} catch (FileNotFoundException e) {
// TODO 自动生成的 catch 块
e.printStackTrace();
}
}
final public SimpleNode Start() throws ParseException {
/*@bgen(jjtree) Start */
SimpleNode jjtn000 = new SimpleNode(JJTSTART);
boolean jjtc000 = true;
jjtree.openNodeScope(jjtn000);
try {
Expression();
jj_consume_token(SEMI);
jjtree.closeNodeScope(jjtn000, true);
jjtc000 = false;
{if (true) return jjtn000;}
} catch (Throwable jjte000) {
if (jjtc000) {
jjtree.clearNodeScope(jjtn000);
jjtc000 = false;
} else {
jjtree.popNode();
}
if (jjte000 instanceof RuntimeException) {
{if (true) throw (RuntimeException)jjte000;}
}
if (jjte000 instanceof ParseException) {
{if (true) throw (ParseException)jjte000;}
}
{if (true) throw (Error)jjte000;}
} finally {
if (jjtc000) {
jjtree.closeNodeScope(jjtn000, true);
}
}
throw new Error("Missing return statement in function");
}
final public void Expression() throws ParseException {
/*@bgen(jjtree) Expression */
SimpleNode jjtn000 = new SimpleNode(JJTEXPRESSION);
boolean jjtc000 = true;
jjtree.openNodeScope(jjtn000);
try {
AdditiveExpression();
} catch (Throwable jjte000) {
if (jjtc000) {
jjtree.clearNodeScope(jjtn000);
jjtc000 = false;
} else {
jjtree.popNode();
}
if (jjte000 instanceof RuntimeException) {
{if (true) throw (RuntimeException)jjte000;}
}
if (jjte000 instanceof ParseException) {
{if (true) throw (ParseException)jjte000;}
}
{if (true) throw (Error)jjte000;}
} finally {
if (jjtc000) {
jjtree.closeNodeScope(jjtn000, true);
}
}
}
final public void AdditiveExpression() throws ParseException {
/*@bgen(jjtree) AdditiveExpression */
SimpleNode jjtn000 = new SimpleNode(JJTADDITIVEEXPRESSION);
boolean jjtc000 = true;
jjtree.openNodeScope(jjtn000);
try {
MultiplicativeExpression();
label_1:
while (true) {
switch ((jj_ntk==-1)?jj_ntk():jj_ntk) {
case ADD:
case 27:
;
break;
default:
jj_la1[0] = jj_gen;
break label_1;
}
switch ((jj_ntk==-1)?jj_ntk():jj_ntk) {
case ADD:
jj_consume_token(ADD);
break;
case 27:
jj_consume_token(27);
break;
default:
jj_la1[1] = jj_gen;
jj_consume_token(-1);
throw new ParseException();
}
MultiplicativeExpression();
}
} catch (Throwable jjte000) {
if (jjtc000) {
jjtree.clearNodeScope(jjtn000);
jjtc000 = false;
} else {
jjtree.popNode();
}
if (jjte000 instanceof RuntimeException) {
{if (true) throw (RuntimeException)jjte000;}
}
if (jjte000 instanceof ParseException) {
{if (true) throw (ParseException)jjte000;}
}
{if (true) throw (Error)jjte000;}
} finally {
if (jjtc000) {
jjtree.closeNodeScope(jjtn000, true);
}
}
}
final public void MultiplicativeExpression() throws ParseException {
/*@bgen(jjtree) MultiplicativeExpression */
SimpleNode jjtn000 = new SimpleNode(JJTMULTIPLICATIVEEXPRESSION);
boolean jjtc000 = true;
jjtree.openNodeScope(jjtn000);
try {
UnaryExpression();
label_2:
while (true) {
switch ((jj_ntk==-1)?jj_ntk():jj_ntk) {
case MUL:
case 28:
case 29:
;
break;
default:
jj_la1[2] = jj_gen;
break label_2;
}
switch ((jj_ntk==-1)?jj_ntk():jj_ntk) {
case MUL:
jj_consume_token(MUL);
break;
case 28:
jj_consume_token(28);
break;
case 29:
jj_consume_token(29);
break;
default:
jj_la1[3] = jj_gen;
jj_consume_token(-1);
throw new ParseException();
}
UnaryExpression();
}
} catch (Throwable jjte000) {
if (jjtc000) {
jjtree.clearNodeScope(jjtn000);
jjtc000 = false;
} else {
jjtree.popNode();
}
if (jjte000 instanceof RuntimeException) {
{if (true) throw (RuntimeException)jjte000;}
}
if (jjte000 instanceof ParseException) {
{if (true) throw (ParseException)jjte000;}
}
{if (true) throw (Error)jjte000;}
} finally {
if (jjtc000) {
jjtree.closeNodeScope(jjtn000, true);
}
}
}
final public void UnaryExpression() throws ParseException {
/*@bgen(jjtree) UnaryExpression */
SimpleNode jjtn000 = new SimpleNode(JJTUNARYEXPRESSION);
boolean jjtc000 = true;
jjtree.openNodeScope(jjtn000);
try {
switch ((jj_ntk==-1)?jj_ntk():jj_ntk) {
case LPAREN:
jj_consume_token(LPAREN);
Expression();
jj_consume_token(RPAREN);
break;
case IDENTIFIER:
Identifier();
break;
case INTEGER_LITERAL:
Integer();
break;
default:
jj_la1[4] = jj_gen;
jj_consume_token(-1);
throw new ParseException();
}
} catch (Throwable jjte000) {
if (jjtc000) {
jjtree.clearNodeScope(jjtn000);
jjtc000 = false;
} else {
jjtree.popNode();
}
if (jjte000 instanceof RuntimeException) {
{if (true) throw (RuntimeException)jjte000;}
}
if (jjte000 instanceof ParseException) {
{if (true) throw (ParseException)jjte000;}
}
{if (true) throw (Error)jjte000;}
} finally {
if (jjtc000) {
jjtree.closeNodeScope(jjtn000, true);
}
}
}
final public void Identifier() throws ParseException {
/*@bgen(jjtree) Identifier */
SimpleNode jjtn000 = new SimpleNode(JJTIDENTIFIER);
boolean jjtc000 = true;
jjtree.openNodeScope(jjtn000);
try {
jj_consume_token(IDENTIFIER);
} finally {
if (jjtc000) {
jjtree.closeNodeScope(jjtn000, true);
}
}
}
final public void Integer() throws ParseException {
/*@bgen(jjtree) Integer */
SimpleNode jjtn000 = new SimpleNode(JJTINTEGER);
boolean jjtc000 = true;
jjtree.openNodeScope(jjtn000);
try {
jj_consume_token(INTEGER_LITERAL);
} finally {
if (jjtc000) {
jjtree.closeNodeScope(jjtn000, true);
}
}
}
/** Generated Token Manager. */
public lexicalTokenManager token_source;
SimpleCharStream jj_input_stream;
/** Current token. */
public Token token;
/** Next token. */
public Token jj_nt;
private int jj_ntk;
private int jj_gen;
final private int[] jj_la1 = new int[5];
static private int[] jj_la1_0;
static {
jj_la1_init_0();
}
private static void jj_la1_init_0() {
jj_la1_0 = new int[] {0x8002000,0x8002000,0x30004000,0x30004000,0x1020080,};
}
/** Constructor with InputStream. */
public lexical(java.io.InputStream stream) {
this(stream, null);
}
/** Constructor with InputStream and supplied encoding */
public lexical(java.io.InputStream stream, String encoding) {
try { jj_input_stream = new SimpleCharStream(stream, encoding, 1, 1); } catch(java.io.UnsupportedEncodingException e) { throw new RuntimeException(e); }
token_source = new lexicalTokenManager(jj_input_stream);
token = new Token();
jj_ntk = -1;
jj_gen = 0;
for (int i = 0; i < 5; i++) jj_la1[i] = -1;
}
/** Reinitialise. */
public void ReInit(java.io.InputStream stream) {
ReInit(stream, null);
}
/** Reinitialise. */
public void ReInit(java.io.InputStream stream, String encoding) {
try { jj_input_stream.ReInit(stream, encoding, 1, 1); } catch(java.io.UnsupportedEncodingException e) { throw new RuntimeException(e); }
token_source.ReInit(jj_input_stream);
token = new Token();
jj_ntk = -1;
jjtree.reset();
jj_gen = 0;
for (int i = 0; i < 5; i++) jj_la1[i] = -1;
}
/** Constructor. */
public lexical(java.io.Reader stream) {
jj_input_stream = new SimpleCharStream(stream, 1, 1);
token_source = new lexicalTokenManager(jj_input_stream);
token = new Token();
jj_ntk = -1;
jj_gen = 0;
for (int i = 0; i < 5; i++) jj_la1[i] = -1;
}
/** Reinitialise. */
public void ReInit(java.io.Reader stream) {
jj_input_stream.ReInit(stream, 1, 1);
token_source.ReInit(jj_input_stream);
token = new Token();
jj_ntk = -1;
jjtree.reset();
jj_gen = 0;
for (int i = 0; i < 5; i++) jj_la1[i] = -1;
}
/** Constructor with generated Token Manager. */
public lexical(lexicalTokenManager tm) {
token_source = tm;
token = new Token();
jj_ntk = -1;
jj_gen = 0;
for (int i = 0; i < 5; i++) jj_la1[i] = -1;
}
/** Reinitialise. */
public void ReInit(lexicalTokenManager tm) {
token_source = tm;
token = new Token();
jj_ntk = -1;
jjtree.reset();
jj_gen = 0;
for (int i = 0; i < 5; i++) jj_la1[i] = -1;
}
private Token jj_consume_token(int kind) throws ParseException {
Token oldToken;
if ((oldToken = token).next != null) token = token.next;
else token = token.next = token_source.getNextToken();
jj_ntk = -1;
if (token.kind == kind) {
jj_gen++;
return token;
}
token = oldToken;
jj_kind = kind;
throw generateParseException();
}
/** Get the next Token. */
final public Token getNextToken() {
if (token.next != null) token = token.next;
else token = token.next = token_source.getNextToken();
jj_ntk = -1;
jj_gen++;
return token;
}
/** Get the specific Token. */
final public Token getToken(int index) {
Token t = token;
for (int i = 0; i < index; i++) {
if (t.next != null) t = t.next;
else t = t.next = token_source.getNextToken();
}
return t;
}
private int jj_ntk() {
if ((jj_nt=token.next) == null)
return (jj_ntk = (token.next=token_source.getNextToken()).kind);
else
return (jj_ntk = jj_nt.kind);
}
private java.util.List<int[]> jj_expentries = new java.util.ArrayList<int[]>();
private int[] jj_expentry;
private int jj_kind = -1;
/** Generate ParseException. */
public ParseException generateParseException() {
jj_expentries.clear();
boolean[] la1tokens = new boolean[30];
if (jj_kind >= 0) {
la1tokens[jj_kind] = true;
jj_kind = -1;
}
for (int i = 0; i < 5; i++) {
if (jj_la1[i] == jj_gen) {
for (int j = 0; j < 32; j++) {
if ((jj_la1_0[i] & (1<<j)) != 0) {
la1tokens[j] = true;
}
}
}
}
for (int i = 0; i < 30; i++) {
if (la1tokens[i]) {
jj_expentry = new int[1];
jj_expentry[0] = i;
jj_expentries.add(jj_expentry);
}
}
int[][] exptokseq = new int[jj_expentries.size()][];
for (int i = 0; i < jj_expentries.size(); i++) {
exptokseq[i] = jj_expentries.get(i);
}
return new ParseException(token, exptokseq, tokenImage);
}
/** Enable tracing. */
final public void enable_tracing() {
}
/** Disable tracing. */
final public void disable_tracing() {
}
}