|
Tokenizer |
|
package ij.macro; import ij.*; import java.io.*; /** This class converts an imageJ macro file file into a token stream. */ public class Tokenizer implements MacroConstants { private StreamTokenizer st; private int token; private String tokenString; private double tokenValue; private Program pgm; private boolean hasUserFunctions; /** Uses a StreamTokenizer to convert an ImageJ macro file into a token stream. */ public Program tokenize(String program) { //IJ.showStatus("tokenizing"); st = new StreamTokenizer(new StringReader(program)); st.eolIsSignificant(true); st.ordinaryChar('-'); st.ordinaryChar('/'); st.ordinaryChar('.'); st.wordChars('_', '_'); st.whitespaceChars(128, 255); st.slashStarComments(true); st.slashSlashComments(true); pgm = new Program(); do { getToken(); addToken(); } while (token!=EOF); if (hasUserFunctions) addUserFunctions(); if (IJ.debugMode) { pgm.dumpSymbolTable(); pgm.dumpProgram(); } return pgm; } final void getToken() { try { token = st.nextToken(); String ret = null; int nextToken; switch (st.ttype) { case StreamTokenizer.TT_EOF: ret = "EOF"; token = EOF; break; case StreamTokenizer.TT_EOL: ret = "EOL"; token = EOL; break; case StreamTokenizer.TT_WORD: ret = st.sval; token = WORD; break; case StreamTokenizer.TT_NUMBER: ret = ""+st.nval; tokenValue = st.nval; if (tokenValue==0.0) tokenValue = getHexConstant(); token = NUMBER; break; case '"': case '\'': ret = ""+st.sval; token = STRING_CONSTANT; break; case '+': nextToken = st.nextToken(); if (nextToken=='+') token = PLUS_PLUS; else if (nextToken=='=') token = PLUS_EQUAL; else st.pushBack(); break; case '-': nextToken = st.nextToken(); if (nextToken=='-') token = MINUS_MINUS; else if (nextToken=='=') token = MINUS_EQUAL; else st.pushBack(); break; case '*': nextToken = st.nextToken(); if (nextToken=='=') token = MUL_EQUAL; else st.pushBack(); break; case '/': nextToken = st.nextToken(); if (nextToken=='=') token = DIV_EQUAL; else st.pushBack(); break; case '=': nextToken = st.nextToken(); if (nextToken=='=') token = EQ; else st.pushBack(); break; case '!': nextToken = st.nextToken(); if (nextToken=='=') token = NEQ; else st.pushBack(); break; case '>': nextToken = st.nextToken(); if (nextToken=='=') token = GTE; else if (nextToken=='>') token = SHIFT_RIGHT; else { st.pushBack(); token = GT; } break; case '<': nextToken = st.nextToken(); if (nextToken=='=') token = LTE; else if (nextToken=='<') token = SHIFT_LEFT; else { st.pushBack(); token = LT; } break; case '&': nextToken = st.nextToken(); if (nextToken=='&') token = LOGICAL_AND; else st.pushBack(); break; case '|': nextToken = st.nextToken(); if (nextToken=='|') token = LOGICAL_OR; else st.pushBack(); break; default: //char s[] = new char[1]; //s[0] = (char)token; //ret = new String(s); } tokenString = ret; } catch (Exception e) { return; } // IJ.log(" "+ret); } final void addToken() { int tok = token; switch (token) { case WORD: Symbol symbol = pgm.lookupWord(tokenString); if (symbol!=null) { int type = symbol.getFunctionType(); if (type==0) { tok = symbol.type; switch (tok) { case FUNCTION: hasUserFunctions=true; break; case VAR: pgm.hasVars=true; break; case MACRO: pgm.macroCount++; break; } } else tok = type; tok += pgm.symTabLoc<<TOK_SHIFT; } else { pgm.addSymbol(new Symbol(token, tokenString)); tok += pgm.stLoc<<TOK_SHIFT; } break; case STRING_CONSTANT: pgm.addSymbol(new Symbol(token, tokenString)); tok += pgm.stLoc<<TOK_SHIFT; break; case NUMBER: pgm.addSymbol(new Symbol(tokenValue)); tok += pgm.stLoc<<TOK_SHIFT; break; default: break; } pgm.addToken(tok); } double getHexConstant() { try { token = st.nextToken(); } catch (Exception e) { return 0.0; } if (st.ttype != StreamTokenizer.TT_WORD) { st.pushBack(); return 0.0; } if (!st.sval.startsWith("x")) { st.pushBack(); return 0.0; } String s = st.sval.substring(1, st.sval.length()); double n = 0.0; try { n = Integer.parseInt(s, 16); } catch (NumberFormatException e) { st.pushBack(); n = 0.0; } return n; } /** Adds user-defined functions to the symbol table. */ void addUserFunctions() { int[] code = pgm.getCode(); Symbol[] symbolTable = pgm.getSymbolTable(); int nextToken, address, address2; for (int i=0; i<code.length; i++) { token = code[i]&TOK_MASK; if (token==FUNCTION) { nextToken = code[i+1]&TOK_MASK; if (nextToken==WORD || (nextToken>=PREDEFINED_FUNCTION&&nextToken<=ARRAY_FUNCTION)) { address = address2 = code[i+1]>>TOK_SHIFT; if (nextToken!=WORD) { // override built in function pgm.addSymbol(new Symbol(WORD, symbolTable[address].str)); address2 = pgm.stLoc; code[i+1] = WORD + (address2<<TOK_SHIFT); } Symbol sym = symbolTable[address2]; sym.type = USER_FUNCTION; sym.value = i+1; //address of function for (int j=0; j<code.length; j++) { token = code[j]&TOK_MASK; if ((token==WORD || (token>=PREDEFINED_FUNCTION&&token<=ARRAY_FUNCTION)) && (code[j]>>TOK_SHIFT)==address && (j==0||(code[j-1]&0xfff)!=FUNCTION)) { code[j] = USER_FUNCTION; code[j] += address2<<TOK_SHIFT; //IJ.log((code[j]&TOK_MASK)+" "+(code[j]>>TOK_SHIFT)+" "+USER_FUNCTION+" "+address); } else if (token==EOF) break; } // IJ.log(pgm.decodeToken(nextToken, address)); } } else if (token==EOF) break; } } } // class Tokenizer
|
Tokenizer |
|