Uploading antlr runtime for doclava use.
Change-Id: Iea8a6dd3a2c46dd3c13a81da6aa94e01bf98b854
diff --git a/Android.mk b/Android.mk
new file mode 100644
index 0000000..efe3a09
--- /dev/null
+++ b/Android.mk
@@ -0,0 +1,32 @@
+# Copyright (C) 2011 The Android Open Source Project
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+LOCAL_PATH := $(call my-dir)
+include $(CLEAR_VARS)
+
+LOCAL_MODULE := antlr
+#LOCAL_SDK_VERSION := 8
+LOCAL_SRC_FILES := $(call all-java-files-under, src)
+LOCAL_MODULE_TAGS := optional
+
+include $(BUILD_HOST_JAVA_LIBRARY)
+
+# Also build a host-side library
+# include $(CLEAR_VARS)
+#
+# LOCAL_SRC_FILES := $(call all-java-files-under, src)
+# LOCAL_MODULE := antlrlib
+#
+# include $(BUILD_HOST_JAVA_LIBRARY)
+
diff --git a/src/org/antlr/runtime/ANTLRFileStream.java b/src/org/antlr/runtime/ANTLRFileStream.java
new file mode 100755
index 0000000..27ef58b
--- /dev/null
+++ b/src/org/antlr/runtime/ANTLRFileStream.java
@@ -0,0 +1,78 @@
+/*
+ [The "BSD license"]
+ Copyright (c) 2005-2009 Terence Parr
+ All rights reserved.
+
+ Redistribution and use in source and binary forms, with or without
+ modification, are permitted provided that the following conditions
+ are met:
+ 1. Redistributions of source code must retain the above copyright
+ notice, this list of conditions and the following disclaimer.
+ 2. Redistributions in binary form must reproduce the above copyright
+ notice, this list of conditions and the following disclaimer in the
+ documentation and/or other materials provided with the distribution.
+ 3. The name of the author may not be used to endorse or promote products
+ derived from this software without specific prior written permission.
+
+ THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
+ IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
+ OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
+ IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
+ INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
+ NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
+ THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+package org.antlr.runtime;
+
+import java.io.*;
+
+/** This is a char buffer stream that is loaded from a file
+ * all at once when you construct the object. This looks very
+ * much like an ANTLReader or ANTLRInputStream, but it's a special case
+ * since we know the exact size of the object to load. We can avoid lots
+ * of data copying.
+ */
+public class ANTLRFileStream extends ANTLRStringStream {
+ protected String fileName;
+
+ public ANTLRFileStream(String fileName) throws IOException {
+ this(fileName, null);
+ }
+
+ public ANTLRFileStream(String fileName, String encoding) throws IOException {
+ this.fileName = fileName;
+ load(fileName, encoding);
+ }
+
+ public void load(String fileName, String encoding)
+ throws IOException
+ {
+ if ( fileName==null ) {
+ return;
+ }
+ File f = new File(fileName);
+ int size = (int)f.length();
+ InputStreamReader isr;
+ FileInputStream fis = new FileInputStream(fileName);
+ if ( encoding!=null ) {
+ isr = new InputStreamReader(fis, encoding);
+ }
+ else {
+ isr = new InputStreamReader(fis);
+ }
+ try {
+ data = new char[size];
+ super.n = isr.read(data);
+ }
+ finally {
+ isr.close();
+ }
+ }
+
+ public String getSourceName() {
+ return fileName;
+ }
+}
diff --git a/src/org/antlr/runtime/ANTLRInputStream.java b/src/org/antlr/runtime/ANTLRInputStream.java
new file mode 100755
index 0000000..e544aae
--- /dev/null
+++ b/src/org/antlr/runtime/ANTLRInputStream.java
@@ -0,0 +1,70 @@
+/*
+ [The "BSD license"]
+ Copyright (c) 2005-2009 Terence Parr
+ All rights reserved.
+
+ Redistribution and use in source and binary forms, with or without
+ modification, are permitted provided that the following conditions
+ are met:
+ 1. Redistributions of source code must retain the above copyright
+ notice, this list of conditions and the following disclaimer.
+ 2. Redistributions in binary form must reproduce the above copyright
+ notice, this list of conditions and the following disclaimer in the
+ documentation and/or other materials provided with the distribution.
+ 3. The name of the author may not be used to endorse or promote products
+ derived from this software without specific prior written permission.
+
+ THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
+ IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
+ OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
+ IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
+ INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
+ NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
+ THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+package org.antlr.runtime;
+
+import java.io.*;
+
+/** A kind of ReaderStream that pulls from an InputStream.
+ * Useful for reading from stdin and specifying file encodings etc...
+ */
+public class ANTLRInputStream extends ANTLRReaderStream {
+ public ANTLRInputStream() {
+ }
+
+ public ANTLRInputStream(InputStream input) throws IOException {
+ this(input, null);
+ }
+
+ public ANTLRInputStream(InputStream input, int size) throws IOException {
+ this(input, size, null);
+ }
+
+ public ANTLRInputStream(InputStream input, String encoding) throws IOException {
+ this(input, INITIAL_BUFFER_SIZE, encoding);
+ }
+
+ public ANTLRInputStream(InputStream input, int size, String encoding) throws IOException {
+ this(input, size, READ_BUFFER_SIZE, encoding);
+ }
+
+ public ANTLRInputStream(InputStream input,
+ int size,
+ int readBufferSize,
+ String encoding)
+ throws IOException
+ {
+ InputStreamReader isr;
+ if ( encoding!=null ) {
+ isr = new InputStreamReader(input, encoding);
+ }
+ else {
+ isr = new InputStreamReader(input);
+ }
+ load(isr, size, readBufferSize);
+ }
+}
diff --git a/src/org/antlr/runtime/ANTLRReaderStream.java b/src/org/antlr/runtime/ANTLRReaderStream.java
new file mode 100755
index 0000000..24d51ad
--- /dev/null
+++ b/src/org/antlr/runtime/ANTLRReaderStream.java
@@ -0,0 +1,95 @@
+/*
+ [The "BSD license"]
+ Copyright (c) 2005-2009 Terence Parr
+ All rights reserved.
+
+ Redistribution and use in source and binary forms, with or without
+ modification, are permitted provided that the following conditions
+ are met:
+ 1. Redistributions of source code must retain the above copyright
+ notice, this list of conditions and the following disclaimer.
+ 2. Redistributions in binary form must reproduce the above copyright
+ notice, this list of conditions and the following disclaimer in the
+ documentation and/or other materials provided with the distribution.
+ 3. The name of the author may not be used to endorse or promote products
+ derived from this software without specific prior written permission.
+
+ THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
+ IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
+ OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
+ IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
+ INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
+ NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
+ THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+package org.antlr.runtime;
+
+import java.io.*;
+
+/** Vacuum all input from a Reader and then treat it like a StringStream.
+ * Manage the buffer manually to avoid unnecessary data copying.
+ *
+ * If you need encoding, use ANTLRInputStream.
+ */
+public class ANTLRReaderStream extends ANTLRStringStream {
+ public static final int READ_BUFFER_SIZE = 1024;
+ public static final int INITIAL_BUFFER_SIZE = 1024;
+
+ public ANTLRReaderStream() {
+ }
+
+ public ANTLRReaderStream(Reader r) throws IOException {
+ this(r, INITIAL_BUFFER_SIZE, READ_BUFFER_SIZE);
+ }
+
+ public ANTLRReaderStream(Reader r, int size) throws IOException {
+ this(r, size, READ_BUFFER_SIZE);
+ }
+
+ public ANTLRReaderStream(Reader r, int size, int readChunkSize) throws IOException {
+ load(r, size, readChunkSize);
+ }
+
+ public void load(Reader r, int size, int readChunkSize)
+ throws IOException
+ {
+ if ( r==null ) {
+ return;
+ }
+ if ( size<=0 ) {
+ size = INITIAL_BUFFER_SIZE;
+ }
+ if ( readChunkSize<=0 ) {
+ readChunkSize = READ_BUFFER_SIZE;
+ }
+ // System.out.println("load "+size+" in chunks of "+readChunkSize);
+ try {
+ // alloc initial buffer size.
+ data = new char[size];
+ // read all the data in chunks of readChunkSize
+ int numRead=0;
+ int p = 0;
+ do {
+ if ( p+readChunkSize > data.length ) { // overflow?
+ // System.out.println("### overflow p="+p+", data.length="+data.length);
+ char[] newdata = new char[data.length*2]; // resize
+ System.arraycopy(data, 0, newdata, 0, data.length);
+ data = newdata;
+ }
+ numRead = r.read(data, p, readChunkSize);
+ // System.out.println("read "+numRead+" chars; p was "+p+" is now "+(p+numRead));
+ p += numRead;
+ } while (numRead!=-1); // while not EOF
+ // set the actual size of the data available;
+ // EOF subtracted one above in p+=numRead; add one back
+ super.n = p+1;
+ //System.out.println("n="+n);
+ }
+ finally {
+ r.close();
+ }
+ }
+}
diff --git a/src/org/antlr/runtime/ANTLRStringStream.java b/src/org/antlr/runtime/ANTLRStringStream.java
new file mode 100755
index 0000000..17af23c
--- /dev/null
+++ b/src/org/antlr/runtime/ANTLRStringStream.java
@@ -0,0 +1,230 @@
+/*
+ [The "BSD license"]
+ Copyright (c) 2005-2009 Terence Parr
+ All rights reserved.
+
+ Redistribution and use in source and binary forms, with or without
+ modification, are permitted provided that the following conditions
+ are met:
+ 1. Redistributions of source code must retain the above copyright
+ notice, this list of conditions and the following disclaimer.
+ 2. Redistributions in binary form must reproduce the above copyright
+ notice, this list of conditions and the following disclaimer in the
+ documentation and/or other materials provided with the distribution.
+ 3. The name of the author may not be used to endorse or promote products
+ derived from this software without specific prior written permission.
+
+ THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
+ IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
+ OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
+ IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
+ INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
+ NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
+ THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+package org.antlr.runtime;
+
+import java.util.ArrayList;
+import java.util.List;
+
+/** A pretty quick CharStream that pulls all data from an array
+ * directly. Every method call counts in the lexer. Java's
+ * strings aren't very good so I'm avoiding.
+ */
+public class ANTLRStringStream implements CharStream {
+ /** The data being scanned */
+ protected char[] data;
+
+ /** How many characters are actually in the buffer */
+ protected int n;
+
+ /** 0..n-1 index into string of next char */
+ protected int p=0;
+
+ /** line number 1..n within the input */
+ protected int line = 1;
+
+ /** The index of the character relative to the beginning of the line 0..n-1 */
+ protected int charPositionInLine = 0;
+
+ /** tracks how deep mark() calls are nested */
+ protected int markDepth = 0;
+
+ /** A list of CharStreamState objects that tracks the stream state
+ * values line, charPositionInLine, and p that can change as you
+ * move through the input stream. Indexed from 1..markDepth.
+ * A null is kept @ index 0. Create upon first call to mark().
+ */
+ protected List markers;
+
+ /** Track the last mark() call result value for use in rewind(). */
+ protected int lastMarker;
+
+ /** What is name or source of this char stream? */
+ public String name;
+
+ public ANTLRStringStream() {
+ }
+
+ /** Copy data in string to a local char array */
+ public ANTLRStringStream(String input) {
+ this();
+ this.data = input.toCharArray();
+ this.n = input.length();
+ }
+
+ /** This is the preferred constructor as no data is copied */
+ public ANTLRStringStream(char[] data, int numberOfActualCharsInArray) {
+ this();
+ this.data = data;
+ this.n = numberOfActualCharsInArray;
+ }
+
+ /** Reset the stream so that it's in the same state it was
+ * when the object was created *except* the data array is not
+ * touched.
+ */
+ public void reset() {
+ p = 0;
+ line = 1;
+ charPositionInLine = 0;
+ markDepth = 0;
+ }
+
+ public void consume() {
+ //System.out.println("prev p="+p+", c="+(char)data[p]);
+ if ( p < n ) {
+ charPositionInLine++;
+ if ( data[p]=='\n' ) {
+ /*
+ System.out.println("newline char found on line: "+line+
+ "@ pos="+charPositionInLine);
+ */
+ line++;
+ charPositionInLine=0;
+ }
+ p++;
+ //System.out.println("p moves to "+p+" (c='"+(char)data[p]+"')");
+ }
+ }
+
+ public int LA(int i) {
+ if ( i==0 ) {
+ return 0; // undefined
+ }
+ if ( i<0 ) {
+ i++; // e.g., translate LA(-1) to use offset i=0; then data[p+0-1]
+ if ( (p+i-1) < 0 ) {
+ return CharStream.EOF; // invalid; no char before first char
+ }
+ }
+
+ if ( (p+i-1) >= n ) {
+ //System.out.println("char LA("+i+")=EOF; p="+p);
+ return CharStream.EOF;
+ }
+ //System.out.println("char LA("+i+")="+(char)data[p+i-1]+"; p="+p);
+ //System.out.println("LA("+i+"); p="+p+" n="+n+" data.length="+data.length);
+ return data[p+i-1];
+ }
+
+ public int LT(int i) {
+ return LA(i);
+ }
+
+ /** Return the current input symbol index 0..n where n indicates the
+ * last symbol has been read. The index is the index of char to
+ * be returned from LA(1).
+ */
+ public int index() {
+ return p;
+ }
+
+ public int size() {
+ return n;
+ }
+
+ public int mark() {
+ if ( markers==null ) {
+ markers = new ArrayList();
+ markers.add(null); // depth 0 means no backtracking, leave blank
+ }
+ markDepth++;
+ CharStreamState state = null;
+ if ( markDepth>=markers.size() ) {
+ state = new CharStreamState();
+ markers.add(state);
+ }
+ else {
+ state = (CharStreamState)markers.get(markDepth);
+ }
+ state.p = p;
+ state.line = line;
+ state.charPositionInLine = charPositionInLine;
+ lastMarker = markDepth;
+ return markDepth;
+ }
+
+ public void rewind(int m) {
+ CharStreamState state = (CharStreamState)markers.get(m);
+ // restore stream state
+ seek(state.p);
+ line = state.line;
+ charPositionInLine = state.charPositionInLine;
+ release(m);
+ }
+
+ public void rewind() {
+ rewind(lastMarker);
+ }
+
+ public void release(int marker) {
+ // unwind any other markers made after m and release m
+ markDepth = marker;
+ // release this marker
+ markDepth--;
+ }
+
+ /** consume() ahead until p==index; can't just set p=index as we must
+ * update line and charPositionInLine.
+ */
+ public void seek(int index) {
+ if ( index<=p ) {
+ p = index; // just jump; don't update stream state (line, ...)
+ return;
+ }
+ // seek forward, consume until p hits index
+ while ( p<index ) {
+ consume();
+ }
+ }
+
+ public String substring(int start, int stop) {
+ return new String(data,start,stop-start+1);
+ }
+
+ public int getLine() {
+ return line;
+ }
+
+ public int getCharPositionInLine() {
+ return charPositionInLine;
+ }
+
+ public void setLine(int line) {
+ this.line = line;
+ }
+
+ public void setCharPositionInLine(int pos) {
+ this.charPositionInLine = pos;
+ }
+
+ public String getSourceName() {
+ return name;
+ }
+
+ public String toString() { return new String(data); }
+}
diff --git a/src/org/antlr/runtime/BaseRecognizer.java b/src/org/antlr/runtime/BaseRecognizer.java
new file mode 100755
index 0000000..667664d
--- /dev/null
+++ b/src/org/antlr/runtime/BaseRecognizer.java
@@ -0,0 +1,886 @@
+/*
+ [The "BSD license"]
+ Copyright (c) 2005-2009 Terence Parr
+ All rights reserved.
+
+ Redistribution and use in source and binary forms, with or without
+ modification, are permitted provided that the following conditions
+ are met:
+ 1. Redistributions of source code must retain the above copyright
+ notice, this list of conditions and the following disclaimer.
+ 2. Redistributions in binary form must reproduce the above copyright
+ notice, this list of conditions and the following disclaimer in the
+ documentation and/or other materials provided with the distribution.
+ 3. The name of the author may not be used to endorse or promote products
+ derived from this software without specific prior written permission.
+
+ THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
+ IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
+ OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
+ IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
+ INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
+ NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
+ THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+package org.antlr.runtime;
+
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+/** A generic recognizer that can handle recognizers generated from
+ * lexer, parser, and tree grammars. This is all the parsing
+ * support code essentially; most of it is error recovery stuff and
+ * backtracking.
+ */
+public abstract class BaseRecognizer {
+ public static final int MEMO_RULE_FAILED = -2;
+ public static final int MEMO_RULE_UNKNOWN = -1;
+ public static final int INITIAL_FOLLOW_STACK_SIZE = 100;
+
+ // copies from Token object for convenience in actions
+ public static final int DEFAULT_TOKEN_CHANNEL = Token.DEFAULT_CHANNEL;
+ public static final int HIDDEN = Token.HIDDEN_CHANNEL;
+
+ public static final String NEXT_TOKEN_RULE_NAME = "nextToken";
+
+ /** State of a lexer, parser, or tree parser are collected into a state
+ * object so the state can be shared. This sharing is needed to
+ * have one grammar import others and share same error variables
+ * and other state variables. It's a kind of explicit multiple
+ * inheritance via delegation of methods and shared state.
+ */
+ protected RecognizerSharedState state;
+
+ public BaseRecognizer() {
+ state = new RecognizerSharedState();
+ }
+
+ public BaseRecognizer(RecognizerSharedState state) {
+ if ( state==null ) {
+ state = new RecognizerSharedState();
+ }
+ this.state = state;
+ }
+
+ /** reset the parser's state; subclasses must rewinds the input stream */
+ public void reset() {
+ // wack everything related to error recovery
+ if ( state==null ) {
+ return; // no shared state work to do
+ }
+ state._fsp = -1;
+ state.errorRecovery = false;
+ state.lastErrorIndex = -1;
+ state.failed = false;
+ state.syntaxErrors = 0;
+ // wack everything related to backtracking and memoization
+ state.backtracking = 0;
+ for (int i = 0; state.ruleMemo!=null && i < state.ruleMemo.length; i++) { // wipe cache
+ state.ruleMemo[i] = null;
+ }
+ }
+
+
+ /** Match current input symbol against ttype. Attempt
+ * single token insertion or deletion error recovery. If
+ * that fails, throw MismatchedTokenException.
+ *
+ * To turn off single token insertion or deletion error
+ * recovery, override recoverFromMismatchedToken() and have it
+ * throw an exception. See TreeParser.recoverFromMismatchedToken().
+ * This way any error in a rule will cause an exception and
+ * immediate exit from rule. Rule would recover by resynchronizing
+ * to the set of symbols that can follow rule ref.
+ */
+ public Object match(IntStream input, int ttype, BitSet follow)
+ throws RecognitionException
+ {
+ //System.out.println("match "+((TokenStream)input).LT(1));
+ Object matchedSymbol = getCurrentInputSymbol(input);
+ if ( input.LA(1)==ttype ) {
+ input.consume();
+ state.errorRecovery = false;
+ state.failed = false;
+ return matchedSymbol;
+ }
+ if ( state.backtracking>0 ) {
+ state.failed = true;
+ return matchedSymbol;
+ }
+ matchedSymbol = recoverFromMismatchedToken(input, ttype, follow);
+ return matchedSymbol;
+ }
+
+ /** Match the wildcard: in a symbol */
+ public void matchAny(IntStream input) {
+ state.errorRecovery = false;
+ state.failed = false;
+ input.consume();
+ }
+
+ public boolean mismatchIsUnwantedToken(IntStream input, int ttype) {
+ return input.LA(2)==ttype;
+ }
+
+ public boolean mismatchIsMissingToken(IntStream input, BitSet follow) {
+ if ( follow==null ) {
+ // we have no information about the follow; we can only consume
+ // a single token and hope for the best
+ return false;
+ }
+ // compute what can follow this grammar element reference
+ if ( follow.member(Token.EOR_TOKEN_TYPE) ) {
+ BitSet viableTokensFollowingThisRule = computeContextSensitiveRuleFOLLOW();
+ follow = follow.or(viableTokensFollowingThisRule);
+ if ( state._fsp>=0 ) { // remove EOR if we're not the start symbol
+ follow.remove(Token.EOR_TOKEN_TYPE);
+ }
+ }
+ // if current token is consistent with what could come after set
+ // then we know we're missing a token; error recovery is free to
+ // "insert" the missing token
+
+ //System.out.println("viable tokens="+follow.toString(getTokenNames()));
+ //System.out.println("LT(1)="+((TokenStream)input).LT(1));
+
+ // BitSet cannot handle negative numbers like -1 (EOF) so I leave EOR
+ // in follow set to indicate that the fall of the start symbol is
+ // in the set (EOF can follow).
+ if ( follow.member(input.LA(1)) || follow.member(Token.EOR_TOKEN_TYPE) ) {
+ //System.out.println("LT(1)=="+((TokenStream)input).LT(1)+" is consistent with what follows; inserting...");
+ return true;
+ }
+ return false;
+ }
+
+ /** Report a recognition problem.
+ *
+ * This method sets errorRecovery to indicate the parser is recovering
+ * not parsing. Once in recovery mode, no errors are generated.
+ * To get out of recovery mode, the parser must successfully match
+ * a token (after a resync). So it will go:
+ *
+ * 1. error occurs
+ * 2. enter recovery mode, report error
+ * 3. consume until token found in resynch set
+ * 4. try to resume parsing
+ * 5. next match() will reset errorRecovery mode
+ *
+ * If you override, make sure to update syntaxErrors if you care about that.
+ */
+ public void reportError(RecognitionException e) {
+ // if we've already reported an error and have not matched a token
+ // yet successfully, don't report any errors.
+ if ( state.errorRecovery ) {
+ //System.err.print("[SPURIOUS] ");
+ return;
+ }
+ state.syntaxErrors++; // don't count spurious
+ state.errorRecovery = true;
+
+ displayRecognitionError(this.getTokenNames(), e);
+ }
+
+ public void displayRecognitionError(String[] tokenNames,
+ RecognitionException e)
+ {
+ String hdr = getErrorHeader(e);
+ String msg = getErrorMessage(e, tokenNames);
+ emitErrorMessage(hdr+" "+msg);
+ }
+
+ /** What error message should be generated for the various
+ * exception types?
+ *
+ * Not very object-oriented code, but I like having all error message
+ * generation within one method rather than spread among all of the
+ * exception classes. This also makes it much easier for the exception
+ * handling because the exception classes do not have to have pointers back
+ * to this object to access utility routines and so on. Also, changing
+ * the message for an exception type would be difficult because you
+ * would have to subclassing exception, but then somehow get ANTLR
+ * to make those kinds of exception objects instead of the default.
+ * This looks weird, but trust me--it makes the most sense in terms
+ * of flexibility.
+ *
+ * For grammar debugging, you will want to override this to add
+ * more information such as the stack frame with
+ * getRuleInvocationStack(e, this.getClass().getName()) and,
+ * for no viable alts, the decision description and state etc...
+ *
+ * Override this to change the message generated for one or more
+ * exception types.
+ */
+ public String getErrorMessage(RecognitionException e, String[] tokenNames) {
+ String msg = e.getMessage();
+ if ( e instanceof UnwantedTokenException ) {
+ UnwantedTokenException ute = (UnwantedTokenException)e;
+ String tokenName="<unknown>";
+ if ( ute.expecting== Token.EOF ) {
+ tokenName = "EOF";
+ }
+ else {
+ tokenName = tokenNames[ute.expecting];
+ }
+ msg = "extraneous input "+getTokenErrorDisplay(ute.getUnexpectedToken())+
+ " expecting "+tokenName;
+ }
+ else if ( e instanceof MissingTokenException ) {
+ MissingTokenException mte = (MissingTokenException)e;
+ String tokenName="<unknown>";
+ if ( mte.expecting== Token.EOF ) {
+ tokenName = "EOF";
+ }
+ else {
+ tokenName = tokenNames[mte.expecting];
+ }
+ msg = "missing "+tokenName+" at "+getTokenErrorDisplay(e.token);
+ }
+ else if ( e instanceof MismatchedTokenException ) {
+ MismatchedTokenException mte = (MismatchedTokenException)e;
+ String tokenName="<unknown>";
+ if ( mte.expecting== Token.EOF ) {
+ tokenName = "EOF";
+ }
+ else {
+ tokenName = tokenNames[mte.expecting];
+ }
+ msg = "mismatched input "+getTokenErrorDisplay(e.token)+
+ " expecting "+tokenName;
+ }
+ else if ( e instanceof MismatchedTreeNodeException ) {
+ MismatchedTreeNodeException mtne = (MismatchedTreeNodeException)e;
+ String tokenName="<unknown>";
+ if ( mtne.expecting==Token.EOF ) {
+ tokenName = "EOF";
+ }
+ else {
+ tokenName = tokenNames[mtne.expecting];
+ }
+ msg = "mismatched tree node: "+mtne.node+
+ " expecting "+tokenName;
+ }
+ else if ( e instanceof NoViableAltException ) {
+ //NoViableAltException nvae = (NoViableAltException)e;
+ // for development, can add "decision=<<"+nvae.grammarDecisionDescription+">>"
+ // and "(decision="+nvae.decisionNumber+") and
+ // "state "+nvae.stateNumber
+ msg = "no viable alternative at input "+getTokenErrorDisplay(e.token);
+ }
+ else if ( e instanceof EarlyExitException ) {
+ //EarlyExitException eee = (EarlyExitException)e;
+ // for development, can add "(decision="+eee.decisionNumber+")"
+ msg = "required (...)+ loop did not match anything at input "+
+ getTokenErrorDisplay(e.token);
+ }
+ else if ( e instanceof MismatchedSetException ) {
+ MismatchedSetException mse = (MismatchedSetException)e;
+ msg = "mismatched input "+getTokenErrorDisplay(e.token)+
+ " expecting set "+mse.expecting;
+ }
+ else if ( e instanceof MismatchedNotSetException ) {
+ MismatchedNotSetException mse = (MismatchedNotSetException)e;
+ msg = "mismatched input "+getTokenErrorDisplay(e.token)+
+ " expecting set "+mse.expecting;
+ }
+ else if ( e instanceof FailedPredicateException ) {
+ FailedPredicateException fpe = (FailedPredicateException)e;
+ msg = "rule "+fpe.ruleName+" failed predicate: {"+
+ fpe.predicateText+"}?";
+ }
+ return msg;
+ }
+
+ /** Get number of recognition errors (lexer, parser, tree parser). Each
+ * recognizer tracks its own number. So parser and lexer each have
+ * separate count. Does not count the spurious errors found between
+ * an error and next valid token match
+ *
+ * See also reportError()
+ */
+ public int getNumberOfSyntaxErrors() {
+ return state.syntaxErrors;
+ }
+
+ /** What is the error header, normally line/character position information? */
+ public String getErrorHeader(RecognitionException e) {
+ if ( getSourceName()!=null )
+ return getSourceName()+" line "+e.line+":"+e.charPositionInLine;
+
+ return "line "+e.line+":"+e.charPositionInLine;
+ }
+
+ /** How should a token be displayed in an error message? The default
+ * is to display just the text, but during development you might
+ * want to have a lot of information spit out. Override in that case
+ * to use t.toString() (which, for CommonToken, dumps everything about
+ * the token). This is better than forcing you to override a method in
+ * your token objects because you don't have to go modify your lexer
+ * so that it creates a new Java type.
+ */
+ public String getTokenErrorDisplay(Token t) {
+ String s = t.getText();
+ if ( s==null ) {
+ if ( t.getType()==Token.EOF ) {
+ s = "<EOF>";
+ }
+ else {
+ s = "<"+t.getType()+">";
+ }
+ }
+ s = s.replaceAll("\n","\\\\n");
+ s = s.replaceAll("\r","\\\\r");
+ s = s.replaceAll("\t","\\\\t");
+ return "'"+s+"'";
+ }
+
+ /** Override this method to change where error messages go */
+ public void emitErrorMessage(String msg) {
+ System.err.println(msg);
+ }
+
+ /** Recover from an error found on the input stream. This is
+ * for NoViableAlt and mismatched symbol exceptions. If you enable
+ * single token insertion and deletion, this will usually not
+ * handle mismatched symbol exceptions but there could be a mismatched
+ * token that the match() routine could not recover from.
+ */
+ public void recover(IntStream input, RecognitionException re) {
+ if ( state.lastErrorIndex==input.index() ) {
+ // uh oh, another error at same token index; must be a case
+ // where LT(1) is in the recovery token set so nothing is
+ // consumed; consume a single token so at least to prevent
+ // an infinite loop; this is a failsafe.
+ input.consume();
+ }
+ state.lastErrorIndex = input.index();
+ BitSet followSet = computeErrorRecoverySet();
+ beginResync();
+ consumeUntil(input, followSet);
+ endResync();
+ }
+
+ /** A hook to listen in on the token consumption during error recovery.
+ * The DebugParser subclasses this to fire events to the listenter.
+ */
+ public void beginResync() {
+ }
+
+ public void endResync() {
+ }
+
+ /* Compute the error recovery set for the current rule. During
+ * rule invocation, the parser pushes the set of tokens that can
+ * follow that rule reference on the stack; this amounts to
+ * computing FIRST of what follows the rule reference in the
+ * enclosing rule. This local follow set only includes tokens
+ * from within the rule; i.e., the FIRST computation done by
+ * ANTLR stops at the end of a rule.
+ *
+ * EXAMPLE
+ *
+ * When you find a "no viable alt exception", the input is not
+ * consistent with any of the alternatives for rule r. The best
+ * thing to do is to consume tokens until you see something that
+ * can legally follow a call to r *or* any rule that called r.
+ * You don't want the exact set of viable next tokens because the
+ * input might just be missing a token--you might consume the
+ * rest of the input looking for one of the missing tokens.
+ *
+ * Consider grammar:
+ *
+ * a : '[' b ']'
+ * | '(' b ')'
+ * ;
+ * b : c '^' INT ;
+ * c : ID
+ * | INT
+ * ;
+ *
+ * At each rule invocation, the set of tokens that could follow
+ * that rule is pushed on a stack. Here are the various "local"
+ * follow sets:
+ *
+ * FOLLOW(b1_in_a) = FIRST(']') = ']'
+ * FOLLOW(b2_in_a) = FIRST(')') = ')'
+ * FOLLOW(c_in_b) = FIRST('^') = '^'
+ *
+ * Upon erroneous input "[]", the call chain is
+ *
+ * a -> b -> c
+ *
+ * and, hence, the follow context stack is:
+ *
+ * depth local follow set after call to rule
+ * 0 <EOF> a (from main())
+ * 1 ']' b
+ * 3 '^' c
+ *
+ * Notice that ')' is not included, because b would have to have
+ * been called from a different context in rule a for ')' to be
+ * included.
+ *
+ * For error recovery, we cannot consider FOLLOW(c)
+ * (context-sensitive or otherwise). We need the combined set of
+ * all context-sensitive FOLLOW sets--the set of all tokens that
+ * could follow any reference in the call chain. We need to
+ * resync to one of those tokens. Note that FOLLOW(c)='^' and if
+ * we resync'd to that token, we'd consume until EOF. We need to
+ * sync to context-sensitive FOLLOWs for a, b, and c: {']','^'}.
+ * In this case, for input "[]", LA(1) is in this set so we would
+ * not consume anything and after printing an error rule c would
+ * return normally. It would not find the required '^' though.
+ * At this point, it gets a mismatched token error and throws an
+ * exception (since LA(1) is not in the viable following token
+ * set). The rule exception handler tries to recover, but finds
+ * the same recovery set and doesn't consume anything. Rule b
+ * exits normally returning to rule a. Now it finds the ']' (and
+ * with the successful match exits errorRecovery mode).
+ *
+ * So, you cna see that the parser walks up call chain looking
+ * for the token that was a member of the recovery set.
+ *
+ * Errors are not generated in errorRecovery mode.
+ *
+ * ANTLR's error recovery mechanism is based upon original ideas:
+ *
+ * "Algorithms + Data Structures = Programs" by Niklaus Wirth
+ *
+ * and
+ *
+ * "A note on error recovery in recursive descent parsers":
+ * http://portal.acm.org/citation.cfm?id=947902.947905
+ *
+ * Later, Josef Grosch had some good ideas:
+ *
+ * "Efficient and Comfortable Error Recovery in Recursive Descent
+ * Parsers":
+ * ftp://www.cocolab.com/products/cocktail/doca4.ps/ell.ps.zip
+ *
+ * Like Grosch I implemented local FOLLOW sets that are combined
+ * at run-time upon error to avoid overhead during parsing.
+ */
+ protected BitSet computeErrorRecoverySet() {
+ return combineFollows(false);
+ }
+
+ /** Compute the context-sensitive FOLLOW set for current rule.
+ * This is set of token types that can follow a specific rule
+ * reference given a specific call chain. You get the set of
+ * viable tokens that can possibly come next (lookahead depth 1)
+ * given the current call chain. Contrast this with the
+ * definition of plain FOLLOW for rule r:
+ *
+ * FOLLOW(r)={x | S=>*alpha r beta in G and x in FIRST(beta)}
+ *
+ * where x in T* and alpha, beta in V*; T is set of terminals and
+ * V is the set of terminals and nonterminals. In other words,
+ * FOLLOW(r) is the set of all tokens that can possibly follow
+ * references to r in *any* sentential form (context). At
+ * runtime, however, we know precisely which context applies as
+ * we have the call chain. We may compute the exact (rather
+ * than covering superset) set of following tokens.
+ *
+ * For example, consider grammar:
+ *
+ * stat : ID '=' expr ';' // FOLLOW(stat)=={EOF}
+ * | "return" expr '.'
+ * ;
+ * expr : atom ('+' atom)* ; // FOLLOW(expr)=={';','.',')'}
+ * atom : INT // FOLLOW(atom)=={'+',')',';','.'}
+ * | '(' expr ')'
+ * ;
+ *
+ * The FOLLOW sets are all inclusive whereas context-sensitive
+ * FOLLOW sets are precisely what could follow a rule reference.
+ * For input input "i=(3);", here is the derivation:
+ *
+ * stat => ID '=' expr ';'
+ * => ID '=' atom ('+' atom)* ';'
+ * => ID '=' '(' expr ')' ('+' atom)* ';'
+ * => ID '=' '(' atom ')' ('+' atom)* ';'
+ * => ID '=' '(' INT ')' ('+' atom)* ';'
+ * => ID '=' '(' INT ')' ';'
+ *
+ * At the "3" token, you'd have a call chain of
+ *
+ * stat -> expr -> atom -> expr -> atom
+ *
+ * What can follow that specific nested ref to atom? Exactly ')'
+ * as you can see by looking at the derivation of this specific
+ * input. Contrast this with the FOLLOW(atom)={'+',')',';','.'}.
+ *
+ * You want the exact viable token set when recovering from a
+ * token mismatch. Upon token mismatch, if LA(1) is member of
+ * the viable next token set, then you know there is most likely
+ * a missing token in the input stream. "Insert" one by just not
+ * throwing an exception.
+ */
+ protected BitSet computeContextSensitiveRuleFOLLOW() {
+ return combineFollows(true);
+ }
+
+ // what is exact? it seems to only add sets from above on stack
+ // if EOR is in set i. When it sees a set w/o EOR, it stops adding.
+ // Why would we ever want them all? Maybe no viable alt instead of
+ // mismatched token?
+ protected BitSet combineFollows(boolean exact) {
+ int top = state._fsp;
+ BitSet followSet = new BitSet();
+ for (int i=top; i>=0; i--) {
+ BitSet localFollowSet = (BitSet)state.following[i];
+ /*
+ System.out.println("local follow depth "+i+"="+
+ localFollowSet.toString(getTokenNames())+")");
+ */
+ followSet.orInPlace(localFollowSet);
+ if ( exact ) {
+ // can we see end of rule?
+ if ( localFollowSet.member(Token.EOR_TOKEN_TYPE) ) {
+ // Only leave EOR in set if at top (start rule); this lets
+ // us know if have to include follow(start rule); i.e., EOF
+ if ( i>0 ) {
+ followSet.remove(Token.EOR_TOKEN_TYPE);
+ }
+ }
+ else { // can't see end of rule, quit
+ break;
+ }
+ }
+ }
+ return followSet;
+ }
+
+ /** Attempt to recover from a single missing or extra token.
+ *
+ * EXTRA TOKEN
+ *
+ * LA(1) is not what we are looking for. If LA(2) has the right token,
+ * however, then assume LA(1) is some extra spurious token. Delete it
+ * and LA(2) as if we were doing a normal match(), which advances the
+ * input.
+ *
+ * MISSING TOKEN
+ *
+ * If current token is consistent with what could come after
+ * ttype then it is ok to "insert" the missing token, else throw
+ * exception For example, Input "i=(3;" is clearly missing the
+ * ')'. When the parser returns from the nested call to expr, it
+ * will have call chain:
+ *
+ * stat -> expr -> atom
+ *
+ * and it will be trying to match the ')' at this point in the
+ * derivation:
+ *
+ * => ID '=' '(' INT ')' ('+' atom)* ';'
+ * ^
+ * match() will see that ';' doesn't match ')' and report a
+ * mismatched token error. To recover, it sees that LA(1)==';'
+ * is in the set of tokens that can follow the ')' token
+ * reference in rule atom. It can assume that you forgot the ')'.
+ */
+ protected Object recoverFromMismatchedToken(IntStream input, int ttype, BitSet follow)
+ throws RecognitionException
+ {
+ RecognitionException e = null;
+ // if next token is what we are looking for then "delete" this token
+ if ( mismatchIsUnwantedToken(input, ttype) ) {
+ e = new UnwantedTokenException(ttype, input);
+ /*
+ System.err.println("recoverFromMismatchedToken deleting "+
+ ((TokenStream)input).LT(1)+
+ " since "+((TokenStream)input).LT(2)+" is what we want");
+ */
+ beginResync();
+ input.consume(); // simply delete extra token
+ endResync();
+ reportError(e); // report after consuming so AW sees the token in the exception
+ // we want to return the token we're actually matching
+ Object matchedSymbol = getCurrentInputSymbol(input);
+ input.consume(); // move past ttype token as if all were ok
+ return matchedSymbol;
+ }
+ // can't recover with single token deletion, try insertion
+ if ( mismatchIsMissingToken(input, follow) ) {
+ Object inserted = getMissingSymbol(input, e, ttype, follow);
+ e = new MissingTokenException(ttype, input, inserted);
+ reportError(e); // report after inserting so AW sees the token in the exception
+ return inserted;
+ }
+ // even that didn't work; must throw the exception
+ e = new MismatchedTokenException(ttype, input);
+ throw e;
+ }
+
+ /** Not currently used */
+ public Object recoverFromMismatchedSet(IntStream input,
+ RecognitionException e,
+ BitSet follow)
+ throws RecognitionException
+ {
+ if ( mismatchIsMissingToken(input, follow) ) {
+ // System.out.println("missing token");
+ reportError(e);
+ // we don't know how to conjure up a token for sets yet
+ return getMissingSymbol(input, e, Token.INVALID_TOKEN_TYPE, follow);
+ }
+ // TODO do single token deletion like above for Token mismatch
+ throw e;
+ }
+
+ /** Match needs to return the current input symbol, which gets put
+ * into the label for the associated token ref; e.g., x=ID. Token
+ * and tree parsers need to return different objects. Rather than test
+ * for input stream type or change the IntStream interface, I use
+ * a simple method to ask the recognizer to tell me what the current
+ * input symbol is.
+ *
+ * This is ignored for lexers.
+ */
+ protected Object getCurrentInputSymbol(IntStream input) { return null; }
+
+ /** Conjure up a missing token during error recovery.
+ *
+ * The recognizer attempts to recover from single missing
+ * symbols. But, actions might refer to that missing symbol.
+ * For example, x=ID {f($x);}. The action clearly assumes
+ * that there has been an identifier matched previously and that
+ * $x points at that token. If that token is missing, but
+ * the next token in the stream is what we want we assume that
+ * this token is missing and we keep going. Because we
+ * have to return some token to replace the missing token,
+ * we have to conjure one up. This method gives the user control
+ * over the tokens returned for missing tokens. Mostly,
+ * you will want to create something special for identifier
+ * tokens. For literals such as '{' and ',', the default
+ * action in the parser or tree parser works. It simply creates
+ * a CommonToken of the appropriate type. The text will be the token.
+ * If you change what tokens must be created by the lexer,
+ * override this method to create the appropriate tokens.
+ */
+ protected Object getMissingSymbol(IntStream input,
+ RecognitionException e,
+ int expectedTokenType,
+ BitSet follow)
+ {
+ return null;
+ }
+
+ public void consumeUntil(IntStream input, int tokenType) {
+ //System.out.println("consumeUntil "+tokenType);
+ int ttype = input.LA(1);
+ while (ttype != Token.EOF && ttype != tokenType) {
+ input.consume();
+ ttype = input.LA(1);
+ }
+ }
+
+ /** Consume tokens until one matches the given token set */
+ public void consumeUntil(IntStream input, BitSet set) {
+ //System.out.println("consumeUntil("+set.toString(getTokenNames())+")");
+ int ttype = input.LA(1);
+ while (ttype != Token.EOF && !set.member(ttype) ) {
+ //System.out.println("consume during recover LA(1)="+getTokenNames()[input.LA(1)]);
+ input.consume();
+ ttype = input.LA(1);
+ }
+ }
+
+ /** Push a rule's follow set using our own hardcoded stack */
+ protected void pushFollow(BitSet fset) {
+ if ( (state._fsp +1)>=state.following.length ) {
+ BitSet[] f = new BitSet[state.following.length*2];
+ System.arraycopy(state.following, 0, f, 0, state.following.length);
+ state.following = f;
+ }
+ state.following[++state._fsp] = fset;
+ }
+
+ /** Return List<String> of the rules in your parser instance
+ * leading up to a call to this method. You could override if
+ * you want more details such as the file/line info of where
+ * in the parser java code a rule is invoked.
+ *
+ * This is very useful for error messages and for context-sensitive
+ * error recovery.
+ */
+ public List getRuleInvocationStack() {
+ String parserClassName = getClass().getName();
+ return getRuleInvocationStack(new Throwable(), parserClassName);
+ }
+
+ /** A more general version of getRuleInvocationStack where you can
+ * pass in, for example, a RecognitionException to get it's rule
+ * stack trace. This routine is shared with all recognizers, hence,
+ * static.
+ *
+ * TODO: move to a utility class or something; weird having lexer call this
+ */
+ public static List getRuleInvocationStack(Throwable e,
+ String recognizerClassName)
+ {
+ List rules = new ArrayList();
+ StackTraceElement[] stack = e.getStackTrace();
+ int i = 0;
+ for (i=stack.length-1; i>=0; i--) {
+ StackTraceElement t = stack[i];
+ if ( t.getClassName().startsWith("org.antlr.runtime.") ) {
+ continue; // skip support code such as this method
+ }
+ if ( t.getMethodName().equals(NEXT_TOKEN_RULE_NAME) ) {
+ continue;
+ }
+ if ( !t.getClassName().equals(recognizerClassName) ) {
+ continue; // must not be part of this parser
+ }
+ rules.add(t.getMethodName());
+ }
+ return rules;
+ }
+
+ public int getBacktrackingLevel() { return state.backtracking; }
+
+ public void setBacktrackingLevel(int n) { state.backtracking = n; }
+
+ /** Return whether or not a backtracking attempt failed. */
+ public boolean failed() { return state.failed; }
+
+ /** Used to print out token names like ID during debugging and
+ * error reporting. The generated parsers implement a method
+ * that overrides this to point to their String[] tokenNames.
+ */
+ public String[] getTokenNames() {
+ return null;
+ }
+
+ /** For debugging and other purposes, might want the grammar name.
+ * Have ANTLR generate an implementation for this method.
+ */
+ public String getGrammarFileName() {
+ return null;
+ }
+
+ public abstract String getSourceName();
+
+ /** A convenience method for use most often with template rewrites.
+ * Convert a List<Token> to List<String>
+ */
+ public List toStrings(List tokens) {
+ if ( tokens==null ) return null;
+ List strings = new ArrayList(tokens.size());
+ for (int i=0; i<tokens.size(); i++) {
+ strings.add(((Token)tokens.get(i)).getText());
+ }
+ return strings;
+ }
+
+ /** Given a rule number and a start token index number, return
+ * MEMO_RULE_UNKNOWN if the rule has not parsed input starting from
+ * start index. If this rule has parsed input starting from the
+ * start index before, then return where the rule stopped parsing.
+ * It returns the index of the last token matched by the rule.
+ *
+ * For now we use a hashtable and just the slow Object-based one.
+ * Later, we can make a special one for ints and also one that
+ * tosses out data after we commit past input position i.
+ */
+ public int getRuleMemoization(int ruleIndex, int ruleStartIndex) {
+ if ( state.ruleMemo[ruleIndex]==null ) {
+ state.ruleMemo[ruleIndex] = new HashMap();
+ }
+ Integer stopIndexI =
+ (Integer)state.ruleMemo[ruleIndex].get(new Integer(ruleStartIndex));
+ if ( stopIndexI==null ) {
+ return MEMO_RULE_UNKNOWN;
+ }
+ return stopIndexI.intValue();
+ }
+
+ /** Has this rule already parsed input at the current index in the
+ * input stream? Return the stop token index or MEMO_RULE_UNKNOWN.
+ * If we attempted but failed to parse properly before, return
+ * MEMO_RULE_FAILED.
+ *
+ * This method has a side-effect: if we have seen this input for
+ * this rule and successfully parsed before, then seek ahead to
+ * 1 past the stop token matched for this rule last time.
+ */
+ public boolean alreadyParsedRule(IntStream input, int ruleIndex) {
+ int stopIndex = getRuleMemoization(ruleIndex, input.index());
+ if ( stopIndex==MEMO_RULE_UNKNOWN ) {
+ return false;
+ }
+ if ( stopIndex==MEMO_RULE_FAILED ) {
+ //System.out.println("rule "+ruleIndex+" will never succeed");
+ state.failed=true;
+ }
+ else {
+ //System.out.println("seen rule "+ruleIndex+" before; skipping ahead to @"+(stopIndex+1)+" failed="+state.failed);
+ input.seek(stopIndex+1); // jump to one past stop token
+ }
+ return true;
+ }
+
+ /** Record whether or not this rule parsed the input at this position
+ * successfully. Use a standard java hashtable for now.
+ */
+ public void memoize(IntStream input,
+ int ruleIndex,
+ int ruleStartIndex)
+ {
+ int stopTokenIndex = state.failed?MEMO_RULE_FAILED:input.index()-1;
+ if ( state.ruleMemo==null ) {
+ System.err.println("!!!!!!!!! memo array is null for "+ getGrammarFileName());
+ }
+ if ( ruleIndex >= state.ruleMemo.length ) {
+ System.err.println("!!!!!!!!! memo size is "+state.ruleMemo.length+", but rule index is "+ruleIndex);
+ }
+ if ( state.ruleMemo[ruleIndex]!=null ) {
+ state.ruleMemo[ruleIndex].put(
+ new Integer(ruleStartIndex), new Integer(stopTokenIndex)
+ );
+ }
+ }
+
+ /** return how many rule/input-index pairs there are in total.
+ * TODO: this includes synpreds. :(
+ */
+ public int getRuleMemoizationCacheSize() {
+ int n = 0;
+ for (int i = 0; state.ruleMemo!=null && i < state.ruleMemo.length; i++) {
+ Map ruleMap = state.ruleMemo[i];
+ if ( ruleMap!=null ) {
+ n += ruleMap.size(); // how many input indexes are recorded?
+ }
+ }
+ return n;
+ }
+
+ public void traceIn(String ruleName, int ruleIndex, Object inputSymbol) {
+ System.out.print("enter "+ruleName+" "+inputSymbol);
+ if ( state.backtracking>0 ) {
+ System.out.print(" backtracking="+state.backtracking);
+ }
+ System.out.println();
+ }
+
+ public void traceOut(String ruleName,
+ int ruleIndex,
+ Object inputSymbol)
+ {
+ System.out.print("exit "+ruleName+" "+inputSymbol);
+ if ( state.backtracking>0 ) {
+ System.out.print(" backtracking="+state.backtracking);
+ if ( state.failed ) System.out.print(" failed");
+ else System.out.print(" succeeded");
+ }
+ System.out.println();
+ }
+
+}
diff --git a/src/org/antlr/runtime/BitSet.java b/src/org/antlr/runtime/BitSet.java
new file mode 100755
index 0000000..f013a26
--- /dev/null
+++ b/src/org/antlr/runtime/BitSet.java
@@ -0,0 +1,325 @@
+/*
+ [The "BSD license"]
+ Copyright (c) 2005-2009 Terence Parr
+ All rights reserved.
+
+ Redistribution and use in source and binary forms, with or without
+ modification, are permitted provided that the following conditions
+ are met:
+ 1. Redistributions of source code must retain the above copyright
+ notice, this list of conditions and the following disclaimer.
+ 2. Redistributions in binary form must reproduce the above copyright
+ notice, this list of conditions and the following disclaimer in the
+ documentation and/or other materials provided with the distribution.
+ 3. The name of the author may not be used to endorse or promote products
+ derived from this software without specific prior written permission.
+
+ THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
+ IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
+ OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
+ IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
+ INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
+ NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
+ THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+package org.antlr.runtime;
+
+import java.util.List;
+
+/**A stripped-down version of org.antlr.misc.BitSet that is just
+ * good enough to handle runtime requirements such as FOLLOW sets
+ * for automatic error recovery.
+ */
+public class BitSet implements Cloneable {
+ protected final static int BITS = 64; // number of bits / long
+ protected final static int LOG_BITS = 6; // 2^6 == 64
+
+ /* We will often need to do a mod operator (i mod nbits). Its
+ * turns out that, for powers of two, this mod operation is
+ * same as (i & (nbits-1)). Since mod is slow, we use a
+ * precomputed mod mask to do the mod instead.
+ */
+ protected final static int MOD_MASK = BITS - 1;
+
+ /** The actual data bits */
+ protected long bits[];
+
+ /** Construct a bitset of size one word (64 bits) */
+ public BitSet() {
+ this(BITS);
+ }
+
+ /** Construction from a static array of longs */
+ public BitSet(long[] bits_) {
+ bits = bits_;
+ }
+
+ /** Construction from a list of integers */
+ public BitSet(List items) {
+ this();
+ for (int i = 0; i < items.size(); i++) {
+ Integer v = (Integer) items.get(i);
+ add(v.intValue());
+ }
+ }
+
+ /** Construct a bitset given the size
+ * @param nbits The size of the bitset in bits
+ */
+ public BitSet(int nbits) {
+ bits = new long[((nbits - 1) >> LOG_BITS) + 1];
+ }
+
+ public static BitSet of(int el) {
+ BitSet s = new BitSet(el + 1);
+ s.add(el);
+ return s;
+ }
+
+ public static BitSet of(int a, int b) {
+ BitSet s = new BitSet(Math.max(a,b)+1);
+ s.add(a);
+ s.add(b);
+ return s;
+ }
+
+ public static BitSet of(int a, int b, int c) {
+ BitSet s = new BitSet();
+ s.add(a);
+ s.add(b);
+ s.add(c);
+ return s;
+ }
+
+ public static BitSet of(int a, int b, int c, int d) {
+ BitSet s = new BitSet();
+ s.add(a);
+ s.add(b);
+ s.add(c);
+ s.add(d);
+ return s;
+ }
+
+ /** return this | a in a new set */
+ public BitSet or(BitSet a) {
+ if ( a==null ) {
+ return this;
+ }
+ BitSet s = (BitSet)this.clone();
+ s.orInPlace(a);
+ return s;
+ }
+
+ /** or this element into this set (grow as necessary to accommodate) */
+ public void add(int el) {
+ int n = wordNumber(el);
+ if (n >= bits.length) {
+ growToInclude(el);
+ }
+ bits[n] |= bitMask(el);
+ }
+
+ /**
+ * Grows the set to a larger number of bits.
+ * @param bit element that must fit in set
+ */
+ public void growToInclude(int bit) {
+ int newSize = Math.max(bits.length << 1, numWordsToHold(bit));
+ long newbits[] = new long[newSize];
+ System.arraycopy(bits, 0, newbits, 0, bits.length);
+ bits = newbits;
+ }
+
+ public void orInPlace(BitSet a) {
+ if ( a==null ) {
+ return;
+ }
+ // If this is smaller than a, grow this first
+ if (a.bits.length > bits.length) {
+ setSize(a.bits.length);
+ }
+ int min = Math.min(bits.length, a.bits.length);
+ for (int i = min - 1; i >= 0; i--) {
+ bits[i] |= a.bits[i];
+ }
+ }
+
+ /**
+ * Sets the size of a set.
+ * @param nwords how many words the new set should be
+ */
+ private void setSize(int nwords) {
+ long newbits[] = new long[nwords];
+ int n = Math.min(nwords, bits.length);
+ System.arraycopy(bits, 0, newbits, 0, n);
+ bits = newbits;
+ }
+
+ private final static long bitMask(int bitNumber) {
+ int bitPosition = bitNumber & MOD_MASK; // bitNumber mod BITS
+ return 1L << bitPosition;
+ }
+
+ public Object clone() {
+ BitSet s;
+ try {
+ s = (BitSet)super.clone();
+ s.bits = new long[bits.length];
+ System.arraycopy(bits, 0, s.bits, 0, bits.length);
+ }
+ catch (CloneNotSupportedException e) {
+ throw new InternalError();
+ }
+ return s;
+ }
+
+ public int size() {
+ int deg = 0;
+ for (int i = bits.length - 1; i >= 0; i--) {
+ long word = bits[i];
+ if (word != 0L) {
+ for (int bit = BITS - 1; bit >= 0; bit--) {
+ if ((word & (1L << bit)) != 0) {
+ deg++;
+ }
+ }
+ }
+ }
+ return deg;
+ }
+
+ public boolean equals(Object other) {
+ if ( other == null || !(other instanceof BitSet) ) {
+ return false;
+ }
+
+ BitSet otherSet = (BitSet)other;
+
+ int n = Math.min(this.bits.length, otherSet.bits.length);
+
+ // for any bits in common, compare
+ for (int i=0; i<n; i++) {
+ if (this.bits[i] != otherSet.bits[i]) {
+ return false;
+ }
+ }
+
+ // make sure any extra bits are off
+
+ if (this.bits.length > n) {
+ for (int i = n+1; i<this.bits.length; i++) {
+ if (this.bits[i] != 0) {
+ return false;
+ }
+ }
+ }
+ else if (otherSet.bits.length > n) {
+ for (int i = n+1; i<otherSet.bits.length; i++) {
+ if (otherSet.bits[i] != 0) {
+ return false;
+ }
+ }
+ }
+
+ return true;
+ }
+
+ public boolean member(int el) {
+ if ( el<0 ) {
+ return false;
+ }
+ int n = wordNumber(el);
+ if (n >= bits.length) return false;
+ return (bits[n] & bitMask(el)) != 0;
+ }
+
+ // remove this element from this set
+ public void remove(int el) {
+ int n = wordNumber(el);
+ if (n < bits.length) {
+ bits[n] &= ~bitMask(el);
+ }
+ }
+
+ public boolean isNil() {
+ for (int i = bits.length - 1; i >= 0; i--) {
+ if (bits[i] != 0) return false;
+ }
+ return true;
+ }
+
+ private final int numWordsToHold(int el) {
+ return (el >> LOG_BITS) + 1;
+ }
+
+ public int numBits() {
+ return bits.length << LOG_BITS; // num words * bits per word
+ }
+
+ /** return how much space is being used by the bits array not
+ * how many actually have member bits on.
+ */
+ public int lengthInLongWords() {
+ return bits.length;
+ }
+
+ /**Is this contained within a? */
+ /*
+ public boolean subset(BitSet a) {
+ if (a == null || !(a instanceof BitSet)) return false;
+ return this.and(a).equals(this);
+ }
+ */
+
+ public int[] toArray() {
+ int[] elems = new int[size()];
+ int en = 0;
+ for (int i = 0; i < (bits.length << LOG_BITS); i++) {
+ if (member(i)) {
+ elems[en++] = i;
+ }
+ }
+ return elems;
+ }
+
+ public long[] toPackedArray() {
+ return bits;
+ }
+
+ private final static int wordNumber(int bit) {
+ return bit >> LOG_BITS; // bit / BITS
+ }
+
+ public String toString() {
+ return toString(null);
+ }
+
+ public String toString(String[] tokenNames) {
+ StringBuffer buf = new StringBuffer();
+ String separator = ",";
+ boolean havePrintedAnElement = false;
+ buf.append('{');
+
+ for (int i = 0; i < (bits.length << LOG_BITS); i++) {
+ if (member(i)) {
+ if (i > 0 && havePrintedAnElement ) {
+ buf.append(separator);
+ }
+ if ( tokenNames!=null ) {
+ buf.append(tokenNames[i]);
+ }
+ else {
+ buf.append(i);
+ }
+ havePrintedAnElement = true;
+ }
+ }
+ buf.append('}');
+ return buf.toString();
+ }
+
+
+}
diff --git a/src/org/antlr/runtime/BufferedTokenStream.java b/src/org/antlr/runtime/BufferedTokenStream.java
new file mode 100755
index 0000000..7742d4f
--- /dev/null
+++ b/src/org/antlr/runtime/BufferedTokenStream.java
@@ -0,0 +1,272 @@
+/*
+ [The "BSD license"]
+ Copyright (c) 2005-2009 Terence Parr
+ All rights reserved.
+
+ Redistribution and use in source and binary forms, with or without
+ modification, are permitted provided that the following conditions
+ are met:
+ 1. Redistributions of source code must retain the above copyright
+ notice, this list of conditions and the following disclaimer.
+ 2. Redistributions in binary form must reproduce the above copyright
+ notice, this list of conditions and the following disclaimer in the
+ documentation and/or other materials provided with the distribution.
+ 3. The name of the author may not be used to endorse or promote products
+ derived from this software without specific prior written permission.
+
+ THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
+ IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
+ OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
+ IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
+ INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
+ NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
+ THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+package org.antlr.runtime;
+
+import java.util.List;
+import java.util.ArrayList;
+import java.util.NoSuchElementException;
+
+/** Buffer all input tokens but do on-demand fetching of new tokens from
+ * lexer. Useful when the parser or lexer has to set context/mode info before
+ * proper lexing of future tokens. The ST template parser needs this,
+ * for example, because it has to constantly flip back and forth between
+ * inside/output templates. E.g., <names:{hi, <it>}> has to parse names
+ * as part of an expression but "hi, <it>" as a nested template.
+ *
+ * You can't use this stream if you pass whitespace or other off-channel
+ * tokens to the parser. The stream can't ignore off-channel tokens.
+ * (UnbufferedTokenStream is the same way.)
+ *
+ * This is not a subclass of UnbufferedTokenStream because I don't want
+ * to confuse small moving window of tokens it uses for the full buffer.
+ */
+public class BufferedTokenStream implements TokenStream {
+ protected TokenSource tokenSource;
+
+ /** Record every single token pulled from the source so we can reproduce
+ * chunks of it later. The buffer in LookaheadStream overlaps sometimes
+ * as its moving window moves through the input. This list captures
+ * everything so we can access complete input text.
+ */
+ protected List<Token> tokens = new ArrayList<Token>(100);
+
+ /** Track the last mark() call result value for use in rewind(). */
+ protected int lastMarker;
+
+ /** The index into the tokens list of the current token (next token
+ * to consume). tokens[p] should be LT(1). p=-1 indicates need
+ * to initialize with first token. The ctor doesn't get a token.
+ * First call to LT(1) or whatever gets the first token and sets p=0;
+ */
+ protected int p = -1;
+
+ protected int range = -1; // how deep have we gone?
+
+ public BufferedTokenStream() {;}
+
+ public BufferedTokenStream(TokenSource tokenSource) {
+ this.tokenSource = tokenSource;
+ }
+
+ public TokenSource getTokenSource() { return tokenSource; }
+
+ public int index() { return p; }
+
+ public int range() { return range; }
+
+ public int mark() {
+ if ( p == -1 ) setup();
+ lastMarker = index();
+ return lastMarker;
+ }
+
+ public void release(int marker) {
+ // no resources to release
+ }
+
+ public void rewind(int marker) {
+ seek(marker);
+ }
+
+ public void rewind() {
+ seek(lastMarker);
+ }
+
+ public void reset() {
+ p = 0;
+ lastMarker = 0;
+ }
+
+ public void seek(int index) { p = index; }
+
+ public int size() { return tokens.size(); }
+
+ /** Move the input pointer to the next incoming token. The stream
+ * must become active with LT(1) available. consume() simply
+ * moves the input pointer so that LT(1) points at the next
+ * input symbol. Consume at least one token.
+ *
+ * Walk past any token not on the channel the parser is listening to.
+ */
+ public void consume() {
+ if ( p == -1 ) setup();
+ p++;
+ sync(p);
+ }
+
+ /** Make sure index i in tokens has a token. */
+ protected void sync(int i) {
+ int n = i - tokens.size() + 1; // how many more elements we need?
+ //System.out.println("sync("+i+") needs "+n);
+ if ( n > 0 ) fetch(n);
+ }
+
+ /** add n elements to buffer */
+ protected void fetch(int n) {
+ for (int i=1; i<=n; i++) {
+ Token t = tokenSource.nextToken();
+ t.setTokenIndex(tokens.size());
+ //System.out.println("adding "+t+" at index "+tokens.size());
+ tokens.add(t);
+ if ( t.getType()==Token.EOF ) break;
+ }
+ }
+
+ public Token get(int i) {
+ if ( i < 0 || i >= tokens.size() ) {
+ throw new NoSuchElementException("token index "+i+" out of range 0.."+(tokens.size()-1));
+ }
+ return tokens.get(i);
+ }
+
+ /** Get all tokens from start..stop inclusively */
+ public List get(int start, int stop) {
+ if ( start<0 || stop<0 ) return null;
+ if ( p == -1 ) setup();
+ List subset = new ArrayList();
+ if ( stop>=tokens.size() ) stop = tokens.size()-1;
+ for (int i = start; i <= stop; i++) {
+ Token t = tokens.get(i);
+ if ( t.getType()==Token.EOF ) break;
+ subset.add(t);
+ }
+ return subset;
+ }
+
+ public int LA(int i) { return LT(i).getType(); }
+
+ protected Token LB(int k) {
+ if ( (p-k)<0 ) return null;
+ return tokens.get(p-k);
+ }
+
+ public Token LT(int k) {
+ if ( p == -1 ) setup();
+ if ( k==0 ) return null;
+ if ( k < 0 ) return LB(-k);
+
+ int i = p + k - 1;
+ sync(i);
+ if ( i >= tokens.size() ) { // return EOF token
+ // EOF must be last token
+ return tokens.get(tokens.size()-1);
+ }
+ if ( i>range ) range = i;
+ return tokens.get(i);
+ }
+
+ protected void setup() { sync(0); p = 0; }
+
+ /** Reset this token stream by setting its token source. */
+ public void setTokenSource(TokenSource tokenSource) {
+ this.tokenSource = tokenSource;
+ tokens.clear();
+ p = -1;
+ }
+
+ public List getTokens() { return tokens; }
+
+ public List getTokens(int start, int stop) {
+ return getTokens(start, stop, (BitSet)null);
+ }
+
+ /** Given a start and stop index, return a List of all tokens in
+ * the token type BitSet. Return null if no tokens were found. This
+ * method looks at both on and off channel tokens.
+ */
+ public List getTokens(int start, int stop, BitSet types) {
+ if ( p == -1 ) setup();
+ if ( stop>=tokens.size() ) stop=tokens.size()-1;
+ if ( start<0 ) start=0;
+ if ( start>stop ) return null;
+
+ // list = tokens[start:stop]:{Token t, t.getType() in types}
+ List<Token> filteredTokens = new ArrayList<Token>();
+ for (int i=start; i<=stop; i++) {
+ Token t = tokens.get(i);
+ if ( types==null || types.member(t.getType()) ) {
+ filteredTokens.add(t);
+ }
+ }
+ if ( filteredTokens.size()==0 ) {
+ filteredTokens = null;
+ }
+ return filteredTokens;
+ }
+
+ public List getTokens(int start, int stop, List types) {
+ return getTokens(start,stop,new BitSet(types));
+ }
+
+ public List getTokens(int start, int stop, int ttype) {
+ return getTokens(start,stop,BitSet.of(ttype));
+ }
+
+ public String getSourceName() { return tokenSource.getSourceName(); }
+
+ /** Grab *all* tokens from stream and return string */
+ public String toString() {
+ if ( p == -1 ) setup();
+ fill();
+ return toString(0, tokens.size()-1);
+ }
+
+ public String toString(int start, int stop) {
+ if ( start<0 || stop<0 ) return null;
+ if ( p == -1 ) setup();
+ if ( stop>=tokens.size() ) stop = tokens.size()-1;
+ StringBuffer buf = new StringBuffer();
+ for (int i = start; i <= stop; i++) {
+ Token t = tokens.get(i);
+ if ( t.getType()==Token.EOF ) break;
+ buf.append(t.getText());
+ }
+ return buf.toString();
+ }
+
+ public String toString(Token start, Token stop) {
+ if ( start!=null && stop!=null ) {
+ return toString(start.getTokenIndex(), stop.getTokenIndex());
+ }
+ return null;
+ }
+
+ /** Get all tokens from lexer until EOF */
+ public void fill() {
+ if ( p == -1 ) setup();
+ if ( tokens.get(p).getType()==Token.EOF ) return;
+
+ int i = p+1;
+ sync(i);
+ while ( tokens.get(i).getType()!=Token.EOF ) {
+ i++;
+ sync(i);
+ }
+ }
+}
diff --git a/src/org/antlr/runtime/CharStream.java b/src/org/antlr/runtime/CharStream.java
new file mode 100755
index 0000000..db55755
--- /dev/null
+++ b/src/org/antlr/runtime/CharStream.java
@@ -0,0 +1,57 @@
+/*
+ [The "BSD license"]
+ Copyright (c) 2005-2009 Terence Parr
+ All rights reserved.
+
+ Redistribution and use in source and binary forms, with or without
+ modification, are permitted provided that the following conditions
+ are met:
+ 1. Redistributions of source code must retain the above copyright
+ notice, this list of conditions and the following disclaimer.
+ 2. Redistributions in binary form must reproduce the above copyright
+ notice, this list of conditions and the following disclaimer in the
+ documentation and/or other materials provided with the distribution.
+ 3. The name of the author may not be used to endorse or promote products
+ derived from this software without specific prior written permission.
+
+ THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
+ IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
+ OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
+ IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
+ INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
+ NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
+ THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+package org.antlr.runtime;
+
+/** A source of characters for an ANTLR lexer */
+public interface CharStream extends IntStream {
+ public static final int EOF = -1;
+
+ /** For infinite streams, you don't need this; primarily I'm providing
+ * a useful interface for action code. Just make sure actions don't
+ * use this on streams that don't support it.
+ */
+ public String substring(int start, int stop);
+
+ /** Get the ith character of lookahead. This is the same usually as
+ * LA(i). This will be used for labels in the generated
+ * lexer code. I'd prefer to return a char here type-wise, but it's
+ * probably better to be 32-bit clean and be consistent with LA.
+ */
+ public int LT(int i);
+
+ /** ANTLR tracks the line information automatically */
+ int getLine();
+
+ /** Because this stream can rewind, we need to be able to reset the line */
+ void setLine(int line);
+
+ void setCharPositionInLine(int pos);
+
+ /** The index of the character relative to the beginning of the line 0..n-1 */
+ int getCharPositionInLine();
+}
diff --git a/src/org/antlr/runtime/CharStreamState.java b/src/org/antlr/runtime/CharStreamState.java
new file mode 100755
index 0000000..04d969a
--- /dev/null
+++ b/src/org/antlr/runtime/CharStreamState.java
@@ -0,0 +1,45 @@
+/*
+ [The "BSD license"]
+ Copyright (c) 2005-2009 Terence Parr
+ All rights reserved.
+
+ Redistribution and use in source and binary forms, with or without
+ modification, are permitted provided that the following conditions
+ are met:
+ 1. Redistributions of source code must retain the above copyright
+ notice, this list of conditions and the following disclaimer.
+ 2. Redistributions in binary form must reproduce the above copyright
+ notice, this list of conditions and the following disclaimer in the
+ documentation and/or other materials provided with the distribution.
+ 3. The name of the author may not be used to endorse or promote products
+ derived from this software without specific prior written permission.
+
+ THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
+ IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
+ OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
+ IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
+ INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
+ NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
+ THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+package org.antlr.runtime;
+
+/** When walking ahead with cyclic DFA or for syntactic predicates,
+ * we need to record the state of the input stream (char index,
+ * line, etc...) so that we can rewind the state after scanning ahead.
+ *
+ * This is the complete state of a stream.
+ */
+public class CharStreamState {
+ /** Index into the char stream of next lookahead char */
+ int p;
+
+ /** What line number is the scanner at before processing buffer[p]? */
+ int line;
+
+ /** What char position 0..n-1 in line is scanner before processing buffer[p]? */
+ int charPositionInLine;
+}
diff --git a/src/org/antlr/runtime/ClassicToken.java b/src/org/antlr/runtime/ClassicToken.java
new file mode 100755
index 0000000..72c2bd9
--- /dev/null
+++ b/src/org/antlr/runtime/ClassicToken.java
@@ -0,0 +1,141 @@
+/*
+ [The "BSD license"]
+ Copyright (c) 2005-2009 Terence Parr
+ All rights reserved.
+
+ Redistribution and use in source and binary forms, with or without
+ modification, are permitted provided that the following conditions
+ are met:
+ 1. Redistributions of source code must retain the above copyright
+ notice, this list of conditions and the following disclaimer.
+ 2. Redistributions in binary form must reproduce the above copyright
+ notice, this list of conditions and the following disclaimer in the
+ documentation and/or other materials provided with the distribution.
+ 3. The name of the author may not be used to endorse or promote products
+ derived from this software without specific prior written permission.
+
+ THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
+ IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
+ OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
+ IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
+ INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
+ NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
+ THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+package org.antlr.runtime;
+
+/** A Token object like we'd use in ANTLR 2.x; has an actual string created
+ * and associated with this object. These objects are needed for imaginary
+ * tree nodes that have payload objects. We need to create a Token object
+ * that has a string; the tree node will point at this token. CommonToken
+ * has indexes into a char stream and hence cannot be used to introduce
+ * new strings.
+ */
+public class ClassicToken implements Token {
+ protected String text;
+ protected int type;
+ protected int line;
+ protected int charPositionInLine;
+ protected int channel=DEFAULT_CHANNEL;
+
+ /** What token number is this from 0..n-1 tokens */
+ protected int index;
+
+ public ClassicToken(int type) {
+ this.type = type;
+ }
+
+ public ClassicToken(Token oldToken) {
+ text = oldToken.getText();
+ type = oldToken.getType();
+ line = oldToken.getLine();
+ charPositionInLine = oldToken.getCharPositionInLine();
+ channel = oldToken.getChannel();
+ }
+
+ public ClassicToken(int type, String text) {
+ this.type = type;
+ this.text = text;
+ }
+
+ public ClassicToken(int type, String text, int channel) {
+ this.type = type;
+ this.text = text;
+ this.channel = channel;
+ }
+
+ public int getType() {
+ return type;
+ }
+
+ public void setLine(int line) {
+ this.line = line;
+ }
+
+ public String getText() {
+ return text;
+ }
+
+ public void setText(String text) {
+ this.text = text;
+ }
+
+ public int getLine() {
+ return line;
+ }
+
+ public int getCharPositionInLine() {
+ return charPositionInLine;
+ }
+
+ public void setCharPositionInLine(int charPositionInLine) {
+ this.charPositionInLine = charPositionInLine;
+ }
+
+ public int getChannel() {
+ return channel;
+ }
+
+ public void setChannel(int channel) {
+ this.channel = channel;
+ }
+
+ public void setType(int type) {
+ this.type = type;
+ }
+
+ public int getTokenIndex() {
+ return index;
+ }
+
+ public void setTokenIndex(int index) {
+ this.index = index;
+ }
+
+ public CharStream getInputStream() {
+ return null;
+ }
+
+ public void setInputStream(CharStream input) {
+ }
+
+ public String toString() {
+ String channelStr = "";
+ if ( channel>0 ) {
+ channelStr=",channel="+channel;
+ }
+ String txt = getText();
+ if ( txt!=null ) {
+ txt = txt.replaceAll("\n","\\\\n");
+ txt = txt.replaceAll("\r","\\\\r");
+ txt = txt.replaceAll("\t","\\\\t");
+ }
+ else {
+ txt = "<no text>";
+ }
+ return "[@"+getTokenIndex()+",'"+txt+"',<"+type+">"+channelStr+","+line+":"+getCharPositionInLine()+"]";
+ }
+}
diff --git a/src/org/antlr/runtime/CommonToken.java b/src/org/antlr/runtime/CommonToken.java
new file mode 100755
index 0000000..f01e1f4
--- /dev/null
+++ b/src/org/antlr/runtime/CommonToken.java
@@ -0,0 +1,191 @@
+/*
+ [The "BSD license"]
+ Copyright (c) 2005-2009 Terence Parr
+ All rights reserved.
+
+ Redistribution and use in source and binary forms, with or without
+ modification, are permitted provided that the following conditions
+ are met:
+ 1. Redistributions of source code must retain the above copyright
+ notice, this list of conditions and the following disclaimer.
+ 2. Redistributions in binary form must reproduce the above copyright
+ notice, this list of conditions and the following disclaimer in the
+ documentation and/or other materials provided with the distribution.
+ 3. The name of the author may not be used to endorse or promote products
+ derived from this software without specific prior written permission.
+
+ THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
+ IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
+ OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
+ IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
+ INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
+ NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
+ THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+package org.antlr.runtime;
+
+import java.io.Serializable;
+
+public class CommonToken implements Token, Serializable {
+ protected int type;
+ protected int line;
+ protected int charPositionInLine = -1; // set to invalid position
+ protected int channel=DEFAULT_CHANNEL;
+ protected transient CharStream input;
+
+ /** We need to be able to change the text once in a while. If
+ * this is non-null, then getText should return this. Note that
+ * start/stop are not affected by changing this.
+ */
+ protected String text;
+
+ /** What token number is this from 0..n-1 tokens; < 0 implies invalid index */
+ protected int index = -1;
+
+ /** The char position into the input buffer where this token starts */
+ protected int start;
+
+ /** The char position into the input buffer where this token stops */
+ protected int stop;
+
+ public CommonToken(int type) {
+ this.type = type;
+ }
+
+ public CommonToken(CharStream input, int type, int channel, int start, int stop) {
+ this.input = input;
+ this.type = type;
+ this.channel = channel;
+ this.start = start;
+ this.stop = stop;
+ }
+
+ public CommonToken(int type, String text) {
+ this.type = type;
+ this.channel = DEFAULT_CHANNEL;
+ this.text = text;
+ }
+
+ public CommonToken(Token oldToken) {
+ text = oldToken.getText();
+ type = oldToken.getType();
+ line = oldToken.getLine();
+ index = oldToken.getTokenIndex();
+ charPositionInLine = oldToken.getCharPositionInLine();
+ channel = oldToken.getChannel();
+ input = oldToken.getInputStream();
+ if ( oldToken instanceof CommonToken ) {
+ start = ((CommonToken)oldToken).start;
+ stop = ((CommonToken)oldToken).stop;
+ }
+ }
+
+ public int getType() {
+ return type;
+ }
+
+ public void setLine(int line) {
+ this.line = line;
+ }
+
+ public String getText() {
+ if ( text!=null ) {
+ return text;
+ }
+ if ( input==null ) {
+ return null;
+ }
+ if ( start<input.size() && stop<input.size() ) {
+ text = input.substring(start,stop);
+ }
+ else {
+ text = "<EOF>";
+ }
+ return text;
+ }
+
+ /** Override the text for this token. getText() will return this text
+ * rather than pulling from the buffer. Note that this does not mean
+ * that start/stop indexes are not valid. It means that that input
+ * was converted to a new string in the token object.
+ */
+ public void setText(String text) {
+ this.text = text;
+ }
+
+ public int getLine() {
+ return line;
+ }
+
+ public int getCharPositionInLine() {
+ return charPositionInLine;
+ }
+
+ public void setCharPositionInLine(int charPositionInLine) {
+ this.charPositionInLine = charPositionInLine;
+ }
+
+ public int getChannel() {
+ return channel;
+ }
+
+ public void setChannel(int channel) {
+ this.channel = channel;
+ }
+
+ public void setType(int type) {
+ this.type = type;
+ }
+
+ public int getStartIndex() {
+ return start;
+ }
+
+ public void setStartIndex(int start) {
+ this.start = start;
+ }
+
+ public int getStopIndex() {
+ return stop;
+ }
+
+ public void setStopIndex(int stop) {
+ this.stop = stop;
+ }
+
+ public int getTokenIndex() {
+ return index;
+ }
+
+ public void setTokenIndex(int index) {
+ this.index = index;
+ }
+
+ public CharStream getInputStream() {
+ return input;
+ }
+
+ public void setInputStream(CharStream input) {
+ this.input = input;
+ }
+
+ public String toString() {
+ String channelStr = "";
+ if ( channel>0 ) {
+ channelStr=",channel="+channel;
+ }
+ String txt = getText();
+ if ( txt!=null ) {
+ txt = txt.replaceAll("\n","\\\\n");
+ txt = txt.replaceAll("\r","\\\\r");
+ txt = txt.replaceAll("\t","\\\\t");
+ }
+ else {
+ txt = "<no text>";
+ }
+ return "[@"+getTokenIndex()+","+start+":"+stop+"='"+txt+"',<"+type+">"+channelStr+","+line+":"+getCharPositionInLine()+"]";
+ }
+}
diff --git a/src/org/antlr/runtime/CommonTokenStream.java b/src/org/antlr/runtime/CommonTokenStream.java
new file mode 100755
index 0000000..28135a6
--- /dev/null
+++ b/src/org/antlr/runtime/CommonTokenStream.java
@@ -0,0 +1,153 @@
+/*
+ [The "BSD license"]
+ Copyright (c) 2005-2009 Terence Parr
+ All rights reserved.
+
+ Redistribution and use in source and binary forms, with or without
+ modification, are permitted provided that the following conditions
+ are met:
+ 1. Redistributions of source code must retain the above copyright
+ notice, this list of conditions and the following disclaimer.
+ 2. Redistributions in binary form must reproduce the above copyright
+ notice, this list of conditions and the following disclaimer in the
+ documentation and/or other materials provided with the distribution.
+ 3. The name of the author may not be used to endorse or promote products
+ derived from this software without specific prior written permission.
+
+ THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
+ IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
+ OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
+ IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
+ INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
+ NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
+ THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+package org.antlr.runtime;
+
+/** The most common stream of tokens where every token is buffered up
+ * and tokens are filtered for a certain channel (the parser will only
+ * see these tokens).
+ *
+ * Even though it buffers all of the tokens, this token stream pulls tokens
+ * from the tokens source on demand. In other words, until you ask for a
+ * token using consume(), LT(), etc. the stream does not pull from the lexer.
+ *
+ * The only difference between this stream and BufferedTokenStream superclass
+ * is that this stream knows how to ignore off channel tokens. There may be
+ * a performance advantage to using the superclass if you don't pass
+ * whitespace and comments etc. to the parser on a hidden channel (i.e.,
+ * you set $channel instead of calling skip() in lexer rules.)
+ *
+ * @see org.antlr.runtime.UnbufferedTokenStream
+ * @see org.antlr.runtime.BufferedTokenStream
+ */
+public class CommonTokenStream extends BufferedTokenStream {
+ /** Skip tokens on any channel but this one; this is how we skip whitespace... */
+ protected int channel = Token.DEFAULT_CHANNEL;
+
+ public CommonTokenStream() { ; }
+
+ public CommonTokenStream(TokenSource tokenSource) {
+ super(tokenSource);
+ }
+
+ public CommonTokenStream(TokenSource tokenSource, int channel) {
+ this(tokenSource);
+ this.channel = channel;
+ }
+
+ /** Always leave p on an on-channel token. */
+ public void consume() {
+ if ( p == -1 ) setup();
+ p++;
+ sync(p);
+ while ( tokens.get(p).getChannel()!=channel ) {
+ p++;
+ sync(p);
+ }
+ }
+
+ protected Token LB(int k) {
+ if ( k==0 || (p-k)<0 ) return null;
+
+ int i = p;
+ int n = 1;
+ // find k good tokens looking backwards
+ while ( n<=k ) {
+ // skip off-channel tokens
+ i = skipOffTokenChannelsReverse(i-1);
+ n++;
+ }
+ if ( i<0 ) return null;
+ return tokens.get(i);
+ }
+
+ public Token LT(int k) {
+ //System.out.println("enter LT("+k+")");
+ if ( p == -1 ) setup();
+ if ( k == 0 ) return null;
+ if ( k < 0 ) return LB(-k);
+ int i = p;
+ int n = 1; // we know tokens[p] is a good one
+ // find k good tokens
+ while ( n<k ) {
+ // skip off-channel tokens
+ i = skipOffTokenChannels(i+1);
+ n++;
+ }
+ if ( i>range ) range = i;
+ return tokens.get(i);
+ }
+
+ /** Given a starting index, return the index of the first on-channel
+ * token.
+ */
+ protected int skipOffTokenChannels(int i) {
+ sync(i);
+ while ( tokens.get(i).getChannel()!=channel ) { // also stops at EOF (it's onchannel)
+ i++;
+ sync(i);
+ }
+ return i;
+ }
+
+ protected int skipOffTokenChannelsReverse(int i) {
+ while ( i>=0 && ((Token)tokens.get(i)).getChannel()!=channel ) {
+ i--;
+ }
+ return i;
+ }
+
+ protected void setup() {
+ p = 0;
+ sync(0);
+ int i = 0;
+ while ( tokens.get(i).getChannel()!=channel ) {
+ i++;
+ sync(i);
+ }
+ p = i;
+ }
+
+ /** Count EOF just once. */
+ public int getNumberOfOnChannelTokens() {
+ int n = 0;
+ fill();
+ for (int i = 0; i < tokens.size(); i++) {
+ Token t = tokens.get(i);
+ if ( t.getChannel()==channel ) n++;
+ if ( t.getType()==Token.EOF ) break;
+ }
+ return n;
+ }
+
+ /** Reset this token stream by setting its token source. */
+ public void setTokenSource(TokenSource tokenSource) {
+ super.setTokenSource(tokenSource);
+ channel = Token.DEFAULT_CHANNEL;
+ }
+}
diff --git a/src/org/antlr/runtime/DFA.java b/src/org/antlr/runtime/DFA.java
new file mode 100755
index 0000000..d1bebbc
--- /dev/null
+++ b/src/org/antlr/runtime/DFA.java
@@ -0,0 +1,229 @@
+/*
+ [The "BSD license"]
+ Copyright (c) 2005-2009 Terence Parr
+ All rights reserved.
+
+ Redistribution and use in source and binary forms, with or without
+ modification, are permitted provided that the following conditions
+ are met:
+ 1. Redistributions of source code must retain the above copyright
+ notice, this list of conditions and the following disclaimer.
+ 2. Redistributions in binary form must reproduce the above copyright
+ notice, this list of conditions and the following disclaimer in the
+ documentation and/or other materials provided with the distribution.
+ 3. The name of the author may not be used to endorse or promote products
+ derived from this software without specific prior written permission.
+
+ THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
+ IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
+ OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
+ IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
+ INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
+ NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
+ THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+package org.antlr.runtime;
+
+/** A DFA implemented as a set of transition tables.
+ *
+ * Any state that has a semantic predicate edge is special; those states
+ * are generated with if-then-else structures in a specialStateTransition()
+ * which is generated by cyclicDFA template.
+ *
+ * There are at most 32767 states (16-bit signed short).
+ * Could get away with byte sometimes but would have to generate different
+ * types and the simulation code too. For a point of reference, the Java
+ * lexer's Tokens rule DFA has 326 states roughly.
+ */
+public class DFA {
+ protected short[] eot;
+ protected short[] eof;
+ protected char[] min;
+ protected char[] max;
+ protected short[] accept;
+ protected short[] special;
+ protected short[][] transition;
+
+ protected int decisionNumber;
+
+ /** Which recognizer encloses this DFA? Needed to check backtracking */
+ protected BaseRecognizer recognizer;
+
+ public static final boolean debug = false;
+
+ /** From the input stream, predict what alternative will succeed
+ * using this DFA (representing the covering regular approximation
+ * to the underlying CFL). Return an alternative number 1..n. Throw
+ * an exception upon error.
+ */
+ public int predict(IntStream input)
+ throws RecognitionException
+ {
+ if ( debug ) {
+ System.err.println("Enter DFA.predict for decision "+decisionNumber);
+ }
+ int mark = input.mark(); // remember where decision started in input
+ int s = 0; // we always start at s0
+ try {
+ while ( true ) {
+ if ( debug ) System.err.println("DFA "+decisionNumber+" state "+s+" LA(1)="+(char)input.LA(1)+"("+input.LA(1)+
+ "), index="+input.index());
+ int specialState = special[s];
+ if ( specialState>=0 ) {
+ if ( debug ) {
+ System.err.println("DFA "+decisionNumber+
+ " state "+s+" is special state "+specialState);
+ }
+ s = specialStateTransition(specialState,input);
+ if ( debug ) {
+ System.err.println("DFA "+decisionNumber+
+ " returns from special state "+specialState+" to "+s);
+ }
+ if ( s==-1 ) {
+ noViableAlt(s,input);
+ return 0;
+ }
+ input.consume();
+ continue;
+ }
+ if ( accept[s] >= 1 ) {
+ if ( debug ) System.err.println("accept; predict "+accept[s]+" from state "+s);
+ return accept[s];
+ }
+ // look for a normal char transition
+ char c = (char)input.LA(1); // -1 == \uFFFF, all tokens fit in 65000 space
+ if (c>=min[s] && c<=max[s]) {
+ int snext = transition[s][c-min[s]]; // move to next state
+ if ( snext < 0 ) {
+ // was in range but not a normal transition
+ // must check EOT, which is like the else clause.
+ // eot[s]>=0 indicates that an EOT edge goes to another
+ // state.
+ if ( eot[s]>=0 ) { // EOT Transition to accept state?
+ if ( debug ) System.err.println("EOT transition");
+ s = eot[s];
+ input.consume();
+ // TODO: I had this as return accept[eot[s]]
+ // which assumed here that the EOT edge always
+ // went to an accept...faster to do this, but
+ // what about predicated edges coming from EOT
+ // target?
+ continue;
+ }
+ noViableAlt(s,input);
+ return 0;
+ }
+ s = snext;
+ input.consume();
+ continue;
+ }
+ if ( eot[s]>=0 ) { // EOT Transition?
+ if ( debug ) System.err.println("EOT transition");
+ s = eot[s];
+ input.consume();
+ continue;
+ }
+ if ( c==(char)Token.EOF && eof[s]>=0 ) { // EOF Transition to accept state?
+ if ( debug ) System.err.println("accept via EOF; predict "+accept[eof[s]]+" from "+eof[s]);
+ return accept[eof[s]];
+ }
+ // not in range and not EOF/EOT, must be invalid symbol
+ if ( debug ) {
+ System.err.println("min["+s+"]="+min[s]);
+ System.err.println("max["+s+"]="+max[s]);
+ System.err.println("eot["+s+"]="+eot[s]);
+ System.err.println("eof["+s+"]="+eof[s]);
+ for (int p=0; p<transition[s].length; p++) {
+ System.err.print(transition[s][p]+" ");
+ }
+ System.err.println();
+ }
+ noViableAlt(s,input);
+ return 0;
+ }
+ }
+ finally {
+ input.rewind(mark);
+ }
+ }
+
+ protected void noViableAlt(int s, IntStream input) throws NoViableAltException {
+ if (recognizer.state.backtracking>0) {
+ recognizer.state.failed=true;
+ return;
+ }
+ NoViableAltException nvae =
+ new NoViableAltException(getDescription(),
+ decisionNumber,
+ s,
+ input);
+ error(nvae);
+ throw nvae;
+ }
+
+ /** A hook for debugging interface */
+ protected void error(NoViableAltException nvae) { ; }
+
+ public int specialStateTransition(int s, IntStream input)
+ throws NoViableAltException
+ {
+ return -1;
+ }
+
+ public String getDescription() {
+ return "n/a";
+ }
+
+ /** Given a String that has a run-length-encoding of some unsigned shorts
+ * like "\1\2\3\9", convert to short[] {2,9,9,9}. We do this to avoid
+ * static short[] which generates so much init code that the class won't
+ * compile. :(
+ */
+ public static short[] unpackEncodedString(String encodedString) {
+ // walk first to find how big it is.
+ int size = 0;
+ for (int i=0; i<encodedString.length(); i+=2) {
+ size += encodedString.charAt(i);
+ }
+ short[] data = new short[size];
+ int di = 0;
+ for (int i=0; i<encodedString.length(); i+=2) {
+ char n = encodedString.charAt(i);
+ char v = encodedString.charAt(i+1);
+ // add v n times to data
+ for (int j=1; j<=n; j++) {
+ data[di++] = (short)v;
+ }
+ }
+ return data;
+ }
+
+ /** Hideous duplication of code, but I need different typed arrays out :( */
+ public static char[] unpackEncodedStringToUnsignedChars(String encodedString) {
+ // walk first to find how big it is.
+ int size = 0;
+ for (int i=0; i<encodedString.length(); i+=2) {
+ size += encodedString.charAt(i);
+ }
+ char[] data = new char[size];
+ int di = 0;
+ for (int i=0; i<encodedString.length(); i+=2) {
+ char n = encodedString.charAt(i);
+ char v = encodedString.charAt(i+1);
+ // add v n times to data
+ for (int j=1; j<=n; j++) {
+ data[di++] = v;
+ }
+ }
+ return data;
+ }
+
+ /*
+ public int specialTransition(int state, int symbol) {
+ return 0;
+ }
+ */
+}
diff --git a/src/org/antlr/runtime/EarlyExitException.java b/src/org/antlr/runtime/EarlyExitException.java
new file mode 100755
index 0000000..1f9c1ec
--- /dev/null
+++ b/src/org/antlr/runtime/EarlyExitException.java
@@ -0,0 +1,41 @@
+/*
+ [The "BSD license"]
+ Copyright (c) 2005-2009 Terence Parr
+ All rights reserved.
+
+ Redistribution and use in source and binary forms, with or without
+ modification, are permitted provided that the following conditions
+ are met:
+ 1. Redistributions of source code must retain the above copyright
+ notice, this list of conditions and the following disclaimer.
+ 2. Redistributions in binary form must reproduce the above copyright
+ notice, this list of conditions and the following disclaimer in the
+ documentation and/or other materials provided with the distribution.
+ 3. The name of the author may not be used to endorse or promote products
+ derived from this software without specific prior written permission.
+
+ THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
+ IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
+ OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
+ IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
+ INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
+ NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
+ THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+package org.antlr.runtime;
+
+/** The recognizer did not match anything for a (..)+ loop. */
+public class EarlyExitException extends RecognitionException {
+ public int decisionNumber;
+
+ /** Used for remote debugger deserialization */
+ public EarlyExitException() {;}
+
+ public EarlyExitException(int decisionNumber, IntStream input) {
+ super(input);
+ this.decisionNumber = decisionNumber;
+ }
+}
diff --git a/src/org/antlr/runtime/FailedPredicateException.java b/src/org/antlr/runtime/FailedPredicateException.java
new file mode 100755
index 0000000..5bef1bd
--- /dev/null
+++ b/src/org/antlr/runtime/FailedPredicateException.java
@@ -0,0 +1,54 @@
+/*
+ [The "BSD license"]
+ Copyright (c) 2005-2009 Terence Parr
+ All rights reserved.
+
+ Redistribution and use in source and binary forms, with or without
+ modification, are permitted provided that the following conditions
+ are met:
+ 1. Redistributions of source code must retain the above copyright
+ notice, this list of conditions and the following disclaimer.
+ 2. Redistributions in binary form must reproduce the above copyright
+ notice, this list of conditions and the following disclaimer in the
+ documentation and/or other materials provided with the distribution.
+ 3. The name of the author may not be used to endorse or promote products
+ derived from this software without specific prior written permission.
+
+ THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
+ IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
+ OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
+ IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
+ INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
+ NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
+ THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+package org.antlr.runtime;
+
+/** A semantic predicate failed during validation. Validation of predicates
+ * occurs when normally parsing the alternative just like matching a token.
+ * Disambiguating predicate evaluation occurs when we hoist a predicate into
+ * a prediction decision.
+ */
+public class FailedPredicateException extends RecognitionException {
+ public String ruleName;
+ public String predicateText;
+
+ /** Used for remote debugger deserialization */
+ public FailedPredicateException() {;}
+
+ public FailedPredicateException(IntStream input,
+ String ruleName,
+ String predicateText)
+ {
+ super(input);
+ this.ruleName = ruleName;
+ this.predicateText = predicateText;
+ }
+
+ public String toString() {
+ return "FailedPredicateException("+ruleName+",{"+predicateText+"}?)";
+ }
+}
diff --git a/src/org/antlr/runtime/IntStream.java b/src/org/antlr/runtime/IntStream.java
new file mode 100755
index 0000000..01c7841
--- /dev/null
+++ b/src/org/antlr/runtime/IntStream.java
@@ -0,0 +1,122 @@
+/*
+ [The "BSD license"]
+ Copyright (c) 2005-2009 Terence Parr
+ All rights reserved.
+
+ Redistribution and use in source and binary forms, with or without
+ modification, are permitted provided that the following conditions
+ are met:
+ 1. Redistributions of source code must retain the above copyright
+ notice, this list of conditions and the following disclaimer.
+ 2. Redistributions in binary form must reproduce the above copyright
+ notice, this list of conditions and the following disclaimer in the
+ documentation and/or other materials provided with the distribution.
+ 3. The name of the author may not be used to endorse or promote products
+ derived from this software without specific prior written permission.
+
+ THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
+ IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
+ OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
+ IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
+ INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
+ NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
+ THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+package org.antlr.runtime;
+
+/** A simple stream of integers used when all I care about is the char
+ * or token type sequence (such as interpretation).
+ */
+public interface IntStream {
+ void consume();
+
+ /** Get int at current input pointer + i ahead where i=1 is next int.
+ * Negative indexes are allowed. LA(-1) is previous token (token
+ * just matched). LA(-i) where i is before first token should
+ * yield -1, invalid char / EOF.
+ */
+ int LA(int i);
+
+ /** Tell the stream to start buffering if it hasn't already. Return
+ * current input position, index(), or some other marker so that
+ * when passed to rewind() you get back to the same spot.
+ * rewind(mark()) should not affect the input cursor. The Lexer
+ * track line/col info as well as input index so its markers are
+ * not pure input indexes. Same for tree node streams.
+ */
+ int mark();
+
+ /** Return the current input symbol index 0..n where n indicates the
+ * last symbol has been read. The index is the symbol about to be
+ * read not the most recently read symbol.
+ */
+ int index();
+
+ /** Reset the stream so that next call to index would return marker.
+ * The marker will usually be index() but it doesn't have to be. It's
+ * just a marker to indicate what state the stream was in. This is
+ * essentially calling release() and seek(). If there are markers
+ * created after this marker argument, this routine must unroll them
+ * like a stack. Assume the state the stream was in when this marker
+ * was created.
+ */
+ void rewind(int marker);
+
+ /** Rewind to the input position of the last marker.
+ * Used currently only after a cyclic DFA and just
+ * before starting a sem/syn predicate to get the
+ * input position back to the start of the decision.
+ * Do not "pop" the marker off the state. mark(i)
+ * and rewind(i) should balance still. It is
+ * like invoking rewind(last marker) but it should not "pop"
+ * the marker off. It's like seek(last marker's input position).
+ */
+ void rewind();
+
+ /** You may want to commit to a backtrack but don't want to force the
+ * stream to keep bookkeeping objects around for a marker that is
+ * no longer necessary. This will have the same behavior as
+ * rewind() except it releases resources without the backward seek.
+ * This must throw away resources for all markers back to the marker
+ * argument. So if you're nested 5 levels of mark(), and then release(2)
+ * you have to release resources for depths 2..5.
+ */
+ void release(int marker);
+
+ /** Set the input cursor to the position indicated by index. This is
+ * normally used to seek ahead in the input stream. No buffering is
+ * required to do this unless you know your stream will use seek to
+ * move backwards such as when backtracking.
+ *
+ * This is different from rewind in its multi-directional
+ * requirement and in that its argument is strictly an input cursor (index).
+ *
+ * For char streams, seeking forward must update the stream state such
+ * as line number. For seeking backwards, you will be presumably
+ * backtracking using the mark/rewind mechanism that restores state and
+ * so this method does not need to update state when seeking backwards.
+ *
+ * Currently, this method is only used for efficient backtracking using
+ * memoization, but in the future it may be used for incremental parsing.
+ *
+ * The index is 0..n-1. A seek to position i means that LA(1) will
+ * return the ith symbol. So, seeking to 0 means LA(1) will return the
+ * first element in the stream.
+ */
+ void seek(int index);
+
+ /** Only makes sense for streams that buffer everything up probably, but
+ * might be useful to display the entire stream or for testing. This
+ * value includes a single EOF.
+ */
+ int size();
+
+ /** Where are you getting symbols from? Normally, implementations will
+ * pass the buck all the way to the lexer who can ask its input stream
+ * for the file name or whatever.
+ */
+ public String getSourceName();
+}
diff --git a/src/org/antlr/runtime/LegacyCommonTokenStream.java b/src/org/antlr/runtime/LegacyCommonTokenStream.java
new file mode 100755
index 0000000..f9c5e39
--- /dev/null
+++ b/src/org/antlr/runtime/LegacyCommonTokenStream.java
@@ -0,0 +1,394 @@
+/*
+ [The "BSD license"]
+ Copyright (c) 2005-2009 Terence Parr
+ All rights reserved.
+
+ Redistribution and use in source and binary forms, with or without
+ modification, are permitted provided that the following conditions
+ are met:
+ 1. Redistributions of source code must retain the above copyright
+ notice, this list of conditions and the following disclaimer.
+ 2. Redistributions in binary form must reproduce the above copyright
+ notice, this list of conditions and the following disclaimer in the
+ documentation and/or other materials provided with the distribution.
+ 3. The name of the author may not be used to endorse or promote products
+ derived from this software without specific prior written permission.
+
+ THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
+ IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
+ OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
+ IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
+ INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
+ NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
+ THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+package org.antlr.runtime;
+
+import java.util.*;
+
+/** The most common stream of tokens is one where every token is buffered up
+ * and tokens are prefiltered for a certain channel (the parser will only
+ * see these tokens and cannot change the filter channel number during the
+ * parse).
+ *
+ * TODO: how to access the full token stream? How to track all tokens matched per rule?
+ */
+public class LegacyCommonTokenStream implements TokenStream {
+ protected TokenSource tokenSource;
+
+ /** Record every single token pulled from the source so we can reproduce
+ * chunks of it later.
+ */
+ protected List tokens;
+
+ /** Map<tokentype, channel> to override some Tokens' channel numbers */
+ protected Map channelOverrideMap;
+
+ /** Set<tokentype>; discard any tokens with this type */
+ protected Set discardSet;
+
+ /** Skip tokens on any channel but this one; this is how we skip whitespace... */
+ protected int channel = Token.DEFAULT_CHANNEL;
+
+ /** By default, track all incoming tokens */
+ protected boolean discardOffChannelTokens = false;
+
+ /** Track the last mark() call result value for use in rewind(). */
+ protected int lastMarker;
+
+ protected int range = -1; // how deep have we gone?
+
+ /** The index into the tokens list of the current token (next token
+ * to consume). p==-1 indicates that the tokens list is empty
+ */
+ protected int p = -1;
+
+ public LegacyCommonTokenStream() {
+ tokens = new ArrayList(500);
+ }
+
+ public LegacyCommonTokenStream(TokenSource tokenSource) {
+ this();
+ this.tokenSource = tokenSource;
+ }
+
+ public LegacyCommonTokenStream(TokenSource tokenSource, int channel) {
+ this(tokenSource);
+ this.channel = channel;
+ }
+
+ /** Reset this token stream by setting its token source. */
+ public void setTokenSource(TokenSource tokenSource) {
+ this.tokenSource = tokenSource;
+ tokens.clear();
+ p = -1;
+ channel = Token.DEFAULT_CHANNEL;
+ }
+
+ /** Load all tokens from the token source and put in tokens.
+ * This is done upon first LT request because you might want to
+ * set some token type / channel overrides before filling buffer.
+ */
+ protected void fillBuffer() {
+ int index = 0;
+ Token t = tokenSource.nextToken();
+ while ( t!=null && t.getType()!=CharStream.EOF ) {
+ boolean discard = false;
+ // is there a channel override for token type?
+ if ( channelOverrideMap!=null ) {
+ Integer channelI = (Integer)
+ channelOverrideMap.get(new Integer(t.getType()));
+ if ( channelI!=null ) {
+ t.setChannel(channelI.intValue());
+ }
+ }
+ if ( discardSet!=null &&
+ discardSet.contains(new Integer(t.getType())) )
+ {
+ discard = true;
+ }
+ else if ( discardOffChannelTokens && t.getChannel()!=this.channel ) {
+ discard = true;
+ }
+ if ( !discard ) {
+ t.setTokenIndex(index);
+ tokens.add(t);
+ index++;
+ }
+ t = tokenSource.nextToken();
+ }
+ // leave p pointing at first token on channel
+ p = 0;
+ p = skipOffTokenChannels(p);
+ }
+
+ /** Move the input pointer to the next incoming token. The stream
+ * must become active with LT(1) available. consume() simply
+ * moves the input pointer so that LT(1) points at the next
+ * input symbol. Consume at least one token.
+ *
+ * Walk past any token not on the channel the parser is listening to.
+ */
+ public void consume() {
+ if ( p<tokens.size() ) {
+ p++;
+ p = skipOffTokenChannels(p); // leave p on valid token
+ }
+ }
+
+ /** Given a starting index, return the index of the first on-channel
+ * token.
+ */
+ protected int skipOffTokenChannels(int i) {
+ int n = tokens.size();
+ while ( i<n && ((Token)tokens.get(i)).getChannel()!=channel ) {
+ i++;
+ }
+ return i;
+ }
+
+ protected int skipOffTokenChannelsReverse(int i) {
+ while ( i>=0 && ((Token)tokens.get(i)).getChannel()!=channel ) {
+ i--;
+ }
+ return i;
+ }
+
+ /** A simple filter mechanism whereby you can tell this token stream
+ * to force all tokens of type ttype to be on channel. For example,
+ * when interpreting, we cannot exec actions so we need to tell
+ * the stream to force all WS and NEWLINE to be a different, ignored
+ * channel.
+ */
+ public void setTokenTypeChannel(int ttype, int channel) {
+ if ( channelOverrideMap==null ) {
+ channelOverrideMap = new HashMap();
+ }
+ channelOverrideMap.put(new Integer(ttype), new Integer(channel));
+ }
+
+ public void discardTokenType(int ttype) {
+ if ( discardSet==null ) {
+ discardSet = new HashSet();
+ }
+ discardSet.add(new Integer(ttype));
+ }
+
+ public void discardOffChannelTokens(boolean discardOffChannelTokens) {
+ this.discardOffChannelTokens = discardOffChannelTokens;
+ }
+
+ public List getTokens() {
+ if ( p == -1 ) {
+ fillBuffer();
+ }
+ return tokens;
+ }
+
+ public List getTokens(int start, int stop) {
+ return getTokens(start, stop, (BitSet)null);
+ }
+
+ /** Given a start and stop index, return a List of all tokens in
+ * the token type BitSet. Return null if no tokens were found. This
+ * method looks at both on and off channel tokens.
+ */
+ public List getTokens(int start, int stop, BitSet types) {
+ if ( p == -1 ) {
+ fillBuffer();
+ }
+ if ( stop>=tokens.size() ) {
+ stop=tokens.size()-1;
+ }
+ if ( start<0 ) {
+ start=0;
+ }
+ if ( start>stop ) {
+ return null;
+ }
+
+ // list = tokens[start:stop]:{Token t, t.getType() in types}
+ List filteredTokens = new ArrayList();
+ for (int i=start; i<=stop; i++) {
+ Token t = (Token)tokens.get(i);
+ if ( types==null || types.member(t.getType()) ) {
+ filteredTokens.add(t);
+ }
+ }
+ if ( filteredTokens.size()==0 ) {
+ filteredTokens = null;
+ }
+ return filteredTokens;
+ }
+
+ public List getTokens(int start, int stop, List types) {
+ return getTokens(start,stop,new BitSet(types));
+ }
+
+ public List getTokens(int start, int stop, int ttype) {
+ return getTokens(start,stop,BitSet.of(ttype));
+ }
+
+ /** Get the ith token from the current position 1..n where k=1 is the
+ * first symbol of lookahead.
+ */
+ public Token LT(int k) {
+ if ( p == -1 ) {
+ fillBuffer();
+ }
+ if ( k==0 ) {
+ return null;
+ }
+ if ( k<0 ) {
+ return LB(-k);
+ }
+ //System.out.print("LT(p="+p+","+k+")=");
+ if ( (p+k-1) >= tokens.size() ) {
+ return (Token)tokens.get(tokens.size()-1);
+ }
+ //System.out.println(tokens.get(p+k-1));
+ int i = p;
+ int n = 1;
+ // find k good tokens
+ while ( n<k ) {
+ // skip off-channel tokens
+ i = skipOffTokenChannels(i+1); // leave p on valid token
+ n++;
+ }
+ if ( i>=tokens.size() ) {
+ return (Token)tokens.get(tokens.size()-1); // must be EOF
+ }
+
+ if ( i>range ) range = i;
+ return (Token)tokens.get(i);
+ }
+
+ /** Look backwards k tokens on-channel tokens */
+ protected Token LB(int k) {
+ //System.out.print("LB(p="+p+","+k+") ");
+ if ( p == -1 ) {
+ fillBuffer();
+ }
+ if ( k==0 ) {
+ return null;
+ }
+ if ( (p-k)<0 ) {
+ return null;
+ }
+
+ int i = p;
+ int n = 1;
+ // find k good tokens looking backwards
+ while ( n<=k ) {
+ // skip off-channel tokens
+ i = skipOffTokenChannelsReverse(i-1); // leave p on valid token
+ n++;
+ }
+ if ( i<0 ) {
+ return null;
+ }
+ return (Token)tokens.get(i);
+ }
+
+ /** Return absolute token i; ignore which channel the tokens are on;
+ * that is, count all tokens not just on-channel tokens.
+ */
+ public Token get(int i) {
+ return (Token)tokens.get(i);
+ }
+
+ /** Get all tokens from start..stop inclusively */
+ public List get(int start, int stop) {
+ if ( p == -1 ) fillBuffer();
+ if ( start<0 || stop<0 ) return null;
+ return tokens.subList(start, stop);
+ }
+
+ public int LA(int i) {
+ return LT(i).getType();
+ }
+
+ public int mark() {
+ if ( p == -1 ) {
+ fillBuffer();
+ }
+ lastMarker = index();
+ return lastMarker;
+ }
+
+ public void release(int marker) {
+ // no resources to release
+ }
+
+ public int size() {
+ return tokens.size();
+ }
+
+ public int index() {
+ return p;
+ }
+
+ public int range() {
+ return range;
+ }
+
+ public void rewind(int marker) {
+ seek(marker);
+ }
+
+ public void rewind() {
+ seek(lastMarker);
+ }
+
+ public void reset() {
+ p = 0;
+ lastMarker = 0;
+ }
+
+ public void seek(int index) {
+ p = index;
+ }
+
+ public TokenSource getTokenSource() {
+ return tokenSource;
+ }
+
+ public String getSourceName() {
+ return getTokenSource().getSourceName();
+ }
+
+ public String toString() {
+ if ( p == -1 ) {
+ fillBuffer();
+ }
+ return toString(0, tokens.size()-1);
+ }
+
+ public String toString(int start, int stop) {
+ if ( start<0 || stop<0 ) {
+ return null;
+ }
+ if ( p == -1 ) {
+ fillBuffer();
+ }
+ if ( stop>=tokens.size() ) {
+ stop = tokens.size()-1;
+ }
+ StringBuffer buf = new StringBuffer();
+ for (int i = start; i <= stop; i++) {
+ Token t = (Token)tokens.get(i);
+ buf.append(t.getText());
+ }
+ return buf.toString();
+ }
+
+ public String toString(Token start, Token stop) {
+ if ( start!=null && stop!=null ) {
+ return toString(start.getTokenIndex(), stop.getTokenIndex());
+ }
+ return null;
+ }
+}
diff --git a/src/org/antlr/runtime/Lexer.java b/src/org/antlr/runtime/Lexer.java
new file mode 100755
index 0000000..6037bcf
--- /dev/null
+++ b/src/org/antlr/runtime/Lexer.java
@@ -0,0 +1,340 @@
+/*
+ [The "BSD license"]
+ Copyright (c) 2005-2009 Terence Parr
+ All rights reserved.
+
+ Redistribution and use in source and binary forms, with or without
+ modification, are permitted provided that the following conditions
+ are met:
+ 1. Redistributions of source code must retain the above copyright
+ notice, this list of conditions and the following disclaimer.
+ 2. Redistributions in binary form must reproduce the above copyright
+ notice, this list of conditions and the following disclaimer in the
+ documentation and/or other materials provided with the distribution.
+ 3. The name of the author may not be used to endorse or promote products
+ derived from this software without specific prior written permission.
+
+ THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
+ IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
+ OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
+ IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
+ INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
+ NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
+ THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+package org.antlr.runtime;
+
+/** A lexer is recognizer that draws input symbols from a character stream.
+ * lexer grammars result in a subclass of this object. A Lexer object
+ * uses simplified match() and error recovery mechanisms in the interest
+ * of speed.
+ */
+public abstract class Lexer extends BaseRecognizer implements TokenSource {
+ /** Where is the lexer drawing characters from? */
+ protected CharStream input;
+
+ public Lexer() {
+ }
+
+ public Lexer(CharStream input) {
+ this.input = input;
+ }
+
+ public Lexer(CharStream input, RecognizerSharedState state) {
+ super(state);
+ this.input = input;
+ }
+
+ public void reset() {
+ super.reset(); // reset all recognizer state variables
+ // wack Lexer state variables
+ if ( input!=null ) {
+ input.seek(0); // rewind the input
+ }
+ if ( state==null ) {
+ return; // no shared state work to do
+ }
+ state.token = null;
+ state.type = Token.INVALID_TOKEN_TYPE;
+ state.channel = Token.DEFAULT_CHANNEL;
+ state.tokenStartCharIndex = -1;
+ state.tokenStartCharPositionInLine = -1;
+ state.tokenStartLine = -1;
+ state.text = null;
+ }
+
+ /** Return a token from this source; i.e., match a token on the char
+ * stream.
+ */
+ public Token nextToken() {
+ while (true) {
+ state.token = null;
+ state.channel = Token.DEFAULT_CHANNEL;
+ state.tokenStartCharIndex = input.index();
+ state.tokenStartCharPositionInLine = input.getCharPositionInLine();
+ state.tokenStartLine = input.getLine();
+ state.text = null;
+ if ( input.LA(1)==CharStream.EOF ) {
+ Token eof = new CommonToken((CharStream)input,Token.EOF,
+ Token.DEFAULT_CHANNEL,
+ input.index(),input.index());
+ eof.setLine(getLine());
+ eof.setCharPositionInLine(getCharPositionInLine());
+ return eof;
+ }
+ try {
+ mTokens();
+ if ( state.token==null ) {
+ emit();
+ }
+ else if ( state.token==Token.SKIP_TOKEN ) {
+ continue;
+ }
+ return state.token;
+ }
+ catch (NoViableAltException nva) {
+ reportError(nva);
+ recover(nva); // throw out current char and try again
+ }
+ catch (RecognitionException re) {
+ reportError(re);
+ // match() routine has already called recover()
+ }
+ }
+ }
+
+ /** Instruct the lexer to skip creating a token for current lexer rule
+ * and look for another token. nextToken() knows to keep looking when
+ * a lexer rule finishes with token set to SKIP_TOKEN. Recall that
+ * if token==null at end of any token rule, it creates one for you
+ * and emits it.
+ */
+ public void skip() {
+ state.token = Token.SKIP_TOKEN;
+ }
+
+ /** This is the lexer entry point that sets instance var 'token' */
+ public abstract void mTokens() throws RecognitionException;
+
+ /** Set the char stream and reset the lexer */
+ public void setCharStream(CharStream input) {
+ this.input = null;
+ reset();
+ this.input = input;
+ }
+
+ public CharStream getCharStream() {
+ return this.input;
+ }
+
+ public String getSourceName() {
+ return input.getSourceName();
+ }
+
+ /** Currently does not support multiple emits per nextToken invocation
+ * for efficiency reasons. Subclass and override this method and
+ * nextToken (to push tokens into a list and pull from that list rather
+ * than a single variable as this implementation does).
+ */
+ public void emit(Token token) {
+ state.token = token;
+ }
+
+ /** The standard method called to automatically emit a token at the
+ * outermost lexical rule. The token object should point into the
+ * char buffer start..stop. If there is a text override in 'text',
+ * use that to set the token's text. Override this method to emit
+ * custom Token objects.
+ *
+ * If you are building trees, then you should also override
+ * Parser or TreeParser.getMissingSymbol().
+ */
+ public Token emit() {
+ Token t = new CommonToken(input, state.type, state.channel, state.tokenStartCharIndex, getCharIndex()-1);
+ t.setLine(state.tokenStartLine);
+ t.setText(state.text);
+ t.setCharPositionInLine(state.tokenStartCharPositionInLine);
+ emit(t);
+ return t;
+ }
+
+ public void match(String s) throws MismatchedTokenException {
+ int i = 0;
+ while ( i<s.length() ) {
+ if ( input.LA(1)!=s.charAt(i) ) {
+ if ( state.backtracking>0 ) {
+ state.failed = true;
+ return;
+ }
+ MismatchedTokenException mte =
+ new MismatchedTokenException(s.charAt(i), input);
+ recover(mte);
+ throw mte;
+ }
+ i++;
+ input.consume();
+ state.failed = false;
+ }
+ }
+
+ public void matchAny() {
+ input.consume();
+ }
+
+ public void match(int c) throws MismatchedTokenException {
+ if ( input.LA(1)!=c ) {
+ if ( state.backtracking>0 ) {
+ state.failed = true;
+ return;
+ }
+ MismatchedTokenException mte =
+ new MismatchedTokenException(c, input);
+ recover(mte); // don't really recover; just consume in lexer
+ throw mte;
+ }
+ input.consume();
+ state.failed = false;
+ }
+
+ public void matchRange(int a, int b)
+ throws MismatchedRangeException
+ {
+ if ( input.LA(1)<a || input.LA(1)>b ) {
+ if ( state.backtracking>0 ) {
+ state.failed = true;
+ return;
+ }
+ MismatchedRangeException mre =
+ new MismatchedRangeException(a,b,input);
+ recover(mre);
+ throw mre;
+ }
+ input.consume();
+ state.failed = false;
+ }
+
+ public int getLine() {
+ return input.getLine();
+ }
+
+ public int getCharPositionInLine() {
+ return input.getCharPositionInLine();
+ }
+
+ /** What is the index of the current character of lookahead? */
+ public int getCharIndex() {
+ return input.index();
+ }
+
+ /** Return the text matched so far for the current token or any
+ * text override.
+ */
+ public String getText() {
+ if ( state.text!=null ) {
+ return state.text;
+ }
+ return input.substring(state.tokenStartCharIndex,getCharIndex()-1);
+ }
+
+ /** Set the complete text of this token; it wipes any previous
+ * changes to the text.
+ */
+ public void setText(String text) {
+ state.text = text;
+ }
+
+ public void reportError(RecognitionException e) {
+ /** TODO: not thought about recovery in lexer yet.
+ *
+ // if we've already reported an error and have not matched a token
+ // yet successfully, don't report any errors.
+ if ( errorRecovery ) {
+ //System.err.print("[SPURIOUS] ");
+ return;
+ }
+ errorRecovery = true;
+ */
+
+ displayRecognitionError(this.getTokenNames(), e);
+ }
+
+ public String getErrorMessage(RecognitionException e, String[] tokenNames) {
+ String msg = null;
+ if ( e instanceof MismatchedTokenException ) {
+ MismatchedTokenException mte = (MismatchedTokenException)e;
+ msg = "mismatched character "+getCharErrorDisplay(e.c)+" expecting "+getCharErrorDisplay(mte.expecting);
+ }
+ else if ( e instanceof NoViableAltException ) {
+ NoViableAltException nvae = (NoViableAltException)e;
+ // for development, can add "decision=<<"+nvae.grammarDecisionDescription+">>"
+ // and "(decision="+nvae.decisionNumber+") and
+ // "state "+nvae.stateNumber
+ msg = "no viable alternative at character "+getCharErrorDisplay(e.c);
+ }
+ else if ( e instanceof EarlyExitException ) {
+ EarlyExitException eee = (EarlyExitException)e;
+ // for development, can add "(decision="+eee.decisionNumber+")"
+ msg = "required (...)+ loop did not match anything at character "+getCharErrorDisplay(e.c);
+ }
+ else if ( e instanceof MismatchedNotSetException ) {
+ MismatchedNotSetException mse = (MismatchedNotSetException)e;
+ msg = "mismatched character "+getCharErrorDisplay(e.c)+" expecting set "+mse.expecting;
+ }
+ else if ( e instanceof MismatchedSetException ) {
+ MismatchedSetException mse = (MismatchedSetException)e;
+ msg = "mismatched character "+getCharErrorDisplay(e.c)+" expecting set "+mse.expecting;
+ }
+ else if ( e instanceof MismatchedRangeException ) {
+ MismatchedRangeException mre = (MismatchedRangeException)e;
+ msg = "mismatched character "+getCharErrorDisplay(e.c)+" expecting set "+
+ getCharErrorDisplay(mre.a)+".."+getCharErrorDisplay(mre.b);
+ }
+ else {
+ msg = super.getErrorMessage(e, tokenNames);
+ }
+ return msg;
+ }
+
+ public String getCharErrorDisplay(int c) {
+ String s = String.valueOf((char)c);
+ switch ( c ) {
+ case Token.EOF :
+ s = "<EOF>";
+ break;
+ case '\n' :
+ s = "\\n";
+ break;
+ case '\t' :
+ s = "\\t";
+ break;
+ case '\r' :
+ s = "\\r";
+ break;
+ }
+ return "'"+s+"'";
+ }
+
+ /** Lexers can normally match any char in it's vocabulary after matching
+ * a token, so do the easy thing and just kill a character and hope
+ * it all works out. You can instead use the rule invocation stack
+ * to do sophisticated error recovery if you are in a fragment rule.
+ */
+ public void recover(RecognitionException re) {
+ //System.out.println("consuming char "+(char)input.LA(1)+" during recovery");
+ //re.printStackTrace();
+ input.consume();
+ }
+
+ public void traceIn(String ruleName, int ruleIndex) {
+ String inputSymbol = ((char)input.LT(1))+" line="+getLine()+":"+getCharPositionInLine();
+ super.traceIn(ruleName, ruleIndex, inputSymbol);
+ }
+
+ public void traceOut(String ruleName, int ruleIndex) {
+ String inputSymbol = ((char)input.LT(1))+" line="+getLine()+":"+getCharPositionInLine();
+ super.traceOut(ruleName, ruleIndex, inputSymbol);
+ }
+}
diff --git a/src/org/antlr/runtime/MismatchedNotSetException.java b/src/org/antlr/runtime/MismatchedNotSetException.java
new file mode 100755
index 0000000..49ceb27
--- /dev/null
+++ b/src/org/antlr/runtime/MismatchedNotSetException.java
@@ -0,0 +1,41 @@
+/*
+ [The "BSD license"]
+ Copyright (c) 2005-2009 Terence Parr
+ All rights reserved.
+
+ Redistribution and use in source and binary forms, with or without
+ modification, are permitted provided that the following conditions
+ are met:
+ 1. Redistributions of source code must retain the above copyright
+ notice, this list of conditions and the following disclaimer.
+ 2. Redistributions in binary form must reproduce the above copyright
+ notice, this list of conditions and the following disclaimer in the
+ documentation and/or other materials provided with the distribution.
+ 3. The name of the author may not be used to endorse or promote products
+ derived from this software without specific prior written permission.
+
+ THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
+ IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
+ OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
+ IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
+ INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
+ NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
+ THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+package org.antlr.runtime;
+
+public class MismatchedNotSetException extends MismatchedSetException {
+ /** Used for remote debugger deserialization */
+ public MismatchedNotSetException() {;}
+
+ public MismatchedNotSetException(BitSet expecting, IntStream input) {
+ super(expecting, input);
+ }
+
+ public String toString() {
+ return "MismatchedNotSetException("+getUnexpectedType()+"!="+expecting+")";
+ }
+}
diff --git a/src/org/antlr/runtime/MismatchedRangeException.java b/src/org/antlr/runtime/MismatchedRangeException.java
new file mode 100755
index 0000000..23b3d87
--- /dev/null
+++ b/src/org/antlr/runtime/MismatchedRangeException.java
@@ -0,0 +1,45 @@
+/*
+ [The "BSD license"]
+ Copyright (c) 2005-2009 Terence Parr
+ All rights reserved.
+
+ Redistribution and use in source and binary forms, with or without
+ modification, are permitted provided that the following conditions
+ are met:
+ 1. Redistributions of source code must retain the above copyright
+ notice, this list of conditions and the following disclaimer.
+ 2. Redistributions in binary form must reproduce the above copyright
+ notice, this list of conditions and the following disclaimer in the
+ documentation and/or other materials provided with the distribution.
+ 3. The name of the author may not be used to endorse or promote products
+ derived from this software without specific prior written permission.
+
+ THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
+ IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
+ OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
+ IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
+ INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
+ NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
+ THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+package org.antlr.runtime;
+
+public class MismatchedRangeException extends RecognitionException {
+ public int a,b;
+
+ /** Used for remote debugger deserialization */
+ public MismatchedRangeException() {;}
+
+ public MismatchedRangeException(int a, int b, IntStream input) {
+ super(input);
+ this.a = a;
+ this.b = b;
+ }
+
+ public String toString() {
+ return "MismatchedNotSetException("+getUnexpectedType()+" not in ["+a+","+b+"])";
+ }
+}
diff --git a/src/org/antlr/runtime/MismatchedSetException.java b/src/org/antlr/runtime/MismatchedSetException.java
new file mode 100755
index 0000000..9bfa530
--- /dev/null
+++ b/src/org/antlr/runtime/MismatchedSetException.java
@@ -0,0 +1,44 @@
+/*
+ [The "BSD license"]
+ Copyright (c) 2005-2009 Terence Parr
+ All rights reserved.
+
+ Redistribution and use in source and binary forms, with or without
+ modification, are permitted provided that the following conditions
+ are met:
+ 1. Redistributions of source code must retain the above copyright
+ notice, this list of conditions and the following disclaimer.
+ 2. Redistributions in binary form must reproduce the above copyright
+ notice, this list of conditions and the following disclaimer in the
+ documentation and/or other materials provided with the distribution.
+ 3. The name of the author may not be used to endorse or promote products
+ derived from this software without specific prior written permission.
+
+ THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
+ IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
+ OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
+ IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
+ INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
+ NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
+ THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+package org.antlr.runtime;
+
+public class MismatchedSetException extends RecognitionException {
+ public BitSet expecting;
+
+ /** Used for remote debugger deserialization */
+ public MismatchedSetException() {;}
+
+ public MismatchedSetException(BitSet expecting, IntStream input) {
+ super(input);
+ this.expecting = expecting;
+ }
+
+ public String toString() {
+ return "MismatchedSetException("+getUnexpectedType()+"!="+expecting+")";
+ }
+}
diff --git a/src/org/antlr/runtime/MismatchedTokenException.java b/src/org/antlr/runtime/MismatchedTokenException.java
new file mode 100755
index 0000000..07ae814
--- /dev/null
+++ b/src/org/antlr/runtime/MismatchedTokenException.java
@@ -0,0 +1,45 @@
+/*
+ [The "BSD license"]
+ Copyright (c) 2005-2009 Terence Parr
+ All rights reserved.
+
+ Redistribution and use in source and binary forms, with or without
+ modification, are permitted provided that the following conditions
+ are met:
+ 1. Redistributions of source code must retain the above copyright
+ notice, this list of conditions and the following disclaimer.
+ 2. Redistributions in binary form must reproduce the above copyright
+ notice, this list of conditions and the following disclaimer in the
+ documentation and/or other materials provided with the distribution.
+ 3. The name of the author may not be used to endorse or promote products
+ derived from this software without specific prior written permission.
+
+ THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
+ IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
+ OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
+ IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
+ INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
+ NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
+ THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+package org.antlr.runtime;
+
+/** A mismatched char or Token or tree node */
+public class MismatchedTokenException extends RecognitionException {
+ public int expecting = Token.INVALID_TOKEN_TYPE;
+
+ /** Used for remote debugger deserialization */
+ public MismatchedTokenException() {;}
+
+ public MismatchedTokenException(int expecting, IntStream input) {
+ super(input);
+ this.expecting = expecting;
+ }
+
+ public String toString() {
+ return "MismatchedTokenException("+getUnexpectedType()+"!="+expecting+")";
+ }
+}
diff --git a/src/org/antlr/runtime/MismatchedTreeNodeException.java b/src/org/antlr/runtime/MismatchedTreeNodeException.java
new file mode 100755
index 0000000..99c834d
--- /dev/null
+++ b/src/org/antlr/runtime/MismatchedTreeNodeException.java
@@ -0,0 +1,49 @@
+/*
+ [The "BSD license"]
+ Copyright (c) 2005-2009 Terence Parr
+ All rights reserved.
+
+ Redistribution and use in source and binary forms, with or without
+ modification, are permitted provided that the following conditions
+ are met:
+ 1. Redistributions of source code must retain the above copyright
+ notice, this list of conditions and the following disclaimer.
+ 2. Redistributions in binary form must reproduce the above copyright
+ notice, this list of conditions and the following disclaimer in the
+ documentation and/or other materials provided with the distribution.
+ 3. The name of the author may not be used to endorse or promote products
+ derived from this software without specific prior written permission.
+
+ THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
+ IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
+ OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
+ IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
+ INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
+ NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
+ THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+package org.antlr.runtime;
+
+import org.antlr.runtime.tree.TreeNodeStream;
+import org.antlr.runtime.tree.Tree;
+
+/**
+ */
+public class MismatchedTreeNodeException extends RecognitionException {
+ public int expecting;
+
+ public MismatchedTreeNodeException() {
+ }
+
+ public MismatchedTreeNodeException(int expecting, TreeNodeStream input) {
+ super(input);
+ this.expecting = expecting;
+ }
+
+ public String toString() {
+ return "MismatchedTreeNodeException("+getUnexpectedType()+"!="+expecting+")";
+ }
+}
diff --git a/src/org/antlr/runtime/MissingTokenException.java b/src/org/antlr/runtime/MissingTokenException.java
new file mode 100755
index 0000000..9eda1f2
--- /dev/null
+++ b/src/org/antlr/runtime/MissingTokenException.java
@@ -0,0 +1,56 @@
+/*
+ [The "BSD license"]
+ Copyright (c) 2005-2009 Terence Parr
+ All rights reserved.
+
+ Redistribution and use in source and binary forms, with or without
+ modification, are permitted provided that the following conditions
+ are met:
+ 1. Redistributions of source code must retain the above copyright
+ notice, this list of conditions and the following disclaimer.
+ 2. Redistributions in binary form must reproduce the above copyright
+ notice, this list of conditions and the following disclaimer in the
+ documentation and/or other materials provided with the distribution.
+ 3. The name of the author may not be used to endorse or promote products
+ derived from this software without specific prior written permission.
+
+ THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
+ IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
+ OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
+ IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
+ INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
+ NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
+ THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+package org.antlr.runtime;
+
+/** We were expecting a token but it's not found. The current token
+ * is actually what we wanted next. Used for tree node errors too.
+ */
+public class MissingTokenException extends MismatchedTokenException {
+ public Object inserted;
+ /** Used for remote debugger deserialization */
+ public MissingTokenException() {;}
+
+ public MissingTokenException(int expecting, IntStream input, Object inserted) {
+ super(expecting, input);
+ this.inserted = inserted;
+ }
+
+ public int getMissingType() {
+ return expecting;
+ }
+
+ public String toString() {
+ if ( inserted!=null && token!=null ) {
+ return "MissingTokenException(inserted "+inserted+" at "+token.getText()+")";
+ }
+ if ( token!=null ) {
+ return "MissingTokenException(at "+token.getText()+")";
+ }
+ return "MissingTokenException";
+ }
+}
diff --git a/src/org/antlr/runtime/NoViableAltException.java b/src/org/antlr/runtime/NoViableAltException.java
new file mode 100755
index 0000000..889045f
--- /dev/null
+++ b/src/org/antlr/runtime/NoViableAltException.java
@@ -0,0 +1,57 @@
+/*
+ [The "BSD license"]
+ Copyright (c) 2005-2009 Terence Parr
+ All rights reserved.
+
+ Redistribution and use in source and binary forms, with or without
+ modification, are permitted provided that the following conditions
+ are met:
+ 1. Redistributions of source code must retain the above copyright
+ notice, this list of conditions and the following disclaimer.
+ 2. Redistributions in binary form must reproduce the above copyright
+ notice, this list of conditions and the following disclaimer in the
+ documentation and/or other materials provided with the distribution.
+ 3. The name of the author may not be used to endorse or promote products
+ derived from this software without specific prior written permission.
+
+ THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
+ IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
+ OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
+ IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
+ INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
+ NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
+ THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+package org.antlr.runtime;
+
+public class NoViableAltException extends RecognitionException {
+ public String grammarDecisionDescription;
+ public int decisionNumber;
+ public int stateNumber;
+
+ /** Used for remote debugger deserialization */
+ public NoViableAltException() {;}
+
+ public NoViableAltException(String grammarDecisionDescription,
+ int decisionNumber,
+ int stateNumber,
+ IntStream input)
+ {
+ super(input);
+ this.grammarDecisionDescription = grammarDecisionDescription;
+ this.decisionNumber = decisionNumber;
+ this.stateNumber = stateNumber;
+ }
+
+ public String toString() {
+ if ( input instanceof CharStream ) {
+ return "NoViableAltException('"+(char)getUnexpectedType()+"'@["+grammarDecisionDescription+"])";
+ }
+ else {
+ return "NoViableAltException("+getUnexpectedType()+"@["+grammarDecisionDescription+"])";
+ }
+ }
+}
diff --git a/src/org/antlr/runtime/Parser.java b/src/org/antlr/runtime/Parser.java
new file mode 100755
index 0000000..9cf937a
--- /dev/null
+++ b/src/org/antlr/runtime/Parser.java
@@ -0,0 +1,98 @@
+/*
+ [The "BSD license"]
+ Copyright (c) 2005-2009 Terence Parr
+ All rights reserved.
+
+ Redistribution and use in source and binary forms, with or without
+ modification, are permitted provided that the following conditions
+ are met:
+ 1. Redistributions of source code must retain the above copyright
+ notice, this list of conditions and the following disclaimer.
+ 2. Redistributions in binary form must reproduce the above copyright
+ notice, this list of conditions and the following disclaimer in the
+ documentation and/or other materials provided with the distribution.
+ 3. The name of the author may not be used to endorse or promote products
+ derived from this software without specific prior written permission.
+
+ THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
+ IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
+ OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
+ IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
+ INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
+ NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
+ THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+package org.antlr.runtime;
+
+/** A parser for TokenStreams. "parser grammars" result in a subclass
+ * of this.
+ */
+public class Parser extends BaseRecognizer {
+ public TokenStream input;
+
+ public Parser(TokenStream input) {
+ super(); // highlight that we go to super to set state object
+ setTokenStream(input);
+ }
+
+ public Parser(TokenStream input, RecognizerSharedState state) {
+ super(state); // share the state object with another parser
+ this.input = input;
+ }
+
+ public void reset() {
+ super.reset(); // reset all recognizer state variables
+ if ( input!=null ) {
+ input.seek(0); // rewind the input
+ }
+ }
+
+ protected Object getCurrentInputSymbol(IntStream input) {
+ return ((TokenStream)input).LT(1);
+ }
+
+ protected Object getMissingSymbol(IntStream input,
+ RecognitionException e,
+ int expectedTokenType,
+ BitSet follow)
+ {
+ String tokenText = null;
+ if ( expectedTokenType==Token.EOF ) tokenText = "<missing EOF>";
+ else tokenText = "<missing "+getTokenNames()[expectedTokenType]+">";
+ CommonToken t = new CommonToken(expectedTokenType, tokenText);
+ Token current = ((TokenStream)input).LT(1);
+ if ( current.getType() == Token.EOF ) {
+ current = ((TokenStream)input).LT(-1);
+ }
+ t.line = current.getLine();
+ t.charPositionInLine = current.getCharPositionInLine();
+ t.channel = DEFAULT_TOKEN_CHANNEL;
+ return t;
+ }
+
+ /** Set the token stream and reset the parser */
+ public void setTokenStream(TokenStream input) {
+ this.input = null;
+ reset();
+ this.input = input;
+ }
+
+ public TokenStream getTokenStream() {
+ return input;
+ }
+
+ public String getSourceName() {
+ return input.getSourceName();
+ }
+
+ public void traceIn(String ruleName, int ruleIndex) {
+ super.traceIn(ruleName, ruleIndex, input.LT(1));
+ }
+
+ public void traceOut(String ruleName, int ruleIndex) {
+ super.traceOut(ruleName, ruleIndex, input.LT(1));
+ }
+}
diff --git a/src/org/antlr/runtime/ParserRuleReturnScope.java b/src/org/antlr/runtime/ParserRuleReturnScope.java
new file mode 100755
index 0000000..9237e4e
--- /dev/null
+++ b/src/org/antlr/runtime/ParserRuleReturnScope.java
@@ -0,0 +1,52 @@
+/*
+ [The "BSD license"]
+ Copyright (c) 2005-2009 Terence Parr
+ All rights reserved.
+
+ Redistribution and use in source and binary forms, with or without
+ modification, are permitted provided that the following conditions
+ are met:
+ 1. Redistributions of source code must retain the above copyright
+ notice, this list of conditions and the following disclaimer.
+ 2. Redistributions in binary form must reproduce the above copyright
+ notice, this list of conditions and the following disclaimer in the
+ documentation and/or other materials provided with the distribution.
+ 3. The name of the author may not be used to endorse or promote products
+ derived from this software without specific prior written permission.
+
+ THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
+ IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
+ OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
+ IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
+ INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
+ NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
+ THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+package org.antlr.runtime;
+
+/** Rules that return more than a single value must return an object
+ * containing all the values. Besides the properties defined in
+ * RuleLabelScope.predefinedRulePropertiesScope there may be user-defined
+ * return values. This class simply defines the minimum properties that
+ * are always defined and methods to access the others that might be
+ * available depending on output option such as template and tree.
+ *
+ * Note text is not an actual property of the return value, it is computed
+ * from start and stop using the input stream's toString() method. I
+ * could add a ctor to this so that we can pass in and store the input
+ * stream, but I'm not sure we want to do that. It would seem to be undefined
+ * to get the .text property anyway if the rule matches tokens from multiple
+ * input streams.
+ *
+ * I do not use getters for fields of objects that are used simply to
+ * group values such as this aggregate. The getters/setters are there to
+ * satisfy the superclass interface.
+ */
+public class ParserRuleReturnScope extends RuleReturnScope {
+ public Token start, stop;
+ public Object getStart() { return start; }
+ public Object getStop() { return stop; }
+}
diff --git a/src/org/antlr/runtime/RecognitionException.java b/src/org/antlr/runtime/RecognitionException.java
new file mode 100755
index 0000000..3e79f99
--- /dev/null
+++ b/src/org/antlr/runtime/RecognitionException.java
@@ -0,0 +1,180 @@
+/*
+ [The "BSD license"]
+ Copyright (c) 2005-2009 Terence Parr
+ All rights reserved.
+
+ Redistribution and use in source and binary forms, with or without
+ modification, are permitted provided that the following conditions
+ are met:
+ 1. Redistributions of source code must retain the above copyright
+ notice, this list of conditions and the following disclaimer.
+ 2. Redistributions in binary form must reproduce the above copyright
+ notice, this list of conditions and the following disclaimer in the
+ documentation and/or other materials provided with the distribution.
+ 3. The name of the author may not be used to endorse or promote products
+ derived from this software without specific prior written permission.
+
+ THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
+ IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
+ OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
+ IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
+ INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
+ NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
+ THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+package org.antlr.runtime;
+
+import org.antlr.runtime.tree.*;
+
+/** The root of the ANTLR exception hierarchy.
+ *
+ * To avoid English-only error messages and to generally make things
+ * as flexible as possible, these exceptions are not created with strings,
+ * but rather the information necessary to generate an error. Then
+ * the various reporting methods in Parser and Lexer can be overridden
+ * to generate a localized error message. For example, MismatchedToken
+ * exceptions are built with the expected token type.
+ * So, don't expect getMessage() to return anything.
+ *
+ * Note that as of Java 1.4, you can access the stack trace, which means
+ * that you can compute the complete trace of rules from the start symbol.
+ * This gives you considerable context information with which to generate
+ * useful error messages.
+ *
+ * ANTLR generates code that throws exceptions upon recognition error and
+ * also generates code to catch these exceptions in each rule. If you
+ * want to quit upon first error, you can turn off the automatic error
+ * handling mechanism using rulecatch action, but you still need to
+ * override methods mismatch and recoverFromMismatchSet.
+ *
+ * In general, the recognition exceptions can track where in a grammar a
+ * problem occurred and/or what was the expected input. While the parser
+ * knows its state (such as current input symbol and line info) that
+ * state can change before the exception is reported so current token index
+ * is computed and stored at exception time. From this info, you can
+ * perhaps print an entire line of input not just a single token, for example.
+ * Better to just say the recognizer had a problem and then let the parser
+ * figure out a fancy report.
+ */
+public class RecognitionException extends Exception {
+ /** What input stream did the error occur in? */
+ public transient IntStream input;
+
+ /** What is index of token/char were we looking at when the error occurred? */
+ public int index;
+
+ /** The current Token when an error occurred. Since not all streams
+ * can retrieve the ith Token, we have to track the Token object.
+ * For parsers. Even when it's a tree parser, token might be set.
+ */
+ public Token token;
+
+ /** If this is a tree parser exception, node is set to the node with
+ * the problem.
+ */
+ public Object node;
+
+ /** The current char when an error occurred. For lexers. */
+ public int c;
+
+ /** Track the line at which the error occurred in case this is
+ * generated from a lexer. We need to track this since the
+ * unexpected char doesn't carry the line info.
+ */
+ public int line;
+
+ public int charPositionInLine;
+
+ /** If you are parsing a tree node stream, you will encounter som
+ * imaginary nodes w/o line/col info. We now search backwards looking
+ * for most recent token with line/col info, but notify getErrorHeader()
+ * that info is approximate.
+ */
+ public boolean approximateLineInfo;
+
+ /** Used for remote debugger deserialization */
+ public RecognitionException() {
+ }
+
+ public RecognitionException(IntStream input) {
+ this.input = input;
+ this.index = input.index();
+ if ( input instanceof TokenStream ) {
+ this.token = ((TokenStream)input).LT(1);
+ this.line = token.getLine();
+ this.charPositionInLine = token.getCharPositionInLine();
+ }
+ if ( input instanceof TreeNodeStream ) {
+ extractInformationFromTreeNodeStream(input);
+ }
+ else if ( input instanceof CharStream ) {
+ this.c = input.LA(1);
+ this.line = ((CharStream)input).getLine();
+ this.charPositionInLine = ((CharStream)input).getCharPositionInLine();
+ }
+ else {
+ this.c = input.LA(1);
+ }
+ }
+
+ protected void extractInformationFromTreeNodeStream(IntStream input) {
+ TreeNodeStream nodes = (TreeNodeStream)input;
+ this.node = nodes.LT(1);
+ TreeAdaptor adaptor = nodes.getTreeAdaptor();
+ Token payload = adaptor.getToken(node);
+ if ( payload!=null ) {
+ this.token = payload;
+ if ( payload.getLine()<= 0 ) {
+ // imaginary node; no line/pos info; scan backwards
+ int i = -1;
+ Object priorNode = nodes.LT(i);
+ while ( priorNode!=null ) {
+ Token priorPayload = adaptor.getToken(priorNode);
+ if ( priorPayload!=null && priorPayload.getLine()>0 ) {
+ // we found the most recent real line / pos info
+ this.line = priorPayload.getLine();
+ this.charPositionInLine = priorPayload.getCharPositionInLine();
+ this.approximateLineInfo = true;
+ break;
+ }
+ --i;
+ priorNode = nodes.LT(i);
+ }
+ }
+ else { // node created from real token
+ this.line = payload.getLine();
+ this.charPositionInLine = payload.getCharPositionInLine();
+ }
+ }
+ else if ( this.node instanceof Tree) {
+ this.line = ((Tree)this.node).getLine();
+ this.charPositionInLine = ((Tree)this.node).getCharPositionInLine();
+ if ( this.node instanceof CommonTree) {
+ this.token = ((CommonTree)this.node).token;
+ }
+ }
+ else {
+ int type = adaptor.getType(this.node);
+ String text = adaptor.getText(this.node);
+ this.token = new CommonToken(type, text);
+ }
+ }
+
+ /** Return the token type or char of the unexpected input element */
+ public int getUnexpectedType() {
+ if ( input instanceof TokenStream ) {
+ return token.getType();
+ }
+ else if ( input instanceof TreeNodeStream ) {
+ TreeNodeStream nodes = (TreeNodeStream)input;
+ TreeAdaptor adaptor = nodes.getTreeAdaptor();
+ return adaptor.getType(node);
+ }
+ else {
+ return c;
+ }
+ }
+}
diff --git a/src/org/antlr/runtime/RecognizerSharedState.java b/src/org/antlr/runtime/RecognizerSharedState.java
new file mode 100755
index 0000000..068ac3b
--- /dev/null
+++ b/src/org/antlr/runtime/RecognizerSharedState.java
@@ -0,0 +1,144 @@
+/*
+ [The "BSD license"]
+ Copyright (c) 2005-2009 Terence Parr
+ All rights reserved.
+
+ Redistribution and use in source and binary forms, with or without
+ modification, are permitted provided that the following conditions
+ are met:
+ 1. Redistributions of source code must retain the above copyright
+ notice, this list of conditions and the following disclaimer.
+ 2. Redistributions in binary form must reproduce the above copyright
+ notice, this list of conditions and the following disclaimer in the
+ documentation and/or other materials provided with the distribution.
+ 3. The name of the author may not be used to endorse or promote products
+ derived from this software without specific prior written permission.
+
+ THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
+ IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
+ OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
+ IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
+ INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
+ NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
+ THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */package org.antlr.runtime;
+
+import java.util.Map;
+
+/** The set of fields needed by an abstract recognizer to recognize input
+ * and recover from errors etc... As a separate state object, it can be
+ * shared among multiple grammars; e.g., when one grammar imports another.
+ *
+ * These fields are publically visible but the actual state pointer per
+ * parser is protected.
+ */
+public class RecognizerSharedState {
+ /** Track the set of token types that can follow any rule invocation.
+ * Stack grows upwards. When it hits the max, it grows 2x in size
+ * and keeps going.
+ */
+ public BitSet[] following = new BitSet[BaseRecognizer.INITIAL_FOLLOW_STACK_SIZE];
+ public int _fsp = -1;
+
+ /** This is true when we see an error and before having successfully
+ * matched a token. Prevents generation of more than one error message
+ * per error.
+ */
+ public boolean errorRecovery = false;
+
+ /** The index into the input stream where the last error occurred.
+ * This is used to prevent infinite loops where an error is found
+ * but no token is consumed during recovery...another error is found,
+ * ad naseum. This is a failsafe mechanism to guarantee that at least
+ * one token/tree node is consumed for two errors.
+ */
+ public int lastErrorIndex = -1;
+
+ /** In lieu of a return value, this indicates that a rule or token
+ * has failed to match. Reset to false upon valid token match.
+ */
+ public boolean failed = false;
+
+ /** Did the recognizer encounter a syntax error? Track how many. */
+ public int syntaxErrors = 0;
+
+ /** If 0, no backtracking is going on. Safe to exec actions etc...
+ * If >0 then it's the level of backtracking.
+ */
+ public int backtracking = 0;
+
+ /** An array[size num rules] of Map<Integer,Integer> that tracks
+ * the stop token index for each rule. ruleMemo[ruleIndex] is
+ * the memoization table for ruleIndex. For key ruleStartIndex, you
+ * get back the stop token for associated rule or MEMO_RULE_FAILED.
+ *
+ * This is only used if rule memoization is on (which it is by default).
+ */
+ public Map[] ruleMemo;
+
+
+ // LEXER FIELDS (must be in same state object to avoid casting
+ // constantly in generated code and Lexer object) :(
+
+
+ /** The goal of all lexer rules/methods is to create a token object.
+ * This is an instance variable as multiple rules may collaborate to
+ * create a single token. nextToken will return this object after
+ * matching lexer rule(s). If you subclass to allow multiple token
+ * emissions, then set this to the last token to be matched or
+ * something nonnull so that the auto token emit mechanism will not
+ * emit another token.
+ */
+ public Token token;
+
+ /** What character index in the stream did the current token start at?
+ * Needed, for example, to get the text for current token. Set at
+ * the start of nextToken.
+ */
+ public int tokenStartCharIndex = -1;
+
+ /** The line on which the first character of the token resides */
+ public int tokenStartLine;
+
+ /** The character position of first character within the line */
+ public int tokenStartCharPositionInLine;
+
+ /** The channel number for the current token */
+ public int channel;
+
+ /** The token type for the current token */
+ public int type;
+
+ /** You can set the text for the current token to override what is in
+ * the input char buffer. Use setText() or can set this instance var.
+ */
+ public String text;
+
+ public RecognizerSharedState() {;}
+
+ public RecognizerSharedState(RecognizerSharedState state) {
+ if ( this.following.length < state.following.length ) {
+ this.following = new BitSet[state.following.length];
+ }
+ System.arraycopy(state.following, 0, this.following, 0, state.following.length);
+ this._fsp = state._fsp;
+ this.errorRecovery = state.errorRecovery;
+ this.lastErrorIndex = state.lastErrorIndex;
+ this.failed = state.failed;
+ this.syntaxErrors = state.syntaxErrors;
+ this.backtracking = state.backtracking;
+ if ( state.ruleMemo!=null ) {
+ this.ruleMemo = new Map[state.ruleMemo.length];
+ System.arraycopy(state.ruleMemo, 0, this.ruleMemo, 0, state.ruleMemo.length);
+ }
+ this.token = state.token;
+ this.tokenStartCharIndex = state.tokenStartCharIndex;
+ this.tokenStartCharPositionInLine = state.tokenStartCharPositionInLine;
+ this.channel = state.channel;
+ this.type = state.type;
+ this.text = state.text;
+ }
+}
diff --git a/src/org/antlr/runtime/RuleReturnScope.java b/src/org/antlr/runtime/RuleReturnScope.java
new file mode 100755
index 0000000..85702ca
--- /dev/null
+++ b/src/org/antlr/runtime/RuleReturnScope.java
@@ -0,0 +1,42 @@
+/*
+ [The "BSD license"]
+ Copyright (c) 2005-2009 Terence Parr
+ All rights reserved.
+
+ Redistribution and use in source and binary forms, with or without
+ modification, are permitted provided that the following conditions
+ are met:
+ 1. Redistributions of source code must retain the above copyright
+ notice, this list of conditions and the following disclaimer.
+ 2. Redistributions in binary form must reproduce the above copyright
+ notice, this list of conditions and the following disclaimer in the
+ documentation and/or other materials provided with the distribution.
+ 3. The name of the author may not be used to endorse or promote products
+ derived from this software without specific prior written permission.
+
+ THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
+ IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
+ OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
+ IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
+ INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
+ NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
+ THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+package org.antlr.runtime;
+
+/** Rules can return start/stop info as well as possible trees and templates */
+public class RuleReturnScope {
+ /** Return the start token or tree */
+ public Object getStart() { return null; }
+ /** Return the stop token or tree */
+ public Object getStop() { return null; }
+ /** Has a value potentially if output=AST; */
+ public Object getTree() { return null; }
+ /** Has a value potentially if output=template; Don't use StringTemplate
+ * type as it then causes a dependency with ST lib.
+ */
+ public Object getTemplate() { return null; }
+}
diff --git a/src/org/antlr/runtime/SerializedGrammar.java b/src/org/antlr/runtime/SerializedGrammar.java
new file mode 100755
index 0000000..a609053
--- /dev/null
+++ b/src/org/antlr/runtime/SerializedGrammar.java
@@ -0,0 +1,198 @@
+/*
+ [The "BSD license"]
+ Copyright (c) 2005-2009 Terence Parr
+ All rights reserved.
+
+ Redistribution and use in source and binary forms, with or without
+ modification, are permitted provided that the following conditions
+ are met:
+ 1. Redistributions of source code must retain the above copyright
+ notice, this list of conditions and the following disclaimer.
+ 2. Redistributions in binary form must reproduce the above copyright
+ notice, this list of conditions and the following disclaimer in the
+ documentation and/or other materials provided with the distribution.
+ 3. The name of the author may not be used to endorse or promote products
+ derived from this software without specific prior written permission.
+
+ THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
+ IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
+ OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
+ IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
+ INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
+ NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
+ THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+package org.antlr.runtime;
+
+import java.io.IOException;
+import java.io.FileInputStream;
+import java.io.BufferedInputStream;
+import java.io.DataInputStream;
+import java.util.List;
+import java.util.ArrayList;
+
+public class SerializedGrammar {
+ public static final String COOKIE = "$ANTLR";
+ public static final int FORMAT_VERSION = 1;
+ //public static org.antlr.tool.Grammar gr; // TESTING ONLY; remove later
+
+ public String name;
+ public char type; // in {l, p, t, c}
+ public List rules;
+
+ class Rule {
+ String name;
+ Block block;
+ public Rule(String name, Block block) {
+ this.name = name;
+ this.block = block;
+ }
+ public String toString() {
+ return name+":"+block;
+ }
+ }
+
+ class Block {
+ List[] alts;
+ public Block(List[] alts) {
+ this.alts = alts;
+ }
+ public String toString() {
+ StringBuffer buf = new StringBuffer();
+ buf.append("(");
+ for (int i = 0; i < alts.length; i++) {
+ List alt = alts[i];
+ if ( i>0 ) buf.append("|");
+ buf.append(alt.toString());
+ }
+ buf.append(")");
+ return buf.toString();
+ }
+ }
+
+ class TokenRef {
+ int ttype;
+ public TokenRef(int ttype) { this.ttype = ttype; }
+ public String toString() { return String.valueOf(ttype); }
+ }
+
+ class RuleRef {
+ int ruleIndex;
+ public RuleRef(int ruleIndex) { this.ruleIndex = ruleIndex; }
+ public String toString() { return String.valueOf(ruleIndex); }
+ }
+
+ public SerializedGrammar(String filename) throws IOException {
+ System.out.println("loading "+filename);
+ FileInputStream fis = new FileInputStream(filename);
+ BufferedInputStream bos = new BufferedInputStream(fis);
+ DataInputStream in = new DataInputStream(bos);
+ readFile(in);
+ in.close();
+ }
+
+ protected void readFile(DataInputStream in) throws IOException {
+ String cookie = readString(in); // get $ANTLR
+ if ( !cookie.equals(COOKIE) ) throw new IOException("not a serialized grammar file");
+ int version = in.readByte();
+ char grammarType = (char)in.readByte();
+ this.type = grammarType;
+ String grammarName = readString(in);
+ this.name = grammarName;
+ System.out.println(grammarType+" grammar "+grammarName);
+ int numRules = in.readShort();
+ System.out.println("num rules = "+numRules);
+ rules = readRules(in, numRules);
+ }
+
+ protected List readRules(DataInputStream in, int numRules) throws IOException {
+ List rules = new ArrayList();
+ for (int i=0; i<numRules; i++) {
+ Rule r = readRule(in);
+ rules.add(r);
+ }
+ return rules;
+ }
+
+ protected Rule readRule(DataInputStream in) throws IOException {
+ byte R = in.readByte();
+ if ( R!='R' ) throw new IOException("missing R on start of rule");
+ String name = readString(in);
+ System.out.println("rule: "+name);
+ byte B = in.readByte();
+ Block b = readBlock(in);
+ byte period = in.readByte();
+ if ( period!='.' ) throw new IOException("missing . on end of rule");
+ return new Rule(name, b);
+ }
+
+ protected Block readBlock(DataInputStream in) throws IOException {
+ int nalts = in.readShort();
+ List[] alts = new List[nalts];
+ //System.out.println("enter block n="+nalts);
+ for (int i=0; i<nalts; i++) {
+ List alt = readAlt(in);
+ alts[i] = alt;
+ }
+ //System.out.println("exit block");
+ return new Block(alts);
+ }
+
+ protected List readAlt(DataInputStream in) throws IOException {
+ List alt = new ArrayList();
+ byte A = in.readByte();
+ if ( A!='A' ) throw new IOException("missing A on start of alt");
+ byte cmd = in.readByte();
+ while ( cmd!=';' ) {
+ switch (cmd) {
+ case 't' :
+ int ttype = in.readShort();
+ alt.add(new TokenRef(ttype));
+ //System.out.println("read token "+gr.getTokenDisplayName(ttype));
+ break;
+ case 'r' :
+ int ruleIndex = in.readShort();
+ alt.add(new RuleRef(ruleIndex));
+ //System.out.println("read rule "+gr.getRuleName(ruleIndex));
+ break;
+ case '.' : // wildcard
+ break;
+ case '-' : // range
+ int from = in.readChar();
+ int to = in.readChar();
+ break;
+ case '~' : // not
+ int notThisTokenType = in.readShort();
+ break;
+ case 'B' : // nested block
+ Block b = readBlock(in);
+ alt.add(b);
+ break;
+ }
+ cmd = in.readByte();
+ }
+ //System.out.println("exit alt");
+ return alt;
+ }
+
+ protected String readString(DataInputStream in) throws IOException {
+ byte c = in.readByte();
+ StringBuffer buf = new StringBuffer();
+ while ( c!=';' ) {
+ buf.append((char)c);
+ c = in.readByte();
+ }
+ return buf.toString();
+ }
+
+ public String toString() {
+ StringBuffer buf = new StringBuffer();
+ buf.append(type+" grammar "+name);
+ buf.append(rules);
+ return buf.toString();
+ }
+}
diff --git a/src/org/antlr/runtime/Token.java b/src/org/antlr/runtime/Token.java
new file mode 100755
index 0000000..b8eb95e
--- /dev/null
+++ b/src/org/antlr/runtime/Token.java
@@ -0,0 +1,92 @@
+/*
+ [The "BSD license"]
+ Copyright (c) 2005-2009 Terence Parr
+ All rights reserved.
+
+ Redistribution and use in source and binary forms, with or without
+ modification, are permitted provided that the following conditions
+ are met:
+ 1. Redistributions of source code must retain the above copyright
+ notice, this list of conditions and the following disclaimer.
+ 2. Redistributions in binary form must reproduce the above copyright
+ notice, this list of conditions and the following disclaimer in the
+ documentation and/or other materials provided with the distribution.
+ 3. The name of the author may not be used to endorse or promote products
+ derived from this software without specific prior written permission.
+
+ THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
+ IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
+ OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
+ IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
+ INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
+ NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
+ THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+package org.antlr.runtime;
+
+public interface Token {
+ public static final int EOR_TOKEN_TYPE = 1;
+
+ /** imaginary tree navigation type; traverse "get child" link */
+ public static final int DOWN = 2;
+ /** imaginary tree navigation type; finish with a child list */
+ public static final int UP = 3;
+
+ public static final int MIN_TOKEN_TYPE = UP+1;
+
+ public static final int EOF = CharStream.EOF;
+ // TODO: remove once we go ANTLR v3.3
+ public static final Token EOF_TOKEN = new CommonToken(EOF);
+
+ public static final int INVALID_TOKEN_TYPE = 0;
+ public static final Token INVALID_TOKEN = new CommonToken(INVALID_TOKEN_TYPE);
+
+ /** In an action, a lexer rule can set token to this SKIP_TOKEN and ANTLR
+ * will avoid creating a token for this symbol and try to fetch another.
+ */
+ public static final Token SKIP_TOKEN = new CommonToken(INVALID_TOKEN_TYPE);
+
+ /** All tokens go to the parser (unless skip() is called in that rule)
+ * on a particular "channel". The parser tunes to a particular channel
+ * so that whitespace etc... can go to the parser on a "hidden" channel.
+ */
+ public static final int DEFAULT_CHANNEL = 0;
+
+ /** Anything on different channel than DEFAULT_CHANNEL is not parsed
+ * by parser.
+ */
+ public static final int HIDDEN_CHANNEL = 99;
+
+ /** Get the text of the token */
+ public String getText();
+ public void setText(String text);
+
+ public int getType();
+ public void setType(int ttype);
+ /** The line number on which this token was matched; line=1..n */
+ public int getLine();
+ public void setLine(int line);
+
+ /** The index of the first character relative to the beginning of the line 0..n-1 */
+ public int getCharPositionInLine();
+ public void setCharPositionInLine(int pos);
+
+ public int getChannel();
+ public void setChannel(int channel);
+
+ /** An index from 0..n-1 of the token object in the input stream.
+ * This must be valid in order to use the ANTLRWorks debugger.
+ */
+ public int getTokenIndex();
+ public void setTokenIndex(int index);
+
+ /** From what character stream was this token created? You don't have to
+ * implement but it's nice to know where a Token comes from if you have
+ * include files etc... on the input.
+ */
+ public CharStream getInputStream();
+ public void setInputStream(CharStream input);
+}
diff --git a/src/org/antlr/runtime/TokenRewriteStream.java b/src/org/antlr/runtime/TokenRewriteStream.java
new file mode 100755
index 0000000..8437441
--- /dev/null
+++ b/src/org/antlr/runtime/TokenRewriteStream.java
@@ -0,0 +1,590 @@
+/*
+ [The "BSD license"]
+ Copyright (c) 2005-2009 Terence Parr
+ All rights reserved.
+
+ Redistribution and use in source and binary forms, with or without
+ modification, are permitted provided that the following conditions
+ are met:
+ 1. Redistributions of source code must retain the above copyright
+ notice, this list of conditions and the following disclaimer.
+ 2. Redistributions in binary form must reproduce the above copyright
+ notice, this list of conditions and the following disclaimer in the
+ documentation and/or other materials provided with the distribution.
+ 3. The name of the author may not be used to endorse or promote products
+ derived from this software without specific prior written permission.
+
+ THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
+ IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
+ OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
+ IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
+ INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
+ NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
+ THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+package org.antlr.runtime;
+
+import java.util.*;
+
+/** Useful for dumping out the input stream after doing some
+ * augmentation or other manipulations.
+ *
+ * You can insert stuff, replace, and delete chunks. Note that the
+ * operations are done lazily--only if you convert the buffer to a
+ * String. This is very efficient because you are not moving data around
+ * all the time. As the buffer of tokens is converted to strings, the
+ * toString() method(s) check to see if there is an operation at the
+ * current index. If so, the operation is done and then normal String
+ * rendering continues on the buffer. This is like having multiple Turing
+ * machine instruction streams (programs) operating on a single input tape. :)
+ *
+ * Since the operations are done lazily at toString-time, operations do not
+ * screw up the token index values. That is, an insert operation at token
+ * index i does not change the index values for tokens i+1..n-1.
+ *
+ * Because operations never actually alter the buffer, you may always get
+ * the original token stream back without undoing anything. Since
+ * the instructions are queued up, you can easily simulate transactions and
+ * roll back any changes if there is an error just by removing instructions.
+ * For example,
+ *
+ * CharStream input = new ANTLRFileStream("input");
+ * TLexer lex = new TLexer(input);
+ * TokenRewriteStream tokens = new TokenRewriteStream(lex);
+ * T parser = new T(tokens);
+ * parser.startRule();
+ *
+ * Then in the rules, you can execute
+ * Token t,u;
+ * ...
+ * input.insertAfter(t, "text to put after t");}
+ * input.insertAfter(u, "text after u");}
+ * System.out.println(tokens.toString());
+ *
+ * Actually, you have to cast the 'input' to a TokenRewriteStream. :(
+ *
+ * You can also have multiple "instruction streams" and get multiple
+ * rewrites from a single pass over the input. Just name the instruction
+ * streams and use that name again when printing the buffer. This could be
+ * useful for generating a C file and also its header file--all from the
+ * same buffer:
+ *
+ * tokens.insertAfter("pass1", t, "text to put after t");}
+ * tokens.insertAfter("pass2", u, "text after u");}
+ * System.out.println(tokens.toString("pass1"));
+ * System.out.println(tokens.toString("pass2"));
+ *
+ * If you don't use named rewrite streams, a "default" stream is used as
+ * the first example shows.
+ */
+public class TokenRewriteStream extends CommonTokenStream {
+ public static final String DEFAULT_PROGRAM_NAME = "default";
+ public static final int PROGRAM_INIT_SIZE = 100;
+ public static final int MIN_TOKEN_INDEX = 0;
+
+ // Define the rewrite operation hierarchy
+
+ class RewriteOperation {
+ /** What index into rewrites List are we? */
+ protected int instructionIndex;
+ /** Token buffer index. */
+ protected int index;
+ protected Object text;
+
+ protected RewriteOperation(int index) {
+ this.index = index;
+ }
+
+ protected RewriteOperation(int index, Object text) {
+ this.index = index;
+ this.text = text;
+ }
+ /** Execute the rewrite operation by possibly adding to the buffer.
+ * Return the index of the next token to operate on.
+ */
+ public int execute(StringBuffer buf) {
+ return index;
+ }
+ public String toString() {
+ String opName = getClass().getName();
+ int $index = opName.indexOf('$');
+ opName = opName.substring($index+1, opName.length());
+ return "<"+opName+"@"+tokens.get(index)+
+ ":\""+text+"\">";
+ }
+ }
+
+ class InsertBeforeOp extends RewriteOperation {
+ public InsertBeforeOp(int index, Object text) {
+ super(index,text);
+ }
+ public int execute(StringBuffer buf) {
+ buf.append(text);
+ if ( tokens.get(index).getType()!=Token.EOF ) {
+ buf.append(tokens.get(index).getText());
+ }
+ return index+1;
+ }
+ }
+
+ /** I'm going to try replacing range from x..y with (y-x)+1 ReplaceOp
+ * instructions.
+ */
+ class ReplaceOp extends RewriteOperation {
+ protected int lastIndex;
+ public ReplaceOp(int from, int to, Object text) {
+ super(from,text);
+ lastIndex = to;
+ }
+ public int execute(StringBuffer buf) {
+ if ( text!=null ) {
+ buf.append(text);
+ }
+ return lastIndex+1;
+ }
+ public String toString() {
+ if ( text==null ) {
+ return "<DeleteOp@"+tokens.get(index)+
+ ".."+tokens.get(lastIndex)+">";
+ }
+ return "<ReplaceOp@"+tokens.get(index)+
+ ".."+tokens.get(lastIndex)+":\""+text+"\">";
+ }
+ }
+
+ /** You may have multiple, named streams of rewrite operations.
+ * I'm calling these things "programs."
+ * Maps String (name) -> rewrite (List)
+ */
+ protected Map programs = null;
+
+ /** Map String (program name) -> Integer index */
+ protected Map lastRewriteTokenIndexes = null;
+
+ public TokenRewriteStream() {
+ init();
+ }
+
+ protected void init() {
+ programs = new HashMap();
+ programs.put(DEFAULT_PROGRAM_NAME, new ArrayList(PROGRAM_INIT_SIZE));
+ lastRewriteTokenIndexes = new HashMap();
+ }
+
+ public TokenRewriteStream(TokenSource tokenSource) {
+ super(tokenSource);
+ init();
+ }
+
+ public TokenRewriteStream(TokenSource tokenSource, int channel) {
+ super(tokenSource, channel);
+ init();
+ }
+
+ public void rollback(int instructionIndex) {
+ rollback(DEFAULT_PROGRAM_NAME, instructionIndex);
+ }
+
+ /** Rollback the instruction stream for a program so that
+ * the indicated instruction (via instructionIndex) is no
+ * longer in the stream. UNTESTED!
+ */
+ public void rollback(String programName, int instructionIndex) {
+ List is = (List)programs.get(programName);
+ if ( is!=null ) {
+ programs.put(programName, is.subList(MIN_TOKEN_INDEX,instructionIndex));
+ }
+ }
+
+ public void deleteProgram() {
+ deleteProgram(DEFAULT_PROGRAM_NAME);
+ }
+
+ /** Reset the program so that no instructions exist */
+ public void deleteProgram(String programName) {
+ rollback(programName, MIN_TOKEN_INDEX);
+ }
+
+ public void insertAfter(Token t, Object text) {
+ insertAfter(DEFAULT_PROGRAM_NAME, t, text);
+ }
+
+ public void insertAfter(int index, Object text) {
+ insertAfter(DEFAULT_PROGRAM_NAME, index, text);
+ }
+
+ public void insertAfter(String programName, Token t, Object text) {
+ insertAfter(programName,t.getTokenIndex(), text);
+ }
+
+ public void insertAfter(String programName, int index, Object text) {
+ // to insert after, just insert before next index (even if past end)
+ insertBefore(programName,index+1, text);
+ }
+
+ public void insertBefore(Token t, Object text) {
+ insertBefore(DEFAULT_PROGRAM_NAME, t, text);
+ }
+
+ public void insertBefore(int index, Object text) {
+ insertBefore(DEFAULT_PROGRAM_NAME, index, text);
+ }
+
+ public void insertBefore(String programName, Token t, Object text) {
+ insertBefore(programName, t.getTokenIndex(), text);
+ }
+
+ public void insertBefore(String programName, int index, Object text) {
+ RewriteOperation op = new InsertBeforeOp(index,text);
+ List rewrites = getProgram(programName);
+ op.instructionIndex = rewrites.size();
+ rewrites.add(op);
+ }
+
+ public void replace(int index, Object text) {
+ replace(DEFAULT_PROGRAM_NAME, index, index, text);
+ }
+
+ public void replace(int from, int to, Object text) {
+ replace(DEFAULT_PROGRAM_NAME, from, to, text);
+ }
+
+ public void replace(Token indexT, Object text) {
+ replace(DEFAULT_PROGRAM_NAME, indexT, indexT, text);
+ }
+
+ public void replace(Token from, Token to, Object text) {
+ replace(DEFAULT_PROGRAM_NAME, from, to, text);
+ }
+
+ public void replace(String programName, int from, int to, Object text) {
+ if ( from > to || from<0 || to<0 || to >= tokens.size() ) {
+ throw new IllegalArgumentException("replace: range invalid: "+from+".."+to+"(size="+tokens.size()+")");
+ }
+ RewriteOperation op = new ReplaceOp(from, to, text);
+ List rewrites = getProgram(programName);
+ op.instructionIndex = rewrites.size();
+ rewrites.add(op);
+ }
+
+ public void replace(String programName, Token from, Token to, Object text) {
+ replace(programName,
+ from.getTokenIndex(),
+ to.getTokenIndex(),
+ text);
+ }
+
+ public void delete(int index) {
+ delete(DEFAULT_PROGRAM_NAME, index, index);
+ }
+
+ public void delete(int from, int to) {
+ delete(DEFAULT_PROGRAM_NAME, from, to);
+ }
+
+ public void delete(Token indexT) {
+ delete(DEFAULT_PROGRAM_NAME, indexT, indexT);
+ }
+
+ public void delete(Token from, Token to) {
+ delete(DEFAULT_PROGRAM_NAME, from, to);
+ }
+
+ public void delete(String programName, int from, int to) {
+ replace(programName,from,to,null);
+ }
+
+ public void delete(String programName, Token from, Token to) {
+ replace(programName,from,to,null);
+ }
+
+ public int getLastRewriteTokenIndex() {
+ return getLastRewriteTokenIndex(DEFAULT_PROGRAM_NAME);
+ }
+
+ protected int getLastRewriteTokenIndex(String programName) {
+ Integer I = (Integer)lastRewriteTokenIndexes.get(programName);
+ if ( I==null ) {
+ return -1;
+ }
+ return I.intValue();
+ }
+
+ protected void setLastRewriteTokenIndex(String programName, int i) {
+ lastRewriteTokenIndexes.put(programName, new Integer(i));
+ }
+
+ protected List getProgram(String name) {
+ List is = (List)programs.get(name);
+ if ( is==null ) {
+ is = initializeProgram(name);
+ }
+ return is;
+ }
+
+ private List initializeProgram(String name) {
+ List is = new ArrayList(PROGRAM_INIT_SIZE);
+ programs.put(name, is);
+ return is;
+ }
+
+ public String toOriginalString() {
+ fill();
+ return toOriginalString(MIN_TOKEN_INDEX, size()-1);
+ }
+
+ public String toOriginalString(int start, int end) {
+ StringBuffer buf = new StringBuffer();
+ for (int i=start; i>=MIN_TOKEN_INDEX && i<=end && i<tokens.size(); i++) {
+ if ( get(i).getType()!=Token.EOF ) buf.append(get(i).getText());
+ }
+ return buf.toString();
+ }
+
+ public String toString() {
+ fill();
+ return toString(MIN_TOKEN_INDEX, size()-1);
+ }
+
+ public String toString(String programName) {
+ fill();
+ return toString(programName, MIN_TOKEN_INDEX, size()-1);
+ }
+
+ public String toString(int start, int end) {
+ return toString(DEFAULT_PROGRAM_NAME, start, end);
+ }
+
+ public String toString(String programName, int start, int end) {
+ List rewrites = (List)programs.get(programName);
+
+ // ensure start/end are in range
+ if ( end>tokens.size()-1 ) end = tokens.size()-1;
+ if ( start<0 ) start = 0;
+
+ if ( rewrites==null || rewrites.size()==0 ) {
+ return toOriginalString(start,end); // no instructions to execute
+ }
+ StringBuffer buf = new StringBuffer();
+
+ // First, optimize instruction stream
+ Map indexToOp = reduceToSingleOperationPerIndex(rewrites);
+
+ // Walk buffer, executing instructions and emitting tokens
+ int i = start;
+ while ( i <= end && i < tokens.size() ) {
+ RewriteOperation op = (RewriteOperation)indexToOp.get(new Integer(i));
+ indexToOp.remove(new Integer(i)); // remove so any left have index size-1
+ Token t = (Token) tokens.get(i);
+ if ( op==null ) {
+ // no operation at that index, just dump token
+ if ( t.getType()!=Token.EOF ) buf.append(t.getText());
+ i++; // move to next token
+ }
+ else {
+ i = op.execute(buf); // execute operation and skip
+ }
+ }
+
+ // include stuff after end if it's last index in buffer
+ // So, if they did an insertAfter(lastValidIndex, "foo"), include
+ // foo if end==lastValidIndex.
+ if ( end==tokens.size()-1 ) {
+ // Scan any remaining operations after last token
+ // should be included (they will be inserts).
+ Iterator it = indexToOp.values().iterator();
+ while (it.hasNext()) {
+ RewriteOperation op = (RewriteOperation)it.next();
+ if ( op.index >= tokens.size()-1 ) buf.append(op.text);
+ }
+ }
+ return buf.toString();
+ }
+
+ /** We need to combine operations and report invalid operations (like
+ * overlapping replaces that are not completed nested). Inserts to
+ * same index need to be combined etc... Here are the cases:
+ *
+ * I.i.u I.j.v leave alone, nonoverlapping
+ * I.i.u I.i.v combine: Iivu
+ *
+ * R.i-j.u R.x-y.v | i-j in x-y delete first R
+ * R.i-j.u R.i-j.v delete first R
+ * R.i-j.u R.x-y.v | x-y in i-j ERROR
+ * R.i-j.u R.x-y.v | boundaries overlap ERROR
+ *
+ * Delete special case of replace (text==null):
+ * D.i-j.u D.x-y.v | boundaries overlap combine to max(min)..max(right)
+ *
+ * I.i.u R.x-y.v | i in (x+1)-y delete I (since insert before
+ * we're not deleting i)
+ * I.i.u R.x-y.v | i not in (x+1)-y leave alone, nonoverlapping
+ * R.x-y.v I.i.u | i in x-y ERROR
+ * R.x-y.v I.x.u R.x-y.uv (combine, delete I)
+ * R.x-y.v I.i.u | i not in x-y leave alone, nonoverlapping
+ *
+ * I.i.u = insert u before op @ index i
+ * R.x-y.u = replace x-y indexed tokens with u
+ *
+ * First we need to examine replaces. For any replace op:
+ *
+ * 1. wipe out any insertions before op within that range.
+ * 2. Drop any replace op before that is contained completely within
+ * that range.
+ * 3. Throw exception upon boundary overlap with any previous replace.
+ *
+ * Then we can deal with inserts:
+ *
+ * 1. for any inserts to same index, combine even if not adjacent.
+ * 2. for any prior replace with same left boundary, combine this
+ * insert with replace and delete this replace.
+ * 3. throw exception if index in same range as previous replace
+ *
+ * Don't actually delete; make op null in list. Easier to walk list.
+ * Later we can throw as we add to index -> op map.
+ *
+ * Note that I.2 R.2-2 will wipe out I.2 even though, technically, the
+ * inserted stuff would be before the replace range. But, if you
+ * add tokens in front of a method body '{' and then delete the method
+ * body, I think the stuff before the '{' you added should disappear too.
+ *
+ * Return a map from token index to operation.
+ */
+ protected Map reduceToSingleOperationPerIndex(List rewrites) {
+// System.out.println("rewrites="+rewrites);
+
+ // WALK REPLACES
+ for (int i = 0; i < rewrites.size(); i++) {
+ RewriteOperation op = (RewriteOperation)rewrites.get(i);
+ if ( op==null ) continue;
+ if ( !(op instanceof ReplaceOp) ) continue;
+ ReplaceOp rop = (ReplaceOp)rewrites.get(i);
+ // Wipe prior inserts within range
+ List inserts = getKindOfOps(rewrites, InsertBeforeOp.class, i);
+ for (int j = 0; j < inserts.size(); j++) {
+ InsertBeforeOp iop = (InsertBeforeOp) inserts.get(j);
+ if ( iop.index == rop.index ) {
+ // E.g., insert before 2, delete 2..2; update replace
+ // text to include insert before, kill insert
+ rewrites.set(iop.instructionIndex, null);
+ rop.text = iop.text.toString() + (rop.text!=null?rop.text.toString():"");
+ }
+ else if ( iop.index > rop.index && iop.index <= rop.lastIndex ) {
+ // delete insert as it's a no-op.
+ rewrites.set(iop.instructionIndex, null);
+ }
+ }
+ // Drop any prior replaces contained within
+ List prevReplaces = getKindOfOps(rewrites, ReplaceOp.class, i);
+ for (int j = 0; j < prevReplaces.size(); j++) {
+ ReplaceOp prevRop = (ReplaceOp) prevReplaces.get(j);
+ if ( prevRop.index>=rop.index && prevRop.lastIndex <= rop.lastIndex ) {
+ // delete replace as it's a no-op.
+ rewrites.set(prevRop.instructionIndex, null);
+ continue;
+ }
+ // throw exception unless disjoint or identical
+ boolean disjoint =
+ prevRop.lastIndex<rop.index || prevRop.index > rop.lastIndex;
+ boolean same =
+ prevRop.index==rop.index && prevRop.lastIndex==rop.lastIndex;
+ // Delete special case of replace (text==null):
+ // D.i-j.u D.x-y.v | boundaries overlap combine to max(min)..max(right)
+ if ( prevRop.text==null && rop.text==null && !disjoint ) {
+ //System.out.println("overlapping deletes: "+prevRop+", "+rop);
+ rewrites.set(prevRop.instructionIndex, null); // kill first delete
+ rop.index = Math.min(prevRop.index, rop.index);
+ rop.lastIndex = Math.max(prevRop.lastIndex, rop.lastIndex);
+ System.out.println("new rop "+rop);
+ }
+ else if ( !disjoint && !same ) {
+ throw new IllegalArgumentException("replace op boundaries of "+rop+
+ " overlap with previous "+prevRop);
+ }
+ }
+ }
+
+ // WALK INSERTS
+ for (int i = 0; i < rewrites.size(); i++) {
+ RewriteOperation op = (RewriteOperation)rewrites.get(i);
+ if ( op==null ) continue;
+ if ( !(op instanceof InsertBeforeOp) ) continue;
+ InsertBeforeOp iop = (InsertBeforeOp)rewrites.get(i);
+ // combine current insert with prior if any at same index
+ List prevInserts = getKindOfOps(rewrites, InsertBeforeOp.class, i);
+ for (int j = 0; j < prevInserts.size(); j++) {
+ InsertBeforeOp prevIop = (InsertBeforeOp) prevInserts.get(j);
+ if ( prevIop.index == iop.index ) { // combine objects
+ // convert to strings...we're in process of toString'ing
+ // whole token buffer so no lazy eval issue with any templates
+ iop.text = catOpText(iop.text,prevIop.text);
+ // delete redundant prior insert
+ rewrites.set(prevIop.instructionIndex, null);
+ }
+ }
+ // look for replaces where iop.index is in range; error
+ List prevReplaces = getKindOfOps(rewrites, ReplaceOp.class, i);
+ for (int j = 0; j < prevReplaces.size(); j++) {
+ ReplaceOp rop = (ReplaceOp) prevReplaces.get(j);
+ if ( iop.index == rop.index ) {
+ rop.text = catOpText(iop.text,rop.text);
+ rewrites.set(i, null); // delete current insert
+ continue;
+ }
+ if ( iop.index >= rop.index && iop.index <= rop.lastIndex ) {
+ throw new IllegalArgumentException("insert op "+iop+
+ " within boundaries of previous "+rop);
+ }
+ }
+ }
+ // System.out.println("rewrites after="+rewrites);
+ Map m = new HashMap();
+ for (int i = 0; i < rewrites.size(); i++) {
+ RewriteOperation op = (RewriteOperation)rewrites.get(i);
+ if ( op==null ) continue; // ignore deleted ops
+ if ( m.get(new Integer(op.index))!=null ) {
+ throw new Error("should only be one op per index");
+ }
+ m.put(new Integer(op.index), op);
+ }
+ //System.out.println("index to op: "+m);
+ return m;
+ }
+
+ protected String catOpText(Object a, Object b) {
+ String x = "";
+ String y = "";
+ if ( a!=null ) x = a.toString();
+ if ( b!=null ) y = b.toString();
+ return x+y;
+ }
+ protected List getKindOfOps(List rewrites, Class kind) {
+ return getKindOfOps(rewrites, kind, rewrites.size());
+ }
+
+ /** Get all operations before an index of a particular kind */
+ protected List getKindOfOps(List rewrites, Class kind, int before) {
+ List ops = new ArrayList();
+ for (int i=0; i<before && i<rewrites.size(); i++) {
+ RewriteOperation op = (RewriteOperation)rewrites.get(i);
+ if ( op==null ) continue; // ignore deleted
+ if ( op.getClass() == kind ) ops.add(op);
+ }
+ return ops;
+ }
+
+ public String toDebugString() {
+ return toDebugString(MIN_TOKEN_INDEX, size()-1);
+ }
+
+ public String toDebugString(int start, int end) {
+ StringBuffer buf = new StringBuffer();
+ for (int i=start; i>=MIN_TOKEN_INDEX && i<=end && i<tokens.size(); i++) {
+ buf.append(get(i));
+ }
+ return buf.toString();
+ }
+}
diff --git a/src/org/antlr/runtime/TokenSource.java b/src/org/antlr/runtime/TokenSource.java
new file mode 100755
index 0000000..1f4dc64
--- /dev/null
+++ b/src/org/antlr/runtime/TokenSource.java
@@ -0,0 +1,54 @@
+/*
+ [The "BSD license"]
+ Copyright (c) 2005-2009 Terence Parr
+ All rights reserved.
+
+ Redistribution and use in source and binary forms, with or without
+ modification, are permitted provided that the following conditions
+ are met:
+ 1. Redistributions of source code must retain the above copyright
+ notice, this list of conditions and the following disclaimer.
+ 2. Redistributions in binary form must reproduce the above copyright
+ notice, this list of conditions and the following disclaimer in the
+ documentation and/or other materials provided with the distribution.
+ 3. The name of the author may not be used to endorse or promote products
+ derived from this software without specific prior written permission.
+
+ THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
+ IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
+ OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
+ IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
+ INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
+ NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
+ THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+package org.antlr.runtime;
+
+/** A source of tokens must provide a sequence of tokens via nextToken()
+ * and also must reveal it's source of characters; CommonToken's text is
+ * computed from a CharStream; it only store indices into the char stream.
+ *
+ * Errors from the lexer are never passed to the parser. Either you want
+ * to keep going or you do not upon token recognition error. If you do not
+ * want to continue lexing then you do not want to continue parsing. Just
+ * throw an exception not under RecognitionException and Java will naturally
+ * toss you all the way out of the recognizers. If you want to continue
+ * lexing then you should not throw an exception to the parser--it has already
+ * requested a token. Keep lexing until you get a valid one. Just report
+ * errors and keep going, looking for a valid token.
+ */
+public interface TokenSource {
+ /** Return a Token object from your input stream (usually a CharStream).
+ * Do not fail/return upon lexing error; keep chewing on the characters
+ * until you get a good one; errors are not passed through to the parser.
+ */
+ public Token nextToken();
+
+ /** Where are you getting tokens from? normally the implication will simply
+ * ask lexers input stream.
+ */
+ public String getSourceName();
+}
diff --git a/src/org/antlr/runtime/TokenStream.java b/src/org/antlr/runtime/TokenStream.java
new file mode 100755
index 0000000..1b43c14
--- /dev/null
+++ b/src/org/antlr/runtime/TokenStream.java
@@ -0,0 +1,75 @@
+/*
+ [The "BSD license"]
+ Copyright (c) 2005-2009 Terence Parr
+ All rights reserved.
+
+ Redistribution and use in source and binary forms, with or without
+ modification, are permitted provided that the following conditions
+ are met:
+ 1. Redistributions of source code must retain the above copyright
+ notice, this list of conditions and the following disclaimer.
+ 2. Redistributions in binary form must reproduce the above copyright
+ notice, this list of conditions and the following disclaimer in the
+ documentation and/or other materials provided with the distribution.
+ 3. The name of the author may not be used to endorse or promote products
+ derived from this software without specific prior written permission.
+
+ THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
+ IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
+ OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
+ IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
+ INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
+ NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
+ THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+package org.antlr.runtime;
+
+import java.util.List;
+
+/** A stream of tokens accessing tokens from a TokenSource */
+public interface TokenStream extends IntStream {
+ /** Get Token at current input pointer + i ahead where i=1 is next Token.
+ * i<0 indicates tokens in the past. So -1 is previous token and -2 is
+ * two tokens ago. LT(0) is undefined. For i>=n, return Token.EOFToken.
+ * Return null for LT(0) and any index that results in an absolute address
+ * that is negative.
+ */
+ public Token LT(int k);
+
+ /** How far ahead has the stream been asked to look? The return
+ * value is a valid index from 0..n-1.
+ */
+ int range();
+
+ /** Get a token at an absolute index i; 0..n-1. This is really only
+ * needed for profiling and debugging and token stream rewriting.
+ * If you don't want to buffer up tokens, then this method makes no
+ * sense for you. Naturally you can't use the rewrite stream feature.
+ * I believe DebugTokenStream can easily be altered to not use
+ * this method, removing the dependency.
+ */
+ public Token get(int i);
+
+ /** Where is this stream pulling tokens from? This is not the name, but
+ * the object that provides Token objects.
+ */
+ public TokenSource getTokenSource();
+
+ /** Return the text of all tokens from start to stop, inclusive.
+ * If the stream does not buffer all the tokens then it can just
+ * return "" or null; Users should not access $ruleLabel.text in
+ * an action of course in that case.
+ */
+ public String toString(int start, int stop);
+
+ /** Because the user is not required to use a token with an index stored
+ * in it, we must provide a means for two token objects themselves to
+ * indicate the start/end location. Most often this will just delegate
+ * to the other toString(int,int). This is also parallel with
+ * the TreeNodeStream.toString(Object,Object).
+ */
+ public String toString(Token start, Token stop);
+}
diff --git a/src/org/antlr/runtime/UnbufferedTokenStream.java b/src/org/antlr/runtime/UnbufferedTokenStream.java
new file mode 100755
index 0000000..0b0e979
--- /dev/null
+++ b/src/org/antlr/runtime/UnbufferedTokenStream.java
@@ -0,0 +1,82 @@
+/*
+ [The "BSD license"]
+ Copyright (c) 2005-2009 Terence Parr
+ All rights reserved.
+
+ Redistribution and use in source and binary forms, with or without
+ modification, are permitted provided that the following conditions
+ are met:
+ 1. Redistributions of source code must retain the above copyright
+ notice, this list of conditions and the following disclaimer.
+ 2. Redistributions in binary form must reproduce the above copyright
+ notice, this list of conditions and the following disclaimer in the
+ documentation and/or other materials provided with the distribution.
+ 3. The name of the author may not be used to endorse or promote products
+ derived from this software without specific prior written permission.
+
+ THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
+ IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
+ OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
+ IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
+ INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
+ NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
+ THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+package org.antlr.runtime;
+
+import org.antlr.runtime.misc.LookaheadStream;
+
+import java.util.List;
+import java.util.NoSuchElementException;
+
+/** A token stream that pulls tokens from the code source on-demand and
+ * without tracking a complete buffer of the tokens. This stream buffers
+ * the minimum number of tokens possible. It's the same as
+ * OnDemandTokenStream except that OnDemandTokenStream buffers all tokens.
+ *
+ * You can't use this stream if you pass whitespace or other off-channel
+ * tokens to the parser. The stream can't ignore off-channel tokens.
+ *
+ * You can only look backwards 1 token: LT(-1).
+ *
+ * Use this when you need to read from a socket or other infinite stream.
+ *
+ * @see BufferedTokenStream
+ * @see CommonTokenStream
+ */
+public class UnbufferedTokenStream extends LookaheadStream<Token> implements TokenStream {
+ protected TokenSource tokenSource;
+ protected int tokenIndex = 0; // simple counter to set token index in tokens
+
+ /** Skip tokens on any channel but this one; this is how we skip whitespace... */
+ protected int channel = Token.DEFAULT_CHANNEL;
+
+ public UnbufferedTokenStream(TokenSource tokenSource) {
+ this.tokenSource = tokenSource;
+ }
+
+ public Token nextElement() {
+ Token t = tokenSource.nextToken();
+ t.setTokenIndex(tokenIndex++);
+ return t;
+ }
+
+ public boolean isEOF(Token o) { return o.getType() == Token.EOF; }
+
+ public TokenSource getTokenSource() { return tokenSource; }
+
+ public String toString(int start, int stop) { return "n/a"; }
+
+ public String toString(Token start, Token stop) { return "n/a"; }
+
+ public int LA(int i) { return LT(i).getType(); }
+
+ public Token get(int i) {
+ throw new UnsupportedOperationException("Absolute token indexes are meaningless in an unbuffered stream");
+ }
+
+ public String getSourceName() { return tokenSource.getSourceName(); }
+}
diff --git a/src/org/antlr/runtime/UnwantedTokenException.java b/src/org/antlr/runtime/UnwantedTokenException.java
new file mode 100755
index 0000000..feb7445
--- /dev/null
+++ b/src/org/antlr/runtime/UnwantedTokenException.java
@@ -0,0 +1,53 @@
+/*
+ [The "BSD license"]
+ Copyright (c) 2005-2009 Terence Parr
+ All rights reserved.
+
+ Redistribution and use in source and binary forms, with or without
+ modification, are permitted provided that the following conditions
+ are met:
+ 1. Redistributions of source code must retain the above copyright
+ notice, this list of conditions and the following disclaimer.
+ 2. Redistributions in binary form must reproduce the above copyright
+ notice, this list of conditions and the following disclaimer in the
+ documentation and/or other materials provided with the distribution.
+ 3. The name of the author may not be used to endorse or promote products
+ derived from this software without specific prior written permission.
+
+ THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
+ IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
+ OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
+ IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
+ INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
+ NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
+ THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+package org.antlr.runtime;
+
+/** An extra token while parsing a TokenStream */
+public class UnwantedTokenException extends MismatchedTokenException {
+ /** Used for remote debugger deserialization */
+ public UnwantedTokenException() {;}
+
+ public UnwantedTokenException(int expecting, IntStream input) {
+ super(expecting, input);
+ }
+
+ public Token getUnexpectedToken() {
+ return token;
+ }
+
+ public String toString() {
+ String exp = ", expected "+expecting;
+ if ( expecting==Token.INVALID_TOKEN_TYPE ) {
+ exp = "";
+ }
+ if ( token==null ) {
+ return "UnwantedTokenException(found="+null+exp+")";
+ }
+ return "UnwantedTokenException(found="+token.getText()+exp+")";
+ }
+}
diff --git a/src/org/antlr/runtime/debug/BlankDebugEventListener.java b/src/org/antlr/runtime/debug/BlankDebugEventListener.java
new file mode 100755
index 0000000..d70aa26
--- /dev/null
+++ b/src/org/antlr/runtime/debug/BlankDebugEventListener.java
@@ -0,0 +1,77 @@
+/*
+ [The "BSD license"]
+ Copyright (c) 2005-2009 Terence Parr
+ All rights reserved.
+
+ Redistribution and use in source and binary forms, with or without
+ modification, are permitted provided that the following conditions
+ are met:
+ 1. Redistributions of source code must retain the above copyright
+ notice, this list of conditions and the following disclaimer.
+ 2. Redistributions in binary form must reproduce the above copyright
+ notice, this list of conditions and the following disclaimer in the
+ documentation and/or other materials provided with the distribution.
+ 3. The name of the author may not be used to endorse or promote products
+ derived from this software without specific prior written permission.
+
+ THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
+ IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
+ OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
+ IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
+ INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
+ NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
+ THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+package org.antlr.runtime.debug;
+
+import org.antlr.runtime.RecognitionException;
+import org.antlr.runtime.Token;
+
+/** A blank listener that does nothing; useful for real classes so
+ * they don't have to have lots of blank methods and are less
+ * sensitive to updates to debug interface.
+ */
+public class BlankDebugEventListener implements DebugEventListener {
+ public void enterRule(String grammarFileName, String ruleName) {}
+ public void exitRule(String grammarFileName, String ruleName) {}
+ public void enterAlt(int alt) {}
+ public void enterSubRule(int decisionNumber) {}
+ public void exitSubRule(int decisionNumber) {}
+ public void enterDecision(int decisionNumber, boolean couldBacktrack) {}
+ public void exitDecision(int decisionNumber) {}
+ public void location(int line, int pos) {}
+ public void consumeToken(Token token) {}
+ public void consumeHiddenToken(Token token) {}
+ public void LT(int i, Token t) {}
+ public void mark(int i) {}
+ public void rewind(int i) {}
+ public void rewind() {}
+ public void beginBacktrack(int level) {}
+ public void endBacktrack(int level, boolean successful) {}
+ public void recognitionException(RecognitionException e) {}
+ public void beginResync() {}
+ public void endResync() {}
+ public void semanticPredicate(boolean result, String predicate) {}
+ public void commence() {}
+ public void terminate() {}
+
+ // Tree parsing stuff
+
+ public void consumeNode(Object t) {}
+ public void LT(int i, Object t) {}
+
+ // AST Stuff
+
+ public void nilNode(Object t) {}
+ public void errorNode(Object t) {}
+ public void createNode(Object t) {}
+ public void createNode(Object node, Token token) {}
+ public void becomeRoot(Object newRoot, Object oldRoot) {}
+ public void addChild(Object root, Object child) {}
+ public void setTokenBoundaries(Object t, int tokenStartIndex, int tokenStopIndex) {}
+}
+
+
diff --git a/src/org/antlr/runtime/debug/DebugEventHub.java b/src/org/antlr/runtime/debug/DebugEventHub.java
new file mode 100755
index 0000000..7bfe6a8
--- /dev/null
+++ b/src/org/antlr/runtime/debug/DebugEventHub.java
@@ -0,0 +1,292 @@
+/*
+ [The "BSD license"]
+ Copyright (c) 2005-2009 Terence Parr
+ All rights reserved.
+
+ Redistribution and use in source and binary forms, with or without
+ modification, are permitted provided that the following conditions
+ are met:
+ 1. Redistributions of source code must retain the above copyright
+ notice, this list of conditions and the following disclaimer.
+ 2. Redistributions in binary form must reproduce the above copyright
+ notice, this list of conditions and the following disclaimer in the
+ documentation and/or other materials provided with the distribution.
+ 3. The name of the author may not be used to endorse or promote products
+ derived from this software without specific prior written permission.
+
+ THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
+ IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
+ OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
+ IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
+ INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
+ NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
+ THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+package org.antlr.runtime.debug;
+
+import org.antlr.runtime.Token;
+import org.antlr.runtime.RecognitionException;
+
+import java.util.List;
+import java.util.ArrayList;
+
+/** Broadcast debug events to multiple listeners. Lets you debug and still
+ * use the event mechanism to build parse trees etc... Not thread-safe.
+ * Don't add events in one thread while parser fires events in another.
+ *
+ * @see also DebugEventRepeater
+ */
+public class DebugEventHub implements DebugEventListener {
+ protected List listeners = new ArrayList();
+
+ public DebugEventHub(DebugEventListener listener) {
+ listeners.add(listener);
+ }
+
+ public DebugEventHub(DebugEventListener a, DebugEventListener b) {
+ listeners.add(a);
+ listeners.add(b);
+ }
+
+ /** Add another listener to broadcast events too. Not thread-safe.
+ * Don't add events in one thread while parser fires events in another.
+ */
+ public void addListener(DebugEventListener listener) {
+ listeners.add(listener);
+ }
+
+ /* To avoid a mess like this:
+ public void enterRule(final String ruleName) {
+ broadcast(new Code(){
+ public void exec(DebugEventListener listener) {listener.enterRule(ruleName);}}
+ );
+ }
+ I am dup'ing the for-loop in each. Where are Java closures!? blech!
+ */
+
+ public void enterRule(String grammarFileName, String ruleName) {
+ for (int i = 0; i < listeners.size(); i++) {
+ DebugEventListener listener = (DebugEventListener)listeners.get(i);
+ listener.enterRule(grammarFileName,ruleName);
+ }
+ }
+
+ public void exitRule(String grammarFileName, String ruleName) {
+ for (int i = 0; i < listeners.size(); i++) {
+ DebugEventListener listener = (DebugEventListener)listeners.get(i);
+ listener.exitRule(grammarFileName, ruleName);
+ }
+ }
+
+ public void enterAlt(int alt) {
+ for (int i = 0; i < listeners.size(); i++) {
+ DebugEventListener listener = (DebugEventListener)listeners.get(i);
+ listener.enterAlt(alt);
+ }
+ }
+
+ public void enterSubRule(int decisionNumber) {
+ for (int i = 0; i < listeners.size(); i++) {
+ DebugEventListener listener = (DebugEventListener)listeners.get(i);
+ listener.enterSubRule(decisionNumber);
+ }
+ }
+
+ public void exitSubRule(int decisionNumber) {
+ for (int i = 0; i < listeners.size(); i++) {
+ DebugEventListener listener = (DebugEventListener)listeners.get(i);
+ listener.exitSubRule(decisionNumber);
+ }
+ }
+
+ public void enterDecision(int decisionNumber, boolean couldBacktrack) {
+ for (int i = 0; i < listeners.size(); i++) {
+ DebugEventListener listener = (DebugEventListener)listeners.get(i);
+ listener.enterDecision(decisionNumber, couldBacktrack);
+ }
+ }
+
+ public void exitDecision(int decisionNumber) {
+ for (int i = 0; i < listeners.size(); i++) {
+ DebugEventListener listener = (DebugEventListener)listeners.get(i);
+ listener.exitDecision(decisionNumber);
+ }
+ }
+
+ public void location(int line, int pos) {
+ for (int i = 0; i < listeners.size(); i++) {
+ DebugEventListener listener = (DebugEventListener)listeners.get(i);
+ listener.location(line, pos);
+ }
+ }
+
+ public void consumeToken(Token token) {
+ for (int i = 0; i < listeners.size(); i++) {
+ DebugEventListener listener = (DebugEventListener)listeners.get(i);
+ listener.consumeToken(token);
+ }
+ }
+
+ public void consumeHiddenToken(Token token) {
+ for (int i = 0; i < listeners.size(); i++) {
+ DebugEventListener listener = (DebugEventListener)listeners.get(i);
+ listener.consumeHiddenToken(token);
+ }
+ }
+
+ public void LT(int index, Token t) {
+ for (int i = 0; i < listeners.size(); i++) {
+ DebugEventListener listener = (DebugEventListener)listeners.get(i);
+ listener.LT(index, t);
+ }
+ }
+
+ public void mark(int index) {
+ for (int i = 0; i < listeners.size(); i++) {
+ DebugEventListener listener = (DebugEventListener)listeners.get(i);
+ listener.mark(index);
+ }
+ }
+
+ public void rewind(int index) {
+ for (int i = 0; i < listeners.size(); i++) {
+ DebugEventListener listener = (DebugEventListener)listeners.get(i);
+ listener.rewind(index);
+ }
+ }
+
+ public void rewind() {
+ for (int i = 0; i < listeners.size(); i++) {
+ DebugEventListener listener = (DebugEventListener)listeners.get(i);
+ listener.rewind();
+ }
+ }
+
+ public void beginBacktrack(int level) {
+ for (int i = 0; i < listeners.size(); i++) {
+ DebugEventListener listener = (DebugEventListener)listeners.get(i);
+ listener.beginBacktrack(level);
+ }
+ }
+
+ public void endBacktrack(int level, boolean successful) {
+ for (int i = 0; i < listeners.size(); i++) {
+ DebugEventListener listener = (DebugEventListener)listeners.get(i);
+ listener.endBacktrack(level, successful);
+ }
+ }
+
+ public void recognitionException(RecognitionException e) {
+ for (int i = 0; i < listeners.size(); i++) {
+ DebugEventListener listener = (DebugEventListener)listeners.get(i);
+ listener.recognitionException(e);
+ }
+ }
+
+ public void beginResync() {
+ for (int i = 0; i < listeners.size(); i++) {
+ DebugEventListener listener = (DebugEventListener)listeners.get(i);
+ listener.beginResync();
+ }
+ }
+
+ public void endResync() {
+ for (int i = 0; i < listeners.size(); i++) {
+ DebugEventListener listener = (DebugEventListener)listeners.get(i);
+ listener.endResync();
+ }
+ }
+
+ public void semanticPredicate(boolean result, String predicate) {
+ for (int i = 0; i < listeners.size(); i++) {
+ DebugEventListener listener = (DebugEventListener)listeners.get(i);
+ listener.semanticPredicate(result, predicate);
+ }
+ }
+
+ public void commence() {
+ for (int i = 0; i < listeners.size(); i++) {
+ DebugEventListener listener = (DebugEventListener)listeners.get(i);
+ listener.commence();
+ }
+ }
+
+ public void terminate() {
+ for (int i = 0; i < listeners.size(); i++) {
+ DebugEventListener listener = (DebugEventListener)listeners.get(i);
+ listener.terminate();
+ }
+ }
+
+
+ // Tree parsing stuff
+
+ public void consumeNode(Object t) {
+ for (int i = 0; i < listeners.size(); i++) {
+ DebugEventListener listener = (DebugEventListener)listeners.get(i);
+ listener.consumeNode(t);
+ }
+ }
+
+ public void LT(int index, Object t) {
+ for (int i = 0; i < listeners.size(); i++) {
+ DebugEventListener listener = (DebugEventListener)listeners.get(i);
+ listener.LT(index, t);
+ }
+ }
+
+
+ // AST Stuff
+
+ public void nilNode(Object t) {
+ for (int i = 0; i < listeners.size(); i++) {
+ DebugEventListener listener = (DebugEventListener)listeners.get(i);
+ listener.nilNode(t);
+ }
+ }
+
+ public void errorNode(Object t) {
+ for (int i = 0; i < listeners.size(); i++) {
+ DebugEventListener listener = (DebugEventListener)listeners.get(i);
+ listener.errorNode(t);
+ }
+ }
+
+ public void createNode(Object t) {
+ for (int i = 0; i < listeners.size(); i++) {
+ DebugEventListener listener = (DebugEventListener)listeners.get(i);
+ listener.createNode(t);
+ }
+ }
+
+ public void createNode(Object node, Token token) {
+ for (int i = 0; i < listeners.size(); i++) {
+ DebugEventListener listener = (DebugEventListener)listeners.get(i);
+ listener.createNode(node, token);
+ }
+ }
+
+ public void becomeRoot(Object newRoot, Object oldRoot) {
+ for (int i = 0; i < listeners.size(); i++) {
+ DebugEventListener listener = (DebugEventListener)listeners.get(i);
+ listener.becomeRoot(newRoot, oldRoot);
+ }
+ }
+
+ public void addChild(Object root, Object child) {
+ for (int i = 0; i < listeners.size(); i++) {
+ DebugEventListener listener = (DebugEventListener)listeners.get(i);
+ listener.addChild(root, child);
+ }
+ }
+
+ public void setTokenBoundaries(Object t, int tokenStartIndex, int tokenStopIndex) {
+ for (int i = 0; i < listeners.size(); i++) {
+ DebugEventListener listener = (DebugEventListener)listeners.get(i);
+ listener.setTokenBoundaries(t, tokenStartIndex, tokenStopIndex);
+ }
+ }
+}
diff --git a/src/org/antlr/runtime/debug/DebugEventListener.java b/src/org/antlr/runtime/debug/DebugEventListener.java
new file mode 100755
index 0000000..163b5cd
--- /dev/null
+++ b/src/org/antlr/runtime/debug/DebugEventListener.java
@@ -0,0 +1,323 @@
+/*
+ [The "BSD license"]
+ Copyright (c) 2005-2009 Terence Parr
+ All rights reserved.
+
+ Redistribution and use in source and binary forms, with or without
+ modification, are permitted provided that the following conditions
+ are met:
+ 1. Redistributions of source code must retain the above copyright
+ notice, this list of conditions and the following disclaimer.
+ 2. Redistributions in binary form must reproduce the above copyright
+ notice, this list of conditions and the following disclaimer in the
+ documentation and/or other materials provided with the distribution.
+ 3. The name of the author may not be used to endorse or promote products
+ derived from this software without specific prior written permission.
+
+ THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
+ IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
+ OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
+ IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
+ INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
+ NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
+ THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+package org.antlr.runtime.debug;
+
+import org.antlr.runtime.RecognitionException;
+import org.antlr.runtime.Token;
+
+/** All debugging events that a recognizer can trigger.
+ *
+ * I did not create a separate AST debugging interface as it would create
+ * lots of extra classes and DebugParser has a dbg var defined, which makes
+ * it hard to change to ASTDebugEventListener. I looked hard at this issue
+ * and it is easier to understand as one monolithic event interface for all
+ * possible events. Hopefully, adding ST debugging stuff won't be bad. Leave
+ * for future. 4/26/2006.
+ */
+public interface DebugEventListener {
+ /** Moved to version 2 for v3.1: added grammar name to enter/exit Rule */
+ public static final String PROTOCOL_VERSION = "2";
+
+ /** serialized version of true */
+ public static final int TRUE = 1;
+ public static final int FALSE = 0;
+
+ /** The parser has just entered a rule. No decision has been made about
+ * which alt is predicted. This is fired AFTER init actions have been
+ * executed. Attributes are defined and available etc...
+ * The grammarFileName allows composite grammars to jump around among
+ * multiple grammar files.
+ */
+ public void enterRule(String grammarFileName, String ruleName);
+
+ /** Because rules can have lots of alternatives, it is very useful to
+ * know which alt you are entering. This is 1..n for n alts.
+ */
+ public void enterAlt(int alt);
+
+ /** This is the last thing executed before leaving a rule. It is
+ * executed even if an exception is thrown. This is triggered after
+ * error reporting and recovery have occurred (unless the exception is
+ * not caught in this rule). This implies an "exitAlt" event.
+ * The grammarFileName allows composite grammars to jump around among
+ * multiple grammar files.
+ */
+ public void exitRule(String grammarFileName, String ruleName);
+
+ /** Track entry into any (...) subrule other EBNF construct */
+ public void enterSubRule(int decisionNumber);
+
+ public void exitSubRule(int decisionNumber);
+
+ /** Every decision, fixed k or arbitrary, has an enter/exit event
+ * so that a GUI can easily track what LT/consume events are
+ * associated with prediction. You will see a single enter/exit
+ * subrule but multiple enter/exit decision events, one for each
+ * loop iteration.
+ */
+ public void enterDecision(int decisionNumber, boolean couldBacktrack);
+
+ public void exitDecision(int decisionNumber);
+
+ /** An input token was consumed; matched by any kind of element.
+ * Trigger after the token was matched by things like match(), matchAny().
+ */
+ public void consumeToken(Token t);
+
+ /** An off-channel input token was consumed.
+ * Trigger after the token was matched by things like match(), matchAny().
+ * (unless of course the hidden token is first stuff in the input stream).
+ */
+ public void consumeHiddenToken(Token t);
+
+ /** Somebody (anybody) looked ahead. Note that this actually gets
+ * triggered by both LA and LT calls. The debugger will want to know
+ * which Token object was examined. Like consumeToken, this indicates
+ * what token was seen at that depth. A remote debugger cannot look
+ * ahead into a file it doesn't have so LT events must pass the token
+ * even if the info is redundant.
+ */
+ public void LT(int i, Token t);
+
+ /** The parser is going to look arbitrarily ahead; mark this location,
+ * the token stream's marker is sent in case you need it.
+ */
+ public void mark(int marker);
+
+ /** After an arbitrairly long lookahead as with a cyclic DFA (or with
+ * any backtrack), this informs the debugger that stream should be
+ * rewound to the position associated with marker.
+ */
+ public void rewind(int marker);
+
+ /** Rewind to the input position of the last marker.
+ * Used currently only after a cyclic DFA and just
+ * before starting a sem/syn predicate to get the
+ * input position back to the start of the decision.
+ * Do not "pop" the marker off the state. mark(i)
+ * and rewind(i) should balance still.
+ */
+ public void rewind();
+
+ public void beginBacktrack(int level);
+
+ public void endBacktrack(int level, boolean successful);
+
+ /** To watch a parser move through the grammar, the parser needs to
+ * inform the debugger what line/charPos it is passing in the grammar.
+ * For now, this does not know how to switch from one grammar to the
+ * other and back for island grammars etc...
+ *
+ * This should also allow breakpoints because the debugger can stop
+ * the parser whenever it hits this line/pos.
+ */
+ public void location(int line, int pos);
+
+ /** A recognition exception occurred such as NoViableAltException. I made
+ * this a generic event so that I can alter the exception hierachy later
+ * without having to alter all the debug objects.
+ *
+ * Upon error, the stack of enter rule/subrule must be properly unwound.
+ * If no viable alt occurs it is within an enter/exit decision, which
+ * also must be rewound. Even the rewind for each mark must be unwount.
+ * In the Java target this is pretty easy using try/finally, if a bit
+ * ugly in the generated code. The rewind is generated in DFA.predict()
+ * actually so no code needs to be generated for that. For languages
+ * w/o this "finally" feature (C++?), the target implementor will have
+ * to build an event stack or something.
+ *
+ * Across a socket for remote debugging, only the RecognitionException
+ * data fields are transmitted. The token object or whatever that
+ * caused the problem was the last object referenced by LT. The
+ * immediately preceding LT event should hold the unexpected Token or
+ * char.
+ *
+ * Here is a sample event trace for grammar:
+ *
+ * b : C ({;}A|B) // {;} is there to prevent A|B becoming a set
+ * | D
+ * ;
+ *
+ * The sequence for this rule (with no viable alt in the subrule) for
+ * input 'c c' (there are 3 tokens) is:
+ *
+ * commence
+ * LT(1)
+ * enterRule b
+ * location 7 1
+ * enter decision 3
+ * LT(1)
+ * exit decision 3
+ * enterAlt1
+ * location 7 5
+ * LT(1)
+ * consumeToken [c/<4>,1:0]
+ * location 7 7
+ * enterSubRule 2
+ * enter decision 2
+ * LT(1)
+ * LT(1)
+ * recognitionException NoViableAltException 2 1 2
+ * exit decision 2
+ * exitSubRule 2
+ * beginResync
+ * LT(1)
+ * consumeToken [c/<4>,1:1]
+ * LT(1)
+ * endResync
+ * LT(-1)
+ * exitRule b
+ * terminate
+ */
+ public void recognitionException(RecognitionException e);
+
+ /** Indicates the recognizer is about to consume tokens to resynchronize
+ * the parser. Any consume events from here until the recovered event
+ * are not part of the parse--they are dead tokens.
+ */
+ public void beginResync();
+
+ /** Indicates that the recognizer has finished consuming tokens in order
+ * to resychronize. There may be multiple beginResync/endResync pairs
+ * before the recognizer comes out of errorRecovery mode (in which
+ * multiple errors are suppressed). This will be useful
+ * in a gui where you want to probably grey out tokens that are consumed
+ * but not matched to anything in grammar. Anything between
+ * a beginResync/endResync pair was tossed out by the parser.
+ */
+ public void endResync();
+
+ /** A semantic predicate was evaluate with this result and action text */
+ public void semanticPredicate(boolean result, String predicate);
+
+ /** Announce that parsing has begun. Not technically useful except for
+ * sending events over a socket. A GUI for example will launch a thread
+ * to connect and communicate with a remote parser. The thread will want
+ * to notify the GUI when a connection is made. ANTLR parsers
+ * trigger this upon entry to the first rule (the ruleLevel is used to
+ * figure this out).
+ */
+ public void commence();
+
+ /** Parsing is over; successfully or not. Mostly useful for telling
+ * remote debugging listeners that it's time to quit. When the rule
+ * invocation level goes to zero at the end of a rule, we are done
+ * parsing.
+ */
+ public void terminate();
+
+
+ // T r e e P a r s i n g
+
+ /** Input for a tree parser is an AST, but we know nothing for sure
+ * about a node except its type and text (obtained from the adaptor).
+ * This is the analog of the consumeToken method. Again, the ID is
+ * the hashCode usually of the node so it only works if hashCode is
+ * not implemented. If the type is UP or DOWN, then
+ * the ID is not really meaningful as it's fixed--there is
+ * just one UP node and one DOWN navigation node.
+ * @param t
+ */
+ public void consumeNode(Object t);
+
+ /** The tree parser lookedahead. If the type is UP or DOWN,
+ * then the ID is not really meaningful as it's fixed--there is
+ * just one UP node and one DOWN navigation node.
+ */
+ public void LT(int i, Object t);
+
+
+ // A S T E v e n t s
+
+ /** A nil was created (even nil nodes have a unique ID...
+ * they are not "null" per se). As of 4/28/2006, this
+ * seems to be uniquely triggered when starting a new subtree
+ * such as when entering a subrule in automatic mode and when
+ * building a tree in rewrite mode.
+ *
+ * If you are receiving this event over a socket via
+ * RemoteDebugEventSocketListener then only t.ID is set.
+ */
+ public void nilNode(Object t);
+
+ /** Upon syntax error, recognizers bracket the error with an error node
+ * if they are building ASTs.
+ * @param t
+ */
+ public void errorNode(Object t);
+
+ /** Announce a new node built from token elements such as type etc...
+ *
+ * If you are receiving this event over a socket via
+ * RemoteDebugEventSocketListener then only t.ID, type, text are
+ * set.
+ */
+ public void createNode(Object t);
+
+ /** Announce a new node built from an existing token.
+ *
+ * If you are receiving this event over a socket via
+ * RemoteDebugEventSocketListener then only node.ID and token.tokenIndex
+ * are set.
+ */
+ public void createNode(Object node, Token token);
+
+ /** Make a node the new root of an existing root. See
+ *
+ * Note: the newRootID parameter is possibly different
+ * than the TreeAdaptor.becomeRoot() newRoot parameter.
+ * In our case, it will always be the result of calling
+ * TreeAdaptor.becomeRoot() and not root_n or whatever.
+ *
+ * The listener should assume that this event occurs
+ * only when the current subrule (or rule) subtree is
+ * being reset to newRootID.
+ *
+ * If you are receiving this event over a socket via
+ * RemoteDebugEventSocketListener then only IDs are set.
+ *
+ * @see org.antlr.runtime.tree.TreeAdaptor.becomeRoot()
+ */
+ public void becomeRoot(Object newRoot, Object oldRoot);
+
+ /** Make childID a child of rootID.
+ *
+ * If you are receiving this event over a socket via
+ * RemoteDebugEventSocketListener then only IDs are set.
+ *
+ * @see org.antlr.runtime.tree.TreeAdaptor.addChild()
+ */
+ public void addChild(Object root, Object child);
+
+ /** Set the token start/stop token index for a subtree root or node.
+ *
+ * If you are receiving this event over a socket via
+ * RemoteDebugEventSocketListener then only t.ID is set.
+ */
+ public void setTokenBoundaries(Object t, int tokenStartIndex, int tokenStopIndex);
+}
diff --git a/src/org/antlr/runtime/debug/DebugEventRepeater.java b/src/org/antlr/runtime/debug/DebugEventRepeater.java
new file mode 100755
index 0000000..8fb6b66
--- /dev/null
+++ b/src/org/antlr/runtime/debug/DebugEventRepeater.java
@@ -0,0 +1,88 @@
+/*
+ [The "BSD license"]
+ Copyright (c) 2005-2009 Terence Parr
+ All rights reserved.
+
+ Redistribution and use in source and binary forms, with or without
+ modification, are permitted provided that the following conditions
+ are met:
+ 1. Redistributions of source code must retain the above copyright
+ notice, this list of conditions and the following disclaimer.
+ 2. Redistributions in binary form must reproduce the above copyright
+ notice, this list of conditions and the following disclaimer in the
+ documentation and/or other materials provided with the distribution.
+ 3. The name of the author may not be used to endorse or promote products
+ derived from this software without specific prior written permission.
+
+ THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
+ IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
+ OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
+ IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
+ INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
+ NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
+ THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+package org.antlr.runtime.debug;
+
+import org.antlr.runtime.Token;
+import org.antlr.runtime.RecognitionException;
+
+/** A simple event repeater (proxy) that delegates all functionality to the
+ * listener sent into the ctor. Useful if you want to listen in on a few
+ * debug events w/o interrupting the debugger. Just subclass the repeater
+ * and override the methods you want to listen in on. Remember to call
+ * the method in this class so the event will continue on to the original
+ * recipient.
+ *
+ * @see DebugEventHub
+ */
+public class DebugEventRepeater implements DebugEventListener {
+ protected DebugEventListener listener;
+
+ public DebugEventRepeater(DebugEventListener listener) {
+ this.listener = listener;
+ }
+
+ public void enterRule(String grammarFileName, String ruleName) { listener.enterRule(grammarFileName, ruleName); }
+ public void exitRule(String grammarFileName, String ruleName) { listener.exitRule(grammarFileName, ruleName); }
+ public void enterAlt(int alt) { listener.enterAlt(alt); }
+ public void enterSubRule(int decisionNumber) { listener.enterSubRule(decisionNumber); }
+ public void exitSubRule(int decisionNumber) { listener.exitSubRule(decisionNumber); }
+ public void enterDecision(int decisionNumber, boolean couldBacktrack) { listener.enterDecision(decisionNumber, couldBacktrack); }
+ public void exitDecision(int decisionNumber) { listener.exitDecision(decisionNumber); }
+ public void location(int line, int pos) { listener.location(line, pos); }
+ public void consumeToken(Token token) { listener.consumeToken(token); }
+ public void consumeHiddenToken(Token token) { listener.consumeHiddenToken(token); }
+ public void LT(int i, Token t) { listener.LT(i, t); }
+ public void mark(int i) { listener.mark(i); }
+ public void rewind(int i) { listener.rewind(i); }
+ public void rewind() { listener.rewind(); }
+ public void beginBacktrack(int level) { listener.beginBacktrack(level); }
+ public void endBacktrack(int level, boolean successful) { listener.endBacktrack(level, successful); }
+ public void recognitionException(RecognitionException e) { listener.recognitionException(e); }
+ public void beginResync() { listener.beginResync(); }
+ public void endResync() { listener.endResync(); }
+ public void semanticPredicate(boolean result, String predicate) { listener.semanticPredicate(result, predicate); }
+ public void commence() { listener.commence(); }
+ public void terminate() { listener.terminate(); }
+
+ // Tree parsing stuff
+
+ public void consumeNode(Object t) { listener.consumeNode(t); }
+ public void LT(int i, Object t) { listener.LT(i, t); }
+
+ // AST Stuff
+
+ public void nilNode(Object t) { listener.nilNode(t); }
+ public void errorNode(Object t) { listener.errorNode(t); }
+ public void createNode(Object t) { listener.createNode(t); }
+ public void createNode(Object node, Token token) { listener.createNode(node, token); }
+ public void becomeRoot(Object newRoot, Object oldRoot) { listener.becomeRoot(newRoot, oldRoot); }
+ public void addChild(Object root, Object child) { listener.addChild(root, child); }
+ public void setTokenBoundaries(Object t, int tokenStartIndex, int tokenStopIndex) {
+ listener.setTokenBoundaries(t, tokenStartIndex, tokenStopIndex);
+ }
+}
diff --git a/src/org/antlr/runtime/debug/DebugEventSocketProxy.java b/src/org/antlr/runtime/debug/DebugEventSocketProxy.java
new file mode 100755
index 0000000..3b480ad
--- /dev/null
+++ b/src/org/antlr/runtime/debug/DebugEventSocketProxy.java
@@ -0,0 +1,354 @@
+/*
+ [The "BSD license"]
+ Copyright (c) 2005-2009 Terence Parr
+ All rights reserved.
+
+ Redistribution and use in source and binary forms, with or without
+ modification, are permitted provided that the following conditions
+ are met:
+ 1. Redistributions of source code must retain the above copyright
+ notice, this list of conditions and the following disclaimer.
+ 2. Redistributions in binary form must reproduce the above copyright
+ notice, this list of conditions and the following disclaimer in the
+ documentation and/or other materials provided with the distribution.
+ 3. The name of the author may not be used to endorse or promote products
+ derived from this software without specific prior written permission.
+
+ THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
+ IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
+ OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
+ IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
+ INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
+ NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
+ THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+package org.antlr.runtime.debug;
+
+import org.antlr.runtime.RecognitionException;
+import org.antlr.runtime.Token;
+import org.antlr.runtime.BaseRecognizer;
+import org.antlr.runtime.tree.TreeAdaptor;
+
+import java.io.*;
+import java.net.ServerSocket;
+import java.net.Socket;
+
+/** A proxy debug event listener that forwards events over a socket to
+ * a debugger (or any other listener) using a simple text-based protocol;
+ * one event per line. ANTLRWorks listens on server socket with a
+ * RemoteDebugEventSocketListener instance. These two objects must therefore
+ * be kept in sync. New events must be handled on both sides of socket.
+ */
+public class DebugEventSocketProxy extends BlankDebugEventListener {
+ public static final int DEFAULT_DEBUGGER_PORT = 49100; // was 49153
+ protected int port = DEFAULT_DEBUGGER_PORT;
+ protected ServerSocket serverSocket;
+ protected Socket socket;
+ protected String grammarFileName;
+ protected PrintWriter out;
+ protected BufferedReader in;
+
+ /** Who am i debugging? */
+ protected BaseRecognizer recognizer;
+
+ /** Almost certainly the recognizer will have adaptor set, but
+ * we don't know how to cast it (Parser or TreeParser) to get
+ * the adaptor field. Must be set with a constructor. :(
+ */
+ protected TreeAdaptor adaptor;
+
+ public DebugEventSocketProxy(BaseRecognizer recognizer, TreeAdaptor adaptor) {
+ this(recognizer, DEFAULT_DEBUGGER_PORT, adaptor);
+ }
+
+ public DebugEventSocketProxy(BaseRecognizer recognizer, int port, TreeAdaptor adaptor) {
+ this.grammarFileName = recognizer.getGrammarFileName();
+ this.adaptor = adaptor;
+ this.port = port;
+ }
+
+ public void handshake() throws IOException {
+ if ( serverSocket==null ) {
+ serverSocket = new ServerSocket(port);
+ socket = serverSocket.accept();
+ socket.setTcpNoDelay(true);
+ OutputStream os = socket.getOutputStream();
+ OutputStreamWriter osw = new OutputStreamWriter(os, "UTF8");
+ out = new PrintWriter(new BufferedWriter(osw));
+ InputStream is = socket.getInputStream();
+ InputStreamReader isr = new InputStreamReader(is, "UTF8");
+ in = new BufferedReader(isr);
+ out.println("ANTLR "+ DebugEventListener.PROTOCOL_VERSION);
+ out.println("grammar \""+ grammarFileName);
+ out.flush();
+ ack();
+ }
+ }
+
+ public void commence() {
+ // don't bother sending event; listener will trigger upon connection
+ }
+
+ public void terminate() {
+ transmit("terminate");
+ out.close();
+ try {
+ socket.close();
+ }
+ catch (IOException ioe) {
+ ioe.printStackTrace(System.err);
+ }
+ }
+
+ protected void ack() {
+ try {
+ in.readLine();
+ }
+ catch (IOException ioe) {
+ ioe.printStackTrace(System.err);
+ }
+ }
+
+ protected void transmit(String event) {
+ out.println(event);
+ out.flush();
+ ack();
+ }
+
+ public void enterRule(String grammarFileName, String ruleName) {
+ transmit("enterRule\t"+grammarFileName+"\t"+ruleName);
+ }
+
+ public void enterAlt(int alt) {
+ transmit("enterAlt\t"+alt);
+ }
+
+ public void exitRule(String grammarFileName, String ruleName) {
+ transmit("exitRule\t"+grammarFileName+"\t"+ruleName);
+ }
+
+ public void enterSubRule(int decisionNumber) {
+ transmit("enterSubRule\t"+decisionNumber);
+ }
+
+ public void exitSubRule(int decisionNumber) {
+ transmit("exitSubRule\t"+decisionNumber);
+ }
+
+ public void enterDecision(int decisionNumber, boolean couldBacktrack) {
+ transmit("enterDecision\t"+decisionNumber+"\t"+couldBacktrack);
+ }
+
+ public void exitDecision(int decisionNumber) {
+ transmit("exitDecision\t"+decisionNumber);
+ }
+
+ public void consumeToken(Token t) {
+ String buf = serializeToken(t);
+ transmit("consumeToken\t"+buf);
+ }
+
+ public void consumeHiddenToken(Token t) {
+ String buf = serializeToken(t);
+ transmit("consumeHiddenToken\t"+buf);
+ }
+
+ public void LT(int i, Token t) {
+ if(t != null)
+ transmit("LT\t"+i+"\t"+serializeToken(t));
+ }
+
+ public void mark(int i) {
+ transmit("mark\t"+i);
+ }
+
+ public void rewind(int i) {
+ transmit("rewind\t"+i);
+ }
+
+ public void rewind() {
+ transmit("rewind");
+ }
+
+ public void beginBacktrack(int level) {
+ transmit("beginBacktrack\t"+level);
+ }
+
+ public void endBacktrack(int level, boolean successful) {
+ transmit("endBacktrack\t"+level+"\t"+(successful?TRUE:FALSE));
+ }
+
+ public void location(int line, int pos) {
+ transmit("location\t"+line+"\t"+pos);
+ }
+
+ public void recognitionException(RecognitionException e) {
+ StringBuffer buf = new StringBuffer(50);
+ buf.append("exception\t");
+ buf.append(e.getClass().getName());
+ // dump only the data common to all exceptions for now
+ buf.append("\t");
+ buf.append(e.index);
+ buf.append("\t");
+ buf.append(e.line);
+ buf.append("\t");
+ buf.append(e.charPositionInLine);
+ transmit(buf.toString());
+ }
+
+ public void beginResync() {
+ transmit("beginResync");
+ }
+
+ public void endResync() {
+ transmit("endResync");
+ }
+
+ public void semanticPredicate(boolean result, String predicate) {
+ StringBuffer buf = new StringBuffer(50);
+ buf.append("semanticPredicate\t");
+ buf.append(result);
+ serializeText(buf, predicate);
+ transmit(buf.toString());
+ }
+
+ // A S T P a r s i n g E v e n t s
+
+ public void consumeNode(Object t) {
+ StringBuffer buf = new StringBuffer(50);
+ buf.append("consumeNode");
+ serializeNode(buf, t);
+ transmit(buf.toString());
+ }
+
+ public void LT(int i, Object t) {
+ int ID = adaptor.getUniqueID(t);
+ String text = adaptor.getText(t);
+ int type = adaptor.getType(t);
+ StringBuffer buf = new StringBuffer(50);
+ buf.append("LN\t"); // lookahead node; distinguish from LT in protocol
+ buf.append(i);
+ serializeNode(buf, t);
+ transmit(buf.toString());
+ }
+
+ protected void serializeNode(StringBuffer buf, Object t) {
+ int ID = adaptor.getUniqueID(t);
+ String text = adaptor.getText(t);
+ int type = adaptor.getType(t);
+ buf.append("\t");
+ buf.append(ID);
+ buf.append("\t");
+ buf.append(type);
+ Token token = adaptor.getToken(t);
+ int line = -1;
+ int pos = -1;
+ if ( token!=null ) {
+ line = token.getLine();
+ pos = token.getCharPositionInLine();
+ }
+ buf.append("\t");
+ buf.append(line);
+ buf.append("\t");
+ buf.append(pos);
+ int tokenIndex = adaptor.getTokenStartIndex(t);
+ buf.append("\t");
+ buf.append(tokenIndex);
+ serializeText(buf, text);
+ }
+
+
+ // A S T E v e n t s
+
+ public void nilNode(Object t) {
+ int ID = adaptor.getUniqueID(t);
+ transmit("nilNode\t"+ID);
+ }
+
+ public void errorNode(Object t) {
+ int ID = adaptor.getUniqueID(t);
+ String text = t.toString();
+ StringBuffer buf = new StringBuffer(50);
+ buf.append("errorNode\t");
+ buf.append(ID);
+ buf.append("\t");
+ buf.append(Token.INVALID_TOKEN_TYPE);
+ serializeText(buf, text);
+ transmit(buf.toString());
+ }
+
+ public void createNode(Object t) {
+ int ID = adaptor.getUniqueID(t);
+ String text = adaptor.getText(t);
+ int type = adaptor.getType(t);
+ StringBuffer buf = new StringBuffer(50);
+ buf.append("createNodeFromTokenElements\t");
+ buf.append(ID);
+ buf.append("\t");
+ buf.append(type);
+ serializeText(buf, text);
+ transmit(buf.toString());
+ }
+
+ public void createNode(Object node, Token token) {
+ int ID = adaptor.getUniqueID(node);
+ int tokenIndex = token.getTokenIndex();
+ transmit("createNode\t"+ID+"\t"+tokenIndex);
+ }
+
+ public void becomeRoot(Object newRoot, Object oldRoot) {
+ int newRootID = adaptor.getUniqueID(newRoot);
+ int oldRootID = adaptor.getUniqueID(oldRoot);
+ transmit("becomeRoot\t"+newRootID+"\t"+oldRootID);
+ }
+
+ public void addChild(Object root, Object child) {
+ int rootID = adaptor.getUniqueID(root);
+ int childID = adaptor.getUniqueID(child);
+ transmit("addChild\t"+rootID+"\t"+childID);
+ }
+
+ public void setTokenBoundaries(Object t, int tokenStartIndex, int tokenStopIndex) {
+ int ID = adaptor.getUniqueID(t);
+ transmit("setTokenBoundaries\t"+ID+"\t"+tokenStartIndex+"\t"+tokenStopIndex);
+ }
+
+
+ // support
+
+ public void setTreeAdaptor(TreeAdaptor adaptor) { this.adaptor = adaptor; }
+ public TreeAdaptor getTreeAdaptor() { return adaptor; }
+
+ protected String serializeToken(Token t) {
+ StringBuffer buf = new StringBuffer(50);
+ buf.append(t.getTokenIndex()); buf.append('\t');
+ buf.append(t.getType()); buf.append('\t');
+ buf.append(t.getChannel()); buf.append('\t');
+ buf.append(t.getLine()); buf.append('\t');
+ buf.append(t.getCharPositionInLine());
+ serializeText(buf, t.getText());
+ return buf.toString();
+ }
+
+ protected void serializeText(StringBuffer buf, String text) {
+ buf.append("\t\"");
+ if ( text==null ) {
+ text = "";
+ }
+ // escape \n and \r all text for token appears to exist on one line
+ // this escape is slow but easy to understand
+ text = escapeNewlines(text);
+ buf.append(text);
+ }
+
+ protected String escapeNewlines(String txt) {
+ txt = txt.replaceAll("%","%25"); // escape all escape char ;)
+ txt = txt.replaceAll("\n","%0A"); // escape \n
+ txt = txt.replaceAll("\r","%0D"); // escape \r
+ return txt;
+ }
+}
+
diff --git a/src/org/antlr/runtime/debug/DebugParser.java b/src/org/antlr/runtime/debug/DebugParser.java
new file mode 100755
index 0000000..49d78e7
--- /dev/null
+++ b/src/org/antlr/runtime/debug/DebugParser.java
@@ -0,0 +1,98 @@
+/*
+ [The "BSD license"]
+ Copyright (c) 2005-2009 Terence Parr
+ All rights reserved.
+
+ Redistribution and use in source and binary forms, with or without
+ modification, are permitted provided that the following conditions
+ are met:
+ 1. Redistributions of source code must retain the above copyright
+ notice, this list of conditions and the following disclaimer.
+ 2. Redistributions in binary form must reproduce the above copyright
+ notice, this list of conditions and the following disclaimer in the
+ documentation and/or other materials provided with the distribution.
+ 3. The name of the author may not be used to endorse or promote products
+ derived from this software without specific prior written permission.
+
+ THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
+ IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
+ OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
+ IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
+ INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
+ NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
+ THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+package org.antlr.runtime.debug;
+
+import org.antlr.runtime.*;
+
+import java.io.IOException;
+
+public class DebugParser extends Parser {
+ /** Who to notify when events in the parser occur. */
+ protected DebugEventListener dbg = null;
+
+ /** Used to differentiate between fixed lookahead and cyclic DFA decisions
+ * while profiling.
+ */
+ public boolean isCyclicDecision = false;
+
+ /** Create a normal parser except wrap the token stream in a debug
+ * proxy that fires consume events.
+ */
+ public DebugParser(TokenStream input, DebugEventListener dbg, RecognizerSharedState state) {
+ super(input instanceof DebugTokenStream?input:new DebugTokenStream(input,dbg), state);
+ setDebugListener(dbg);
+ }
+
+ public DebugParser(TokenStream input, RecognizerSharedState state) {
+ super(input instanceof DebugTokenStream?input:new DebugTokenStream(input,null), state);
+ }
+
+ public DebugParser(TokenStream input, DebugEventListener dbg) {
+ this(input instanceof DebugTokenStream?input:new DebugTokenStream(input,dbg), dbg, null);
+ }
+
+ /** Provide a new debug event listener for this parser. Notify the
+ * input stream too that it should send events to this listener.
+ */
+ public void setDebugListener(DebugEventListener dbg) {
+ if ( input instanceof DebugTokenStream ) {
+ ((DebugTokenStream)input).setDebugListener(dbg);
+ }
+ this.dbg = dbg;
+ }
+
+ public DebugEventListener getDebugListener() {
+ return dbg;
+ }
+
+ public void reportError(IOException e) {
+ System.err.println(e);
+ e.printStackTrace(System.err);
+ }
+
+ public void beginResync() {
+ dbg.beginResync();
+ }
+
+ public void endResync() {
+ dbg.endResync();
+ }
+
+ public void beginBacktrack(int level) {
+ dbg.beginBacktrack(level);
+ }
+
+ public void endBacktrack(int level, boolean successful) {
+ dbg.endBacktrack(level,successful);
+ }
+
+ public void reportError(RecognitionException e) {
+ super.reportError(e);
+ dbg.recognitionException(e);
+ }
+}
diff --git a/src/org/antlr/runtime/debug/DebugTokenStream.java b/src/org/antlr/runtime/debug/DebugTokenStream.java
new file mode 100755
index 0000000..9a7a75f
--- /dev/null
+++ b/src/org/antlr/runtime/debug/DebugTokenStream.java
@@ -0,0 +1,156 @@
+/*
+ [The "BSD license"]
+ Copyright (c) 2005-2009 Terence Parr
+ All rights reserved.
+
+ Redistribution and use in source and binary forms, with or without
+ modification, are permitted provided that the following conditions
+ are met:
+ 1. Redistributions of source code must retain the above copyright
+ notice, this list of conditions and the following disclaimer.
+ 2. Redistributions in binary form must reproduce the above copyright
+ notice, this list of conditions and the following disclaimer in the
+ documentation and/or other materials provided with the distribution.
+ 3. The name of the author may not be used to endorse or promote products
+ derived from this software without specific prior written permission.
+
+ THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
+ IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
+ OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
+ IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
+ INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
+ NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
+ THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+package org.antlr.runtime.debug;
+
+import org.antlr.runtime.*;
+
+import java.util.List;
+
+public class DebugTokenStream implements TokenStream {
+ protected DebugEventListener dbg;
+ public TokenStream input;
+ protected boolean initialStreamState = true;
+
+ /** Track the last mark() call result value for use in rewind(). */
+ protected int lastMarker;
+
+ public DebugTokenStream(TokenStream input, DebugEventListener dbg) {
+ this.input = input;
+ setDebugListener(dbg);
+ // force TokenStream to get at least first valid token
+ // so we know if there are any hidden tokens first in the stream
+ input.LT(1);
+ }
+
+ public void setDebugListener(DebugEventListener dbg) {
+ this.dbg = dbg;
+ }
+
+ public void consume() {
+ if ( initialStreamState ) {
+ consumeInitialHiddenTokens();
+ }
+ int a = input.index();
+ Token t = input.LT(1);
+ input.consume();
+ int b = input.index();
+ dbg.consumeToken(t);
+ if ( b>a+1 ) {
+ // then we consumed more than one token; must be off channel tokens
+ for (int i=a+1; i<b; i++) {
+ dbg.consumeHiddenToken(input.get(i));
+ }
+ }
+ }
+
+ /* consume all initial off-channel tokens */
+ protected void consumeInitialHiddenTokens() {
+ int firstOnChannelTokenIndex = input.index();
+ for (int i=0; i<firstOnChannelTokenIndex; i++) {
+ dbg.consumeHiddenToken(input.get(i));
+ }
+ initialStreamState = false;
+ }
+
+ public Token LT(int i) {
+ if ( initialStreamState ) {
+ consumeInitialHiddenTokens();
+ }
+ dbg.LT(i, input.LT(i));
+ return input.LT(i);
+ }
+
+ public int LA(int i) {
+ if ( initialStreamState ) {
+ consumeInitialHiddenTokens();
+ }
+ dbg.LT(i, input.LT(i));
+ return input.LA(i);
+ }
+
+ public Token get(int i) {
+ return input.get(i);
+ }
+
+ public int mark() {
+ lastMarker = input.mark();
+ dbg.mark(lastMarker);
+ return lastMarker;
+ }
+
+ public int index() {
+ return input.index();
+ }
+
+ public int range() {
+ return input.range();
+ }
+
+ public void rewind(int marker) {
+ dbg.rewind(marker);
+ input.rewind(marker);
+ }
+
+ public void rewind() {
+ dbg.rewind();
+ input.rewind(lastMarker);
+ }
+
+ public void release(int marker) {
+ }
+
+ public void seek(int index) {
+ // TODO: implement seek in dbg interface
+ // db.seek(index);
+ input.seek(index);
+ }
+
+ public int size() {
+ return input.size();
+ }
+
+ public TokenSource getTokenSource() {
+ return input.getTokenSource();
+ }
+
+ public String getSourceName() {
+ return getTokenSource().getSourceName();
+ }
+
+ public String toString() {
+ return input.toString();
+ }
+
+ public String toString(int start, int stop) {
+ return input.toString(start,stop);
+ }
+
+ public String toString(Token start, Token stop) {
+ return input.toString(start,stop);
+ }
+}
diff --git a/src/org/antlr/runtime/debug/DebugTreeAdaptor.java b/src/org/antlr/runtime/debug/DebugTreeAdaptor.java
new file mode 100755
index 0000000..c72a2b6
--- /dev/null
+++ b/src/org/antlr/runtime/debug/DebugTreeAdaptor.java
@@ -0,0 +1,250 @@
+/*
+ [The "BSD license"]
+ Copyright (c) 2005-2009 Terence Parr
+ All rights reserved.
+
+ Redistribution and use in source and binary forms, with or without
+ modification, are permitted provided that the following conditions
+ are met:
+ 1. Redistributions of source code must retain the above copyright
+ notice, this list of conditions and the following disclaimer.
+ 2. Redistributions in binary form must reproduce the above copyright
+ notice, this list of conditions and the following disclaimer in the
+ documentation and/or other materials provided with the distribution.
+ 3. The name of the author may not be used to endorse or promote products
+ derived from this software without specific prior written permission.
+
+ THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
+ IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
+ OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
+ IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
+ INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
+ NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
+ THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+package org.antlr.runtime.debug;
+
+import org.antlr.runtime.Token;
+import org.antlr.runtime.TokenStream;
+import org.antlr.runtime.RecognitionException;
+import org.antlr.runtime.tree.TreeAdaptor;
+
+/** A TreeAdaptor proxy that fires debugging events to a DebugEventListener
+ * delegate and uses the TreeAdaptor delegate to do the actual work. All
+ * AST events are triggered by this adaptor; no code gen changes are needed
+ * in generated rules. Debugging events are triggered *after* invoking
+ * tree adaptor routines.
+ *
+ * Trees created with actions in rewrite actions like "-> ^(ADD {foo} {bar})"
+ * cannot be tracked as they might not use the adaptor to create foo, bar.
+ * The debug listener has to deal with tree node IDs for which it did
+ * not see a createNode event. A single <unknown> node is sufficient even
+ * if it represents a whole tree.
+ */
+public class DebugTreeAdaptor implements TreeAdaptor {
+ protected DebugEventListener dbg;
+ protected TreeAdaptor adaptor;
+
+ public DebugTreeAdaptor(DebugEventListener dbg, TreeAdaptor adaptor) {
+ this.dbg = dbg;
+ this.adaptor = adaptor;
+ }
+
+ public Object create(Token payload) {
+ if ( payload.getTokenIndex() < 0 ) {
+ // could be token conjured up during error recovery
+ return create(payload.getType(), payload.getText());
+ }
+ Object node = adaptor.create(payload);
+ dbg.createNode(node, payload);
+ return node;
+ }
+
+ public Object errorNode(TokenStream input, Token start, Token stop,
+ RecognitionException e)
+ {
+ Object node = adaptor.errorNode(input, start, stop, e);
+ if ( node!=null ) {
+ dbg.errorNode(node);
+ }
+ return node;
+ }
+
+ public Object dupTree(Object tree) {
+ Object t = adaptor.dupTree(tree);
+ // walk the tree and emit create and add child events
+ // to simulate what dupTree has done. dupTree does not call this debug
+ // adapter so I must simulate.
+ simulateTreeConstruction(t);
+ return t;
+ }
+
+ /** ^(A B C): emit create A, create B, add child, ...*/
+ protected void simulateTreeConstruction(Object t) {
+ dbg.createNode(t);
+ int n = adaptor.getChildCount(t);
+ for (int i=0; i<n; i++) {
+ Object child = adaptor.getChild(t, i);
+ simulateTreeConstruction(child);
+ dbg.addChild(t, child);
+ }
+ }
+
+ public Object dupNode(Object treeNode) {
+ Object d = adaptor.dupNode(treeNode);
+ dbg.createNode(d);
+ return d;
+ }
+
+ public Object nil() {
+ Object node = adaptor.nil();
+ dbg.nilNode(node);
+ return node;
+ }
+
+ public boolean isNil(Object tree) {
+ return adaptor.isNil(tree);
+ }
+
+ public void addChild(Object t, Object child) {
+ if ( t==null || child==null ) {
+ return;
+ }
+ adaptor.addChild(t,child);
+ dbg.addChild(t, child);
+ }
+
+ public Object becomeRoot(Object newRoot, Object oldRoot) {
+ Object n = adaptor.becomeRoot(newRoot, oldRoot);
+ dbg.becomeRoot(newRoot, oldRoot);
+ return n;
+ }
+
+ public Object rulePostProcessing(Object root) {
+ return adaptor.rulePostProcessing(root);
+ }
+
+ public void addChild(Object t, Token child) {
+ Object n = this.create(child);
+ this.addChild(t, n);
+ }
+
+ public Object becomeRoot(Token newRoot, Object oldRoot) {
+ Object n = this.create(newRoot);
+ adaptor.becomeRoot(n, oldRoot);
+ dbg.becomeRoot(newRoot, oldRoot);
+ return n;
+ }
+
+ public Object create(int tokenType, Token fromToken) {
+ Object node = adaptor.create(tokenType, fromToken);
+ dbg.createNode(node);
+ return node;
+ }
+
+ public Object create(int tokenType, Token fromToken, String text) {
+ Object node = adaptor.create(tokenType, fromToken, text);
+ dbg.createNode(node);
+ return node;
+ }
+
+ public Object create(int tokenType, String text) {
+ Object node = adaptor.create(tokenType, text);
+ dbg.createNode(node);
+ return node;
+ }
+
+ public int getType(Object t) {
+ return adaptor.getType(t);
+ }
+
+ public void setType(Object t, int type) {
+ adaptor.setType(t, type);
+ }
+
+ public String getText(Object t) {
+ return adaptor.getText(t);
+ }
+
+ public void setText(Object t, String text) {
+ adaptor.setText(t, text);
+ }
+
+ public Token getToken(Object t) {
+ return adaptor.getToken(t);
+ }
+
+ public void setTokenBoundaries(Object t, Token startToken, Token stopToken) {
+ adaptor.setTokenBoundaries(t, startToken, stopToken);
+ if ( t!=null && startToken!=null && stopToken!=null ) {
+ dbg.setTokenBoundaries(
+ t, startToken.getTokenIndex(),
+ stopToken.getTokenIndex());
+ }
+ }
+
+ public int getTokenStartIndex(Object t) {
+ return adaptor.getTokenStartIndex(t);
+ }
+
+ public int getTokenStopIndex(Object t) {
+ return adaptor.getTokenStopIndex(t);
+ }
+
+ public Object getChild(Object t, int i) {
+ return adaptor.getChild(t, i);
+ }
+
+ public void setChild(Object t, int i, Object child) {
+ adaptor.setChild(t, i, child);
+ }
+
+ public Object deleteChild(Object t, int i) {
+ return deleteChild(t, i);
+ }
+
+ public int getChildCount(Object t) {
+ return adaptor.getChildCount(t);
+ }
+
+ public int getUniqueID(Object node) {
+ return adaptor.getUniqueID(node);
+ }
+
+ public Object getParent(Object t) {
+ return adaptor.getParent(t);
+ }
+
+ public int getChildIndex(Object t) {
+ return adaptor.getChildIndex(t);
+ }
+
+ public void setParent(Object t, Object parent) {
+ adaptor.setParent(t, parent);
+ }
+
+ public void setChildIndex(Object t, int index) {
+ adaptor.setChildIndex(t, index);
+ }
+
+ public void replaceChildren(Object parent, int startChildIndex, int stopChildIndex, Object t) {
+ adaptor.replaceChildren(parent, startChildIndex, stopChildIndex, t);
+ }
+
+ // support
+
+ public DebugEventListener getDebugListener() {
+ return dbg;
+ }
+
+ public void setDebugListener(DebugEventListener dbg) {
+ this.dbg = dbg;
+ }
+
+ public TreeAdaptor getTreeAdaptor() {
+ return adaptor;
+ }
+}
diff --git a/src/org/antlr/runtime/debug/DebugTreeNodeStream.java b/src/org/antlr/runtime/debug/DebugTreeNodeStream.java
new file mode 100755
index 0000000..92ff009
--- /dev/null
+++ b/src/org/antlr/runtime/debug/DebugTreeNodeStream.java
@@ -0,0 +1,155 @@
+/*
+ [The "BSD license"]
+ Copyright (c) 2005-2009 Terence Parr
+ All rights reserved.
+
+ Redistribution and use in source and binary forms, with or without
+ modification, are permitted provided that the following conditions
+ are met:
+ 1. Redistributions of source code must retain the above copyright
+ notice, this list of conditions and the following disclaimer.
+ 2. Redistributions in binary form must reproduce the above copyright
+ notice, this list of conditions and the following disclaimer in the
+ documentation and/or other materials provided with the distribution.
+ 3. The name of the author may not be used to endorse or promote products
+ derived from this software without specific prior written permission.
+
+ THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
+ IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
+ OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
+ IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
+ INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
+ NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
+ THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+package org.antlr.runtime.debug;
+
+import org.antlr.runtime.tree.TreeAdaptor;
+import org.antlr.runtime.tree.TreeNodeStream;
+import org.antlr.runtime.TokenStream;
+
+/** Debug any tree node stream. The constructor accepts the stream
+ * and a debug listener. As node stream calls come in, debug events
+ * are triggered.
+ */
+public class DebugTreeNodeStream implements TreeNodeStream {
+ protected DebugEventListener dbg;
+ protected TreeAdaptor adaptor;
+ protected TreeNodeStream input;
+ protected boolean initialStreamState = true;
+
+ /** Track the last mark() call result value for use in rewind(). */
+ protected int lastMarker;
+
+ public DebugTreeNodeStream(TreeNodeStream input,
+ DebugEventListener dbg)
+ {
+ this.input = input;
+ this.adaptor = input.getTreeAdaptor();
+ this.input.setUniqueNavigationNodes(true);
+ setDebugListener(dbg);
+ }
+
+ public void setDebugListener(DebugEventListener dbg) {
+ this.dbg = dbg;
+ }
+
+ public TreeAdaptor getTreeAdaptor() {
+ return adaptor;
+ }
+
+ public void consume() {
+ Object node = input.LT(1);
+ input.consume();
+ dbg.consumeNode(node);
+ }
+
+ public Object get(int i) {
+ return input.get(i);
+ }
+
+ public Object LT(int i) {
+ Object node = input.LT(i);
+ int ID = adaptor.getUniqueID(node);
+ String text = adaptor.getText(node);
+ int type = adaptor.getType(node);
+ dbg.LT(i, node);
+ return node;
+ }
+
+ public int LA(int i) {
+ Object node = input.LT(i);
+ int ID = adaptor.getUniqueID(node);
+ String text = adaptor.getText(node);
+ int type = adaptor.getType(node);
+ dbg.LT(i, node);
+ return type;
+ }
+
+ public int mark() {
+ lastMarker = input.mark();
+ dbg.mark(lastMarker);
+ return lastMarker;
+ }
+
+ public int index() {
+ return input.index();
+ }
+
+ public void rewind(int marker) {
+ dbg.rewind(marker);
+ input.rewind(marker);
+ }
+
+ public void rewind() {
+ dbg.rewind();
+ input.rewind(lastMarker);
+ }
+
+ public void release(int marker) {
+ }
+
+ public void seek(int index) {
+ // TODO: implement seek in dbg interface
+ // db.seek(index);
+ input.seek(index);
+ }
+
+ public int size() {
+ return input.size();
+ }
+
+ public void reset() { ; }
+
+ public Object getTreeSource() {
+ return input;
+ }
+
+ public String getSourceName() {
+ return getTokenStream().getSourceName();
+ }
+
+ public TokenStream getTokenStream() {
+ return input.getTokenStream();
+ }
+
+ /** It is normally this object that instructs the node stream to
+ * create unique nav nodes, but to satisfy interface, we have to
+ * define it. It might be better to ignore the parameter but
+ * there might be a use for it later, so I'll leave.
+ */
+ public void setUniqueNavigationNodes(boolean uniqueNavigationNodes) {
+ input.setUniqueNavigationNodes(uniqueNavigationNodes);
+ }
+
+ public void replaceChildren(Object parent, int startChildIndex, int stopChildIndex, Object t) {
+ input.replaceChildren(parent, startChildIndex, stopChildIndex, t);
+ }
+
+ public String toString(Object start, Object stop) {
+ return input.toString(start,stop);
+ }
+}
diff --git a/src/org/antlr/runtime/debug/DebugTreeParser.java b/src/org/antlr/runtime/debug/DebugTreeParser.java
new file mode 100755
index 0000000..6e1ece8
--- /dev/null
+++ b/src/org/antlr/runtime/debug/DebugTreeParser.java
@@ -0,0 +1,109 @@
+/*
+ [The "BSD license"]
+ Copyright (c) 2005-2009 Terence Parr
+ All rights reserved.
+
+ Redistribution and use in source and binary forms, with or without
+ modification, are permitted provided that the following conditions
+ are met:
+ 1. Redistributions of source code must retain the above copyright
+ notice, this list of conditions and the following disclaimer.
+ 2. Redistributions in binary form must reproduce the above copyright
+ notice, this list of conditions and the following disclaimer in the
+ documentation and/or other materials provided with the distribution.
+ 3. The name of the author may not be used to endorse or promote products
+ derived from this software without specific prior written permission.
+
+ THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
+ IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
+ OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
+ IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
+ INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
+ NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
+ THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+package org.antlr.runtime.debug;
+
+import org.antlr.runtime.*;
+import org.antlr.runtime.tree.TreeNodeStream;
+import org.antlr.runtime.tree.TreeParser;
+
+import java.io.IOException;
+
+public class DebugTreeParser extends TreeParser {
+ /** Who to notify when events in the parser occur. */
+ protected DebugEventListener dbg = null;
+
+ /** Used to differentiate between fixed lookahead and cyclic DFA decisions
+ * while profiling.
+ */
+ public boolean isCyclicDecision = false;
+
+ /** Create a normal parser except wrap the token stream in a debug
+ * proxy that fires consume events.
+ */
+ public DebugTreeParser(TreeNodeStream input, DebugEventListener dbg, RecognizerSharedState state) {
+ super(input instanceof DebugTreeNodeStream?input:new DebugTreeNodeStream(input,dbg), state);
+ setDebugListener(dbg);
+ }
+
+ public DebugTreeParser(TreeNodeStream input, RecognizerSharedState state) {
+ super(input instanceof DebugTreeNodeStream?input:new DebugTreeNodeStream(input,null), state);
+ }
+
+ public DebugTreeParser(TreeNodeStream input, DebugEventListener dbg) {
+ this(input instanceof DebugTreeNodeStream?input:new DebugTreeNodeStream(input,dbg), dbg, null);
+ }
+
+ /** Provide a new debug event listener for this parser. Notify the
+ * input stream too that it should send events to this listener.
+ */
+ public void setDebugListener(DebugEventListener dbg) {
+ if ( input instanceof DebugTreeNodeStream ) {
+ ((DebugTreeNodeStream)input).setDebugListener(dbg);
+ }
+ this.dbg = dbg;
+ }
+
+ public DebugEventListener getDebugListener() {
+ return dbg;
+ }
+
+ public void reportError(IOException e) {
+ System.err.println(e);
+ e.printStackTrace(System.err);
+ }
+
+ public void reportError(RecognitionException e) {
+ dbg.recognitionException(e);
+ }
+
+ protected Object getMissingSymbol(IntStream input,
+ RecognitionException e,
+ int expectedTokenType,
+ BitSet follow)
+ {
+ Object o = super.getMissingSymbol(input, e, expectedTokenType, follow);
+ dbg.consumeNode(o);
+ return o;
+ }
+
+ public void beginResync() {
+ dbg.beginResync();
+ }
+
+ public void endResync() {
+ dbg.endResync();
+ }
+
+ public void beginBacktrack(int level) {
+ dbg.beginBacktrack(level);
+ }
+
+ public void endBacktrack(int level, boolean successful) {
+ dbg.endBacktrack(level,successful);
+ }
+}
diff --git a/src/org/antlr/runtime/debug/ParseTreeBuilder.java b/src/org/antlr/runtime/debug/ParseTreeBuilder.java
new file mode 100755
index 0000000..13c6ed0
--- /dev/null
+++ b/src/org/antlr/runtime/debug/ParseTreeBuilder.java
@@ -0,0 +1,109 @@
+/*
+ [The "BSD license"]
+ Copyright (c) 2005-2009 Terence Parr
+ All rights reserved.
+
+ Redistribution and use in source and binary forms, with or without
+ modification, are permitted provided that the following conditions
+ are met:
+ 1. Redistributions of source code must retain the above copyright
+ notice, this list of conditions and the following disclaimer.
+ 2. Redistributions in binary form must reproduce the above copyright
+ notice, this list of conditions and the following disclaimer in the
+ documentation and/or other materials provided with the distribution.
+ 3. The name of the author may not be used to endorse or promote products
+ derived from this software without specific prior written permission.
+
+ THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
+ IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
+ OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
+ IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
+ INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
+ NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
+ THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+package org.antlr.runtime.debug;
+
+import org.antlr.runtime.RecognitionException;
+import org.antlr.runtime.Token;
+import org.antlr.runtime.tree.ParseTree;
+
+import java.util.Stack;
+import java.util.ArrayList;
+import java.util.List;
+
+/** This parser listener tracks rule entry/exit and token matches
+ * to build a simple parse tree using ParseTree nodes.
+ */
+public class ParseTreeBuilder extends BlankDebugEventListener {
+ public static final String EPSILON_PAYLOAD = "<epsilon>";
+
+ Stack callStack = new Stack();
+ List hiddenTokens = new ArrayList();
+ int backtracking = 0;
+
+ public ParseTreeBuilder(String grammarName) {
+ ParseTree root = create("<grammar "+grammarName+">");
+ callStack.push(root);
+ }
+
+ public ParseTree getTree() {
+ return (ParseTree)callStack.elementAt(0);
+ }
+
+ /** What kind of node to create. You might want to override
+ * so I factored out creation here.
+ */
+ public ParseTree create(Object payload) {
+ return new ParseTree(payload);
+ }
+
+ public ParseTree epsilonNode() {
+ return create(EPSILON_PAYLOAD);
+ }
+
+ /** Backtracking or cyclic DFA, don't want to add nodes to tree */
+ public void enterDecision(int d, boolean couldBacktrack) { backtracking++; }
+ public void exitDecision(int i) { backtracking--; }
+
+ public void enterRule(String filename, String ruleName) {
+ if ( backtracking>0 ) return;
+ ParseTree parentRuleNode = (ParseTree)callStack.peek();
+ ParseTree ruleNode = create(ruleName);
+ parentRuleNode.addChild(ruleNode);
+ callStack.push(ruleNode);
+ }
+
+ public void exitRule(String filename, String ruleName) {
+ if ( backtracking>0 ) return;
+ ParseTree ruleNode = (ParseTree)callStack.peek();
+ if ( ruleNode.getChildCount()==0 ) {
+ ruleNode.addChild(epsilonNode());
+ }
+ callStack.pop();
+ }
+
+ public void consumeToken(Token token) {
+ if ( backtracking>0 ) return;
+ ParseTree ruleNode = (ParseTree)callStack.peek();
+ ParseTree elementNode = create(token);
+ elementNode.hiddenTokens = this.hiddenTokens;
+ this.hiddenTokens = new ArrayList();
+ ruleNode.addChild(elementNode);
+ }
+
+ public void consumeHiddenToken(Token token) {
+ if ( backtracking>0 ) return;
+ hiddenTokens.add(token);
+ }
+
+ public void recognitionException(RecognitionException e) {
+ if ( backtracking>0 ) return;
+ ParseTree ruleNode = (ParseTree)callStack.peek();
+ ParseTree errorNode = create(e);
+ ruleNode.addChild(errorNode);
+ }
+}
diff --git a/src/org/antlr/runtime/debug/Profiler.java b/src/org/antlr/runtime/debug/Profiler.java
new file mode 100755
index 0000000..aea9a17
--- /dev/null
+++ b/src/org/antlr/runtime/debug/Profiler.java
@@ -0,0 +1,734 @@
+/*
+ [The "BSD license"]
+ Copyright (c) 2005-2009 Terence Parr
+ All rights reserved.
+
+ Redistribution and use in source and binary forms, with or without
+ modification, are permitted provided that the following conditions
+ are met:
+ 1. Redistributions of source code must retain the above copyright
+ notice, this list of conditions and the following disclaimer.
+ 2. Redistributions in binary form must reproduce the above copyright
+ notice, this list of conditions and the following disclaimer in the
+ documentation and/or other materials provided with the distribution.
+ 3. The name of the author may not be used to endorse or promote products
+ derived from this software without specific prior written permission.
+
+ THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
+ IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
+ OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
+ IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
+ INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
+ NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
+ THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+package org.antlr.runtime.debug;
+
+import org.antlr.runtime.*;
+import org.antlr.runtime.misc.DoubleKeyMap;
+
+import java.util.*;
+
+/** Using the debug event interface, track what is happening in the parser
+ * and record statistics about the runtime.
+ */
+public class Profiler extends BlankDebugEventListener {
+ public static final String DATA_SEP = "\t";
+ public static final String newline = System.getProperty("line.separator");
+
+ static boolean dump = false;
+
+ public static class ProfileStats {
+ public String Version;
+ public String name;
+ public int numRuleInvocations;
+ public int numUniqueRulesInvoked;
+ public int numDecisionEvents;
+ public int numDecisionsCovered;
+ public int numDecisionsThatPotentiallyBacktrack;
+ public int numDecisionsThatDoBacktrack;
+ public int maxRuleInvocationDepth;
+ public float avgkPerDecisionEvent;
+ public float avgkPerBacktrackingDecisionEvent;
+ public float averageDecisionPercentBacktracks;
+ public int numBacktrackOccurrences; // doesn't count gated DFA edges
+
+ public int numFixedDecisions;
+ public int minDecisionMaxFixedLookaheads;
+ public int maxDecisionMaxFixedLookaheads;
+ public int avgDecisionMaxFixedLookaheads;
+ public int stddevDecisionMaxFixedLookaheads;
+ public int numCyclicDecisions;
+ public int minDecisionMaxCyclicLookaheads;
+ public int maxDecisionMaxCyclicLookaheads;
+ public int avgDecisionMaxCyclicLookaheads;
+ public int stddevDecisionMaxCyclicLookaheads;
+// int Stats.min(toArray(decisionMaxSynPredLookaheads);
+// int Stats.max(toArray(decisionMaxSynPredLookaheads);
+// int Stats.avg(toArray(decisionMaxSynPredLookaheads);
+// int Stats.stddev(toArray(decisionMaxSynPredLookaheads);
+ public int numSemanticPredicates;
+ public int numTokens;
+ public int numHiddenTokens;
+ public int numCharsMatched;
+ public int numHiddenCharsMatched;
+ public int numReportedErrors;
+ public int numMemoizationCacheHits;
+ public int numMemoizationCacheMisses;
+ public int numGuessingRuleInvocations;
+ public int numMemoizationCacheEntries;
+ }
+
+ public static class DecisionDescriptor {
+ public int decision;
+ public String fileName;
+ public String ruleName;
+ public int line;
+ public int pos;
+ public boolean couldBacktrack;
+
+ public int n;
+ public float avgk; // avg across all decision events
+ public int maxk;
+ public int numBacktrackOccurrences;
+ public int numSemPredEvals;
+ }
+
+ // all about a specific exec of a single decision
+ public static class DecisionEvent {
+ public DecisionDescriptor decision;
+ public int startIndex;
+ public int k;
+ public boolean backtracks; // doesn't count gated DFA edges
+ public boolean evalSemPred;
+ public long startTime;
+ public long stopTime;
+ public int numMemoizationCacheHits;
+ public int numMemoizationCacheMisses;
+ }
+
+ /** Because I may change the stats, I need to track that for later
+ * computations to be consistent.
+ */
+ public static final String Version = "3";
+ public static final String RUNTIME_STATS_FILENAME = "runtime.stats";
+
+ /** Ack, should not store parser; can't do remote stuff. Well, we pass
+ * input stream around too so I guess it's ok.
+ */
+ public DebugParser parser = null;
+
+ // working variables
+
+ protected int ruleLevel = 0;
+ //protected int decisionLevel = 0;
+ protected Token lastRealTokenTouchedInDecision;
+ protected Set<String> uniqueRules = new HashSet<String>();
+ protected Stack<String> currentGrammarFileName = new Stack();
+ protected Stack<String> currentRuleName = new Stack();
+ protected Stack<Integer> currentLine = new Stack();
+ protected Stack<Integer> currentPos = new Stack();
+
+ // Vector<DecisionStats>
+ //protected Vector decisions = new Vector(200); // need setSize
+ protected DoubleKeyMap<String,Integer, DecisionDescriptor> decisions =
+ new DoubleKeyMap<String,Integer, DecisionDescriptor>();
+
+ // Record a DecisionData for each decision we hit while parsing
+ protected List<DecisionEvent> decisionEvents = new ArrayList<DecisionEvent>();
+ protected Stack<DecisionEvent> decisionStack = new Stack<DecisionEvent>();
+
+ protected int backtrackDepth;
+
+ ProfileStats stats = new ProfileStats();
+
+ public Profiler() {
+ }
+
+ public Profiler(DebugParser parser) {
+ this.parser = parser;
+ }
+
+ public void enterRule(String grammarFileName, String ruleName) {
+// System.out.println("enterRule "+grammarFileName+":"+ruleName);
+ ruleLevel++;
+ stats.numRuleInvocations++;
+ uniqueRules.add(grammarFileName+":"+ruleName);
+ stats.maxRuleInvocationDepth = Math.max(stats.maxRuleInvocationDepth, ruleLevel);
+ currentGrammarFileName.push( grammarFileName );
+ currentRuleName.push( ruleName );
+ }
+
+ public void exitRule(String grammarFileName, String ruleName) {
+ ruleLevel--;
+ currentGrammarFileName.pop();
+ currentRuleName.pop();
+ }
+
+ /** Track memoization; this is not part of standard debug interface
+ * but is triggered by profiling. Code gen inserts an override
+ * for this method in the recognizer, which triggers this method.
+ * Called from alreadyParsedRule().
+ */
+ public void examineRuleMemoization(IntStream input,
+ int ruleIndex,
+ int stopIndex, // index or MEMO_RULE_UNKNOWN...
+ String ruleName)
+ {
+ if (dump) System.out.println("examine memo "+ruleName+" at "+input.index()+": "+stopIndex);
+ if ( stopIndex==BaseRecognizer.MEMO_RULE_UNKNOWN ) {
+ //System.out.println("rule "+ruleIndex+" missed @ "+input.index());
+ stats.numMemoizationCacheMisses++;
+ stats.numGuessingRuleInvocations++; // we'll have to enter
+ currentDecision().numMemoizationCacheMisses++;
+ }
+ else {
+ // regardless of rule success/failure, if in cache, we have a cache hit
+ //System.out.println("rule "+ruleIndex+" hit @ "+input.index());
+ stats.numMemoizationCacheHits++;
+ currentDecision().numMemoizationCacheHits++;
+ }
+ }
+
+ /** Warning: doesn't track success/failure, just unique recording event */
+ public void memoize(IntStream input,
+ int ruleIndex,
+ int ruleStartIndex,
+ String ruleName)
+ {
+ // count how many entries go into table
+ if (dump) System.out.println("memoize "+ruleName);
+ stats.numMemoizationCacheEntries++;
+ }
+
+ @Override
+ public void location(int line, int pos) {
+ currentLine.push(line);
+ currentPos.push(pos);
+ }
+
+ public void enterDecision(int decisionNumber, boolean couldBacktrack) {
+ lastRealTokenTouchedInDecision = null;
+ stats.numDecisionEvents++;
+ int startingLookaheadIndex = parser.getTokenStream().index();
+ TokenStream input = parser.getTokenStream();
+ if ( dump ) System.out.println("enterDecision canBacktrack="+couldBacktrack+" "+ decisionNumber +
+ " backtrack depth " + backtrackDepth +
+ " @ " + input.get(input.index()) +
+ " rule " +locationDescription());
+ String g = (String) currentGrammarFileName.peek();
+ DecisionDescriptor descriptor = decisions.get(g, decisionNumber);
+ if ( descriptor == null ) {
+ descriptor = new DecisionDescriptor();
+ decisions.put(g, decisionNumber, descriptor);
+ descriptor.decision = decisionNumber;
+ descriptor.fileName = (String)currentGrammarFileName.peek();
+ descriptor.ruleName = (String)currentRuleName.peek();
+ descriptor.line = (Integer)currentLine.peek();
+ descriptor.pos = (Integer)currentPos.peek();
+ descriptor.couldBacktrack = couldBacktrack;
+ }
+ descriptor.n++;
+
+ DecisionEvent d = new DecisionEvent();
+ decisionStack.push(d);
+ d.decision = descriptor;
+ d.startTime = System.currentTimeMillis();
+ d.startIndex = startingLookaheadIndex;
+ }
+
+ public void exitDecision(int decisionNumber) {
+ DecisionEvent d = decisionStack.pop();
+ d.stopTime = System.currentTimeMillis();
+
+ int lastTokenIndex = lastRealTokenTouchedInDecision.getTokenIndex();
+ int numHidden = getNumberOfHiddenTokens(d.startIndex, lastTokenIndex);
+ int depth = lastTokenIndex - d.startIndex - numHidden + 1; // +1 counts consuming start token as 1
+ d.k = depth;
+ d.decision.maxk = Math.max(d.decision.maxk, depth);
+
+ if (dump) System.out.println("exitDecision "+decisionNumber+" in "+d.decision.ruleName+
+ " lookahead "+d.k +" max token "+lastRealTokenTouchedInDecision);
+ decisionEvents.add(d); // done with decision; track all
+ }
+
+ public void consumeToken(Token token) {
+ if (dump) System.out.println("consume token "+token);
+ if ( !inDecision() ) {
+ stats.numTokens++;
+ return;
+ }
+ if ( lastRealTokenTouchedInDecision==null ||
+ lastRealTokenTouchedInDecision.getTokenIndex() < token.getTokenIndex() )
+ {
+ lastRealTokenTouchedInDecision = token;
+ }
+ DecisionEvent d = currentDecision();
+ // compute lookahead depth
+ int thisRefIndex = token.getTokenIndex();
+ int numHidden = getNumberOfHiddenTokens(d.startIndex, thisRefIndex);
+ int depth = thisRefIndex - d.startIndex - numHidden + 1; // +1 counts consuming start token as 1
+ //d.maxk = Math.max(d.maxk, depth);
+ if (dump) System.out.println("consume "+thisRefIndex+" "+depth+" tokens ahead in "+
+ d.decision.ruleName+"-"+d.decision.decision+" start index "+d.startIndex);
+ }
+
+ /** The parser is in a decision if the decision depth > 0. This
+ * works for backtracking also, which can have nested decisions.
+ */
+ public boolean inDecision() {
+ return decisionStack.size()>0;
+ }
+
+ public void consumeHiddenToken(Token token) {
+ //System.out.println("consume hidden token "+token);
+ if ( !inDecision() ) stats.numHiddenTokens++;
+ }
+
+ /** Track refs to lookahead if in a fixed/nonfixed decision.
+ */
+ public void LT(int i, Token t) {
+ if ( inDecision() && i>0 ) {
+ DecisionEvent d = currentDecision();
+ if (dump) System.out.println("LT("+i+")="+t+" index "+t.getTokenIndex()+" relative to "+d.decision.ruleName+"-"+
+ d.decision.decision+" start index "+d.startIndex);
+ if ( lastRealTokenTouchedInDecision==null ||
+ lastRealTokenTouchedInDecision.getTokenIndex() < t.getTokenIndex() )
+ {
+ lastRealTokenTouchedInDecision = t;
+ if (dump) System.out.println("set last token "+lastRealTokenTouchedInDecision);
+ }
+ // get starting index off stack
+// int stackTop = lookaheadStack.size()-1;
+// Integer startingIndex = (Integer)lookaheadStack.get(stackTop);
+// // compute lookahead depth
+// int thisRefIndex = parser.getTokenStream().index();
+// int numHidden =
+// getNumberOfHiddenTokens(startingIndex.intValue(), thisRefIndex);
+// int depth = i + thisRefIndex - startingIndex.intValue() - numHidden;
+// /*
+// System.out.println("LT("+i+") @ index "+thisRefIndex+" is depth "+depth+
+// " max is "+maxLookaheadInCurrentDecision);
+// */
+// if ( depth>maxLookaheadInCurrentDecision ) {
+// maxLookaheadInCurrentDecision = depth;
+// }
+// d.maxk = currentDecision()/
+ }
+ }
+
+ /** Track backtracking decisions. You'll see a fixed or cyclic decision
+ * and then a backtrack.
+ *
+ * enter rule
+ * ...
+ * enter decision
+ * LA and possibly consumes (for cyclic DFAs)
+ * begin backtrack level
+ * mark m
+ * rewind m
+ * end backtrack level, success
+ * exit decision
+ * ...
+ * exit rule
+ */
+ public void beginBacktrack(int level) {
+ if (dump) System.out.println("enter backtrack "+level);
+ backtrackDepth++;
+ DecisionEvent e = currentDecision();
+ if ( e.decision.couldBacktrack ) {
+ stats.numBacktrackOccurrences++;
+ e.decision.numBacktrackOccurrences++;
+ e.backtracks = true;
+ }
+ }
+
+ /** Successful or not, track how much lookahead synpreds use */
+ public void endBacktrack(int level, boolean successful) {
+ if (dump) System.out.println("exit backtrack "+level+": "+successful);
+ backtrackDepth--;
+ }
+
+ @Override
+ public void mark(int i) {
+ if (dump) System.out.println("mark "+i);
+ }
+
+ @Override
+ public void rewind(int i) {
+ if (dump) System.out.println("rewind "+i);
+ }
+
+ @Override
+ public void rewind() {
+ if (dump) System.out.println("rewind");
+ }
+
+
+
+ protected DecisionEvent currentDecision() {
+ return decisionStack.peek();
+ }
+
+ public void recognitionException(RecognitionException e) {
+ stats.numReportedErrors++;
+ }
+
+ public void semanticPredicate(boolean result, String predicate) {
+ stats.numSemanticPredicates++;
+ if ( inDecision() ) {
+ DecisionEvent d = currentDecision();
+ d.evalSemPred = true;
+ d.decision.numSemPredEvals++;
+ if (dump) System.out.println("eval "+predicate+" in "+d.decision.ruleName+"-"+
+ d.decision.decision);
+ }
+ }
+
+ public void terminate() {
+ for (DecisionEvent e : decisionEvents) {
+ //System.out.println("decision "+e.decision.decision+": k="+e.k);
+ e.decision.avgk += e.k;
+ stats.avgkPerDecisionEvent += e.k;
+ if ( e.backtracks ) { // doesn't count gated syn preds on DFA edges
+ stats.avgkPerBacktrackingDecisionEvent += e.k;
+ }
+ }
+ stats.averageDecisionPercentBacktracks = 0.0f;
+ for (DecisionDescriptor d : decisions.values()) {
+ stats.numDecisionsCovered++;
+ d.avgk /= (double)d.n;
+ if ( d.couldBacktrack ) {
+ stats.numDecisionsThatPotentiallyBacktrack++;
+ float percentBacktracks = d.numBacktrackOccurrences / (float)d.n;
+ //System.out.println("dec "+d.decision+" backtracks "+percentBacktracks*100+"%");
+ stats.averageDecisionPercentBacktracks += percentBacktracks;
+ }
+ // ignore rules that backtrack along gated DFA edges
+ if ( d.numBacktrackOccurrences > 0 ) {
+ stats.numDecisionsThatDoBacktrack++;
+ }
+ }
+ stats.averageDecisionPercentBacktracks /= stats.numDecisionsThatPotentiallyBacktrack;
+ stats.averageDecisionPercentBacktracks *= 100; // it's a percentage
+ stats.avgkPerDecisionEvent /= stats.numDecisionEvents;
+ stats.avgkPerBacktrackingDecisionEvent /= (double)stats.numBacktrackOccurrences;
+
+ System.err.println(toString());
+ System.err.println(getDecisionStatsDump());
+
+// String stats = toNotifyString();
+// try {
+// Stats.writeReport(RUNTIME_STATS_FILENAME,stats);
+// }
+// catch (IOException ioe) {
+// System.err.println(ioe);
+// ioe.printStackTrace(System.err);
+// }
+ }
+
+ public void setParser(DebugParser parser) {
+ this.parser = parser;
+ }
+
+ // R E P O R T I N G
+
+ public String toNotifyString() {
+ StringBuffer buf = new StringBuffer();
+ buf.append(Version);
+ buf.append('\t');
+ buf.append(parser.getClass().getName());
+// buf.append('\t');
+// buf.append(numRuleInvocations);
+// buf.append('\t');
+// buf.append(maxRuleInvocationDepth);
+// buf.append('\t');
+// buf.append(numFixedDecisions);
+// buf.append('\t');
+// buf.append(Stats.min(decisionMaxFixedLookaheads));
+// buf.append('\t');
+// buf.append(Stats.max(decisionMaxFixedLookaheads));
+// buf.append('\t');
+// buf.append(Stats.avg(decisionMaxFixedLookaheads));
+// buf.append('\t');
+// buf.append(Stats.stddev(decisionMaxFixedLookaheads));
+// buf.append('\t');
+// buf.append(numCyclicDecisions);
+// buf.append('\t');
+// buf.append(Stats.min(decisionMaxCyclicLookaheads));
+// buf.append('\t');
+// buf.append(Stats.max(decisionMaxCyclicLookaheads));
+// buf.append('\t');
+// buf.append(Stats.avg(decisionMaxCyclicLookaheads));
+// buf.append('\t');
+// buf.append(Stats.stddev(decisionMaxCyclicLookaheads));
+// buf.append('\t');
+// buf.append(numBacktrackDecisions);
+// buf.append('\t');
+// buf.append(Stats.min(toArray(decisionMaxSynPredLookaheads)));
+// buf.append('\t');
+// buf.append(Stats.max(toArray(decisionMaxSynPredLookaheads)));
+// buf.append('\t');
+// buf.append(Stats.avg(toArray(decisionMaxSynPredLookaheads)));
+// buf.append('\t');
+// buf.append(Stats.stddev(toArray(decisionMaxSynPredLookaheads)));
+// buf.append('\t');
+// buf.append(numSemanticPredicates);
+// buf.append('\t');
+// buf.append(parser.getTokenStream().size());
+// buf.append('\t');
+// buf.append(numHiddenTokens);
+// buf.append('\t');
+// buf.append(numCharsMatched);
+// buf.append('\t');
+// buf.append(numHiddenCharsMatched);
+// buf.append('\t');
+// buf.append(numberReportedErrors);
+// buf.append('\t');
+// buf.append(numMemoizationCacheHits);
+// buf.append('\t');
+// buf.append(numMemoizationCacheMisses);
+// buf.append('\t');
+// buf.append(numGuessingRuleInvocations);
+// buf.append('\t');
+// buf.append(numMemoizationCacheEntries);
+ return buf.toString();
+ }
+
+ public String toString() {
+ return toString(getReport());
+ }
+
+ public ProfileStats getReport() {
+// TokenStream input = parser.getTokenStream();
+// for (int i=0; i<input.size()&& lastRealTokenTouchedInDecision !=null&&i<= lastRealTokenTouchedInDecision.getTokenIndex(); i++) {
+// Token t = input.get(i);
+// if ( t.getChannel()!=Token.DEFAULT_CHANNEL ) {
+// stats.numHiddenTokens++;
+// stats.numHiddenCharsMatched += t.getText().length();
+// }
+// }
+ stats.Version = Version;
+ stats.name = parser.getClass().getName();
+ stats.numUniqueRulesInvoked = uniqueRules.size();
+ //stats.numCharsMatched = lastTokenConsumed.getStopIndex() + 1;
+ return stats;
+ }
+
+ public DoubleKeyMap getDecisionStats() {
+ return decisions;
+ }
+
+ public List getDecisionEvents() {
+ return decisionEvents;
+ }
+
+ public static String toString(ProfileStats stats) {
+ StringBuffer buf = new StringBuffer();
+ buf.append("ANTLR Runtime Report; Profile Version ");
+ buf.append(stats.Version);
+ buf.append(newline);
+ buf.append("parser name ");
+ buf.append(stats.name);
+ buf.append(newline);
+ buf.append("Number of rule invocations ");
+ buf.append(stats.numRuleInvocations);
+ buf.append(newline);
+ buf.append("Number of unique rules visited ");
+ buf.append(stats.numUniqueRulesInvoked);
+ buf.append(newline);
+ buf.append("Number of decision events ");
+ buf.append(stats.numDecisionEvents);
+ buf.append(newline);
+ buf.append("Overall average k per decision event ");
+ buf.append(stats.avgkPerDecisionEvent);
+ buf.append(newline);
+ buf.append("Number of backtracking occurrences (can be multiple per decision) ");
+ buf.append(stats.numBacktrackOccurrences);
+ buf.append(newline);
+ buf.append("Overall average k per decision event that backtracks ");
+ buf.append(stats.avgkPerBacktrackingDecisionEvent);
+ buf.append(newline);
+ buf.append("Number of rule invocations while backtracking ");
+ buf.append(stats.numGuessingRuleInvocations);
+ buf.append(newline);
+ buf.append("num decisions that potentially backtrack ");
+ buf.append(stats.numDecisionsThatPotentiallyBacktrack);
+ buf.append(newline);
+ buf.append("num decisions that do backtrack ");
+ buf.append(stats.numDecisionsThatDoBacktrack);
+ buf.append(newline);
+ buf.append("num decisions that potentially backtrack but don't ");
+ buf.append(stats.numDecisionsThatPotentiallyBacktrack - stats.numDecisionsThatDoBacktrack);
+ buf.append(newline);
+ buf.append("average % of time a potentially backtracking decision backtracks ");
+ buf.append(stats.averageDecisionPercentBacktracks);
+ buf.append(newline);
+ buf.append("num unique decisions covered ");
+ buf.append(stats.numDecisionsCovered);
+ buf.append(newline);
+ buf.append("max rule invocation nesting depth ");
+ buf.append(stats.maxRuleInvocationDepth);
+ buf.append(newline);
+
+// buf.append("number of fixed lookahead decisions ");
+// buf.append();
+// buf.append('\n');
+// buf.append("min lookahead used in a fixed lookahead decision ");
+// buf.append();
+// buf.append('\n');
+// buf.append("max lookahead used in a fixed lookahead decision ");
+// buf.append();
+// buf.append('\n');
+// buf.append("average lookahead depth used in fixed lookahead decisions ");
+// buf.append();
+// buf.append('\n');
+// buf.append("standard deviation of depth used in fixed lookahead decisions ");
+// buf.append();
+// buf.append('\n');
+// buf.append("number of arbitrary lookahead decisions ");
+// buf.append();
+// buf.append('\n');
+// buf.append("min lookahead used in an arbitrary lookahead decision ");
+// buf.append();
+// buf.append('\n');
+// buf.append("max lookahead used in an arbitrary lookahead decision ");
+// buf.append();
+// buf.append('\n');
+// buf.append("average lookahead depth used in arbitrary lookahead decisions ");
+// buf.append();
+// buf.append('\n');
+// buf.append("standard deviation of depth used in arbitrary lookahead decisions ");
+// buf.append();
+// buf.append('\n');
+// buf.append("number of evaluated syntactic predicates ");
+// buf.append();
+// buf.append('\n');
+// buf.append("min lookahead used in a syntactic predicate ");
+// buf.append();
+// buf.append('\n');
+// buf.append("max lookahead used in a syntactic predicate ");
+// buf.append();
+// buf.append('\n');
+// buf.append("average lookahead depth used in syntactic predicates ");
+// buf.append();
+// buf.append('\n');
+// buf.append("standard deviation of depth used in syntactic predicates ");
+// buf.append();
+// buf.append('\n');
+ buf.append("rule memoization cache size ");
+ buf.append(stats.numMemoizationCacheEntries);
+ buf.append(newline);
+ buf.append("number of rule memoization cache hits ");
+ buf.append(stats.numMemoizationCacheHits);
+ buf.append(newline);
+ buf.append("number of rule memoization cache misses ");
+ buf.append(stats.numMemoizationCacheMisses);
+ buf.append(newline);
+// buf.append("number of evaluated semantic predicates ");
+// buf.append();
+// buf.append(newline);
+ buf.append("number of tokens ");
+ buf.append(stats.numTokens);
+ buf.append(newline);
+ buf.append("number of hidden tokens ");
+ buf.append(stats.numHiddenTokens);
+ buf.append(newline);
+ buf.append("number of char ");
+ buf.append(stats.numCharsMatched);
+ buf.append(newline);
+ buf.append("number of hidden char ");
+ buf.append(stats.numHiddenCharsMatched);
+ buf.append(newline);
+ buf.append("number of syntax errors ");
+ buf.append(stats.numReportedErrors);
+ buf.append(newline);
+ return buf.toString();
+ }
+
+ public String getDecisionStatsDump() {
+ StringBuffer buf = new StringBuffer();
+ buf.append("location");
+ buf.append(DATA_SEP);
+ buf.append("n");
+ buf.append(DATA_SEP);
+ buf.append("avgk");
+ buf.append(DATA_SEP);
+ buf.append("maxk");
+ buf.append(DATA_SEP);
+ buf.append("synpred");
+ buf.append(DATA_SEP);
+ buf.append("sempred");
+ buf.append(DATA_SEP);
+ buf.append("canbacktrack");
+ buf.append("\n");
+ for (String fileName : decisions.keySet()) {
+ for (int d : decisions.keySet(fileName)) {
+ DecisionDescriptor s = decisions.get(fileName, d);
+ buf.append(s.decision);
+ buf.append("@");
+ buf.append(locationDescription(s.fileName,s.ruleName,s.line,s.pos)); // decision number
+ buf.append(DATA_SEP);
+ buf.append(s.n);
+ buf.append(DATA_SEP);
+ buf.append(String.format("%.2f",s.avgk));
+ buf.append(DATA_SEP);
+ buf.append(s.maxk);
+ buf.append(DATA_SEP);
+ buf.append(s.numBacktrackOccurrences);
+ buf.append(DATA_SEP);
+ buf.append(s.numSemPredEvals);
+ buf.append(DATA_SEP);
+ buf.append(s.couldBacktrack ?"1":"0");
+ buf.append(newline);
+ }
+ }
+ return buf.toString();
+ }
+
+ protected int[] trim(int[] X, int n) {
+ if ( n<X.length ) {
+ int[] trimmed = new int[n];
+ System.arraycopy(X,0,trimmed,0,n);
+ X = trimmed;
+ }
+ return X;
+ }
+
+ protected int[] toArray(List a) {
+ int[] x = new int[a.size()];
+ for (int i = 0; i < a.size(); i++) {
+ Integer I = (Integer) a.get(i);
+ x[i] = I.intValue();
+ }
+ return x;
+ }
+
+ /** Get num hidden tokens between i..j inclusive */
+ public int getNumberOfHiddenTokens(int i, int j) {
+ int n = 0;
+ TokenStream input = parser.getTokenStream();
+ for (int ti = i; ti<input.size() && ti <= j; ti++) {
+ Token t = input.get(ti);
+ if ( t.getChannel()!=Token.DEFAULT_CHANNEL ) {
+ n++;
+ }
+ }
+ return n;
+ }
+
+ protected String locationDescription() {
+ return locationDescription(
+ currentGrammarFileName.peek(),
+ currentRuleName.peek(),
+ currentLine.peek(),
+ currentPos.peek());
+ }
+
+ protected String locationDescription(String file, String rule, int line, int pos) {
+ return file+":"+line+":"+pos+"(" + rule + ")";
+ }
+}
diff --git a/src/org/antlr/runtime/debug/RemoteDebugEventSocketListener.java b/src/org/antlr/runtime/debug/RemoteDebugEventSocketListener.java
new file mode 100755
index 0000000..933fdae
--- /dev/null
+++ b/src/org/antlr/runtime/debug/RemoteDebugEventSocketListener.java
@@ -0,0 +1,527 @@
+/*
+ [The "BSD license"]
+ Copyright (c) 2005-2009 Terence Parr
+ All rights reserved.
+
+ Redistribution and use in source and binary forms, with or without
+ modification, are permitted provided that the following conditions
+ are met:
+ 1. Redistributions of source code must retain the above copyright
+ notice, this list of conditions and the following disclaimer.
+ 2. Redistributions in binary form must reproduce the above copyright
+ notice, this list of conditions and the following disclaimer in the
+ documentation and/or other materials provided with the distribution.
+ 3. The name of the author may not be used to endorse or promote products
+ derived from this software without specific prior written permission.
+
+ THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
+ IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
+ OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
+ IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
+ INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
+ NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
+ THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+package org.antlr.runtime.debug;
+
+import org.antlr.runtime.RecognitionException;
+import org.antlr.runtime.Token;
+import org.antlr.runtime.CharStream;
+import org.antlr.runtime.tree.BaseTree;
+import org.antlr.runtime.tree.Tree;
+
+import java.io.*;
+import java.net.ConnectException;
+import java.net.Socket;
+import java.util.StringTokenizer;
+
+public class RemoteDebugEventSocketListener implements Runnable {
+ static final int MAX_EVENT_ELEMENTS = 8;
+ DebugEventListener listener;
+ String machine;
+ int port;
+ Socket channel = null;
+ PrintWriter out;
+ BufferedReader in;
+ String event;
+ /** Version of ANTLR (dictates events) */
+ public String version;
+ public String grammarFileName;
+ /** Track the last token index we saw during a consume. If same, then
+ * set a flag that we have a problem.
+ */
+ int previousTokenIndex = -1;
+ boolean tokenIndexesInvalid = false;
+
+ public static class ProxyToken implements Token {
+ int index;
+ int type;
+ int channel;
+ int line;
+ int charPos;
+ String text;
+ public ProxyToken(int index) { this.index = index; }
+ public ProxyToken(int index, int type, int channel,
+ int line, int charPos, String text)
+ {
+ this.index = index;
+ this.type = type;
+ this.channel = channel;
+ this.line = line;
+ this.charPos = charPos;
+ this.text = text;
+ }
+ public String getText() {
+ return text;
+ }
+ public void setText(String text) {
+ this.text = text;
+ }
+ public int getType() {
+ return type;
+ }
+ public void setType(int ttype) {
+ this.type = ttype;
+ }
+ public int getLine() {
+ return line;
+ }
+ public void setLine(int line) {
+ this.line = line;
+ }
+ public int getCharPositionInLine() {
+ return charPos;
+ }
+ public void setCharPositionInLine(int pos) {
+ this.charPos = pos;
+ }
+ public int getChannel() {
+ return channel;
+ }
+ public void setChannel(int channel) {
+ this.channel = channel;
+ }
+ public int getTokenIndex() {
+ return index;
+ }
+ public void setTokenIndex(int index) {
+ this.index = index;
+ }
+ public CharStream getInputStream() {
+ return null;
+ }
+ public void setInputStream(CharStream input) {
+ }
+ public String toString() {
+ String channelStr = "";
+ if ( channel!=Token.DEFAULT_CHANNEL ) {
+ channelStr=",channel="+channel;
+ }
+ return "["+getText()+"/<"+type+">"+channelStr+","+line+":"+getCharPositionInLine()+",@"+index+"]";
+ }
+ }
+
+ public static class ProxyTree extends BaseTree {
+ public int ID;
+ public int type;
+ public int line = 0;
+ public int charPos = -1;
+ public int tokenIndex = -1;
+ public String text;
+
+ public ProxyTree(int ID, int type, int line, int charPos, int tokenIndex, String text) {
+ this.ID = ID;
+ this.type = type;
+ this.line = line;
+ this.charPos = charPos;
+ this.tokenIndex = tokenIndex;
+ this.text = text;
+ }
+
+ public ProxyTree(int ID) { this.ID = ID; }
+
+ public int getTokenStartIndex() { return tokenIndex; }
+ public void setTokenStartIndex(int index) { }
+ public int getTokenStopIndex() { return 0; }
+ public void setTokenStopIndex(int index) { }
+ public Tree dupNode() { return null; }
+ public int getType() { return type; }
+ public String getText() { return text; }
+ public String toString() {
+ return "fix this";
+ }
+ }
+
+ public RemoteDebugEventSocketListener(DebugEventListener listener,
+ String machine,
+ int port) throws IOException
+ {
+ this.listener = listener;
+ this.machine = machine;
+ this.port = port;
+
+ if( !openConnection() ) {
+ throw new ConnectException();
+ }
+ }
+
+ protected void eventHandler() {
+ try {
+ handshake();
+ event = in.readLine();
+ while ( event!=null ) {
+ dispatch(event);
+ ack();
+ event = in.readLine();
+ }
+ }
+ catch (Exception e) {
+ System.err.println(e);
+ e.printStackTrace(System.err);
+ }
+ finally {
+ closeConnection();
+ }
+ }
+
+ protected boolean openConnection() {
+ boolean success = false;
+ try {
+ channel = new Socket(machine, port);
+ channel.setTcpNoDelay(true);
+ OutputStream os = channel.getOutputStream();
+ OutputStreamWriter osw = new OutputStreamWriter(os, "UTF8");
+ out = new PrintWriter(new BufferedWriter(osw));
+ InputStream is = channel.getInputStream();
+ InputStreamReader isr = new InputStreamReader(is, "UTF8");
+ in = new BufferedReader(isr);
+ success = true;
+ } catch(Exception e) {
+ System.err.println(e);
+ }
+ return success;
+ }
+
+ protected void closeConnection() {
+ try {
+ in.close(); in = null;
+ out.close(); out = null;
+ channel.close(); channel=null;
+ }
+ catch (Exception e) {
+ System.err.println(e);
+ e.printStackTrace(System.err);
+ }
+ finally {
+ if ( in!=null ) {
+ try {in.close();} catch (IOException ioe) {
+ System.err.println(ioe);
+ }
+ }
+ if ( out!=null ) {
+ out.close();
+ }
+ if ( channel!=null ) {
+ try {channel.close();} catch (IOException ioe) {
+ System.err.println(ioe);
+ }
+ }
+ }
+
+ }
+
+ protected void handshake() throws IOException {
+ String antlrLine = in.readLine();
+ String[] antlrElements = getEventElements(antlrLine);
+ version = antlrElements[1];
+ String grammarLine = in.readLine();
+ String[] grammarElements = getEventElements(grammarLine);
+ grammarFileName = grammarElements[1];
+ ack();
+ listener.commence(); // inform listener after handshake
+ }
+
+ protected void ack() {
+ out.println("ack");
+ out.flush();
+ }
+
+ protected void dispatch(String line) {
+ //System.out.println("event: "+line);
+ String[] elements = getEventElements(line);
+ if ( elements==null || elements[0]==null ) {
+ System.err.println("unknown debug event: "+line);
+ return;
+ }
+ if ( elements[0].equals("enterRule") ) {
+ listener.enterRule(elements[1], elements[2]);
+ }
+ else if ( elements[0].equals("exitRule") ) {
+ listener.exitRule(elements[1], elements[2]);
+ }
+ else if ( elements[0].equals("enterAlt") ) {
+ listener.enterAlt(Integer.parseInt(elements[1]));
+ }
+ else if ( elements[0].equals("enterSubRule") ) {
+ listener.enterSubRule(Integer.parseInt(elements[1]));
+ }
+ else if ( elements[0].equals("exitSubRule") ) {
+ listener.exitSubRule(Integer.parseInt(elements[1]));
+ }
+ else if ( elements[0].equals("enterDecision") ) {
+ listener.enterDecision(Integer.parseInt(elements[1]), elements[2].equals("true"));
+ }
+ else if ( elements[0].equals("exitDecision") ) {
+ listener.exitDecision(Integer.parseInt(elements[1]));
+ }
+ else if ( elements[0].equals("location") ) {
+ listener.location(Integer.parseInt(elements[1]),
+ Integer.parseInt(elements[2]));
+ }
+ else if ( elements[0].equals("consumeToken") ) {
+ ProxyToken t = deserializeToken(elements, 1);
+ if ( t.getTokenIndex() == previousTokenIndex ) {
+ tokenIndexesInvalid = true;
+ }
+ previousTokenIndex = t.getTokenIndex();
+ listener.consumeToken(t);
+ }
+ else if ( elements[0].equals("consumeHiddenToken") ) {
+ ProxyToken t = deserializeToken(elements, 1);
+ if ( t.getTokenIndex() == previousTokenIndex ) {
+ tokenIndexesInvalid = true;
+ }
+ previousTokenIndex = t.getTokenIndex();
+ listener.consumeHiddenToken(t);
+ }
+ else if ( elements[0].equals("LT") ) {
+ Token t = deserializeToken(elements, 2);
+ listener.LT(Integer.parseInt(elements[1]), t);
+ }
+ else if ( elements[0].equals("mark") ) {
+ listener.mark(Integer.parseInt(elements[1]));
+ }
+ else if ( elements[0].equals("rewind") ) {
+ if ( elements[1]!=null ) {
+ listener.rewind(Integer.parseInt(elements[1]));
+ }
+ else {
+ listener.rewind();
+ }
+ }
+ else if ( elements[0].equals("beginBacktrack") ) {
+ listener.beginBacktrack(Integer.parseInt(elements[1]));
+ }
+ else if ( elements[0].equals("endBacktrack") ) {
+ int level = Integer.parseInt(elements[1]);
+ int successI = Integer.parseInt(elements[2]);
+ listener.endBacktrack(level, successI==DebugEventListener.TRUE);
+ }
+ else if ( elements[0].equals("exception") ) {
+ String excName = elements[1];
+ String indexS = elements[2];
+ String lineS = elements[3];
+ String posS = elements[4];
+ Class excClass = null;
+ try {
+ excClass = Class.forName(excName);
+ RecognitionException e =
+ (RecognitionException)excClass.newInstance();
+ e.index = Integer.parseInt(indexS);
+ e.line = Integer.parseInt(lineS);
+ e.charPositionInLine = Integer.parseInt(posS);
+ listener.recognitionException(e);
+ }
+ catch (ClassNotFoundException cnfe) {
+ System.err.println("can't find class "+cnfe);
+ cnfe.printStackTrace(System.err);
+ }
+ catch (InstantiationException ie) {
+ System.err.println("can't instantiate class "+ie);
+ ie.printStackTrace(System.err);
+ }
+ catch (IllegalAccessException iae) {
+ System.err.println("can't access class "+iae);
+ iae.printStackTrace(System.err);
+ }
+ }
+ else if ( elements[0].equals("beginResync") ) {
+ listener.beginResync();
+ }
+ else if ( elements[0].equals("endResync") ) {
+ listener.endResync();
+ }
+ else if ( elements[0].equals("terminate") ) {
+ listener.terminate();
+ }
+ else if ( elements[0].equals("semanticPredicate") ) {
+ Boolean result = Boolean.valueOf(elements[1]);
+ String predicateText = elements[2];
+ predicateText = unEscapeNewlines(predicateText);
+ listener.semanticPredicate(result.booleanValue(),
+ predicateText);
+ }
+ else if ( elements[0].equals("consumeNode") ) {
+ ProxyTree node = deserializeNode(elements, 1);
+ listener.consumeNode(node);
+ }
+ else if ( elements[0].equals("LN") ) {
+ int i = Integer.parseInt(elements[1]);
+ ProxyTree node = deserializeNode(elements, 2);
+ listener.LT(i, node);
+ }
+ else if ( elements[0].equals("createNodeFromTokenElements") ) {
+ int ID = Integer.parseInt(elements[1]);
+ int type = Integer.parseInt(elements[2]);
+ String text = elements[3];
+ text = unEscapeNewlines(text);
+ ProxyTree node = new ProxyTree(ID, type, -1, -1, -1, text);
+ listener.createNode(node);
+ }
+ else if ( elements[0].equals("createNode") ) {
+ int ID = Integer.parseInt(elements[1]);
+ int tokenIndex = Integer.parseInt(elements[2]);
+ // create dummy node/token filled with ID, tokenIndex
+ ProxyTree node = new ProxyTree(ID);
+ ProxyToken token = new ProxyToken(tokenIndex);
+ listener.createNode(node, token);
+ }
+ else if ( elements[0].equals("nilNode") ) {
+ int ID = Integer.parseInt(elements[1]);
+ ProxyTree node = new ProxyTree(ID);
+ listener.nilNode(node);
+ }
+ else if ( elements[0].equals("errorNode") ) {
+ // TODO: do we need a special tree here?
+ int ID = Integer.parseInt(elements[1]);
+ int type = Integer.parseInt(elements[2]);
+ String text = elements[3];
+ text = unEscapeNewlines(text);
+ ProxyTree node = new ProxyTree(ID, type, -1, -1, -1, text);
+ listener.errorNode(node);
+ }
+ else if ( elements[0].equals("becomeRoot") ) {
+ int newRootID = Integer.parseInt(elements[1]);
+ int oldRootID = Integer.parseInt(elements[2]);
+ ProxyTree newRoot = new ProxyTree(newRootID);
+ ProxyTree oldRoot = new ProxyTree(oldRootID);
+ listener.becomeRoot(newRoot, oldRoot);
+ }
+ else if ( elements[0].equals("addChild") ) {
+ int rootID = Integer.parseInt(elements[1]);
+ int childID = Integer.parseInt(elements[2]);
+ ProxyTree root = new ProxyTree(rootID);
+ ProxyTree child = new ProxyTree(childID);
+ listener.addChild(root, child);
+ }
+ else if ( elements[0].equals("setTokenBoundaries") ) {
+ int ID = Integer.parseInt(elements[1]);
+ ProxyTree node = new ProxyTree(ID);
+ listener.setTokenBoundaries(
+ node,
+ Integer.parseInt(elements[2]),
+ Integer.parseInt(elements[3]));
+ }
+ else {
+ System.err.println("unknown debug event: "+line);
+ }
+ }
+
+ protected ProxyTree deserializeNode(String[] elements, int offset) {
+ int ID = Integer.parseInt(elements[offset+0]);
+ int type = Integer.parseInt(elements[offset+1]);
+ int tokenLine = Integer.parseInt(elements[offset+2]);
+ int charPositionInLine = Integer.parseInt(elements[offset+3]);
+ int tokenIndex = Integer.parseInt(elements[offset+4]);
+ String text = elements[offset+5];
+ text = unEscapeNewlines(text);
+ return new ProxyTree(ID, type, tokenLine, charPositionInLine, tokenIndex, text);
+ }
+
+ protected ProxyToken deserializeToken(String[] elements,
+ int offset)
+ {
+ String indexS = elements[offset+0];
+ String typeS = elements[offset+1];
+ String channelS = elements[offset+2];
+ String lineS = elements[offset+3];
+ String posS = elements[offset+4];
+ String text = elements[offset+5];
+ text = unEscapeNewlines(text);
+ int index = Integer.parseInt(indexS);
+ ProxyToken t =
+ new ProxyToken(index,
+ Integer.parseInt(typeS),
+ Integer.parseInt(channelS),
+ Integer.parseInt(lineS),
+ Integer.parseInt(posS),
+ text);
+ return t;
+ }
+
+ /** Create a thread to listen to the remote running recognizer */
+ public void start() {
+ Thread t = new Thread(this);
+ t.start();
+ }
+
+ public void run() {
+ eventHandler();
+ }
+
+ // M i s c
+
+ public String[] getEventElements(String event) {
+ if ( event==null ) {
+ return null;
+ }
+ String[] elements = new String[MAX_EVENT_ELEMENTS];
+ String str = null; // a string element if present (must be last)
+ try {
+ int firstQuoteIndex = event.indexOf('"');
+ if ( firstQuoteIndex>=0 ) {
+ // treat specially; has a string argument like "a comment\n
+ // Note that the string is terminated by \n not end quote.
+ // Easier to parse that way.
+ String eventWithoutString = event.substring(0,firstQuoteIndex);
+ str = event.substring(firstQuoteIndex+1,event.length());
+ event = eventWithoutString;
+ }
+ StringTokenizer st = new StringTokenizer(event, "\t", false);
+ int i = 0;
+ while ( st.hasMoreTokens() ) {
+ if ( i>=MAX_EVENT_ELEMENTS ) {
+ // ErrorManager.internalError("event has more than "+MAX_EVENT_ELEMENTS+" args: "+event);
+ return elements;
+ }
+ elements[i] = st.nextToken();
+ i++;
+ }
+ if ( str!=null ) {
+ elements[i] = str;
+ }
+ }
+ catch (Exception e) {
+ e.printStackTrace(System.err);
+ }
+ return elements;
+ }
+
+ protected String unEscapeNewlines(String txt) {
+ // this unescape is slow but easy to understand
+ txt = txt.replaceAll("%0A","\n"); // unescape \n
+ txt = txt.replaceAll("%0D","\r"); // unescape \r
+ txt = txt.replaceAll("%25","%"); // undo escaped escape chars
+ return txt;
+ }
+
+ public boolean tokenIndexesAreInvalid() {
+ return false;
+ //return tokenIndexesInvalid;
+ }
+
+}
+
diff --git a/src/org/antlr/runtime/debug/TraceDebugEventListener.java b/src/org/antlr/runtime/debug/TraceDebugEventListener.java
new file mode 100755
index 0000000..de9366d
--- /dev/null
+++ b/src/org/antlr/runtime/debug/TraceDebugEventListener.java
@@ -0,0 +1,96 @@
+/*
+ [The "BSD license"]
+ Copyright (c) 2005-2009 Terence Parr
+ All rights reserved.
+
+ Redistribution and use in source and binary forms, with or without
+ modification, are permitted provided that the following conditions
+ are met:
+ 1. Redistributions of source code must retain the above copyright
+ notice, this list of conditions and the following disclaimer.
+ 2. Redistributions in binary form must reproduce the above copyright
+ notice, this list of conditions and the following disclaimer in the
+ documentation and/or other materials provided with the distribution.
+ 3. The name of the author may not be used to endorse or promote products
+ derived from this software without specific prior written permission.
+
+ THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
+ IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
+ OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
+ IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
+ INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
+ NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
+ THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+package org.antlr.runtime.debug;
+
+import org.antlr.runtime.Token;
+import org.antlr.runtime.tree.TreeAdaptor;
+
+/** Print out (most of) the events... Useful for debugging, testing... */
+public class TraceDebugEventListener extends BlankDebugEventListener {
+ TreeAdaptor adaptor;
+
+ public TraceDebugEventListener(TreeAdaptor adaptor) {
+ this.adaptor = adaptor;
+ }
+
+ public void enterRule(String ruleName) { System.out.println("enterRule "+ruleName); }
+ public void exitRule(String ruleName) { System.out.println("exitRule "+ruleName); }
+ public void enterSubRule(int decisionNumber) { System.out.println("enterSubRule"); }
+ public void exitSubRule(int decisionNumber) { System.out.println("exitSubRule"); }
+ public void location(int line, int pos) {System.out.println("location "+line+":"+pos);}
+
+ // Tree parsing stuff
+
+ public void consumeNode(Object t) {
+ int ID = adaptor.getUniqueID(t);
+ String text = adaptor.getText(t);
+ int type = adaptor.getType(t);
+ System.out.println("consumeNode "+ID+" "+text+" "+type);
+ }
+
+ public void LT(int i, Object t) {
+ int ID = adaptor.getUniqueID(t);
+ String text = adaptor.getText(t);
+ int type = adaptor.getType(t);
+ System.out.println("LT "+i+" "+ID+" "+text+" "+type);
+ }
+
+
+ // AST stuff
+ public void nilNode(Object t) {System.out.println("nilNode "+adaptor.getUniqueID(t));}
+
+ public void createNode(Object t) {
+ int ID = adaptor.getUniqueID(t);
+ String text = adaptor.getText(t);
+ int type = adaptor.getType(t);
+ System.out.println("create "+ID+": "+text+", "+type);
+ }
+
+ public void createNode(Object node, Token token) {
+ int ID = adaptor.getUniqueID(node);
+ String text = adaptor.getText(node);
+ int tokenIndex = token.getTokenIndex();
+ System.out.println("create "+ID+": "+tokenIndex);
+ }
+
+ public void becomeRoot(Object newRoot, Object oldRoot) {
+ System.out.println("becomeRoot "+adaptor.getUniqueID(newRoot)+", "+
+ adaptor.getUniqueID(oldRoot));
+ }
+
+ public void addChild(Object root, Object child) {
+ System.out.println("addChild "+adaptor.getUniqueID(root)+", "+
+ adaptor.getUniqueID(child));
+ }
+
+ public void setTokenBoundaries(Object t, int tokenStartIndex, int tokenStopIndex) {
+ System.out.println("setTokenBoundaries "+adaptor.getUniqueID(t)+", "+
+ tokenStartIndex+", "+tokenStopIndex);
+ }
+}
+
diff --git a/src/org/antlr/runtime/debug/Tracer.java b/src/org/antlr/runtime/debug/Tracer.java
new file mode 100755
index 0000000..c2c73da
--- /dev/null
+++ b/src/org/antlr/runtime/debug/Tracer.java
@@ -0,0 +1,65 @@
+/*
+ [The "BSD license"]
+ Copyright (c) 2005-2009 Terence Parr
+ All rights reserved.
+
+ Redistribution and use in source and binary forms, with or without
+ modification, are permitted provided that the following conditions
+ are met:
+ 1. Redistributions of source code must retain the above copyright
+ notice, this list of conditions and the following disclaimer.
+ 2. Redistributions in binary form must reproduce the above copyright
+ notice, this list of conditions and the following disclaimer in the
+ documentation and/or other materials provided with the distribution.
+ 3. The name of the author may not be used to endorse or promote products
+ derived from this software without specific prior written permission.
+
+ THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
+ IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
+ OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
+ IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
+ INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
+ NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
+ THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+package org.antlr.runtime.debug;
+
+import org.antlr.runtime.IntStream;
+import org.antlr.runtime.TokenStream;
+
+/** The default tracer mimics the traceParser behavior of ANTLR 2.x.
+ * This listens for debugging events from the parser and implies
+ * that you cannot debug and trace at the same time.
+ */
+public class Tracer extends BlankDebugEventListener {
+ public IntStream input;
+ protected int level = 0;
+
+ public Tracer(IntStream input) {
+ this.input = input;
+ }
+
+ public void enterRule(String ruleName) {
+ for (int i=1; i<=level; i++) {System.out.print(" ");}
+ System.out.println("> "+ruleName+" lookahead(1)="+getInputSymbol(1));
+ level++;
+ }
+
+ public void exitRule(String ruleName) {
+ level--;
+ for (int i=1; i<=level; i++) {System.out.print(" ");}
+ System.out.println("< "+ruleName+" lookahead(1)="+getInputSymbol(1));
+ }
+
+ public Object getInputSymbol(int k) {
+ if ( input instanceof TokenStream ) {
+ return ((TokenStream)input).LT(k);
+ }
+ return new Character((char)input.LA(k));
+ }
+}
+
+
diff --git a/src/org/antlr/runtime/misc/DoubleKeyMap.java b/src/org/antlr/runtime/misc/DoubleKeyMap.java
new file mode 100755
index 0000000..b69ae32
--- /dev/null
+++ b/src/org/antlr/runtime/misc/DoubleKeyMap.java
@@ -0,0 +1,62 @@
+package org.antlr.runtime.misc;
+
+import java.util.*;
+
+/** Sometimes we need to map a key to a value but key is two pieces of data.
+ * This nested hash table saves creating a single key each time we access
+ * map; avoids mem creation.
+ */
+public class DoubleKeyMap<Key1, Key2, Value> {
+ Map<Key1, Map<Key2, Value>> data = new LinkedHashMap<Key1, Map<Key2, Value>>();
+
+ public Value put(Key1 k1, Key2 k2, Value v) {
+ Map<Key2, Value> data2 = data.get(k1);
+ Value prev = null;
+ if ( data2==null ) {
+ data2 = new LinkedHashMap<Key2, Value>();
+ data.put(k1, data2);
+ }
+ else {
+ prev = data2.get(k2);
+ }
+ data2.put(k2, v);
+ return prev;
+ }
+
+ public Value get(Key1 k1, Key2 k2) {
+ Map<Key2, Value> data2 = data.get(k1);
+ if ( data2==null ) return null;
+ return data2.get(k2);
+ }
+
+ public Map<Key2, Value> get(Key1 k1) { return data.get(k1); }
+
+ /** Get all values associated with primary key */
+ public Collection<Value> values(Key1 k1) {
+ Map<Key2, Value> data2 = data.get(k1);
+ if ( data2==null ) return null;
+ return data2.values();
+ }
+
+ /** get all primary keys */
+ public Set<Key1> keySet() {
+ return data.keySet();
+ }
+
+ /** get all secondary keys associated with a primary key */
+ public Set<Key2> keySet(Key1 k1) {
+ Map<Key2, Value> data2 = data.get(k1);
+ if ( data2==null ) return null;
+ return data2.keySet();
+ }
+
+ public Collection<Value> values() {
+ Set<Value> s = new HashSet<Value>();
+ for (Map<Key2, Value> k2 : data.values()) {
+ for (Value v : k2.values()) {
+ s.add(v);
+ }
+ }
+ return s;
+ }
+}
diff --git a/src/org/antlr/runtime/misc/FastQueue.java b/src/org/antlr/runtime/misc/FastQueue.java
new file mode 100755
index 0000000..08843dd
--- /dev/null
+++ b/src/org/antlr/runtime/misc/FastQueue.java
@@ -0,0 +1,100 @@
+/*
+ [The "BSD license"]
+ Copyright (c) 2005-2009 Terence Parr
+ All rights reserved.
+
+ Redistribution and use in source and binary forms, with or without
+ modification, are permitted provided that the following conditions
+ are met:
+ 1. Redistributions of source code must retain the above copyright
+ notice, this list of conditions and the following disclaimer.
+ 2. Redistributions in binary form must reproduce the above copyright
+ notice, this list of conditions and the following disclaimer in the
+ documentation and/or other materials provided with the distribution.
+ 3. The name of the author may not be used to endorse or promote products
+ derived from this software without specific prior written permission.
+
+ THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
+ IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
+ OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
+ IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
+ INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
+ NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
+ THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+package org.antlr.runtime.misc;
+
+import java.util.List;
+import java.util.ArrayList;
+import java.util.NoSuchElementException;
+
+/** A queue that can dequeue and get(i) in O(1) and grow arbitrarily large.
+ * A linked list is fast at dequeue but slow at get(i). An array is
+ * the reverse. This is O(1) for both operations.
+ *
+ * List grows until you dequeue last element at end of buffer. Then
+ * it resets to start filling at 0 again. If adds/removes are balanced, the
+ * buffer will not grow too large.
+ *
+ * No iterator stuff as that's not how we'll use it.
+ */
+public class FastQueue<T> {
+ /** dynamically-sized buffer of elements */
+ protected List<T> data = new ArrayList<T>();
+ /** index of next element to fill */
+ protected int p = 0;
+ protected int range = -1; // how deep have we gone?
+
+ public void reset() { clear(); }
+ public void clear() { p = 0; data.clear(); }
+
+ /** Get and remove first element in queue */
+ public T remove() {
+ T o = elementAt(0);
+ p++;
+ // have we hit end of buffer?
+ if ( p == data.size() ) {
+ // if so, it's an opportunity to start filling at index 0 again
+ clear(); // size goes to 0, but retains memory
+ }
+ return o;
+ }
+
+ public void add(T o) { data.add(o); }
+
+ public int size() { return data.size() - p; }
+
+ public int range() { return range; }
+
+ public T head() { return elementAt(0); }
+
+ /** Return element i elements ahead of current element. i==0 gets
+ * current element. This is not an absolute index into the data list
+ * since p defines the start of the real list.
+ */
+ public T elementAt(int i) {
+ int absIndex = p + i;
+ if ( absIndex >= data.size() ) {
+ throw new NoSuchElementException("queue index "+ absIndex +" > last index "+(data.size()-1));
+ }
+ if ( absIndex < 0 ) {
+ throw new NoSuchElementException("queue index "+ absIndex +" < 0");
+ }
+ if ( absIndex>range ) range = absIndex;
+ return data.get(absIndex);
+ }
+
+ /** Return string of current buffer contents; non-destructive */
+ public String toString() {
+ StringBuffer buf = new StringBuffer();
+ int n = size();
+ for (int i=0; i<n; i++) {
+ buf.append(elementAt(i));
+ if ( (i+1)<n ) buf.append(" ");
+ }
+ return buf.toString();
+ }
+}
\ No newline at end of file
diff --git a/src/org/antlr/runtime/misc/IntArray.java b/src/org/antlr/runtime/misc/IntArray.java
new file mode 100755
index 0000000..bc484aa
--- /dev/null
+++ b/src/org/antlr/runtime/misc/IntArray.java
@@ -0,0 +1,87 @@
+/*
+ [The "BSD license"]
+ Copyright (c) 2005-2009 Terence Parr
+ All rights reserved.
+
+ Redistribution and use in source and binary forms, with or without
+ modification, are permitted provided that the following conditions
+ are met:
+ 1. Redistributions of source code must retain the above copyright
+ notice, this list of conditions and the following disclaimer.
+ 2. Redistributions in binary form must reproduce the above copyright
+ notice, this list of conditions and the following disclaimer in the
+ documentation and/or other materials provided with the distribution.
+ 3. The name of the author may not be used to endorse or promote products
+ derived from this software without specific prior written permission.
+
+ THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
+ IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
+ OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
+ IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
+ INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
+ NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
+ THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+package org.antlr.runtime.misc;
+
+/** A dynamic array that uses int not Integer objects. In principle this
+ * is more efficient in time, but certainly in space.
+ *
+ * This is simple enough that you can access the data array directly,
+ * but make sure that you append elements only with add() so that you
+ * get dynamic sizing. Make sure to call ensureCapacity() when you are
+ * manually adding new elements.
+ *
+ * Doesn't impl List because it doesn't return objects and I mean this
+ * really as just an array not a List per se. Manipulate the elements
+ * at will. This has stack methods too.
+ *
+ * When runtime can be 1.5, I'll make this generic.
+ */
+public class IntArray {
+ public static final int INITIAL_SIZE = 10;
+ public int[] data;
+ protected int p = -1;
+
+ public void add(int v) {
+ ensureCapacity(p+1);
+ data[++p] = v;
+ }
+
+ public void push(int v) {
+ add(v);
+ }
+
+ public int pop() {
+ int v = data[p];
+ p--;
+ return v;
+ }
+
+ /** This only tracks elements added via push/add. */
+ public int size() {
+ return p;
+ }
+
+ public void clear() {
+ p = -1;
+ }
+
+ public void ensureCapacity(int index) {
+ if ( data==null ) {
+ data = new int[INITIAL_SIZE];
+ }
+ else if ( (index+1)>=data.length ) {
+ int newSize = data.length*2;
+ if ( index>newSize ) {
+ newSize = index+1;
+ }
+ int[] newData = new int[newSize];
+ System.arraycopy(data, 0, newData, 0, data.length);
+ data = newData;
+ }
+ }
+}
diff --git a/src/org/antlr/runtime/misc/LookaheadStream.java b/src/org/antlr/runtime/misc/LookaheadStream.java
new file mode 100755
index 0000000..6f19c44
--- /dev/null
+++ b/src/org/antlr/runtime/misc/LookaheadStream.java
@@ -0,0 +1,161 @@
+/*
+ [The "BSD license"]
+ Copyright (c) 2005-2009 Terence Parr
+ All rights reserved.
+
+ Redistribution and use in source and binary forms, with or without
+ modification, are permitted provided that the following conditions
+ are met:
+ 1. Redistributions of source code must retain the above copyright
+ notice, this list of conditions and the following disclaimer.
+ 2. Redistributions in binary form must reproduce the above copyright
+ notice, this list of conditions and the following disclaimer in the
+ documentation and/or other materials provided with the distribution.
+ 3. The name of the author may not be used to endorse or promote products
+ derived from this software without specific prior written permission.
+
+ THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
+ IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
+ OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
+ IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
+ INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
+ NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
+ THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+package org.antlr.runtime.misc;
+
+import org.antlr.runtime.Token;
+
+import java.util.NoSuchElementException;
+
+/** A lookahead queue that knows how to mark/release locations
+ * in the buffer for backtracking purposes. Any markers force the FastQueue
+ * superclass to keep all tokens until no more markers; then can reset
+ * to avoid growing a huge buffer.
+ */
+public abstract class LookaheadStream<T> extends FastQueue<T> {
+ public static final int UNINITIALIZED_EOF_ELEMENT_INDEX = Integer.MAX_VALUE;
+
+ /** Absolute token index. It's the index of the symbol about to be
+ * read via LT(1). Goes from 0 to numtokens.
+ */
+ protected int currentElementIndex = 0;
+
+ protected T prevElement;
+
+ /** Track object returned by nextElement upon end of stream;
+ * Return it later when they ask for LT passed end of input.
+ */
+ public T eof = null;
+
+ /** Track the last mark() call result value for use in rewind(). */
+ protected int lastMarker;
+
+ /** tracks how deep mark() calls are nested */
+ protected int markDepth = 0;
+
+ public void reset() {
+ super.reset();
+ currentElementIndex = 0;
+ p = 0;
+ prevElement=null;
+ }
+
+ /** Implement nextElement to supply a stream of elements to this
+ * lookahead buffer. Return eof upon end of the stream we're pulling from.
+ */
+ public abstract T nextElement();
+
+ public abstract boolean isEOF(T o);
+
+ /** Get and remove first element in queue; override FastQueue.remove();
+ * it's the same, just checks for backtracking.
+ */
+ public T remove() {
+ T o = elementAt(0);
+ p++;
+ // have we hit end of buffer and not backtracking?
+ if ( p == data.size() && markDepth==0 ) {
+ // if so, it's an opportunity to start filling at index 0 again
+ clear(); // size goes to 0, but retains memory
+ }
+ return o;
+ }
+
+ /** Make sure we have at least one element to remove, even if EOF */
+ public void consume() {
+ syncAhead(1);
+ prevElement = remove();
+ currentElementIndex++;
+ }
+
+ /** Make sure we have 'need' elements from current position p. Last valid
+ * p index is data.size()-1. p+need-1 is the data index 'need' elements
+ * ahead. If we need 1 element, (p+1-1)==p must be < data.size().
+ */
+ protected void syncAhead(int need) {
+ int n = (p+need-1) - data.size() + 1; // how many more elements we need?
+ if ( n > 0 ) fill(n); // out of elements?
+ }
+
+ /** add n elements to buffer */
+ public void fill(int n) {
+ for (int i=1; i<=n; i++) {
+ T o = nextElement();
+ if ( isEOF(o) ) eof = o;
+ data.add(o);
+ }
+ }
+
+ /** Size of entire stream is unknown; we only know buffer size from FastQueue */
+ public int size() { throw new UnsupportedOperationException("streams are of unknown size"); }
+
+ public T LT(int k) {
+ if ( k==0 ) {
+ return null;
+ }
+ if ( k<0 ) return LB(-k);
+ //System.out.print("LT(p="+p+","+k+")=");
+ syncAhead(k);
+ if ( (p+k-1) > data.size() ) return eof;
+ return elementAt(k-1);
+ }
+
+ public int index() { return currentElementIndex; }
+
+ public int mark() {
+ markDepth++;
+ lastMarker = p; // track where we are in buffer not absolute token index
+ return lastMarker;
+ }
+
+ public void release(int marker) {
+ // no resources to release
+ }
+
+ public void rewind(int marker) {
+ markDepth--;
+ seek(marker); // assume marker is top
+ // release(marker); // waste of call; it does nothing in this class
+ }
+
+ public void rewind() {
+ seek(lastMarker); // rewind but do not release marker
+ }
+
+ /** Seek to a 0-indexed position within data buffer. Can't handle
+ * case where you seek beyond end of existing buffer. Normally used
+ * to seek backwards in the buffer. Does not force loading of nodes.
+ * Doesn't see to absolute position in input stream since this stream
+ * is unbuffered. Seeks only into our moving window of elements.
+ */
+ public void seek(int index) { p = index; }
+
+ protected T LB(int k) {
+ if ( k==1 ) return prevElement;
+ throw new NoSuchElementException("can't look backwards more than one token in this stream");
+ }
+}
\ No newline at end of file
diff --git a/src/org/antlr/runtime/misc/Stats.java b/src/org/antlr/runtime/misc/Stats.java
new file mode 100755
index 0000000..9fdd21e
--- /dev/null
+++ b/src/org/antlr/runtime/misc/Stats.java
@@ -0,0 +1,189 @@
+/*
+ [The "BSD license"]
+ Copyright (c) 2005-2009 Terence Parr
+ All rights reserved.
+
+ Redistribution and use in source and binary forms, with or without
+ modification, are permitted provided that the following conditions
+ are met:
+ 1. Redistributions of source code must retain the above copyright
+ notice, this list of conditions and the following disclaimer.
+ 2. Redistributions in binary form must reproduce the above copyright
+ notice, this list of conditions and the following disclaimer in the
+ documentation and/or other materials provided with the distribution.
+ 3. The name of the author may not be used to endorse or promote products
+ derived from this software without specific prior written permission.
+
+ THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
+ IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
+ OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
+ IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
+ INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
+ NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
+ THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+package org.antlr.runtime.misc;
+
+import java.io.*;
+import java.util.List;
+
+/** Stats routines needed by profiler etc...
+
+ // note that these routines return 0.0 if no values exist in the X[]
+ // which is not "correct", but it is useful so I don't generate NaN
+ // in my output
+
+ */
+public class Stats {
+ public static final String ANTLRWORKS_DIR = "antlrworks";
+
+ /** Compute the sample (unbiased estimator) standard deviation following:
+ *
+ * Computing Deviations: Standard Accuracy
+ * Tony F. Chan and John Gregg Lewis
+ * Stanford University
+ * Communications of ACM September 1979 of Volume 22 the ACM Number 9
+ *
+ * The "two-pass" method from the paper; supposed to have better
+ * numerical properties than the textbook summation/sqrt. To me
+ * this looks like the textbook method, but I ain't no numerical
+ * methods guy.
+ */
+ public static double stddev(int[] X) {
+ int m = X.length;
+ if ( m<=1 ) {
+ return 0;
+ }
+ double xbar = avg(X);
+ double s2 = 0.0;
+ for (int i=0; i<m; i++){
+ s2 += (X[i] - xbar)*(X[i] - xbar);
+ }
+ s2 = s2/(m-1);
+ return Math.sqrt(s2);
+ }
+
+ /** Compute the sample mean */
+ public static double avg(int[] X) {
+ double xbar = 0.0;
+ int m = X.length;
+ if ( m==0 ) {
+ return 0;
+ }
+ for (int i=0; i<m; i++){
+ xbar += X[i];
+ }
+ if ( xbar>=0.0 ) {
+ return xbar / m;
+ }
+ return 0.0;
+ }
+
+ public static int min(int[] X) {
+ int min = Integer.MAX_VALUE;
+ int m = X.length;
+ if ( m==0 ) {
+ return 0;
+ }
+ for (int i=0; i<m; i++){
+ if ( X[i] < min ) {
+ min = X[i];
+ }
+ }
+ return min;
+ }
+
+ public static int max(int[] X) {
+ int max = Integer.MIN_VALUE;
+ int m = X.length;
+ if ( m==0 ) {
+ return 0;
+ }
+ for (int i=0; i<m; i++){
+ if ( X[i] > max ) {
+ max = X[i];
+ }
+ }
+ return max;
+ }
+
+ /** Compute the sample mean */
+ public static double avg(List<Integer> X) {
+ double xbar = 0.0;
+ int m = X.size();
+ if ( m==0 ) {
+ return 0;
+ }
+ for (int i=0; i<m; i++){
+ xbar += X.get(i);
+ }
+ if ( xbar>=0.0 ) {
+ return xbar / m;
+ }
+ return 0.0;
+ }
+
+ public static int min(List<Integer> X) {
+ int min = Integer.MAX_VALUE;
+ int m = X.size();
+ if ( m==0 ) {
+ return 0;
+ }
+ for (int i=0; i<m; i++){
+ if ( X.get(i) < min ) {
+ min = X.get(i);
+ }
+ }
+ return min;
+ }
+
+ public static int max(List<Integer> X) {
+ int max = Integer.MIN_VALUE;
+ int m = X.size();
+ if ( m==0 ) {
+ return 0;
+ }
+ for (int i=0; i<m; i++){
+ if ( X.get(i) > max ) {
+ max = X.get(i);
+ }
+ }
+ return max;
+ }
+
+ public static int sum(int[] X) {
+ int s = 0;
+ int m = X.length;
+ if ( m==0 ) {
+ return 0;
+ }
+ for (int i=0; i<m; i++){
+ s += X[i];
+ }
+ return s;
+ }
+
+ public static void writeReport(String filename, String data) throws IOException {
+ String absoluteFilename = getAbsoluteFileName(filename);
+ File f = new File(absoluteFilename);
+ File parent = f.getParentFile();
+ parent.mkdirs(); // ensure parent dir exists
+ // write file
+ FileOutputStream fos = new FileOutputStream(f, true); // append
+ BufferedOutputStream bos = new BufferedOutputStream(fos);
+ PrintStream ps = new PrintStream(bos);
+ ps.println(data);
+ ps.close();
+ bos.close();
+ fos.close();
+ }
+
+ public static String getAbsoluteFileName(String filename) {
+ return System.getProperty("user.home")+File.separator+
+ ANTLRWORKS_DIR +File.separator+
+ filename;
+ }
+}
diff --git a/src/org/antlr/runtime/tree/.DS_Store b/src/org/antlr/runtime/tree/.DS_Store
new file mode 100644
index 0000000..5008ddf
--- /dev/null
+++ b/src/org/antlr/runtime/tree/.DS_Store
Binary files differ
diff --git a/src/org/antlr/runtime/tree/BaseTree.java b/src/org/antlr/runtime/tree/BaseTree.java
new file mode 100755
index 0000000..ebffdb3
--- /dev/null
+++ b/src/org/antlr/runtime/tree/BaseTree.java
@@ -0,0 +1,349 @@
+/*
+ [The "BSD license"]
+ Copyright (c) 2005-2009 Terence Parr
+ All rights reserved.
+
+ Redistribution and use in source and binary forms, with or without
+ modification, are permitted provided that the following conditions
+ are met:
+ 1. Redistributions of source code must retain the above copyright
+ notice, this list of conditions and the following disclaimer.
+ 2. Redistributions in binary form must reproduce the above copyright
+ notice, this list of conditions and the following disclaimer in the
+ documentation and/or other materials provided with the distribution.
+ 3. The name of the author may not be used to endorse or promote products
+ derived from this software without specific prior written permission.
+
+ THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
+ IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
+ OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
+ IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
+ INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
+ NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
+ THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+package org.antlr.runtime.tree;
+
+import java.util.ArrayList;
+import java.util.List;
+
+/** A generic tree implementation with no payload. You must subclass to
+ * actually have any user data. ANTLR v3 uses a list of children approach
+ * instead of the child-sibling approach in v2. A flat tree (a list) is
+ * an empty node whose children represent the list. An empty, but
+ * non-null node is called "nil".
+ */
+public abstract class BaseTree implements Tree {
+ protected List children;
+
+ public BaseTree() {
+ }
+
+ /** Create a new node from an existing node does nothing for BaseTree
+ * as there are no fields other than the children list, which cannot
+ * be copied as the children are not considered part of this node.
+ */
+ public BaseTree(Tree node) {
+ }
+
+ public Tree getChild(int i) {
+ if ( children==null || i>=children.size() ) {
+ return null;
+ }
+ return (Tree)children.get(i);
+ }
+
+ /** Get the children internal List; note that if you directly mess with
+ * the list, do so at your own risk.
+ */
+ public List getChildren() {
+ return children;
+ }
+
+ public Tree getFirstChildWithType(int type) {
+ for (int i = 0; children!=null && i < children.size(); i++) {
+ Tree t = (Tree) children.get(i);
+ if ( t.getType()==type ) {
+ return t;
+ }
+ }
+ return null;
+ }
+
+ public int getChildCount() {
+ if ( children==null ) {
+ return 0;
+ }
+ return children.size();
+ }
+
+ /** Add t as child of this node.
+ *
+ * Warning: if t has no children, but child does
+ * and child isNil then this routine moves children to t via
+ * t.children = child.children; i.e., without copying the array.
+ */
+ public void addChild(Tree t) {
+ //System.out.println("add child "+t.toStringTree()+" "+this.toStringTree());
+ //System.out.println("existing children: "+children);
+ if ( t==null ) {
+ return; // do nothing upon addChild(null)
+ }
+ BaseTree childTree = (BaseTree)t;
+ if ( childTree.isNil() ) { // t is an empty node possibly with children
+ if ( this.children!=null && this.children == childTree.children ) {
+ throw new RuntimeException("attempt to add child list to itself");
+ }
+ // just add all of childTree's children to this
+ if ( childTree.children!=null ) {
+ if ( this.children!=null ) { // must copy, this has children already
+ int n = childTree.children.size();
+ for (int i = 0; i < n; i++) {
+ Tree c = (Tree)childTree.children.get(i);
+ this.children.add(c);
+ // handle double-link stuff for each child of nil root
+ c.setParent(this);
+ c.setChildIndex(children.size()-1);
+ }
+ }
+ else {
+ // no children for this but t has children; just set pointer
+ // call general freshener routine
+ this.children = childTree.children;
+ this.freshenParentAndChildIndexes();
+ }
+ }
+ }
+ else { // child is not nil (don't care about children)
+ if ( children==null ) {
+ children = createChildrenList(); // create children list on demand
+ }
+ children.add(t);
+ childTree.setParent(this);
+ childTree.setChildIndex(children.size()-1);
+ }
+ // System.out.println("now children are: "+children);
+ }
+
+ /** Add all elements of kids list as children of this node */
+ public void addChildren(List kids) {
+ for (int i = 0; i < kids.size(); i++) {
+ Tree t = (Tree) kids.get(i);
+ addChild(t);
+ }
+ }
+
+ public void setChild(int i, Tree t) {
+ if ( t==null ) {
+ return;
+ }
+ if ( t.isNil() ) {
+ throw new IllegalArgumentException("Can't set single child to a list");
+ }
+ if ( children==null ) {
+ children = createChildrenList();
+ }
+ children.set(i, t);
+ t.setParent(this);
+ t.setChildIndex(i);
+ }
+
+ public Object deleteChild(int i) {
+ if ( children==null ) {
+ return null;
+ }
+ Tree killed = (Tree)children.remove(i);
+ // walk rest and decrement their child indexes
+ this.freshenParentAndChildIndexes(i);
+ return killed;
+ }
+
+ /** Delete children from start to stop and replace with t even if t is
+ * a list (nil-root tree). num of children can increase or decrease.
+ * For huge child lists, inserting children can force walking rest of
+ * children to set their childindex; could be slow.
+ */
+ public void replaceChildren(int startChildIndex, int stopChildIndex, Object t) {
+ /*
+ System.out.println("replaceChildren "+startChildIndex+", "+stopChildIndex+
+ " with "+((BaseTree)t).toStringTree());
+ System.out.println("in="+toStringTree());
+ */
+ if ( children==null ) {
+ throw new IllegalArgumentException("indexes invalid; no children in list");
+ }
+ int replacingHowMany = stopChildIndex - startChildIndex + 1;
+ int replacingWithHowMany;
+ BaseTree newTree = (BaseTree)t;
+ List newChildren = null;
+ // normalize to a list of children to add: newChildren
+ if ( newTree.isNil() ) {
+ newChildren = newTree.children;
+ }
+ else {
+ newChildren = new ArrayList(1);
+ newChildren.add(newTree);
+ }
+ replacingWithHowMany = newChildren.size();
+ int numNewChildren = newChildren.size();
+ int delta = replacingHowMany - replacingWithHowMany;
+ // if same number of nodes, do direct replace
+ if ( delta == 0 ) {
+ int j = 0; // index into new children
+ for (int i=startChildIndex; i<=stopChildIndex; i++) {
+ BaseTree child = (BaseTree)newChildren.get(j);
+ children.set(i, child);
+ child.setParent(this);
+ child.setChildIndex(i);
+ j++;
+ }
+ }
+ else if ( delta > 0 ) { // fewer new nodes than there were
+ // set children and then delete extra
+ for (int j=0; j<numNewChildren; j++) {
+ children.set(startChildIndex+j, newChildren.get(j));
+ }
+ int indexToDelete = startChildIndex+numNewChildren;
+ for (int c=indexToDelete; c<=stopChildIndex; c++) {
+ // delete same index, shifting everybody down each time
+ children.remove(indexToDelete);
+ }
+ freshenParentAndChildIndexes(startChildIndex);
+ }
+ else { // more new nodes than were there before
+ // fill in as many children as we can (replacingHowMany) w/o moving data
+ for (int j=0; j<replacingHowMany; j++) {
+ children.set(startChildIndex+j, newChildren.get(j));
+ }
+ int numToInsert = replacingWithHowMany-replacingHowMany;
+ for (int j=replacingHowMany; j<replacingWithHowMany; j++) {
+ children.add(startChildIndex+j, newChildren.get(j));
+ }
+ freshenParentAndChildIndexes(startChildIndex);
+ }
+ //System.out.println("out="+toStringTree());
+ }
+
+ /** Override in a subclass to change the impl of children list */
+ protected List createChildrenList() {
+ return new ArrayList();
+ }
+
+ public boolean isNil() {
+ return false;
+ }
+
+ /** Set the parent and child index values for all child of t */
+ public void freshenParentAndChildIndexes() {
+ freshenParentAndChildIndexes(0);
+ }
+
+ public void freshenParentAndChildIndexes(int offset) {
+ int n = getChildCount();
+ for (int c = offset; c < n; c++) {
+ Tree child = (Tree)getChild(c);
+ child.setChildIndex(c);
+ child.setParent(this);
+ }
+ }
+
+ public void sanityCheckParentAndChildIndexes() {
+ sanityCheckParentAndChildIndexes(null, -1);
+ }
+
+ public void sanityCheckParentAndChildIndexes(Tree parent, int i) {
+ if ( parent!=this.getParent() ) {
+ throw new IllegalStateException("parents don't match; expected "+parent+" found "+this.getParent());
+ }
+ if ( i!=this.getChildIndex() ) {
+ throw new IllegalStateException("child indexes don't match; expected "+i+" found "+this.getChildIndex());
+ }
+ int n = this.getChildCount();
+ for (int c = 0; c < n; c++) {
+ CommonTree child = (CommonTree)this.getChild(c);
+ child.sanityCheckParentAndChildIndexes(this, c);
+ }
+ }
+
+ /** BaseTree doesn't track child indexes. */
+ public int getChildIndex() {
+ return 0;
+ }
+ public void setChildIndex(int index) {
+ }
+
+ /** BaseTree doesn't track parent pointers. */
+ public Tree getParent() {
+ return null;
+ }
+
+ public void setParent(Tree t) {
+ }
+
+ /** Walk upwards looking for ancestor with this token type. */
+ public boolean hasAncestor(int ttype) { return getAncestor(ttype)!=null; }
+
+ /** Walk upwards and get first ancestor with this token type. */
+ public Tree getAncestor(int ttype) {
+ Tree t = this;
+ t = t.getParent();
+ while ( t!=null ) {
+ if ( t.getType()==ttype ) return t;
+ t = t.getParent();
+ }
+ return null;
+ }
+
+ /** Return a list of all ancestors of this node. The first node of
+ * list is the root and the last is the parent of this node.
+ */
+ public List getAncestors() {
+ if ( getParent()==null ) return null;
+ List ancestors = new ArrayList();
+ Tree t = this;
+ t = t.getParent();
+ while ( t!=null ) {
+ ancestors.add(0, t); // insert at start
+ t = t.getParent();
+ }
+ return ancestors;
+ }
+
+ /** Print out a whole tree not just a node */
+ public String toStringTree() {
+ if ( children==null || children.size()==0 ) {
+ return this.toString();
+ }
+ StringBuffer buf = new StringBuffer();
+ if ( !isNil() ) {
+ buf.append("(");
+ buf.append(this.toString());
+ buf.append(' ');
+ }
+ for (int i = 0; children!=null && i < children.size(); i++) {
+ Tree t = (Tree)children.get(i);
+ if ( i>0 ) {
+ buf.append(' ');
+ }
+ buf.append(t.toStringTree());
+ }
+ if ( !isNil() ) {
+ buf.append(")");
+ }
+ return buf.toString();
+ }
+
+ public int getLine() {
+ return 0;
+ }
+
+ public int getCharPositionInLine() {
+ return 0;
+ }
+
+ /** Override to say how a node (not a tree) should look as text */
+ public abstract String toString();
+}
diff --git a/src/org/antlr/runtime/tree/BaseTreeAdaptor.java b/src/org/antlr/runtime/tree/BaseTreeAdaptor.java
new file mode 100755
index 0000000..33140b1
--- /dev/null
+++ b/src/org/antlr/runtime/tree/BaseTreeAdaptor.java
@@ -0,0 +1,279 @@
+/*
+ [The "BSD license"]
+ Copyright (c) 2005-2009 Terence Parr
+ All rights reserved.
+
+ Redistribution and use in source and binary forms, with or without
+ modification, are permitted provided that the following conditions
+ are met:
+ 1. Redistributions of source code must retain the above copyright
+ notice, this list of conditions and the following disclaimer.
+ 2. Redistributions in binary form must reproduce the above copyright
+ notice, this list of conditions and the following disclaimer in the
+ documentation and/or other materials provided with the distribution.
+ 3. The name of the author may not be used to endorse or promote products
+ derived from this software without specific prior written permission.
+
+ THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
+ IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
+ OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
+ IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
+ INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
+ NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
+ THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+package org.antlr.runtime.tree;
+
+import org.antlr.runtime.Token;
+import org.antlr.runtime.TokenStream;
+import org.antlr.runtime.RecognitionException;
+
+import java.util.HashMap;
+import java.util.Map;
+
+/** A TreeAdaptor that works with any Tree implementation. */
+public abstract class BaseTreeAdaptor implements TreeAdaptor {
+ /** System.identityHashCode() is not always unique; we have to
+ * track ourselves. That's ok, it's only for debugging, though it's
+ * expensive: we have to create a hashtable with all tree nodes in it.
+ */
+ protected Map treeToUniqueIDMap;
+ protected int uniqueNodeID = 1;
+
+ public Object nil() {
+ return create(null);
+ }
+
+ /** create tree node that holds the start and stop tokens associated
+ * with an error.
+ *
+ * If you specify your own kind of tree nodes, you will likely have to
+ * override this method. CommonTree returns Token.INVALID_TOKEN_TYPE
+ * if no token payload but you might have to set token type for diff
+ * node type.
+ *
+ * You don't have to subclass CommonErrorNode; you will likely need to
+ * subclass your own tree node class to avoid class cast exception.
+ */
+ public Object errorNode(TokenStream input, Token start, Token stop,
+ RecognitionException e)
+ {
+ CommonErrorNode t = new CommonErrorNode(input, start, stop, e);
+ //System.out.println("returning error node '"+t+"' @index="+input.index());
+ return t;
+ }
+
+ public boolean isNil(Object tree) {
+ return ((Tree)tree).isNil();
+ }
+
+ public Object dupTree(Object tree) {
+ return dupTree(tree, null);
+ }
+
+ /** This is generic in the sense that it will work with any kind of
+ * tree (not just Tree interface). It invokes the adaptor routines
+ * not the tree node routines to do the construction.
+ */
+ public Object dupTree(Object t, Object parent) {
+ if ( t==null ) {
+ return null;
+ }
+ Object newTree = dupNode(t);
+ // ensure new subtree root has parent/child index set
+ setChildIndex(newTree, getChildIndex(t)); // same index in new tree
+ setParent(newTree, parent);
+ int n = getChildCount(t);
+ for (int i = 0; i < n; i++) {
+ Object child = getChild(t, i);
+ Object newSubTree = dupTree(child, t);
+ addChild(newTree, newSubTree);
+ }
+ return newTree;
+ }
+
+ /** Add a child to the tree t. If child is a flat tree (a list), make all
+ * in list children of t. Warning: if t has no children, but child does
+ * and child isNil then you can decide it is ok to move children to t via
+ * t.children = child.children; i.e., without copying the array. Just
+ * make sure that this is consistent with have the user will build
+ * ASTs.
+ */
+ public void addChild(Object t, Object child) {
+ if ( t!=null && child!=null ) {
+ ((Tree)t).addChild((Tree)child);
+ }
+ }
+
+ /** If oldRoot is a nil root, just copy or move the children to newRoot.
+ * If not a nil root, make oldRoot a child of newRoot.
+ *
+ * old=^(nil a b c), new=r yields ^(r a b c)
+ * old=^(a b c), new=r yields ^(r ^(a b c))
+ *
+ * If newRoot is a nil-rooted single child tree, use the single
+ * child as the new root node.
+ *
+ * old=^(nil a b c), new=^(nil r) yields ^(r a b c)
+ * old=^(a b c), new=^(nil r) yields ^(r ^(a b c))
+ *
+ * If oldRoot was null, it's ok, just return newRoot (even if isNil).
+ *
+ * old=null, new=r yields r
+ * old=null, new=^(nil r) yields ^(nil r)
+ *
+ * Return newRoot. Throw an exception if newRoot is not a
+ * simple node or nil root with a single child node--it must be a root
+ * node. If newRoot is ^(nil x) return x as newRoot.
+ *
+ * Be advised that it's ok for newRoot to point at oldRoot's
+ * children; i.e., you don't have to copy the list. We are
+ * constructing these nodes so we should have this control for
+ * efficiency.
+ */
+ public Object becomeRoot(Object newRoot, Object oldRoot) {
+ //System.out.println("becomeroot new "+newRoot.toString()+" old "+oldRoot);
+ Tree newRootTree = (Tree)newRoot;
+ Tree oldRootTree = (Tree)oldRoot;
+ if ( oldRoot==null ) {
+ return newRoot;
+ }
+ // handle ^(nil real-node)
+ if ( newRootTree.isNil() ) {
+ int nc = newRootTree.getChildCount();
+ if ( nc==1 ) newRootTree = (Tree)newRootTree.getChild(0);
+ else if ( nc >1 ) {
+ // TODO: make tree run time exceptions hierarchy
+ throw new RuntimeException("more than one node as root (TODO: make exception hierarchy)");
+ }
+ }
+ // add oldRoot to newRoot; addChild takes care of case where oldRoot
+ // is a flat list (i.e., nil-rooted tree). All children of oldRoot
+ // are added to newRoot.
+ newRootTree.addChild(oldRootTree);
+ return newRootTree;
+ }
+
+ /** Transform ^(nil x) to x and nil to null */
+ public Object rulePostProcessing(Object root) {
+ //System.out.println("rulePostProcessing: "+((Tree)root).toStringTree());
+ Tree r = (Tree)root;
+ if ( r!=null && r.isNil() ) {
+ if ( r.getChildCount()==0 ) {
+ r = null;
+ }
+ else if ( r.getChildCount()==1 ) {
+ r = (Tree)r.getChild(0);
+ // whoever invokes rule will set parent and child index
+ r.setParent(null);
+ r.setChildIndex(-1);
+ }
+ }
+ return r;
+ }
+
+ public Object becomeRoot(Token newRoot, Object oldRoot) {
+ return becomeRoot(create(newRoot), oldRoot);
+ }
+
+ public Object create(int tokenType, Token fromToken) {
+ fromToken = createToken(fromToken);
+ //((ClassicToken)fromToken).setType(tokenType);
+ fromToken.setType(tokenType);
+ Tree t = (Tree)create(fromToken);
+ return t;
+ }
+
+ public Object create(int tokenType, Token fromToken, String text) {
+ if (fromToken == null) return create(tokenType, text);
+ fromToken = createToken(fromToken);
+ fromToken.setType(tokenType);
+ fromToken.setText(text);
+ Tree t = (Tree)create(fromToken);
+ return t;
+ }
+
+ public Object create(int tokenType, String text) {
+ Token fromToken = createToken(tokenType, text);
+ Tree t = (Tree)create(fromToken);
+ return t;
+ }
+
+ public int getType(Object t) {
+ return ((Tree)t).getType();
+ }
+
+ public void setType(Object t, int type) {
+ throw new NoSuchMethodError("don't know enough about Tree node");
+ }
+
+ public String getText(Object t) {
+ return ((Tree)t).getText();
+ }
+
+ public void setText(Object t, String text) {
+ throw new NoSuchMethodError("don't know enough about Tree node");
+ }
+
+ public Object getChild(Object t, int i) {
+ return ((Tree)t).getChild(i);
+ }
+
+ public void setChild(Object t, int i, Object child) {
+ ((Tree)t).setChild(i, (Tree)child);
+ }
+
+ public Object deleteChild(Object t, int i) {
+ return ((Tree)t).deleteChild(i);
+ }
+
+ public int getChildCount(Object t) {
+ return ((Tree)t).getChildCount();
+ }
+
+ public int getUniqueID(Object node) {
+ if ( treeToUniqueIDMap==null ) {
+ treeToUniqueIDMap = new HashMap();
+ }
+ Integer prevID = (Integer)treeToUniqueIDMap.get(node);
+ if ( prevID!=null ) {
+ return prevID.intValue();
+ }
+ int ID = uniqueNodeID;
+ treeToUniqueIDMap.put(node, new Integer(ID));
+ uniqueNodeID++;
+ return ID;
+ // GC makes these nonunique:
+ // return System.identityHashCode(node);
+ }
+
+ /** Tell me how to create a token for use with imaginary token nodes.
+ * For example, there is probably no input symbol associated with imaginary
+ * token DECL, but you need to create it as a payload or whatever for
+ * the DECL node as in ^(DECL type ID).
+ *
+ * If you care what the token payload objects' type is, you should
+ * override this method and any other createToken variant.
+ */
+ public abstract Token createToken(int tokenType, String text);
+
+ /** Tell me how to create a token for use with imaginary token nodes.
+ * For example, there is probably no input symbol associated with imaginary
+ * token DECL, but you need to create it as a payload or whatever for
+ * the DECL node as in ^(DECL type ID).
+ *
+ * This is a variant of createToken where the new token is derived from
+ * an actual real input token. Typically this is for converting '{'
+ * tokens to BLOCK etc... You'll see
+ *
+ * r : lc='{' ID+ '}' -> ^(BLOCK[$lc] ID+) ;
+ *
+ * If you care what the token payload objects' type is, you should
+ * override this method and any other createToken variant.
+ */
+ public abstract Token createToken(Token fromToken);
+}
+
diff --git a/src/org/antlr/runtime/tree/BufferedTreeNodeStream.java b/src/org/antlr/runtime/tree/BufferedTreeNodeStream.java
new file mode 100755
index 0000000..d9a2a7e
--- /dev/null
+++ b/src/org/antlr/runtime/tree/BufferedTreeNodeStream.java
@@ -0,0 +1,478 @@
+/*
+ [The "BSD license"]
+ Copyright (c) 2005-2009 Terence Parr
+ All rights reserved.
+
+ Redistribution and use in source and binary forms, with or without
+ modification, are permitted provided that the following conditions
+ are met:
+ 1. Redistributions of source code must retain the above copyright
+ notice, this list of conditions and the following disclaimer.
+ 2. Redistributions in binary form must reproduce the above copyright
+ notice, this list of conditions and the following disclaimer in the
+ documentation and/or other materials provided with the distribution.
+ 3. The name of the author may not be used to endorse or promote products
+ derived from this software without specific prior written permission.
+
+ THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
+ IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
+ OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
+ IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
+ INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
+ NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
+ THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+package org.antlr.runtime.tree;
+
+import org.antlr.runtime.Token;
+import org.antlr.runtime.TokenStream;
+import org.antlr.runtime.misc.IntArray;
+import java.util.*;
+
+/** A buffered stream of tree nodes. Nodes can be from a tree of ANY kind.
+ *
+ * This node stream sucks all nodes out of the tree specified in
+ * the constructor during construction and makes pointers into
+ * the tree using an array of Object pointers. The stream necessarily
+ * includes pointers to DOWN and UP and EOF nodes.
+ *
+ * This stream knows how to mark/release for backtracking.
+ *
+ * This stream is most suitable for tree interpreters that need to
+ * jump around a lot or for tree parsers requiring speed (at cost of memory).
+ * There is some duplicated functionality here with UnBufferedTreeNodeStream
+ * but just in bookkeeping, not tree walking etc...
+ *
+ * TARGET DEVELOPERS:
+ *
+ * This is the old CommonTreeNodeStream that buffered up entire node stream.
+ * No need to implement really as new CommonTreeNodeStream is much better
+ * and covers what we need.
+ *
+ * @see CommonTreeNodeStream
+ */
+public class BufferedTreeNodeStream implements TreeNodeStream {
+ public static final int DEFAULT_INITIAL_BUFFER_SIZE = 100;
+ public static final int INITIAL_CALL_STACK_SIZE = 10;
+
+ protected class StreamIterator implements Iterator {
+ int i = 0;
+ public boolean hasNext() {
+ return i<nodes.size();
+ }
+
+ public Object next() {
+ int current = i;
+ i++;
+ if ( current < nodes.size() ) {
+ return nodes.get(current);
+ }
+ return eof;
+ }
+
+ public void remove() {
+ throw new RuntimeException("cannot remove nodes from stream");
+ }
+ }
+
+ // all these navigation nodes are shared and hence they
+ // cannot contain any line/column info
+
+ protected Object down;
+ protected Object up;
+ protected Object eof;
+
+ /** The complete mapping from stream index to tree node.
+ * This buffer includes pointers to DOWN, UP, and EOF nodes.
+ * It is built upon ctor invocation. The elements are type
+ * Object as we don't what the trees look like.
+ *
+ * Load upon first need of the buffer so we can set token types
+ * of interest for reverseIndexing. Slows us down a wee bit to
+ * do all of the if p==-1 testing everywhere though.
+ */
+ protected List nodes;
+
+ /** Pull nodes from which tree? */
+ protected Object root;
+
+ /** IF this tree (root) was created from a token stream, track it. */
+ protected TokenStream tokens;
+
+ /** What tree adaptor was used to build these trees */
+ TreeAdaptor adaptor;
+
+ /** Reuse same DOWN, UP navigation nodes unless this is true */
+ protected boolean uniqueNavigationNodes = false;
+
+ /** The index into the nodes list of the current node (next node
+ * to consume). If -1, nodes array not filled yet.
+ */
+ protected int p = -1;
+
+ /** Track the last mark() call result value for use in rewind(). */
+ protected int lastMarker;
+
+ /** Stack of indexes used for push/pop calls */
+ protected IntArray calls;
+
+ public BufferedTreeNodeStream(Object tree) {
+ this(new CommonTreeAdaptor(), tree);
+ }
+
+ public BufferedTreeNodeStream(TreeAdaptor adaptor, Object tree) {
+ this(adaptor, tree, DEFAULT_INITIAL_BUFFER_SIZE);
+ }
+
+ public BufferedTreeNodeStream(TreeAdaptor adaptor, Object tree, int initialBufferSize) {
+ this.root = tree;
+ this.adaptor = adaptor;
+ nodes = new ArrayList(initialBufferSize);
+ down = adaptor.create(Token.DOWN, "DOWN");
+ up = adaptor.create(Token.UP, "UP");
+ eof = adaptor.create(Token.EOF, "EOF");
+ }
+
+ /** Walk tree with depth-first-search and fill nodes buffer.
+ * Don't do DOWN, UP nodes if its a list (t is isNil).
+ */
+ protected void fillBuffer() {
+ fillBuffer(root);
+ //System.out.println("revIndex="+tokenTypeToStreamIndexesMap);
+ p = 0; // buffer of nodes intialized now
+ }
+
+ public void fillBuffer(Object t) {
+ boolean nil = adaptor.isNil(t);
+ if ( !nil ) {
+ nodes.add(t); // add this node
+ }
+ // add DOWN node if t has children
+ int n = adaptor.getChildCount(t);
+ if ( !nil && n>0 ) {
+ addNavigationNode(Token.DOWN);
+ }
+ // and now add all its children
+ for (int c=0; c<n; c++) {
+ Object child = adaptor.getChild(t,c);
+ fillBuffer(child);
+ }
+ // add UP node if t has children
+ if ( !nil && n>0 ) {
+ addNavigationNode(Token.UP);
+ }
+ }
+
+ /** What is the stream index for node? 0..n-1
+ * Return -1 if node not found.
+ */
+ protected int getNodeIndex(Object node) {
+ if ( p==-1 ) {
+ fillBuffer();
+ }
+ for (int i = 0; i < nodes.size(); i++) {
+ Object t = (Object) nodes.get(i);
+ if ( t==node ) {
+ return i;
+ }
+ }
+ return -1;
+ }
+
+ /** As we flatten the tree, we use UP, DOWN nodes to represent
+ * the tree structure. When debugging we need unique nodes
+ * so instantiate new ones when uniqueNavigationNodes is true.
+ */
+ protected void addNavigationNode(final int ttype) {
+ Object navNode = null;
+ if ( ttype==Token.DOWN ) {
+ if ( hasUniqueNavigationNodes() ) {
+ navNode = adaptor.create(Token.DOWN, "DOWN");
+ }
+ else {
+ navNode = down;
+ }
+ }
+ else {
+ if ( hasUniqueNavigationNodes() ) {
+ navNode = adaptor.create(Token.UP, "UP");
+ }
+ else {
+ navNode = up;
+ }
+ }
+ nodes.add(navNode);
+ }
+
+ public Object get(int i) {
+ if ( p==-1 ) {
+ fillBuffer();
+ }
+ return nodes.get(i);
+ }
+
+ public Object LT(int k) {
+ if ( p==-1 ) {
+ fillBuffer();
+ }
+ if ( k==0 ) {
+ return null;
+ }
+ if ( k<0 ) {
+ return LB(-k);
+ }
+ //System.out.print("LT(p="+p+","+k+")=");
+ if ( (p+k-1) >= nodes.size() ) {
+ return eof;
+ }
+ return nodes.get(p+k-1);
+ }
+
+ public Object getCurrentSymbol() { return LT(1); }
+
+/*
+ public Object getLastTreeNode() {
+ int i = index();
+ if ( i>=size() ) {
+ i--; // if at EOF, have to start one back
+ }
+ System.out.println("start last node: "+i+" size=="+nodes.size());
+ while ( i>=0 &&
+ (adaptor.getType(get(i))==Token.EOF ||
+ adaptor.getType(get(i))==Token.UP ||
+ adaptor.getType(get(i))==Token.DOWN) )
+ {
+ i--;
+ }
+ System.out.println("stop at node: "+i+" "+nodes.get(i));
+ return nodes.get(i);
+ }
+*/
+
+ /** Look backwards k nodes */
+ protected Object LB(int k) {
+ if ( k==0 ) {
+ return null;
+ }
+ if ( (p-k)<0 ) {
+ return null;
+ }
+ return nodes.get(p-k);
+ }
+
+ public Object getTreeSource() {
+ return root;
+ }
+
+ public String getSourceName() {
+ return getTokenStream().getSourceName();
+ }
+
+ public TokenStream getTokenStream() {
+ return tokens;
+ }
+
+ public void setTokenStream(TokenStream tokens) {
+ this.tokens = tokens;
+ }
+
+ public TreeAdaptor getTreeAdaptor() {
+ return adaptor;
+ }
+
+ public void setTreeAdaptor(TreeAdaptor adaptor) {
+ this.adaptor = adaptor;
+ }
+
+ public boolean hasUniqueNavigationNodes() {
+ return uniqueNavigationNodes;
+ }
+
+ public void setUniqueNavigationNodes(boolean uniqueNavigationNodes) {
+ this.uniqueNavigationNodes = uniqueNavigationNodes;
+ }
+
+ public void consume() {
+ if ( p==-1 ) {
+ fillBuffer();
+ }
+ p++;
+ }
+
+ public int LA(int i) {
+ return adaptor.getType(LT(i));
+ }
+
+ public int mark() {
+ if ( p==-1 ) {
+ fillBuffer();
+ }
+ lastMarker = index();
+ return lastMarker;
+ }
+
+ public void release(int marker) {
+ // no resources to release
+ }
+
+ public int index() {
+ return p;
+ }
+
+ public void rewind(int marker) {
+ seek(marker);
+ }
+
+ public void rewind() {
+ seek(lastMarker);
+ }
+
+ public void seek(int index) {
+ if ( p==-1 ) {
+ fillBuffer();
+ }
+ p = index;
+ }
+
+ /** Make stream jump to a new location, saving old location.
+ * Switch back with pop().
+ */
+ public void push(int index) {
+ if ( calls==null ) {
+ calls = new IntArray();
+ }
+ calls.push(p); // save current index
+ seek(index);
+ }
+
+ /** Seek back to previous index saved during last push() call.
+ * Return top of stack (return index).
+ */
+ public int pop() {
+ int ret = calls.pop();
+ seek(ret);
+ return ret;
+ }
+
+ public void reset() {
+ p = 0;
+ lastMarker = 0;
+ if (calls != null) {
+ calls.clear();
+ }
+ }
+
+ public int size() {
+ if ( p==-1 ) {
+ fillBuffer();
+ }
+ return nodes.size();
+ }
+
+ public Iterator iterator() {
+ if ( p==-1 ) {
+ fillBuffer();
+ }
+ return new StreamIterator();
+ }
+
+ // TREE REWRITE INTERFACE
+
+ public void replaceChildren(Object parent, int startChildIndex, int stopChildIndex, Object t) {
+ if ( parent!=null ) {
+ adaptor.replaceChildren(parent, startChildIndex, stopChildIndex, t);
+ }
+ }
+
+ /** Used for testing, just return the token type stream */
+ public String toTokenTypeString() {
+ if ( p==-1 ) {
+ fillBuffer();
+ }
+ StringBuffer buf = new StringBuffer();
+ for (int i = 0; i < nodes.size(); i++) {
+ Object t = (Object) nodes.get(i);
+ buf.append(" ");
+ buf.append(adaptor.getType(t));
+ }
+ return buf.toString();
+ }
+
+ /** Debugging */
+ public String toTokenString(int start, int stop) {
+ if ( p==-1 ) {
+ fillBuffer();
+ }
+ StringBuffer buf = new StringBuffer();
+ for (int i = start; i < nodes.size() && i <= stop; i++) {
+ Object t = (Object) nodes.get(i);
+ buf.append(" ");
+ buf.append(adaptor.getToken(t));
+ }
+ return buf.toString();
+ }
+
+ public String toString(Object start, Object stop) {
+ System.out.println("toString");
+ if ( start==null || stop==null ) {
+ return null;
+ }
+ if ( p==-1 ) {
+ fillBuffer();
+ }
+ //System.out.println("stop: "+stop);
+ if ( start instanceof CommonTree )
+ System.out.print("toString: "+((CommonTree)start).getToken()+", ");
+ else
+ System.out.println(start);
+ if ( stop instanceof CommonTree )
+ System.out.println(((CommonTree)stop).getToken());
+ else
+ System.out.println(stop);
+ // if we have the token stream, use that to dump text in order
+ if ( tokens!=null ) {
+ int beginTokenIndex = adaptor.getTokenStartIndex(start);
+ int endTokenIndex = adaptor.getTokenStopIndex(stop);
+ // if it's a tree, use start/stop index from start node
+ // else use token range from start/stop nodes
+ if ( adaptor.getType(stop)==Token.UP ) {
+ endTokenIndex = adaptor.getTokenStopIndex(start);
+ }
+ else if ( adaptor.getType(stop)==Token.EOF ) {
+ endTokenIndex = size()-2; // don't use EOF
+ }
+ return tokens.toString(beginTokenIndex, endTokenIndex);
+ }
+ // walk nodes looking for start
+ Object t = null;
+ int i = 0;
+ for (; i < nodes.size(); i++) {
+ t = nodes.get(i);
+ if ( t==start ) {
+ break;
+ }
+ }
+ // now walk until we see stop, filling string buffer with text
+ StringBuffer buf = new StringBuffer();
+ t = nodes.get(i);
+ while ( t!=stop ) {
+ String text = adaptor.getText(t);
+ if ( text==null ) {
+ text = " "+String.valueOf(adaptor.getType(t));
+ }
+ buf.append(text);
+ i++;
+ t = nodes.get(i);
+ }
+ // include stop node too
+ String text = adaptor.getText(stop);
+ if ( text==null ) {
+ text = " "+String.valueOf(adaptor.getType(stop));
+ }
+ buf.append(text);
+ return buf.toString();
+ }
+}
diff --git a/src/org/antlr/runtime/tree/CommonErrorNode.java b/src/org/antlr/runtime/tree/CommonErrorNode.java
new file mode 100755
index 0000000..26b9933
--- /dev/null
+++ b/src/org/antlr/runtime/tree/CommonErrorNode.java
@@ -0,0 +1,108 @@
+/*
+ [The "BSD license"]
+ Copyright (c) 2005-2009 Terence Parr
+ All rights reserved.
+
+ Redistribution and use in source and binary forms, with or without
+ modification, are permitted provided that the following conditions
+ are met:
+ 1. Redistributions of source code must retain the above copyright
+ notice, this list of conditions and the following disclaimer.
+ 2. Redistributions in binary form must reproduce the above copyright
+ notice, this list of conditions and the following disclaimer in the
+ documentation and/or other materials provided with the distribution.
+ 3. The name of the author may not be used to endorse or promote products
+ derived from this software without specific prior written permission.
+
+ THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
+ IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
+ OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
+ IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
+ INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
+ NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
+ THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+package org.antlr.runtime.tree;
+
+import org.antlr.runtime.*;
+
+/** A node representing erroneous token range in token stream */
+public class CommonErrorNode extends CommonTree {
+ public IntStream input;
+ public Token start;
+ public Token stop;
+ public RecognitionException trappedException;
+
+ public CommonErrorNode(TokenStream input, Token start, Token stop,
+ RecognitionException e)
+ {
+ //System.out.println("start: "+start+", stop: "+stop);
+ if ( stop==null ||
+ (stop.getTokenIndex() < start.getTokenIndex() &&
+ stop.getType()!=Token.EOF) )
+ {
+ // sometimes resync does not consume a token (when LT(1) is
+ // in follow set. So, stop will be 1 to left to start. adjust.
+ // Also handle case where start is the first token and no token
+ // is consumed during recovery; LT(-1) will return null.
+ stop = start;
+ }
+ this.input = input;
+ this.start = start;
+ this.stop = stop;
+ this.trappedException = e;
+ }
+
+ public boolean isNil() {
+ return false;
+ }
+
+ public int getType() {
+ return Token.INVALID_TOKEN_TYPE;
+ }
+
+ public String getText() {
+ String badText = null;
+ if ( start instanceof Token ) {
+ int i = ((Token)start).getTokenIndex();
+ int j = ((Token)stop).getTokenIndex();
+ if ( ((Token)stop).getType() == Token.EOF ) {
+ j = ((TokenStream)input).size();
+ }
+ badText = ((TokenStream)input).toString(i, j);
+ }
+ else if ( start instanceof Tree ) {
+ badText = ((TreeNodeStream)input).toString(start, stop);
+ }
+ else {
+ // people should subclass if they alter the tree type so this
+ // next one is for sure correct.
+ badText = "<unknown>";
+ }
+ return badText;
+ }
+
+ public String toString() {
+ if ( trappedException instanceof MissingTokenException ) {
+ return "<missing type: "+
+ ((MissingTokenException)trappedException).getMissingType()+
+ ">";
+ }
+ else if ( trappedException instanceof UnwantedTokenException ) {
+ return "<extraneous: "+
+ ((UnwantedTokenException)trappedException).getUnexpectedToken()+
+ ", resync="+getText()+">";
+ }
+ else if ( trappedException instanceof MismatchedTokenException ) {
+ return "<mismatched token: "+trappedException.token+", resync="+getText()+">";
+ }
+ else if ( trappedException instanceof NoViableAltException ) {
+ return "<unexpected: "+trappedException.token+
+ ", resync="+getText()+">";
+ }
+ return "<error: "+getText()+">";
+ }
+}
diff --git a/src/org/antlr/runtime/tree/CommonTree.java b/src/org/antlr/runtime/tree/CommonTree.java
new file mode 100755
index 0000000..91c59de
--- /dev/null
+++ b/src/org/antlr/runtime/tree/CommonTree.java
@@ -0,0 +1,185 @@
+/*
+ [The "BSD license"]
+ Copyright (c) 2005-2009 Terence Parr
+ All rights reserved.
+
+ Redistribution and use in source and binary forms, with or without
+ modification, are permitted provided that the following conditions
+ are met:
+ 1. Redistributions of source code must retain the above copyright
+ notice, this list of conditions and the following disclaimer.
+ 2. Redistributions in binary form must reproduce the above copyright
+ notice, this list of conditions and the following disclaimer in the
+ documentation and/or other materials provided with the distribution.
+ 3. The name of the author may not be used to endorse or promote products
+ derived from this software without specific prior written permission.
+
+ THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
+ IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
+ OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
+ IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
+ INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
+ NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
+ THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+package org.antlr.runtime.tree;
+
+import org.antlr.runtime.Token;
+
+/** A tree node that is wrapper for a Token object. After 3.0 release
+ * while building tree rewrite stuff, it became clear that computing
+ * parent and child index is very difficult and cumbersome. Better to
+ * spend the space in every tree node. If you don't want these extra
+ * fields, it's easy to cut them out in your own BaseTree subclass.
+ */
+public class CommonTree extends BaseTree {
+ /** A single token is the payload */
+ public Token token;
+
+ /** What token indexes bracket all tokens associated with this node
+ * and below?
+ */
+ protected int startIndex=-1, stopIndex=-1;
+
+ /** Who is the parent node of this node; if null, implies node is root */
+ public CommonTree parent;
+
+ /** What index is this node in the child list? Range: 0..n-1 */
+ public int childIndex = -1;
+
+ public CommonTree() { }
+
+ public CommonTree(CommonTree node) {
+ super(node);
+ this.token = node.token;
+ this.startIndex = node.startIndex;
+ this.stopIndex = node.stopIndex;
+ }
+
+ public CommonTree(Token t) {
+ this.token = t;
+ }
+
+ public Token getToken() {
+ return token;
+ }
+
+ public Tree dupNode() {
+ return new CommonTree(this);
+ }
+
+ public boolean isNil() {
+ return token==null;
+ }
+
+ public int getType() {
+ if ( token==null ) {
+ return Token.INVALID_TOKEN_TYPE;
+ }
+ return token.getType();
+ }
+
+ public String getText() {
+ if ( token==null ) {
+ return null;
+ }
+ return token.getText();
+ }
+
+ public int getLine() {
+ if ( token==null || token.getLine()==0 ) {
+ if ( getChildCount()>0 ) {
+ return getChild(0).getLine();
+ }
+ return 0;
+ }
+ return token.getLine();
+ }
+
+ public int getCharPositionInLine() {
+ if ( token==null || token.getCharPositionInLine()==-1 ) {
+ if ( getChildCount()>0 ) {
+ return getChild(0).getCharPositionInLine();
+ }
+ return 0;
+ }
+ return token.getCharPositionInLine();
+ }
+
+ public int getTokenStartIndex() {
+ if ( startIndex==-1 && token!=null ) {
+ return token.getTokenIndex();
+ }
+ return startIndex;
+ }
+
+ public void setTokenStartIndex(int index) {
+ startIndex = index;
+ }
+
+ public int getTokenStopIndex() {
+ if ( stopIndex==-1 && token!=null ) {
+ return token.getTokenIndex();
+ }
+ return stopIndex;
+ }
+
+ public void setTokenStopIndex(int index) {
+ stopIndex = index;
+ }
+
+ /** For every node in this subtree, make sure it's start/stop token's
+ * are set. Walk depth first, visit bottom up. Only updates nodes
+ * with at least one token index < 0.
+ */
+ public void setUnknownTokenBoundaries() {
+ if ( children==null ) {
+ if ( startIndex<0 || stopIndex<0 ) {
+ startIndex = stopIndex = token.getTokenIndex();
+ }
+ return;
+ }
+ for (int i=0; i<children.size(); i++) {
+ ((CommonTree)children.get(i)).setUnknownTokenBoundaries();
+ }
+ if ( startIndex>=0 && stopIndex>=0 ) return; // already set
+ if ( children.size() > 0 ) {
+ CommonTree firstChild = (CommonTree)children.get(0);
+ CommonTree lastChild = (CommonTree)children.get(children.size()-1);
+ startIndex = firstChild.getTokenStartIndex();
+ stopIndex = lastChild.getTokenStopIndex();
+ }
+ }
+
+ public int getChildIndex() {
+ return childIndex;
+ }
+
+ public Tree getParent() {
+ return parent;
+ }
+
+ public void setParent(Tree t) {
+ this.parent = (CommonTree)t;
+ }
+
+ public void setChildIndex(int index) {
+ this.childIndex = index;
+ }
+
+ public String toString() {
+ if ( isNil() ) {
+ return "nil";
+ }
+ if ( getType()==Token.INVALID_TOKEN_TYPE ) {
+ return "<errornode>";
+ }
+ if ( token==null ) {
+ return null;
+ }
+ return token.getText();
+ }
+}
diff --git a/src/org/antlr/runtime/tree/CommonTreeAdaptor.java b/src/org/antlr/runtime/tree/CommonTreeAdaptor.java
new file mode 100755
index 0000000..ebf560b
--- /dev/null
+++ b/src/org/antlr/runtime/tree/CommonTreeAdaptor.java
@@ -0,0 +1,168 @@
+/*
+ [The "BSD license"]
+ Copyright (c) 2005-2009 Terence Parr
+ All rights reserved.
+
+ Redistribution and use in source and binary forms, with or without
+ modification, are permitted provided that the following conditions
+ are met:
+ 1. Redistributions of source code must retain the above copyright
+ notice, this list of conditions and the following disclaimer.
+ 2. Redistributions in binary form must reproduce the above copyright
+ notice, this list of conditions and the following disclaimer in the
+ documentation and/or other materials provided with the distribution.
+ 3. The name of the author may not be used to endorse or promote products
+ derived from this software without specific prior written permission.
+
+ THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
+ IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
+ OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
+ IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
+ INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
+ NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
+ THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+package org.antlr.runtime.tree;
+
+import org.antlr.runtime.CommonToken;
+import org.antlr.runtime.Token;
+
+/** A TreeAdaptor that works with any Tree implementation. It provides
+ * really just factory methods; all the work is done by BaseTreeAdaptor.
+ * If you would like to have different tokens created than ClassicToken
+ * objects, you need to override this and then set the parser tree adaptor to
+ * use your subclass.
+ *
+ * To get your parser to build nodes of a different type, override
+ * create(Token), errorNode(), and to be safe, YourTreeClass.dupNode().
+ * dupNode is called to duplicate nodes during rewrite operations.
+ */
+public class CommonTreeAdaptor extends BaseTreeAdaptor {
+ /** Duplicate a node. This is part of the factory;
+ * override if you want another kind of node to be built.
+ *
+ * I could use reflection to prevent having to override this
+ * but reflection is slow.
+ */
+ public Object dupNode(Object t) {
+ if ( t==null ) return null;
+ return ((Tree)t).dupNode();
+ }
+
+ public Object create(Token payload) {
+ return new CommonTree(payload);
+ }
+
+ /** Tell me how to create a token for use with imaginary token nodes.
+ * For example, there is probably no input symbol associated with imaginary
+ * token DECL, but you need to create it as a payload or whatever for
+ * the DECL node as in ^(DECL type ID).
+ *
+ * If you care what the token payload objects' type is, you should
+ * override this method and any other createToken variant.
+ */
+ public Token createToken(int tokenType, String text) {
+ return new CommonToken(tokenType, text);
+ }
+
+ /** Tell me how to create a token for use with imaginary token nodes.
+ * For example, there is probably no input symbol associated with imaginary
+ * token DECL, but you need to create it as a payload or whatever for
+ * the DECL node as in ^(DECL type ID).
+ *
+ * This is a variant of createToken where the new token is derived from
+ * an actual real input token. Typically this is for converting '{'
+ * tokens to BLOCK etc... You'll see
+ *
+ * r : lc='{' ID+ '}' -> ^(BLOCK[$lc] ID+) ;
+ *
+ * If you care what the token payload objects' type is, you should
+ * override this method and any other createToken variant.
+ */
+ public Token createToken(Token fromToken) {
+ return new CommonToken(fromToken);
+ }
+
+ /** Track start/stop token for subtree root created for a rule.
+ * Only works with Tree nodes. For rules that match nothing,
+ * seems like this will yield start=i and stop=i-1 in a nil node.
+ * Might be useful info so I'll not force to be i..i.
+ */
+ public void setTokenBoundaries(Object t, Token startToken, Token stopToken) {
+ if ( t==null ) return;
+ int start = 0;
+ int stop = 0;
+ if ( startToken!=null ) start = startToken.getTokenIndex();
+ if ( stopToken!=null ) stop = stopToken.getTokenIndex();
+ ((Tree)t).setTokenStartIndex(start);
+ ((Tree)t).setTokenStopIndex(stop);
+ }
+
+ public int getTokenStartIndex(Object t) {
+ if ( t==null ) return -1;
+ return ((Tree)t).getTokenStartIndex();
+ }
+
+ public int getTokenStopIndex(Object t) {
+ if ( t==null ) return -1;
+ return ((Tree)t).getTokenStopIndex();
+ }
+
+ public String getText(Object t) {
+ if ( t==null ) return null;
+ return ((Tree)t).getText();
+ }
+
+ public int getType(Object t) {
+ if ( t==null ) return Token.INVALID_TOKEN_TYPE;
+ return ((Tree)t).getType();
+ }
+
+ /** What is the Token associated with this node? If
+ * you are not using CommonTree, then you must
+ * override this in your own adaptor.
+ */
+ public Token getToken(Object t) {
+ if ( t instanceof CommonTree ) {
+ return ((CommonTree)t).getToken();
+ }
+ return null; // no idea what to do
+ }
+
+ public Object getChild(Object t, int i) {
+ if ( t==null ) return null;
+ return ((Tree)t).getChild(i);
+ }
+
+ public int getChildCount(Object t) {
+ if ( t==null ) return 0;
+ return ((Tree)t).getChildCount();
+ }
+
+ public Object getParent(Object t) {
+ if ( t==null ) return null;
+ return ((Tree)t).getParent();
+ }
+
+ public void setParent(Object t, Object parent) {
+ if ( t!=null ) ((Tree)t).setParent((Tree)parent);
+ }
+
+ public int getChildIndex(Object t) {
+ if ( t==null ) return 0;
+ return ((Tree)t).getChildIndex();
+ }
+
+ public void setChildIndex(Object t, int index) {
+ if ( t!=null ) ((Tree)t).setChildIndex(index);
+ }
+
+ public void replaceChildren(Object parent, int startChildIndex, int stopChildIndex, Object t) {
+ if ( parent!=null ) {
+ ((Tree)parent).replaceChildren(startChildIndex, stopChildIndex, t);
+ }
+ }
+}
diff --git a/src/org/antlr/runtime/tree/CommonTreeNodeStream.java b/src/org/antlr/runtime/tree/CommonTreeNodeStream.java
new file mode 100755
index 0000000..05dbbdd
--- /dev/null
+++ b/src/org/antlr/runtime/tree/CommonTreeNodeStream.java
@@ -0,0 +1,171 @@
+/*
+ [The "BSD license"]
+ Copyright (c) 2005-2009 Terence Parr
+ All rights reserved.
+
+ Redistribution and use in source and binary forms, with or without
+ modification, are permitted provided that the following conditions
+ are met:
+ 1. Redistributions of source code must retain the above copyright
+ notice, this list of conditions and the following disclaimer.
+ 2. Redistributions in binary form must reproduce the above copyright
+ notice, this list of conditions and the following disclaimer in the
+ documentation and/or other materials provided with the distribution.
+ 3. The name of the author may not be used to endorse or promote products
+ derived from this software without specific prior written permission.
+
+ THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
+ IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
+ OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
+ IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
+ INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
+ NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
+ THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+package org.antlr.runtime.tree;
+
+import org.antlr.runtime.Token;
+import org.antlr.runtime.TokenStream;
+import org.antlr.runtime.misc.LookaheadStream;
+import org.antlr.runtime.misc.IntArray;
+
+import java.util.*;
+
+public class CommonTreeNodeStream extends LookaheadStream<Object> implements TreeNodeStream {
+ public static final int DEFAULT_INITIAL_BUFFER_SIZE = 100;
+ public static final int INITIAL_CALL_STACK_SIZE = 10;
+
+ /** Pull nodes from which tree? */
+ protected Object root;
+
+ /** If this tree (root) was created from a token stream, track it. */
+ protected TokenStream tokens;
+
+ /** What tree adaptor was used to build these trees */
+ TreeAdaptor adaptor;
+
+ /** The tree iterator we using */
+ protected TreeIterator it;
+
+ /** Stack of indexes used for push/pop calls */
+ protected IntArray calls;
+
+ /** Tree (nil A B C) trees like flat A B C streams */
+ protected boolean hasNilRoot = false;
+
+ /** Tracks tree depth. Level=0 means we're at root node level. */
+ protected int level = 0;
+
+ public CommonTreeNodeStream(Object tree) {
+ this(new CommonTreeAdaptor(), tree);
+ }
+
+ public CommonTreeNodeStream(TreeAdaptor adaptor, Object tree) {
+ this.root = tree;
+ this.adaptor = adaptor;
+ it = new TreeIterator(adaptor,root);
+ }
+
+ public void reset() {
+ super.reset();
+ it.reset();
+ hasNilRoot = false;
+ level = 0;
+ if ( calls != null ) calls.clear();
+ }
+
+ /** Pull elements from tree iterator. Track tree level 0..max_level.
+ * If nil rooted tree, don't give initial nil and DOWN nor final UP.
+ */
+ public Object nextElement() {
+ Object t = it.next();
+ //System.out.println("pulled "+adaptor.getType(t));
+ if ( t == it.up ) {
+ level--;
+ if ( level==0 && hasNilRoot ) return it.next(); // don't give last UP; get EOF
+ }
+ else if ( t == it.down ) level++;
+ if ( level==0 && adaptor.isNil(t) ) { // if nil root, scarf nil, DOWN
+ hasNilRoot = true;
+ t = it.next(); // t is now DOWN, so get first real node next
+ level++;
+ t = it.next();
+ }
+ return t;
+ }
+
+ public boolean isEOF(Object o) { return adaptor.getType(o) == Token.EOF; }
+
+ public void setUniqueNavigationNodes(boolean uniqueNavigationNodes) { }
+
+ public Object getTreeSource() { return root; }
+
+ public String getSourceName() { return getTokenStream().getSourceName(); }
+
+ public TokenStream getTokenStream() { return tokens; }
+
+ public void setTokenStream(TokenStream tokens) { this.tokens = tokens; }
+
+ public TreeAdaptor getTreeAdaptor() { return adaptor; }
+
+ public void setTreeAdaptor(TreeAdaptor adaptor) { this.adaptor = adaptor; }
+
+ public Object get(int i) {
+ throw new UnsupportedOperationException("Absolute node indexes are meaningless in an unbuffered stream");
+ }
+
+ public int LA(int i) { return adaptor.getType(LT(i)); }
+
+ /** Make stream jump to a new location, saving old location.
+ * Switch back with pop().
+ */
+ public void push(int index) {
+ if ( calls==null ) {
+ calls = new IntArray();
+ }
+ calls.push(p); // save current index
+ seek(index);
+ }
+
+ /** Seek back to previous index saved during last push() call.
+ * Return top of stack (return index).
+ */
+ public int pop() {
+ int ret = calls.pop();
+ seek(ret);
+ return ret;
+ }
+
+ // TREE REWRITE INTERFACE
+
+ public void replaceChildren(Object parent, int startChildIndex, int stopChildIndex, Object t) {
+ if ( parent!=null ) {
+ adaptor.replaceChildren(parent, startChildIndex, stopChildIndex, t);
+ }
+ }
+
+ public String toString(Object start, Object stop) {
+ // we'll have to walk from start to stop in tree; we're not keeping
+ // a complete node stream buffer
+ return "n/a";
+ }
+
+ /** For debugging; destructive: moves tree iterator to end. */
+ public String toTokenTypeString() {
+ reset();
+ StringBuffer buf = new StringBuffer();
+ Object o = LT(1);
+ int type = adaptor.getType(o);
+ while ( type!=Token.EOF ) {
+ buf.append(" ");
+ buf.append(type);
+ consume();
+ o = LT(1);
+ type = adaptor.getType(o);
+ }
+ return buf.toString();
+ }
+}
diff --git a/src/org/antlr/runtime/tree/ParseTree.java b/src/org/antlr/runtime/tree/ParseTree.java
new file mode 100755
index 0000000..5811c55
--- /dev/null
+++ b/src/org/antlr/runtime/tree/ParseTree.java
@@ -0,0 +1,119 @@
+/*
+ [The "BSD license"]
+ Copyright (c) 2005-2009 Terence Parr
+ All rights reserved.
+
+ Redistribution and use in source and binary forms, with or without
+ modification, are permitted provided that the following conditions
+ are met:
+ 1. Redistributions of source code must retain the above copyright
+ notice, this list of conditions and the following disclaimer.
+ 2. Redistributions in binary form must reproduce the above copyright
+ notice, this list of conditions and the following disclaimer in the
+ documentation and/or other materials provided with the distribution.
+ 3. The name of the author may not be used to endorse or promote products
+ derived from this software without specific prior written permission.
+
+ THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
+ IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
+ OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
+ IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
+ INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
+ NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
+ THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+package org.antlr.runtime.tree;
+
+import org.antlr.runtime.Token;
+
+import java.util.List;
+
+/** A record of the rules used to match a token sequence. The tokens
+ * end up as the leaves of this tree and rule nodes are the interior nodes.
+ * This really adds no functionality, it is just an alias for CommonTree
+ * that is more meaningful (specific) and holds a String to display for a node.
+ */
+public class ParseTree extends BaseTree {
+ public Object payload;
+ public List hiddenTokens;
+
+ public ParseTree(Object label) {
+ this.payload = label;
+ }
+
+ public Tree dupNode() {
+ return null;
+ }
+
+ public int getType() {
+ return 0;
+ }
+
+ public String getText() {
+ return toString();
+ }
+
+ public int getTokenStartIndex() {
+ return 0;
+ }
+
+ public void setTokenStartIndex(int index) {
+ }
+
+ public int getTokenStopIndex() {
+ return 0;
+ }
+
+ public void setTokenStopIndex(int index) {
+ }
+
+ public String toString() {
+ if ( payload instanceof Token ) {
+ Token t = (Token)payload;
+ if ( t.getType() == Token.EOF ) {
+ return "<EOF>";
+ }
+ return t.getText();
+ }
+ return payload.toString();
+ }
+
+ /** Emit a token and all hidden nodes before. EOF node holds all
+ * hidden tokens after last real token.
+ */
+ public String toStringWithHiddenTokens() {
+ StringBuffer buf = new StringBuffer();
+ if ( hiddenTokens!=null ) {
+ for (int i = 0; i < hiddenTokens.size(); i++) {
+ Token hidden = (Token) hiddenTokens.get(i);
+ buf.append(hidden.getText());
+ }
+ }
+ String nodeText = this.toString();
+ if ( !nodeText.equals("<EOF>") ) buf.append(nodeText);
+ return buf.toString();
+ }
+
+ /** Print out the leaves of this tree, which means printing original
+ * input back out.
+ */
+ public String toInputString() {
+ StringBuffer buf = new StringBuffer();
+ _toStringLeaves(buf);
+ return buf.toString();
+ }
+
+ public void _toStringLeaves(StringBuffer buf) {
+ if ( payload instanceof Token ) { // leaf node token?
+ buf.append(this.toStringWithHiddenTokens());
+ return;
+ }
+ for (int i = 0; children!=null && i < children.size(); i++) {
+ ParseTree t = (ParseTree)children.get(i);
+ t._toStringLeaves(buf);
+ }
+ }
+}
diff --git a/src/org/antlr/runtime/tree/RewriteCardinalityException.java b/src/org/antlr/runtime/tree/RewriteCardinalityException.java
new file mode 100755
index 0000000..7f909cd
--- /dev/null
+++ b/src/org/antlr/runtime/tree/RewriteCardinalityException.java
@@ -0,0 +1,47 @@
+/*
+ [The "BSD license"]
+ Copyright (c) 2005-2009 Terence Parr
+ All rights reserved.
+
+ Redistribution and use in source and binary forms, with or without
+ modification, are permitted provided that the following conditions
+ are met:
+ 1. Redistributions of source code must retain the above copyright
+ notice, this list of conditions and the following disclaimer.
+ 2. Redistributions in binary form must reproduce the above copyright
+ notice, this list of conditions and the following disclaimer in the
+ documentation and/or other materials provided with the distribution.
+ 3. The name of the author may not be used to endorse or promote products
+ derived from this software without specific prior written permission.
+
+ THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
+ IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
+ OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
+ IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
+ INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
+ NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
+ THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+package org.antlr.runtime.tree;
+
+/** Base class for all exceptions thrown during AST rewrite construction.
+ * This signifies a case where the cardinality of two or more elements
+ * in a subrule are different: (ID INT)+ where |ID|!=|INT|
+ */
+public class RewriteCardinalityException extends RuntimeException {
+ public String elementDescription;
+
+ public RewriteCardinalityException(String elementDescription) {
+ this.elementDescription = elementDescription;
+ }
+
+ public String getMessage() {
+ if ( elementDescription!=null ) {
+ return elementDescription;
+ }
+ return null;
+ }
+}
diff --git a/src/org/antlr/runtime/tree/RewriteEarlyExitException.java b/src/org/antlr/runtime/tree/RewriteEarlyExitException.java
new file mode 100755
index 0000000..9dbcc76
--- /dev/null
+++ b/src/org/antlr/runtime/tree/RewriteEarlyExitException.java
@@ -0,0 +1,39 @@
+/*
+ [The "BSD license"]
+ Copyright (c) 2005-2009 Terence Parr
+ All rights reserved.
+
+ Redistribution and use in source and binary forms, with or without
+ modification, are permitted provided that the following conditions
+ are met:
+ 1. Redistributions of source code must retain the above copyright
+ notice, this list of conditions and the following disclaimer.
+ 2. Redistributions in binary form must reproduce the above copyright
+ notice, this list of conditions and the following disclaimer in the
+ documentation and/or other materials provided with the distribution.
+ 3. The name of the author may not be used to endorse or promote products
+ derived from this software without specific prior written permission.
+
+ THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
+ IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
+ OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
+ IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
+ INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
+ NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
+ THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+package org.antlr.runtime.tree;
+
+/** No elements within a (...)+ in a rewrite rule */
+public class RewriteEarlyExitException extends RewriteCardinalityException {
+ public RewriteEarlyExitException() {
+ super(null);
+ }
+ public RewriteEarlyExitException(String elementDescription) {
+ super(elementDescription);
+ }
+
+}
diff --git a/src/org/antlr/runtime/tree/RewriteEmptyStreamException.java b/src/org/antlr/runtime/tree/RewriteEmptyStreamException.java
new file mode 100755
index 0000000..8f3a3fc
--- /dev/null
+++ b/src/org/antlr/runtime/tree/RewriteEmptyStreamException.java
@@ -0,0 +1,35 @@
+/*
+ [The "BSD license"]
+ Copyright (c) 2005-2009 Terence Parr
+ All rights reserved.
+
+ Redistribution and use in source and binary forms, with or without
+ modification, are permitted provided that the following conditions
+ are met:
+ 1. Redistributions of source code must retain the above copyright
+ notice, this list of conditions and the following disclaimer.
+ 2. Redistributions in binary form must reproduce the above copyright
+ notice, this list of conditions and the following disclaimer in the
+ documentation and/or other materials provided with the distribution.
+ 3. The name of the author may not be used to endorse or promote products
+ derived from this software without specific prior written permission.
+
+ THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
+ IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
+ OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
+ IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
+ INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
+ NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
+ THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+package org.antlr.runtime.tree;
+
+/** Ref to ID or expr but no tokens in ID stream or subtrees in expr stream */
+public class RewriteEmptyStreamException extends RewriteCardinalityException {
+ public RewriteEmptyStreamException(String elementDescription) {
+ super(elementDescription);
+ }
+}
diff --git a/src/org/antlr/runtime/tree/RewriteRuleElementStream.java b/src/org/antlr/runtime/tree/RewriteRuleElementStream.java
new file mode 100755
index 0000000..61f1860
--- /dev/null
+++ b/src/org/antlr/runtime/tree/RewriteRuleElementStream.java
@@ -0,0 +1,210 @@
+/*
+ [The "BSD license"]
+ Copyright (c) 2005-2009 Terence Parr
+ All rights reserved.
+
+ Redistribution and use in source and binary forms, with or without
+ modification, are permitted provided that the following conditions
+ are met:
+ 1. Redistributions of source code must retain the above copyright
+ notice, this list of conditions and the following disclaimer.
+ 2. Redistributions in binary form must reproduce the above copyright
+ notice, this list of conditions and the following disclaimer in the
+ documentation and/or other materials provided with the distribution.
+ 3. The name of the author may not be used to endorse or promote products
+ derived from this software without specific prior written permission.
+
+ THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
+ IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
+ OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
+ IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
+ INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
+ NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
+ THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+package org.antlr.runtime.tree;
+
+import java.util.ArrayList;
+import java.util.List;
+
+/** A generic list of elements tracked in an alternative to be used in
+ * a -> rewrite rule. We need to subclass to fill in the next() method,
+ * which returns either an AST node wrapped around a token payload or
+ * an existing subtree.
+ *
+ * Once you start next()ing, do not try to add more elements. It will
+ * break the cursor tracking I believe.
+ *
+ * @see org.antlr.runtime.tree.RewriteRuleSubtreeStream
+ * @see org.antlr.runtime.tree.RewriteRuleTokenStream
+ *
+ * TODO: add mechanism to detect/puke on modification after reading from stream
+ */
+public abstract class RewriteRuleElementStream {
+ /** Cursor 0..n-1. If singleElement!=null, cursor is 0 until you next(),
+ * which bumps it to 1 meaning no more elements.
+ */
+ protected int cursor = 0;
+
+ /** Track single elements w/o creating a list. Upon 2nd add, alloc list */
+ protected Object singleElement;
+
+ /** The list of tokens or subtrees we are tracking */
+ protected List elements;
+
+ /** Once a node / subtree has been used in a stream, it must be dup'd
+ * from then on. Streams are reset after subrules so that the streams
+ * can be reused in future subrules. So, reset must set a dirty bit.
+ * If dirty, then next() always returns a dup.
+ *
+ * I wanted to use "naughty bit" here, but couldn't think of a way
+ * to use "naughty".
+ *
+ * TODO: unused?
+ */
+ protected boolean dirty = false;
+
+ /** The element or stream description; usually has name of the token or
+ * rule reference that this list tracks. Can include rulename too, but
+ * the exception would track that info.
+ */
+ protected String elementDescription;
+ protected TreeAdaptor adaptor;
+
+ public RewriteRuleElementStream(TreeAdaptor adaptor, String elementDescription) {
+ this.elementDescription = elementDescription;
+ this.adaptor = adaptor;
+ }
+
+ /** Create a stream with one element */
+ public RewriteRuleElementStream(TreeAdaptor adaptor,
+ String elementDescription,
+ Object oneElement)
+ {
+ this(adaptor, elementDescription);
+ add(oneElement);
+ }
+
+ /** Create a stream, but feed off an existing list */
+ public RewriteRuleElementStream(TreeAdaptor adaptor,
+ String elementDescription,
+ List elements)
+ {
+ this(adaptor, elementDescription);
+ this.singleElement = null;
+ this.elements = elements;
+ }
+
+ /** Reset the condition of this stream so that it appears we have
+ * not consumed any of its elements. Elements themselves are untouched.
+ * Once we reset the stream, any future use will need duplicates. Set
+ * the dirty bit.
+ */
+ public void reset() {
+ cursor = 0;
+ dirty = true;
+ }
+
+ public void add(Object el) {
+ //System.out.println("add '"+elementDescription+"' is "+el);
+ if ( el==null ) {
+ return;
+ }
+ if ( elements!=null ) { // if in list, just add
+ elements.add(el);
+ return;
+ }
+ if ( singleElement == null ) { // no elements yet, track w/o list
+ singleElement = el;
+ return;
+ }
+ // adding 2nd element, move to list
+ elements = new ArrayList(5);
+ elements.add(singleElement);
+ singleElement = null;
+ elements.add(el);
+ }
+
+ /** Return the next element in the stream. If out of elements, throw
+ * an exception unless size()==1. If size is 1, then return elements[0].
+ * Return a duplicate node/subtree if stream is out of elements and
+ * size==1. If we've already used the element, dup (dirty bit set).
+ */
+ public Object nextTree() {
+ int n = size();
+ if ( dirty || (cursor>=n && n==1) ) {
+ // if out of elements and size is 1, dup
+ Object el = _next();
+ return dup(el);
+ }
+ // test size above then fetch
+ Object el = _next();
+ return el;
+ }
+
+ /** do the work of getting the next element, making sure that it's
+ * a tree node or subtree. Deal with the optimization of single-
+ * element list versus list of size > 1. Throw an exception
+ * if the stream is empty or we're out of elements and size>1.
+ * protected so you can override in a subclass if necessary.
+ */
+ protected Object _next() {
+ int n = size();
+ if ( n ==0 ) {
+ throw new RewriteEmptyStreamException(elementDescription);
+ }
+ if ( cursor>= n) { // out of elements?
+ if ( n ==1 ) { // if size is 1, it's ok; return and we'll dup
+ return toTree(singleElement);
+ }
+ // out of elements and size was not 1, so we can't dup
+ throw new RewriteCardinalityException(elementDescription);
+ }
+ // we have elements
+ if ( singleElement!=null ) {
+ cursor++; // move cursor even for single element list
+ return toTree(singleElement);
+ }
+ // must have more than one in list, pull from elements
+ Object o = toTree(elements.get(cursor));
+ cursor++;
+ return o;
+ }
+
+ /** When constructing trees, sometimes we need to dup a token or AST
+ * subtree. Dup'ing a token means just creating another AST node
+ * around it. For trees, you must call the adaptor.dupTree() unless
+ * the element is for a tree root; then it must be a node dup.
+ */
+ protected abstract Object dup(Object el);
+
+ /** Ensure stream emits trees; tokens must be converted to AST nodes.
+ * AST nodes can be passed through unmolested.
+ */
+ protected Object toTree(Object el) {
+ return el;
+ }
+
+ public boolean hasNext() {
+ return (singleElement != null && cursor < 1) ||
+ (elements!=null && cursor < elements.size());
+ }
+
+ public int size() {
+ int n = 0;
+ if ( singleElement != null ) {
+ n = 1;
+ }
+ if ( elements!=null ) {
+ return elements.size();
+ }
+ return n;
+ }
+
+ public String getDescription() {
+ return elementDescription;
+ }
+}
diff --git a/src/org/antlr/runtime/tree/RewriteRuleNodeStream.java b/src/org/antlr/runtime/tree/RewriteRuleNodeStream.java
new file mode 100755
index 0000000..713e9ff
--- /dev/null
+++ b/src/org/antlr/runtime/tree/RewriteRuleNodeStream.java
@@ -0,0 +1,70 @@
+/*
+ [The "BSD license"]
+ Copyright (c) 2005-2009 Terence Parr
+ All rights reserved.
+
+ Redistribution and use in source and binary forms, with or without
+ modification, are permitted provided that the following conditions
+ are met:
+ 1. Redistributions of source code must retain the above copyright
+ notice, this list of conditions and the following disclaimer.
+ 2. Redistributions in binary form must reproduce the above copyright
+ notice, this list of conditions and the following disclaimer in the
+ documentation and/or other materials provided with the distribution.
+ 3. The name of the author may not be used to endorse or promote products
+ derived from this software without specific prior written permission.
+
+ THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
+ IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
+ OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
+ IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
+ INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
+ NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
+ THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+package org.antlr.runtime.tree;
+
+import java.util.List;
+
+/** Queues up nodes matched on left side of -> in a tree parser. This is
+ * the analog of RewriteRuleTokenStream for normal parsers.
+ */
+public class RewriteRuleNodeStream extends RewriteRuleElementStream {
+
+ public RewriteRuleNodeStream(TreeAdaptor adaptor, String elementDescription) {
+ super(adaptor, elementDescription);
+ }
+
+ /** Create a stream with one element */
+ public RewriteRuleNodeStream(TreeAdaptor adaptor,
+ String elementDescription,
+ Object oneElement)
+ {
+ super(adaptor, elementDescription, oneElement);
+ }
+
+ /** Create a stream, but feed off an existing list */
+ public RewriteRuleNodeStream(TreeAdaptor adaptor,
+ String elementDescription,
+ List elements)
+ {
+ super(adaptor, elementDescription, elements);
+ }
+
+ public Object nextNode() {
+ return _next();
+ }
+
+ protected Object toTree(Object el) {
+ return adaptor.dupNode(el);
+ }
+
+ protected Object dup(Object el) {
+ // we dup every node, so don't have to worry about calling dup; short-
+ // circuited next() so it doesn't call.
+ throw new UnsupportedOperationException("dup can't be called for a node stream.");
+ }
+}
diff --git a/src/org/antlr/runtime/tree/RewriteRuleSubtreeStream.java b/src/org/antlr/runtime/tree/RewriteRuleSubtreeStream.java
new file mode 100755
index 0000000..5189f21
--- /dev/null
+++ b/src/org/antlr/runtime/tree/RewriteRuleSubtreeStream.java
@@ -0,0 +1,88 @@
+/*
+ [The "BSD license"]
+ Copyright (c) 2005-2009 Terence Parr
+ All rights reserved.
+
+ Redistribution and use in source and binary forms, with or without
+ modification, are permitted provided that the following conditions
+ are met:
+ 1. Redistributions of source code must retain the above copyright
+ notice, this list of conditions and the following disclaimer.
+ 2. Redistributions in binary form must reproduce the above copyright
+ notice, this list of conditions and the following disclaimer in the
+ documentation and/or other materials provided with the distribution.
+ 3. The name of the author may not be used to endorse or promote products
+ derived from this software without specific prior written permission.
+
+ THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
+ IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
+ OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
+ IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
+ INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
+ NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
+ THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+package org.antlr.runtime.tree;
+
+import java.util.List;
+
+public class RewriteRuleSubtreeStream extends RewriteRuleElementStream {
+
+ public RewriteRuleSubtreeStream(TreeAdaptor adaptor, String elementDescription) {
+ super(adaptor, elementDescription);
+ }
+
+ /** Create a stream with one element */
+ public RewriteRuleSubtreeStream(TreeAdaptor adaptor,
+ String elementDescription,
+ Object oneElement)
+ {
+ super(adaptor, elementDescription, oneElement);
+ }
+
+ /** Create a stream, but feed off an existing list */
+ public RewriteRuleSubtreeStream(TreeAdaptor adaptor,
+ String elementDescription,
+ List elements)
+ {
+ super(adaptor, elementDescription, elements);
+ }
+
+ /** Treat next element as a single node even if it's a subtree.
+ * This is used instead of next() when the result has to be a
+ * tree root node. Also prevents us from duplicating recently-added
+ * children; e.g., ^(type ID)+ adds ID to type and then 2nd iteration
+ * must dup the type node, but ID has been added.
+ *
+ * Referencing a rule result twice is ok; dup entire tree as
+ * we can't be adding trees as root; e.g., expr expr.
+ *
+ * Hideous code duplication here with super.next(). Can't think of
+ * a proper way to refactor. This needs to always call dup node
+ * and super.next() doesn't know which to call: dup node or dup tree.
+ */
+ public Object nextNode() {
+ //System.out.println("nextNode: elements="+elements+", singleElement="+((Tree)singleElement).toStringTree());
+ int n = size();
+ if ( dirty || (cursor>=n && n==1) ) {
+ // if out of elements and size is 1, dup (at most a single node
+ // since this is for making root nodes).
+ Object el = _next();
+ return adaptor.dupNode(el);
+ }
+ // test size above then fetch
+ Object tree = _next();
+ while (adaptor.isNil(tree) && adaptor.getChildCount(tree) == 1)
+ tree = adaptor.getChild(tree, 0);
+ //System.out.println("_next="+((Tree)tree).toStringTree());
+ Object el = adaptor.dupNode(tree); // dup just the root (want node here)
+ return el;
+ }
+
+ protected Object dup(Object el) {
+ return adaptor.dupTree(el);
+ }
+}
\ No newline at end of file
diff --git a/src/org/antlr/runtime/tree/RewriteRuleTokenStream.java b/src/org/antlr/runtime/tree/RewriteRuleTokenStream.java
new file mode 100755
index 0000000..4cd7b08
--- /dev/null
+++ b/src/org/antlr/runtime/tree/RewriteRuleTokenStream.java
@@ -0,0 +1,76 @@
+/*
+ [The "BSD license"]
+ Copyright (c) 2005-2009 Terence Parr
+ All rights reserved.
+
+ Redistribution and use in source and binary forms, with or without
+ modification, are permitted provided that the following conditions
+ are met:
+ 1. Redistributions of source code must retain the above copyright
+ notice, this list of conditions and the following disclaimer.
+ 2. Redistributions in binary form must reproduce the above copyright
+ notice, this list of conditions and the following disclaimer in the
+ documentation and/or other materials provided with the distribution.
+ 3. The name of the author may not be used to endorse or promote products
+ derived from this software without specific prior written permission.
+
+ THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
+ IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
+ OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
+ IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
+ INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
+ NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
+ THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+package org.antlr.runtime.tree;
+
+import org.antlr.runtime.Token;
+
+import java.util.List;
+
+public class RewriteRuleTokenStream extends RewriteRuleElementStream {
+
+ public RewriteRuleTokenStream(TreeAdaptor adaptor, String elementDescription) {
+ super(adaptor, elementDescription);
+ }
+
+ /** Create a stream with one element */
+ public RewriteRuleTokenStream(TreeAdaptor adaptor,
+ String elementDescription,
+ Object oneElement)
+ {
+ super(adaptor, elementDescription, oneElement);
+ }
+
+ /** Create a stream, but feed off an existing list */
+ public RewriteRuleTokenStream(TreeAdaptor adaptor,
+ String elementDescription,
+ List elements)
+ {
+ super(adaptor, elementDescription, elements);
+ }
+
+ /** Get next token from stream and make a node for it */
+ public Object nextNode() {
+ Token t = (Token)_next();
+ return adaptor.create(t);
+ }
+
+ public Token nextToken() {
+ return (Token)_next();
+ }
+
+ /** Don't convert to a tree unless they explicitly call nextTree.
+ * This way we can do hetero tree nodes in rewrite.
+ */
+ protected Object toTree(Object el) {
+ return el;
+ }
+
+ protected Object dup(Object el) {
+ throw new UnsupportedOperationException("dup can't be called for a token stream.");
+ }
+}
diff --git a/src/org/antlr/runtime/tree/Tree.java b/src/org/antlr/runtime/tree/Tree.java
new file mode 100755
index 0000000..7875be3
--- /dev/null
+++ b/src/org/antlr/runtime/tree/Tree.java
@@ -0,0 +1,127 @@
+/*
+ [The "BSD license"]
+ Copyright (c) 2005-2009 Terence Parr
+ All rights reserved.
+
+ Redistribution and use in source and binary forms, with or without
+ modification, are permitted provided that the following conditions
+ are met:
+ 1. Redistributions of source code must retain the above copyright
+ notice, this list of conditions and the following disclaimer.
+ 2. Redistributions in binary form must reproduce the above copyright
+ notice, this list of conditions and the following disclaimer in the
+ documentation and/or other materials provided with the distribution.
+ 3. The name of the author may not be used to endorse or promote products
+ derived from this software without specific prior written permission.
+
+ THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
+ IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
+ OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
+ IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
+ INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
+ NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
+ THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+package org.antlr.runtime.tree;
+
+import org.antlr.runtime.Token;
+
+import java.util.List;
+
+/** What does a tree look like? ANTLR has a number of support classes
+ * such as CommonTreeNodeStream that work on these kinds of trees. You
+ * don't have to make your trees implement this interface, but if you do,
+ * you'll be able to use more support code.
+ *
+ * NOTE: When constructing trees, ANTLR can build any kind of tree; it can
+ * even use Token objects as trees if you add a child list to your tokens.
+ *
+ * This is a tree node without any payload; just navigation and factory stuff.
+ */
+public interface Tree {
+ public static final Tree INVALID_NODE = new CommonTree(Token.INVALID_TOKEN);
+
+ Tree getChild(int i);
+
+ int getChildCount();
+
+ // Tree tracks parent and child index now > 3.0
+
+ public Tree getParent();
+
+ public void setParent(Tree t);
+
+ /** Is there is a node above with token type ttype? */
+ public boolean hasAncestor(int ttype);
+
+ /** Walk upwards and get first ancestor with this token type. */
+ public Tree getAncestor(int ttype);
+
+ /** Return a list of all ancestors of this node. The first node of
+ * list is the root and the last is the parent of this node.
+ */
+ public List getAncestors();
+
+ /** This node is what child index? 0..n-1 */
+ public int getChildIndex();
+
+ public void setChildIndex(int index);
+
+ /** Set the parent and child index values for all children */
+ public void freshenParentAndChildIndexes();
+
+ /** Add t as a child to this node. If t is null, do nothing. If t
+ * is nil, add all children of t to this' children.
+ */
+ void addChild(Tree t);
+
+ /** Set ith child (0..n-1) to t; t must be non-null and non-nil node */
+ public void setChild(int i, Tree t);
+
+ public Object deleteChild(int i);
+
+ /** Delete children from start to stop and replace with t even if t is
+ * a list (nil-root tree). num of children can increase or decrease.
+ * For huge child lists, inserting children can force walking rest of
+ * children to set their childindex; could be slow.
+ */
+ public void replaceChildren(int startChildIndex, int stopChildIndex, Object t);
+
+ /** Indicates the node is a nil node but may still have children, meaning
+ * the tree is a flat list.
+ */
+ boolean isNil();
+
+ /** What is the smallest token index (indexing from 0) for this node
+ * and its children?
+ */
+ int getTokenStartIndex();
+
+ void setTokenStartIndex(int index);
+
+ /** What is the largest token index (indexing from 0) for this node
+ * and its children?
+ */
+ int getTokenStopIndex();
+
+ void setTokenStopIndex(int index);
+
+ Tree dupNode();
+
+ /** Return a token type; needed for tree parsing */
+ int getType();
+
+ String getText();
+
+ /** In case we don't have a token payload, what is the line for errors? */
+ int getLine();
+
+ int getCharPositionInLine();
+
+ String toStringTree();
+
+ String toString();
+}
diff --git a/src/org/antlr/runtime/tree/TreeAdaptor.java b/src/org/antlr/runtime/tree/TreeAdaptor.java
new file mode 100755
index 0000000..8e889f8
--- /dev/null
+++ b/src/org/antlr/runtime/tree/TreeAdaptor.java
@@ -0,0 +1,263 @@
+/*
+ [The "BSD license"]
+ Copyright (c) 2005-2009 Terence Parr
+ All rights reserved.
+
+ Redistribution and use in source and binary forms, with or without
+ modification, are permitted provided that the following conditions
+ are met:
+ 1. Redistributions of source code must retain the above copyright
+ notice, this list of conditions and the following disclaimer.
+ 2. Redistributions in binary form must reproduce the above copyright
+ notice, this list of conditions and the following disclaimer in the
+ documentation and/or other materials provided with the distribution.
+ 3. The name of the author may not be used to endorse or promote products
+ derived from this software without specific prior written permission.
+
+ THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
+ IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
+ OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
+ IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
+ INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
+ NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
+ THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+package org.antlr.runtime.tree;
+
+import org.antlr.runtime.Token;
+import org.antlr.runtime.TokenStream;
+import org.antlr.runtime.RecognitionException;
+
+/** How to create and navigate trees. Rather than have a separate factory
+ * and adaptor, I've merged them. Makes sense to encapsulate.
+ *
+ * This takes the place of the tree construction code generated in the
+ * generated code in 2.x and the ASTFactory.
+ *
+ * I do not need to know the type of a tree at all so they are all
+ * generic Objects. This may increase the amount of typecasting needed. :(
+ */
+public interface TreeAdaptor {
+ // C o n s t r u c t i o n
+
+ /** Create a tree node from Token object; for CommonTree type trees,
+ * then the token just becomes the payload. This is the most
+ * common create call.
+ *
+ * Override if you want another kind of node to be built.
+ */
+ public Object create(Token payload);
+
+ /** Duplicate a single tree node.
+ * Override if you want another kind of node to be built.
+ */
+ public Object dupNode(Object treeNode);
+
+ /** Duplicate tree recursively, using dupNode() for each node */
+ public Object dupTree(Object tree);
+
+ /** Return a nil node (an empty but non-null node) that can hold
+ * a list of element as the children. If you want a flat tree (a list)
+ * use "t=adaptor.nil(); t.addChild(x); t.addChild(y);"
+ */
+ public Object nil();
+
+ /** Return a tree node representing an error. This node records the
+ * tokens consumed during error recovery. The start token indicates the
+ * input symbol at which the error was detected. The stop token indicates
+ * the last symbol consumed during recovery.
+ *
+ * You must specify the input stream so that the erroneous text can
+ * be packaged up in the error node. The exception could be useful
+ * to some applications; default implementation stores ptr to it in
+ * the CommonErrorNode.
+ *
+ * This only makes sense during token parsing, not tree parsing.
+ * Tree parsing should happen only when parsing and tree construction
+ * succeed.
+ */
+ public Object errorNode(TokenStream input, Token start, Token stop, RecognitionException e);
+
+ /** Is tree considered a nil node used to make lists of child nodes? */
+ public boolean isNil(Object tree);
+
+ /** Add a child to the tree t. If child is a flat tree (a list), make all
+ * in list children of t. Warning: if t has no children, but child does
+ * and child isNil then you can decide it is ok to move children to t via
+ * t.children = child.children; i.e., without copying the array. Just
+ * make sure that this is consistent with have the user will build
+ * ASTs. Do nothing if t or child is null.
+ */
+ public void addChild(Object t, Object child);
+
+ /** If oldRoot is a nil root, just copy or move the children to newRoot.
+ * If not a nil root, make oldRoot a child of newRoot.
+ *
+ * old=^(nil a b c), new=r yields ^(r a b c)
+ * old=^(a b c), new=r yields ^(r ^(a b c))
+ *
+ * If newRoot is a nil-rooted single child tree, use the single
+ * child as the new root node.
+ *
+ * old=^(nil a b c), new=^(nil r) yields ^(r a b c)
+ * old=^(a b c), new=^(nil r) yields ^(r ^(a b c))
+ *
+ * If oldRoot was null, it's ok, just return newRoot (even if isNil).
+ *
+ * old=null, new=r yields r
+ * old=null, new=^(nil r) yields ^(nil r)
+ *
+ * Return newRoot. Throw an exception if newRoot is not a
+ * simple node or nil root with a single child node--it must be a root
+ * node. If newRoot is ^(nil x) return x as newRoot.
+ *
+ * Be advised that it's ok for newRoot to point at oldRoot's
+ * children; i.e., you don't have to copy the list. We are
+ * constructing these nodes so we should have this control for
+ * efficiency.
+ */
+ public Object becomeRoot(Object newRoot, Object oldRoot);
+
+ /** Given the root of the subtree created for this rule, post process
+ * it to do any simplifications or whatever you want. A required
+ * behavior is to convert ^(nil singleSubtree) to singleSubtree
+ * as the setting of start/stop indexes relies on a single non-nil root
+ * for non-flat trees.
+ *
+ * Flat trees such as for lists like "idlist : ID+ ;" are left alone
+ * unless there is only one ID. For a list, the start/stop indexes
+ * are set in the nil node.
+ *
+ * This method is executed after all rule tree construction and right
+ * before setTokenBoundaries().
+ */
+ public Object rulePostProcessing(Object root);
+
+ /** For identifying trees.
+ *
+ * How to identify nodes so we can say "add node to a prior node"?
+ * Even becomeRoot is an issue. Use System.identityHashCode(node)
+ * usually.
+ */
+ public int getUniqueID(Object node);
+
+
+ // R e w r i t e R u l e s
+
+ /** Create a node for newRoot make it the root of oldRoot.
+ * If oldRoot is a nil root, just copy or move the children to newRoot.
+ * If not a nil root, make oldRoot a child of newRoot.
+ *
+ * Return node created for newRoot.
+ *
+ * Be advised: when debugging ASTs, the DebugTreeAdaptor manually
+ * calls create(Token child) and then plain becomeRoot(node, node)
+ * because it needs to trap calls to create, but it can't since it delegates
+ * to not inherits from the TreeAdaptor.
+ */
+ public Object becomeRoot(Token newRoot, Object oldRoot);
+
+ /** Create a new node derived from a token, with a new token type.
+ * This is invoked from an imaginary node ref on right side of a
+ * rewrite rule as IMAG[$tokenLabel].
+ *
+ * This should invoke createToken(Token).
+ */
+ public Object create(int tokenType, Token fromToken);
+
+ /** Same as create(tokenType,fromToken) except set the text too.
+ * This is invoked from an imaginary node ref on right side of a
+ * rewrite rule as IMAG[$tokenLabel, "IMAG"].
+ *
+ * This should invoke createToken(Token).
+ */
+ public Object create(int tokenType, Token fromToken, String text);
+
+ /** Create a new node derived from a token, with a new token type.
+ * This is invoked from an imaginary node ref on right side of a
+ * rewrite rule as IMAG["IMAG"].
+ *
+ * This should invoke createToken(int,String).
+ */
+ public Object create(int tokenType, String text);
+
+
+ // C o n t e n t
+
+ /** For tree parsing, I need to know the token type of a node */
+ public int getType(Object t);
+
+ /** Node constructors can set the type of a node */
+ public void setType(Object t, int type);
+
+ public String getText(Object t);
+
+ /** Node constructors can set the text of a node */
+ public void setText(Object t, String text);
+
+ /** Return the token object from which this node was created.
+ * Currently used only for printing an error message.
+ * The error display routine in BaseRecognizer needs to
+ * display where the input the error occurred. If your
+ * tree of limitation does not store information that can
+ * lead you to the token, you can create a token filled with
+ * the appropriate information and pass that back. See
+ * BaseRecognizer.getErrorMessage().
+ */
+ public Token getToken(Object t);
+
+ /** Where are the bounds in the input token stream for this node and
+ * all children? Each rule that creates AST nodes will call this
+ * method right before returning. Flat trees (i.e., lists) will
+ * still usually have a nil root node just to hold the children list.
+ * That node would contain the start/stop indexes then.
+ */
+ public void setTokenBoundaries(Object t, Token startToken, Token stopToken);
+
+ /** Get the token start index for this subtree; return -1 if no such index */
+ public int getTokenStartIndex(Object t);
+
+ /** Get the token stop index for this subtree; return -1 if no such index */
+ public int getTokenStopIndex(Object t);
+
+
+ // N a v i g a t i o n / T r e e P a r s i n g
+
+ /** Get a child 0..n-1 node */
+ public Object getChild(Object t, int i);
+
+ /** Set ith child (0..n-1) to t; t must be non-null and non-nil node */
+ public void setChild(Object t, int i, Object child);
+
+ /** Remove ith child and shift children down from right. */
+ public Object deleteChild(Object t, int i);
+
+ /** How many children? If 0, then this is a leaf node */
+ public int getChildCount(Object t);
+
+ /** Who is the parent node of this node; if null, implies node is root.
+ * If your node type doesn't handle this, it's ok but the tree rewrites
+ * in tree parsers need this functionality.
+ */
+ public Object getParent(Object t);
+ public void setParent(Object t, Object parent);
+
+ /** What index is this node in the child list? Range: 0..n-1
+ * If your node type doesn't handle this, it's ok but the tree rewrites
+ * in tree parsers need this functionality.
+ */
+ public int getChildIndex(Object t);
+ public void setChildIndex(Object t, int index);
+
+ /** Replace from start to stop child index of parent with t, which might
+ * be a list. Number of children may be different
+ * after this call.
+ *
+ * If parent is null, don't do anything; must be at root of overall tree.
+ * Can't replace whatever points to the parent externally. Do nothing.
+ */
+ public void replaceChildren(Object parent, int startChildIndex, int stopChildIndex, Object t);
+}
diff --git a/src/org/antlr/runtime/tree/TreeFilter.java b/src/org/antlr/runtime/tree/TreeFilter.java
new file mode 100755
index 0000000..b6a7e05
--- /dev/null
+++ b/src/org/antlr/runtime/tree/TreeFilter.java
@@ -0,0 +1,135 @@
+/*
+ [The "BSD license"]
+ Copyright (c) 2005-2009 Terence Parr
+ All rights reserved.
+
+ Redistribution and use in source and binary forms, with or without
+ modification, are permitted provided that the following conditions
+ are met:
+ 1. Redistributions of source code must retain the above copyright
+ notice, this list of conditions and the following disclaimer.
+ 2. Redistributions in binary form must reproduce the above copyright
+ notice, this list of conditions and the following disclaimer in the
+ documentation and/or other materials provided with the distribution.
+ 3. The name of the author may not be used to endorse or promote products
+ derived from this software without specific prior written permission.
+
+ THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
+ IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
+ OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
+ IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
+ INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
+ NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
+ THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+package org.antlr.runtime.tree;
+
+import org.antlr.runtime.RecognizerSharedState;
+import org.antlr.runtime.RecognitionException;
+import org.antlr.runtime.TokenStream;
+
+/**
+ Cut-n-paste from material I'm not using in the book anymore (edit later
+ to make sense):
+
+ Now, how are we going to test these tree patterns against every
+subtree in our original tree? In what order should we visit nodes?
+For this application, it turns out we need a simple ``apply once''
+rule application strategy and a ``down then up'' tree traversal
+strategy. Let's look at rule application first.
+
+As we visit each node, we need to see if any of our patterns match. If
+a pattern matches, we execute the associated tree rewrite and move on
+to the next node. In other words, we only look for a single rule
+application opportunity (we'll see below that we sometimes need to
+repeatedly apply rules). The following method applies a rule in a @cl
+TreeParser (derived from a tree grammar) to a tree:
+
+here is where weReferenced code/walking/patterns/TreePatternMatcher.java
+
+It uses reflection to lookup the appropriate rule within the generated
+tree parser class (@cl Simplify in this case). Most of the time, the
+rule will not match the tree. To avoid issuing syntax errors and
+attempting error recovery, it bumps up the backtracking level. Upon
+failure, the invoked rule immediately returns. If you don't plan on
+using this technique in your own ANTLR-based application, don't sweat
+the details. This method boils down to ``call a rule to match a tree,
+executing any embedded actions and rewrite rules.''
+
+At this point, we know how to define tree grammar rules and how to
+apply them to a particular subtree. The final piece of the tree
+pattern matcher is the actual tree traversal. We have to get the
+correct node visitation order. In particular, we need to perform the
+scalar-vector multiply transformation on the way down (preorder) and
+we need to reduce multiply-by-zero subtrees on the way up (postorder).
+
+To implement a top-down visitor, we do a depth first walk of the tree,
+executing an action in the preorder position. To get a bottom-up
+visitor, we execute an action in the postorder position. ANTLR
+provides a standard @cl TreeVisitor class with a depth first search @v
+visit method. That method executes either a @m pre or @m post method
+or both. In our case, we need to call @m applyOnce in both. On the way
+down, we'll look for @r vmult patterns. On the way up,
+we'll look for @r mult0 patterns.
+ */
+public class TreeFilter extends TreeParser {
+ public interface fptr {
+ public void rule() throws RecognitionException;
+ }
+
+ protected TokenStream originalTokenStream;
+ protected TreeAdaptor originalAdaptor;
+
+ public TreeFilter(TreeNodeStream input) {
+ this(input, new RecognizerSharedState());
+ }
+ public TreeFilter(TreeNodeStream input, RecognizerSharedState state) {
+ super(input, state);
+ originalAdaptor = input.getTreeAdaptor();
+ originalTokenStream = input.getTokenStream();
+ }
+
+ public void applyOnce(Object t, fptr whichRule) {
+ if ( t==null ) return;
+ try {
+ // share TreeParser object but not parsing-related state
+ state = new RecognizerSharedState();
+ input = new CommonTreeNodeStream(originalAdaptor, t);
+ ((CommonTreeNodeStream)input).setTokenStream(originalTokenStream);
+ setBacktrackingLevel(1);
+ whichRule.rule();
+ setBacktrackingLevel(0);
+ }
+ catch (RecognitionException e) { ; }
+ }
+
+ public void downup(Object t) {
+ TreeVisitor v = new TreeVisitor(new CommonTreeAdaptor());
+ TreeVisitorAction actions = new TreeVisitorAction() {
+ public Object pre(Object t) { applyOnce(t, topdown_fptr); return t; }
+ public Object post(Object t) { applyOnce(t, bottomup_fptr); return t; }
+ };
+ v.visit(t, actions);
+ }
+
+ fptr topdown_fptr = new fptr() {
+ public void rule() throws RecognitionException {
+ topdown();
+ }
+ };
+
+ fptr bottomup_fptr = new fptr() {
+ public void rule() throws RecognitionException {
+ bottomup();
+ }
+ };
+
+ // methods the downup strategy uses to do the up and down rules.
+ // to override, just define tree grammar rule topdown and turn on
+ // filter=true.
+ public void topdown() throws RecognitionException {;}
+ public void bottomup() throws RecognitionException {;}
+}
diff --git a/src/org/antlr/runtime/tree/TreeIterator.java b/src/org/antlr/runtime/tree/TreeIterator.java
new file mode 100755
index 0000000..43ead6d
--- /dev/null
+++ b/src/org/antlr/runtime/tree/TreeIterator.java
@@ -0,0 +1,132 @@
+/*
+ [The "BSD license"]
+ Copyright (c) 2005-2009 Terence Parr
+ All rights reserved.
+
+ Redistribution and use in source and binary forms, with or without
+ modification, are permitted provided that the following conditions
+ are met:
+ 1. Redistributions of source code must retain the above copyright
+ notice, this list of conditions and the following disclaimer.
+ 2. Redistributions in binary form must reproduce the above copyright
+ notice, this list of conditions and the following disclaimer in the
+ documentation and/or other materials provided with the distribution.
+ 3. The name of the author may not be used to endorse or promote products
+ derived from this software without specific prior written permission.
+
+ THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
+ IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
+ OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
+ IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
+ INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
+ NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
+ THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+package org.antlr.runtime.tree;
+
+import org.antlr.runtime.Token;
+import org.antlr.runtime.CommonToken;
+import org.antlr.runtime.misc.FastQueue;
+
+import java.util.Iterator;
+
+/** Return a node stream from a doubly-linked tree whose nodes
+ * know what child index they are. No remove() is supported.
+ *
+ * Emit navigation nodes (DOWN, UP, and EOF) to let show tree structure.
+ */
+public class TreeIterator implements Iterator {
+ protected TreeAdaptor adaptor;
+ protected Object root;
+ protected Object tree;
+ protected boolean firstTime = true;
+
+ // navigation nodes to return during walk and at end
+ public Object up;
+ public Object down;
+ public Object eof;
+
+ /** If we emit UP/DOWN nodes, we need to spit out multiple nodes per
+ * next() call.
+ */
+ protected FastQueue nodes;
+
+ public TreeIterator(Object tree) {
+ this(new CommonTreeAdaptor(),tree);
+ }
+
+ public TreeIterator(TreeAdaptor adaptor, Object tree) {
+ this.adaptor = adaptor;
+ this.tree = tree;
+ this.root = tree;
+ nodes = new FastQueue();
+ down = adaptor.create(Token.DOWN, "DOWN");
+ up = adaptor.create(Token.UP, "UP");
+ eof = adaptor.create(Token.EOF, "EOF");
+ }
+
+ public void reset() {
+ firstTime = true;
+ tree = root;
+ nodes.clear();
+ }
+
+ public boolean hasNext() {
+ if ( firstTime ) return root!=null;
+ if ( nodes!=null && nodes.size()>0 ) return true;
+ if ( tree==null ) return false;
+ if ( adaptor.getChildCount(tree)>0 ) return true;
+ return adaptor.getParent(tree)!=null; // back at root?
+ }
+
+ public Object next() {
+ if ( firstTime ) { // initial condition
+ firstTime = false;
+ if ( adaptor.getChildCount(tree)==0 ) { // single node tree (special)
+ nodes.add(eof);
+ return tree;
+ }
+ return tree;
+ }
+ // if any queued up, use those first
+ if ( nodes!=null && nodes.size()>0 ) return nodes.remove();
+
+ // no nodes left?
+ if ( tree==null ) return eof;
+
+ // next node will be child 0 if any children
+ if ( adaptor.getChildCount(tree)>0 ) {
+ tree = adaptor.getChild(tree, 0);
+ nodes.add(tree); // real node is next after DOWN
+ return down;
+ }
+ // if no children, look for next sibling of tree or ancestor
+ Object parent = adaptor.getParent(tree);
+ // while we're out of siblings, keep popping back up towards root
+ while ( parent!=null &&
+ adaptor.getChildIndex(tree)+1 >= adaptor.getChildCount(parent) )
+ {
+ nodes.add(up); // we're moving back up
+ tree = parent;
+ parent = adaptor.getParent(tree);
+ }
+ // no nodes left?
+ if ( parent==null ) {
+ tree = null; // back at root? nothing left then
+ nodes.add(eof); // add to queue, might have UP nodes in there
+ return nodes.remove();
+ }
+
+ // must have found a node with an unvisited sibling
+ // move to it and return it
+ int nextSiblingIndex = adaptor.getChildIndex(tree) + 1;
+ tree = adaptor.getChild(parent, nextSiblingIndex);
+ nodes.add(tree); // add to queue, might have UP nodes in there
+ return nodes.remove();
+ }
+
+ public void remove() { throw new UnsupportedOperationException(); }
+}
diff --git a/src/org/antlr/runtime/tree/TreeNodeStream.java b/src/org/antlr/runtime/tree/TreeNodeStream.java
new file mode 100755
index 0000000..df0ad34
--- /dev/null
+++ b/src/org/antlr/runtime/tree/TreeNodeStream.java
@@ -0,0 +1,106 @@
+/*
+ [The "BSD license"]
+ Copyright (c) 2005-2009 Terence Parr
+ All rights reserved.
+
+ Redistribution and use in source and binary forms, with or without
+ modification, are permitted provided that the following conditions
+ are met:
+ 1. Redistributions of source code must retain the above copyright
+ notice, this list of conditions and the following disclaimer.
+ 2. Redistributions in binary form must reproduce the above copyright
+ notice, this list of conditions and the following disclaimer in the
+ documentation and/or other materials provided with the distribution.
+ 3. The name of the author may not be used to endorse or promote products
+ derived from this software without specific prior written permission.
+
+ THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
+ IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
+ OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
+ IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
+ INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
+ NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
+ THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+package org.antlr.runtime.tree;
+
+import org.antlr.runtime.IntStream;
+import org.antlr.runtime.TokenStream;
+
+/** A stream of tree nodes, accessing nodes from a tree of some kind */
+public interface TreeNodeStream extends IntStream {
+ /** Get a tree node at an absolute index i; 0..n-1.
+ * If you don't want to buffer up nodes, then this method makes no
+ * sense for you.
+ */
+ public Object get(int i);
+
+ /** Get tree node at current input pointer + i ahead where i=1 is next node.
+ * i<0 indicates nodes in the past. So LT(-1) is previous node, but
+ * implementations are not required to provide results for k < -1.
+ * LT(0) is undefined. For i>=n, return null.
+ * Return null for LT(0) and any index that results in an absolute address
+ * that is negative.
+ *
+ * This is analogus to the LT() method of the TokenStream, but this
+ * returns a tree node instead of a token. Makes code gen identical
+ * for both parser and tree grammars. :)
+ */
+ public Object LT(int k);
+
+ /** Where is this stream pulling nodes from? This is not the name, but
+ * the object that provides node objects.
+ */
+ public Object getTreeSource();
+
+ /** If the tree associated with this stream was created from a TokenStream,
+ * you can specify it here. Used to do rule $text attribute in tree
+ * parser. Optional unless you use tree parser rule text attribute
+ * or output=template and rewrite=true options.
+ */
+ public TokenStream getTokenStream();
+
+ /** What adaptor can tell me how to interpret/navigate nodes and
+ * trees. E.g., get text of a node.
+ */
+ public TreeAdaptor getTreeAdaptor();
+
+ /** As we flatten the tree, we use UP, DOWN nodes to represent
+ * the tree structure. When debugging we need unique nodes
+ * so we have to instantiate new ones. When doing normal tree
+ * parsing, it's slow and a waste of memory to create unique
+ * navigation nodes. Default should be false;
+ */
+ public void setUniqueNavigationNodes(boolean uniqueNavigationNodes);
+
+ /** Reset the tree node stream in such a way that it acts like
+ * a freshly constructed stream.
+ */
+ public void reset();
+
+ /** Return the text of all nodes from start to stop, inclusive.
+ * If the stream does not buffer all the nodes then it can still
+ * walk recursively from start until stop. You can always return
+ * null or "" too, but users should not access $ruleLabel.text in
+ * an action of course in that case.
+ */
+ public String toString(Object start, Object stop);
+
+
+ // REWRITING TREES (used by tree parser)
+
+ /** Replace from start to stop child index of parent with t, which might
+ * be a list. Number of children may be different
+ * after this call. The stream is notified because it is walking the
+ * tree and might need to know you are monkeying with the underlying
+ * tree. Also, it might be able to modify the node stream to avoid
+ * restreaming for future phases.
+ *
+ * If parent is null, don't do anything; must be at root of overall tree.
+ * Can't replace whatever points to the parent externally. Do nothing.
+ */
+ public void replaceChildren(Object parent, int startChildIndex, int stopChildIndex, Object t);
+}
diff --git a/src/org/antlr/runtime/tree/TreeParser.java b/src/org/antlr/runtime/tree/TreeParser.java
new file mode 100755
index 0000000..e568bc9
--- /dev/null
+++ b/src/org/antlr/runtime/tree/TreeParser.java
@@ -0,0 +1,169 @@
+/*
+ [The "BSD license"]
+ Copyright (c) 2005-2009 Terence Parr
+ All rights reserved.
+
+ Redistribution and use in source and binary forms, with or without
+ modification, are permitted provided that the following conditions
+ are met:
+ 1. Redistributions of source code must retain the above copyright
+ notice, this list of conditions and the following disclaimer.
+ 2. Redistributions in binary form must reproduce the above copyright
+ notice, this list of conditions and the following disclaimer in the
+ documentation and/or other materials provided with the distribution.
+ 3. The name of the author may not be used to endorse or promote products
+ derived from this software without specific prior written permission.
+
+ THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
+ IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
+ OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
+ IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
+ INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
+ NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
+ THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+package org.antlr.runtime.tree;
+
+import org.antlr.runtime.*;
+
+import java.util.regex.Pattern;
+import java.util.regex.Matcher;
+
+/** A parser for a stream of tree nodes. "tree grammars" result in a subclass
+ * of this. All the error reporting and recovery is shared with Parser via
+ * the BaseRecognizer superclass.
+*/
+public class TreeParser extends BaseRecognizer {
+ public static final int DOWN = Token.DOWN;
+ public static final int UP = Token.UP;
+
+ // precompiled regex used by inContext
+ static String dotdot = ".*[^.]\\.\\.[^.].*";
+ static String doubleEtc = ".*\\.\\.\\.\\s+\\.\\.\\..*";
+ static Pattern dotdotPattern = Pattern.compile(dotdot);
+ static Pattern doubleEtcPattern = Pattern.compile(doubleEtc);
+
+ protected TreeNodeStream input;
+
+ public TreeParser(TreeNodeStream input) {
+ super(); // highlight that we go to super to set state object
+ setTreeNodeStream(input);
+ }
+
+ public TreeParser(TreeNodeStream input, RecognizerSharedState state) {
+ super(state); // share the state object with another parser
+ setTreeNodeStream(input);
+ }
+
+ public void reset() {
+ super.reset(); // reset all recognizer state variables
+ if ( input!=null ) {
+ input.seek(0); // rewind the input
+ }
+ }
+
+ /** Set the input stream */
+ public void setTreeNodeStream(TreeNodeStream input) {
+ this.input = input;
+ }
+
+ public TreeNodeStream getTreeNodeStream() {
+ return input;
+ }
+
+ public String getSourceName() {
+ return input.getSourceName();
+ }
+
+ protected Object getCurrentInputSymbol(IntStream input) {
+ return ((TreeNodeStream)input).LT(1);
+ }
+
+ protected Object getMissingSymbol(IntStream input,
+ RecognitionException e,
+ int expectedTokenType,
+ BitSet follow)
+ {
+ String tokenText =
+ "<missing "+getTokenNames()[expectedTokenType]+">";
+ TreeAdaptor adaptor = ((TreeNodeStream)e.input).getTreeAdaptor();
+ return adaptor.create(new CommonToken(expectedTokenType, tokenText));
+ }
+
+ /** Match '.' in tree parser has special meaning. Skip node or
+ * entire tree if node has children. If children, scan until
+ * corresponding UP node.
+ */
+ public void matchAny(IntStream ignore) { // ignore stream, copy of input
+ state.errorRecovery = false;
+ state.failed = false;
+ Object look = input.LT(1);
+ if ( input.getTreeAdaptor().getChildCount(look)==0 ) {
+ input.consume(); // not subtree, consume 1 node and return
+ return;
+ }
+ // current node is a subtree, skip to corresponding UP.
+ // must count nesting level to get right UP
+ int level=0;
+ int tokenType = input.getTreeAdaptor().getType(look);
+ while ( tokenType!=Token.EOF && !(tokenType==UP && level==0) ) {
+ input.consume();
+ look = input.LT(1);
+ tokenType = input.getTreeAdaptor().getType(look);
+ if ( tokenType == DOWN ) {
+ level++;
+ }
+ else if ( tokenType == UP ) {
+ level--;
+ }
+ }
+ input.consume(); // consume UP
+ }
+
+ /** We have DOWN/UP nodes in the stream that have no line info; override.
+ * plus we want to alter the exception type. Don't try to recover
+ * from tree parser errors inline...
+ */
+ protected Object recoverFromMismatchedToken(IntStream input,
+ int ttype,
+ BitSet follow)
+ throws RecognitionException
+ {
+ throw new MismatchedTreeNodeException(ttype, (TreeNodeStream)input);
+ }
+
+ /** Prefix error message with the grammar name because message is
+ * always intended for the programmer because the parser built
+ * the input tree not the user.
+ */
+ public String getErrorHeader(RecognitionException e) {
+ return getGrammarFileName()+": node from "+
+ (e.approximateLineInfo?"after ":"")+"line "+e.line+":"+e.charPositionInLine;
+ }
+
+ /** Tree parsers parse nodes they usually have a token object as
+ * payload. Set the exception token and do the default behavior.
+ */
+ public String getErrorMessage(RecognitionException e, String[] tokenNames) {
+ if ( this instanceof TreeParser ) {
+ TreeAdaptor adaptor = ((TreeNodeStream)e.input).getTreeAdaptor();
+ e.token = adaptor.getToken(e.node);
+ if ( e.token==null ) { // could be an UP/DOWN node
+ e.token = new CommonToken(adaptor.getType(e.node),
+ adaptor.getText(e.node));
+ }
+ }
+ return super.getErrorMessage(e, tokenNames);
+ }
+
+ public void traceIn(String ruleName, int ruleIndex) {
+ super.traceIn(ruleName, ruleIndex, input.LT(1));
+ }
+
+ public void traceOut(String ruleName, int ruleIndex) {
+ super.traceOut(ruleName, ruleIndex, input.LT(1));
+ }
+}
diff --git a/src/org/antlr/runtime/tree/TreePatternLexer.java b/src/org/antlr/runtime/tree/TreePatternLexer.java
new file mode 100755
index 0000000..2677c4e
--- /dev/null
+++ b/src/org/antlr/runtime/tree/TreePatternLexer.java
@@ -0,0 +1,135 @@
+/*
+ [The "BSD license"]
+ Copyright (c) 2005-2009 Terence Parr
+ All rights reserved.
+
+ Redistribution and use in source and binary forms, with or without
+ modification, are permitted provided that the following conditions
+ are met:
+ 1. Redistributions of source code must retain the above copyright
+ notice, this list of conditions and the following disclaimer.
+ 2. Redistributions in binary form must reproduce the above copyright
+ notice, this list of conditions and the following disclaimer in the
+ documentation and/or other materials provided with the distribution.
+ 3. The name of the author may not be used to endorse or promote products
+ derived from this software without specific prior written permission.
+
+ THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
+ IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
+ OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
+ IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
+ INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
+ NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
+ THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+package org.antlr.runtime.tree;
+
+public class TreePatternLexer {
+ public static final int EOF = -1;
+ public static final int BEGIN = 1;
+ public static final int END = 2;
+ public static final int ID = 3;
+ public static final int ARG = 4;
+ public static final int PERCENT = 5;
+ public static final int COLON = 6;
+ public static final int DOT = 7;
+
+ /** The tree pattern to lex like "(A B C)" */
+ protected String pattern;
+
+ /** Index into input string */
+ protected int p = -1;
+
+ /** Current char */
+ protected int c;
+
+ /** How long is the pattern in char? */
+ protected int n;
+
+ /** Set when token type is ID or ARG (name mimics Java's StreamTokenizer) */
+ public StringBuffer sval = new StringBuffer();
+
+ public boolean error = false;
+
+ public TreePatternLexer(String pattern) {
+ this.pattern = pattern;
+ this.n = pattern.length();
+ consume();
+ }
+
+ public int nextToken() {
+ sval.setLength(0); // reset, but reuse buffer
+ while ( c != EOF ) {
+ if ( c==' ' || c=='\n' || c=='\r' || c=='\t' ) {
+ consume();
+ continue;
+ }
+ if ( (c>='a' && c<='z') || (c>='A' && c<='Z') || c=='_' ) {
+ sval.append((char)c);
+ consume();
+ while ( (c>='a' && c<='z') || (c>='A' && c<='Z') ||
+ (c>='0' && c<='9') || c=='_' )
+ {
+ sval.append((char)c);
+ consume();
+ }
+ return ID;
+ }
+ if ( c=='(' ) {
+ consume();
+ return BEGIN;
+ }
+ if ( c==')' ) {
+ consume();
+ return END;
+ }
+ if ( c=='%' ) {
+ consume();
+ return PERCENT;
+ }
+ if ( c==':' ) {
+ consume();
+ return COLON;
+ }
+ if ( c=='.' ) {
+ consume();
+ return DOT;
+ }
+ if ( c=='[' ) { // grab [x] as a string, returning x
+ consume();
+ while ( c!=']' ) {
+ if ( c=='\\' ) {
+ consume();
+ if ( c!=']' ) {
+ sval.append('\\');
+ }
+ sval.append((char)c);
+ }
+ else {
+ sval.append((char)c);
+ }
+ consume();
+ }
+ consume();
+ return ARG;
+ }
+ consume();
+ error = true;
+ return EOF;
+ }
+ return EOF;
+ }
+
+ protected void consume() {
+ p++;
+ if ( p>=n ) {
+ c = EOF;
+ }
+ else {
+ c = pattern.charAt(p);
+ }
+ }
+}
\ No newline at end of file
diff --git a/src/org/antlr/runtime/tree/TreePatternParser.java b/src/org/antlr/runtime/tree/TreePatternParser.java
new file mode 100755
index 0000000..14983ab
--- /dev/null
+++ b/src/org/antlr/runtime/tree/TreePatternParser.java
@@ -0,0 +1,154 @@
+/*
+ [The "BSD license"]
+ Copyright (c) 2005-2009 Terence Parr
+ All rights reserved.
+
+ Redistribution and use in source and binary forms, with or without
+ modification, are permitted provided that the following conditions
+ are met:
+ 1. Redistributions of source code must retain the above copyright
+ notice, this list of conditions and the following disclaimer.
+ 2. Redistributions in binary form must reproduce the above copyright
+ notice, this list of conditions and the following disclaimer in the
+ documentation and/or other materials provided with the distribution.
+ 3. The name of the author may not be used to endorse or promote products
+ derived from this software without specific prior written permission.
+
+ THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
+ IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
+ OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
+ IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
+ INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
+ NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
+ THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+package org.antlr.runtime.tree;
+
+import org.antlr.runtime.Token;
+import org.antlr.runtime.CommonToken;
+
+public class TreePatternParser {
+ protected TreePatternLexer tokenizer;
+ protected int ttype;
+ protected TreeWizard wizard;
+ protected TreeAdaptor adaptor;
+
+ public TreePatternParser(TreePatternLexer tokenizer, TreeWizard wizard, TreeAdaptor adaptor) {
+ this.tokenizer = tokenizer;
+ this.wizard = wizard;
+ this.adaptor = adaptor;
+ ttype = tokenizer.nextToken(); // kickstart
+ }
+
+ public Object pattern() {
+ if ( ttype==TreePatternLexer.BEGIN ) {
+ return parseTree();
+ }
+ else if ( ttype==TreePatternLexer.ID ) {
+ Object node = parseNode();
+ if ( ttype==TreePatternLexer.EOF ) {
+ return node;
+ }
+ return null; // extra junk on end
+ }
+ return null;
+ }
+
+ public Object parseTree() {
+ if ( ttype != TreePatternLexer.BEGIN ) {
+ throw new RuntimeException("no BEGIN");
+ }
+ ttype = tokenizer.nextToken();
+ Object root = parseNode();
+ if ( root==null ) {
+ return null;
+ }
+ while ( ttype==TreePatternLexer.BEGIN ||
+ ttype==TreePatternLexer.ID ||
+ ttype==TreePatternLexer.PERCENT ||
+ ttype==TreePatternLexer.DOT )
+ {
+ if ( ttype==TreePatternLexer.BEGIN ) {
+ Object subtree = parseTree();
+ adaptor.addChild(root, subtree);
+ }
+ else {
+ Object child = parseNode();
+ if ( child==null ) {
+ return null;
+ }
+ adaptor.addChild(root, child);
+ }
+ }
+ if ( ttype != TreePatternLexer.END ) {
+ throw new RuntimeException("no END");
+ }
+ ttype = tokenizer.nextToken();
+ return root;
+ }
+
+ public Object parseNode() {
+ // "%label:" prefix
+ String label = null;
+ if ( ttype == TreePatternLexer.PERCENT ) {
+ ttype = tokenizer.nextToken();
+ if ( ttype != TreePatternLexer.ID ) {
+ return null;
+ }
+ label = tokenizer.sval.toString();
+ ttype = tokenizer.nextToken();
+ if ( ttype != TreePatternLexer.COLON ) {
+ return null;
+ }
+ ttype = tokenizer.nextToken(); // move to ID following colon
+ }
+
+ // Wildcard?
+ if ( ttype == TreePatternLexer.DOT ) {
+ ttype = tokenizer.nextToken();
+ Token wildcardPayload = new CommonToken(0, ".");
+ TreeWizard.TreePattern node =
+ new TreeWizard.WildcardTreePattern(wildcardPayload);
+ if ( label!=null ) {
+ node.label = label;
+ }
+ return node;
+ }
+
+ // "ID" or "ID[arg]"
+ if ( ttype != TreePatternLexer.ID ) {
+ return null;
+ }
+ String tokenName = tokenizer.sval.toString();
+ ttype = tokenizer.nextToken();
+ if ( tokenName.equals("nil") ) {
+ return adaptor.nil();
+ }
+ String text = tokenName;
+ // check for arg
+ String arg = null;
+ if ( ttype == TreePatternLexer.ARG ) {
+ arg = tokenizer.sval.toString();
+ text = arg;
+ ttype = tokenizer.nextToken();
+ }
+
+ // create node
+ int treeNodeType = wizard.getTokenType(tokenName);
+ if ( treeNodeType==Token.INVALID_TOKEN_TYPE ) {
+ return null;
+ }
+ Object node;
+ node = adaptor.create(treeNodeType, text);
+ if ( label!=null && node.getClass()==TreeWizard.TreePattern.class ) {
+ ((TreeWizard.TreePattern)node).label = label;
+ }
+ if ( arg!=null && node.getClass()==TreeWizard.TreePattern.class ) {
+ ((TreeWizard.TreePattern)node).hasTextArg = true;
+ }
+ return node;
+ }
+}
diff --git a/src/org/antlr/runtime/tree/TreeRewriter.java b/src/org/antlr/runtime/tree/TreeRewriter.java
new file mode 100755
index 0000000..91aee93
--- /dev/null
+++ b/src/org/antlr/runtime/tree/TreeRewriter.java
@@ -0,0 +1,120 @@
+/*
+ [The "BSD license"]
+ Copyright (c) 2005-2009 Terence Parr
+ All rights reserved.
+
+ Redistribution and use in source and binary forms, with or without
+ modification, are permitted provided that the following conditions
+ are met:
+ 1. Redistributions of source code must retain the above copyright
+ notice, this list of conditions and the following disclaimer.
+ 2. Redistributions in binary form must reproduce the above copyright
+ notice, this list of conditions and the following disclaimer in the
+ documentation and/or other materials provided with the distribution.
+ 3. The name of the author may not be used to endorse or promote products
+ derived from this software without specific prior written permission.
+
+ THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
+ IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
+ OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
+ IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
+ INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
+ NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
+ THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+package org.antlr.runtime.tree;
+
+import org.antlr.runtime.RecognizerSharedState;
+import org.antlr.runtime.RecognitionException;
+import org.antlr.runtime.TokenStream;
+
+public class TreeRewriter extends TreeParser {
+ public interface fptr {
+ public Object rule() throws RecognitionException;
+ }
+
+ protected boolean showTransformations = false;
+
+ protected TokenStream originalTokenStream;
+ protected TreeAdaptor originalAdaptor;
+
+ public TreeRewriter(TreeNodeStream input) {
+ this(input, new RecognizerSharedState());
+ }
+ public TreeRewriter(TreeNodeStream input, RecognizerSharedState state) {
+ super(input, state);
+ originalAdaptor = input.getTreeAdaptor();
+ originalTokenStream = input.getTokenStream();
+ }
+
+ public Object applyOnce(Object t, fptr whichRule) {
+ if ( t==null ) return null;
+ try {
+ // share TreeParser object but not parsing-related state
+ state = new RecognizerSharedState();
+ input = new CommonTreeNodeStream(originalAdaptor, t);
+ ((CommonTreeNodeStream)input).setTokenStream(originalTokenStream);
+ setBacktrackingLevel(1);
+ TreeRuleReturnScope r = (TreeRuleReturnScope)whichRule.rule();
+ setBacktrackingLevel(0);
+ if ( failed() ) return t;
+ if ( showTransformations &&
+ r!=null && !t.equals(r.getTree()) && r.getTree()!=null )
+ {
+ reportTransformation(t, r.getTree());
+ }
+ if ( r!=null && r.getTree()!=null ) return r.getTree();
+ else return t;
+ }
+ catch (RecognitionException e) { ; }
+ return t;
+ }
+
+ public Object applyRepeatedly(Object t, fptr whichRule) {
+ boolean treeChanged = true;
+ while ( treeChanged ) {
+ Object u = applyOnce(t, whichRule);
+ treeChanged = !t.equals(u);
+ t = u;
+ }
+ return t;
+ }
+
+ public Object downup(Object t) { return downup(t, false); }
+
+ public Object downup(Object t, boolean showTransformations) {
+ this.showTransformations = showTransformations;
+ TreeVisitor v = new TreeVisitor(new CommonTreeAdaptor());
+ TreeVisitorAction actions = new TreeVisitorAction() {
+ public Object pre(Object t) { return applyOnce(t, topdown_fptr); }
+ public Object post(Object t) { return applyRepeatedly(t, bottomup_ftpr); }
+ };
+ t = v.visit(t, actions);
+ return t;
+ }
+
+ /** Override this if you need transformation tracing to go somewhere
+ * other than stdout or if you're not using Tree-derived trees.
+ */
+ public void reportTransformation(Object oldTree, Object newTree) {
+ System.out.println(((Tree)oldTree).toStringTree()+" -> "+
+ ((Tree)newTree).toStringTree());
+ }
+
+ fptr topdown_fptr = new fptr() {
+ public Object rule() throws RecognitionException { return topdown(); }
+ };
+
+ fptr bottomup_ftpr = new fptr() {
+ public Object rule() throws RecognitionException { return bottomup(); }
+ };
+
+ // methods the downup strategy uses to do the up and down rules.
+ // to override, just define tree grammar rule topdown and turn on
+ // filter=true.
+ public Object topdown() throws RecognitionException { return null; }
+ public Object bottomup() throws RecognitionException { return null; }
+}
diff --git a/src/org/antlr/runtime/tree/TreeRuleReturnScope.java b/src/org/antlr/runtime/tree/TreeRuleReturnScope.java
new file mode 100755
index 0000000..4ea65c0
--- /dev/null
+++ b/src/org/antlr/runtime/tree/TreeRuleReturnScope.java
@@ -0,0 +1,41 @@
+/*
+ [The "BSD license"]
+ Copyright (c) 2005-2009 Terence Parr
+ All rights reserved.
+
+ Redistribution and use in source and binary forms, with or without
+ modification, are permitted provided that the following conditions
+ are met:
+ 1. Redistributions of source code must retain the above copyright
+ notice, this list of conditions and the following disclaimer.
+ 2. Redistributions in binary form must reproduce the above copyright
+ notice, this list of conditions and the following disclaimer in the
+ documentation and/or other materials provided with the distribution.
+ 3. The name of the author may not be used to endorse or promote products
+ derived from this software without specific prior written permission.
+
+ THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
+ IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
+ OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
+ IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
+ INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
+ NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
+ THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+package org.antlr.runtime.tree;
+
+import org.antlr.runtime.RuleReturnScope;
+
+/** This is identical to the ParserRuleReturnScope except that
+ * the start property is a tree nodes not Token object
+ * when you are parsing trees. To be generic the tree node types
+ * have to be Object.
+ */
+public class TreeRuleReturnScope extends RuleReturnScope {
+ /** First node or root node of tree matched for this rule. */
+ public Object start;
+ public Object getStart() { return start; }
+}
diff --git a/src/org/antlr/runtime/tree/TreeVisitor.java b/src/org/antlr/runtime/tree/TreeVisitor.java
new file mode 100755
index 0000000..8c5a717
--- /dev/null
+++ b/src/org/antlr/runtime/tree/TreeVisitor.java
@@ -0,0 +1,69 @@
+/*
+ [The "BSD license"]
+ Copyright (c) 2005-2009 Terence Parr
+ All rights reserved.
+
+ Redistribution and use in source and binary forms, with or without
+ modification, are permitted provided that the following conditions
+ are met:
+ 1. Redistributions of source code must retain the above copyright
+ notice, this list of conditions and the following disclaimer.
+ 2. Redistributions in binary form must reproduce the above copyright
+ notice, this list of conditions and the following disclaimer in the
+ documentation and/or other materials provided with the distribution.
+ 3. The name of the author may not be used to endorse or promote products
+ derived from this software without specific prior written permission.
+
+ THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
+ IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
+ OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
+ IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
+ INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
+ NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
+ THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+package org.antlr.runtime.tree;
+
+/** Do a depth first walk of a tree, applying pre() and post() actions
+ * as we discover and finish nodes.
+ */
+public class TreeVisitor {
+ protected TreeAdaptor adaptor;
+
+ public TreeVisitor(TreeAdaptor adaptor) {
+ this.adaptor = adaptor;
+ }
+ public TreeVisitor() { this(new CommonTreeAdaptor()); }
+
+ /** Visit every node in tree t and trigger an action for each node
+ * before/after having visited all of its children.
+ * Execute both actions even if t has no children.
+ * If a child visit yields a new child, it can update its
+ * parent's child list or just return the new child. The
+ * child update code works even if the child visit alters its parent
+ * and returns the new tree.
+ *
+ * Return result of applying post action to this node.
+ */
+ public Object visit(Object t, TreeVisitorAction action) {
+ // System.out.println("visit "+((Tree)t).toStringTree());
+ boolean isNil = adaptor.isNil(t);
+ if ( action!=null && !isNil ) {
+ t = action.pre(t); // if rewritten, walk children of new t
+ }
+ for (int i=0; i<adaptor.getChildCount(t); i++) {
+ Object child = adaptor.getChild(t, i);
+ Object visitResult = visit(child, action);
+ Object childAfterVisit = adaptor.getChild(t, i);
+ if ( visitResult != childAfterVisit ) { // result & child differ?
+ adaptor.setChild(t, i, visitResult);
+ }
+ }
+ if ( action!=null && !isNil ) t = action.post(t);
+ return t;
+ }
+}
diff --git a/src/org/antlr/runtime/tree/TreeVisitorAction.java b/src/org/antlr/runtime/tree/TreeVisitorAction.java
new file mode 100755
index 0000000..ef0f93c
--- /dev/null
+++ b/src/org/antlr/runtime/tree/TreeVisitorAction.java
@@ -0,0 +1,47 @@
+/*
+ [The "BSD license"]
+ Copyright (c) 2005-2009 Terence Parr
+ All rights reserved.
+
+ Redistribution and use in source and binary forms, with or without
+ modification, are permitted provided that the following conditions
+ are met:
+ 1. Redistributions of source code must retain the above copyright
+ notice, this list of conditions and the following disclaimer.
+ 2. Redistributions in binary form must reproduce the above copyright
+ notice, this list of conditions and the following disclaimer in the
+ documentation and/or other materials provided with the distribution.
+ 3. The name of the author may not be used to endorse or promote products
+ derived from this software without specific prior written permission.
+
+ THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
+ IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
+ OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
+ IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
+ INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
+ NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
+ THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+package org.antlr.runtime.tree;
+
+/** How to execute code for node t when a visitor visits node t. Execute
+ * pre() before visiting children and execute post() after visiting children.
+ */
+public interface TreeVisitorAction {
+ /** Execute an action before visiting children of t. Return t or
+ * a rewritten t. It is up to the visitor to decide what to do
+ * with the return value. Children of returned value will be
+ * visited if using TreeVisitor.visit().
+ */
+ public Object pre(Object t);
+
+ /** Execute an action after visiting children of t. Return t or
+ * a rewritten t. It is up to the visitor to decide what to do
+ * with the return value.
+ */
+ public Object post(Object t);
+}
diff --git a/src/org/antlr/runtime/tree/TreeWizard.java b/src/org/antlr/runtime/tree/TreeWizard.java
new file mode 100755
index 0000000..666cfd6
--- /dev/null
+++ b/src/org/antlr/runtime/tree/TreeWizard.java
@@ -0,0 +1,531 @@
+/*
+ [The "BSD license"]
+ Copyright (c) 2005-2009 Terence Parr
+ All rights reserved.
+
+ Redistribution and use in source and binary forms, with or without
+ modification, are permitted provided that the following conditions
+ are met:
+ 1. Redistributions of source code must retain the above copyright
+ notice, this list of conditions and the following disclaimer.
+ 2. Redistributions in binary form must reproduce the above copyright
+ notice, this list of conditions and the following disclaimer in the
+ documentation and/or other materials provided with the distribution.
+ 3. The name of the author may not be used to endorse or promote products
+ derived from this software without specific prior written permission.
+
+ THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
+ IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
+ OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
+ IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
+ INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
+ NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
+ THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+package org.antlr.runtime.tree;
+
+import org.antlr.runtime.Token;
+
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+/** Build and navigate trees with this object. Must know about the names
+ * of tokens so you have to pass in a map or array of token names (from which
+ * this class can build the map). I.e., Token DECL means nothing unless the
+ * class can translate it to a token type.
+ *
+ * In order to create nodes and navigate, this class needs a TreeAdaptor.
+ *
+ * This class can build a token type -> node index for repeated use or for
+ * iterating over the various nodes with a particular type.
+ *
+ * This class works in conjunction with the TreeAdaptor rather than moving
+ * all this functionality into the adaptor. An adaptor helps build and
+ * navigate trees using methods. This class helps you do it with string
+ * patterns like "(A B C)". You can create a tree from that pattern or
+ * match subtrees against it.
+ */
+public class TreeWizard {
+ protected TreeAdaptor adaptor;
+ protected Map tokenNameToTypeMap;
+
+ public interface ContextVisitor {
+ // TODO: should this be called visit or something else?
+ public void visit(Object t, Object parent, int childIndex, Map labels);
+ }
+
+ public static abstract class Visitor implements ContextVisitor {
+ public void visit(Object t, Object parent, int childIndex, Map labels) {
+ visit(t);
+ }
+ public abstract void visit(Object t);
+ }
+
+ /** When using %label:TOKENNAME in a tree for parse(), we must
+ * track the label.
+ */
+ public static class TreePattern extends CommonTree {
+ public String label;
+ public boolean hasTextArg;
+ public TreePattern(Token payload) {
+ super(payload);
+ }
+ public String toString() {
+ if ( label!=null ) {
+ return "%"+label+":"+super.toString();
+ }
+ else {
+ return super.toString();
+ }
+ }
+ }
+
+ public static class WildcardTreePattern extends TreePattern {
+ public WildcardTreePattern(Token payload) {
+ super(payload);
+ }
+ }
+
+ /** This adaptor creates TreePattern objects for use during scan() */
+ public static class TreePatternTreeAdaptor extends CommonTreeAdaptor {
+ public Object create(Token payload) {
+ return new TreePattern(payload);
+ }
+ }
+
+ // TODO: build indexes for the wizard
+
+ /** During fillBuffer(), we can make a reverse index from a set
+ * of token types of interest to the list of indexes into the
+ * node stream. This lets us convert a node pointer to a
+ * stream index semi-efficiently for a list of interesting
+ * nodes such as function definition nodes (you'll want to seek
+ * to their bodies for an interpreter). Also useful for doing
+ * dynamic searches; i.e., go find me all PLUS nodes.
+ protected Map tokenTypeToStreamIndexesMap;
+
+ /** If tokenTypesToReverseIndex set to INDEX_ALL then indexing
+ * occurs for all token types.
+ public static final Set INDEX_ALL = new HashSet();
+
+ /** A set of token types user would like to index for faster lookup.
+ * If this is INDEX_ALL, then all token types are tracked. If null,
+ * then none are indexed.
+ protected Set tokenTypesToReverseIndex = null;
+ */
+
+ public TreeWizard(TreeAdaptor adaptor) {
+ this.adaptor = adaptor;
+ }
+
+ public TreeWizard(TreeAdaptor adaptor, Map tokenNameToTypeMap) {
+ this.adaptor = adaptor;
+ this.tokenNameToTypeMap = tokenNameToTypeMap;
+ }
+
+ public TreeWizard(TreeAdaptor adaptor, String[] tokenNames) {
+ this.adaptor = adaptor;
+ this.tokenNameToTypeMap = computeTokenTypes(tokenNames);
+ }
+
+ public TreeWizard(String[] tokenNames) {
+ this(new CommonTreeAdaptor(), tokenNames);
+ }
+
+ /** Compute a Map<String, Integer> that is an inverted index of
+ * tokenNames (which maps int token types to names).
+ */
+ public Map computeTokenTypes(String[] tokenNames) {
+ Map m = new HashMap();
+ if ( tokenNames==null ) {
+ return m;
+ }
+ for (int ttype = Token.MIN_TOKEN_TYPE; ttype < tokenNames.length; ttype++) {
+ String name = tokenNames[ttype];
+ m.put(name, new Integer(ttype));
+ }
+ return m;
+ }
+
+ /** Using the map of token names to token types, return the type. */
+ public int getTokenType(String tokenName) {
+ if ( tokenNameToTypeMap==null ) {
+ return Token.INVALID_TOKEN_TYPE;
+ }
+ Integer ttypeI = (Integer)tokenNameToTypeMap.get(tokenName);
+ if ( ttypeI!=null ) {
+ return ttypeI.intValue();
+ }
+ return Token.INVALID_TOKEN_TYPE;
+ }
+
+ /** Walk the entire tree and make a node name to nodes mapping.
+ * For now, use recursion but later nonrecursive version may be
+ * more efficient. Returns Map<Integer, List> where the List is
+ * of your AST node type. The Integer is the token type of the node.
+ *
+ * TODO: save this index so that find and visit are faster
+ */
+ public Map index(Object t) {
+ Map m = new HashMap();
+ _index(t, m);
+ return m;
+ }
+
+ /** Do the work for index */
+ protected void _index(Object t, Map m) {
+ if ( t==null ) {
+ return;
+ }
+ int ttype = adaptor.getType(t);
+ List elements = (List)m.get(new Integer(ttype));
+ if ( elements==null ) {
+ elements = new ArrayList();
+ m.put(new Integer(ttype), elements);
+ }
+ elements.add(t);
+ int n = adaptor.getChildCount(t);
+ for (int i=0; i<n; i++) {
+ Object child = adaptor.getChild(t, i);
+ _index(child, m);
+ }
+ }
+
+ /** Return a List of tree nodes with token type ttype */
+ public List find(Object t, int ttype) {
+ final List nodes = new ArrayList();
+ visit(t, ttype, new TreeWizard.Visitor() {
+ public void visit(Object t) {
+ nodes.add(t);
+ }
+ });
+ return nodes;
+ }
+
+ /** Return a List of subtrees matching pattern. */
+ public List find(Object t, String pattern) {
+ final List subtrees = new ArrayList();
+ // Create a TreePattern from the pattern
+ TreePatternLexer tokenizer = new TreePatternLexer(pattern);
+ TreePatternParser parser =
+ new TreePatternParser(tokenizer, this, new TreePatternTreeAdaptor());
+ final TreePattern tpattern = (TreePattern)parser.pattern();
+ // don't allow invalid patterns
+ if ( tpattern==null ||
+ tpattern.isNil() ||
+ tpattern.getClass()==WildcardTreePattern.class )
+ {
+ return null;
+ }
+ int rootTokenType = tpattern.getType();
+ visit(t, rootTokenType, new TreeWizard.ContextVisitor() {
+ public void visit(Object t, Object parent, int childIndex, Map labels) {
+ if ( _parse(t, tpattern, null) ) {
+ subtrees.add(t);
+ }
+ }
+ });
+ return subtrees;
+ }
+
+ public Object findFirst(Object t, int ttype) {
+ return null;
+ }
+
+ public Object findFirst(Object t, String pattern) {
+ return null;
+ }
+
+ /** Visit every ttype node in t, invoking the visitor. This is a quicker
+ * version of the general visit(t, pattern) method. The labels arg
+ * of the visitor action method is never set (it's null) since using
+ * a token type rather than a pattern doesn't let us set a label.
+ */
+ public void visit(Object t, int ttype, ContextVisitor visitor) {
+ _visit(t, null, 0, ttype, visitor);
+ }
+
+ /** Do the recursive work for visit */
+ protected void _visit(Object t, Object parent, int childIndex, int ttype, ContextVisitor visitor) {
+ if ( t==null ) {
+ return;
+ }
+ if ( adaptor.getType(t)==ttype ) {
+ visitor.visit(t, parent, childIndex, null);
+ }
+ int n = adaptor.getChildCount(t);
+ for (int i=0; i<n; i++) {
+ Object child = adaptor.getChild(t, i);
+ _visit(child, t, i, ttype, visitor);
+ }
+ }
+
+ /** For all subtrees that match the pattern, execute the visit action.
+ * The implementation uses the root node of the pattern in combination
+ * with visit(t, ttype, visitor) so nil-rooted patterns are not allowed.
+ * Patterns with wildcard roots are also not allowed.
+ */
+ public void visit(Object t, final String pattern, final ContextVisitor visitor) {
+ // Create a TreePattern from the pattern
+ TreePatternLexer tokenizer = new TreePatternLexer(pattern);
+ TreePatternParser parser =
+ new TreePatternParser(tokenizer, this, new TreePatternTreeAdaptor());
+ final TreePattern tpattern = (TreePattern)parser.pattern();
+ // don't allow invalid patterns
+ if ( tpattern==null ||
+ tpattern.isNil() ||
+ tpattern.getClass()==WildcardTreePattern.class )
+ {
+ return;
+ }
+ final Map labels = new HashMap(); // reused for each _parse
+ int rootTokenType = tpattern.getType();
+ visit(t, rootTokenType, new TreeWizard.ContextVisitor() {
+ public void visit(Object t, Object parent, int childIndex, Map unusedlabels) {
+ // the unusedlabels arg is null as visit on token type doesn't set.
+ labels.clear();
+ if ( _parse(t, tpattern, labels) ) {
+ visitor.visit(t, parent, childIndex, labels);
+ }
+ }
+ });
+ }
+
+ /** Given a pattern like (ASSIGN %lhs:ID %rhs:.) with optional labels
+ * on the various nodes and '.' (dot) as the node/subtree wildcard,
+ * return true if the pattern matches and fill the labels Map with
+ * the labels pointing at the appropriate nodes. Return false if
+ * the pattern is malformed or the tree does not match.
+ *
+ * If a node specifies a text arg in pattern, then that must match
+ * for that node in t.
+ *
+ * TODO: what's a better way to indicate bad pattern? Exceptions are a hassle
+ */
+ public boolean parse(Object t, String pattern, Map labels) {
+ TreePatternLexer tokenizer = new TreePatternLexer(pattern);
+ TreePatternParser parser =
+ new TreePatternParser(tokenizer, this, new TreePatternTreeAdaptor());
+ TreePattern tpattern = (TreePattern)parser.pattern();
+ /*
+ System.out.println("t="+((Tree)t).toStringTree());
+ System.out.println("scant="+tpattern.toStringTree());
+ */
+ boolean matched = _parse(t, tpattern, labels);
+ return matched;
+ }
+
+ public boolean parse(Object t, String pattern) {
+ return parse(t, pattern, null);
+ }
+
+ /** Do the work for parse. Check to see if the t2 pattern fits the
+ * structure and token types in t1. Check text if the pattern has
+ * text arguments on nodes. Fill labels map with pointers to nodes
+ * in tree matched against nodes in pattern with labels.
+ */
+ protected boolean _parse(Object t1, TreePattern tpattern, Map labels) {
+ // make sure both are non-null
+ if ( t1==null || tpattern==null ) {
+ return false;
+ }
+ // check roots (wildcard matches anything)
+ if ( tpattern.getClass() != WildcardTreePattern.class ) {
+ if ( adaptor.getType(t1) != tpattern.getType() ) return false;
+ // if pattern has text, check node text
+ if ( tpattern.hasTextArg && !adaptor.getText(t1).equals(tpattern.getText()) ) {
+ return false;
+ }
+ }
+ if ( tpattern.label!=null && labels!=null ) {
+ // map label in pattern to node in t1
+ labels.put(tpattern.label, t1);
+ }
+ // check children
+ int n1 = adaptor.getChildCount(t1);
+ int n2 = tpattern.getChildCount();
+ if ( n1 != n2 ) {
+ return false;
+ }
+ for (int i=0; i<n1; i++) {
+ Object child1 = adaptor.getChild(t1, i);
+ TreePattern child2 = (TreePattern)tpattern.getChild(i);
+ if ( !_parse(child1, child2, labels) ) {
+ return false;
+ }
+ }
+ return true;
+ }
+
+ /** Create a tree or node from the indicated tree pattern that closely
+ * follows ANTLR tree grammar tree element syntax:
+ *
+ * (root child1 ... child2).
+ *
+ * You can also just pass in a node: ID
+ *
+ * Any node can have a text argument: ID[foo]
+ * (notice there are no quotes around foo--it's clear it's a string).
+ *
+ * nil is a special name meaning "give me a nil node". Useful for
+ * making lists: (nil A B C) is a list of A B C.
+ */
+ public Object create(String pattern) {
+ TreePatternLexer tokenizer = new TreePatternLexer(pattern);
+ TreePatternParser parser = new TreePatternParser(tokenizer, this, adaptor);
+ Object t = parser.pattern();
+ return t;
+ }
+
+ /** Compare t1 and t2; return true if token types/text, structure match exactly.
+ * The trees are examined in their entirety so that (A B) does not match
+ * (A B C) nor (A (B C)).
+ // TODO: allow them to pass in a comparator
+ * TODO: have a version that is nonstatic so it can use instance adaptor
+ *
+ * I cannot rely on the tree node's equals() implementation as I make
+ * no constraints at all on the node types nor interface etc...
+ */
+ public static boolean equals(Object t1, Object t2, TreeAdaptor adaptor) {
+ return _equals(t1, t2, adaptor);
+ }
+
+ /** Compare type, structure, and text of two trees, assuming adaptor in
+ * this instance of a TreeWizard.
+ */
+ public boolean equals(Object t1, Object t2) {
+ return _equals(t1, t2, adaptor);
+ }
+
+ protected static boolean _equals(Object t1, Object t2, TreeAdaptor adaptor) {
+ // make sure both are non-null
+ if ( t1==null || t2==null ) {
+ return false;
+ }
+ // check roots
+ if ( adaptor.getType(t1) != adaptor.getType(t2) ) {
+ return false;
+ }
+ if ( !adaptor.getText(t1).equals(adaptor.getText(t2)) ) {
+ return false;
+ }
+ // check children
+ int n1 = adaptor.getChildCount(t1);
+ int n2 = adaptor.getChildCount(t2);
+ if ( n1 != n2 ) {
+ return false;
+ }
+ for (int i=0; i<n1; i++) {
+ Object child1 = adaptor.getChild(t1, i);
+ Object child2 = adaptor.getChild(t2, i);
+ if ( !_equals(child1, child2, adaptor) ) {
+ return false;
+ }
+ }
+ return true;
+ }
+
+ // TODO: next stuff taken from CommonTreeNodeStream
+
+ /** Given a node, add this to the reverse index tokenTypeToStreamIndexesMap.
+ * You can override this method to alter how indexing occurs. The
+ * default is to create a
+ *
+ * Map<Integer token type,ArrayList<Integer stream index>>
+ *
+ * This data structure allows you to find all nodes with type INT in order.
+ *
+ * If you really need to find a node of type, say, FUNC quickly then perhaps
+ *
+ * Map<Integertoken type,Map<Object tree node,Integer stream index>>
+ *
+ * would be better for you. The interior maps map a tree node to
+ * the index so you don't have to search linearly for a specific node.
+ *
+ * If you change this method, you will likely need to change
+ * getNodeIndex(), which extracts information.
+ protected void fillReverseIndex(Object node, int streamIndex) {
+ //System.out.println("revIndex "+node+"@"+streamIndex);
+ if ( tokenTypesToReverseIndex==null ) {
+ return; // no indexing if this is empty (nothing of interest)
+ }
+ if ( tokenTypeToStreamIndexesMap==null ) {
+ tokenTypeToStreamIndexesMap = new HashMap(); // first indexing op
+ }
+ int tokenType = adaptor.getType(node);
+ Integer tokenTypeI = new Integer(tokenType);
+ if ( !(tokenTypesToReverseIndex==INDEX_ALL ||
+ tokenTypesToReverseIndex.contains(tokenTypeI)) )
+ {
+ return; // tokenType not of interest
+ }
+ Integer streamIndexI = new Integer(streamIndex);
+ ArrayList indexes = (ArrayList)tokenTypeToStreamIndexesMap.get(tokenTypeI);
+ if ( indexes==null ) {
+ indexes = new ArrayList(); // no list yet for this token type
+ indexes.add(streamIndexI); // not there yet, add
+ tokenTypeToStreamIndexesMap.put(tokenTypeI, indexes);
+ }
+ else {
+ if ( !indexes.contains(streamIndexI) ) {
+ indexes.add(streamIndexI); // not there yet, add
+ }
+ }
+ }
+
+ /** Track the indicated token type in the reverse index. Call this
+ * repeatedly for each type or use variant with Set argument to
+ * set all at once.
+ * @param tokenType
+ public void reverseIndex(int tokenType) {
+ if ( tokenTypesToReverseIndex==null ) {
+ tokenTypesToReverseIndex = new HashSet();
+ }
+ else if ( tokenTypesToReverseIndex==INDEX_ALL ) {
+ return;
+ }
+ tokenTypesToReverseIndex.add(new Integer(tokenType));
+ }
+
+ /** Track the indicated token types in the reverse index. Set
+ * to INDEX_ALL to track all token types.
+ public void reverseIndex(Set tokenTypes) {
+ tokenTypesToReverseIndex = tokenTypes;
+ }
+
+ /** Given a node pointer, return its index into the node stream.
+ * This is not its Token stream index. If there is no reverse map
+ * from node to stream index or the map does not contain entries
+ * for node's token type, a linear search of entire stream is used.
+ *
+ * Return -1 if exact node pointer not in stream.
+ public int getNodeIndex(Object node) {
+ //System.out.println("get "+node);
+ if ( tokenTypeToStreamIndexesMap==null ) {
+ return getNodeIndexLinearly(node);
+ }
+ int tokenType = adaptor.getType(node);
+ Integer tokenTypeI = new Integer(tokenType);
+ ArrayList indexes = (ArrayList)tokenTypeToStreamIndexesMap.get(tokenTypeI);
+ if ( indexes==null ) {
+ //System.out.println("found linearly; stream index = "+getNodeIndexLinearly(node));
+ return getNodeIndexLinearly(node);
+ }
+ for (int i = 0; i < indexes.size(); i++) {
+ Integer streamIndexI = (Integer)indexes.get(i);
+ Object n = get(streamIndexI.intValue());
+ if ( n==node ) {
+ //System.out.println("found in index; stream index = "+streamIndexI);
+ return streamIndexI.intValue(); // found it!
+ }
+ }
+ return -1;
+ }
+
+ */
+}
