package slangc.parser; import slang.streams.SyncOutput; import slang.streams.SyncInput; public final class Token extends Node { private Scan owner; private TokenType type; private SourceSnippet snippet; public Token(TokenType type, SourceSnippet snippet) { reset(type, snippet); } public static void encInt(byte[] t, int index, int val) { t[index] = (byte) val; t[index+1] = (byte) (val >> 8); t[index+2] = (byte) (val >> 16); t[index+3] = (byte) (val >> 24); } public static int decInt(byte[] t, int index) { int v = 0; v |= (((int)t[index]) & 0xFF); v |= ((((int)t[index+1]) & 0xFF) << 8); v |= ((((int)t[index+2]) & 0xFF) << 16); v |= ((((int)t[index+3]) & 0xFF) << 24); return v; } static byte[] outbuffer = new byte[1 + 1 + 4 + 4 + 4 + 4]; public static Token loadBinaryShort(Scan scn, TokenType typ, SyncInput inp) { inp.readBuffer(outbuffer, 0, 6); // First byte is already decoded as the typ argument int idx = (((int)outbuffer[0]) & 0xFF) | ((((int)outbuffer[1]) & 0xFF) << 8); int lin = (((int)outbuffer[2]) & 0xFF) | ((((int)outbuffer[3]) & 0xFF) << 8); int chr = ((int)outbuffer[4]) & 0xFF; int len = ((int)outbuffer[5]) & 0xFF; Token t = new Token(typ, new SourceSnippet(new SourcePosition(scn.getSource(),idx,lin,chr), len)); t.setOwner(scn); return t; } public static Token loadBinaryLong(Scan scn, TokenType typ, SyncInput inp) { inp.readBuffer(outbuffer, 0, 16); // First bytes are already decoded as the typ argument int idx = decInt(outbuffer, 0); int lin = decInt(outbuffer, 4); int chr = decInt(outbuffer, 8); int len = decInt(outbuffer, 12); Token t = new Token(typ, new SourceSnippet(new SourcePosition(scn.getSource(),idx,lin,chr), len)); t.setOwner(scn); return t; } public void dumpBinary(SyncOutput o) { if (type.value > 100) { throw new Error("Too many token types for this simple format!"); } if (owner != null) owner.reset();//owner = null; // Owner will be cleared in cached runs if (snippet.start.index > 65535 || snippet.start.line > 65535 || snippet.start.character > 255 || snippet.length > 255) { outbuffer[0] = (byte) (-128); outbuffer[1] = (byte) (-(type.value+1)); encInt(outbuffer, 2, snippet.start.index); encInt(outbuffer, 6, snippet.start.line); encInt(outbuffer, 10, snippet.start.character); encInt(outbuffer, 14, snippet.length); o.writeBuffer(outbuffer, 0, 18); } else { outbuffer[0] = (byte) (-(type.value+1)); outbuffer[1] = (byte) snippet.start.index; outbuffer[2] = (byte) (snippet.start.index >> 8); outbuffer[3] = (byte) snippet.start.line; outbuffer[4] = (byte) (snippet.start.line >> 8); outbuffer[5] = (byte) snippet.start.character; outbuffer[6] = (byte) snippet.length; o.writeBuffer(outbuffer, 0, 7); } //throw new Error("Can't dump AST nodes of type " + Type.of(this).fullName()); } public void setOwner(Scan scan) { this.owner = scan; } public Scan getOwner() { return owner; } public void reset(TokenType type, SourceSnippet snippet) { this.type = type; this.snippet = snippet; } public TokenType getType() { return type; } public SourceSnippet getSnippet() { return snippet; } public boolean is(TokenType t) { return type == t; } public boolean is(TokenType t, String match) { return type == t && snippet.getSource().equals(match); } @Override public String toString() { // TODO: Refactor return "Token(" + type.name() + ", " + snippet.toString() + ")"; } @Override public NodeType getNodeType() { return NodeType.TOKEN; } public boolean is(String value) { return snippet.getSource().equals(value); } }