126 lines
3.9 KiB
Plaintext
126 lines
3.9 KiB
Plaintext
package slangc.parser;
|
|
|
|
import slang.streams.SyncOutput;
|
|
import slang.streams.SyncInput;
|
|
|
|
public final class Token extends Node {
|
|
private Scan owner;
|
|
private TokenType type;
|
|
private SourceSnippet snippet;
|
|
|
|
public Token(TokenType type, SourceSnippet snippet) {
|
|
reset(type, snippet);
|
|
}
|
|
|
|
public static void encInt(byte[] t, int index, int val) {
|
|
t[index] = (byte) val;
|
|
t[index+1] = (byte) (val >> 8);
|
|
t[index+2] = (byte) (val >> 16);
|
|
t[index+3] = (byte) (val >> 24);
|
|
}
|
|
|
|
public static int decInt(byte[] t, int index) {
|
|
int v = 0;
|
|
v |= (((int)t[index]) & 0xFF);
|
|
v |= ((((int)t[index+1]) & 0xFF) << 8);
|
|
v |= ((((int)t[index+2]) & 0xFF) << 16);
|
|
v |= ((((int)t[index+3]) & 0xFF) << 24);
|
|
return v;
|
|
}
|
|
|
|
static byte[] outbuffer = new byte[1 + 1 + 4 + 4 + 4 + 4];
|
|
|
|
public static Token loadBinaryShort(Scan scn, TokenType typ, SyncInput<byte> inp) {
|
|
inp.readBuffer(outbuffer, 0, 6); // First byte is already decoded as the typ argument
|
|
|
|
int idx = (((int)outbuffer[0]) & 0xFF) | ((((int)outbuffer[1]) & 0xFF) << 8);
|
|
int lin = (((int)outbuffer[2]) & 0xFF) | ((((int)outbuffer[3]) & 0xFF) << 8);
|
|
int chr = ((int)outbuffer[4]) & 0xFF;
|
|
int len = ((int)outbuffer[5]) & 0xFF;
|
|
|
|
Token t = new Token(typ, new SourceSnippet(new SourcePosition(scn.getSource(),idx,lin,chr), len));
|
|
t.setOwner(scn);
|
|
return t;
|
|
}
|
|
|
|
public static Token loadBinaryLong(Scan scn, TokenType typ, SyncInput<byte> inp) {
|
|
inp.readBuffer(outbuffer, 0, 16); // First bytes are already decoded as the typ argument
|
|
int idx = decInt(outbuffer, 0);
|
|
int lin = decInt(outbuffer, 4);
|
|
int chr = decInt(outbuffer, 8);
|
|
int len = decInt(outbuffer, 12);
|
|
Token t = new Token(typ, new SourceSnippet(new SourcePosition(scn.getSource(),idx,lin,chr), len));
|
|
t.setOwner(scn);
|
|
return t;
|
|
}
|
|
|
|
public void dumpBinary(SyncOutput<byte> o) {
|
|
if (type.value > 100) {
|
|
throw new Error("Too many token types for this simple format!");
|
|
}
|
|
if (owner != null) owner.reset();//owner = null; // Owner will be cleared in cached runs
|
|
if (snippet.start.index > 65535 || snippet.start.line > 65535 || snippet.start.character > 255 || snippet.length > 255) {
|
|
outbuffer[0] = (byte) (-128);
|
|
outbuffer[1] = (byte) (-(type.value+1));
|
|
encInt(outbuffer, 2, snippet.start.index);
|
|
encInt(outbuffer, 6, snippet.start.line);
|
|
encInt(outbuffer, 10, snippet.start.character);
|
|
encInt(outbuffer, 14, snippet.length);
|
|
o.writeBuffer(outbuffer, 0, 18);
|
|
} else {
|
|
outbuffer[0] = (byte) (-(type.value+1));
|
|
outbuffer[1] = (byte) snippet.start.index;
|
|
outbuffer[2] = (byte) (snippet.start.index >> 8);
|
|
outbuffer[3] = (byte) snippet.start.line;
|
|
outbuffer[4] = (byte) (snippet.start.line >> 8);
|
|
outbuffer[5] = (byte) snippet.start.character;
|
|
outbuffer[6] = (byte) snippet.length;
|
|
o.writeBuffer(outbuffer, 0, 7);
|
|
}
|
|
|
|
//throw new Error("Can't dump AST nodes of type " + Type.of(this).fullName());
|
|
}
|
|
|
|
|
|
public void setOwner(Scan scan) {
|
|
this.owner = scan;
|
|
}
|
|
|
|
public Scan getOwner() {
|
|
return owner;
|
|
}
|
|
|
|
public void reset(TokenType type, SourceSnippet snippet) {
|
|
this.type = type;
|
|
this.snippet = snippet;
|
|
}
|
|
|
|
public TokenType getType() {
|
|
return type;
|
|
}
|
|
|
|
public SourceSnippet getSnippet() {
|
|
return snippet;
|
|
}
|
|
|
|
public boolean is(TokenType t) {
|
|
return type == t;
|
|
}
|
|
|
|
public boolean is(TokenType t, String match) {
|
|
return type == t && snippet.getSource().equals(match);
|
|
}
|
|
|
|
@Override
|
|
public String toString() { // TODO: Refactor
|
|
return "Token(" + type.name() + ", " + snippet.toString() + ")";
|
|
}
|
|
@Override
|
|
public NodeType getNodeType() {
|
|
return NodeType.TOKEN;
|
|
}
|
|
|
|
public boolean is(String value) {
|
|
return snippet.getSource().equals(value);
|
|
}
|
|
} |