feat: adds tokens

adds TokenType, Token and tokens

1 files changed, 34 insertions, 0 deletions

diff --git a/token/token.go b/token/token.go
new file mode 100644
index 0000000..92bc1ee
--- /dev/null
+++ b/token/token.go
@@ -0,0 +1,34 @@
+package token
+
+const (
+	ILLEGAL = "ILLEGAL"
+	EOF     = "EOF"
+
+	// Identifiers and literals
+	IDENT = "IDENT"
+	INT   = "INT"
+
+	// Operators
+	ASSIGN = "="
+	PLUS   = "+"
+
+	// Delimiters
+	COMMA     = ","
+	SEMICOLON = ";"
+
+	LPAREN = "("
+	RPAREN = ")"
+	LBRACE = "{"
+	RBRACE = "}"
+
+	// Keywords
+	FUNCTION = "FUNCTION"
+	LET      = "LET"
+)
+
+type TokenType string
+
+type Token struct {
+	Type    TokenType
+	Literal string
+}