added a load of blabber

10 files changed, 236 insertions, 3 deletions

diff --git a/comp/lucas-standen-NEA/code/ads/ast/Makefile b/comp/lucas-standen-NEA/code/ads/ast/Makefile
deleted file mode 100644
index e69de29..0000000
--- a/comp/lucas-standen-NEA/code/ads/ast/Makefile
+++ /dev/null
diff --git a/comp/lucas-standen-NEA/code/ads/ast/ast.c b/comp/lucas-standen-NEA/code/ads/ast/ast.c
deleted file mode 100644
index e69de29..0000000
--- a/comp/lucas-standen-NEA/code/ads/ast/ast.c
+++ /dev/null
diff --git a/comp/lucas-standen-NEA/code/ads/ast/ast.h b/comp/lucas-standen-NEA/code/ads/ast/ast.h
deleted file mode 100644
index e69de29..0000000
--- a/comp/lucas-standen-NEA/code/ads/ast/ast.h
+++ /dev/null
diff --git a/comp/lucas-standen-NEA/code/ads/ast/types.c b/comp/lucas-standen-NEA/code/ads/ast/types.c
deleted file mode 100644
index e69de29..0000000
--- a/comp/lucas-standen-NEA/code/ads/ast/types.c
+++ /dev/null
diff --git a/comp/lucas-standen-NEA/code/ads/ast/types.h b/comp/lucas-standen-NEA/code/ads/ast/types.h
deleted file mode 100644
index e69de29..0000000
--- a/comp/lucas-standen-NEA/code/ads/ast/types.h
+++ /dev/null
diff --git a/comp/lucas-standen-NEA/code/tokenizer/Makefile b/comp/lucas-standen-NEA/code/tokenizer/Makefile
index b5cfe56..479b838 100644
--- a/comp/lucas-standen-NEA/code/tokenizer/Makefile
+++ b/comp/lucas-standen-NEA/code/tokenizer/Makefile
@@ -1,5 +1,11 @@
-
-parser: parser.c util
-	cc -O3 parser.c util.o -c -o parser.o
+tokenizer: parser util tokenizer.c
+	cc -O3 tokenizer.c parser.o util.o -o tokenizer
+parser: parser.c
+	cc -O3 parser.c -c -o parser.o
 util:	util.c
 	cc -O3 util.c -c -o util.o
+
+clean:
+	rm -rf *.o
+	rm -rf tokenizer
+	
diff --git a/comp/lucas-standen-NEA/code/tokenizer/tokenizer b/comp/lucas-standen-NEA/code/tokenizer/tokenizer
new file mode 100755
index 0000000..726ee21
--- /dev/null
+++ b/comp/lucas-standen-NEA/code/tokenizer/tokenizer
diff --git a/comp/lucas-standen-NEA/code/tokenizer/tokenizer.c b/comp/lucas-standen-NEA/code/tokenizer/tokenizer.c
index a643455..5cc596f 100644
--- a/comp/lucas-standen-NEA/code/tokenizer/tokenizer.c
+++ b/comp/lucas-standen-NEA/code/tokenizer/tokenizer.c
@@ -1,3 +1,154 @@
+#include <stdlib.h>
+#include <stdio.h>
+#include <string.h>
+
 #include "parser.h"
+#include "util.h"
+#include "types.h"
+
+int functionIdCounter = 0;
+
+ast_node *GenAst(char *exp); // generates the ast of 1 expression
+int getArgCount(char *exp); // counts how many args are pressent in exp
+char **GetStringArgs(char *exp); // gets the string args of an expression
+char *GetFunction(char *exp); // gets the function used in 1 expression
+builtInFuncs IsBuiltIn(char *func); // returns the id of a function thats built in, or -1 if its not
+
+
+builtInFuncs IsBuiltIn(char *func){
+	if (strcmp(func, "defun") == 0){
+		return DEFUN;
+	}else if (strcmp(func, "let") == 0){
+		return LET;
+	}else if (strcmp(func, "set") == 0){
+		return SET;
+	}else if (strcmp(func, "if") == 0){
+		return IF;
+	}else if (strcmp(func, "elif") == 0){
+		return ELIF;
+	}else if (strcmp(func, "else") == 0){
+		return ELSE;
+	}else if (strcmp(func, "for") == 0){
+		return FOR;
+	}else if (strcmp(func, "while") == 0){
+		return WHILE;
+	}else if (strcmp(func, "symbol") == 0){
+		return SYMBOL;
+	}else if (strcmp(func, "struct") == 0){
+		return STRUCT;
+	}else if (strcmp(func, "+") == 0){
+		return ADD;
+	}else if (strcmp(func, "-") == 0){
+		return SUB;
+	}else if (strcmp(func, "*") == 0){
+		return MUL;
+	}else if (strcmp(func, "/") == 0){
+		return DIV;
+	}else if (strcmp(func, "=") == 0){
+		return EQ;
+	}else if (strcmp(func, "!=") == 0){
+		return NEQ;
+	}else if (strcmp(func, ">") == 0){
+		return GT;
+	}else if (strcmp(func, "<") == 0){
+		return LT;
+	}else if (strcmp(func, ">=") == 0){
+		return GTEQ;
+	}else if (strcmp(func, "<=") == 0){
+		return LTEQ;
+	}else if (strcmp(func, "cast") == 0){
+		return CAST;
+	}else if (strcmp(func, "typeof") == 0){
+		return TYPEOF;
+	}else if (strcmp(func, "terminate") == 0){
+		return TERMINATE;
+	}else if (strcmp(func, "return") == 0){
+		return RETURN;
+	}
+	else {
+		return -1;
+	}
+}
+
+char *GetFunction(char *exp){ // takes exp with brackets
+	char *out = CheckedMalloc(strlen(exp));
+	int i = 1;
+	char c = exp[i];
+	while (c != ' '){
+		out[i-1] = c;
+		i++;
+		c = exp[i];
+	}
+	i++;
+	out[i] = '\0';
+	out = CheckedRealloc(out, i);
+	return out;
+}
+
+// TODO make it count any arg inside () as one arg
+char **GetStringArgs(char *exp){ // takes exp without brackets
+	int spaceCount = 0;
+	int i = 0;
+	char c = exp[i];
+	while (c != '\0'){
+		spaceCount++;
+		i++;
+		c = exp[i];
+
+	}
+
+	char **out = CheckedMalloc(spaceCount);
+	for (int i = 0; i < spaceCount; i++){
+		out[i] = CheckedMalloc(strlen(exp));
+	}
+
+	int tokCounter = 0;
+	i = 0;
+	int charCounter = 0;
+	while (exp[i] != '\0'){
+		if (exp[i] != ' '){
+			if (tokCounter != 0){
+				out[tokCounter-1][charCounter] = exp[i];
+				charCounter++;
+			}
+		} else{
+			out[tokCounter][i] = '\0';
+			charCounter = 0;
+			tokCounter++;
+		}
+		i++;
+	}
+
+	return out;
+}
+
+ast_node *GenAst(char *exp){ // takes exp with brackets
+	ast_node *head = CheckedMalloc(sizeof(ast_node));
+	char *function = GetFunction(exp);
+	head->builtInFunc = IsBuiltIn(function);	
+	free(function);
+
+	if (head->builtInFunc == -1){
+		head->func = CheckedMalloc(sizeof(functionToken));
+		head->func->id = functionIdCounter;
+		functionIdCounter++;
+	}else {
+		head->func = NULL;
+	}
+
+	return head;
+}
+
+int main(){
+	ast_node *node = GenAst("(+ 1 2)");
+	printf("%d\n", node->builtInFunc);
+
+	char **args = GetStringArgs("+ 1 2");
+	for (int i = 0; i < 2; i++){
+		printf("%s\n", args[i]);
+	}
 
+	free(args);
 
+	free(node);
+}
diff --git a/comp/lucas-standen-NEA/code/tokenizer/types.c b/comp/lucas-standen-NEA/code/tokenizer/types.c
deleted file mode 100644
index e69de29..0000000
--- a/comp/lucas-standen-NEA/code/tokenizer/types.c
+++ /dev/null
diff --git a/comp/lucas-standen-NEA/code/tokenizer/types.h b/comp/lucas-standen-NEA/code/tokenizer/types.h
index e69de29..034dc04 100644
--- a/comp/lucas-standen-NEA/code/tokenizer/types.h
+++ b/comp/lucas-standen-NEA/code/tokenizer/types.h
@@ -0,0 +1,76 @@
+#include <stdint.h>
+#include "../ads/ll/ll.h"
+
+// all language types
+typedef enum types { 
+	I32_T = 0,
+	I64_T = 1,
+	U32_T = 2,
+	U64_T = 3,
+	FLOAT_t = 4,
+	CHAR_T = 5,
+	FUNCTION_T = 6,
+	STRUCT_T = 7,
+	OBJ_T = 8,
+} types;
+
+// int types
+typedef int32_t i32;
+typedef int64_t i64;
+
+// uint types
+typedef uint32_t u32;
+typedef uint64_t u64;
+
+// char and float types are still called char and float so no typedef needed
+
+// function type
+typedef struct functionToken { 
+	int id; // a function id to avoid strings
+	types returnType; // what the function returns
+	types *args; // the types of args a function takes
+	ll_t astHead; // the code for the function
+} functionToken;
+
+// built in functions
+typedef enum builtInFuncs {
+	DEFUN = 0,
+	LET = 1,
+	SET = 2,
+	IF = 3,
+	ELIF = 4,
+	ELSE = 5,
+	FOR = 6,
+	WHILE = 7,
+	SYMBOL = 8,
+	STRUCT = 9,
+	
+	// arithmetic
+	ADD = 10,
+	SUB = 11,
+	MUL = 12,
+	DIV = 13,
+	
+	// comparison
+	EQ = 14,
+	NEQ = 15,
+	GT = 16,
+	LT = 17,
+	GTEQ = 18,
+	LTEQ = 19,
+
+	CAST = 20,
+	TYPEOF = 21,
+	TERMINATE = 22,
+	RETURN = 23, 
+} builtInFuncs;
+
+typedef struct ast_node ast_node;
+
+typedef struct ast_node {
+	builtInFuncs builtInFunc; // if it's a builtin function call use this, else -1
+	functionToken *func; // if it's not builtin then use this
+	void **literalArgs; // the args of the node, this will be an array of litteral values
+	ast_node **args; // the non litteral tokens
+	// if litteralArgs[x] is real then args[x] should be NULL, and vice versa
+} ast_node;