Squash commits for public release

utils/compilers/ConnectionCompiler/Lexer/lexer.py

@@ -0,0 +1,108 @@
+from token import Token
+from Lexer.reserved_symbols import *
+from type_file import Type
+
+
+class Lexer():
+    MAX_OPERATION_SIZE = 2
+
+    def __init__(self, code: [str]):
+        self.code = code
+        self.current_code_part = code[0]
+        self.current_code_line = 0
+        self.code_lines = len(code)
+        self.current_position = 0
+        self.current_char = self.current_code_part[self.current_position]
+
+    def advance(self):
+        self.current_position += 1
+        if self.current_position < len(self.current_code_part):
+            self.current_char = self.current_code_part[self.current_position]
+        else:
+            self.current_code_line += 1
+            if self.current_code_line < self.code_lines:
+                self.current_code_part = self.code[self.current_code_line]
+                self.current_position = 0
+                self.current_char = self.current_code_part[0]
+            else:
+                self.current_char = None
+
+    def skip_rest_of_line(self):
+        start_with_line = self.current_code_line
+        while self.current_char is not None and self.current_code_line == start_with_line:
+            self.advance()
+
+    def lookup(self, count):
+        peek_pos = self.current_position + count
+        if peek_pos < len(self.current_code_part):
+            return self.current_code_part[peek_pos]
+        else:
+            return None
+
+    def skip_gaps(self):
+        while self.current_char is not None and self.current_char == ' ':
+            self.advance()
+
+    def read_number(self):
+        result = ""
+        was_dot = False
+        while self.current_char is not None and (self.current_char.isdigit() or self.current_char == '.'):
+            if self.current_char == '.':
+                if was_dot:
+                    print(">2 dots in number")
+                    exit(0)
+                was_dot = True
+            result += self.current_char
+            self.advance()
+        if was_dot:
+            return Token(Type.Number.Real, float(result))
+        else:
+            return Token(Type.Number.Integer, int(result))
+
+    def read_word(self):
+        type = Type.Word
+        result = ""
+        while self.current_char is not None and (self.current_char.isalpha() or self.current_char.isdigit() or self.current_char == '_' or self.current_char == '<' or self.current_char == '>' or (self.current_char == ':' and self.lookup(1) == ':')):
+            if (self.current_char == ':'):
+                result += self.current_char
+                self.advance()
+            result += self.current_char
+            self.advance()
+
+        if result.upper() in reserved_words:
+            return Token(reserved_words[result.upper()], result.upper())
+
+        return Token(type, result)
+
+    def read_operation(self):
+        token = Token()
+        operation = ""
+        for i in range(self.MAX_OPERATION_SIZE):
+            next_element = self.lookup(i)
+            if next_element is None:
+                break
+            operation += next_element
+            if operation in reserved_symbols.keys():
+                token = Token(reserved_symbols[operation], operation)
+
+        if token.value is not None:
+            for i in range(len(token.value)):
+                self.advance()
+
+        return token
+
+    def next_token(self):
+        self.skip_gaps()
+
+        while self.current_char == '#':
+            self.skip_rest_of_line()
+            self.skip_gaps()
+
+        if self.current_char is None:
+            return Token(Type.Special.EOF, None)
+        elif self.current_char.isdigit():
+            return self.read_number()
+        elif self.current_char.isalpha():
+            return self.read_word()
+        else:
+            return self.read_operation()

utils/compilers/ConnectionCompiler/Lexer/reserved_symbols.py

@@ -0,0 +1,24 @@
+from type_file import Type
+
+reserved_symbols = {
+    '(': Type.Lang.LeftBracket,
+    ')': Type.Lang.RightBracket,
+    ';': Type.Lang.Semi,
+    '.': Type.Lang.Dot,
+    ',': Type.Lang.Comma,
+    ':': Type.Lang.Colon,
+
+    '=>': Type.Reserved.Return,
+
+    '{': Type.Reserved.Begin,
+    '}': Type.Reserved.End,
+}
+
+reserved_words = {
+    'NAME': Type.Reserved.Name,
+    'MAGIC': Type.Reserved.Magic,
+    'KEYPROTECTED': Type.Reserved.KeyProtected,
+}
+
+available_var_types = [Type.Number.Integer,
+                       Type.Number.Real, Type.Number.Boolean]