Squash commits for public release

utils/codeassistant/clang_tidy.py

@@ -0,0 +1,143 @@
+#!/usr/bin/env python3
+
+import subprocess
+import os
+import sys
+
+target = "all"  # all, kernel, userland
+target_arch = "x86_64"
+
+
+class ClassTidyLauncher():
+
+    backend_flags = {
+        "x86": ["-c", "-m32",
+                "-D_LIBCXX_BUILD_XOS_EXTENSIONS"],
+        "arm32": [
+            "-fno-builtin",
+            "-march=armv7-a",
+            "-mfpu=neon-vfpv4",
+            "-mfloat-abi=soft",
+            "-D_LIBCXX_BUILD_XOS_EXTENSIONS",
+        ],
+        "arm64": [
+            "-fno-builtin",
+            "-mcpu=cortex-a53+nofp+nosimd+nocrypto+nocrc",
+            "-D_LIBCXX_BUILD_XOS_EXTENSIONS",
+        ],
+        "x86_64": [
+            "-c",
+            "-D_LIBCXX_BUILD_XOS_EXTENSIONS",
+        ]
+    }
+
+    def __init__(self, dir, includes):
+        self.path_dir = dir
+        self.include = includes
+        self.front_flags = ["--use-color", "--fix"]
+        self.back_flags = self.backend_flags[target_arch]
+
+    def run_clang_tidy(self, ff, files, bf):
+        cmd = ["clang-tidy"]
+        cmd.extend(ff)
+        cmd.extend(files)
+        cmd.extend(["--"])
+        cmd.extend(bf)
+        result = subprocess.run(cmd, stdout=subprocess.PIPE)
+        return result.stdout
+
+    def process_includes(self):
+        for i in self.include:
+            self.back_flags.append("-I")
+            self.back_flags.append(i)
+
+    def get_files(self):
+        self.c_files = []
+        self.cpp_files = []
+
+        platforms = ['x86', 'i386', 'x86_64', 'arm',
+                     'arm32', 'arm64', 'aarch32', 'aarch64']
+        bits = ['bits32', 'bits64']
+
+        platform_to_bits = {
+            "x86": "bits32",
+            "x86_64": "bits64",
+            "arm32": "bits32",
+            "arm64": "bits64",
+        }
+
+        allowed_paths = {
+            "x86": ["x86", "i386"],
+            "x86_64": ["x86", "x86_64"],
+            "arm32": ["aarch32", "arm32", "arm"],
+            "arm64": ["aarch64", "arm64", "arm"],
+        }
+
+        ignore_platforms = []
+        ignore_bits = []
+
+        allowed_paths_for_target = allowed_paths.get(target_arch, None)
+        if allowed_paths_for_target is None:
+            print("Unknown platform {0}".format(target_arch))
+            exit(1)
+
+        for platform in platforms:
+            if not (platform in allowed_paths_for_target):
+                ignore_platforms.append(platform)
+
+        for bit in bits:
+            if platform_to_bits[target_arch] != bit:
+                ignore_bits.append(bit)
+
+        def is_file_type(name, ending):
+            if len(name) <= len(ending):
+                return False
+            return (name[-len(ending)-1::] == '.'+ending)
+
+        def is_file_blocked(name):
+            for platform in ignore_platforms:
+                if (name.find(platform) != -1):
+                    return True
+            for bit in ignore_bits:
+                if (name.find(bit) != -1):
+                    return True
+            return False
+
+        for path, subdirs, files in os.walk(self.path_dir):
+            for name in files:
+                # It runs from out dir, at least it should
+                file = path + "/" + name
+                if is_file_blocked(file):
+                    continue
+                if is_file_type(file, 'c'):
+                    self.c_files.append(file)
+                if is_file_type(file, 'cpp'):
+                    self.cpp_files.append(file)
+
+    def process(self):
+        self.process_includes()
+        self.get_files()
+        self.c_back_flags = self.back_flags
+        self.c_back_flags += ["-std=gnu98"]
+        ret = ""
+        if len(self.c_files) > 0:
+            ret += self.run_clang_tidy(self.front_flags,
+                                       self.c_files, self.c_back_flags).decode("ascii")
+        self.cpp_back_flags = self.back_flags
+        self.cpp_back_flags += ["-std=c++2a"]
+        if len(self.cpp_files) > 0:
+            ret += self.run_clang_tidy(self.front_flags,
+                                       self.cpp_files, self.cpp_back_flags).decode("ascii")
+        return ret
+
+
+kernel_includes = ["kernel/include"]
+app_includes = ["libs/libc/include", "libs/libcxx/include", "libs/libfoundation/include",
+                "libs/libipc/include", "libs/libg/include", "libs/libui/include"]
+
+if target == "all" or target == "kernel":
+    print(ClassTidyLauncher("kernel/kernel", kernel_includes).process())
+
+if target == "all" or target == "userland":
+    print(ClassTidyLauncher("servers/", app_includes).process())
+    print(ClassTidyLauncher("libs/", app_includes).process())

utils/codeassistant/include_guards_fix.py

@@ -0,0 +1,63 @@
+#!/usr/bin/env python
+# Launch the script from root of the project to have the correct paths
+
+import os
+import sys
+from os import fdopen, remove
+
+walk_dir = sys.argv[1]
+
+print('walk_dir = ' + walk_dir)
+print('walk_dir (absolute) = ' + os.path.abspath(walk_dir))
+
+all_includes = []
+
+
+def is_guard(line):
+    if line.startswith("#ifndef _"):
+        return True
+    return False
+
+
+def get_guard(line):
+    return line[8:-1]
+
+
+def new_guard(line, path):
+    gen = path.split('/')
+    gen = list(filter(lambda a: a != "libs", gen))
+    gen = list(filter(lambda a: a != "include", gen))
+    line = "_"
+    for l in gen:
+        line += l + "_"
+    line = line.replace(".", "_")
+    line = line.replace("-", "_")
+    line = line.upper()
+    return line[:-1]
+
+
+def fix_guards(file):
+    print("prc ", file)
+    data = []
+    guard = None
+    with open(file) as old_file:
+        for line in old_file:
+            data.append(line)
+            if is_guard(line) and guard is None:
+                guard = get_guard(line)
+
+    if guard is None:
+        return
+
+    ng = new_guard(guard, file)
+
+    with open(file, 'w') as new_file:
+        for i in data:
+            i = i.replace(guard, ng)
+            new_file.write(i)
+
+
+for root, subdirs, files in os.walk(walk_dir):
+    for x in files:
+        if x.endswith(".h") or x.endswith(".hpp") or root.find("/libcxx/include") != -1:
+            fix_guards(os.path.join(root, x))

utils/codeassistant/libkern_libc_compat.py

@@ -0,0 +1,92 @@
+#!/usr/bin/env python
+# The tool checks the compatability of linkern and libc bits/ structs
+
+import sys
+import os
+
+
+def list_of_files_relative(opath):
+    res = []
+
+    def is_file_type(name, ending):
+        if len(name) <= len(ending):
+            return False
+        return (name[-len(ending)-1::] == '.'+ending)
+
+    for path, subdirs, files in os.walk(opath):
+        for name in files:
+            # It runs from out dir, at least it should
+            file = path + "/" + name
+            if is_file_type(file, 'h'):
+                res.append(file[len(opath):])
+    return res
+
+
+def process_file(file, res_map):
+    def accept_line(line):
+        if line.endswith("\n"):
+            line = line[:-1]
+
+        if (len(line) == 0):
+            return False
+
+        block = [
+            "#include",
+            "#ifndef",
+            "#endif",
+            "#define _KERNEL_LIBKERN",
+            "#define _LIBC",
+        ]
+
+        for b in block:
+            if line.startswith(b):
+                return False
+
+        return True
+
+    with open(file) as ofile:
+        for line in ofile:
+            if line.endswith("\n"):
+                line = line[:-1]
+            if accept_line(line):
+                if line in res_map:
+                    res_map[line] += 1
+                else:
+                    res_map[line] = 1
+
+
+def create_map_of_lines(files):
+    res_map = {}
+    for f in files:
+        process_file(f, res_map)
+    return res_map
+
+
+def check_files(pbase, pslave, files):
+    filesbase = [pbase+x for x in files]
+    filesslave = [pslave+x for x in files]
+    libkern_map = create_map_of_lines(filesbase)
+    libc_map = create_map_of_lines(filesslave)
+
+    for i, x in libkern_map.items():
+        if i in libc_map:
+            if x != libc_map[i]:
+                return False
+        else:
+            print("Can't find {0} in LibC".format(i))
+            return False
+
+    return True
+
+
+libkern_files = list_of_files_relative("kernel/include/libkern/bits")
+libc_files = list_of_files_relative("libs/libc/include/bits")
+
+if len(libkern_files) != len(libc_files):
+    print("Note: LibC and LibKern might not be compatible, taking LibKern as base")
+
+if check_files("kernel/include/libkern/bits", "libs/libc/include/bits", libkern_files):
+    print("OK")
+else:
+    print("Failed")
+    exit(1)

utils/codeassistant/pongo_startup.py

@@ -0,0 +1,40 @@
+import os
+import subprocess
+from git import Repo
+
+rootdir = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
+pongodir = rootdir + "/.cached/pongoOS"
+
+if not os.path.exists(pongodir):
+    url = "https://github.com/xOS-Project/pongoOS.git"
+    print("Cloning pongoOS to ", pongodir)
+    Repo.clone_from(url, pongodir)
+
+
+def run_command(cmd, cwd="."):
+    result = subprocess.run(
+        cmd, stdout=subprocess.PIPE, shell=True, cwd=cwd, env=os.environ.copy())
+    return (result.stdout.decode("ascii"), result.returncode)
+
+
+print("Rebuilding pongoOS")
+run_command("make -j16", pongodir)
+
+print("Ready: pongoOS")
+checkrain_exec = os.environ.get('CHECKRAIN')
+if checkrain_exec is None:
+    print("Error: No provided $CHECKRAIN env variable.")
+    print("Please get a checkrain binary at http://checkra.in/ and set $CHECKRAIN to the binary path.")
+    print("")
+    print("E.g on macOS after getting binary and installing it to /Application, run:")
+    print("\texport CHECKRAIN=/Applications/checkra1n.app/Contents/MacOS/checkra1n")
+    exit(1)
+
+print("Checkrain is found. Connect your device and switch it to DFU mode.")
+run_command("$CHECKRAIN -k " + pongodir +
+            "/build/PongoConsolidated.bin -cpE", pongodir)
+
+xos_outdir = os.path.dirname(rootdir) + "/out/"
+pathrun = "python3 scripts/load_xos.py -k {0}/rawImage.bin -r {0}/one.img".format(
+    xos_outdir)
+run_command(pathrun, pongodir)

utils/codeassistant/recompile_connections.py

@@ -0,0 +1,21 @@
+#!/usr/bin/env python3
+# Launch the script from root of the project to have the correct paths
+
+import subprocess
+import os
+import sys
+
+connections = [
+    ["libs/libapi/includes/libapi/window_server/Connections/ws_connection.ipc",
+        "libs/libapi/includes/libapi/window_server/Connections/WSConnection.h"],
+]
+
+for conn in connections:
+    inf = conn[0]
+    outf = conn[1]
+    print("Compiling {0} -> {1}", inf, outf)
+    cmd = ["utils/compilers/ConnectionCompiler/connc"]
+    cmd.extend([inf, outf])
+    result = subprocess.run(cmd, stdout=subprocess.PIPE)
+    print(result.stdout.decode("ascii"))
+    print()

utils/codeassistant/syscall_parser.py

@@ -0,0 +1,33 @@
+import requests
+import lxml.html as lh
+import pandas as pd
+import argparse
+
+parser = argparse.ArgumentParser()
+parser.add_argument('arch', type=str, help='Arch [arm, x86]')
+args = parser.parse_args()
+
+url='https://chromium.googlesource.com/chromiumos/docs/+/master/constants/syscalls.md'
+page = requests.get(url)
+doc = lh.fromstring(page.content)
+tr_elements = doc.xpath('//tr')
+
+start = 0
+
+for i, tr in enumerate(tr_elements):
+    if tr[0].text_content() == "syscall name":
+        start = i + 1
+
+
+data = []
+for tr in tr_elements[start:]:
+    table_col = 2
+    if args.arch == "x86":
+        table_col = 4
+    if tr[table_col].text_content() != "-":
+        data.append([tr[0].text_content(), int(tr[table_col].text_content())])
+
+data.sort(key=lambda x: x[1])
+
+for i in data:
+    print("SYS_{0} = {1},".format(i[0].upper(), i[1]))