scu_builders.py

"""Functions used to generate scu build source files during build time
"""

import glob, os
import math
from pathlib import Path
from os.path import normpath, basename

base_folder_path = str(Path(__file__).parent) + "/"
base_folder_only = os.path.basename(os.path.normpath(base_folder_path))
_verbose = False
_is_release_build = False
_scu_folders = set()


def clear_out_existing_files(output_folder, extension):
    output_folder = os.path.abspath(output_folder)
    # print("clear_out_existing_files from folder: " + output_folder)

    if not os.path.isdir(output_folder):
        # folder does not exist or has not been created yet,
        # no files to clearout. (this is not an error)
        return

    os.chdir(output_folder)

    for file in glob.glob("*." + extension):
        # print("removed pre-existing file: " + file)
        os.remove(file)


def folder_not_found(folder):
    abs_folder = base_folder_path + folder + "/"
    return not os.path.isdir(abs_folder)


def find_files_in_folder(folder, sub_folder, include_list, extension, sought_exceptions, found_exceptions):
    abs_folder = base_folder_path + folder + "/" + sub_folder

    if not os.path.isdir(abs_folder):
        print("ERROR " + abs_folder + " not found.")
        return include_list, found_exceptions

    os.chdir(abs_folder)

    sub_folder_slashed = ""
    if sub_folder != "":
        sub_folder_slashed = sub_folder + "/"

    for file in glob.glob("*." + extension):

        simple_name = Path(file).stem

        if file.endswith(".gen.cpp"):
            continue

        li = '#include "' + folder + "/" + sub_folder_slashed + file + '"'

        if not simple_name in sought_exceptions:
            include_list.append(li)
        else:
            found_exceptions.append(li)

    return include_list, found_exceptions


def write_output_file(file_count, include_list, start_line, end_line, output_folder, output_filename_prefix, extension):

    output_folder = os.path.abspath(output_folder)

    if not os.path.isdir(output_folder):
        # create
        os.mkdir(output_folder)
        if not os.path.isdir(output_folder):
            print("ERROR " + output_folder + " could not be created.")
            return
        print("CREATING folder " + output_folder)

    file_text = ""

    for l in range(start_line, end_line):
        if l < len(include_list):
            line = include_list[l]
            li = line + "\n"
            file_text += li

    num_string = ""
    if file_count > 0:
        num_string = "_" + str(file_count)

    short_filename = output_filename_prefix + num_string + ".gen." + extension
    output_filename = output_folder + "/" + short_filename
    if _verbose:
        print("generating: " + short_filename)

    output_path = Path(output_filename)
    output_path.write_text(file_text, encoding="utf8")


def write_exception_output_file(file_count, exception_string, output_folder, output_filename_prefix, extension):
    output_folder = os.path.abspath(output_folder)
    if not os.path.isdir(output_folder):
        print("ERROR " + output_folder + " does not exist.")
        return

    file_text = exception_string + "\n"

    num_string = ""
    if file_count > 0:
        num_string = "_" + str(file_count)

    short_filename = output_filename_prefix + "_exception" + num_string + ".gen." + extension
    output_filename = output_folder + "/" + short_filename

    if _verbose:
        print("generating: " + short_filename)

    output_path = Path(output_filename)
    output_path.write_text(file_text, encoding="utf8")


def find_section_name(sub_folder):
    # Construct a useful name for the section from the path for debug logging
    section_path = os.path.abspath(base_folder_path + sub_folder) + "/"

    folders = []
    folder = ""

    for i in range(8):
        folder = os.path.dirname(section_path)
        folder = os.path.basename(folder)
        if folder == base_folder_only:
            break
        folders.append(folder)
        section_path += "../"
        section_path = os.path.abspath(section_path) + "/"

    section_name = ""
    for n in range(len(folders)):
        section_name += folders[len(folders) - n - 1]
        if n != (len(folders) - 1):
            section_name += "_"

    return section_name


# "folders" is a list of folders to add all the files from to add to the SCU
# "section (like a module)". The name of the scu file will be derived from the first folder
# (thus e.g. scene/3d becomes scu_scene_3d.gen.cpp)

# "includes_per_scu" limits the number of includes in a single scu file.
# This allows the module to be built in several translation units instead of just 1.
# This will usually be slower to compile but will use less memory per compiler instance, which
# is most relevant in release builds.

# "sought_exceptions" are a list of files (without extension) that contain
# e.g. naming conflicts, and are therefore not suitable for the scu build.
# These will automatically be placed in their own separate scu file,
# which is slow like a normal build, but prevents the naming conflicts.
# Ideally in these situations, the source code should be changed to prevent naming conflicts.


# "extension" will usually be cpp, but can also be set to c (for e.g. third party libraries that use c)
def process_folder(folders, sought_exceptions=[], includes_per_scu=0, extension="cpp"):
    if len(folders) == 0:
        return

    # Construct the filename prefix from the FIRST folder name
    # e.g. "scene_3d"
    out_filename = find_section_name(folders[0])

    found_includes = []
    found_exceptions = []

    main_folder = folders[0]
    abs_main_folder = base_folder_path + main_folder

    # Keep a record of all folders that have been processed for SCU,
    # this enables deciding what to do when we call "add_source_files()"
    global _scu_folders
    _scu_folders.add(main_folder)

    # main folder (first)
    found_includes, found_exceptions = find_files_in_folder(
        main_folder, "", found_includes, extension, sought_exceptions, found_exceptions
    )

    # sub folders
    for d in range(1, len(folders)):
        found_includes, found_exceptions = find_files_in_folder(
            main_folder, folders[d], found_includes, extension, sought_exceptions, found_exceptions
        )

    found_includes = sorted(found_includes)

    # calculate how many lines to write in each file
    total_lines = len(found_includes)

    # adjust number of output files according to whether DEV or release
    num_output_files = 1
    if _is_release_build:
        # always have a maximum in release
        includes_per_scu = 8
        num_output_files = max(math.ceil(total_lines / includes_per_scu), 1)
    else:
        if includes_per_scu > 0:
            num_output_files = max(math.ceil(total_lines / includes_per_scu), 1)

    # error condition
    if total_lines == 0:
        return

    lines_per_file = math.ceil(total_lines / num_output_files)
    lines_per_file = max(lines_per_file, 1)

    start_line = 0
    file_number = 0

    # These do not vary throughout the loop
    output_folder = abs_main_folder + "/.scu/"
    output_filename_prefix = "scu_" + out_filename

    # Clear out any existing files (usually we will be overwriting,
    # but we want to remove any that are pre-existing that will not be
    # overwritten, so as to not compile anything stale)
    clear_out_existing_files(output_folder, extension)

    for file_count in range(0, num_output_files):
        end_line = start_line + lines_per_file

        # special case to cover rounding error in final file
        if file_count == (num_output_files - 1):
            end_line = len(found_includes)

        write_output_file(
            file_count, found_includes, start_line, end_line, output_folder, output_filename_prefix, extension
        )

        start_line = end_line

    # Write the exceptions each in their own scu gen file,
    # so they can effectively compile in "old style / normal build".
    for exception_count in range(len(found_exceptions)):
        write_exception_output_file(
            exception_count, found_exceptions[exception_count], output_folder, output_filename_prefix, extension
        )


def generate_scu_files(verbose, is_release_build):

    print("=============================")
    print("Single Compilation Unit Build")
    print("=============================")
    print("Generating SCU build files")
    global _verbose
    _verbose = verbose
    global _is_release_build
    _is_release_build = is_release_build

    curr_folder = os.path.abspath("./")

    # check we are running from the correct folder
    if folder_not_found("core") or folder_not_found("platform") or folder_not_found("scene"):
        raise RuntimeError("scu_builders.py must be run from the godot folder.")
        return

    process_folder(["core"])
    process_folder(["core/bind"])
    process_folder(["core/config"])
    process_folder(["core/containers"])
    process_folder(["core/error"])
    process_folder(["core/input"])
    process_folder(["core/log"])
    process_folder(["core/math"])
    process_folder(["core/object"])
    process_folder(["core/os"])
    process_folder(["core/string"])
    process_folder(["core/variant"])
    process_folder(["core/io"])
    process_folder(["core/crypto"])

    process_folder(["drivers/gles2"])
    process_folder(["drivers/unix"])
    process_folder(["drivers/png"])

    process_folder(["editor"])
    process_folder(["editor/import"])
    process_folder(["editor/plugins"])

    process_folder(["main"])
    process_folder(["main/tests"])

    process_folder(
        [
            "platform",
            "android/export",
            "iphone/export",
            "javascript/export",
            "osx/export",
            #"uwp/export",
            "windows/export",
            "x11/export",
        ]
    )

    #TODO These should be moved to module conpig.py s

    process_folder(["modules/broken_seals_module"])
    process_folder(["modules/cscript"])
    process_folder(["modules/csg"])

    process_folder(["modules/database"])
    process_folder(["modules/database_sqlite"])

    process_folder(["modules/entity_spell_system",
        "data/atlases",
        "data/auras",
        "data/items",
        "data/loot",
        "data/species",
        "data/spells",
        "database",
        "drag_and_drop",
        "editor",
        "entities",
        "entities/ai",
        "entities/auras",
        "entities/data",
        "entities/resources",
        "entities/skills",
        "entities/stats",
        "formations",
        "infos",
        "inventory",
        "material_cache",
        "pipelines",
        "profiles",
        "profiles/actionbar",
        "profiles/input",
        "projectiles/3d",
        "props",
        "singletons",
        "skeleton",
        "spawners",
        "utility",
    ])

    process_folder(["modules/fastnoise"])

    process_folder(["modules/gdnative",
        "gdnative",
        "nativescript",
        #"arvr",
        "pluginscript",
        "net",
    ])

    process_folder(["modules/gdscript"])
    process_folder(["modules/gdscript/language_server"])

    process_folder(["modules/gridmap"])
    process_folder(["modules/gridmap/geometry_parser"])

    process_folder(["modules/http_server_simple"])

    process_folder(["modules/lz4"])

    process_folder(["modules/material_maker",
        "algos",
        "editor",
        "editor/widgets/color_picker_popup",
        "editor/widgets/curve_edit",
        "editor/widgets/file_dialog",
        "editor/widgets/float_edit",
        "editor/widgets/gradient_editor",
        "editor/widgets/image_picker_button",
        "editor/widgets/mm_dnd_color_picker_button",
        "editor/widgets/polygon_edit",
        "editor/widgets/tones_editor",
        "nodes",
        "nodes/bases",
        "nodes/filter",
        "nodes/gradient",
        "nodes/noise",
        "nodes/other",
        "nodes/pattern",
        "nodes/sdf2d",
        "nodes/sdf3d",
        "nodes/simple",
        "nodes/transform",
        "nodes/uniform",
    ])

    process_folder(["modules/mbedtls"])

    process_folder(["modules/mesh_data_resource",
        "editor",
        "editor/utilities",
        "editor/uv_editor",
        "nodes",
        "plugin",
        "plugin_gltf",
        "props",
        "props_2d",
    ])

    process_folder(["modules/mesh_utils"])

    process_folder(["modules/navigation"])
    process_folder(["modules/navigation_dummy"])
    process_folder(["modules/navigation_geometry_parsers",
        "geometry_parser_2d",
        "geometry_parser_3d",
    ])
    process_folder(["modules/navigation_mesh_generator"])
    process_folder(["modules/navigation_mesh_generator/editor"])

    process_folder(["modules/network_synchronizer"])

    process_folder(["modules/paint",
        "actions",
        "editor",
        "nodes",
        "ui",
        "ui/property_inspectors",
    ])

    process_folder(["modules/props",
        "clutter",
        "editor",
        "jobs",
        "lights",
        "material_cache",
        "props",
        "singleton",
        "tiled_wall",
    ])

    process_folder(["modules/props_2d",
        "clutter",
        "editor",
        "jobs",
        "lights",
        "material_cache",
        "props",
        "singleton",
        "tiled_wall",
    ])

    process_folder(["modules/skeleton_2d",
        "editor",
        "nodes",
        "resources",
    ])
    process_folder(["modules/skeleton_3d",
        "editor",
        "nodes",
        "resources",
    ])

    process_folder(["modules/steering_ai",
        "agents",
        "behaviors",
        "proximities",
    ])

    process_folder(["modules/terraman",
        "areas",
        "data",
        "level_generator",
        "library",
        "meshers",
        "meshers/blocky",
        "meshers/default",
        "nodes",
        "world",
        "world/blocky",
        "world/default",
        "world/jobs",
    ])

    process_folder(["modules/terraman_2d",
        "areas",
        "data",
        "level_generator",
        "library",
        "meshers",
        "meshers/default",
        "meshers/isometric",
        "meshers/simple",
        "nodes",
        "world",
        "world/default",
        "world/isometric",
        "world/jobs",
        "world/simple",
    ])

    process_folder(["modules/texture_packer",
        "layers",
        "rectpack2D",
        "texture_resource",
    ])

    process_folder(["modules/tile_map",
        "geometry_parser",
    ])

    process_folder(["modules/ui_extensions"])

    process_folder(["modules/unit_test"])

    process_folder(["modules/users",
        "managers",
        "singleton",
        "users",
        "web/middleware",
        "web/web_nodes",
    ])

    process_folder(["modules/voxelman",
        "areas",
        "data",
        "level_generator",
        "library",
        "meshers",
        "meshers/blocky",
        "meshers/cubic",
        "meshers/default",
        "meshers/marching_cubes",
        "nodes",
        "world",
        "world/blocky",
        "world/cubic",
        "world/default",
        "world/jobs",
        "world/marching_cubes",
    ])

    process_folder(["modules/web",
        "database",
        "editor",
        "html",
        "http",
        "nodes/admin_panel",
        "nodes/folder_serve_nodes",
        "nodes/list_page",
        "nodes/message_page",
        "nodes/paged_article",
        "nodes/redirect",
        "nodes/static_pages",
    ])

    process_folder(["modules/websocket"])

    process_folder(["modules/wfc"])

    #Editor Modules
    process_folder(["editor_modules/editor_code_editor"])

    process_folder(["editor_modules/gltf"])
    process_folder(["editor_modules/gltf/structures"])
    process_folder(["editor_modules/gltf/extensions"])
    process_folder(["editor_modules/gltf/extensions/physics"])

    process_folder(["editor_modules/plugin_refresher"])

    process_folder(["editor_modules/shader_editor"])
    process_folder(["editor_modules/text_editor"])

    #process_folder(["modules/fbx"])
    #process_folder(["modules/fbx/tools"])
    #process_folder(["modules/fbx/data"])
    #process_folder(["modules/fbx/fbx_parser"])

    process_folder(["scene"])
    process_folder(["scene/audio"])
    process_folder(["scene/debugger"])
    process_folder(["scene/2d"])
    process_folder(["scene/3d"])
    process_folder(["scene/animation"])
    process_folder(["scene/gui"])
    process_folder(["scene/main"])
    process_folder(["scene/resources"])

    process_folder(["servers"])
    process_folder(["servers/rendering"])
    process_folder(["servers/rendering/portals"])
    process_folder(["servers/physics_2d"])
    process_folder(["servers/physics"])
    process_folder(["servers/physics/joints"])
    process_folder(["servers/audio"])
    process_folder(["servers/audio/effects"])

    # Finally change back the path to the calling folder
    os.chdir(curr_folder)

    return _scu_folders