Source code for batools.assetsmakefile

# Released under the MIT License. See LICENSE for details.
#
"""Updates src/assets/Makefile based on source assets present."""

from __future__ import annotations

import json
import os
from typing import TYPE_CHECKING

from efrotools.pyver import PYVER

if TYPE_CHECKING:
    pass

ASSETS_SRC = 'src/assets'
BUILD_DIR = 'build/assets'


def _get_targets(
    projroot: str,
    varname: str,
    inext: str,
    outext: str,
    all_targets: set,
    limit_to_prefix: str | None = None,
) -> str:
    """Generic function to map source extension to dst files."""
    # pylint: disable=too-many-locals
    # pylint: disable=too-many-positional-arguments

    src = ASSETS_SRC
    dst = BUILD_DIR
    targets = []

    # Create outext targets for all inext files we find.
    for root, _dname, fnames in os.walk(os.path.join(projroot, src)):
        src_abs = os.path.join(projroot, src)
        if limit_to_prefix is not None and not root.startswith(
            os.path.join(src_abs, limit_to_prefix)
        ):
            continue

        # Write the target to make sense from within src/assets/
        assert root.startswith(src_abs)
        dstrootvar = '$(BUILD_DIR)' + root.removeprefix(src_abs)
        dstfin = dst + root.removeprefix(src_abs)
        for fname in fnames:
            outname = fname[: -len(inext)] + outext
            if fname.endswith(inext):
                all_targets.add(os.path.join(dstfin, outname))
                targets.append(os.path.join(dstrootvar, outname))

    return '\n' + varname + ' = \\\n  ' + ' \\\n  '.join(sorted(targets))


def _get_py_targets(
    projroot: str,
    meta_manifests: dict[str, str],
    explicit_sources: set[str],
    src: str,
    dst: str,
    py_targets: list[str],
    # pyc_targets: list[str],
    so_targets: list[str],
    all_targets: set[str],
    subset: str,
) -> None:
    # pylint: disable=too-many-positional-arguments
    # pylint: disable=too-many-branches
    # pylint: disable=too-many-locals
    # pylint: disable=too-many-statements

    py_generated_root = f'{ASSETS_SRC}/ba_data/python/babase/_mgen'

    def _do_get_targets(
        proot: str, fnames: list[str], is_explicit: bool = False
    ) -> None:
        # Special case: don't make targets for stuff in specific dirs.
        if proot in {
            f'{ASSETS_SRC}/ba_data/data/maps',
            f'{ASSETS_SRC}/mac_disk_image',
            f'{ASSETS_SRC}/workspace',
        }:
            return

        # Special case: exclude test modules.
        if f'/python{PYVER}/test/' in f'{proot}/':
            return

        assert proot.startswith(src), f'{proot} does not start with {src}'
        assert dst.startswith(BUILD_DIR)
        dstrootvar = (
            '$(BUILD_DIR)'
            + dst.removeprefix(BUILD_DIR)
            + proot.removeprefix(src)
        )
        dstfin = dst + proot[len(src) :]

        for fname in fnames:
            # Ignore non-python files and flycheck/emacs temp files.
            if (
                (not fname.endswith('.py') and not fname.endswith('.so'))
                or fname.startswith('flycheck_')
                or fname.startswith('.#')
            ):
                continue

            # Ignore any files in the list of explicit sources we got;
            # we explicitly add those at the end and don't want to do it
            # twice (since we don't know if this one will always exist
            # anyway).
            if (
                os.path.join(proot, fname) in explicit_sources
                and not is_explicit
            ):
                continue

            if proot.startswith(f'{ASSETS_SRC}/ba_data/python-site-packages'):
                in_subset = 'private-common'
            elif proot.startswith(f'{ASSETS_SRC}/ba_data') or proot.startswith(
                f'{ASSETS_SRC}/server'
            ):
                in_subset = 'public'
            elif proot.startswith('tools/efro') and not proot.startswith(
                'tools/efrotools'
            ):
                # We want to pull just 'efro' out of tools; not efrotools.
                in_subset = 'public_tools'
            elif proot.startswith('tools/bacommon'):
                in_subset = 'public_tools'
            elif proot.startswith(f'{ASSETS_SRC}/windows/x64'):
                in_subset = 'private-windows-x64'
            elif proot.startswith(f'{ASSETS_SRC}/windows/Win32'):
                in_subset = 'private-windows-Win32'
            elif proot.startswith(
                f'src/external/python-apple/macos/Python.xcframework/'
                f'macos-arm64_x86_64/Python.framework/'
                f'Versions/{PYVER}/lib/python{PYVER}'
            ):
                in_subset = 'private-apple-mac'
            # elif proot.startswith(f'{ASSETS_SRC}/pylib-apple'):
            #     in_subset = 'private-apple'
            elif proot.startswith(f'{ASSETS_SRC}/pylib-android'):
                in_subset = 'private-android'
            else:
                in_subset = 'private-common'

            if subset == 'all':
                pass
            elif subset != in_subset:
                continue

            if fname.endswith('.so'):
                # .so:
                targetpath = os.path.join(dstfin, fname)
                assert targetpath not in all_targets
                all_targets.add(targetpath)
                so_targets.append(os.path.join(dstrootvar, fname))
            else:
                # .py:
                targetpath = os.path.join(dstfin, fname)
                assert targetpath not in all_targets
                all_targets.add(targetpath)
                py_targets.append(os.path.join(dstrootvar, fname))

                # and .pyc:
                # fname_pyc = fname[:-3] + PYC_SUFFIX
                # all_targets.add(os.path.join(dstfin,
                # '__pycache__', fname_pyc))
                # pyc_targets.append(
                #     os.path.join(dstrootvar, '__pycache__', fname_pyc)
                # )

    # Create py and pyc targets for all physical scripts in src, with
    # the exception of our dynamically generated stuff.
    for physical_root, _dname, physical_fnames in os.walk(
        os.path.join(projroot, src)
    ):
        # Skip any generated files; we'll add those from the meta manifest.
        # (dont want our results to require a meta build beforehand)
        if physical_root == os.path.join(
            projroot, py_generated_root
        ) or physical_root.startswith(
            os.path.join(projroot, py_generated_root) + '/'
        ):
            continue

        _do_get_targets(
            physical_root.removeprefix(projroot + '/'), physical_fnames
        )

    # Now create targets for any of our dynamically generated stuff that
    # lives under this dir.
    meta_targets: list[str] = []
    for manifest in meta_manifests.values():
        # Sanity check; make sure meta system is giving actual paths;
        # no accidental makefile vars.
        if '$' in manifest:
            raise RuntimeError(
                'meta-manifest value contains a $; probably a bug.'
            )
        meta_targets += json.loads(manifest)

    meta_targets = [
        t
        for t in meta_targets
        if t.startswith(src + '/') and t.startswith(py_generated_root + '/')
    ]

    for target in meta_targets:
        _do_get_targets(
            proot=os.path.dirname(target), fnames=[os.path.basename(target)]
        )

    # Now create targets for any explicitly passed paths.
    for expsrc in explicit_sources:
        if expsrc.startswith(f'{src}/'):
            _do_get_targets(
                proot=os.path.dirname(expsrc),
                fnames=[os.path.basename(expsrc)],
                is_explicit=True,
            )


def _get_py_targets_subset(
    projroot: str,
    meta_manifests: dict[str, str],
    explicit_sources: set[str],
    all_targets: set[str],
    subset: str,
    suffix: str,
) -> str:
    # pylint: disable=too-many-positional-arguments

    copyrule_so: str | None = None

    # Map stuff from tools/ to build/assets/ba_data/python/
    if subset == 'public_tools':
        src = 'tools'
        dst = f'{BUILD_DIR}/ba_data/python'
        copyrule = '$(BUILD_DIR)/ba_data/python/%.py : $(TOOLS_DIR)/%.py'

    # Map stuff from mac python xcframework's lib dir to
    # build/assets/pylib-apple-mac
    elif subset == 'private-apple-mac':
        src = (
            f'src/external/python-apple/macos/Python.xcframework/'
            f'macos-arm64_x86_64/Python.framework/'
            f'Versions/{PYVER}/lib/python{PYVER}'
        )
        dst = f'{BUILD_DIR}/python-apple/macos/pylib'
        copyrule = (
            f'$(BUILD_DIR)/python-apple/macos/pylib/%.py :'
            f' $(SRC_DIR)/external/python-apple/macos/Python.xcframework/'
            f'macos-arm64_x86_64/Python.framework/'
            f'Versions/{PYVER}/lib/python{PYVER}/%.py'
        )
        copyrule_so = (
            f'$(BUILD_DIR)/python-apple/macos/pylib/%.so :'
            f' $(SRC_DIR)/external/python-apple/macos/Python.xcframework/'
            f'macos-arm64_x86_64/Python.framework/'
            f'Versions/{PYVER}/lib/python{PYVER}/%.so'
        )

    # Default - map stuff from src/assets/ to build/assets/
    else:
        src = ASSETS_SRC
        dst = BUILD_DIR
        copyrule = '$(BUILD_DIR)/%.py : %.py'

    # This could be a nice sanity check but some src paths aren't present
    # on various cloud-builds we do. Perhaps we could somehow only check
    # when we know everything is present?..
    if bool(False):
        if not os.path.exists(os.path.join(projroot, src)):
            raise RuntimeError(
                f'Expected src path not found in project: "{src}"'
            )

    py_targets: list[str] = []
    # pyc_targets: list[str] = []
    so_targets: list[str] = []

    _get_py_targets(
        projroot,
        meta_manifests,
        explicit_sources,
        src,
        dst,
        py_targets,
        # pyc_targets,
        so_targets,
        all_targets,
        subset=subset,
    )

    # Need to sort py and pyc combined to keep pairs together.
    # combined_targets = [
    #     (py_targets[i], pyc_targets[i]) for i in range(len(py_targets))
    # ]
    # combined_targets.sort()
    py_targets.sort()
    so_targets.sort()

    # py_targets = [t[0] for t in combined_targets]
    # pyc_targets = [t[1] for t in combined_targets]

    out = (
        f'\nSCRIPT_TARGETS_PY{suffix} = \\\n  '
        + ' \\\n  '.join(py_targets)
        + '\n'
    )

    # out += (
    #     f'\nSCRIPT_TARGETS_PYC{suffix} = \\\n  '
    #     + ' \\\n  '.join(pyc_targets)
    #     + '\n'
    # )

    out += (
        f'\nSCRIPT_TARGETS_SO{suffix} = \\\n  '
        + ' \\\n  '.join(so_targets)
        + '\n'
    )

    # We transform all non-public targets into efrocache-fetches in public.
    efc = '' if subset.startswith('public') else '# __EFROCACHE_TARGET__\n'

    out += (
        '\n# Rule to copy src asset scripts to dst.\n'
        '# (and make non-writable so I\'m less likely to '
        'accidentally edit them there)\n'
        f'{efc}$(SCRIPT_TARGETS_PY{suffix}) : {copyrule}\n'
        '\t@$(PCOMMANDBATCH) copy_python_file $^ $@\n'
    )

    if so_targets:
        assert copyrule_so is not None
        out += (
            '\n# Rule to copy src asset binary modules to dst.\n'
            '# (and make non-writable so I\'m less likely to '
            'accidentally edit them there)\n'
            f'{efc}$(SCRIPT_TARGETS_SO{suffix}) : {copyrule_so}\n'
            '\t@$(PCOMMANDBATCH) copy_python_file $^ $@\n'
        )

    # out += (
    #     '\n# Rule to copy src asset scripts to dst.\n'
    #     '# (and make non-writable so I\'m less likely to '
    #     'accidentally edit them there)\n'
    #     f'{efc}$(SCRIPT_TARGETS_PY{suffix}) : {copyrule}\n'
    #     '\t@echo Copying script: $(subst $(BUILD_DIR)/,,$@)\n'
    #     '\t@mkdir -p $(dir $@)\n'
    #     '\t@rm -f $@\n'
    #     '\t@cp $^ $@\n'
    #     '\t@chmod 444 $@\n'
    # )

    # Fancy new simple loop-based target generation.
    # out += (
    #     f'\n# These are too complex to define in a pattern rule;\n'
    #     f'# Instead we generate individual targets in a loop.\n'
    #     f'$(foreach element,$(SCRIPT_TARGETS_PYC{suffix}),\\\n'
    #     f'$(eval $(call make-opt-pyc-target,$(element))))'
    # )

    # Old code to explicitly emit individual targets.
    # if bool(False):
    #     out += (
    #         '\n# Looks like path mangling from py to pyc is too complex for'
    #         ' pattern rules so\n# just generating explicit targets'
    #         ' for each. Could perhaps look into using a\n# fancy for-loop'
    #         ' instead, but perhaps listing these explicitly isn\'t so bad.\n'
    #     )
    #     for i, target in enumerate(pyc_targets):
    #         # Note: there's currently a bug which can cause python bytecode
    #         # generation to be non-deterministic. This can break our blessing
    #         # process since we bless in core but then regenerate bytecode in
    #         # spinoffs. See https://bugs.python.org/issue34722
    #         # For now setting PYTHONHASHSEED=1 is a workaround.
    #         out += (
    #             '\n'
    #             + target
    #             + ': \\\n      '
    #             + py_targets[i]
    #             + '\n\t@echo Compiling script: $(subst $(BUILD_DIR),,$^)\n'
    #             '\t@rm -rf $@ && PYTHONHASHSEED=1 $(TOOLS_DIR)/pcommand'
    #             ' compile_python_file $^'
    #             ' && chmod 444 $@\n'
    #         )

    return out


def _get_extras_targets_win(
    projroot: str, all_targets: set[str], platform: str
) -> str:
    targets: list[str] = []
    base = f'{ASSETS_SRC}/windows'
    dstbase = 'windows'
    for root, _dnames, fnames in os.walk(os.path.join(projroot, base)):
        for fname in fnames:
            # Only include the platform we were passed.
            if not root.startswith(
                os.path.join(projroot, f'{ASSETS_SRC}/windows/{platform}')
            ):
                continue

            ext = os.path.splitext(fname)[-1]

            # "I don't like .DS_Store files. They're coarse and rough and
            # irritating and they get everywhere."
            if fname == '.DS_Store':
                continue

            # Ignore python files as they're handled separately.
            if ext in ['.py', '.pyc']:
                continue

            # Various stuff we expect to be there...
            if ext in [
                '.exe',
                '.dll',
                '.bat',
                '.txt',
                '.whl',
                '.ps1',
                '.css',
                '.sample',
                '.ico',
                '.pyd',
                '.ctypes',
                '.rst',
                '.fish',
                '.csh',
                '.cat',
                '.pdb',
                '.lib',
                '.html',
            ] or fname in [
                'activate',
                'README',
                'command_template',
                'fetch_macholib',
            ]:
                base_abs = os.path.join(projroot, base)
                assert root.startswith(base_abs)
                targetpath = os.path.join(
                    dstbase + root.removeprefix(base_abs), fname
                )
                # print(f'DSTBASE {dstbase} ROOT {root}
                # TARGETPATH {targetpath}')
                targets.append('$(BUILD_DIR)/' + targetpath)
                all_targets.add(BUILD_DIR + '/' + targetpath)
                continue

            # Complain if something new shows up instead of blindly
            # including it.
            raise RuntimeError(f'Unexpected extras file: {root}/{fname}')

    targets.sort()
    p_up = platform.upper()
    out = (
        f'\nEXTRAS_TARGETS_WIN_{p_up} = \\\n  ' + ' \\\n  '.join(targets) + '\n'
    )

    # We transform all these targets into efrocache-fetches in public.
    out += (
        '\n# Rule to copy src extras to build.\n'
        f'# __EFROCACHE_TARGET__\n'
        f'$(EXTRAS_TARGETS_WIN_{p_up}) : $(BUILD_DIR)/% :'
        ' %\n'
        '\t@$(PCOMMANDBATCH) copy_win_extra_file $^ $@\n'
        # '\t@echo Copying file: $(subst $(BUILD_DIR)/,,$@)\n'
        # '\t@mkdir -p $(dir $@)\n'
        # '\t@rm -f $@\n'
        # '\t@cp $^ $@\n'
    )

    return out


[docs] def generate_assets_makefile( projroot: str, fname: str, existing_data: str, meta_manifests: dict[str, str], explicit_sources: set[str], ) -> dict[str, str]: """Main script entry point.""" # pylint: disable=too-many-locals from efrotools.project import getprojectconfig from pathlib import Path public = getprojectconfig(Path(projroot))['public'] assert isinstance(public, bool) original = existing_data lines = original.splitlines() auto_start_public = lines.index('# __AUTOGENERATED_PUBLIC_BEGIN__') auto_end_public = lines.index('# __AUTOGENERATED_PUBLIC_END__') auto_start_private = lines.index('# __AUTOGENERATED_PRIVATE_BEGIN__') auto_end_private = lines.index('# __AUTOGENERATED_PRIVATE_END__') all_targets_public: set[str] = set() all_targets_private: set[str] = set() # We always auto-generate the public section. our_lines_public = [ _get_py_targets_subset( projroot, meta_manifests, explicit_sources, all_targets_public, subset='public', suffix='_PUBLIC', ), _get_py_targets_subset( projroot, meta_manifests, explicit_sources, all_targets_public, subset='public_tools', suffix='_PUBLIC_TOOLS', ), ] # Only auto-generate the private section in the private repo. if public: our_lines_private = lines[auto_start_private + 1 : auto_end_private] else: our_lines_private = [ _get_py_targets_subset( projroot, meta_manifests, explicit_sources, all_targets_private, subset='private-apple-mac', suffix='_PRIVATE_APPLE_MAC', ), _get_py_targets_subset( projroot, meta_manifests, explicit_sources, all_targets_private, subset='private-android', suffix='_PRIVATE_ANDROID', ), _get_py_targets_subset( projroot, meta_manifests, explicit_sources, all_targets_private, subset='private-common', suffix='_PRIVATE_COMMON', ), _get_py_targets_subset( projroot, meta_manifests, explicit_sources, all_targets_private, subset='private-windows-Win32', suffix='_PRIVATE_WIN_WIN32', ), _get_py_targets_subset( projroot, meta_manifests, explicit_sources, all_targets_private, subset='private-windows-x64', suffix='_PRIVATE_WIN_X64', ), _get_targets( projroot, 'COB_TARGETS', '.collisionmesh.obj', '.cob', all_targets_private, ), _get_targets( projroot, 'BOB_TARGETS', '.mesh.obj', '.bob', all_targets_private, ), _get_targets( projroot, 'FONT_TARGETS', '.fdata', '.fdata', all_targets_private, ), _get_targets( projroot, 'PEM_TARGETS', '.pem', '.pem', all_targets_private, ), _get_targets( projroot, 'DATA_TARGETS', '.json', '.json', all_targets_private, limit_to_prefix='ba_data/data', ), _get_targets( projroot, 'AUDIO_TARGETS', '.wav', '.ogg', all_targets_private, ), _get_targets( projroot, 'TEX2D_DDS_TARGETS', '.tex2d.png', '.dds', all_targets_private, ), _get_targets( projroot, 'TEX2D_PVR_TARGETS', '.tex2d.png', '.pvr', all_targets_private, ), _get_targets( projroot, 'TEX2D_KTX_TARGETS', '.tex2d.png', '.ktx', all_targets_private, ), _get_targets( projroot, 'TEX2D_PREVIEW_PNG_TARGETS', '.tex2d.png', '_preview.png', all_targets_private, ), _get_extras_targets_win(projroot, all_targets_private, 'Win32'), _get_extras_targets_win(projroot, all_targets_private, 'x64'), ] filtered = ( lines[: auto_start_public + 1] + our_lines_public + lines[auto_end_public : auto_start_private + 1] + our_lines_private + lines[auto_end_private:] ) out_files: dict[str, str] = {} out = '\n'.join(filtered) + '\n' out_files[fname] = out # Write a simple manifest of the things we expect to have in build. # We can use this to clear out orphaned files as part of builds. out_files['src/assets/.asset_manifest_public.json'] = _gen_manifest( all_targets_public ) # Only *generate* the private manifest in the private repo. In public # we just give what's already on disk. manprivpath = 'src/assets/.asset_manifest_private.json' if not public: out_files[manprivpath] = _gen_manifest(all_targets_private) return out_files
def _gen_manifest(all_targets: set[str]) -> str: # Lastly, write a simple manifest of the things we expect to have # in build. We can use this to clear out orphaned files as part of builds. assert all(t.startswith(BUILD_DIR) for t in all_targets) manifest = sorted(t[13:] for t in all_targets) return json.dumps(manifest, indent=1) # Docs-generation hack; import some stuff that we likely only forward-declared # in our actual source code so that docs tools can find it. from typing import (Coroutine, Any, Literal, Callable, Generator, Awaitable, Sequence, Self) import asyncio from concurrent.futures import Future from pathlib import Path from enum import Enum