# Released under the MIT License. See LICENSE for details.
#
"""Updates src/assets/Makefile based on source assets present."""
from __future__ import annotations
import json
import os
from typing import TYPE_CHECKING
from efrotools.pyver import PYVERNODOT
if TYPE_CHECKING:
pass
PYC_SUFFIX = f'.cpython-{PYVERNODOT}.opt-1.pyc'
ASSETS_SRC = 'src/assets'
BUILD_DIR = 'build/assets'
def _get_targets(
projroot: str,
varname: str,
inext: str,
outext: str,
all_targets: set,
limit_to_prefix: str | None = None,
) -> str:
"""Generic function to map source extension to dst files."""
# pylint: disable=too-many-locals
# pylint: disable=too-many-positional-arguments
src = ASSETS_SRC
dst = BUILD_DIR
targets = []
# Create outext targets for all inext files we find.
for root, _dname, fnames in os.walk(os.path.join(projroot, src)):
src_abs = os.path.join(projroot, src)
if limit_to_prefix is not None and not root.startswith(
os.path.join(src_abs, limit_to_prefix)
):
continue
# Write the target to make sense from within src/assets/
assert root.startswith(src_abs)
dstrootvar = '$(BUILD_DIR)' + root.removeprefix(src_abs)
dstfin = dst + root.removeprefix(src_abs)
for fname in fnames:
outname = fname[: -len(inext)] + outext
if fname.endswith(inext):
all_targets.add(os.path.join(dstfin, outname))
targets.append(os.path.join(dstrootvar, outname))
return '\n' + varname + ' = \\\n ' + ' \\\n '.join(sorted(targets))
def _get_py_targets(
projroot: str,
meta_manifests: dict[str, str],
explicit_sources: set[str],
src: str,
dst: str,
py_targets: list[str],
pyc_targets: list[str],
all_targets: set[str],
subset: str,
) -> None:
# pylint: disable=too-many-positional-arguments
# pylint: disable=too-many-branches
# pylint: disable=too-many-locals
# pylint: disable=too-many-statements
py_generated_root = f'{ASSETS_SRC}/ba_data/python/babase/_mgen'
def _do_get_targets(
proot: str, fnames: list[str], is_explicit: bool = False
) -> None:
# Special case: don't make targets for stuff in specific dirs.
if proot in {
f'{ASSETS_SRC}/ba_data/data/maps',
f'{ASSETS_SRC}/mac_disk_image',
f'{ASSETS_SRC}/workspace',
}:
return
assert proot.startswith(src), f'{proot} does not start with {src}'
assert dst.startswith(BUILD_DIR)
dstrootvar = (
'$(BUILD_DIR)'
+ dst.removeprefix(BUILD_DIR)
+ proot.removeprefix(src)
)
dstfin = dst + proot[len(src) :]
for fname in fnames:
# Ignore non-python files and flycheck/emacs temp files.
if (
not fname.endswith('.py')
or fname.startswith('flycheck_')
or fname.startswith('.#')
):
continue
# Ignore any files in the list of explicit sources we got;
# we explicitly add those at the end and don't want to do it
# twice (since we don't know if this one will always exist
# anyway).
if (
os.path.join(proot, fname) in explicit_sources
and not is_explicit
):
continue
if proot.startswith(f'{ASSETS_SRC}/ba_data/python-site-packages'):
in_subset = 'private-common'
elif proot.startswith(f'{ASSETS_SRC}/ba_data') or proot.startswith(
f'{ASSETS_SRC}/server'
):
in_subset = 'public'
elif proot.startswith('tools/efro') and not proot.startswith(
'tools/efrotools'
):
# We want to pull just 'efro' out of tools; not efrotools.
in_subset = 'public_tools'
elif proot.startswith('tools/bacommon'):
in_subset = 'public_tools'
elif proot.startswith(f'{ASSETS_SRC}/windows/x64'):
in_subset = 'private-windows-x64'
elif proot.startswith(f'{ASSETS_SRC}/windows/Win32'):
in_subset = 'private-windows-Win32'
elif proot.startswith(f'{ASSETS_SRC}/pylib-apple'):
in_subset = 'private-apple'
elif proot.startswith(f'{ASSETS_SRC}/pylib-android'):
in_subset = 'private-android'
else:
in_subset = 'private-common'
if subset == 'all':
pass
elif subset != in_subset:
continue
# gamedata pass includes only data; otherwise do all else
# .py:
targetpath = os.path.join(dstfin, fname)
assert targetpath not in all_targets
all_targets.add(targetpath)
py_targets.append(os.path.join(dstrootvar, fname))
# and .pyc:
fname_pyc = fname[:-3] + PYC_SUFFIX
all_targets.add(os.path.join(dstfin, '__pycache__', fname_pyc))
pyc_targets.append(
os.path.join(dstrootvar, '__pycache__', fname_pyc)
)
# Create py and pyc targets for all physical scripts in src, with
# the exception of our dynamically generated stuff.
for physical_root, _dname, physical_fnames in os.walk(
os.path.join(projroot, src)
):
# Skip any generated files; we'll add those from the meta manifest.
# (dont want our results to require a meta build beforehand)
if physical_root == os.path.join(
projroot, py_generated_root
) or physical_root.startswith(
os.path.join(projroot, py_generated_root) + '/'
):
continue
_do_get_targets(
physical_root.removeprefix(projroot + '/'), physical_fnames
)
# Now create targets for any of our dynamically generated stuff that
# lives under this dir.
meta_targets: list[str] = []
for manifest in meta_manifests.values():
# Sanity check; make sure meta system is giving actual paths;
# no accidental makefile vars.
if '$' in manifest:
raise RuntimeError(
'meta-manifest value contains a $; probably a bug.'
)
meta_targets += json.loads(manifest)
meta_targets = [
t
for t in meta_targets
if t.startswith(src + '/') and t.startswith(py_generated_root + '/')
]
for target in meta_targets:
_do_get_targets(
proot=os.path.dirname(target), fnames=[os.path.basename(target)]
)
# Now create targets for any explicitly passed paths.
for expsrc in explicit_sources:
if expsrc.startswith(f'{src}/'):
_do_get_targets(
proot=os.path.dirname(expsrc),
fnames=[os.path.basename(expsrc)],
is_explicit=True,
)
def _get_py_targets_subset(
projroot: str,
meta_manifests: dict[str, str],
explicit_sources: set[str],
all_targets: set[str],
subset: str,
suffix: str,
) -> str:
# pylint: disable=too-many-locals
# pylint: disable=too-many-positional-arguments
if subset == 'public_tools':
src = 'tools'
dst = f'{BUILD_DIR}/ba_data/python'
copyrule = '$(BUILD_DIR)/ba_data/python/%.py : $(TOOLS_DIR)/%.py'
else:
src = ASSETS_SRC
dst = BUILD_DIR
copyrule = '$(BUILD_DIR)/%.py : %.py'
# Separate these into '1' and '2'.
py_targets: list[str] = []
pyc_targets: list[str] = []
_get_py_targets(
projroot,
meta_manifests,
explicit_sources,
src,
dst,
py_targets,
pyc_targets,
all_targets,
subset=subset,
)
# Need to sort these combined to keep pairs together.
combined_targets = [
(py_targets[i], pyc_targets[i]) for i in range(len(py_targets))
]
combined_targets.sort()
py_targets = [t[0] for t in combined_targets]
pyc_targets = [t[1] for t in combined_targets]
out = (
f'\nSCRIPT_TARGETS_PY{suffix} = \\\n '
+ ' \\\n '.join(py_targets)
+ '\n'
)
out += (
f'\nSCRIPT_TARGETS_PYC{suffix} = \\\n '
+ ' \\\n '.join(pyc_targets)
+ '\n'
)
# We transform all non-public targets into efrocache-fetches in public.
efc = '' if subset.startswith('public') else '# __EFROCACHE_TARGET__\n'
out += (
'\n# Rule to copy src asset scripts to dst.\n'
'# (and make non-writable so I\'m less likely to '
'accidentally edit them there)\n'
f'{efc}$(SCRIPT_TARGETS_PY{suffix}) : {copyrule}\n'
# '#\t@echo Copying script: $(subst $(BUILD_DIR)/,,$@)\n'
'\t@$(PCOMMANDBATCH) copy_python_file $^ $@\n'
)
# out += (
# '\n# Rule to copy src asset scripts to dst.\n'
# '# (and make non-writable so I\'m less likely to '
# 'accidentally edit them there)\n'
# f'{efc}$(SCRIPT_TARGETS_PY{suffix}) : {copyrule}\n'
# '\t@echo Copying script: $(subst $(BUILD_DIR)/,,$@)\n'
# '\t@mkdir -p $(dir $@)\n'
# '\t@rm -f $@\n'
# '\t@cp $^ $@\n'
# '\t@chmod 444 $@\n'
# )
# Fancy new simple loop-based target generation.
out += (
f'\n# These are too complex to define in a pattern rule;\n'
f'# Instead we generate individual targets in a loop.\n'
f'$(foreach element,$(SCRIPT_TARGETS_PYC{suffix}),\\\n'
f'$(eval $(call make-opt-pyc-target,$(element))))'
)
# Old code to explicitly emit individual targets.
if bool(False):
out += (
'\n# Looks like path mangling from py to pyc is too complex for'
' pattern rules so\n# just generating explicit targets'
' for each. Could perhaps look into using a\n# fancy for-loop'
' instead, but perhaps listing these explicitly isn\'t so bad.\n'
)
for i, target in enumerate(pyc_targets):
# Note: there's currently a bug which can cause python bytecode
# generation to be non-deterministic. This can break our blessing
# process since we bless in core but then regenerate bytecode in
# spinoffs. See https://bugs.python.org/issue34722
# For now setting PYTHONHASHSEED=1 is a workaround.
out += (
'\n'
+ target
+ ': \\\n '
+ py_targets[i]
+ '\n\t@echo Compiling script: $(subst $(BUILD_DIR),,$^)\n'
'\t@rm -rf $@ && PYTHONHASHSEED=1 $(TOOLS_DIR)/pcommand'
' compile_python_file $^'
' && chmod 444 $@\n'
)
return out
def _get_extras_targets_win(
projroot: str, all_targets: set[str], platform: str
) -> str:
targets: list[str] = []
base = f'{ASSETS_SRC}/windows'
dstbase = 'windows'
for root, _dnames, fnames in os.walk(os.path.join(projroot, base)):
for fname in fnames:
# Only include the platform we were passed.
if not root.startswith(
os.path.join(projroot, f'{ASSETS_SRC}/windows/{platform}')
):
continue
ext = os.path.splitext(fname)[-1]
# "I don't like .DS_Store files. They're coarse and rough and
# irritating and they get everywhere."
if fname == '.DS_Store':
continue
# Ignore python files as they're handled separately.
if ext in ['.py', '.pyc']:
continue
# Various stuff we expect to be there...
if ext in [
'.exe',
'.dll',
'.bat',
'.txt',
'.whl',
'.ps1',
'.css',
'.sample',
'.ico',
'.pyd',
'.ctypes',
'.rst',
'.fish',
'.csh',
'.cat',
'.pdb',
'.lib',
'.html',
] or fname in [
'activate',
'README',
'command_template',
'fetch_macholib',
]:
base_abs = os.path.join(projroot, base)
assert root.startswith(base_abs)
targetpath = os.path.join(
dstbase + root.removeprefix(base_abs), fname
)
# print(f'DSTBASE {dstbase} ROOT {root}
# TARGETPATH {targetpath}')
targets.append('$(BUILD_DIR)/' + targetpath)
all_targets.add(BUILD_DIR + '/' + targetpath)
continue
# Complain if something new shows up instead of blindly
# including it.
raise RuntimeError(f'Unexpected extras file: {fname}')
targets.sort()
p_up = platform.upper()
out = (
f'\nEXTRAS_TARGETS_WIN_{p_up} = \\\n ' + ' \\\n '.join(targets) + '\n'
)
# We transform all these targets into efrocache-fetches in public.
out += (
'\n# Rule to copy src extras to build.\n'
f'# __EFROCACHE_TARGET__\n'
f'$(EXTRAS_TARGETS_WIN_{p_up}) : $(BUILD_DIR)/% :'
' %\n'
'\t@$(PCOMMANDBATCH) copy_win_extra_file $^ $@\n'
# '\t@echo Copying file: $(subst $(BUILD_DIR)/,,$@)\n'
# '\t@mkdir -p $(dir $@)\n'
# '\t@rm -f $@\n'
# '\t@cp $^ $@\n'
)
return out
[docs]
def generate_assets_makefile(
projroot: str,
fname: str,
existing_data: str,
meta_manifests: dict[str, str],
explicit_sources: set[str],
) -> dict[str, str]:
"""Main script entry point."""
# pylint: disable=too-many-locals
from efrotools.project import getprojectconfig
from pathlib import Path
public = getprojectconfig(Path(projroot))['public']
assert isinstance(public, bool)
original = existing_data
lines = original.splitlines()
auto_start_public = lines.index('# __AUTOGENERATED_PUBLIC_BEGIN__')
auto_end_public = lines.index('# __AUTOGENERATED_PUBLIC_END__')
auto_start_private = lines.index('# __AUTOGENERATED_PRIVATE_BEGIN__')
auto_end_private = lines.index('# __AUTOGENERATED_PRIVATE_END__')
all_targets_public: set[str] = set()
all_targets_private: set[str] = set()
# We always auto-generate the public section.
our_lines_public = [
_get_py_targets_subset(
projroot,
meta_manifests,
explicit_sources,
all_targets_public,
subset='public',
suffix='_PUBLIC',
),
_get_py_targets_subset(
projroot,
meta_manifests,
explicit_sources,
all_targets_public,
subset='public_tools',
suffix='_PUBLIC_TOOLS',
),
]
# Only auto-generate the private section in the private repo.
if public:
our_lines_private = lines[auto_start_private + 1 : auto_end_private]
else:
our_lines_private = [
_get_py_targets_subset(
projroot,
meta_manifests,
explicit_sources,
all_targets_private,
subset='private-apple',
suffix='_PRIVATE_APPLE',
),
_get_py_targets_subset(
projroot,
meta_manifests,
explicit_sources,
all_targets_private,
subset='private-android',
suffix='_PRIVATE_ANDROID',
),
_get_py_targets_subset(
projroot,
meta_manifests,
explicit_sources,
all_targets_private,
subset='private-common',
suffix='_PRIVATE_COMMON',
),
_get_py_targets_subset(
projroot,
meta_manifests,
explicit_sources,
all_targets_private,
subset='private-windows-Win32',
suffix='_PRIVATE_WIN_WIN32',
),
_get_py_targets_subset(
projroot,
meta_manifests,
explicit_sources,
all_targets_private,
subset='private-windows-x64',
suffix='_PRIVATE_WIN_X64',
),
_get_targets(
projroot,
'COB_TARGETS',
'.collisionmesh.obj',
'.cob',
all_targets_private,
),
_get_targets(
projroot,
'BOB_TARGETS',
'.mesh.obj',
'.bob',
all_targets_private,
),
_get_targets(
projroot,
'FONT_TARGETS',
'.fdata',
'.fdata',
all_targets_private,
),
_get_targets(
projroot,
'PEM_TARGETS',
'.pem',
'.pem',
all_targets_private,
),
_get_targets(
projroot,
'DATA_TARGETS',
'.json',
'.json',
all_targets_private,
limit_to_prefix='ba_data/data',
),
_get_targets(
projroot,
'AUDIO_TARGETS',
'.wav',
'.ogg',
all_targets_private,
),
_get_targets(
projroot,
'TEX2D_DDS_TARGETS',
'.tex2d.png',
'.dds',
all_targets_private,
),
_get_targets(
projroot,
'TEX2D_PVR_TARGETS',
'.tex2d.png',
'.pvr',
all_targets_private,
),
_get_targets(
projroot,
'TEX2D_KTX_TARGETS',
'.tex2d.png',
'.ktx',
all_targets_private,
),
_get_targets(
projroot,
'TEX2D_PREVIEW_PNG_TARGETS',
'.tex2d.png',
'_preview.png',
all_targets_private,
),
_get_extras_targets_win(projroot, all_targets_private, 'Win32'),
_get_extras_targets_win(projroot, all_targets_private, 'x64'),
]
filtered = (
lines[: auto_start_public + 1]
+ our_lines_public
+ lines[auto_end_public : auto_start_private + 1]
+ our_lines_private
+ lines[auto_end_private:]
)
out_files: dict[str, str] = {}
out = '\n'.join(filtered) + '\n'
out_files[fname] = out
# Write a simple manifest of the things we expect to have in build.
# We can use this to clear out orphaned files as part of builds.
out_files['src/assets/.asset_manifest_public.json'] = _gen_manifest(
all_targets_public
)
# Only *generate* the private manifest in the private repo. In public
# we just give what's already on disk.
manprivpath = 'src/assets/.asset_manifest_private.json'
if not public:
out_files[manprivpath] = _gen_manifest(all_targets_private)
return out_files
def _gen_manifest(all_targets: set[str]) -> str:
# Lastly, write a simple manifest of the things we expect to have
# in build. We can use this to clear out orphaned files as part of builds.
assert all(t.startswith(BUILD_DIR) for t in all_targets)
manifest = sorted(t[13:] for t in all_targets)
return json.dumps(manifest, indent=1)