# Released under the MIT License. See LICENSE for details.
#
"""Procedurally regenerates our code Makefile.
This Makefiles builds our generated code such as encrypted python strings,
node types, etc). Outputs land in ``generated/`` / ``_generated/`` dirs per
feature-set; see docs/design/codegen.md for why that's the convention.
"""
from __future__ import annotations # Docs-generation hack.
import os
import json
from pathlib import Path
from typing import TYPE_CHECKING
from dataclasses import dataclass
from efro.error import CleanError
from efrotools.project import getprojectconfig
if TYPE_CHECKING:
pass
# These paths need to be relative to the dir we're writing the Makefile to.
PROJ_DIR = '../..'
TOOLS_DIR = f'{PROJ_DIR}/tools'
PROJ_SRC_DIR = '..'
# These should only be used for make targets since they use makefile
# vars. Our makefile vars have the same names as above. We could inline
# vars ourself but it's nice to build a makefile that feels like one
# we'd build by hand.
OUT_DIR_ROOT_CPP = '$(PROJ_SRC_DIR)/ballistica'
#: The asset-set codegen groups, keyed by the feature-set that OWNS
#: each set's outputs. Registration gates are *derived* from these
#: keys (present featureset -> targets registered), and every
#: registered dst is validated against the owner's generated dirs --
#: so a set can't be registered under the wrong featureset
#: conditional. (The 2026-08-31 bug: all three groups sat under a
#: hand-written ui_v1 gate, so core+base spinoffs never generated
#: babase's asset set and failed to import babase; only public CI's
#: spinoff test caught it.)
ASSET_SET_GROUPS: dict[str, str] = {
'base': '_add_base_asset_set_targets',
'scene_v1': '_add_scene_asset_set_targets',
'ui_v1': '_add_ui_asset_set_targets',
}
def _featureset_python_package(fsname: str) -> str:
"""A feature set's python package name (base -> babase etc.)."""
return 'ba' + fsname.replace('_', '')
OUT_DIR_BASE_PYTHON = '$(PROJ_SRC_DIR)/assets/ba_data/python/babase/_generated'
OUT_DIR_UI_V1_PYTHON = '$(PROJ_SRC_DIR)/assets/ba_data/python/bauiv1/_generated'
OUT_DIR_SCENE_V1_PYTHON = (
'$(PROJ_SRC_DIR)/assets/ba_data/python/bascenev1/_generated'
)
[docs]
@dataclass
class Target:
"""A target to be added to the Makefile."""
src: list[str]
dst: str
cmd: str
mkdir: bool = False
[docs]
def emit(self) -> str:
"""Gen a Makefile target."""
out: str = self.dst.replace(' ', '\\ ')
out += (
' : '
+ ' '.join(s for s in self.src)
+ (
('\n\t@mkdir -p "' + os.path.dirname(self.dst) + '"')
if self.mkdir
else ''
)
+ '\n\t@'
+ self.cmd
+ '\n'
)
return out
[docs]
def generate_codegen_makefile(
projroot: str, existing_data: str
) -> dict[str, str]:
"""Update the project codegen Makefile.
Returns file names and contents.
"""
return CodegenMakefileGenerator(projroot, existing_data).run()
[docs]
class CodegenMakefileGenerator:
"""Thing that does the thing."""
def __init__(self, projroot: str, existing_data: str) -> None:
from batools.featureset import FeatureSet
self._existing_data = existing_data
self._projroot = projroot
self._featuresets = FeatureSet.get_all_for_project(projroot)
[docs]
def run(self) -> dict[str, str]:
"""Do the thing."""
public = getprojectconfig(Path(self._projroot))['public']
assert isinstance(public, bool)
fname = 'src/codegen/Makefile'
fname_pub_man = 'src/codegen/.codegen_manifest_public.json'
fname_priv_man = 'src/codegen/.codegen_manifest_private.json'
original = self._existing_data
lines = original.splitlines()
# We'll generate manifests of all public/private files we
# generate (not private-internal though).
all_dsts_public: set[str] = set()
all_dsts_private: set[str] = set()
auto_start_public = lines.index('# __AUTOGENERATED_PUBLIC_BEGIN__')
auto_end_public = lines.index('# __AUTOGENERATED_PUBLIC_END__')
auto_start_private = lines.index('# __AUTOGENERATED_PRIVATE_BEGIN__')
auto_end_private = lines.index('# __AUTOGENERATED_PRIVATE_END__')
# Public targets (stuff with full sources available in public
# repo).
targets: list[Target] = []
pubtargets = targets
self._add_monolithic_register_modules_target(targets)
self._add_pyembed_targets(targets)
# Base feature set bits.
if os.path.exists(
f'{self._projroot}/pconfig/featuresets/featureset_base.py'
):
self._add_init_module_target(targets, moduledir=OUT_DIR_BASE_PYTHON)
self._add_base_enums_module_target(targets)
# Asset-set groups: gates derived from each group's owning
# feature set, and every dst validated against that owner's
# generated dirs (see ASSET_SET_GROUPS).
self._add_asset_set_groups(targets)
our_lines_public = (
_empty_line_if(bool(targets))
+ self._emit_sources_lines(targets)
+ [t.emit() for t in targets]
)
all_dsts_public.update(t.dst for t in targets)
# Only rewrite the private section in the private repo;
# otherwise keep the existing one intact.
if public:
our_lines_private = lines[auto_start_private + 1 : auto_end_private]
else:
# Private targets (but available in public through
# efrocache).
targets = []
our_lines_private_1 = (
_empty_line_if(bool(targets))
+ self._emit_sources_lines(targets)
+ ['# __EFROCACHE_TARGET__\n' + t.emit() for t in targets]
+ [
'\n#'
' Note: we include our public targets in efrocache even\n'
'# though they are buildable in public. This allows us to\n'
'# fetch them to bootstrap binary builds in cases where\n'
'# we can\'t use our full Makefiles (like Windows CI).\n'
]
+ self._emit_efrocache_lines(pubtargets + targets)
)
all_dsts_private.update(t.dst for t in targets)
# Private-internal targets (not available at all in public).
targets = []
self._add_pyembed_targets_internal(targets)
self._add_extra_targets_internal(targets)
our_lines_private_2 = (
['# __PUBSYNC_STRIP_BEGIN__']
+ _empty_line_if(bool(targets))
+ self._emit_sources_lines(targets)
+ [t.emit() for t in targets]
+ ['# __PUBSYNC_STRIP_END__']
)
our_lines_private = our_lines_private_1 + our_lines_private_2
filtered = (
lines[: auto_start_public + 1]
+ our_lines_public
+ lines[auto_end_public : auto_start_private + 1]
+ our_lines_private
+ lines[auto_end_private:]
)
out = '\n'.join(filtered) + '\n'
out_files: dict[str, str] = {}
out_pub_man = json.dumps(
sorted(self._filter_manifest_path(p) for p in all_dsts_public),
indent=1,
)
out_priv_man = json.dumps(
sorted(self._filter_manifest_path(p) for p in all_dsts_private),
indent=1,
)
out_files[fname] = out
out_files[fname_pub_man] = out_pub_man
out_files[fname_priv_man] = out_priv_man
return out_files
def _emit_sources_lines(self, targets: list[Target]) -> list[str]:
"""Gen lines to build provided targets."""
out: list[str] = []
if not targets:
return out
all_dsts = set()
for target in targets:
all_dsts.add(target.dst)
out.append(
'sources: \\\n '
+ ' \\\n '.join(
dst.replace(' ', '\\ ') for dst in sorted(all_dsts)
)
+ '\n'
)
return out
def _emit_efrocache_lines(self, targets: list[Target]) -> list[str]:
"""Gen lines to cache provided targets."""
out: list[str] = []
if not targets:
return out
all_dsts = set()
for target in targets:
# We may need to make pipeline adjustments if/when we get
# filenames with spaces in them.
if ' ' in target.dst:
raise CleanError(
'FIXME: need to account for spaces in filename'
f' "{target.dst}".'
)
all_dsts.add(target.dst)
out.append(
'efrocache-list:\n\t@echo '
+ ' \\\n '.join('"' + dst + '"' for dst in sorted(all_dsts))
+ '\n'
)
out.append('efrocache-build: sources\n')
return out
def _add_base_enums_module_target(self, targets: list[Target]) -> None:
targets.append(
Target(
src=[
'$(PROJ_DIR)/src/ballistica/shared/ballistica.h',
'$(TOOLS_DIR)/batools/enumspython.py',
],
dst=os.path.join(OUT_DIR_BASE_PYTHON, 'enums.py'),
cmd='$(PCOMMAND) gen_python_enums_module $< $@',
)
)
def _add_init_module_target(
self, targets: list[Target], moduledir: str
) -> None:
targets.append(
Target(
src=['$(TOOLS_DIR)/batools/pcommands.py'],
dst=os.path.join(moduledir, '__init__.py'),
cmd='$(PCOMMAND) gen_python_init_module $@',
)
)
def _add_monolithic_register_modules_target(
self, targets: list[Target]
) -> None:
# When any of our featuresets configs changes, rebuild our
# snippet of code that registers them all.
featureset_fnames = [
n
for n in os.listdir(
os.path.join(self._projroot, 'pconfig/featuresets')
)
if n.startswith('featureset_') and n.endswith('.py')
]
targets.append(
Target(
src=[
f'$(PROJ_DIR)/pconfig/featuresets/{n}'
for n in sorted(featureset_fnames)
],
dst=(
f'{OUT_DIR_ROOT_CPP}/core/generated/'
'python_modules_monolithic.h'
),
cmd='$(PCOMMAND) gen_monolithic_register_modules $@',
)
)
def _add_featureset_entries(
self, entries: list[tuple[str, str]], internal: bool
) -> None:
featuresets = [f for f in self._featuresets if internal == f.internal]
# For featureset 'foo_bar', stuff under 'bafoobarcodegen' goes
# into 'ballistica/foo_bar/generated'.
for featureset in featuresets:
entries.append(
(
featureset.name_python_package_codegen,
os.path.join(
OUT_DIR_ROOT_CPP, featureset.name, 'generated'
),
)
)
def _create_featureset_targets(
self,
entries: list[tuple[str, str]],
targets: list[Target],
internal: bool,
) -> None:
for pkg, out_dir in entries:
base_src_dir = os.path.join(self._projroot, f'src/codegen/{pkg}')
if not os.path.exists(base_src_dir):
continue
# Note: sort to keep things deterministic.
for fname in sorted(os.listdir(f'{base_src_dir}/pyembed')):
if (
not fname.endswith('.py')
or fname == '__init__.py'
or 'flycheck' in fname
):
continue
name = os.path.splitext(fname)[0]
src = [
f'{pkg}/pyembed/{name}.py',
]
dst = os.path.join(out_dir, 'pyembed', f'{name}.inc')
if name.startswith('binding_'):
targets.append(
Target(
src=src,
dst=dst,
cmd='$(PCOMMAND) gen_binding_code $< $@',
)
)
else:
# Route all non-binding pyembed through the unified
# bytecode-based gen_pyembed. Internal builds encrypt
# (casual obfuscation); spinoff builds emit plain
# bytecode. env.py runs before the core context is
# fully plumbed so uses a caller-supplied ``ctx``
# variable instead of ``internal_py_context``.
encrypt_flag = '1' if internal else '0'
extra_args = f' encrypt={encrypt_flag}'
if name == 'env':
extra_args += ' ctx=*ctx'
targets.append(
Target(
src=src,
dst=dst,
cmd=(
f'$(PCOMMAND) gen_pyembed' f' $< $@{extra_args}'
),
)
)
def _add_asset_set_groups(self, targets: list[Target]) -> None:
"""Register asset-set codegen for each present owning fs.
Gates come from ASSET_SET_GROUPS keys and each group's newly
added dsts get validated against its owner's generated dirs.
"""
for fsname, addername in ASSET_SET_GROUPS.items():
if not os.path.exists(
f'{self._projroot}/pconfig/featuresets/featureset_{fsname}.py'
):
continue
count_before = len(targets)
getattr(self, addername)(targets)
self._check_asset_set_dsts(fsname, targets[count_before:])
def _check_asset_set_dsts(self, fsname: str, added: list[Target]) -> None:
"""Validate an asset-set group's dsts against its owning fs.
Python outputs must land in the owner's ``_generated`` dir and
C++ outputs in its ``generated`` dir. Anything else either
belongs to a different feature set (a wrong ASSET_SET_GROUPS
pairing) or falls outside the generated-dir convention -- in
both cases the assets-staging machinery would silently
mis-handle it on some project flavor, so fail loudly here.
"""
pkg = _featureset_python_package(fsname)
py_prefix = f'$(PROJ_SRC_DIR)/assets/ba_data/python/{pkg}/_generated/'
cpp_prefix = f'{OUT_DIR_ROOT_CPP}/{fsname}/generated/'
for target in added:
if not target.dst.startswith((py_prefix, cpp_prefix)):
raise RuntimeError(
f"Asset-set target '{target.dst}' for feature set"
f" '{fsname}' is outside that set's generated dirs"
f' ({py_prefix} / {cpp_prefix}); check'
' ASSET_SET_GROUPS ownership and the'
' generated-dir convention (docs/design/codegen.md).'
)
def _add_base_asset_set_targets(self, targets: list[Target]) -> None:
"""Targets for the base asset-set spec.
The base sibling of _add_scene_asset_set_targets; also emits
the placeholder-fill include unique to this set.
"""
spec = 'babasecodegen/base_assets.py'
targets.append(
Target(
src=[spec, '$(TOOLS_DIR)/batools/base_assets.py'],
dst=os.path.join(OUT_DIR_BASE_PYTHON, 'base_asset_set.py'),
cmd='$(PCOMMAND) gen_base_asset_set_py $@',
mkdir=True,
)
)
for name in (
'base_asset_set.h',
'base_asset_set_unpack.inc',
'base_asset_set_placeholders.inc',
):
targets.append(
Target(
src=[spec, '$(TOOLS_DIR)/batools/base_assets.py'],
dst=os.path.join(
OUT_DIR_ROOT_CPP, 'base', 'generated', name
),
cmd='$(PCOMMAND) gen_base_asset_set_cpp $@',
mkdir=True,
)
)
# (No _add_init_module_target here: the enums targets already
# register the __init__ for this dir.)
def _add_scene_asset_set_targets(self, targets: list[Target]) -> None:
"""Targets for the scene_v1 asset-set spec.
The scene sibling of _add_ui_asset_set_targets; same
one-spec-three-outputs reasoning.
"""
spec = 'bascenev1codegen/scene_assets.py'
targets.append(
Target(
src=[spec, '$(TOOLS_DIR)/batools/scene_assets.py'],
dst=os.path.join(OUT_DIR_SCENE_V1_PYTHON, 'scene_asset_set.py'),
cmd='$(PCOMMAND) gen_scene_asset_set_py $@',
mkdir=True,
)
)
for name in (
'scene_asset_set.h',
'scene_asset_set_unpack.inc',
):
targets.append(
Target(
src=[spec, '$(TOOLS_DIR)/batools/scene_assets.py'],
dst=os.path.join(
OUT_DIR_ROOT_CPP, 'scene_v1', 'generated', name
),
cmd='$(PCOMMAND) gen_scene_asset_set_cpp $@',
mkdir=True,
)
)
self._add_init_module_target(targets, moduledir=OUT_DIR_SCENE_V1_PYTHON)
def _add_ui_asset_set_targets(self, targets: list[Target]) -> None:
"""Targets for the ui_v1 asset-set spec.
One spec drives the Python dataclass an app-mode fills in, the
native struct it lands in, and the unpacker between them --
which is what makes a renamed or added slot a build error on
both sides rather than silent drift.
Public (not internal): ui_v1 is a public feature-set, so public
builds need these generated to compile.
"""
spec = 'bauiv1codegen/ui_assets.py'
targets.append(
Target(
src=[spec, '$(TOOLS_DIR)/batools/ui_assets.py'],
dst=os.path.join(OUT_DIR_UI_V1_PYTHON, 'ui_asset_set.py'),
cmd='$(PCOMMAND) gen_ui_asset_set_py $@',
mkdir=True,
)
)
for name in (
'ui_asset_set.h',
'ui_asset_set_unpack.inc',
):
targets.append(
Target(
src=[spec, '$(TOOLS_DIR)/batools/ui_assets.py'],
dst=os.path.join(
OUT_DIR_ROOT_CPP, 'ui_v1', 'generated', name
),
cmd='$(PCOMMAND) gen_ui_asset_set_cpp $@',
mkdir=True,
)
)
self._add_init_module_target(targets, moduledir=OUT_DIR_UI_V1_PYTHON)
def _add_pyembed_targets(self, targets: list[Target]) -> None:
entries: list[tuple[str, str]] = []
# Map stuff from other featureset codegen packages to a
# generated dir under their C++ root.
self._add_featureset_entries(entries, internal=False)
self._create_featureset_targets(entries, targets, internal=False)
def _add_pyembed_targets_internal(self, targets: list[Target]) -> None:
entries: list[tuple[str, str]] = []
self._add_featureset_entries(entries, internal=True)
self._create_featureset_targets(entries, targets, internal=True)
def _add_extra_targets_internal(self, targets: list[Target]) -> None:
if os.path.exists(
f'{self._projroot}/pconfig/featuresets/featureset_plus.py'
):
# Add targets to generate message sender/receiver classes
# for our basn/client protocols. Their outputs go to
# 'generated' so they don't get added to git.
self._add_init_module_target(
targets, moduledir='bapluscodegen/generated'
)
for srcname, dstname, gencmd in [
('batocloud', 'basnmessagesender', 'gen_basn_msg_sender'),
('cloudtoba', 'basnmessagereceiver', 'gen_basn_msg_receiver'),
]:
targets.append(
Target(
src=[f'bapluscodegen/pyembed/{srcname}.py'],
dst=f'bapluscodegen/generated/{dstname}.py',
cmd=f'$(PCOMMAND) {gencmd} $@',
)
)
# Now add explicit targets to generate embedded code for the
# resulting classes. We can't simply place them in a scanned
# dir like pyembed because they might not exist yet at
# update time.
for name in ['basnmessagesender', 'basnmessagereceiver']:
targets.append(
Target(
src=[f'bapluscodegen/generated/{name}.py'],
dst=os.path.join(
OUT_DIR_ROOT_CPP,
'plus',
'generated',
'pyembed',
f'{name}.inc',
),
cmd='$(PCOMMAND) gen_pyembed $< $@ encrypt=1',
)
)
# Typed Android JNI message bus. Spec lives in babasecodegen;
# codegen emits one Java bridge class and a pair of C++
# .inc files (decl + impl) included by hand-written glue
# files under src/ballistica/base/platform/android/.
#
# The Java file lives under src/ballistica/base/generated/java
# (a project-name-independent path) rather than the Android
# app tree. Gradle's build.gradle adds this dir to its
# sourceSets.main.java.srcDirs. This avoids needing project-
# name-specific paths (ballisticakit-android/BallisticaKit/...
# vs template-android/Template/...) in the generated Makefile,
# which the spinoff sync doesn't substitute correctly in
# auto-generated sections.
if os.path.exists(
f'{self._projroot}/pconfig/featuresets/featureset_base.py'
):
spec = 'babasecodegen/android_messages.py'
targets.append(
Target(
src=[spec],
dst=os.path.join(
OUT_DIR_ROOT_CPP,
'base',
'generated',
'java',
'com',
'ericfroemling',
'ballistica',
'generated',
'BallisticaJniBridge.java',
),
cmd='$(PCOMMAND) gen_android_message_java $@',
mkdir=True,
)
)
for inc in (
'android_messages_decl.inc',
'android_messages_impl.inc',
):
targets.append(
Target(
src=[spec],
dst=os.path.join(
OUT_DIR_ROOT_CPP,
'base',
'generated',
'android',
inc,
),
cmd='$(PCOMMAND) gen_android_message_cpp $@',
mkdir=True,
)
)
def _filter_manifest_path(self, path: str) -> str:
"""Given a path we dumped into our makefile, generate an abs one."""
# Our makefile paths contain vars to be subbed by the makefile.
# We need to do those same subs now.
for pair in [
('$(PROJ_DIR)', PROJ_DIR),
('$(TOOLS_DIR)', TOOLS_DIR),
('$(PROJ_SRC_DIR)', PROJ_SRC_DIR),
]:
path = path.replace(pair[0], pair[1])
projpath = f'{self._projroot}/'
assert '\\' not in projpath # Don't expect to work on windows.
abspath = os.path.abspath(
os.path.join(self._projroot, 'src', 'codegen', path)
)
if not abspath.startswith(projpath):
raise RuntimeError(
f'Path "{abspath}" is not under project root "{projpath}"'
)
return abspath[len(projpath) :]
def _empty_line_if(condition: bool) -> list[str]:
return [''] if condition else []
# Docs-generation hack; import some stuff that we likely only forward-declared
# in our actual source code so that docs tools can find it.
from typing import (Coroutine, Any, Literal, Callable,
Generator, Awaitable, Sequence, Self)
import asyncio
from concurrent.futures import Future
from pathlib import Path
from enum import Enum