Source code for batools.pcommands2

# Released under the MIT License. See LICENSE for details.
#
"""A nice collection of ready-to-use pcommands for this package."""

from __future__ import annotations  # Docs-generation hack.

# Note: import as little as possible here at the module level to
# keep launch times fast for small snippets.
import sys
from typing import TYPE_CHECKING

from efrotools import pcommand

if TYPE_CHECKING:
    from typing import Any

    from libcst.metadata import CodeRange


[docs] def gen_builtin_asset_ids() -> None: """Generate builtin-asset C++ (id-enums, load-block, strings API). Fetches the asset listing for the projectconfig ``"assets"`` pin via ``assetpackage _listing`` (assembly-free) and splices generated content into the ``// __AUTOGENERATED_BUILTIN_ASSET_IDS_*__`` autogen section in ``src/ballistica/base/base.h`` and the ``// __AUTOGENERATED_BUILTIN_ASSET_LOAD_*__`` section inside ``Assets::StartLoading()`` in ``src/ballistica/base/assets/assets.cc``, and fully regenerates the ``BuiltinStrings`` typed accessor files ``src/ballistica/base/assets/builtin_strings.{h,cc}``. Idempotent — only rewrites a file if its generated content differs from on-disk. Normally invoked indirectly by ``assetpins`` when the pin changes (it talks to the cloud, so it lives there, NOT in ``make update`` -- ``update_project`` only *checks* the splice matches the pin and stays offline). Standalone invocation exists for manual regen. Pass ``--check`` to fail (without writing) if either file would change. """ from pathlib import Path from efro.error import CleanError from efrotools.project import getprojectconfig from batools.builtinassetids import generate projroot = Path(pcommand.PROJROOT) apverid = getprojectconfig(projroot).get('assets') if not isinstance(apverid, str) or not apverid: raise CleanError( "projectconfig has no string 'assets' pin to generate" ' builtin-asset ids from.' ) check = '--check' in sys.argv changed = generate(projroot, apverid, check=check) if check and changed: raise CleanError( 'Builtin-asset id autogen sections are out of date; ' 'run `make assetpins-latest` to regenerate.' )
[docs] def gen_monolithic_register_modules() -> None: """Generate .h file for registering py modules.""" import os import textwrap from efro.error import CleanError from batools.featureset import FeatureSet if len(sys.argv) != 3: raise CleanError('Expected 1 arg.') outpath = sys.argv[2] featuresets = FeatureSet.get_all_for_project(str(pcommand.PROJROOT)) # Filter out ones without native modules. featuresets = [f for f in featuresets if f.has_python_binary_module] pymodulenames = sorted(f.name_python_binary_module for f in featuresets) def initname(mname: str) -> str: # plus is a special case since we need to define that symbol # ourself. return f'DoPyInit_{mname}' if mname == '_baplus' else f'PyInit_{mname}' extern_def_code = '\n'.join( f'auto {initname(n)}() -> PyObject*;' for n in pymodulenames ) py_register_code = '\n'.join( f'PyImport_AppendInittab("{n}", &{initname(n)});' for n in pymodulenames ) if '_baplus' in pymodulenames: init_plus_code = ( '\n' '// Slight hack: because we are currently building baplus as a' ' static module\n' '// and linking it in, symbols exported there (namely' ' PyInit__baplus) do not\n' '// seem to be available through us when we are compiled as' ' a dynamic\n' '// library. This leads to Python being unable to load baplus.' ' While I\'m sure\n' '// there is some way to get those symbols exported, I\'m worried' ' it might be\n' '// a messy platform-specific affair. So instead we\'re just' ' defining that\n' '// function here when baplus is present and forwarding it through' ' to the\n' '// static library version.\n' 'extern "C" auto PyInit__baplus() -> PyObject* {\n' ' return DoPyInit__baplus();\n' '}\n' ) else: init_plus_code = '' base_code = """ // Released under the MIT License. See LICENSE for details. #ifndef BALLISTICA_CORE_GENERATED_PYTHON_MODULES_MONOLITHIC_H_ #define BALLISTICA_CORE_GENERATED_PYTHON_MODULES_MONOLITHIC_H_ // THIS CODE IS AUTOGENERATED BY CODEGEN BUILD; DO NOT EDIT BY HAND. #include "ballistica/shared/ballistica.h" #include "ballistica/shared/python/python_macros.h" extern "C" { ${EXTERN_DEF_CODE} } namespace ballistica { /// Register init calls for all of our built-in Python modules. /// Should only be used in monolithic builds. In modular builds /// binary modules get located as .so files on disk as per regular /// Python behavior. void MonolithicRegisterPythonModules() { if (g_buildconfig.monolithic_build()) { ${PY_REGISTER_CODE} } else { FatalError( "MonolithicRegisterPythonModules should not be called" " in modular builds."); } } ${PY_INIT_PLUS} } // namespace ballistica #endif // BALLISTICA_CORE_GENERATED_PYTHON_MODULES_MONOLITHIC_H_ """ out = ( textwrap.dedent(base_code) .replace('${EXTERN_DEF_CODE}', extern_def_code) .replace( '${PY_REGISTER_CODE}', textwrap.indent(py_register_code, ' ') ) .replace('${PY_INIT_PLUS}', init_plus_code) .strip() + '\n' ) os.makedirs(os.path.dirname(outpath), exist_ok=True) with open(outpath, 'w', encoding='utf-8') as outfile: outfile.write(out)
[docs] def py_examine() -> None: """Run a python examination at a given point in a given file.""" import os from pathlib import Path import efrotools.emacs if len(sys.argv) != 7: print('ERROR: expected 7 args') sys.exit(255) filename = Path(sys.argv[2]) line = int(sys.argv[3]) column = int(sys.argv[4]) selection: str | None = None if sys.argv[5] == '' else sys.argv[5] operation = sys.argv[6] # This stuff assumes it is being run from project root. os.chdir(pcommand.PROJROOT) # Set up pypaths so our main distro stuff works. scriptsdir = os.path.abspath( os.path.join( os.path.dirname(sys.argv[0]), '../src/assets/ba_data/python' ) ) toolsdir = os.path.abspath( os.path.join(os.path.dirname(sys.argv[0]), '../tools') ) if scriptsdir not in sys.path: sys.path.append(scriptsdir) if toolsdir not in sys.path: sys.path.append(toolsdir) efrotools.emacs.py_examine( pcommand.PROJROOT, filename, line, column, selection, operation )
[docs] def clean_orphaned_assets() -> None: """Remove asset files that are no longer part of the build.""" import os import json import subprocess # Operate from dist root.. os.chdir(pcommand.PROJROOT) # Our manifest is split into 2 files (public and private) with open( 'src/assets/.asset_manifest_public.json', encoding='utf-8' ) as infile: manifest = set(json.loads(infile.read())) with open( 'src/assets/.asset_manifest_private.json', encoding='utf-8' ) as infile: manifest.update(set(json.loads(infile.read()))) for root, _dirs, fnames in os.walk('build/assets'): for fname in fnames: fpath = os.path.join(root, fname) fpathrel = fpath[13:] # paths are relative to build/assets if fpathrel not in manifest: print(f'Removing orphaned asset file: {fpath}') os.unlink(fpath) # Lastly, clear empty dirs. subprocess.run( 'find build/assets -depth -empty -type d -delete', shell=True, check=True, )
[docs] def win_ci_install_prereqs() -> None: """Install bits needed for basic win ci.""" import json from efrotools.efrocache import get_target # We'll need to pull a handful of things out of efrocache for the # build to succeed. Normally this would happen through our Makefile # targets but we can't use them under raw Windows so we need to just # hard-code whatever we need here. lib_dbg_x64 = 'build/prefab/lib/windows/Debug_x64' needed_targets: set[str] = { f'{lib_dbg_x64}/BallisticaKitGenericPlus.lib', f'{lib_dbg_x64}/BallisticaKitGenericPlus.pdb', 'ballisticakit-windows/Generic/BallisticaKit.ico', } # Look through everything that gets generated by our codegen builds # and pick out anything we need for our basic builds/tests. with open( 'src/codegen/.codegen_manifest_public.json', encoding='utf-8' ) as infile: codegen_public: list[str] = json.loads(infile.read()) with open( 'src/codegen/.codegen_manifest_private.json', encoding='utf-8' ) as infile: codegen_private: list[str] = json.loads(infile.read()) for target in codegen_public + codegen_private: if ( target.startswith('src/ballistica/') and '/generated/' in target ) or ( target.startswith('src/assets/ba_data/python/') and '/_generated/' in target ): needed_targets.add(target) for target in needed_targets: get_target(target, batch=pcommand.is_batch(), clr=pcommand.clr())
[docs] def win_ci_binary_build() -> None: """Simple windows binary build for ci.""" import subprocess # Do the thing. subprocess.run( [ 'C:\\Program Files\\Microsoft Visual Studio\\18\\' 'Enterprise\\MSBuild\\Current\\Bin\\MSBuild.exe', 'ballisticakit-windows\\Generic\\BallisticaKitGeneric.vcxproj', '-target:Build', '-property:Configuration=Debug', '-property:Platform=x64', '-property:VisualStudioVersion=18', ], check=True, )
[docs] def update_cmake_prefab_lib() -> None: """Update prefab internal libs; run as part of a build.""" import subprocess import os from efro.error import CleanError from batools.build import PrefabPlatform if len(sys.argv) != 5: raise CleanError( 'Expected 3 args (standard/server, debug/release, build-dir)' ) buildtype = sys.argv[2] mode = sys.argv[3] builddir = sys.argv[4] if buildtype not in {'standard', 'server'}: raise CleanError(f'Invalid buildtype: {buildtype}') if mode not in {'debug', 'release'}: raise CleanError(f'Invalid mode: {mode}') # Our 'cmake' build targets use the Linux side of WSL; not native # Windows. platform = PrefabPlatform.get_current(wsl_targets_windows=False) suffix = '_server' if buildtype == 'server' else '_gui' target = ( f'build/prefab/lib/{platform.value}{suffix}/{mode}/libballisticaplus.a' ) # Build the target and then copy it to dst if it doesn't exist there # yet or the existing one is older than our target. subprocess.run(['make', target], check=True) libdir = os.path.join(builddir, 'prefablib') libpath = os.path.join(libdir, 'libballisticaplus.a') update = True time1 = os.path.getmtime(target) if os.path.exists(libpath): time2 = os.path.getmtime(libpath) if time1 <= time2: update = False if update: if not os.path.exists(libdir): os.makedirs(libdir, exist_ok=True) subprocess.run(['cp', target, libdir], check=True)
[docs] def android_archive_unstripped_libs() -> None: """Copy libs to a build archive.""" import subprocess from pathlib import Path from efro.error import CleanError from efro.terminal import Clr if len(sys.argv) != 4: raise CleanError('Expected 2 args; src-dir and dst-dir') src = Path(sys.argv[2]) dst = Path(sys.argv[3]) if dst.exists(): subprocess.run(['rm', '-rf', dst], check=True) dst.mkdir(parents=True, exist_ok=True) if not src.is_dir(): raise CleanError(f"Source dir not found: '{src}'") libname = 'libmain' libext = '.so' for abi, abishort in [ ('armeabi-v7a', 'arm'), ('arm64-v8a', 'arm64'), ('x86', 'x86'), ('x86_64', 'x86-64'), ]: srcpath = Path(src, abi, libname + libext) dstname = f'{libname}_{abishort}{libext}' dstpath = Path(dst, dstname) if srcpath.exists(): print(f'Archiving unstripped library: {Clr.BLD}{dstname}{Clr.RST}') subprocess.run(['cp', srcpath, dstpath], check=True) subprocess.run( ['tar', '-zcf', dstname + '.tgz', dstname], cwd=dst, check=True ) subprocess.run(['rm', dstpath], check=True)
[docs] def spinoff_test() -> None: """Test spinoff functionality.""" import batools.spinoff batools.spinoff.spinoff_test(sys.argv[2:])
[docs] def spinoff_check_submodule_parent() -> None: """Make sure this dst proj has a submodule parent.""" import os from efro.error import CleanError # Make sure we're a spinoff dst project. The spinoff command will be # a symlink if this is the case. if not os.path.exists('tools/spinoff'): raise CleanError( 'This does not appear to be a spinoff-enabled project.' ) if not os.path.islink('tools/spinoff'): raise CleanError('This project is a spinoff parent; we require a dst.') if not os.path.isdir('submodules/ballistica'): raise CleanError( 'This project is not using a submodule for its parent.\n' 'To set one up, run `tools/spinoff add-submodule-parent`' )
[docs] def gen_python_init_module() -> None: """Generate a basic __init__.py.""" import os from efro.error import CleanError from efro.terminal import Clr from batools.project import project_centric_path if len(sys.argv) != 3: raise CleanError('Expected an outfile arg.') outfilename = sys.argv[2] os.makedirs(os.path.dirname(outfilename), exist_ok=True) prettypath = project_centric_path( projroot=str(pcommand.PROJROOT), path=outfilename ) print(f'Codegen-building {Clr.BLD}{prettypath}{Clr.RST}') with open(outfilename, 'w', encoding='utf-8') as outfile: outfile.write( '# Released under the MIT License.' ' See LICENSE for details.\n' '#\n' )
[docs] def tests_warm_start() -> None: """Warm-start some stuff needed by tests. This keeps logs clearer by showing any binary builds/downloads we need to do instead of having those silently happen as part of tests. """ from batools import apprun # We do lots of apprun.python_command() within test. Pre-build the # binary that they need to do their thing. if not apprun.test_runs_disabled(): apprun.acquire_binary(purpose='running tests')
[docs] def wsl_build_check() -> None: """Sanity-check the environment before we hand off to MSBuild. Usage: ``tools/pcommand wsl_build_check <msbuild-path>``. Verifies both that we're building from a Windows drive and that the Visual Studio we build with is actually present, explaining what to do about it if not. """ from efro.error import CleanError from batools import wslbuild # We use env vars to influence our behavior and thus can't support # batch. args = pcommand.get_args() if len(args) != 1: raise CleanError('Expected a single msbuild-path arg.') wslbuild.check_win_drive() wslbuild.check_msbuild(args[0])
[docs] def wsl_path_to_win() -> None: """Forward escape slashes in a provided win path arg.""" import subprocess import logging import os from efro.error import CleanError try: create = False escape = False if len(sys.argv) < 3: raise CleanError('Expected at least 1 path arg.') wsl_path: str | None = None for arg in sys.argv[2:]: if arg == '--create': create = True elif arg == '--escape': escape = True else: if wsl_path is not None: raise CleanError('More than one path provided.') wsl_path = arg if wsl_path is None: raise CleanError('No path provided.') # wslpath fails on nonexistent paths; make it clear when that happens. if create: os.makedirs(wsl_path, exist_ok=True) if not os.path.exists(wsl_path): raise CleanError(f'Path \'{wsl_path}\' does not exist.') results = subprocess.run( ['wslpath', '-w', '-a', wsl_path], capture_output=True, check=True ) except Exception: # This gets used in a makefile so our returncode is ignored; # let's try to make our failure known in other ways. logging.exception('wsl_to_escaped_win_path failed.') print('wsl_to_escaped_win_path_error_occurred', end='') return out = results.stdout.decode().strip() # If our input ended with a slash, match in the output. if wsl_path.endswith('/') and not out.endswith('\\'): out += '\\' if escape: out = out.replace('\\', '\\\\') print(out, end='')
[docs] def get_modern_make() -> None: """Print name of a modern make command.""" import platform import subprocess # Mac gnu make is outdated (due to newer versions using GPL3 I believe). # so let's return 'gmake' there which will point to homebrew make which # should be up to date. if platform.system() == 'Darwin': if ( subprocess.run( ['which', 'gmake'], check=False, capture_output=True ).returncode != 0 ): print( 'WARNING: this requires gmake (mac system make is too old).' " Install it with 'brew install make'", file=sys.stderr, flush=True, ) print('gmake') else: print('make')
[docs] def assetpins() -> None: """Inspect and update asset-package pins. **Usage**:: tools/pcommand assetpins tools/pcommand assetpins update <TARGET> <VERSION> [--force] TARGET: all | <package-name> | <file-path> VERSION: latest | prod | test | dev | <version> | <account-or-tag>.<package>.<version> **Examples:** ``make assetpins`` Show current pins. Same as ``tools/pcommand assetpins``. ``make assetpins-latest`` Pin everything to the latest version in its current track (dev/test/prod). Same as ``tools/pcommand assetpins update all latest``. ``tools/pcommand assetpins update myassetpack dev`` Pin every myassetpack to the latest dev version of itself. ``tools/pcommand assetpins update all prod`` Pin everything to the latest prod version of itself. ``tools/pcommand assetpins update pconfig/projectconfig.json test260513`` Pin one specific file to a specific version. ``tools/pcommand assetpins update myoldassets efro.newassets.test260518`` Pin every myoldassets to a specific version. With this long form you can switch assetpacks completely. Full semantics (pin discovery, apverid schema/tracks, ``--force``) live in the ``batools.assetpins`` module docstring. """ from pathlib import Path from efro.error import CleanError from efro.terminal import Clr from batools import assetpins as assetpins_module args = sys.argv[2:] if not args or args == ['list']: assetpins_module.do_list(Path(pcommand.PROJROOT)) return if args == ['help']: # Full usage lives in this docstring; the pcommand framework's # built-in help renders it. print( f'For usage, run {Clr.BLD}tools/pcommand help' f' assetpins{Clr.RST} (or {Clr.BLD}make' f' assetpins-help{Clr.RST}).' ) return if args[0] == 'update': rest = args[1:] force = '--force' in rest rest = [a for a in rest if a != '--force'] if len(rest) != 2: raise CleanError( f'assetpins update: expected exactly two args' f' (TARGET VERSION), got {len(rest)}.' f' Try {Clr.BLD}assetpins update all latest{Clr.RST}.' ) target_str, version_str = rest assetpins_module.do_update( Path(pcommand.PROJROOT), target_str, version_str, force=force ) return # Tooling-only (blessing/pubsync gates); deliberately undocumented # in the help docstring. if args[0] == 'assert-prod-only': if len(args) > 1: raise CleanError('assetpins assert-prod-only: takes no args.') offenders = assetpins_module.do_check(Path(pcommand.PROJROOT)) if not offenders: print(f'{Clr.GRN}assetpins assert-prod-only: clean.{Clr.RST}') return lines = [ f'{Clr.RED}assetpins assert-prod-only: found' f' non-prod pin(s):{Clr.RST}' ] for pin in offenders: lines.append( f' {Clr.BLD}{pin.file_path}{Clr.RST}:' f' {Clr.CYN}{pin.apverid}{Clr.RST}' f' ({pin.pin_type.value})' ) lines.append( f'{Clr.YLW}Fix: run `make assetpins-latest` to move to the' f' current-track latest, or `tools/pcommand assetpins update' f' <target> prod` to switch to prod.{Clr.RST}' ) raise CleanError('\n'.join(lines)) raise CleanError( f'Unknown assetpins subcommand: {args[0]!r}.' f' Try {Clr.BLD}assetpins{Clr.RST} (list),' f' {Clr.BLD}assetpins update <TARGET> <VERSION>{Clr.RST}, or' f' {Clr.BLD}make assetpins-help{Clr.RST} (full usage).' )
[docs] def asset_bundle_build() -> None: """Assemble a named asset-bundle profile for the projectconfig pins. Args: a bundle-profile name (see :mod:`batools.assetbundleprofiles`) plus ``--form-factor <desktop|mobile>`` for form-factor-specific profiles (``store``). See ``batools._assetbundlebuild.build()`` for the full story. """ from batools._assetbundlebuild import build build(str(pcommand.PROJROOT), pcommand.get_args())
[docs] def gen_base_asset_set_py() -> None: """Generate babase.BaseAssetSet from the base-asset spec. Emits ``babase/_generated/base_asset_set.py`` from ``src/codegen/babasecodegen/base_assets.py``. Companion to ``gen_base_asset_set_cpp``; the base sibling of ``gen_scene_asset_set_py``. """ import batools.base_assets from efro.error import CleanError if len(sys.argv) != 3: raise CleanError('Expected 1 arg (output path).') batools.base_assets.generate_python( projroot=str(pcommand.PROJROOT), out_path=sys.argv[2] )
[docs] def gen_base_asset_set_cpp() -> None: """Generate the base_asset_set .h / _unpack.inc / _placeholders.inc.""" import batools.base_assets from efro.error import CleanError if len(sys.argv) != 3: raise CleanError('Expected 1 arg (output path).') batools.base_assets.generate_cpp( projroot=str(pcommand.PROJROOT), out_path=sys.argv[2] )
[docs] def gen_character_ranges_cpp() -> None: """Generate character_ranges.h from the shared span table. Emits ``base/generated/character_ranges.h`` from ``tools/bacommon/characterranges.py`` (see :mod:`batools.character_ranges`). """ import batools.character_ranges from efro.error import CleanError if len(sys.argv) != 3: raise CleanError('Expected 1 arg (output path).') batools.character_ranges.write_cpp(sys.argv[2])
[docs] def gen_special_chars_cpp() -> None: """Generate special_chars.h from the shared special-char enum. Emits ``base/generated/special_chars.h`` from ``tools/bacommon/text.py`` (see :mod:`batools.special_chars`). """ import batools.special_chars from efro.error import CleanError if len(sys.argv) != 3: raise CleanError('Expected 1 arg (output path).') batools.special_chars.write_cpp(sys.argv[2])
[docs] def gen_character_rig_cpp() -> None: """Generate one of the engine's character rig files. Emits C++ from ``tools/bacommon/characterrig.py``; which file is chosen by the output path's name (see :mod:`batools.character_rig`). """ import batools.character_rig from efro.error import CleanError if len(sys.argv) != 3: raise CleanError('Expected 1 arg (output path).') batools.character_rig.write_cpp(sys.argv[2])
[docs] def gen_scene_asset_set_py() -> None: """Generate bascenev1.SceneV1AssetSet from the scene-asset spec. Emits ``bascenev1/_generated/scene_asset_set.py`` from ``src/codegen/bascenev1codegen/scene_assets.py``. Companion to ``gen_scene_asset_set_cpp``; the scene sibling of ``gen_ui_asset_set_py``. """ import batools.scene_assets from efro.error import CleanError if len(sys.argv) != 3: raise CleanError('Expected 1 arg (output path).') batools.scene_assets.generate_python( projroot=str(pcommand.PROJROOT), out_path=sys.argv[2] )
[docs] def gen_scene_asset_set_cpp() -> None: """Generate scene_asset_set.h / scene_asset_set_unpack.inc.""" import batools.scene_assets from efro.error import CleanError if len(sys.argv) != 3: raise CleanError('Expected 1 arg (output path).') batools.scene_assets.generate_cpp( projroot=str(pcommand.PROJROOT), out_path=sys.argv[2] )
[docs] def gen_ui_asset_set_py() -> None: """Generate the bauiv1.UIAssetSet dataclass from the ui-asset spec. Emits ``bauiv1/_generated/<set>.py`` from ``src/codegen/bauiv1codegen/ui_assets.py``. Companion to ``gen_ui_asset_set_cpp``, which emits the matching native struct and the unpacker between them; generating all three from one spec is what keeps a renamed or added slot from silently drifting. """ import batools.ui_assets from efro.error import CleanError if len(sys.argv) != 3: raise CleanError('Expected 1 arg (output path).') batools.ui_assets.generate_python( projroot=str(pcommand.PROJROOT), out_path=sys.argv[2] )
[docs] def gen_ui_asset_set_cpp() -> None: """Generate ui_asset_set.h / ui_asset_set_unpack.inc from spec.""" import batools.ui_assets from efro.error import CleanError if len(sys.argv) != 3: raise CleanError('Expected 1 arg (output path).') batools.ui_assets.generate_cpp( projroot=str(pcommand.PROJROOT), out_path=sys.argv[2] )
# Docs-generation hack; import some stuff that we likely only forward-declared # in our actual source code so that docs tools can find it. from typing import (Coroutine, Any, Literal, Callable, Generator, Awaitable, Sequence, Self, TypeIs) import asyncio from concurrent.futures import Future from pathlib import Path from enum import Enum