# Released under the MIT License. See LICENSE for details.
#
"""Bundle-manifest loading for the asset-packages CAS pipeline.
The native build pipeline stages a top-level ``ba_data/manifest.json``
plus per-bucket manifest blobs in the CAS store
(``ba_data/assets/<aa>/<rest>``). At startup we parse those and push
the resolved ``logical_path → CAS hash`` mappings into the C++
:class:`AssetPackageRegistry` via
:func:`_babase.register_asset_package_bucket`, so subsequent
``aptextureget(``'apvernum:asset'``)``-style lookups can resolve
GIL-free in C++.
"""
from __future__ import annotations # Docs-generation hack.
import os
import json
import logging
import traceback
from typing import TYPE_CHECKING
import _babase
from bacommon.assetpackage import ApverNum
from efro.util import strip_exception_tracebacks
if TYPE_CHECKING:
from typing import Any, Callable
from bacommon.assetspec import AssetBucketKind
_lifecyclelog = logging.getLogger('ba.lifecycle')
# Numeric ids of the BUILTIN (bundled) packages, populated at startup by
# :func:`load_bundled_asset_packages` (one per bundled package).
# Membership here means "builtin", which drives builtin-only resolve
# semantics (bundled-fallback flavor; see ``AssetSubsystem._is_builtin``)
# and qualified-ref construction -- so a runtime-resolved *non-builtin*
# package must NOT land here (see ``_resolved_apvernums``).
_builtin_apvernums: list[ApverNum] = []
# Numeric ids of non-builtin packages brought in by a runtime resolve
# (``babase.App.assets.resolve``). Tracked separately so the language
# table merges their strings too WITHOUT making them count as builtin.
_resolved_apvernums: list[ApverNum] = []
# Set once construct-mode has resolved every required asset-package (see
# :func:`mark_construct_complete`). Until then, only the construct package
# itself is legitimately loadable.
_g_construct_complete = False
# The construct/builtin package's apvernum, cached on first check.
_g_construct_apvernum: ApverNum | None = None
def mark_construct_complete() -> None:
"""Note that construct-mode has finished resolving asset-packages.
Called from construct-mode's hand-off to the real app-mode -- the one
point at which every package the meta-scan requires is guaranteed
resolved and registered. Opens the gate enforced by
:func:`check_asset_package_load`.
Also opens the native gate
(``AssetPackageRegistry::CheckPreConstructAccess``), which covers the
load paths that never touch Python: direct C++ ``Assets::Get*``
calls, and scene_v1 wire traffic / replays arriving as legacy bare
names. Both open here so they cannot disagree about when bring-up
ended.
"""
global _g_construct_complete # pylint: disable=global-statement
_g_construct_complete = True
_babase.mark_construct_assets_complete()
# Logged because this is otherwise an invisible state change on a
# load-ordering invariant -- and because it is the marker to key any
# 'did bring-up finish?' check on: it sits on ba.lifecycle alongside
# the rest of the boot trace, where construct-mode's own hand-off
# log is over on ba.assetmanager.
_lifecyclelog.info('Construct-mode asset gate opened.')
# Run whatever was waiting on us (see call_after_construct()), each
# as its own call so one failing can't stop the rest or the
# hand-off itself.
pending = list(_g_after_construct_calls)
_g_after_construct_calls.clear()
for call in pending:
_babase.pushcall(call)
# Calls waiting on construct-mode success (see call_after_construct()).
_g_after_construct_calls: list[Callable[[], None]] = []
def call_after_construct(call: Callable[[], None]) -> None:
"""Run a call once construct-mode has succeeded.
Runs it soon (on the logic thread) if construct-mode already has;
otherwise queues it until the asset gate opens. For things that use
non-builtin assets but can come up during app loading -- a notice
that plays a sound, say -- and would rather wait than go without.
Queued calls run only on *success*: if construct-mode never
finishes (the user quits, or this build is too old to load current
assets), they never run. Logic thread only.
"""
assert _babase.in_logic_thread()
if _g_construct_complete:
_babase.pushcall(call)
else:
_g_after_construct_calls.append(call)
def construct_assets_complete() -> bool:
"""Whether construct-mode has finished resolving asset-packages.
For callers that need to *avoid* doing something too early rather
than be scolded for it -- notably the dev console's AppModes tab,
which would otherwise exec app-mode modules (wrapper modules
included) while their packages are still unresolved.
"""
return _g_construct_complete
[docs]
def check_asset_package_load(apvernum: ApverNum, path: str) -> None:
"""Flag an asset load from a package that is not up yet.
Before construct-mode hands off, the only package guaranteed
registered is the construct/builtin one; loading from any other is a
bug even when it happens to work. It works whenever the package is
*bundled* into this particular build (bundled packages register
during native bootstrapping, via
``load_bundled_asset_packages()``), so the same code silently
succeeds or fails depending on the build's bundle profile -- and
headless never notices at all, since texture loads there
short-circuit before the package registry is consulted. Hence this
check keys on the construct package rather than on what is merely
registered.
Raises on debug builds so the offending call site fails loudly in
dev and CI; logs an error elsewhere. The underlying failure is worse
either way -- a dead ``on_app_loading`` hook, or an asset that goes
permanently ``kFailed`` and draws blank forever.
Hold the wrapper's *reference* and load it later (on first access,
once the ui that wants it exists) rather than loading at
construction time; see :class:`bascenev1.Level`.
"""
if _g_construct_complete:
return
global _g_construct_apvernum # pylint: disable=global-statement
if _g_construct_apvernum is None:
# Deferred: this module is imported while babase itself is still
# coming up, well before the wrapper is importable.
# pylint: disable-next=cyclic-import
from babase import _builtinassets
# Package identity for the construct pin, not a path.
# pylint: disable-next=protected-access
_g_construct_apvernum = _builtinassets._ASSET_PACKAGE
if apvernum == _g_construct_apvernum:
return
msg = (
f"Asset '{apvernum}:{path}' loaded before construct-mode finished"
f' resolving asset-packages; only {_g_construct_apvernum} is'
f' available this early. Hold the wrapper reference and load it'
f' on first use instead.'
)
# Keyed on debug-build, matching the native gate
# (``AssetPackageRegistry::CheckPreConstructAccess``) so the two
# cannot disagree about how loud to be. Note the *check* itself runs
# everywhere -- release builds log this error rather than skipping
# it; only the raise-vs-log severity varies.
if _babase.app.env.debug_build:
raise RuntimeError(msg)
_lifecyclelog.error(msg)
def builtin_asset_package_apvernums() -> list[ApverNum]:
"""Apverids of the bundled/builtin packages (registered at startup).
The builtin-only set: drives ``_is_builtin`` (bundled-fallback resolve
semantics) and qualified-ref construction. Use
:func:`loaded_asset_package_apvernums` instead for "every loaded
package" (e.g. rebuilding the language table).
"""
return list(_builtin_apvernums)
[docs]
def loaded_asset_package_apvernums() -> list[ApverNum]:
"""Return every currently-loaded apvernum: builtin + runtime-resolved.
This is the set the native language table is rebuilt from (so every
loaded package's strings merge) and that a locale switch re-resolves.
Builtins are populated at startup by ``load_bundled_asset_packages``;
runtime-resolved packages are added by ``register_resolved_apvernums``
after a successful ``resolve``. Distinct from
``builtin_asset_package_apvernums``, which alone must drive
builtin-only behavior.
"""
out = list(_builtin_apvernums)
out.extend(a for a in _resolved_apvernums if a not in _builtin_apvernums)
return out
[docs]
def asset_package_bucket_paths(
apvernum: ApverNum, kind: 'AssetBucketKind'
) -> list[str] | None:
"""Canonical sorted logical paths in a loaded package's bucket.
This is the list integer asset references address -- doc-ui's flat
refs and scene_v1's indexed wire refs both index into it -- and it
is portable across flavors by the identical-key-set invariant
(asset-packages D23/D24), so two ends holding different texture
profiles still agree on what an index means.
``None`` means the package isn't registered, which is a
resolve-ordering fault worth surfacing; distinguish it from ``[]``,
a package that genuinely has no assets of that kind.
"""
return _babase.get_asset_package_bucket_paths(apvernum, kind.value)
[docs]
def asset_package_string_count(apvernum: ApverNum) -> int | None:
"""How many language-strings a loaded package holds, this locale.
The size of the canonical sorted name list that string indices
address -- enough to fold a flat wire index back into the
``(package, string)`` pair the native decoder consumes, without
surfacing the names themselves.
``None`` when the package has no language table loaded, which must
be distinguished from a package holding zero strings: treating the
first as zero would silently shift every later package's offset.
"""
return _babase.get_asset_package_string_count(apvernum)
def register_resolved_apvernums(apvernums: list[ApverNum]) -> None:
"""Record runtime-resolved (non-builtin) packages as loaded.
Called after a successful downloading/offline ``resolve`` commit so
:meth:`~babase.AssetSubsystem._reload_language` merges the package's
``language`` bucket into the native table automatically (no caller-side
reload needed). Builtins are skipped (already loaded) and duplicates
ignored, so it's safe to pass the whole resolve batch.
"""
for apvernum in apvernums:
if apvernum in _builtin_apvernums or apvernum in _resolved_apvernums:
continue
_resolved_apvernums.append(apvernum)
def load_bundled_asset_packages() -> str | None:
"""Register builtin asset-packages, reporting failure as a string.
The native bootstrapping entry point. Returns ``None`` on success
and a short one-line description of the cause on failure (exception
type, message, and the innermost few frames), which the native side
puts in its fatal-error text. That text is the only part of a
failure here that reaches us from the field; the logged exception
stays on the device.
"""
try:
_load_bundled_asset_packages()
except Exception as exc:
_lifecyclelog.exception('Error loading bundled asset-packages.')
frames = traceback.extract_tb(exc.__traceback__)[-4:]
where = ' < '.join(
f'{os.path.basename(f.filename)}:{f.lineno} {f.name}'
for f in reversed(frames)
)
return f'{type(exc).__name__}: {str(exc)[:400]} ({where})'
return None
def load_bundled_language() -> None:
"""Register the construct package's strings ahead of everything else.
Builtin strings are usable from the moment the boot ``setlanguage``
runs; this is what makes that true. The full bundled load
(:func:`load_bundled_asset_packages`) has to wait for graphics,
since picking a texture flavor needs GPU caps, and on a slow or
backgrounded Android launch that wait can last seconds. Strings
need no graphics, so the construct package's ``language`` bucket
is registered here, before the first table rebuild. The full load
later replaces this registration with the package's complete
bucket set.
A no-op once any builtin package is registered (or when the build
bundles none). Failure is logged, not fatal: the full load runs the
same resolve moments later and fails loudly if it's real.
"""
if _builtin_apvernums:
return
manifest_text = _babase.bundled_asset_manifest_text()
if manifest_text is None:
return
# Deferred: this module is imported while babase itself is still
# coming up, well before the wrapper is importable.
# pylint: disable-next=cyclic-import
from babase import _builtinassets
# Package identity for the construct pin, not a path.
# pylint: disable-next=protected-access
construct = _builtinassets._ASSET_PACKAGE
bundle = json.loads(manifest_text)
if construct not in (a for a, _ in _iter_manifest_packages(bundle)):
return
_builtin_apvernums.append(construct)
try:
_babase.app.assets.resolve_local([construct], language_only=True)
except Exception as exc:
_lifecyclelog.exception('Error registering builtin language strings.')
strip_exception_tracebacks(exc)
def _load_bundled_asset_packages() -> None:
"""Register builtin asset-packages at their best LOCAL flavor.
Called once during native bootstrapping, *before* ``StartLoading``'s
builtin asset loads. Discovers the builtin packages from the bundled
``manifest.json`` (its keys) and hands them to
:meth:`~babase.AssetSubsystem.resolve_local`, which registers the ideal
flavor of each when its blobs are already cached (warm starts) and the
bundled fallback otherwise (cold starts -- until a later downloading
resolve fetches and swaps the ideal flavor in).
A missing ``manifest.json`` is treated as "no bundled CAS assets" and
logged at debug level -- headless/server builds and tests may run without
one.
"""
# Served natively; the manifest may live inside an archive (the
# apk on Android) rather than as a plain file.
manifest_text = _babase.bundled_asset_manifest_text()
if manifest_text is None:
_lifecyclelog.debug(
'No bundled asset-package manifest present; skipping CAS init.',
)
return
bundle = json.loads(manifest_text)
# Bundled packages start out builtin (those that turn out to have
# no usable local flavor are dropped again below). Record them (so
# _is_builtin and ref-construction see them) before resolving, then let
# the AssetSubsystem register the best-local flavor of each.
apvernums = [apvernum for apvernum, _ in _iter_manifest_packages(bundle)]
for apvernum in apvernums:
if apvernum not in _builtin_apvernums:
_builtin_apvernums.append(apvernum)
if apvernums:
# Only the construct package is mandatory this early: it alone
# is guaranteed a usable flavor in every bundle, and it is all
# construct-mode needs to come up and download the rest. Any
# other bundled package is a bonus that registers when the
# bundle happens to hold a flavor this device can use (a store
# bundle carries only the native texture flavor of those).
#
# Deferred: this module is imported while babase itself is still
# coming up, well before the wrapper is importable.
# pylint: disable-next=cyclic-import
from babase import _builtinassets
# Package identity for the construct pin, not a path.
# pylint: disable-next=protected-access
construct = _builtinassets._ASSET_PACKAGE
optional = {a for a in apvernums if a != construct}
# resolve_local registers the packages' buckets (including
# ``language/<locale>``) and rebuilds the native language string
# table from them — so this is what actually populates the table
# at startup (the boot-time ``setlanguage`` may have run earlier,
# before any packages were loaded).
result = _babase.app.assets.resolve_local(apvernums, optional=optional)
# A skipped package is not builtin for this run: nothing of it
# is registered and it has no bundled floor to fall back on, so
# construct-mode's resolve must treat it as a plain download.
for apvernum in apvernums:
if apvernum not in result.apvernums:
_builtin_apvernums.remove(apvernum)
def _iter_manifest_packages(
bundle: dict[str, Any],
) -> list[tuple[ApverNum, dict[str, str]]]:
"""Return ``(apvernum, flavor_manifests)`` pairs from a parsed manifest.
Entries are keyed by numeric id (as text, being json object keys).
"""
return [
(ApverNum(int(key)), entry['flavor_manifests'])
for key, entry in bundle.get('asset_package_versions', {}).items()
]
# Docs-generation hack; import some stuff that we likely only forward-declared
# in our actual source code so that docs tools can find it.
from typing import (Coroutine, Any, Literal, Callable,
Generator, Awaitable, Sequence, Self, TypeIs)
import asyncio
from concurrent.futures import Future
from pathlib import Path
from enum import Enum