From 22dd92982e11724091eee7db77adc0ce46f21fa6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=B6=BE=E7=80=AC=E6=A1=83=E6=A1=83?= Date: Wed, 4 Nov 2026 13:14:25 +0800 Subject: [PATCH] fix(android): bind metadata recovery to its application --- senbei-io/src/android/context.rs | 512 +++++++++++++ senbei-io/src/android/manifest.rs | 713 +++++++++++++++++++ senbei-io/src/android/mod.rs | 813 +++++++++++++++++---- senbei-io/src/android/tests.rs | 1107 +++++++++++++++++++++++++++++ senbei-io/src/job.rs | 286 +++++--- 5 files changed, 3173 insertions(+), 258 deletions(-) create mode 100644 senbei-io/src/android/context.rs create mode 100644 senbei-io/src/android/manifest.rs diff --git a/senbei-io/src/android/context.rs b/senbei-io/src/android/context.rs new file mode 100644 index 0000000..81d76bc --- /dev/null +++ b/senbei-io/src/android/context.rs @@ -0,0 +1,512 @@ +//! Application-scoped module `0x0C` context for Android metadata recovery. +//! +//! One decoded module belongs to one application. Split APKs of that application +//! share it; unrelated games under the same scan do not. A metadata blob is +//! restored only after its application context is unambiguous, and the identity +//! of a successful restore includes that context: the same encrypted bytes can +//! decode differently under two module profiles. + +use std::collections::{BTreeMap, HashMap}; +use std::path::{Component, Path, PathBuf}; + +use anyhow::Result; + +/// Where a decoded module `0x0C` image was read from. +#[derive(Debug, Clone, PartialEq, Eq)] +pub(crate) struct ModuleSource { + /// Protected library this module was recovered from (package label or path). + pub label: String, + /// Stage-2 image name inside that library, when the restorer reported one. + pub image: Option, +} + +/// One application's collected module `0x0C` images. +/// +/// Distinct byte profiles are kept. Duplicate libraries populate the same +/// context instead of replacing it, so a later game cannot win by being last. +#[derive(Debug, Default, Clone)] +pub(crate) struct ApplicationModules { + profiles: BTreeMap>, + sources: Vec, +} + +impl ApplicationModules { + pub(crate) fn record(&mut self, module: &[u8], source: ModuleSource) { + let identity = crate::android::content_identity(module); + self.profiles + .entry(identity) + .or_insert_with(|| module.to_vec()); + self.sources.push(source); + } + + pub(crate) fn is_empty(&self) -> bool { + self.profiles.is_empty() + } + + /// The single module image, or `None` when this application recovered none + /// or more than one distinct profile. + pub(crate) fn unambiguous_module(&self) -> Option<&[u8]> { + let mut values = self.profiles.values(); + let first = values.next()?; + values.next().is_none().then_some(first.as_slice()) + } + + pub(crate) fn profile_count(&self) -> usize { + self.profiles.len() + } + + pub(crate) fn sources(&self) -> &[ModuleSource] { + &self.sources + } +} + +/// Why a metadata blob cannot be paired with exactly one module `0x0C`. +#[derive(Debug, Clone, PartialEq, Eq)] +pub(crate) enum ContextAmbiguity { + /// No library in this application yielded a module. + Missing, + /// Libraries in this application yielded more than one distinct module. + Conflicting { profiles: usize }, +} + +/// Run-scoped cache of restored libraries and application module contexts. +/// +/// The public package API still takes a content-identity set. This cache is +/// internal: a library hit reuses the restored image, while a metadata hit is +/// keyed by the recovery context as well as the encrypted bytes. +#[derive(Debug, Default)] +pub(crate) struct AndroidRunCache { + apps: HashMap, + /// Successfully restored protected libraries, keyed by encrypted content. + libraries: HashMap, + /// Successfully handled metadata, keyed by encrypted content and context. + metadata: HashMap, + /// Content identities already restored before this cache existed (the public + /// `HashSet` contract). They skip a repeat of the same bytes but carry no + /// module image, so they never become another application's profile. + prior_success: std::collections::HashSet, +} + +#[derive(Debug, Clone, PartialEq, Eq, Hash)] +struct MetadataCacheKey { + content: String, + context: String, +} + +#[derive(Debug, Clone)] +pub(crate) struct CachedLibrary { + /// Module `0x0C` from a library whose restore, including any embedded + /// metadata write, succeeded. The restored image is not kept: a duplicate + /// writes nothing. + pub method_index_module: Option>, + pub module_image: Option, +} + +#[derive(Debug, Clone)] +pub(crate) enum CachedMetadata { + /// Successfully written package metadata. A duplicate does not write again. + Restored { remapped: usize }, + /// The blob was already canonical under this context. + Unchanged, +} + +impl AndroidRunCache { + pub(crate) fn modules_mut(&mut self, key: &AppKey) -> &mut ApplicationModules { + self.apps.entry(key.clone()).or_default() + } + + pub(crate) fn modules(&self, key: &AppKey) -> Option<&ApplicationModules> { + self.apps.get(key) + } + + pub(crate) fn library(&self, content: &str) -> Option<&CachedLibrary> { + self.libraries.get(content) + } + + pub(crate) fn insert_prior_success(&mut self, content: String) { + self.prior_success.insert(content); + } + + pub(crate) fn is_prior_success(&self, content: &str) -> bool { + self.prior_success.contains(content) + } + + /// Content identities safe to publish back to the public successful-only set: + /// prior successes plus libraries and metadata handled successfully here. + /// Metadata identities are included because the public set still dedups a + /// repeated blob inside one package call; cross-application recovery uses + /// the context-keyed cache instead of this set. + pub(crate) fn successful_content_identities(&self) -> Vec { + let mut identities: Vec = self.prior_success.iter().cloned().collect(); + identities.extend(self.libraries.keys().cloned()); + identities.extend(self.metadata.keys().map(|key| key.content.clone())); + identities.sort(); + identities.dedup(); + identities + } + + pub(crate) fn insert_library(&mut self, content: String, cached: CachedLibrary) { + self.libraries.entry(content).or_insert(cached); + } + + pub(crate) fn metadata(&self, content: &str, context: &str) -> Option<&CachedMetadata> { + self.metadata.get(&MetadataCacheKey { + content: content.to_owned(), + context: context.to_owned(), + }) + } + + pub(crate) fn insert_metadata( + &mut self, + content: String, + context: String, + cached: CachedMetadata, + ) { + self.metadata + .entry(MetadataCacheKey { content, context }) + .or_insert(cached); + } +} + +/// Identity of one Android application inside a folder run or one package set. +/// +/// Constructed only from manifest package name **and** version, or the +/// conservative extracted `lib//` + `assets/` boundary. The same package +/// name in a different version, or in a different bundle / loose tree, is a +/// different application. Scan-root names and ZIP entry names are never used +/// as a guess. +#[derive(Debug, Clone, PartialEq, Eq, Hash)] +pub(crate) enum AppKey { + /// Manifest `package` plus `versionCode` (or `versionName` when the code is + /// absent). `package_set` is the directory of the package file, or the + /// bundle file itself, so the same name and version elsewhere do not merge. + Manifest { + package: String, + version: String, + package_set: PathBuf, + }, + /// Parent of a real `lib//` or `assets/` boundary. + Extracted(PathBuf), + /// No manifest and no extracted boundary. Isolated so it cannot see another + /// application's modules. Not shared with any other unidentifiable input. + Unidentifiable { source: String }, +} + +/// A metadata or library input assigned to one application context. +#[derive(Debug, Clone, PartialEq, Eq)] +pub(crate) struct ScopedInput { + pub path: PathBuf, + pub key: AppKey, +} + +/// Group loose Android targets by application without crossing unrelated games. +/// +/// A `lib//` library and an `assets/` metadata blob share a context only +/// when both sit directly under the same application root. Anything else is +/// isolated. Order of `libraries` and `metadata` does not matter. +pub(crate) fn group_loose_inputs(libraries: &[PathBuf], metadata: &[PathBuf]) -> Vec { + let mut scoped = Vec::with_capacity(libraries.len() + metadata.len()); + for path in libraries { + scoped.push(ScopedInput { + key: loose_library_key(path), + path: path.clone(), + }); + } + for path in metadata { + scoped.push(ScopedInput { + key: loose_metadata_key(path), + path: path.clone(), + }); + } + scoped.sort_by(|left, right| left.path.cmp(&right.path)); + scoped +} + +fn loose_library_key(path: &Path) -> AppKey { + extracted_app_root(path).map_or_else( + || AppKey::Unidentifiable { + source: path.to_string_lossy().into_owned(), + }, + AppKey::Extracted, + ) +} + +fn loose_metadata_key(path: &Path) -> AppKey { + // `assets/bin/Data/Managed/Metadata/global-metadata.dat` and the shorter + // `assets/bin/Data/Managed/global-metadata.dat` both end at `assets/`. + assets_app_root(path).map_or_else( + || AppKey::Unidentifiable { + source: path.to_string_lossy().into_owned(), + }, + AppKey::Extracted, + ) +} + +/// Application root when `path` is `/lib//.so`. +fn extracted_app_root(path: &Path) -> Option { + let mut components: Vec> = path.components().collect(); + let file = components.pop()?; + let abi = components.pop()?; + let lib = components.pop()?; + if !component_is(file, path.file_name()?.to_str()?) { + return None; + } + // Compare components, not `Path::parent` display. A path joined from mixed + // separators still yields `lib` and `` as two normal components. + if !component_is(lib, "lib") || !matches!(abi, Component::Normal(_)) { + return None; + } + let root: PathBuf = components.into_iter().collect(); + Some(if root.as_os_str().is_empty() { + PathBuf::from(".") + } else { + root + }) +} + +/// Application root when `path` is `/assets/**/global-metadata.dat`. +fn assets_app_root(path: &Path) -> Option { + let mut components: Vec> = path.components().collect(); + let name = components.pop()?; + if !component_is(name, crate::METADATA_FILE_NAME) { + return None; + } + let assets_at = components + .iter() + .rposition(|component| component_is(*component, "assets"))?; + // `assets/global-metadata.dat` and `assets/**/global-metadata.dat` both + // belong to the directory that contains `assets`. + let root: PathBuf = components[..assets_at].iter().collect(); + Some(if root.as_os_str().is_empty() { + PathBuf::from(".") + } else { + root + }) +} + +fn component_eq(path: &Path, expected: &str) -> bool { + // The final component must be exactly `expected`. `file_name` on a directory + // path is that component; requiring the whole path to equal it would reject + // every application root that is not a single path segment. + path.file_name() + .and_then(|name| name.to_str()) + .is_some_and(|name| name.eq_ignore_ascii_case(expected)) +} + +fn component_is(component: Component<'_>, expected: &str) -> bool { + match component { + Component::Normal(name) => name + .to_str() + .is_some_and(|name| name.eq_ignore_ascii_case(expected)), + _ => false, + } +} + +/// One ZIP whose central directory was listed while naming an application. +/// +/// `reopen_key` is the on-disk package (or extracted nested APK) to open again +/// when entries are restored. A later resource budget must charge this archive's +/// physical entry count on the first inventory pass only; reopening it for +/// entry extraction is not a second package. +#[derive(Debug, Clone, PartialEq, Eq)] +pub(crate) struct InventoriedArchive { + pub reopen_key: PathBuf, + /// Central-directory index of `AndroidManifest.xml`, when one was present. + /// `None` means the manifest entry was absent and nothing was read. + pub manifest_index: Option, + /// Central-directory entry count observed by this inventory pass. + pub entry_count: usize, +} + +/// Inventory of one package (or nested split) that can name its application. +#[derive(Debug, Clone)] +pub(crate) struct PackageInventory { + pub key: AppKey, + /// True when `key` came from a parsed manifest `package` and version. + /// Distinguishes a named package from an isolated one for a later report. + #[allow(dead_code)] + pub manifest_named: bool, + /// Reopen handle for this archive. Entry extraction uses the already-open + /// handle; this records the central directory that must be charged once. + pub archive: InventoriedArchive, +} + +/// Manifest-first application identity for one ZIP package. +/// +/// Reads only the root `AndroidManifest.xml` entry, and only the leading window +/// needed for the plain-XML `package` and version attributes. A missing or +/// binary manifest does not fall back to the ZIP file name; the caller isolates +/// that package. `package_set` is the bundle or directory that owns this set of +/// splits — the same package name outside that boundary is a different app. +/// +/// The returned [`InventoriedArchive`] is the reopen handle. Callers restore +/// entries from `archive.reopen_key` and must not treat that second open as a +/// newly discovered package. +pub(crate) fn inventory_manifest_package( + archive: &mut zip::ZipArchive, + reopen_key: &Path, + package_set: &Path, + isolate_as: &str, +) -> Result { + let entry_count = archive.len(); + let mut manifest_index = None; + for index in 0..entry_count { + let entry = archive.by_index(index)?; + if entry.is_dir() { + continue; + } + let Some(name) = entry.enclosed_name() else { + continue; + }; + // Root manifest only. Nested paths and every other entry stay unread. + if name.components().count() == 1 && component_eq(&name, "AndroidManifest.xml") { + manifest_index = Some(index); + break; + } + } + let Some(index) = manifest_index else { + return Ok(PackageInventory { + key: AppKey::Unidentifiable { + source: isolate_as.to_owned(), + }, + manifest_named: false, + archive: InventoriedArchive { + reopen_key: reopen_key.to_path_buf(), + manifest_index: None, + entry_count, + }, + }); + }; + let mut entry = archive.by_index(index)?; + // The package and version attributes are in the root ` break, + read => filled += read, + } + } + buffer.truncate(filled); + let truncated = entry.size() > MANIFEST_PROBE as u64; + let named = super::manifest::identity(&buffer, truncated); + Ok(PackageInventory { + key: match &named { + Some((package, version)) => AppKey::Manifest { + package: package.clone(), + version: version.clone(), + package_set: package_set.to_path_buf(), + }, + None => AppKey::Unidentifiable { + source: isolate_as.to_owned(), + }, + }, + manifest_named: named.is_some(), + archive: InventoriedArchive { + reopen_key: reopen_key.to_path_buf(), + manifest_index: Some(index), + entry_count, + }, + }) +} + +/// Recovery-context identity mixed into metadata dedup. +/// +/// Same encrypted bytes under two modules (or under "no module" versus a module) +/// must not collapse to one cached output. +pub(crate) fn metadata_context_identity(module: Option<&[u8]>) -> String { + match module { + Some(module) => format!("module:{}", crate::android::content_identity(module)), + None => "module:none".to_owned(), + } +} + +/// Resolve the module a metadata blob may use, refusing an ambiguous context +/// before any output is written. +pub(crate) fn binding_module( + modules: Option<&ApplicationModules>, +) -> std::result::Result, ContextAmbiguity> { + let Some(modules) = modules else { + return Ok(None); + }; + if modules.is_empty() { + return Err(ContextAmbiguity::Missing); + } + match modules.unambiguous_module() { + Some(module) => Ok(Some(module)), + None => Err(ContextAmbiguity::Conflicting { + profiles: modules.profile_count(), + }), + } +} + +/// Provenance recorded on a metadata outcome: every library that contributed +/// the module actually used, or the reason no module was bound. +#[derive(Debug, Clone, PartialEq, Eq, Default)] +pub(crate) struct MetadataProvenance { + pub sources: Vec, + pub note: Option, +} + +pub(crate) fn provenance_for( + modules: Option<&ApplicationModules>, + ambiguity: Option<&ContextAmbiguity>, +) -> MetadataProvenance { + match ambiguity { + Some(ContextAmbiguity::Missing) => MetadataProvenance { + sources: modules + .map(|modules| modules.sources().to_vec()) + .unwrap_or_default(), + note: Some("no module 0x0C in this application".to_owned()), + }, + Some(ContextAmbiguity::Conflicting { profiles }) => MetadataProvenance { + sources: modules + .map(|modules| modules.sources().to_vec()) + .unwrap_or_default(), + note: Some(format!( + "ambiguous module 0x0C binding ({profiles} distinct profiles)" + )), + }, + None => MetadataProvenance { + sources: modules + .map(|modules| modules.sources().to_vec()) + .unwrap_or_default(), + note: None, + }, + } +} + +/// What a library restore contributes to an application context. +#[derive(Debug, Clone)] +pub(crate) struct RestoredLibrary { + pub embedded_metadata: Option>, + pub method_index_module: Option>, + pub module_image: Option, +} + +/// Injectable library restore. Tests supply a fake so application-context +/// behaviour does not need a protected ELF fixture. Production calls the +/// stage-2 extractor. +pub(crate) trait LibraryRestorer { + fn restore(&mut self, input: &Path, dest: &Path, verbose: bool) -> Result; + + /// Bytes that should be treated as a protected library without the ELF + /// probe. Production returns `None`; tests return the synthetic payload so + /// a package entry can reach the restorer. + fn protected_override(&mut self, _input: &Path) -> Option> { + None + } +} + +/// Production restorer: stage-2 extract, ELF rebuild, embedded metadata unwrap. +pub(crate) struct EngineRestorer; + +impl LibraryRestorer for EngineRestorer { + fn restore(&mut self, input: &Path, dest: &Path, verbose: bool) -> Result { + crate::android::restore_so_with_engine(input, dest, verbose) + } +} diff --git a/senbei-io/src/android/manifest.rs b/senbei-io/src/android/manifest.rs new file mode 100644 index 0000000..fe1109f --- /dev/null +++ b/senbei-io/src/android/manifest.rs @@ -0,0 +1,713 @@ +//! Bounded identity of one Android manifest. +//! +//! The only result is `(package, version)`. Version is `android:versionCode`, +//! with `android:versionCodeMajor` prefixed when it is present and non-zero. +//! `android:versionName` is not an identity. A truncated or corrupt manifest +//! returns `None` rather than a partial guess. +//! +//! Chunk layout follows `androidfw/ResourceTypes.h`: a start element is a +//! `ResXMLTree_node` (`headerSize` is normally 16) followed by `ResXMLTree_attrExt` +//! at that header size. Attribute records begin at `headerSize + attributeStart`. +//! `Res_value::dataType` is the byte immediately before the data word. + +/// `(package, version)` when the root element is `manifest` and carries a +/// package plus a version code. Truncated probes never supply a binding. +pub(super) fn identity(bytes: &[u8], truncated: bool) -> Option<(String, String)> { + if truncated { + return None; + } + if bytes.len() >= 8 && u16::from_le_bytes([bytes[0], bytes[1]]) == 0x0003 { + return binary_identity(bytes); + } + plain_identity(bytes, truncated) +} + +const TYPE_STRING: u8 = 0x03; +const TYPE_INT_DEC: u8 = 0x10; +const TYPE_INT_HEX: u8 = 0x11; +const VERSION_CODE: u32 = 0x0101_021b; +const VERSION_CODE_MAJOR: u32 = 0x0101_0576; +const NO_STRING: u32 = 0xffff_ffff; + +fn binary_identity(bytes: &[u8]) -> Option<(String, String)> { + if bytes.len() < 8 { + return None; + } + let file_size = u32::from_le_bytes(bytes.get(4..8)?.try_into().ok()?) as usize; + if file_size < 8 || file_size > bytes.len() { + return None; + } + let bytes = bytes.get(..file_size)?; + let mut offset = read_u16(bytes, 2)? as usize; + if offset < 8 || offset > bytes.len() { + return None; + } + let mut pool: Option> = None; + let mut resource_map = Vec::new(); + while offset + 8 <= bytes.len() { + let kind = read_u16(bytes, offset)?; + let header = read_u16(bytes, offset + 2)? as usize; + let size = read_u32(bytes, offset + 4)? as usize; + if header < 8 || size < header || offset.checked_add(size)? > bytes.len() { + return None; + } + let chunk = bytes.get(offset..offset + size)?; + match kind { + 0x0001 => pool = Some(StringPool::parse(chunk)?), + 0x0180 => resource_map = resource_map_of(chunk)?, + 0x0102 => return start_element_identity(chunk, pool.as_ref()?, &resource_map), + _ => {} + } + offset += size; + } + None +} + +fn start_element_identity( + chunk: &[u8], + pool: &StringPool<'_>, + resource_map: &[u32], +) -> Option<(String, String)> { + let header = read_u16(chunk, 2)? as usize; + // ResXMLTree_attrExt sits headerSize bytes after the chunk start. + if header < 16 || chunk.len() < header.checked_add(20)? { + return None; + } + let ext = header; + let name = read_u32(chunk, ext + 4)?; + if pool.get(name)?.as_str() != "manifest" { + return None; + } + let attribute_start = read_u16(chunk, ext + 8)? as usize; + let attribute_size = read_u16(chunk, ext + 10)? as usize; + let attribute_count = read_u16(chunk, ext + 12)? as usize; + if attribute_size < 20 || attribute_start < 20 { + return None; + } + let base = header.checked_add(attribute_start)?; + let bytes = attribute_count.checked_mul(attribute_size)?; + if base.checked_add(bytes)? > chunk.len() { + return None; + } + let mut package = None; + let mut version_code = None; + let mut version_major = None; + for index in 0..attribute_count { + let at = base + index * attribute_size; + let record = chunk.get(at..at + 20)?; + let namespace = read_u32(record, 0)?; + let name = read_u32(record, 4)?; + let raw = read_u32(record, 8)?; + // Res_value: size u16, res0 u8, dataType u8, data u32. + let data_type = *record.get(15)?; + let data = read_u32(record, 16)?; + let name_text = pool.get(name)?; + let resource = resource_map.get(name as usize).copied(); + // `package` is the un-namespaced attribute. A resource id is never + // used for it: `0x01010003` is `android:name`, not the package. + if namespace == NO_STRING && name_text == "package" { + package = Some(attribute_text(pool, raw, data_type, data)?); + } else if name_text == "versionCode" || resource == Some(VERSION_CODE) { + version_code = Some(version_number(pool, raw, data_type, data)?); + } else if name_text == "versionCodeMajor" || resource == Some(VERSION_CODE_MAJOR) { + version_major = Some(version_number(pool, raw, data_type, data)?); + } + } + finish_identity(package?, version_code, version_major) +} + +fn attribute_text(pool: &StringPool<'_>, raw: u32, data_type: u8, data: u32) -> Option { + if data_type == TYPE_STRING { + return pool.get(data); + } + if raw != NO_STRING { + return pool.get(raw); + } + None +} + +fn version_number(pool: &StringPool<'_>, raw: u32, data_type: u8, data: u32) -> Option { + match data_type { + TYPE_INT_DEC | TYPE_INT_HEX => Some(data.to_string()), + TYPE_STRING => normalize_version(&pool.get(data)?), + _ if raw != NO_STRING => normalize_version(&pool.get(raw)?), + _ => None, + } +} + +fn finish_identity( + package: String, + version_code: Option, + version_major: Option, +) -> Option<(String, String)> { + if !is_package_token(&package) { + return None; + } + let code = version_code?; + let version = match version_major { + Some(major) if major != "0" => format!("{major}.{code}"), + _ => code, + }; + if !is_version_token(&version) { + return None; + } + Some((package, version)) +} + +struct StringPool<'a> { + bytes: &'a [u8], + offsets: Vec, + strings_start: usize, + utf8: bool, +} + +impl<'a> StringPool<'a> { + fn parse(chunk: &'a [u8]) -> Option { + if chunk.len() < 28 { + return None; + } + let count = read_u32(chunk, 8)? as usize; + let flags = read_u32(chunk, 16)?; + let strings_start = read_u32(chunk, 20)? as usize; + // Offsets begin after the 28-byte pool header. A count whose offset + // table does not fit is corrupt. There is no fixed cap on the count: + // the bound is that the table fits and later decoding stays within + // twice the chunk. + let table = 28_usize.checked_add(count.checked_mul(4)?)?; + if table > chunk.len() || strings_start < table || strings_start > chunk.len() { + return None; + } + let mut offsets = Vec::new(); + offsets.try_reserve_exact(count).ok()?; + for index in 0..count { + offsets.push(read_u32(chunk, 28 + index * 4)?); + } + Some(Self { + bytes: chunk, + offsets, + strings_start, + utf8: flags & (1 << 8) != 0, + }) + } + + fn get(&self, index: u32) -> Option { + if index == NO_STRING { + return None; + } + let start = self + .strings_start + .checked_add(*self.offsets.get(index as usize)? as usize)?; + let text = if self.utf8 { + utf8_at(self.bytes, start)?.to_owned() + } else { + utf16_at(self.bytes, start)? + }; + // One looked-up string cannot be larger than the pool that holds it. + if text.len() > self.bytes.len().saturating_mul(2) { + return None; + } + Some(text) + } +} + +fn utf8_at(bytes: &[u8], mut offset: usize) -> Option<&str> { + let (_, next) = utf8_len(bytes, offset)?; + offset = next; + let (byte_len, next) = utf8_len(bytes, offset)?; + offset = next; + let end = offset.checked_add(byte_len)?; + let text = bytes.get(offset..end)?; + // The character after the bytes is a NUL terminator. + if *bytes.get(end)? != 0 { + return None; + } + std::str::from_utf8(text).ok() +} + +fn utf8_len(bytes: &[u8], offset: usize) -> Option<(usize, usize)> { + let first = *bytes.get(offset)?; + if first & 0x80 == 0 { + Some((first as usize, offset + 1)) + } else { + let second = *bytes.get(offset + 1)? as usize; + Some((((first as usize & 0x7f) << 8) | second, offset + 2)) + } +} + +fn utf16_at(bytes: &[u8], offset: usize) -> Option { + let first = read_u16(bytes, offset)?; + let (len, data_at) = if first & 0x8000 == 0 { + (first as usize, offset + 2) + } else { + let second = read_u16(bytes, offset + 2)? as usize; + ((((first as usize) & 0x7fff) << 16) | second, offset + 4) + }; + let end = data_at.checked_add(len.checked_mul(2)?)?; + if read_u16(bytes, end)? != 0 { + return None; + } + let raw = bytes.get(data_at..end)?; + let units = raw.as_chunks::<2>().0; + // Avoid allocating a UTF-16 copy larger than the chunk that holds it. + if units.len().saturating_mul(2) > bytes.len().saturating_mul(2) { + return None; + } + let owned: Vec = units.iter().map(|pair| u16::from_le_bytes(*pair)).collect(); + String::from_utf16(&owned).ok() +} + +fn resource_map_of(chunk: &[u8]) -> Option> { + let body = chunk.get(8..)?; + let mut ids = Vec::new(); + ids.try_reserve_exact(body.len() / 4).ok()?; + for item in body.as_chunks::<4>().0 { + ids.push(u32::from_le_bytes(*item)); + } + Some(ids) +} + +fn plain_identity(bytes: &[u8], truncated: bool) -> Option<(String, String)> { + let text = std::str::from_utf8(bytes).ok()?; + let text = text.strip_prefix('\u{feff}').unwrap_or(text); + let rest = skip_preamble(text)?; + if !rest.starts_with("') || cursor.starts_with("/>") { + closed = true; + break; + } + if cursor.starts_with("")?; + cursor = &cursor[close + 3..]; + continue; + } + let name_end = cursor + .find(|ch: char| ch.is_whitespace() || ch == '=') + .unwrap_or(cursor.len()); + let name = &cursor[..name_end]; + if name.is_empty() { + return None; + } + cursor = cursor[name_end..].trim_start(); + if !cursor.starts_with('=') { + return None; + } + cursor = cursor[1..].trim_start(); + let quote = cursor.chars().next()?; + if quote != '"' && quote != '\'' { + return None; + } + cursor = &cursor[quote.len_utf8()..]; + let value_end = cursor.find(quote)?; + let value = &cursor[..value_end]; + cursor = &cursor[value_end + quote.len_utf8()..]; + match name { + "package" => package = Some(value.to_owned()), + "android:versionCode" => version_code = normalize_version(value), + "android:versionCodeMajor" => version_major = normalize_version(value), + _ => {} + } + } + if !closed || truncated { + return None; + } + finish_identity(package?, version_code, version_major) +} + +/// Skip a BOM (already removed), XML declarations, and comments. A comment +/// that contains ` Option<&str> { + loop { + text = text.trim_start(); + if text.starts_with("")?; + text = &text[close + 2..]; + continue; + } + if text.starts_with("")?; + text = &text[close + 3..]; + continue; + } + break; + } + Some(text) +} + +fn normalize_version(value: &str) -> Option { + if value.starts_with(['-', '+']) { + return None; + } + let number = if let Some(hex) = value + .strip_prefix("0x") + .or_else(|| value.strip_prefix("0X")) + { + u32::from_str_radix(hex, 16).ok()? + } else { + value.parse::().ok()? + }; + Some(number.to_string()) +} + +fn is_package_token(value: &str) -> bool { + !value.is_empty() + && value + .chars() + .all(|ch| ch.is_ascii_alphanumeric() || matches!(ch, '.' | '_' | '-')) +} + +fn is_version_token(value: &str) -> bool { + !value.is_empty() + && value.len() <= 80 + && value + .chars() + .all(|ch| ch.is_ascii_digit() || matches!(ch, '.' | '-' | 'x' | 'a'..='f')) +} + +fn read_u16(bytes: &[u8], offset: usize) -> Option { + Some(u16::from_le_bytes( + bytes.get(offset..offset + 2)?.try_into().ok()?, + )) +} + +fn read_u32(bytes: &[u8], offset: usize) -> Option { + Some(u32::from_le_bytes( + bytes.get(offset..offset + 4)?.try_into().ok()?, + )) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn plain(body: &str) -> Vec { + body.as_bytes().to_vec() + } + + #[test] + fn plain_requires_version_code_and_skips_comments() { + let text = plain( + "\n\n\n\n", + ); + assert_eq!( + identity(&text, false), + Some(("example.game".to_owned(), "3.12".to_owned())) + ); + let named = plain(""); + assert_eq!(identity(&named, false), None); + let substring = plain(""); + assert_eq!(identity(&substring, false), None); + } + + #[test] + fn plain_truncated_root_is_not_an_identity() { + let cut = plain(""); + assert_eq!( + identity(&text, false), + Some(("example.game".to_owned(), "16".to_owned())) + ); + } + + #[test] + fn binary_typed_package_and_nonzero_major() { + // rawValue is 0xFFFFFFFF. The package exists only as a TYPE_STRING + // data index, and versionCodeMajor is a real non-zero integer. + let bytes = axml_with_data( + "manifest", + &[ + ("package", NO_STRING, "example.typed", TYPE_STRING, 0), + ("versionCode", NO_STRING, "", TYPE_INT_DEC, 7), + ("versionCodeMajor", NO_STRING, "", TYPE_INT_DEC, 2), + ], + Encoding::Utf8, + ); + assert_eq!( + identity(&bytes, false), + Some(("example.typed".to_owned(), "2.7".to_owned())) + ); + } + + #[test] + fn binary_utf16_and_hex_version() { + let bytes = axml_with_data( + "manifest", + &[ + ("package", NO_STRING, "example.utf16", 0, 0), + ("versionCode", NO_STRING, "", TYPE_INT_HEX, 0x20), + ], + Encoding::Utf16, + ); + assert_eq!( + identity(&bytes, false), + Some(("example.utf16".to_owned(), "32".to_owned())) + ); + let _ = bytes; + } + + #[test] + fn binary_rejects_other_root_and_android_name_resource() { + let other = axml( + "application", + &[ + attr(NO_STRING, "package", "example.nope", 0, ""), + attr(NO_STRING, "versionCode", "1", 0, ""), + ], + Encoding::Utf8, + ); + assert_eq!(identity(&other, false), None); + // Resource id 0x01010003 is android:name. It must not become the package + // even when its string is a package-looking value. + let named = axml_resources( + "manifest", + &[("name", 0x0101_0003, "example.fake", TYPE_STRING)], + &[], + Encoding::Utf8, + ); + assert_eq!(identity(&named, false), None); + } + + #[test] + fn binary_truncated_and_corrupt_ranges() { + let bytes = axml( + "manifest", + &[ + attr(NO_STRING, "package", "example.cut", 0, ""), + attr(NO_STRING, "versionCode", "1", 0, ""), + ], + Encoding::Utf8, + ); + let mut short = bytes.clone(); + short.truncate(20); + assert_eq!(identity(&short, true), None); + let mut bad = bytes.clone(); + bad[4..8].copy_from_slice(&4_u32.to_le_bytes()); + assert_eq!(identity(&bad, false), None); + } + + #[test] + fn zero_major_is_omitted() { + let bytes = axml_with_data( + "manifest", + &[ + ("package", NO_STRING, "example.zero", TYPE_STRING, 0), + ("versionCode", NO_STRING, "", TYPE_INT_DEC, 9), + ("versionCodeMajor", NO_STRING, "", TYPE_INT_DEC, 0), + ], + Encoding::Utf8, + ); + assert_eq!( + identity(&bytes, false), + Some(("example.zero".to_owned(), "9".to_owned())) + ); + } + + struct BuiltAttr { + name: &'static str, + raw: u32, + raw_text: &'static str, + data_type: u8, + data_text: &'static str, + } + + fn attr( + _namespace: u32, + name: &'static str, + raw_text: &'static str, + data_type: u8, + data_text: &'static str, + ) -> BuiltAttr { + BuiltAttr { + name, + raw: if raw_text.is_empty() { NO_STRING } else { 0 }, + raw_text, + data_type, + data_text, + } + } + + #[derive(Clone, Copy)] + enum Encoding { + Utf8, + Utf16, + } + + fn axml(root: &str, attributes: &[BuiltAttr], encoding: Encoding) -> Vec { + let specs: Vec<(&str, u32, &str, u8, u32)> = attributes + .iter() + .map(|item| { + let data = if item.data_type == TYPE_INT_DEC || item.data_type == TYPE_INT_HEX { + item.data_text.parse().unwrap_or(0) + } else { + 0 + }; + (item.name, item.raw, item.raw_text, item.data_type, data) + }) + .collect(); + axml_with_data(root, &specs, encoding) + } + + fn axml_with_data( + root: &str, + attributes: &[(&str, u32, &str, u8, u32)], + encoding: Encoding, + ) -> Vec { + axml_resources(root, &[], attributes, encoding) + } + + fn axml_resources( + root: &str, + named_resources: &[(&str, u32, &str, u8)], + attributes: &[(&str, u32, &str, u8, u32)], + encoding: Encoding, + ) -> Vec { + let mut strings = vec![root.to_owned()]; + let mut specs: Vec<(u32, u32, u32, u8, u32)> = Vec::new(); + for (name, raw, raw_text, data_type, data) in attributes { + let name_index = push_string(&mut strings, name); + let raw_index = if raw_text.is_empty() { + *raw + } else { + push_string(&mut strings, raw_text) + }; + let data_index = if *data_type == TYPE_STRING { + if raw_text.is_empty() { + *data + } else { + push_string(&mut strings, raw_text) + } + } else { + *data + }; + specs.push((NO_STRING, name_index, raw_index, *data_type, data_index)); + } + for (name, _resource, text, data_type) in named_resources { + let name_index = push_string(&mut strings, name); + let value_index = push_string(&mut strings, text); + specs.push((NO_STRING, name_index, NO_STRING, *data_type, value_index)); + } + let resources: Vec = { + let mut ids = vec![0_u32; strings.len()]; + for (name, resource, _, _) in named_resources { + if let Some(index) = strings.iter().position(|item| item == name) { + ids[index] = *resource; + } + } + ids + }; + let pool = encode_pool(&strings, encoding); + let mut map = Vec::new(); + map.extend(u16::to_le_bytes(0x0180)); + map.extend(u16::to_le_bytes(8)); + map.extend(u32::to_le_bytes((8 + resources.len() * 4) as u32)); + for id in &resources { + map.extend(u32::to_le_bytes(*id)); + } + let mut element = Vec::new(); + element.extend(u16::to_le_bytes(0x0102)); + element.extend(u16::to_le_bytes(16)); + let attribute_bytes = specs.len() * 20; + let size = 16 + 20 + attribute_bytes; + element.extend(u32::to_le_bytes(size as u32)); + element.extend(u32::to_le_bytes(1)); // line + element.extend(u32::to_le_bytes(NO_STRING)); // comment + element.extend(u32::to_le_bytes(NO_STRING)); // namespace + element.extend(u32::to_le_bytes(0)); // name = root + element.extend(u16::to_le_bytes(20)); // attributeStart from attrExt + element.extend(u16::to_le_bytes(20)); + element.extend(u16::to_le_bytes(specs.len() as u16)); + element.extend(u16::to_le_bytes(0)); + element.extend(u16::to_le_bytes(0)); + element.extend(u16::to_le_bytes(0)); + for (namespace, name, raw, data_type, data) in specs { + element.extend(u32::to_le_bytes(namespace)); + element.extend(u32::to_le_bytes(name)); + element.extend(u32::to_le_bytes(raw)); + element.extend(u16::to_le_bytes(8)); + element.push(0); + element.push(data_type); + element.extend(u32::to_le_bytes(data)); + } + let mut bytes = Vec::new(); + bytes.extend(u16::to_le_bytes(0x0003)); + bytes.extend(u16::to_le_bytes(8)); + bytes.extend(u32::to_le_bytes( + (8 + pool.len() + map.len() + element.len()) as u32, + )); + bytes.extend(pool); + bytes.extend(map); + bytes.extend(element); + bytes + } + + fn push_string(strings: &mut Vec, value: &str) -> u32 { + if let Some(index) = strings.iter().position(|item| item == value) { + return index as u32; + } + strings.push(value.to_owned()); + (strings.len() - 1) as u32 + } + + fn encode_pool(strings: &[String], encoding: Encoding) -> Vec { + let mut data = Vec::new(); + let mut offsets = Vec::new(); + for value in strings { + offsets.push(data.len() as u32); + match encoding { + Encoding::Utf8 => { + data.push(value.chars().count() as u8); + data.push(value.len() as u8); + data.extend(value.as_bytes()); + data.push(0); + } + Encoding::Utf16 => { + let units: Vec = value.encode_utf16().collect(); + data.extend(u16::to_le_bytes(units.len() as u16)); + for unit in units { + data.extend(u16::to_le_bytes(unit)); + } + data.extend(u16::to_le_bytes(0)); + } + } + } + let mut chunk = Vec::new(); + chunk.extend(u16::to_le_bytes(0x0001)); + chunk.extend(u16::to_le_bytes(28)); + let size = 28 + strings.len() * 4 + data.len(); + chunk.extend(u32::to_le_bytes(size as u32)); + chunk.extend(u32::to_le_bytes(strings.len() as u32)); + chunk.extend(u32::to_le_bytes(0)); + let flags: u32 = match encoding { + Encoding::Utf8 => 1 << 8, + Encoding::Utf16 => 0, + }; + chunk.extend(u32::to_le_bytes(flags)); + chunk.extend(u32::to_le_bytes((28 + strings.len() * 4) as u32)); + chunk.extend(u32::to_le_bytes(0)); + for offset in offsets { + chunk.extend(u32::to_le_bytes(offset)); + } + chunk.extend(data); + chunk + } +} diff --git a/senbei-io/src/android/mod.rs b/senbei-io/src/android/mod.rs index 4e5c10f..a101c04 100644 --- a/senbei-io/src/android/mod.rs +++ b/senbei-io/src/android/mod.rs @@ -18,12 +18,17 @@ #[cfg(test)] mod tests; +mod context; +mod manifest; + +use context::LibraryRestorer; + use std::collections::HashSet; use std::fs::File; use std::io::{BufWriter, Read, Seek, Write}; use std::path::{Path, PathBuf}; -use anyhow::{Context, Result, bail}; +use anyhow::{Context, Result, bail, ensure}; use memmap2::{Mmap, MmapOptions}; use senbei_crypto::hex_digest; use senbei_engine::android::{ExtractOptions, extract_stage2, is_protected_libil2cpp}; @@ -103,16 +108,16 @@ pub fn file_content_identity(path: &Path) -> std::io::Result { /// implementation detail of the two-phase restore, not user-facing output). /// Returns the unwrapped embedded metadata blob when the restored image /// carries one (see the module docs); the caller decides where to write it. -struct SoRestoreContext { - embedded_metadata: Option>, - method_index_module: Option>, -} - -fn restore_so_file_with_context( +/// +/// Module `0x0C` recovered here belongs to this library's application. Callers +/// that restore more than one application must keep those modules apart; this +/// single-file entry point has only one application, so it keeps the module +/// only long enough to unwrap embedded metadata (which does not consume it). +pub(crate) fn restore_so_with_engine( input: &Path, dest: &Path, verbose: bool, -) -> Result { +) -> Result { let temporary = tempfile::tempdir().context("create stage-2 workspace")?; let stage2_dir = temporary.path().join("stage2"); let extraction = extract_stage2(&ExtractOptions::with_defaults( @@ -120,15 +125,33 @@ fn restore_so_file_with_context( stage2_dir.clone(), )) .context("extract stage-1/stage-2 payload")?; - let method_index_module = extraction + // Every module 0x0C is collected. Two distinct images in one library are a + // conflict for that library's application, not a last-writer-wins update. + let mut modules = Vec::new(); + for module in extraction .module_registry .iter() - .find(|module| module.command_id == 0x0c) - .map(|module| { - std::fs::read(stage2_dir.join(&module.image_path)) - .with_context(|| format!("read decoded module 0x0C `{}`", module.image_path)) - }) - .transpose()?; + .filter(|module| module.command_id == 0x0c) + { + let bytes = std::fs::read(stage2_dir.join(&module.image_path)) + .with_context(|| format!("read decoded module 0x0C `{}`", module.image_path))?; + if modules + .iter() + .all(|(_, existing): &(_, Vec)| existing != &bytes) + { + modules.push((module.image_path.clone(), bytes)); + } + } + ensure!( + modules.len() <= 1, + "library `{}` contains {} distinct module 0x0C images", + input.display(), + modules.len() + ); + let (module_image, method_index_module) = modules + .pop() + .map(|(image, bytes)| (Some(image), Some(bytes))) + .unwrap_or((None, None)); restore_libil2cpp(&RestoreOptions { input: input.to_path_buf(), output: dest.to_path_buf(), @@ -139,30 +162,289 @@ fn restore_so_file_with_context( verbose, }) .context("restore protected library")?; - let restored = - std::fs::read(dest).with_context(|| format!("read restored `{}`", dest.display()))?; - Ok(SoRestoreContext { + let restored_file = + File::open(dest).with_context(|| format!("read restored `{}`", dest.display()))?; + let restored = map_read_only(&restored_file, dest)?; + Ok(context::RestoredLibrary { embedded_metadata: senbei_metadata::android::extract_embedded_metadata(&restored), method_index_module, + module_image, }) } pub fn restore_so_file(input: &Path, dest: &Path, verbose: bool) -> Result>> { - let mut method_index_module = None; - restore_so_file_with_method_index_module(input, dest, verbose, &mut method_index_module) + Ok(context::EngineRestorer + .restore(input, dest, verbose)? + .embedded_metadata) } -pub(crate) fn restore_so_file_with_method_index_module( +/// Fresh run-scoped cache. Library hits reuse a restored image; metadata hits +/// are keyed by encrypted bytes and the module that recovered them. +pub(crate) fn run_cache() -> context::AndroidRunCache { + context::AndroidRunCache::default() +} + +/// Assign loose libraries and metadata to application contexts. +pub(crate) fn loose_scopes( + libraries: &[PathBuf], + metadata: &[PathBuf], +) -> Vec { + context::group_loose_inputs(libraries, metadata) +} + +/// One loose library restore, including the embedded metadata blob when the +/// rebuilt image carried one. `Duplicate` means an earlier successful restore +/// of the same encrypted bytes already populated this application. +#[derive(Debug)] +pub(crate) struct LooseLibraryOutcome { + pub status: EntryStatus, + /// Embedded metadata already written as part of this successful restore. + pub embedded: Option, +} + +pub(crate) fn restore_loose_library( input: &Path, dest: &Path, + label: &str, + app: &context::AppKey, verbose: bool, - method_index_module: &mut Option>, -) -> Result>> { - let restored = restore_so_file_with_context(input, dest, verbose)?; - if let Some(module) = restored.method_index_module { - *method_index_module = Some(module); + cache: &mut context::AndroidRunCache, +) -> Result { + restore_loose_library_with( + input, + dest, + label, + app, + verbose, + cache, + &mut context::EngineRestorer, + ) +} + +fn restore_loose_library_with( + input: &Path, + dest: &Path, + label: &str, + app: &context::AppKey, + verbose: bool, + cache: &mut context::AndroidRunCache, + restorer: &mut R, +) -> Result { + let identity = file_content_identity(input).with_context(|| format!("hash `{label}`"))?; + if let Some(cached) = cache.library(&identity).cloned() { + // Same encrypted library. Record its module on this application and + // write nothing: a duplicate has no output of its own. + record_module(cache, app, &cached, label); + return Ok(LooseLibraryOutcome { + status: EntryStatus::Duplicate, + embedded: None, + }); } - Ok(restored.embedded_metadata) + // A prior content-only success still needs context extraction. Restore it + // in a private workspace, never over a destination that must stay untouched. + let prior = cache.is_prior_success(&identity); + let workspace = prior.then(tempfile::tempdir).transpose().context("create duplicate context workspace")?; + let scratch = workspace.as_ref().map(|directory| directory.path().join("restored.so")); + let restore_dest = scratch.as_deref().unwrap_or(dest); + match restorer.restore(input, restore_dest, verbose) { + Ok(restored) => { + // The library file is already at `dest`. Embedded metadata is part + // of the same success: cache the module only after that write. + let mut embedded = None; + if !prior && let Some(blob) = &restored.embedded_metadata { + let meta_dest = embedded_metadata_dest(dest); + if let Err(error) = write_metadata_blob(&meta_dest, blob) { + return Ok(LooseLibraryOutcome { + status: EntryStatus::Failed(error), + embedded: None, + }); + } + embedded = Some(meta_dest); + } + let cached = context::CachedLibrary { + method_index_module: restored.method_index_module, + module_image: restored.module_image, + }; + record_module(cache, app, &cached, label); + cache.insert_library(identity, cached); + Ok(LooseLibraryOutcome { + status: if prior { EntryStatus::Duplicate } else { EntryStatus::Restored }, + embedded, + }) + } + Err(error) => Ok(LooseLibraryOutcome { + status: EntryStatus::Failed(error), + embedded: None, + }), + } +} + +/// Metadata restored under one application context, plus the libraries whose +/// module was paired with it. +#[derive(Debug)] +pub(crate) struct LooseMetadataOutcome { + pub version: u32, + pub remapped: usize, + pub provenance: Vec, +} + +pub(crate) fn restore_loose_metadata( + input: &Path, + dest: &Path, + label: &str, + app: &context::AppKey, + verbose: bool, + cache: &mut context::AndroidRunCache, +) -> Result { + let data = std::fs::read(input).with_context(|| format!("read `{label}`"))?; + let version = metadata_version(&data); + // Only raw v24 consumes module 0x0C. v29/v31/v39 and the structural path + // must not fail, and must not record a module they did not use, when the + // application's libraries disagree or have no module. + let (module, provenance) = if version == Some(24) { + let modules = cache.modules(app).cloned(); + match context::binding_module(modules.as_ref()) { + Ok(module) => ( + module.map(|bytes| bytes.to_vec()), + provenance_labels(&context::provenance_for(modules.as_ref(), None)), + ), + Err(ambiguity) => { + let provenance = context::provenance_for(modules.as_ref(), Some(&ambiguity)); + bail!(provenance_error(&provenance)); + } + } + } else { + (None, Vec::new()) + }; + let module = module.as_deref(); + // Loose metadata retains one output per path, as before. Do not keep all + // restored blobs in the folder cache just to avoid a repeated permutation. + let (out, report) = + restore_metadata_bytes_with_module(&data, module).with_context(|| label.to_owned())?; + if report.remapped > 0 { + write_metadata_blob(dest, &out)?; + if verbose { println!("Write to {}", dest.display()); } + } + Ok(LooseMetadataOutcome { + version: report.version, + remapped: report.remapped, + provenance, + }) +} + +/// Group loose `.apk` files that are a split set: same parent directory and +/// the same manifest package name plus version. Bundles are never grouped +/// with anything. Order inside a group does not matter; the caller restores +/// every library in the group before any of its metadata. +pub(crate) fn loose_apk_groups(packages: &[PathBuf]) -> Vec> { + let mut groups: Vec<(context::AppKey, Vec)> = Vec::new(); + let mut alone = Vec::new(); + for package in packages { + let extension = package.extension().and_then(|value| value.to_str()); + let is_apk = extension.is_some_and(|value| value.eq_ignore_ascii_case("apk")); + if !is_apk { + alone.push(vec![package.clone()]); + continue; + } + let key = apk_manifest_key(package); + if let Some(key) = key { + if let Some((_, members)) = groups.iter_mut().find(|(existing, _)| existing == &key) { + members.push(package.clone()); + } else { + groups.push((key, vec![package.clone()])); + } + } else { + alone.push(vec![package.clone()]); + } + } + let mut grouped: Vec<_> = groups.into_iter().map(|(_, members)| members).collect(); + grouped.append(&mut alone); + grouped +} + +fn apk_manifest_key(package: &Path) -> Option { + let file = std::fs::File::open(package).ok()?; + let mut archive = zip::ZipArchive::new(file).ok()?; + let parent = package + .parent() + .filter(|parent| !parent.as_os_str().is_empty()) + .unwrap_or_else(|| Path::new(".")); + let inventory = context::inventory_manifest_package( + &mut archive, + package, + parent, + &package.display().to_string(), + ) + .ok()?; + match inventory.key { + context::AppKey::Manifest { .. } => Some(inventory.key), + _ => None, + } +} + +/// Restore one package against the folder run's cache. Successful library +/// identities are published back through the cache itself; the public +/// [`restore_package`] wrapper remains for callers that only have a `HashSet`. +pub(crate) fn restore_package_cached( + package: &Path, + rel: &Path, + out_root: &Path, + cache: &mut context::AndroidRunCache, + verbose: bool, +) -> Result> { + restore_package_with_restorer( + package, + rel, + out_root, + cache, + verbose, + &mut context::EngineRestorer, + ) +} + +/// One folder split-set: every library in the group, then every metadata blob. +/// A library-only pass records modules without writing metadata, so a metadata +/// blob in an earlier sibling cannot run before a library in a later one. +pub(crate) fn restore_apk_group( + packages: &[PathBuf], + root: &Path, + out_root: &Path, + cache: &mut context::AndroidRunCache, + verbose: bool, +) -> Result> { + let mut outcomes = Vec::new(); + for package in packages { + let rel = package.strip_prefix(root).unwrap_or(package); + outcomes.extend(restore_package_pass( + package, + rel, + out_root, + cache, + verbose, + &mut context::EngineRestorer, + PackagePass::Libraries, + )?); + } + for package in packages { + let rel = package.strip_prefix(root).unwrap_or(package); + outcomes.extend(restore_package_pass( + package, + rel, + out_root, + cache, + verbose, + &mut context::EngineRestorer, + PackagePass::Metadata, + )?); + } + Ok(outcomes) +} + +#[derive(Clone, Copy, PartialEq, Eq)] +enum PackagePass { + All, + Libraries, + Metadata, } /// Content identity for cross-source deduplication: the same library may @@ -198,9 +480,13 @@ pub(crate) fn restore_metadata_bytes_with_module( .get(4..8) .and_then(|bytes| <[u8; 4]>::try_from(bytes).ok()) .map(u32::from_le_bytes); - if version == Some(24) - && let Some(module) = method_index_module - { + if version == Some(24) { + // v24.1 method indices come only from this application's module 0x0C. + // Falling through to the structural remap would report "unsupported + // version" and could be mistaken for a clean skip. No module is a + // failed binding, not an unsupported layout. + let module = method_index_module + .ok_or_else(|| anyhow::anyhow!("no module 0x0C for v24.1 metadata"))?; let (out, report) = senbei_metadata::android::restore_method_indices_v24_1(data, module) .map_err(anyhow::Error::new)?; return Ok(( @@ -252,6 +538,9 @@ pub struct EntryOutcome { pub dest: PathBuf, pub kind: EntryKind, pub status: EntryStatus, + /// Libraries whose module `0x0C` was paired with this metadata entry. + /// Empty for libraries and for metadata that had no paired module. + pub provenance: Vec, } #[derive(Debug, Clone, Copy, PartialEq, Eq)] @@ -278,20 +567,12 @@ pub enum EntryStatus { Failed(anyhow::Error), } -struct PackageRestoreContext<'a> { - method_index_module: &'a mut Option>, +struct PackageRestoreState<'a, R> { + restorer: &'a mut R, + cache: &'a mut context::AndroidRunCache, verbose: bool, } -impl<'a> PackageRestoreContext<'a> { - fn new(verbose: bool, method_index_module: &'a mut Option>) -> Self { - Self { - method_index_module, - verbose, - } - } -} - fn package_entry_priority(path: &Path) -> u8 { if is_so_name(path) { 0 } else { 1 } } @@ -310,24 +591,60 @@ pub fn restore_package( seen: &mut HashSet, verbose: bool, ) -> Result> { - let mut method_index_module = None; - restore_package_with_method_index_module( + // `seen` is the public successful-content set shared with loose libraries. + // Metadata recovery context does not fit in that set, so this wrapper keeps + // a run-scoped cache and only publishes content identities of entries that + // actually succeeded. A caller that passes identities restored by an earlier + // package still skips those bytes; module context for metadata is rebuilt + // inside this package (one package, or one manifest-scoped bundle, is one + // application boundary). + let mut cache = context::AndroidRunCache::default(); + for identity in seen.iter().cloned() { + cache.insert_prior_success(identity); + } + let outcomes = restore_package_with_restorer( package, rel, out_root, - seen, + &mut cache, verbose, - &mut method_index_module, - ) + &mut context::EngineRestorer, + ); + // Publish successes even when the package returns `Err`. The caller's set + // is only replaced after that, so a panic still leaves the old successes. + let succeeded = cache.successful_content_identities(); + seen.clear(); + seen.extend(succeeded); + outcomes } -pub(crate) fn restore_package_with_method_index_module( +pub(crate) fn restore_package_with_restorer( package: &Path, rel: &Path, out_root: &Path, - seen: &mut HashSet, + cache: &mut context::AndroidRunCache, verbose: bool, - method_index_module: &mut Option>, + restorer: &mut R, +) -> Result> { + restore_package_pass( + package, + rel, + out_root, + cache, + verbose, + restorer, + PackagePass::All, + ) +} + +fn restore_package_pass( + package: &Path, + rel: &Path, + out_root: &Path, + cache: &mut context::AndroidRunCache, + verbose: bool, + restorer: &mut R, + pass: PackagePass, ) -> Result> { let bundle = package .extension() @@ -338,7 +655,11 @@ pub(crate) fn restore_package_with_method_index_module( let mut archive = open_package(package)?; let temporary = tempfile::tempdir().context("create package workspace")?; let mut outcomes = Vec::new(); - let mut context = PackageRestoreContext::new(verbose, method_index_module); + let mut state = PackageRestoreState { + restorer, + cache, + verbose, + }; let mut direct = Vec::new(); let mut nested = Vec::new(); @@ -361,32 +682,60 @@ pub(crate) fn restore_package_with_method_index_module( { nested.push((index, name)); } - } else { - if is_android_entry_name(&name) { - direct.push((index, name)); - } + } else if is_android_entry_name(&name) { + direct.push((index, name)); } } - direct.sort_by_key(|(_, name)| package_entry_priority(name)); - for (index, name) in direct { - let label = format!("{}::{}", rel.display(), name.display()); - let dest = out_root.join(rel).join(crate::job::out_name(&name)); - let mut entry_outcomes = restore_package_entry( + // One direct package is one application inside its own package set (the + // package file). A plain manifest names it by package plus version; without + // both, the package is isolated. The ZIP file name is not a package id. + // Inventory already walked this central directory; entry extraction below + // reuses the same open and must not be charged as a second package. + let direct_inventory = if direct.is_empty() { + None + } else { + Some(context::inventory_manifest_package( &mut archive, - index, - &label, - &dest, - &temporary, - seen, - &mut context, - ) - .with_context(|| format!("extract `{label}`"))?; - outcomes.append(&mut entry_outcomes); + package, + package + .parent() + .filter(|parent| !parent.as_os_str().is_empty()) + .unwrap_or_else(|| Path::new(".")), + &package.display().to_string(), + )?) + }; + direct.retain(|(_, name)| match pass { + PackagePass::Libraries => package_entry_priority(name) == 0, + PackagePass::Metadata => package_entry_priority(name) == 1, + PackagePass::All => true, + }); + direct.sort_by_key(|(_, name)| package_entry_priority(name)); + if let Some(inventory) = direct_inventory { + for (index, name) in direct { + let label = format!("{}::{}", rel.display(), name.display()); + let dest = out_root.join(rel).join(crate::job::out_name(&name)); + let mut entry_outcomes = restore_package_entry( + &mut archive, + index, + &label, + &dest, + &temporary, + &inventory.key, + &mut state, + ) + .with_context(|| format!("extract `{label}`"))?; + outcomes.append(&mut entry_outcomes); + } } struct NestedPackage { label: PathBuf, path: PathBuf, base: PathBuf, + app: context::AppKey, + /// Manifest inventory, including the reopen key. Extraction opens + /// `path` again; `inventory.archive` is the pass a later budget must + /// not charge a second time. + inventory: context::PackageInventory, entries: Vec<(usize, PathBuf)>, } @@ -396,6 +745,18 @@ pub(crate) fn restore_package_with_method_index_module( let nested_path = extract_entry(&mut archive, index, &temporary, &nested_label) .with_context(|| format!("extract `{}`", nested_label.display()))?; let mut nested_archive = open_package(&nested_path)?; + // Manifest first, before any library or metadata entry is chosen. Splits + // that name the same package and version, inside this bundle only, share + // one context. A split with no plain package name or no version is + // isolated and cannot borrow a sibling's module. `nested_path` is the + // reopen key: the central directory listed here is not charged again + // when entries are extracted below. + let inventory = context::inventory_manifest_package( + &mut nested_archive, + &nested_path, + package, + &nested_path.display().to_string(), + )?; let mut entries = Vec::new(); for nested_index in 0..nested_archive.len() { let (entry_name, is_dir) = { @@ -415,40 +776,61 @@ pub(crate) fn restore_package_with_method_index_module( label: nested_label, path: nested_path, base: rel.join(name.with_extension("")), + app: inventory.key.clone(), + inventory, entries, }); } // A bundle can put libil2cpp.so in an ABI split while metadata stays in - // base.apk. Share one context across the whole bundle and process all SOs - // before any metadata, regardless of which nested APK owns each entry. - for priority in [0_u8, 1_u8] { - for nested_package in &nested_packages { - let mut nested_archive = open_package(&nested_package.path)?; - for (nested_index, entry_name) in nested_package - .entries - .iter() - .filter(|(_, entry_name)| package_entry_priority(entry_name) == priority) - { - let label = format!( - "{}::{}", - nested_package.label.display(), - entry_name.display() + // base.apk. Splits of the same manifest package share one context. All of + // that application's libraries are restored before its metadata, independent + // of nested-APK order. Different package names never share a module. + let mut app_order: Vec = Vec::new(); + for nested_package in &nested_packages { + if !app_order.contains(&nested_package.app) { + app_order.push(nested_package.app.clone()); + } + } + let priorities: &[u8] = match pass { + PackagePass::Libraries => &[0], + PackagePass::Metadata => &[1], + PackagePass::All => &[0, 1], + }; + for app in &app_order { + for priority in priorities { + for nested_package in nested_packages.iter().filter(|nested| &nested.app == app) { + // Same file the inventory listed. Reopening is not a new package. + debug_assert_eq!( + nested_package.inventory.archive.reopen_key, + nested_package.path ); - let dest = out_root - .join(&nested_package.base) - .join(crate::job::out_name(entry_name)); - let mut entry_outcomes = restore_package_entry( - &mut nested_archive, - *nested_index, - &label, - &dest, - &temporary, - seen, - &mut context, - ) - .with_context(|| format!("extract `{label}`"))?; - outcomes.append(&mut entry_outcomes); + let mut nested_archive = open_package(&nested_package.path)?; + for (nested_index, entry_name) in nested_package + .entries + .iter() + .filter(|(_, entry_name)| package_entry_priority(entry_name) == *priority) + { + let label = format!( + "{}::{}", + nested_package.label.display(), + entry_name.display() + ); + let dest = out_root + .join(&nested_package.base) + .join(crate::job::out_name(entry_name)); + let mut entry_outcomes = restore_package_entry( + &mut nested_archive, + *nested_index, + &label, + &dest, + &temporary, + app, + &mut state, + ) + .with_context(|| format!("extract `{label}`"))?; + outcomes.append(&mut entry_outcomes); + } } } } @@ -458,105 +840,248 @@ pub(crate) fn restore_package_with_method_index_module( /// Probe one extracted package entry and restore it when it is a target. /// Returns one outcome per produced/consumed artifact: the entry itself, plus /// an `EmbeddedMetadata` outcome when the restored library carried a blob. -fn restore_package_entry( +/// +/// `app` is the application this entry was inventoried into. Libraries record +/// their module there; metadata reads it back and refuses a conflicting or +/// missing binding before writing. +fn restore_package_entry( archive: &mut ZipArchive, index: usize, label: &str, dest: &Path, temporary: &tempfile::TempDir, - seen: &mut HashSet, - context: &mut PackageRestoreContext<'_>, + app: &context::AppKey, + state: &mut PackageRestoreState<'_, T>, ) -> Result> { let entry_path = extract_entry(archive, index, temporary, Path::new(label))?; let entry_file = File::open(&entry_path).with_context(|| format!("open extracted `{label}`"))?; let entry_data = map_read_only(&entry_file, &entry_path)?; - let is_so = is_elf64_aarch64(&entry_data) && is_protected_libil2cpp(&entry_data); + let protected_override = state.restorer.protected_override(&entry_path); + let is_so = protected_override.is_some() + || (is_elf64_aarch64(&entry_data) && is_protected_libil2cpp(&entry_data)); let is_meta = !is_so && senbei_metadata::is_metadata(&entry_data); - let outcome = |kind, status| EntryOutcome { + let outcome = |kind, status, provenance: Vec| EntryOutcome { label: label.to_owned(), dest: dest.to_path_buf(), kind, status, + provenance, }; if !is_so && !is_meta { - return Ok(vec![outcome(EntryKind::So, EntryStatus::NotTarget)]); + return Ok(vec![outcome( + EntryKind::So, + EntryStatus::NotTarget, + Vec::new(), + )]); } let identity = content_identity(&entry_data); - if seen.contains(&identity) { - let kind = if is_so { - EntryKind::So - } else { - EntryKind::Metadata { remapped: 0 } - }; - return Ok(vec![outcome(kind, EntryStatus::Duplicate)]); + if is_so && state.cache.library(&identity).is_some() { + if let Some(cached) = state.cache.library(&identity).cloned() { + // Duplicate libraries still belong to the application that contains + // them. The cached module is recorded here and no output is written. + record_module(state.cache, app, &cached, label); + } + return Ok(vec![outcome( + EntryKind::So, + EntryStatus::Duplicate, + Vec::new(), + )]); } if is_so { drop(entry_data); drop(entry_file); + let prior = state.cache.is_prior_success(&identity); + let workspace = prior.then(|| tempfile::tempdir_in(temporary.path())).transpose() + .context("create duplicate context workspace")?; + let scratch = workspace.as_ref().map(|directory| directory.path().join("restored.so")); + let restore_dest = scratch.as_deref().unwrap_or(dest); return Ok( - match restore_so_file_with_context(&entry_path, dest, context.verbose) { + match state.restorer.restore(&entry_path, restore_dest, state.verbose) { Ok(restored) => { - if let Some(module) = restored.method_index_module { - *context.method_index_module = Some(module); + // A prior content-only success is restored only in the + // private workspace to recover its module context. + let mut outcomes = Vec::new(); + let mut embedded_failed = None; + if !prior { + outcomes.push(outcome(EntryKind::So, EntryStatus::Restored, Vec::new())); } - let mut outcomes = vec![outcome(EntryKind::So, EntryStatus::Restored)]; - if let Some(blob) = restored.embedded_metadata { + if !prior && let Some(blob) = &restored.embedded_metadata { let meta_dest = embedded_metadata_dest(dest); - let status = match write_metadata_blob(&meta_dest, &blob) { - Ok(()) => EntryStatus::Restored, - Err(error) => EntryStatus::Failed(error), + match write_metadata_blob(&meta_dest, blob) { + Ok(()) => outcomes.push(EntryOutcome { + label: format!("{label} (embedded metadata)"), + dest: meta_dest, + kind: EntryKind::EmbeddedMetadata, + status: EntryStatus::Restored, + provenance: Vec::new(), + }), + Err(error) => embedded_failed = Some(error), + } + } + if let Some(error) = embedded_failed { + vec![outcome( + EntryKind::So, + EntryStatus::Failed(error), + Vec::new(), + )] + } else { + let cached = context::CachedLibrary { + method_index_module: restored.method_index_module.clone(), + module_image: restored.module_image.clone(), }; - outcomes.push(EntryOutcome { - label: format!("{label} (embedded metadata)"), - dest: meta_dest, - kind: EntryKind::EmbeddedMetadata, - status, - }); + record_module(state.cache, app, &cached, label); + state.cache.insert_library(identity, cached); + if prior { + vec![outcome(EntryKind::So, EntryStatus::Duplicate, Vec::new())] + } else { + outcomes + } } - if outcomes - .iter() - .all(|outcome| !matches!(outcome.status, EntryStatus::Failed(_))) - { - seen.insert(identity); - } - outcomes } - Err(error) => vec![outcome(EntryKind::So, EntryStatus::Failed(error))], + Err(error) => vec![outcome( + EntryKind::So, + EntryStatus::Failed(error), + Vec::new(), + )], }, ); } - // Metadata entry: write only when the restore actually changed protected method fields — - // a clean blob needs no copy (same contract as loose metadata files). - let kind_and_status = match restore_metadata_bytes_with_module( - &entry_data, - context.method_index_module.as_deref(), - ) { + let version = metadata_version(&entry_data); + // Module 0x0C is a v24.1 input only. Later layouts ignore it, so a missing + // or conflicting module does not block them and is not listed as paired. + let (module, provenance) = if version == Some(24) { + let modules = state.cache.modules(app).cloned(); + match context::binding_module(modules.as_ref()) { + Ok(module) => ( + module.map(|bytes| bytes.to_vec()), + context::provenance_for(modules.as_ref(), None), + ), + Err(ambiguity) => { + let provenance = context::provenance_for(modules.as_ref(), Some(&ambiguity)); + return Ok(vec![outcome( + EntryKind::Metadata { remapped: 0 }, + EntryStatus::Failed(anyhow::anyhow!(provenance_error(&provenance))), + provenance_labels(&provenance), + )]); + } + } + } else { + (None, context::MetadataProvenance::default()) + }; + let module_bytes = module; + let module = module_bytes.as_deref(); + let context_identity = context::metadata_context_identity(module); + if let Some(cached) = state.cache.metadata(&identity, &context_identity).cloned() { + // A repeated blob is not written again. The cached bytes stay available + // for a *different* recovery context; this context already succeeded. + let kind = match &cached { + context::CachedMetadata::Unchanged => EntryKind::Metadata { remapped: 0 }, + context::CachedMetadata::Restored { remapped, .. } => EntryKind::Metadata { + remapped: *remapped, + }, + }; + return Ok(vec![outcome( + kind, + EntryStatus::Duplicate, + provenance_labels(&provenance), + )]); + } + + // Metadata entry: write only when the restore actually changed protected + // method fields — a clean blob needs no copy (same contract as loose files). + // The binding was resolved before this write. + let kind_and_status = match restore_metadata_bytes_with_module(&entry_data, module) { Ok((out, report)) if report.remapped > 0 => { let kind = EntryKind::Metadata { remapped: report.remapped, }; match write_metadata_blob(dest, &out) { - Ok(()) => (kind, EntryStatus::Restored), + Ok(()) => { + state.cache.insert_metadata( + identity, + context_identity, + context::CachedMetadata::Restored { + remapped: report.remapped, + }, + ); + (kind, EntryStatus::Restored) + } Err(error) => (kind, EntryStatus::Failed(error)), } } - Ok(_) => (EntryKind::Metadata { remapped: 0 }, EntryStatus::Unchanged), + Ok(_) => { + state.cache.insert_metadata( + identity, + context_identity, + context::CachedMetadata::Unchanged, + ); + (EntryKind::Metadata { remapped: 0 }, EntryStatus::Unchanged) + } Err(error) => ( EntryKind::Metadata { remapped: 0 }, EntryStatus::Failed(error), ), }; - if matches!( + Ok(vec![outcome( + kind_and_status.0, kind_and_status.1, - EntryStatus::Restored | EntryStatus::Unchanged - ) { - seen.insert(identity); + provenance_labels(&provenance), + )]) +} + +fn record_module( + cache: &mut context::AndroidRunCache, + app: &context::AppKey, + restored: &context::CachedLibrary, + label: &str, +) { + if let Some(module) = &restored.method_index_module { + cache.modules_mut(app).record( + module, + context::ModuleSource { + label: label.to_owned(), + image: restored.module_image.clone(), + }, + ); } - Ok(vec![outcome(kind_and_status.0, kind_and_status.1)]) +} + +fn metadata_version(data: &[u8]) -> Option { + data.get(4..8) + .and_then(|bytes| <[u8; 4]>::try_from(bytes).ok()) + .map(u32::from_le_bytes) +} + +fn provenance_labels(provenance: &context::MetadataProvenance) -> Vec { + provenance + .sources + .iter() + .map(|source| match &source.image { + Some(image) => format!("{} ({image})", source.label), + None => source.label.clone(), + }) + .collect() +} + +fn provenance_error(provenance: &context::MetadataProvenance) -> String { + let why = provenance + .note + .clone() + .unwrap_or_else(|| "ambiguous module 0x0C binding".to_owned()); + if provenance.sources.is_empty() { + return why; + } + let sources = provenance + .sources + .iter() + .map(|source| source.label.as_str()) + .collect::>() + .join(", "); + format!("{why}; modules from {sources}") } /// Output path for a metadata blob unwrapped from a restored library: next to diff --git a/senbei-io/src/android/tests.rs b/senbei-io/src/android/tests.rs index 29304d9..7634ea5 100644 --- a/senbei-io/src/android/tests.rs +++ b/senbei-io/src/android/tests.rs @@ -1,6 +1,130 @@ use super::*; +use crate::android::context::{self, AppKey, LibraryRestorer, RestoredLibrary}; use zip::write::SimpleFileOptions; +#[derive(Default)] +struct ScriptedRestorer { + /// Encrypted library bytes -> module 0x0C profile. + modules: Vec<(Vec, Vec)>, + embedded: Option>, +} + +impl LibraryRestorer for ScriptedRestorer { + fn restore( + &mut self, + input: &Path, + dest: &Path, + _verbose: bool, + ) -> anyhow::Result { + let bytes = std::fs::read(input)?; + let module = self + .modules + .iter() + .find(|(library, _)| library == &bytes) + .map(|(_, module)| module.clone()) + .ok_or_else(|| anyhow::anyhow!("no scripted module for library"))?; + if let Some(parent) = dest.parent() { + std::fs::create_dir_all(parent)?; + } + // The restored image is not an ELF; tests only check which metadata the + // module recovered. A stable marker keeps duplicate reuse observable. + std::fs::write(dest, b"restored")?; + Ok(RestoredLibrary { + embedded_metadata: self.embedded.clone(), + method_index_module: Some(module), + module_image: Some("module-0c.bin".to_owned()), + }) + } + + fn protected_override(&mut self, input: &Path) -> Option> { + let bytes = std::fs::read(input).ok()?; + self.modules + .iter() + .any(|(library, _)| library == &bytes) + .then_some(bytes) + } +} + +fn pair(seed: u32) -> (Vec, Vec) { + // Local copy of the v24.1 fixture transform. Seeds 1..=5 and `0x77700DCC` + // miss exactly `0..255` after the repeated high-half image, which is the + // exception table written below. The metadata crate does not export this. + const STRIDE: usize = 0x34; + const INDEX_AT: usize = 0x14; + fn transform(mut value: u32, count: u32, seed: u32) -> u32 { + let key = seed % (count / 2) + count / 4; + for _ in 0..5 { + let mirror = count * 2 - 1; + if value & 1 != 0 { + value = mirror - value; + } + value >>= 1; + if value >= count { + value = mirror - value; + } + let adjusted = i64::from(value) - i64::from(key); + value = if adjusted < 0 { + (adjusted + i64::from(count)) as u32 + } else { + adjusted as u32 + }; + } + value + } + let active = 512_u32; + let mut inverse = vec![u32::MAX; active as usize]; + for value in 0..active { + inverse[transform(value, active, seed) as usize] = value; + } + let header = 0x100_usize; + let methods = active as usize * STRIDE; + let mut metadata = vec![0_u8; header + methods]; + metadata[0..4].copy_from_slice(&0xfab1_1baf_u32.to_le_bytes()); + metadata[4..8].copy_from_slice(&24_u32.to_le_bytes()); + metadata[0x30..0x34].copy_from_slice(&(header as u32).to_le_bytes()); + metadata[0x34..0x38].copy_from_slice(&(methods as u32).to_le_bytes()); + for method in 0..active as usize { + let image = 256 + (method % 256) as u32; + let protected = inverse[image as usize] as i32; + let offset = header + method * STRIDE + INDEX_AT; + metadata[offset..offset + 4].copy_from_slice(&protected.to_le_bytes()); + } + let mut module = vec![0_u8; 0x1000]; + module[0x100..0x104].copy_from_slice(&seed.to_le_bytes()); + module[0x104..0x108].copy_from_slice(&24_u32.to_le_bytes()); + module[0x3f8..0x3fc].copy_from_slice(&1_u32.to_le_bytes()); + module[0x3fc..0x400].copy_from_slice(&(active - 1).to_le_bytes()); + for value in 0..256_u32 { + let offset = 0x400 + value as usize * 4; + module[offset..offset + 4].copy_from_slice(&value.to_le_bytes()); + } + (metadata, module) +} + +fn restored_with(metadata: &[u8], module: &[u8]) -> Vec { + match senbei_metadata::android::restore_method_indices_v24_1(metadata, module) { + Ok((bytes, _)) => bytes, + Err(error) => panic!("fixture restore failed: {error:?}"), + } +} + +fn manifest(package: &str, version: &str) -> String { + format!( + r#" + + +"# + ) +} + +fn library_bytes(marker: &[u8]) -> Vec { + // Distinct from metadata magic so a mis-classified entry cannot take the + // metadata path. The scripted restorer recognises these exact bytes. + let mut bytes = b"SENBEI-SYNTHETIC-LIBRARY".to_vec(); + bytes.extend_from_slice(marker); + bytes +} + fn metadata_fixture() -> Vec { let mut data = vec![0; 0x1a4]; let mut put = |offset: usize, value: u32| { @@ -116,3 +240,986 @@ fn unchanged_metadata_is_a_successful_deduplication_source() { assert_eq!(seen.len(), 1); assert!(outcomes.iter().all(|outcome| !outcome.dest.exists())); } + +fn restore_scripted( + package: &Path, + rel: &Path, + out: &Path, + cache: &mut context::AndroidRunCache, + restorer: &mut ScriptedRestorer, +) -> Vec { + restore_package_with_restorer(package, rel, out, cache, false, restorer).unwrap() +} + +fn metadata_outcomes(outcomes: &[EntryOutcome]) -> Vec<&EntryOutcome> { + outcomes + .iter() + .filter(|outcome| matches!(outcome.kind, EntryKind::Metadata { .. })) + .collect() +} + +#[test] +fn two_extracted_apps_do_not_share_module_context() { + let temp = tempfile::tempdir().unwrap(); + let root = temp.path(); + let (meta_a, module_a) = pair(1); + let (meta_b, module_b) = pair(2); + let lib_a = library_bytes(b"app-a"); + let lib_b = library_bytes(b"app-b"); + let app_a = root.join("games/alpha"); + let app_b = root.join("games/beta"); + std::fs::create_dir_all(app_a.join("lib/arm64-v8a")).unwrap(); + std::fs::create_dir_all(app_a.join("assets/bin/Data/Managed/Metadata")).unwrap(); + std::fs::create_dir_all(app_b.join("lib/arm64-v8a")).unwrap(); + std::fs::create_dir_all(app_b.join("assets/bin/Data/Managed/Metadata")).unwrap(); + let so_a = app_a.join("lib/arm64-v8a/libil2cpp.so"); + let so_b = app_b.join("lib/arm64-v8a/libil2cpp.so"); + let md_a = app_a.join("assets/bin/Data/Managed/Metadata/global-metadata.dat"); + let md_b = app_b.join("assets/bin/Data/Managed/Metadata/global-metadata.dat"); + std::fs::write(&so_a, &lib_a).unwrap(); + std::fs::write(&so_b, &lib_b).unwrap(); + std::fs::write(&md_a, &meta_a).unwrap(); + std::fs::write(&md_b, &meta_b).unwrap(); + + let scoped = loose_scopes(&[so_b.clone(), so_a.clone()], &[md_a.clone(), md_b.clone()]); + assert!( + scoped + .iter() + .any(|item| item.path == so_a && item.key == AppKey::Extracted(app_a.clone())), + "scopes: {:?}", + scoped + .iter() + .map(|item| (item.path.display().to_string(), format!("{:?}", item.key))) + .collect::>() + ); + assert!( + scoped + .iter() + .any(|item| item.path == md_b && item.key == AppKey::Extracted(app_b.clone())) + ); + + let mut cache = run_cache(); + let mut restorer = ScriptedRestorer { + embedded: None, + modules: vec![(lib_a, module_a.clone()), (lib_b, module_b.clone())], + }; + let out = root.join("unpack"); + // Restore beta's library first. A last-writer cache would then poison alpha. + for (library, app) in [(&so_b, &app_b), (&so_a, &app_a)] { + let scoped = scoped.iter().find(|item| &item.path == library).unwrap(); + assert_eq!(&scoped.key, &AppKey::Extracted(app.clone())); + let outcome = restore_loose_library_with( + library, + &out.join(library.file_name().unwrap()), + &library.display().to_string(), + &scoped.key, + false, + &mut cache, + &mut restorer, + ) + .unwrap(); + assert!(matches!(outcome.status, EntryStatus::Restored)); + } + let restored_a = restore_loose_metadata( + &md_a, + &out.join("a.dat"), + "alpha metadata", + &AppKey::Extracted(app_a.clone()), + false, + &mut cache, + ) + .unwrap_or_else(|error| panic!("alpha metadata: {error:#}; cache apps will not print")); + let restored_b = restore_loose_metadata( + &md_b, + &out.join("b.dat"), + "beta metadata", + &AppKey::Extracted(app_b), + false, + &mut cache, + ) + .unwrap(); + assert_eq!( + std::fs::read(out.join("a.dat")).unwrap(), + restored_with(&meta_a, &module_a) + ); + assert_eq!( + std::fs::read(out.join("b.dat")).unwrap(), + restored_with(&meta_b, &module_b) + ); + assert!( + senbei_metadata::android::restore_method_indices_v24_1(&meta_a, &module_b).is_err(), + "beta's module must not restore alpha's metadata" + ); + assert!( + restored_a + .provenance + .iter() + .any(|source| source.contains("alpha") || source.contains("libil2cpp")) + ); + assert!( + restored_b + .provenance + .iter() + .all(|source| !source.contains("games/alpha") && !source.contains("games\\alpha")) + ); +} + +#[test] +fn bundle_splits_share_context_independent_of_order() { + let temp = tempfile::tempdir().unwrap(); + let (metadata, module) = pair(3); + let library = library_bytes(b"split-lib"); + let manifest_xml = manifest("example.game", "123"); + let write = |name: &str, base_first: bool| { + let path = temp.path().join(name); + let base = ( + "base.apk", + entries(&[ + ("AndroidManifest.xml", manifest_xml.as_bytes()), + ( + "assets/bin/Data/Managed/Metadata/global-metadata.dat", + &metadata, + ), + ]), + ); + let split = ( + "split_config.arm64_v8a.apk", + entries(&[ + ("AndroidManifest.xml", manifest_xml.as_bytes()), + ("lib/arm64-v8a/libil2cpp.so", &library), + ]), + ); + let nested = if base_first { + vec![base, split] + } else { + vec![split, base] + }; + write_nested(&path, &nested); + path + }; + let mut cache_forward = run_cache(); + let mut cache_reverse = run_cache(); + let mut restorer = ScriptedRestorer { + embedded: None, + modules: vec![(library.clone(), module.clone())], + }; + let forward = write("forward.apks", true); + let reverse = write("reverse.apks", false); + let out_forward = temp.path().join("out-forward"); + let out_reverse = temp.path().join("out-reverse"); + let first = restore_scripted( + &forward, + Path::new("forward.apks"), + &out_forward, + &mut cache_forward, + &mut restorer, + ); + let second = restore_scripted( + &reverse, + Path::new("reverse.apks"), + &out_reverse, + &mut cache_reverse, + &mut restorer, + ); + let meta_first = metadata_outcomes(&first); + let meta_second = metadata_outcomes(&second); + assert_eq!(meta_first.len(), 1); + assert_eq!(meta_second.len(), 1); + assert!(matches!(meta_first[0].status, EntryStatus::Restored)); + assert!(matches!(meta_second[0].status, EntryStatus::Restored)); + let expected = restored_with(&metadata, &module); + assert_eq!(std::fs::read(&meta_first[0].dest).unwrap(), expected); + assert_eq!(std::fs::read(&meta_second[0].dest).unwrap(), expected); + assert!(!meta_first[0].provenance.is_empty()); + assert!(!meta_second[0].provenance.is_empty()); +} + +#[test] +fn mixed_bundle_does_not_cross_package_identities() { + let temp = tempfile::tempdir().unwrap(); + let (meta_a, module_a) = pair(1); + let (meta_b, module_b) = pair(2); + let lib_a = library_bytes(b"mix-a"); + let lib_b = library_bytes(b"mix-b"); + let package = temp.path().join("mixed.apks"); + write_nested( + &package, + &[ + ( + "first.apk", + entries(&[ + ( + "AndroidManifest.xml", + manifest("example.one", "1").as_bytes(), + ), + ("lib/arm64-v8a/libil2cpp.so", &lib_a), + ( + "assets/bin/Data/Managed/Metadata/global-metadata.dat", + &meta_a, + ), + ]), + ), + ( + "second.apk", + entries(&[ + ( + "AndroidManifest.xml", + manifest("example.two", "1").as_bytes(), + ), + ("lib/arm64-v8a/libil2cpp.so", &lib_b), + ( + "assets/bin/Data/Managed/Metadata/global-metadata.dat", + &meta_b, + ), + ]), + ), + ], + ); + let mut cache = run_cache(); + let mut restorer = ScriptedRestorer { + embedded: None, + modules: vec![(lib_a, module_a.clone()), (lib_b, module_b.clone())], + }; + let outcomes = restore_scripted( + &package, + Path::new("mixed.apks"), + &temp.path().join("out"), + &mut cache, + &mut restorer, + ); + let metas = metadata_outcomes(&outcomes); + assert_eq!(metas.len(), 2); + assert!( + metas + .iter() + .all(|outcome| matches!(outcome.status, EntryStatus::Restored)), + "outcomes: {:?}", + metas + .iter() + .map(|outcome| { + let status = match &outcome.status { + EntryStatus::Failed(error) => format!("failed: {error:#}"), + other => format!("{other:?}"), + }; + (outcome.label.clone(), status, outcome.provenance.clone()) + }) + .collect::>() + ); + let bytes: Vec<_> = metas + .iter() + .map(|outcome| std::fs::read(&outcome.dest).unwrap()) + .collect(); + // Canonical method order does not depend on the seed, so both restored + // blobs match their own module. Isolation is the provenance: each metadata + // entry records only the library inside its own package. + assert!( + bytes + .iter() + .all(|item| item == &restored_with(&meta_a, &module_a) + || item == &restored_with(&meta_b, &module_b)) + ); + assert!(metas.iter().all(|outcome| outcome.provenance.len() == 1)); + assert_ne!(metas[0].provenance, metas[1].provenance); +} + +#[test] +fn duplicate_library_populates_the_owning_context() { + let temp = tempfile::tempdir().unwrap(); + let (metadata, module) = pair(4); + let library = library_bytes(b"dup-lib"); + let package = temp.path().join("dups.apk"); + let xml = manifest("example.dups", "9"); + write_package( + &package, + &[ + ("AndroidManifest.xml", xml.as_bytes()), + ("lib/arm64-v8a/libil2cpp.so", &library), + ("lib/armeabi-v7a/libil2cpp.so", &library), + ( + "assets/bin/Data/Managed/Metadata/global-metadata.dat", + &metadata, + ), + ], + ); + let mut cache = run_cache(); + let mut restorer = ScriptedRestorer { + embedded: None, + modules: vec![(library, module.clone())], + }; + let outcomes = restore_scripted( + &package, + Path::new("dups.apk"), + &temp.path().join("out"), + &mut cache, + &mut restorer, + ); + let libraries: Vec<_> = outcomes + .iter() + .filter(|outcome| matches!(outcome.kind, EntryKind::So)) + .collect(); + assert!(matches!(libraries[0].status, EntryStatus::Restored)); + assert!(matches!(libraries[1].status, EntryStatus::Duplicate)); + let meta = metadata_outcomes(&outcomes); + assert_eq!(meta.len(), 1); + assert!(matches!(meta[0].status, EntryStatus::Restored)); + assert_eq!( + std::fs::read(&meta[0].dest).unwrap(), + restored_with(&metadata, &module) + ); + assert!( + meta[0].provenance.len() >= 2, + "both library copies must be recorded, got {:?}", + meta[0].provenance + ); +} + +#[test] +fn missing_module_writes_no_v24_metadata() { + let temp = tempfile::tempdir().unwrap(); + let (metadata, module) = pair(5); + let package = temp.path().join("bare.apk"); + write_package( + &package, + &[( + "assets/bin/Data/Managed/Metadata/global-metadata.dat", + metadata.as_slice(), + )], + ); + let mut cache = run_cache(); + let mut restorer = ScriptedRestorer { + embedded: None, + modules: vec![(library_bytes(b"unused"), module)], + }; + let outcomes = restore_scripted( + &package, + Path::new("bare.apk"), + &temp.path().join("out"), + &mut cache, + &mut restorer, + ); + assert!( + outcomes + .iter() + .all(|outcome| matches!(outcome.status, EntryStatus::Failed(_))) + ); + assert!(outcomes.iter().all(|outcome| !outcome.dest.exists())); +} + +#[test] +fn conflicting_modules_write_no_metadata() { + let temp = tempfile::tempdir().unwrap(); + let (metadata, module_a) = pair(1); + let (_, module_b) = pair(2); + let lib_a = library_bytes(b"conflict-a"); + let lib_b = library_bytes(b"conflict-b"); + let package = temp.path().join("conflict.apk"); + let xml = manifest("example.conflict", "1"); + write_package( + &package, + &[ + ("AndroidManifest.xml", xml.as_bytes()), + ("lib/arm64-v8a/one.so", &lib_a), + ("lib/arm64-v8a/two.so", &lib_b), + ( + "assets/bin/Data/Managed/Metadata/global-metadata.dat", + &metadata, + ), + ], + ); + let mut cache = run_cache(); + let mut restorer = ScriptedRestorer { + embedded: None, + modules: vec![(lib_a, module_a), (lib_b, module_b)], + }; + let outcomes = restore_scripted( + &package, + Path::new("conflict.apk"), + &temp.path().join("out"), + &mut cache, + &mut restorer, + ); + let meta = metadata_outcomes(&outcomes); + assert_eq!(meta.len(), 1); + assert!(matches!(meta[0].status, EntryStatus::Failed(_))); + assert!(!meta[0].dest.exists()); + assert!(meta[0].provenance.len() >= 2); +} + +#[test] +fn same_metadata_bytes_cache_separately_per_module() { + let temp = tempfile::tempdir().unwrap(); + let (metadata, module_a) = pair(3); + // A second copy of the same module is still one profile. A different seed + // that does not restore this metadata must not be substituted for it. + let module_copy = module_a.clone(); + let (_other_metadata, other_module) = pair(4); + let lib_a = library_bytes(b"cache-a"); + let lib_b = library_bytes(b"cache-b"); + let bundle = temp.path().join("same-bytes.apks"); + write_nested( + &bundle, + &[ + ( + "one.apk", + entries(&[ + ( + "AndroidManifest.xml", + manifest("example.cache", "1").as_bytes(), + ), + ("lib/arm64-v8a/libil2cpp.so", &lib_a), + ( + "assets/bin/Data/Managed/Metadata/global-metadata.dat", + &metadata, + ), + ]), + ), + ( + "two.apk", + entries(&[ + ( + "AndroidManifest.xml", + manifest("example.cache", "2").as_bytes(), + ), + ("lib/arm64-v8a/libil2cpp.so", &lib_b), + ( + "assets/bin/Data/Managed/Metadata/global-metadata.dat", + &metadata, + ), + ]), + ), + ], + ); + let mut cache = run_cache(); + let mut restorer = ScriptedRestorer { + embedded: None, + modules: vec![(lib_a, module_copy), (lib_b, other_module)], + }; + let outcomes = restore_scripted( + &bundle, + Path::new("same-bytes.apks"), + &temp.path().join("out"), + &mut cache, + &mut restorer, + ); + let metas = metadata_outcomes(&outcomes); + assert_eq!(metas.len(), 2); + // Version 1 has this metadata's module and restores. Version 2 has a + // different module; the shared encrypted bytes must not reuse version 1's + // output, and version 2 writes nothing. + assert!(matches!(metas[0].status, EntryStatus::Restored)); + assert!(matches!(metas[1].status, EntryStatus::Failed(_))); + assert_eq!( + std::fs::read(&metas[0].dest).unwrap(), + restored_with(&metadata, &module_a) + ); + assert!(!metas[1].dest.exists()); + assert!(metas.iter().all(|outcome| outcome.provenance.len() == 1)); + assert_ne!(metas[0].provenance, metas[1].provenance); +} + +#[test] +fn loose_metadata_outside_an_extracted_boundary_is_isolated() { + let temp = tempfile::tempdir().unwrap(); + let (metadata, module) = pair(5); + let library = library_bytes(b"loose-boundary"); + let app = temp.path().join("game"); + std::fs::create_dir_all(app.join("lib/arm64-v8a")).unwrap(); + let so = app.join("lib/arm64-v8a/libil2cpp.so"); + // Same scan root, but not under `/assets/`. Must not borrow the module. + let stray = temp.path().join("global-metadata.dat"); + std::fs::write(&so, &library).unwrap(); + std::fs::write(&stray, &metadata).unwrap(); + let scoped = loose_scopes(std::slice::from_ref(&so), std::slice::from_ref(&stray)); + let stray_key = scoped + .iter() + .find(|item| item.path == stray) + .unwrap() + .key + .clone(); + assert!(matches!(stray_key, AppKey::Unidentifiable { .. })); + let mut cache = run_cache(); + let mut restorer = ScriptedRestorer { + embedded: None, + modules: vec![(library, module)], + }; + restore_loose_library_with( + &so, + &temp.path().join("out.so"), + "library", + &AppKey::Extracted(app), + false, + &mut cache, + &mut restorer, + ) + .unwrap(); + let error = restore_loose_metadata( + &stray, + &temp.path().join("out.dat"), + "stray", + &stray_key, + false, + &mut cache, + ) + .unwrap_err(); + let message = format!("{error:#}"); + assert!( + message.contains("no module 0x0C"), + "unexpected error: {message}" + ); + assert!(!temp.path().join("out.dat").exists()); +} + +type NestedEntries = Vec<(String, Vec)>; + +fn entries(pairs: &[(&str, &[u8])]) -> NestedEntries { + pairs + .iter() + .map(|(name, data)| ((*name).to_owned(), (*data).to_vec())) + .collect() +} + +fn write_nested(path: &Path, apks: &[(&str, NestedEntries)]) { + let mut archive = zip::ZipWriter::new(File::create(path).unwrap()); + for (name, nested) in apks { + let mut payload = Vec::new(); + { + let mut inner = zip::ZipWriter::new(std::io::Cursor::new(&mut payload)); + for (entry, data) in nested { + inner + .start_file(entry, SimpleFileOptions::default()) + .unwrap(); + inner.write_all(data).unwrap(); + } + inner.finish().unwrap(); + } + archive + .start_file(*name, SimpleFileOptions::default()) + .unwrap(); + archive.write_all(&payload).unwrap(); + } + archive.finish().unwrap(); +} + +#[test] +fn duplicate_library_writes_no_output() { + let temp = tempfile::tempdir().unwrap(); + let (metadata, module) = pair(1); + let library = library_bytes(b"dup-output"); + let package = temp.path().join("once.apk"); + write_package( + &package, + &[ + ( + "AndroidManifest.xml", + manifest("example.once", "1").as_bytes(), + ), + ("lib/arm64-v8a/libil2cpp.so", &library), + ("lib/armeabi-v7a/libil2cpp.so", &library), + ( + "assets/bin/Data/Managed/Metadata/global-metadata.dat", + &metadata, + ), + ], + ); + let mut cache = run_cache(); + let mut restorer = ScriptedRestorer { + embedded: None, + modules: vec![(library, module)], + }; + let outcomes = restore_scripted( + &package, + Path::new("once.apk"), + &temp.path().join("out"), + &mut cache, + &mut restorer, + ); + let libraries: Vec<_> = outcomes + .iter() + .filter(|outcome| matches!(outcome.kind, EntryKind::So)) + .collect(); + assert!(matches!(libraries[0].status, EntryStatus::Restored)); + assert!(libraries[0].dest.exists()); + assert!(matches!(libraries[1].status, EntryStatus::Duplicate)); + assert!( + !libraries[1].dest.exists(), + "a duplicate library must not write {}", + libraries[1].dest.display() + ); +} + +#[test] +fn embedded_metadata_failure_is_retried() { + let temp = tempfile::tempdir().unwrap(); + let library = library_bytes(b"embedded-retry"); + let package = temp.path().join("embed.apk"); + write_package(&package, &[("lib/arm64-v8a/libil2cpp.so", &library)]); + let first_root = temp.path().join("first"); + let library_dest = first_root.join("embed.apk/lib/arm64-v8a/libil2cpp.unpack.so"); + let metadata_dest = embedded_metadata_dest(&library_dest); + // Let the library write succeed, but prevent replacing its metadata file. + std::fs::create_dir_all(&metadata_dest).unwrap(); + let mut cache = run_cache(); + let mut restorer = ScriptedRestorer { + modules: vec![(library.clone(), pair(1).1)], + embedded: Some(b"embedded-blob".to_vec()), + }; + let first = restore_scripted( + &package, + Path::new("embed.apk"), + &first_root, + &mut cache, + &mut restorer, + ); + assert!( + first + .iter() + .any(|outcome| matches!(outcome.status, EntryStatus::Failed(_))), + "embedded write must fail, got {first:?}" + ); + assert!(cache.library(&content_identity(&library)).is_none()); + assert_eq!(std::fs::read(&library_dest).unwrap(), b"restored"); + assert!(metadata_dest.is_dir()); + let retry_root = temp.path().join("retry"); + let second = restore_scripted( + &package, + Path::new("embed.apk"), + &retry_root, + &mut cache, + &mut restorer, + ); + assert!( + second + .iter() + .any(|outcome| matches!(outcome.status, EntryStatus::Restored)) + ); +} + +#[test] +fn content_only_duplicate_recovers_context_without_touching_existing_output() { + let temp = tempfile::tempdir().unwrap(); + let library = library_bytes(b"prior-success"); + let (metadata, module) = pair(1); + let package = temp.path().join("app.apk"); + write_package( + &package, + &[ + ("lib/arm64-v8a/libil2cpp.so", &library), + ("assets/global-metadata.dat", &metadata), + ], + ); + let out = temp.path().join("out"); + let dest = out.join("app.apk/lib/arm64-v8a/libil2cpp.unpack.so"); + std::fs::create_dir_all(dest.parent().unwrap()).unwrap(); + std::fs::write(&dest, b"keep existing output").unwrap(); + let mut cache = run_cache(); + cache.insert_prior_success(content_identity(&library)); + let mut restorer = ScriptedRestorer { + modules: vec![(library, module.clone())], + embedded: None, + }; + let outcomes = restore_scripted( + &package, + Path::new("app.apk"), + &out, + &mut cache, + &mut restorer, + ); + assert!(matches!(outcomes[0].status, EntryStatus::Duplicate)); + assert_eq!(std::fs::read(&dest).unwrap(), b"keep existing output"); + let meta = metadata_outcomes(&outcomes); + assert!(matches!(meta[0].status, EntryStatus::Restored)); + assert_eq!( + std::fs::read(&meta[0].dest).unwrap(), + restored_with(&metadata, &module) + ); +} + +#[test] +fn loose_embedded_metadata_reports_its_written_path() { + let temp = tempfile::tempdir().unwrap(); + let input = temp.path().join("library.so"); + let library = library_bytes(b"loose-embedded"); + std::fs::write(&input, &library).unwrap(); + let dest = temp.path().join("out/library.unpack.so"); + let mut cache = run_cache(); + let mut restorer = ScriptedRestorer { + modules: vec![(library, pair(1).1)], + embedded: Some(b"metadata".to_vec()), + }; + let outcome = restore_loose_library_with( + &input, + &dest, + "library.so", + &AppKey::Unidentifiable { + source: "one".to_owned(), + }, + false, + &mut cache, + &mut restorer, + ) + .unwrap(); + assert!(matches!(outcome.status, EntryStatus::Restored)); + assert_eq!(outcome.embedded, Some(embedded_metadata_dest(&dest))); + assert_eq!( + std::fs::read(outcome.embedded.unwrap()).unwrap(), + b"metadata" + ); +} + +#[test] +fn public_seen_survives_package_error() { + let temp = tempfile::tempdir().unwrap(); + let missing = temp.path().join("missing.apk"); + let mut seen = std::collections::HashSet::from(["already-restored".to_owned()]); + let error = restore_package( + &missing, + Path::new("missing.apk"), + &temp.path().join("out"), + &mut seen, + false, + ); + assert!(error.is_err()); + assert!( + seen.contains("already-restored"), + "an error must not drop successes the caller already had: {seen:?}" + ); +} + +#[test] +fn binary_manifest_splits_share_and_versions_do_not() { + let temp = tempfile::tempdir().unwrap(); + let (metadata, _module) = pair(1); + let library = library_bytes(b"axml-lib"); + let same = axml_manifest("example.axml", 7, None, None); + let other = axml_manifest("example.axml", 8, None, None); + let base = temp.path().join("base.apk"); + let split = temp.path().join("split.apk"); + let newer = temp.path().join("newer.apk"); + write_package( + &base, + &[ + ("AndroidManifest.xml", &same), + ( + "assets/bin/Data/Managed/Metadata/global-metadata.dat", + &metadata, + ), + ], + ); + write_package( + &split, + &[ + ("AndroidManifest.xml", &same), + ("lib/arm64-v8a/libil2cpp.so", &library), + ], + ); + write_package( + &newer, + &[ + ("AndroidManifest.xml", &other), + ("lib/arm64-v8a/libil2cpp.so", &library), + ], + ); + let groups = loose_apk_groups(&[newer.clone(), base.clone(), split.clone()]); + let together = groups.iter().find(|group| group.contains(&base)).unwrap(); + assert!(together.contains(&split), "groups: {groups:?}"); + assert!(!together.contains(&newer), "groups: {groups:?}"); +} + +#[test] +fn truncated_binary_manifest_does_not_name_a_package() { + let temp = tempfile::tempdir().unwrap(); + let package = temp.path().join("cut.apk"); + let mut truncated = axml_manifest("example.cut", 1, None, None); + truncated.truncate(12); + write_package(&package, &[("AndroidManifest.xml", &truncated)]); + let groups = loose_apk_groups(std::slice::from_ref(&package)); + assert_eq!(groups, vec![vec![package]]); +} + +#[test] +fn non_v24_metadata_ignores_conflicting_modules() { + let temp = tempfile::tempdir().unwrap(); + let package = temp.path().join("later.apk"); + let lib_a = library_bytes(b"later-a"); + let lib_b = library_bytes(b"later-b"); + let metadata = metadata_fixture(); + write_package( + &package, + &[ + ( + "AndroidManifest.xml", + manifest("example.later", "1").as_bytes(), + ), + ("lib/arm64-v8a/one.so", &lib_a), + ("lib/arm64-v8a/two.so", &lib_b), + ( + "assets/bin/Data/Managed/Metadata/global-metadata.dat", + &metadata, + ), + ], + ); + let mut cache = run_cache(); + let mut restorer = ScriptedRestorer { + embedded: None, + modules: vec![(lib_a, pair(1).1), (lib_b, pair(2).1)], + }; + let outcomes = restore_scripted( + &package, + Path::new("later.apk"), + &temp.path().join("out"), + &mut cache, + &mut restorer, + ); + let meta = metadata_outcomes(&outcomes); + assert_eq!(meta.len(), 1); + assert!( + matches!( + meta[0].status, + EntryStatus::Restored | EntryStatus::Unchanged + ), + "v31 must not be blocked by conflicting modules: {:?}", + meta[0].status + ); + assert!( + meta[0].provenance.is_empty(), + "a module that was not used must not be recorded: {:?}", + meta[0].provenance + ); +} + +#[test] +fn version_name_alone_does_not_merge_builds() { + let temp = tempfile::tempdir().unwrap(); + let named = r#""#; + let first = temp.path().join("a.apk"); + let second = temp.path().join("b.apk"); + write_package( + &first, + &[( + "AndroidManifest.xml", + named.replace("{code}", "1").as_bytes(), + )], + ); + write_package( + &second, + &[( + "AndroidManifest.xml", + named.replace("{code}", "2").as_bytes(), + )], + ); + let groups = loose_apk_groups(&[first.clone(), second.clone()]); + assert!(groups.iter().all(|group| group.len() == 1), "{groups:?}"); +} + +#[test] +fn direct_assets_metadata_shares_the_extracted_root() { + let temp = tempfile::tempdir().unwrap(); + let app = temp.path().join("game"); + std::fs::create_dir_all(app.join("lib/arm64-v8a")).unwrap(); + std::fs::create_dir_all(app.join("assets")).unwrap(); + let so = app.join("lib/arm64-v8a/libil2cpp.so"); + let meta = app.join("assets/global-metadata.dat"); + std::fs::write(&so, b"so").unwrap(); + std::fs::write(&meta, b"meta").unwrap(); + let scoped = loose_scopes(std::slice::from_ref(&so), std::slice::from_ref(&meta)); + assert_eq!(scoped[0].key, scoped[1].key); + assert!(matches!(scoped[0].key, AppKey::Extracted(_))); +} + +fn axml_manifest( + package: &str, + version_code: u32, + major: Option, + version_name: Option<&str>, +) -> Vec { + let mut strings = vec![ + "manifest".to_owned(), + "package".to_owned(), + package.to_owned(), + "versionCode".to_owned(), + ]; + if major.is_some() { + strings.push("versionCodeMajor".to_owned()); + } + if let Some(version_name) = version_name { + strings.push("versionName".to_owned()); + strings.push(version_name.to_owned()); + } + let pool = axml_pool(&strings); + let mut attributes = Vec::new(); + attributes.extend(axml_attr(1, 2, 0x03, 2)); + attributes.extend(axml_attr(3, 0xffff_ffff, 0x10, version_code)); + if let Some(major) = major { + attributes.extend(axml_attr(4, 0xffff_ffff, 0x10, major)); + } + if version_name.is_some() { + let name_index = (strings.len() - 2) as u32; + let value_index = (strings.len() - 1) as u32; + attributes.extend(axml_attr(name_index, value_index, 0x03, value_index)); + } + let attr_count = attributes.len() / 20; + let mut element = Vec::new(); + element.extend(u16::to_le_bytes(0x0102)); + element.extend(u16::to_le_bytes(16)); + let element_size = 8 + 28 + attributes.len(); + element.extend(u32::to_le_bytes(element_size as u32)); + element.extend(u32::to_le_bytes(0)); + element.extend(u32::to_le_bytes(0xffff_ffff)); + element.extend(u32::to_le_bytes(0xffff_ffff)); + element.extend(u32::to_le_bytes(0)); + element.extend(u16::to_le_bytes(0x14)); // attribute start, from the node + element.extend(u16::to_le_bytes(20)); + element.extend(u16::to_le_bytes(attr_count as u16)); + element.extend(u16::to_le_bytes(0)); + element.extend(u16::to_le_bytes(0)); + element.extend(u16::to_le_bytes(0)); + element.extend(attributes); + let mut bytes = Vec::new(); + bytes.extend(u16::to_le_bytes(0x0003)); + bytes.extend(u16::to_le_bytes(8)); + bytes.extend(u32::to_le_bytes((8 + pool.len() + element.len()) as u32)); + bytes.extend(pool); + bytes.extend(element); + bytes +} + +fn axml_pool(strings: &[String]) -> Vec { + let mut data = Vec::new(); + let mut offsets = Vec::new(); + for value in strings { + offsets.push(data.len() as u32); + let units: Vec = value.encode_utf16().collect(); + data.extend(u16::to_le_bytes(units.len() as u16)); + for unit in units { + data.extend(u16::to_le_bytes(unit)); + } + data.extend(u16::to_le_bytes(0)); + } + let mut chunk = Vec::new(); + chunk.extend(u16::to_le_bytes(0x0001)); + chunk.extend(u16::to_le_bytes(28)); + let size = 28 + strings.len() * 4 + data.len(); + chunk.extend(u32::to_le_bytes(size as u32)); + chunk.extend(u32::to_le_bytes(strings.len() as u32)); + chunk.extend(u32::to_le_bytes(0)); // style count + chunk.extend(u32::to_le_bytes(0)); // flags + chunk.extend(u32::to_le_bytes((28 + strings.len() * 4) as u32)); + chunk.extend(u32::to_le_bytes(0)); // styles start + for offset in offsets { + chunk.extend(u32::to_le_bytes(offset)); + } + chunk.extend(data); + chunk +} + +fn axml_attr(name: u32, raw: u32, value_type: u8, data: u32) -> Vec { + let mut record = Vec::new(); + record.extend(u32::to_le_bytes(0xffff_ffff)); + record.extend(u32::to_le_bytes(name)); + record.extend(u32::to_le_bytes(raw)); + record.extend(u16::to_le_bytes(8)); + record.push(0); + record.push(value_type); + record.extend(u32::to_le_bytes(data)); + record +} diff --git a/senbei-io/src/job.rs b/senbei-io/src/job.rs index 87a3dd2..c9ea25c 100644 --- a/senbei-io/src/job.rs +++ b/senbei-io/src/job.rs @@ -213,64 +213,62 @@ pub fn run_folder_opts( bar.inc(1); } - // Android pass: protected AArch64 libraries and app packages. Loose `.so` - // files restore first so the cross-source dedup keeps them over a copy - // inside a package (loose beats `.apk` beats `.apks`/`.xapk` bundle). - let mut android_seen = std::collections::HashSet::new(); - let mut android_method_index_module = None; - // Hashing a protected library costs a full read, so only pay it when a - // duplicate source can actually exist in this run. - let android_dedup = scan.android_so.len() > 1 || !scan.android_packages.is_empty(); - for input in &scan.android_so { - let rel = rel_in_tree(root, input); + // Android pass. Loose libraries restore before packages so an identical + // library keeps the loose destination (loose beats `.apk` beats bundle). + // Module 0x0C stays inside the application that produced it: an extracted + // `lib//` tree shares with that tree's `assets/` metadata only, and a + // package shares only with splits of the same package name and version. + let mut android_cache = crate::android::run_cache(); + let loose = crate::android::loose_scopes(&scan.android_so, metas); + for scoped in loose.iter().filter(|scoped| { + scan.android_so + .iter() + .any(|library| library == &scoped.path) + }) { + let rel = rel_in_tree(root, &scoped.path); let dest = out_root.join(out_name(&rel)); - // Failed restores must not hide an identical input encountered later. - // Unreadable here is fine: the restore reports the same error. - let identity = android_dedup - .then(|| crate::android::file_content_identity(input).ok()) - .flatten(); - if identity - .as_ref() - .is_some_and(|id| android_seen.contains(id)) - { - s.skipped += 1; - if let Some(log) = &log { - log.step(&format!("SKIP {rel:?}: duplicate of an earlier target")); - } - bar.inc(1); - continue; - } - let input_owned = input.clone(); + let input_owned = scoped.path.clone(); let dest_owned = dest.clone(); + let key = scoped.key.clone(); + let label = rel.display().to_string(); let result = senbei_engine::catch_unwind_quiet(|| { - crate::android::restore_so_file_with_method_index_module( + crate::android::restore_loose_library( &input_owned, &dest_owned, + &label, + &key, verbose_steps, - &mut android_method_index_module, + &mut android_cache, ) }); match result { - Ok(Ok(embedded)) => { - s.unpacked += 1; - crate::ui::ok_label( - &bar, - suppress_file_lines, - &rel.display().to_string(), - "So", - &dest, - ); - if let Some(log) = &log { - log.step(&format!("OK {rel:?} -> {dest:?} (Android SO)")); + Ok(Ok(outcome)) => match outcome.status { + crate::android::EntryStatus::Duplicate => { + s.skipped += 1; + if let Some(log) = &log { + log.step(&format!("SKIP {rel:?}: duplicate of an earlier target")); + } } - let embedded_result = write_embedded_metadata(embedded, &dest); - if embedded_result.is_ok() - && let Some(identity) = identity - { - android_seen.insert(identity); + crate::android::EntryStatus::Failed(e) => { + s.errors += 1; + crate::ui::err(&bar, suppress_file_lines, &rel, &e); + if let Some(log) = &log { + log.step(&format!("ERR {rel:?}: {e:#}")); + } } - match embedded_result { - Ok(Some(meta_dest)) => { + crate::android::EntryStatus::Restored => { + s.unpacked += 1; + crate::ui::ok_label( + &bar, + suppress_file_lines, + &rel.display().to_string(), + "So", + &dest, + ); + if let Some(log) = &log { + log.step(&format!("OK {rel:?} -> {dest:?} (Android SO)")); + } + if let Some(meta_dest) = outcome.embedded { s.metadata += 1; crate::ui::ok_label( &bar, @@ -283,16 +281,11 @@ pub fn run_folder_opts( log.step(&format!("META {rel:?} (embedded) -> {meta_dest:?}")); } } - Ok(None) => {} - Err(e) => { - s.errors += 1; - crate::ui::err(&bar, suppress_file_lines, &rel, &e); - if let Some(log) = &log { - log.step(&format!("ERR {rel:?}: embedded metadata: {e:#}")); - } - } } - } + crate::android::EntryStatus::NotTarget | crate::android::EntryStatus::Unchanged => { + s.skipped += 1; + } + }, Ok(Err(e)) => { s.errors += 1; crate::ui::err(&bar, suppress_file_lines, &rel, &e); @@ -314,71 +307,120 @@ pub fn run_folder_opts( } bar.inc(1); } - for package in &scan.android_packages { - let rel = rel_in_tree(root, package); - s.packages += 1; - let package_owned = package.clone(); - let rel_owned = rel.clone().into_owned(); - let out_root_owned = out_root.clone(); - let mut seen_taken = std::mem::take(&mut android_seen); - let result = senbei_engine::catch_unwind_quiet(|| { - let outcomes = crate::android::restore_package_with_method_index_module( - &package_owned, - &rel_owned, - &out_root_owned, - &mut seen_taken, - verbose_steps, - &mut android_method_index_module, - ); - (outcomes, seen_taken) - }); - match result { - Ok((Ok(outcomes), seen_back)) => { - android_seen = seen_back; - apply_package_outcomes(outcomes, &mut s, &bar, suppress_file_lines, &log); - } - Ok((Err(e), seen_back)) => { - android_seen = seen_back; - s.errors += 1; - crate::ui::err(&bar, suppress_file_lines, &rel, &e); - if let Some(log) = &log { - log.step(&format!("ERR {rel:?}: {e:#}")); + // Loose sibling APKs that name the same package and version are one split + // set. Their libraries run before their metadata because each package + // restores its own libraries first and the shared cache collects modules + // across the group. Bundles stay one package each. + let package_groups = crate::android::loose_apk_groups(&scan.android_packages); + for group in &package_groups { + if group.len() > 1 { + s.packages += group.len(); + let group_owned = group.clone(); + let root_owned = root.to_path_buf(); + let out_root_owned = out_root.clone(); + let result = senbei_engine::catch_unwind_quiet(|| { + crate::android::restore_apk_group( + &group_owned, + &root_owned, + &out_root_owned, + &mut android_cache, + verbose_steps, + ) + }); + let rel = rel_in_tree(root, &group[0]); + match result { + Ok(Ok(outcomes)) => { + apply_package_outcomes(outcomes, &mut s, &bar, suppress_file_lines, &log); } - } - Err(panic) => { - // The dedup set may be in an unknown state after a panic; a - // re-scan costs a duplicate restore at worst, never corruption. - let e = anyhow::anyhow!("unexpected panic: {}", panic_payload(&panic)); - s.errors += 1; - crate::ui::err(&bar, suppress_file_lines, &rel, &e); - if let Some(log) = &log { - log.step(&format!( - "ERR {rel:?}: panic during package restore: {}", - panic_payload(&panic) - )); + Ok(Err(e)) => { + s.errors += 1; + crate::ui::err(&bar, suppress_file_lines, &rel, &e); + if let Some(log) = &log { + log.step(&format!("ERR {rel:?}: {e:#}")); + } + } + Err(panic) => { + let e = anyhow::anyhow!("unexpected panic: {}", panic_payload(&panic)); + s.errors += 1; + crate::ui::err(&bar, suppress_file_lines, &rel, &e); + if let Some(log) = &log { + log.step(&format!( + "ERR {rel:?}: panic during package restore: {}", + panic_payload(&panic) + )); + } } } + bar.inc(group.len() as u64); + continue; + } + for package in group { + let rel = rel_in_tree(root, package); + s.packages += 1; + let package_owned = package.clone(); + let rel_owned = rel.clone().into_owned(); + let out_root_owned = out_root.clone(); + let result = senbei_engine::catch_unwind_quiet(|| { + crate::android::restore_package_cached( + &package_owned, + &rel_owned, + &out_root_owned, + &mut android_cache, + verbose_steps, + ) + }); + match result { + Ok(Ok(outcomes)) => { + apply_package_outcomes(outcomes, &mut s, &bar, suppress_file_lines, &log); + } + Ok(Err(e)) => { + s.errors += 1; + crate::ui::err(&bar, suppress_file_lines, &rel, &e); + if let Some(log) = &log { + log.step(&format!("ERR {rel:?}: {e:#}")); + } + } + Err(panic) => { + // A panic may leave a partial module recorded. Keeping the cache + // preserves successful dedup; a conflict still fails closed. + let e = anyhow::anyhow!("unexpected panic: {}", panic_payload(&panic)); + s.errors += 1; + crate::ui::err(&bar, suppress_file_lines, &rel, &e); + if let Some(log) = &log { + log.step(&format!( + "ERR {rel:?}: panic during package restore: {}", + panic_payload(&panic) + )); + } + } + } + bar.inc(1); } - bar.inc(1); } - // il2cpp metadata pass. Crackproof's `-GMD` option obfuscates the method - // tokens in `global-metadata.dat`; de-obfuscate any we find so the unpacked - // il2cpp game assembly resolves methods instead of indexing its per-module - // tables out of bounds (see [`senbei_metadata`]). This is additive to the - // Crackproof module unpack above — the metadata blob is not itself a - // Crackproof file. - for meta in metas.iter() { - let rel = rel_in_tree(root, meta); + // il2cpp metadata pass. A loose blob is paired only with the application + // assigned when the tree was grouped. v24.1 with a missing or conflicting + // module writes nothing. Other versions still restore when the application + // has no conflicting module. The paired libraries are recorded on the log + // line. + for scoped in loose + .iter() + .filter(|scoped| metas.iter().any(|meta| meta == &scoped.path)) + { + let rel = rel_in_tree(root, &scoped.path); let dest = out_root.join(out_name(&rel)); - let meta_owned = meta.clone(); + let meta_owned = scoped.path.clone(); let dest_owned = dest.clone(); + let key = scoped.key.clone(); + let label = rel.display().to_string(); let result = senbei_engine::catch_unwind_quiet(|| { - deobfuscate_metadata_to_with_module( + crate::android::restore_loose_metadata( &meta_owned, &dest_owned, + &label, + &key, verbose_steps, - android_method_index_module.as_deref(), + &mut android_cache, ) }); match result { @@ -387,8 +429,10 @@ pub fn run_folder_opts( crate::ui::metadata(&bar, suppress_file_lines, &rel, report.remapped, &dest); if let Some(log) = &log { log.step(&format!( - "META {rel:?} -> {dest:?}: v{} remapped {} method fields", - report.version, report.remapped + "META {rel:?} -> {dest:?}: v{} remapped {} method fields{}", + report.version, + report.remapped, + provenance_suffix(&report.provenance) )); } } @@ -397,8 +441,9 @@ pub fn run_folder_opts( Ok(Ok(report)) => { if let Some(log) = &log { log.step(&format!( - "META {rel:?}: v{} already de-obfuscated", - report.version + "META {rel:?}: v{} already de-obfuscated{}", + report.version, + provenance_suffix(&report.provenance) )); } } @@ -709,6 +754,14 @@ pub fn out_name(input: &Path) -> PathBuf { } } +fn provenance_suffix(sources: &[String]) -> String { + if sources.is_empty() { + String::new() + } else { + format!("; module 0x0C from {}", sources.join(", ")) + } +} + /// Extract a printable message from a caught panic payload. fn panic_payload(panic: &(dyn std::any::Any + Send)) -> String { if let Some(s) = panic.downcast_ref::<&str>() { @@ -832,7 +885,12 @@ fn apply_package_outcomes( } } if let Some(log) = log { - log.step(&format!("OK {} -> {:?}", outcome.label, outcome.dest)); + log.step(&format!( + "OK {} -> {:?}{}", + outcome.label, + outcome.dest, + provenance_suffix(&outcome.provenance) + )); } } EntryStatus::Duplicate | EntryStatus::NotTarget | EntryStatus::Unchanged => {