fix(android): bind metadata recovery to its application

This commit is contained in:
Momoko-Ayase committed 2026-11-04 13:14:25 +08:00
1 parent 303aaa6283
commit 22dd92982e
5 files changed
+3104 -189

No files matched your search

+512
View File
@@ -0,0 +1,512 @@
//! Application-scoped module `0x0C` context for Android metadata recovery.
//!
//! One decoded module belongs to one application. Split APKs of that application
//! share it; unrelated games under the same scan do not. A metadata blob is
//! restored only after its application context is unambiguous, and the identity
//! of a successful restore includes that context: the same encrypted bytes can
//! decode differently under two module profiles.
use std::collections::{BTreeMap, HashMap};
use std::path::{Component, Path, PathBuf};
use anyhow::Result;
/// Where a decoded module `0x0C` image was read from.
#[derive(Debug, Clone, PartialEq, Eq)]
pub(crate) struct ModuleSource {
/// Protected library this module was recovered from (package label or path).
pub label: String,
/// Stage-2 image name inside that library, when the restorer reported one.
pub image: Option<String>,
}
/// One application's collected module `0x0C` images.
///
/// Distinct byte profiles are kept. Duplicate libraries populate the same
/// context instead of replacing it, so a later game cannot win by being last.
#[derive(Debug, Default, Clone)]
pub(crate) struct ApplicationModules {
profiles: BTreeMap<String, Vec<u8>>,
sources: Vec<ModuleSource>,
}
impl ApplicationModules {
pub(crate) fn record(&mut self, module: &[u8], source: ModuleSource) {
let identity = crate::android::content_identity(module);
self.profiles
.entry(identity)
.or_insert_with(|| module.to_vec());
self.sources.push(source);
}
pub(crate) fn is_empty(&self) -> bool {
self.profiles.is_empty()
}
/// The single module image, or `None` when this application recovered none
/// or more than one distinct profile.
pub(crate) fn unambiguous_module(&self) -> Option<&[u8]> {
let mut values = self.profiles.values();
let first = values.next()?;
values.next().is_none().then_some(first.as_slice())
}
pub(crate) fn profile_count(&self) -> usize {
self.profiles.len()
}
pub(crate) fn sources(&self) -> &[ModuleSource] {
&self.sources
}
}
/// Why a metadata blob cannot be paired with exactly one module `0x0C`.
#[derive(Debug, Clone, PartialEq, Eq)]
pub(crate) enum ContextAmbiguity {
/// No library in this application yielded a module.
Missing,
/// Libraries in this application yielded more than one distinct module.
Conflicting { profiles: usize },
}
/// Run-scoped cache of restored libraries and application module contexts.
///
/// The public package API still takes a content-identity set. This cache is
/// internal: a library hit reuses the restored image, while a metadata hit is
/// keyed by the recovery context as well as the encrypted bytes.
#[derive(Debug, Default)]
pub(crate) struct AndroidRunCache {
apps: HashMap<AppKey, ApplicationModules>,
/// Successfully restored protected libraries, keyed by encrypted content.
libraries: HashMap<String, CachedLibrary>,
/// Successfully handled metadata, keyed by encrypted content and context.
metadata: HashMap<MetadataCacheKey, CachedMetadata>,
/// Content identities already restored before this cache existed (the public
/// `HashSet` contract). They skip a repeat of the same bytes but carry no
/// module image, so they never become another application's profile.
prior_success: std::collections::HashSet<String>,
}
#[derive(Debug, Clone, PartialEq, Eq, Hash)]
struct MetadataCacheKey {
content: String,
context: String,
}
#[derive(Debug, Clone)]
pub(crate) struct CachedLibrary {
/// Module `0x0C` from a library whose restore, including any embedded
/// metadata write, succeeded. The restored image is not kept: a duplicate
/// writes nothing.
pub method_index_module: Option<Vec<u8>>,
pub module_image: Option<String>,
}
#[derive(Debug, Clone)]
pub(crate) enum CachedMetadata {
/// Successfully written package metadata. A duplicate does not write again.
Restored { remapped: usize },
/// The blob was already canonical under this context.
Unchanged,
}
impl AndroidRunCache {
pub(crate) fn modules_mut(&mut self, key: &AppKey) -> &mut ApplicationModules {
self.apps.entry(key.clone()).or_default()
}
pub(crate) fn modules(&self, key: &AppKey) -> Option<&ApplicationModules> {
self.apps.get(key)
}
pub(crate) fn library(&self, content: &str) -> Option<&CachedLibrary> {
self.libraries.get(content)
}
pub(crate) fn insert_prior_success(&mut self, content: String) {
self.prior_success.insert(content);
}
pub(crate) fn is_prior_success(&self, content: &str) -> bool {
self.prior_success.contains(content)
}
/// Content identities safe to publish back to the public successful-only set:
/// prior successes plus libraries and metadata handled successfully here.
/// Metadata identities are included because the public set still dedups a
/// repeated blob inside one package call; cross-application recovery uses
/// the context-keyed cache instead of this set.
pub(crate) fn successful_content_identities(&self) -> Vec<String> {
let mut identities: Vec<String> = self.prior_success.iter().cloned().collect();
identities.extend(self.libraries.keys().cloned());
identities.extend(self.metadata.keys().map(|key| key.content.clone()));
identities.sort();
identities.dedup();
identities
}
pub(crate) fn insert_library(&mut self, content: String, cached: CachedLibrary) {
self.libraries.entry(content).or_insert(cached);
}
pub(crate) fn metadata(&self, content: &str, context: &str) -> Option<&CachedMetadata> {
self.metadata.get(&MetadataCacheKey {
content: content.to_owned(),
context: context.to_owned(),
})
}
pub(crate) fn insert_metadata(
&mut self,
content: String,
context: String,
cached: CachedMetadata,
) {
self.metadata
.entry(MetadataCacheKey { content, context })
.or_insert(cached);
}
}
/// Identity of one Android application inside a folder run or one package set.
///
/// Constructed only from manifest package name **and** version, or the
/// conservative extracted `lib/<abi>/` + `assets/` boundary. The same package
/// name in a different version, or in a different bundle / loose tree, is a
/// different application. Scan-root names and ZIP entry names are never used
/// as a guess.
#[derive(Debug, Clone, PartialEq, Eq, Hash)]
pub(crate) enum AppKey {
/// Manifest `package` plus `versionCode` (or `versionName` when the code is
/// absent). `package_set` is the directory of the package file, or the
/// bundle file itself, so the same name and version elsewhere do not merge.
Manifest {
package: String,
version: String,
package_set: PathBuf,
},
/// Parent of a real `lib/<abi>/` or `assets/` boundary.
Extracted(PathBuf),
/// No manifest and no extracted boundary. Isolated so it cannot see another
/// application's modules. Not shared with any other unidentifiable input.
Unidentifiable { source: String },
}
/// A metadata or library input assigned to one application context.
#[derive(Debug, Clone, PartialEq, Eq)]
pub(crate) struct ScopedInput {
pub path: PathBuf,
pub key: AppKey,
}
/// Group loose Android targets by application without crossing unrelated games.
///
/// A `lib/<abi>/` library and an `assets/` metadata blob share a context only
/// when both sit directly under the same application root. Anything else is
/// isolated. Order of `libraries` and `metadata` does not matter.
pub(crate) fn group_loose_inputs(libraries: &[PathBuf], metadata: &[PathBuf]) -> Vec<ScopedInput> {
let mut scoped = Vec::with_capacity(libraries.len() + metadata.len());
for path in libraries {
scoped.push(ScopedInput {
key: loose_library_key(path),
path: path.clone(),
});
}
for path in metadata {
scoped.push(ScopedInput {
key: loose_metadata_key(path),
path: path.clone(),
});
}
scoped.sort_by(|left, right| left.path.cmp(&right.path));
scoped
}
fn loose_library_key(path: &Path) -> AppKey {
extracted_app_root(path).map_or_else(
|| AppKey::Unidentifiable {
source: path.to_string_lossy().into_owned(),
},
AppKey::Extracted,
)
}
fn loose_metadata_key(path: &Path) -> AppKey {
// `assets/bin/Data/Managed/Metadata/global-metadata.dat` and the shorter
// `assets/bin/Data/Managed/global-metadata.dat` both end at `assets/`.
assets_app_root(path).map_or_else(
|| AppKey::Unidentifiable {
source: path.to_string_lossy().into_owned(),
},
AppKey::Extracted,
)
}
/// Application root when `path` is `<root>/lib/<abi>/<library>.so`.
fn extracted_app_root(path: &Path) -> Option<PathBuf> {
let mut components: Vec<Component<'_>> = path.components().collect();
let file = components.pop()?;
let abi = components.pop()?;
let lib = components.pop()?;
if !component_is(file, path.file_name()?.to_str()?) {
return None;
}
// Compare components, not `Path::parent` display. A path joined from mixed
// separators still yields `lib` and `<abi>` as two normal components.
if !component_is(lib, "lib") || !matches!(abi, Component::Normal(_)) {
return None;
}
let root: PathBuf = components.into_iter().collect();
Some(if root.as_os_str().is_empty() {
PathBuf::from(".")
} else {
root
})
}
/// Application root when `path` is `<root>/assets/**/global-metadata.dat`.
fn assets_app_root(path: &Path) -> Option<PathBuf> {
let mut components: Vec<Component<'_>> = path.components().collect();
let name = components.pop()?;
if !component_is(name, crate::METADATA_FILE_NAME) {
return None;
}
let assets_at = components
.iter()
.rposition(|component| component_is(*component, "assets"))?;
// `assets/global-metadata.dat` and `assets/**/global-metadata.dat` both
// belong to the directory that contains `assets`.
let root: PathBuf = components[..assets_at].iter().collect();
Some(if root.as_os_str().is_empty() {
PathBuf::from(".")
} else {
root
})
}
fn component_eq(path: &Path, expected: &str) -> bool {
// The final component must be exactly `expected`. `file_name` on a directory
// path is that component; requiring the whole path to equal it would reject
// every application root that is not a single path segment.
path.file_name()
.and_then(|name| name.to_str())
.is_some_and(|name| name.eq_ignore_ascii_case(expected))
}
fn component_is(component: Component<'_>, expected: &str) -> bool {
match component {
Component::Normal(name) => name
.to_str()
.is_some_and(|name| name.eq_ignore_ascii_case(expected)),
_ => false,
}
}
/// One ZIP whose central directory was listed while naming an application.
///
/// `reopen_key` is the on-disk package (or extracted nested APK) to open again
/// when entries are restored. A later resource budget must charge this archive's
/// physical entry count on the first inventory pass only; reopening it for
/// entry extraction is not a second package.
#[derive(Debug, Clone, PartialEq, Eq)]
pub(crate) struct InventoriedArchive {
pub reopen_key: PathBuf,
/// Central-directory index of `AndroidManifest.xml`, when one was present.
/// `None` means the manifest entry was absent and nothing was read.
pub manifest_index: Option<usize>,
/// Central-directory entry count observed by this inventory pass.
pub entry_count: usize,
}
/// Inventory of one package (or nested split) that can name its application.
#[derive(Debug, Clone)]
pub(crate) struct PackageInventory {
pub key: AppKey,
/// True when `key` came from a parsed manifest `package` and version.
/// Distinguishes a named package from an isolated one for a later report.
#[allow(dead_code)]
pub manifest_named: bool,
/// Reopen handle for this archive. Entry extraction uses the already-open
/// handle; this records the central directory that must be charged once.
pub archive: InventoriedArchive,
}
/// Manifest-first application identity for one ZIP package.
///
/// Reads only the root `AndroidManifest.xml` entry, and only the leading window
/// needed for the plain-XML `package` and version attributes. A missing or
/// binary manifest does not fall back to the ZIP file name; the caller isolates
/// that package. `package_set` is the bundle or directory that owns this set of
/// splits — the same package name outside that boundary is a different app.
///
/// The returned [`InventoriedArchive`] is the reopen handle. Callers restore
/// entries from `archive.reopen_key` and must not treat that second open as a
/// newly discovered package.
pub(crate) fn inventory_manifest_package<R: std::io::Read + std::io::Seek>(
archive: &mut zip::ZipArchive<R>,
reopen_key: &Path,
package_set: &Path,
isolate_as: &str,
) -> Result<PackageInventory> {
let entry_count = archive.len();
let mut manifest_index = None;
for index in 0..entry_count {
let entry = archive.by_index(index)?;
if entry.is_dir() {
continue;
}
let Some(name) = entry.enclosed_name() else {
continue;
};
// Root manifest only. Nested paths and every other entry stay unread.
if name.components().count() == 1 && component_eq(&name, "AndroidManifest.xml") {
manifest_index = Some(index);
break;
}
}
let Some(index) = manifest_index else {
return Ok(PackageInventory {
key: AppKey::Unidentifiable {
source: isolate_as.to_owned(),
},
manifest_named: false,
archive: InventoriedArchive {
reopen_key: reopen_key.to_path_buf(),
manifest_index: None,
entry_count,
},
});
};
let mut entry = archive.by_index(index)?;
// The package and version attributes are in the root `<manifest` start tag.
// Bound the read. A later shared ZIP budget should charge this probe; it
// never opens assets or other entries. A manifest larger than the bound is
// truncated and produces no identity rather than a partial guess.
const MANIFEST_PROBE: usize = 4 * 1024 * 1024;
let mut buffer = vec![0_u8; MANIFEST_PROBE];
let mut filled = 0_usize;
while filled < buffer.len() {
match std::io::Read::read(&mut entry, &mut buffer[filled..])? {
0 => break,
read => filled += read,
}
}
buffer.truncate(filled);
let truncated = entry.size() > MANIFEST_PROBE as u64;
let named = super::manifest::identity(&buffer, truncated);
Ok(PackageInventory {
key: match &named {
Some((package, version)) => AppKey::Manifest {
package: package.clone(),
version: version.clone(),
package_set: package_set.to_path_buf(),
},
None => AppKey::Unidentifiable {
source: isolate_as.to_owned(),
},
},
manifest_named: named.is_some(),
archive: InventoriedArchive {
reopen_key: reopen_key.to_path_buf(),
manifest_index: Some(index),
entry_count,
},
})
}
/// Recovery-context identity mixed into metadata dedup.
///
/// Same encrypted bytes under two modules (or under "no module" versus a module)
/// must not collapse to one cached output.
pub(crate) fn metadata_context_identity(module: Option<&[u8]>) -> String {
match module {
Some(module) => format!("module:{}", crate::android::content_identity(module)),
None => "module:none".to_owned(),
}
}
/// Resolve the module a metadata blob may use, refusing an ambiguous context
/// before any output is written.
pub(crate) fn binding_module(
modules: Option<&ApplicationModules>,
) -> std::result::Result<Option<&[u8]>, ContextAmbiguity> {
let Some(modules) = modules else {
return Ok(None);
};
if modules.is_empty() {
return Err(ContextAmbiguity::Missing);
}
match modules.unambiguous_module() {
Some(module) => Ok(Some(module)),
None => Err(ContextAmbiguity::Conflicting {
profiles: modules.profile_count(),
}),
}
}
/// Provenance recorded on a metadata outcome: every library that contributed
/// the module actually used, or the reason no module was bound.
#[derive(Debug, Clone, PartialEq, Eq, Default)]
pub(crate) struct MetadataProvenance {
pub sources: Vec<ModuleSource>,
pub note: Option<String>,
}
pub(crate) fn provenance_for(
modules: Option<&ApplicationModules>,
ambiguity: Option<&ContextAmbiguity>,
) -> MetadataProvenance {
match ambiguity {
Some(ContextAmbiguity::Missing) => MetadataProvenance {
sources: modules
.map(|modules| modules.sources().to_vec())
.unwrap_or_default(),
note: Some("no module 0x0C in this application".to_owned()),
},
Some(ContextAmbiguity::Conflicting { profiles }) => MetadataProvenance {
sources: modules
.map(|modules| modules.sources().to_vec())
.unwrap_or_default(),
note: Some(format!(
"ambiguous module 0x0C binding ({profiles} distinct profiles)"
)),
},
None => MetadataProvenance {
sources: modules
.map(|modules| modules.sources().to_vec())
.unwrap_or_default(),
note: None,
},
}
}
/// What a library restore contributes to an application context.
#[derive(Debug, Clone)]
pub(crate) struct RestoredLibrary {
pub embedded_metadata: Option<Vec<u8>>,
pub method_index_module: Option<Vec<u8>>,
pub module_image: Option<String>,
}
/// Injectable library restore. Tests supply a fake so application-context
/// behaviour does not need a protected ELF fixture. Production calls the
/// stage-2 extractor.
pub(crate) trait LibraryRestorer {
fn restore(&mut self, input: &Path, dest: &Path, verbose: bool) -> Result<RestoredLibrary>;
/// Bytes that should be treated as a protected library without the ELF
/// probe. Production returns `None`; tests return the synthetic payload so
/// a package entry can reach the restorer.
fn protected_override(&mut self, _input: &Path) -> Option<Vec<u8>> {
None
}
}
/// Production restorer: stage-2 extract, ELF rebuild, embedded metadata unwrap.
pub(crate) struct EngineRestorer;
impl LibraryRestorer for EngineRestorer {
fn restore(&mut self, input: &Path, dest: &Path, verbose: bool) -> Result<RestoredLibrary> {
crate::android::restore_so_with_engine(input, dest, verbose)
}
}
+713
View File
@@ -0,0 +1,713 @@
//! Bounded identity of one Android manifest.
//!
//! The only result is `(package, version)`. Version is `android:versionCode`,
//! with `android:versionCodeMajor` prefixed when it is present and non-zero.
//! `android:versionName` is not an identity. A truncated or corrupt manifest
//! returns `None` rather than a partial guess.
//!
//! Chunk layout follows `androidfw/ResourceTypes.h`: a start element is a
//! `ResXMLTree_node` (`headerSize` is normally 16) followed by `ResXMLTree_attrExt`
//! at that header size. Attribute records begin at `headerSize + attributeStart`.
//! `Res_value::dataType` is the byte immediately before the data word.
/// `(package, version)` when the root element is `manifest` and carries a
/// package plus a version code. Truncated probes never supply a binding.
pub(super) fn identity(bytes: &[u8], truncated: bool) -> Option<(String, String)> {
if truncated {
return None;
}
if bytes.len() >= 8 && u16::from_le_bytes([bytes[0], bytes[1]]) == 0x0003 {
return binary_identity(bytes);
}
plain_identity(bytes, truncated)
}
const TYPE_STRING: u8 = 0x03;
const TYPE_INT_DEC: u8 = 0x10;
const TYPE_INT_HEX: u8 = 0x11;
const VERSION_CODE: u32 = 0x0101_021b;
const VERSION_CODE_MAJOR: u32 = 0x0101_0576;
const NO_STRING: u32 = 0xffff_ffff;
fn binary_identity(bytes: &[u8]) -> Option<(String, String)> {
if bytes.len() < 8 {
return None;
}
let file_size = u32::from_le_bytes(bytes.get(4..8)?.try_into().ok()?) as usize;
if file_size < 8 || file_size > bytes.len() {
return None;
}
let bytes = bytes.get(..file_size)?;
let mut offset = read_u16(bytes, 2)? as usize;
if offset < 8 || offset > bytes.len() {
return None;
}
let mut pool: Option<StringPool<'_>> = None;
let mut resource_map = Vec::new();
while offset + 8 <= bytes.len() {
let kind = read_u16(bytes, offset)?;
let header = read_u16(bytes, offset + 2)? as usize;
let size = read_u32(bytes, offset + 4)? as usize;
if header < 8 || size < header || offset.checked_add(size)? > bytes.len() {
return None;
}
let chunk = bytes.get(offset..offset + size)?;
match kind {
0x0001 => pool = Some(StringPool::parse(chunk)?),
0x0180 => resource_map = resource_map_of(chunk)?,
0x0102 => return start_element_identity(chunk, pool.as_ref()?, &resource_map),
_ => {}
}
offset += size;
}
None
}
fn start_element_identity(
chunk: &[u8],
pool: &StringPool<'_>,
resource_map: &[u32],
) -> Option<(String, String)> {
let header = read_u16(chunk, 2)? as usize;
// ResXMLTree_attrExt sits headerSize bytes after the chunk start.
if header < 16 || chunk.len() < header.checked_add(20)? {
return None;
}
let ext = header;
let name = read_u32(chunk, ext + 4)?;
if pool.get(name)?.as_str() != "manifest" {
return None;
}
let attribute_start = read_u16(chunk, ext + 8)? as usize;
let attribute_size = read_u16(chunk, ext + 10)? as usize;
let attribute_count = read_u16(chunk, ext + 12)? as usize;
if attribute_size < 20 || attribute_start < 20 {
return None;
}
let base = header.checked_add(attribute_start)?;
let bytes = attribute_count.checked_mul(attribute_size)?;
if base.checked_add(bytes)? > chunk.len() {
return None;
}
let mut package = None;
let mut version_code = None;
let mut version_major = None;
for index in 0..attribute_count {
let at = base + index * attribute_size;
let record = chunk.get(at..at + 20)?;
let namespace = read_u32(record, 0)?;
let name = read_u32(record, 4)?;
let raw = read_u32(record, 8)?;
// Res_value: size u16, res0 u8, dataType u8, data u32.
let data_type = *record.get(15)?;
let data = read_u32(record, 16)?;
let name_text = pool.get(name)?;
let resource = resource_map.get(name as usize).copied();
// `package` is the un-namespaced attribute. A resource id is never
// used for it: `0x01010003` is `android:name`, not the package.
if namespace == NO_STRING && name_text == "package" {
package = Some(attribute_text(pool, raw, data_type, data)?);
} else if name_text == "versionCode" || resource == Some(VERSION_CODE) {
version_code = Some(version_number(pool, raw, data_type, data)?);
} else if name_text == "versionCodeMajor" || resource == Some(VERSION_CODE_MAJOR) {
version_major = Some(version_number(pool, raw, data_type, data)?);
}
}
finish_identity(package?, version_code, version_major)
}
fn attribute_text(pool: &StringPool<'_>, raw: u32, data_type: u8, data: u32) -> Option<String> {
if data_type == TYPE_STRING {
return pool.get(data);
}
if raw != NO_STRING {
return pool.get(raw);
}
None
}
fn version_number(pool: &StringPool<'_>, raw: u32, data_type: u8, data: u32) -> Option<String> {
match data_type {
TYPE_INT_DEC | TYPE_INT_HEX => Some(data.to_string()),
TYPE_STRING => normalize_version(&pool.get(data)?),
_ if raw != NO_STRING => normalize_version(&pool.get(raw)?),
_ => None,
}
}
fn finish_identity(
package: String,
version_code: Option<String>,
version_major: Option<String>,
) -> Option<(String, String)> {
if !is_package_token(&package) {
return None;
}
let code = version_code?;
let version = match version_major {
Some(major) if major != "0" => format!("{major}.{code}"),
_ => code,
};
if !is_version_token(&version) {
return None;
}
Some((package, version))
}
struct StringPool<'a> {
bytes: &'a [u8],
offsets: Vec<u32>,
strings_start: usize,
utf8: bool,
}
impl<'a> StringPool<'a> {
fn parse(chunk: &'a [u8]) -> Option<Self> {
if chunk.len() < 28 {
return None;
}
let count = read_u32(chunk, 8)? as usize;
let flags = read_u32(chunk, 16)?;
let strings_start = read_u32(chunk, 20)? as usize;
// Offsets begin after the 28-byte pool header. A count whose offset
// table does not fit is corrupt. There is no fixed cap on the count:
// the bound is that the table fits and later decoding stays within
// twice the chunk.
let table = 28_usize.checked_add(count.checked_mul(4)?)?;
if table > chunk.len() || strings_start < table || strings_start > chunk.len() {
return None;
}
let mut offsets = Vec::new();
offsets.try_reserve_exact(count).ok()?;
for index in 0..count {
offsets.push(read_u32(chunk, 28 + index * 4)?);
}
Some(Self {
bytes: chunk,
offsets,
strings_start,
utf8: flags & (1 << 8) != 0,
})
}
fn get(&self, index: u32) -> Option<String> {
if index == NO_STRING {
return None;
}
let start = self
.strings_start
.checked_add(*self.offsets.get(index as usize)? as usize)?;
let text = if self.utf8 {
utf8_at(self.bytes, start)?.to_owned()
} else {
utf16_at(self.bytes, start)?
};
// One looked-up string cannot be larger than the pool that holds it.
if text.len() > self.bytes.len().saturating_mul(2) {
return None;
}
Some(text)
}
}
fn utf8_at(bytes: &[u8], mut offset: usize) -> Option<&str> {
let (_, next) = utf8_len(bytes, offset)?;
offset = next;
let (byte_len, next) = utf8_len(bytes, offset)?;
offset = next;
let end = offset.checked_add(byte_len)?;
let text = bytes.get(offset..end)?;
// The character after the bytes is a NUL terminator.
if *bytes.get(end)? != 0 {
return None;
}
std::str::from_utf8(text).ok()
}
fn utf8_len(bytes: &[u8], offset: usize) -> Option<(usize, usize)> {
let first = *bytes.get(offset)?;
if first & 0x80 == 0 {
Some((first as usize, offset + 1))
} else {
let second = *bytes.get(offset + 1)? as usize;
Some((((first as usize & 0x7f) << 8) | second, offset + 2))
}
}
fn utf16_at(bytes: &[u8], offset: usize) -> Option<String> {
let first = read_u16(bytes, offset)?;
let (len, data_at) = if first & 0x8000 == 0 {
(first as usize, offset + 2)
} else {
let second = read_u16(bytes, offset + 2)? as usize;
((((first as usize) & 0x7fff) << 16) | second, offset + 4)
};
let end = data_at.checked_add(len.checked_mul(2)?)?;
if read_u16(bytes, end)? != 0 {
return None;
}
let raw = bytes.get(data_at..end)?;
let units = raw.as_chunks::<2>().0;
// Avoid allocating a UTF-16 copy larger than the chunk that holds it.
if units.len().saturating_mul(2) > bytes.len().saturating_mul(2) {
return None;
}
let owned: Vec<u16> = units.iter().map(|pair| u16::from_le_bytes(*pair)).collect();
String::from_utf16(&owned).ok()
}
fn resource_map_of(chunk: &[u8]) -> Option<Vec<u32>> {
let body = chunk.get(8..)?;
let mut ids = Vec::new();
ids.try_reserve_exact(body.len() / 4).ok()?;
for item in body.as_chunks::<4>().0 {
ids.push(u32::from_le_bytes(*item));
}
Some(ids)
}
fn plain_identity(bytes: &[u8], truncated: bool) -> Option<(String, String)> {
let text = std::str::from_utf8(bytes).ok()?;
let text = text.strip_prefix('\u{feff}').unwrap_or(text);
let rest = skip_preamble(text)?;
if !rest.starts_with("<manifest") {
return None;
}
let mut cursor = &rest["<manifest".len()..];
if cursor.starts_with(|ch: char| ch.is_ascii_alphanumeric() || matches!(ch, '-' | '_' | ':')) {
return None;
}
let mut package = None;
let mut version_code = None;
let mut version_major = None;
let mut closed = false;
loop {
cursor = cursor.trim_start();
if cursor.is_empty() {
break;
}
if cursor.starts_with('>') || cursor.starts_with("/>") {
closed = true;
break;
}
if cursor.starts_with("<!--") {
let close = cursor.find("-->")?;
cursor = &cursor[close + 3..];
continue;
}
let name_end = cursor
.find(|ch: char| ch.is_whitespace() || ch == '=')
.unwrap_or(cursor.len());
let name = &cursor[..name_end];
if name.is_empty() {
return None;
}
cursor = cursor[name_end..].trim_start();
if !cursor.starts_with('=') {
return None;
}
cursor = cursor[1..].trim_start();
let quote = cursor.chars().next()?;
if quote != '"' && quote != '\'' {
return None;
}
cursor = &cursor[quote.len_utf8()..];
let value_end = cursor.find(quote)?;
let value = &cursor[..value_end];
cursor = &cursor[value_end + quote.len_utf8()..];
match name {
"package" => package = Some(value.to_owned()),
"android:versionCode" => version_code = normalize_version(value),
"android:versionCodeMajor" => version_major = normalize_version(value),
_ => {}
}
}
if !closed || truncated {
return None;
}
finish_identity(package?, version_code, version_major)
}
/// Skip a BOM (already removed), XML declarations, and comments. A comment
/// that contains `<manifest` is not the root.
fn skip_preamble(mut text: &str) -> Option<&str> {
loop {
text = text.trim_start();
if text.starts_with("<?") {
let close = text.find("?>")?;
text = &text[close + 2..];
continue;
}
if text.starts_with("<!--") {
let close = text.find("-->")?;
text = &text[close + 3..];
continue;
}
break;
}
Some(text)
}
fn normalize_version(value: &str) -> Option<String> {
if value.starts_with(['-', '+']) {
return None;
}
let number = if let Some(hex) = value
.strip_prefix("0x")
.or_else(|| value.strip_prefix("0X"))
{
u32::from_str_radix(hex, 16).ok()?
} else {
value.parse::<u32>().ok()?
};
Some(number.to_string())
}
fn is_package_token(value: &str) -> bool {
!value.is_empty()
&& value
.chars()
.all(|ch| ch.is_ascii_alphanumeric() || matches!(ch, '.' | '_' | '-'))
}
fn is_version_token(value: &str) -> bool {
!value.is_empty()
&& value.len() <= 80
&& value
.chars()
.all(|ch| ch.is_ascii_digit() || matches!(ch, '.' | '-' | 'x' | 'a'..='f'))
}
fn read_u16(bytes: &[u8], offset: usize) -> Option<u16> {
Some(u16::from_le_bytes(
bytes.get(offset..offset + 2)?.try_into().ok()?,
))
}
fn read_u32(bytes: &[u8], offset: usize) -> Option<u32> {
Some(u32::from_le_bytes(
bytes.get(offset..offset + 4)?.try_into().ok()?,
))
}
#[cfg(test)]
mod tests {
use super::*;
fn plain(body: &str) -> Vec<u8> {
body.as_bytes().to_vec()
}
#[test]
fn plain_requires_version_code_and_skips_comments() {
let text = plain(
"<?xml version=\"1.0\"?>\n<!-- <manifest package=\"fake.comment\" android:versionCode=\"9\"/> -->\n<manifest package='example.game' android:versionName=\"same\" android:versionCode=\"12\" android:versionCodeMajor=\"3\">\n</manifest>\n",
);
assert_eq!(
identity(&text, false),
Some(("example.game".to_owned(), "3.12".to_owned()))
);
let named = plain("<manifest package=\"example.game\" android:versionName=\"1\"/>");
assert_eq!(identity(&named, false), None);
let substring = plain("<notmanifest package=\"example.game\" android:versionCode=\"1\"/>");
assert_eq!(identity(&substring, false), None);
}
#[test]
fn plain_truncated_root_is_not_an_identity() {
let cut = plain("<manifest package=\"example.game\" android:versionCode=\"4\"");
assert_eq!(identity(&cut, false), None);
assert_eq!(identity(&cut, true), None);
}
#[test]
fn plain_hex_version_normalizes() {
let text = plain("<manifest package=\"example.game\" android:versionCode=\"0x10\"/>");
assert_eq!(
identity(&text, false),
Some(("example.game".to_owned(), "16".to_owned()))
);
}
#[test]
fn binary_typed_package_and_nonzero_major() {
// rawValue is 0xFFFFFFFF. The package exists only as a TYPE_STRING
// data index, and versionCodeMajor is a real non-zero integer.
let bytes = axml_with_data(
"manifest",
&[
("package", NO_STRING, "example.typed", TYPE_STRING, 0),
("versionCode", NO_STRING, "", TYPE_INT_DEC, 7),
("versionCodeMajor", NO_STRING, "", TYPE_INT_DEC, 2),
],
Encoding::Utf8,
);
assert_eq!(
identity(&bytes, false),
Some(("example.typed".to_owned(), "2.7".to_owned()))
);
}
#[test]
fn binary_utf16_and_hex_version() {
let bytes = axml_with_data(
"manifest",
&[
("package", NO_STRING, "example.utf16", 0, 0),
("versionCode", NO_STRING, "", TYPE_INT_HEX, 0x20),
],
Encoding::Utf16,
);
assert_eq!(
identity(&bytes, false),
Some(("example.utf16".to_owned(), "32".to_owned()))
);
let _ = bytes;
}
#[test]
fn binary_rejects_other_root_and_android_name_resource() {
let other = axml(
"application",
&[
attr(NO_STRING, "package", "example.nope", 0, ""),
attr(NO_STRING, "versionCode", "1", 0, ""),
],
Encoding::Utf8,
);
assert_eq!(identity(&other, false), None);
// Resource id 0x01010003 is android:name. It must not become the package
// even when its string is a package-looking value.
let named = axml_resources(
"manifest",
&[("name", 0x0101_0003, "example.fake", TYPE_STRING)],
&[],
Encoding::Utf8,
);
assert_eq!(identity(&named, false), None);
}
#[test]
fn binary_truncated_and_corrupt_ranges() {
let bytes = axml(
"manifest",
&[
attr(NO_STRING, "package", "example.cut", 0, ""),
attr(NO_STRING, "versionCode", "1", 0, ""),
],
Encoding::Utf8,
);
let mut short = bytes.clone();
short.truncate(20);
assert_eq!(identity(&short, true), None);
let mut bad = bytes.clone();
bad[4..8].copy_from_slice(&4_u32.to_le_bytes());
assert_eq!(identity(&bad, false), None);
}
#[test]
fn zero_major_is_omitted() {
let bytes = axml_with_data(
"manifest",
&[
("package", NO_STRING, "example.zero", TYPE_STRING, 0),
("versionCode", NO_STRING, "", TYPE_INT_DEC, 9),
("versionCodeMajor", NO_STRING, "", TYPE_INT_DEC, 0),
],
Encoding::Utf8,
);
assert_eq!(
identity(&bytes, false),
Some(("example.zero".to_owned(), "9".to_owned()))
);
}
struct BuiltAttr {
name: &'static str,
raw: u32,
raw_text: &'static str,
data_type: u8,
data_text: &'static str,
}
fn attr(
_namespace: u32,
name: &'static str,
raw_text: &'static str,
data_type: u8,
data_text: &'static str,
) -> BuiltAttr {
BuiltAttr {
name,
raw: if raw_text.is_empty() { NO_STRING } else { 0 },
raw_text,
data_type,
data_text,
}
}
#[derive(Clone, Copy)]
enum Encoding {
Utf8,
Utf16,
}
fn axml(root: &str, attributes: &[BuiltAttr], encoding: Encoding) -> Vec<u8> {
let specs: Vec<(&str, u32, &str, u8, u32)> = attributes
.iter()
.map(|item| {
let data = if item.data_type == TYPE_INT_DEC || item.data_type == TYPE_INT_HEX {
item.data_text.parse().unwrap_or(0)
} else {
0
};
(item.name, item.raw, item.raw_text, item.data_type, data)
})
.collect();
axml_with_data(root, &specs, encoding)
}
fn axml_with_data(
root: &str,
attributes: &[(&str, u32, &str, u8, u32)],
encoding: Encoding,
) -> Vec<u8> {
axml_resources(root, &[], attributes, encoding)
}
fn axml_resources(
root: &str,
named_resources: &[(&str, u32, &str, u8)],
attributes: &[(&str, u32, &str, u8, u32)],
encoding: Encoding,
) -> Vec<u8> {
let mut strings = vec![root.to_owned()];
let mut specs: Vec<(u32, u32, u32, u8, u32)> = Vec::new();
for (name, raw, raw_text, data_type, data) in attributes {
let name_index = push_string(&mut strings, name);
let raw_index = if raw_text.is_empty() {
*raw
} else {
push_string(&mut strings, raw_text)
};
let data_index = if *data_type == TYPE_STRING {
if raw_text.is_empty() {
*data
} else {
push_string(&mut strings, raw_text)
}
} else {
*data
};
specs.push((NO_STRING, name_index, raw_index, *data_type, data_index));
}
for (name, _resource, text, data_type) in named_resources {
let name_index = push_string(&mut strings, name);
let value_index = push_string(&mut strings, text);
specs.push((NO_STRING, name_index, NO_STRING, *data_type, value_index));
}
let resources: Vec<u32> = {
let mut ids = vec![0_u32; strings.len()];
for (name, resource, _, _) in named_resources {
if let Some(index) = strings.iter().position(|item| item == name) {
ids[index] = *resource;
}
}
ids
};
let pool = encode_pool(&strings, encoding);
let mut map = Vec::new();
map.extend(u16::to_le_bytes(0x0180));
map.extend(u16::to_le_bytes(8));
map.extend(u32::to_le_bytes((8 + resources.len() * 4) as u32));
for id in &resources {
map.extend(u32::to_le_bytes(*id));
}
let mut element = Vec::new();
element.extend(u16::to_le_bytes(0x0102));
element.extend(u16::to_le_bytes(16));
let attribute_bytes = specs.len() * 20;
let size = 16 + 20 + attribute_bytes;
element.extend(u32::to_le_bytes(size as u32));
element.extend(u32::to_le_bytes(1)); // line
element.extend(u32::to_le_bytes(NO_STRING)); // comment
element.extend(u32::to_le_bytes(NO_STRING)); // namespace
element.extend(u32::to_le_bytes(0)); // name = root
element.extend(u16::to_le_bytes(20)); // attributeStart from attrExt
element.extend(u16::to_le_bytes(20));
element.extend(u16::to_le_bytes(specs.len() as u16));
element.extend(u16::to_le_bytes(0));
element.extend(u16::to_le_bytes(0));
element.extend(u16::to_le_bytes(0));
for (namespace, name, raw, data_type, data) in specs {
element.extend(u32::to_le_bytes(namespace));
element.extend(u32::to_le_bytes(name));
element.extend(u32::to_le_bytes(raw));
element.extend(u16::to_le_bytes(8));
element.push(0);
element.push(data_type);
element.extend(u32::to_le_bytes(data));
}
let mut bytes = Vec::new();
bytes.extend(u16::to_le_bytes(0x0003));
bytes.extend(u16::to_le_bytes(8));
bytes.extend(u32::to_le_bytes(
(8 + pool.len() + map.len() + element.len()) as u32,
));
bytes.extend(pool);
bytes.extend(map);
bytes.extend(element);
bytes
}
fn push_string(strings: &mut Vec<String>, value: &str) -> u32 {
if let Some(index) = strings.iter().position(|item| item == value) {
return index as u32;
}
strings.push(value.to_owned());
(strings.len() - 1) as u32
}
fn encode_pool(strings: &[String], encoding: Encoding) -> Vec<u8> {
let mut data = Vec::new();
let mut offsets = Vec::new();
for value in strings {
offsets.push(data.len() as u32);
match encoding {
Encoding::Utf8 => {
data.push(value.chars().count() as u8);
data.push(value.len() as u8);
data.extend(value.as_bytes());
data.push(0);
}
Encoding::Utf16 => {
let units: Vec<u16> = value.encode_utf16().collect();
data.extend(u16::to_le_bytes(units.len() as u16));
for unit in units {
data.extend(u16::to_le_bytes(unit));
}
data.extend(u16::to_le_bytes(0));
}
}
}
let mut chunk = Vec::new();
chunk.extend(u16::to_le_bytes(0x0001));
chunk.extend(u16::to_le_bytes(28));
let size = 28 + strings.len() * 4 + data.len();
chunk.extend(u32::to_le_bytes(size as u32));
chunk.extend(u32::to_le_bytes(strings.len() as u32));
chunk.extend(u32::to_le_bytes(0));
let flags: u32 = match encoding {
Encoding::Utf8 => 1 << 8,
Encoding::Utf16 => 0,
};
chunk.extend(u32::to_le_bytes(flags));
chunk.extend(u32::to_le_bytes((28 + strings.len() * 4) as u32));
chunk.extend(u32::to_le_bytes(0));
for offset in offsets {
chunk.extend(u32::to_le_bytes(offset));
}
chunk.extend(data);
chunk
}
}
+630 -105
View File
@@ -18,12 +18,17 @@
#[cfg(test)] #[cfg(test)]
mod tests; mod tests;
mod context;
mod manifest;
use context::LibraryRestorer;
use std::collections::HashSet; use std::collections::HashSet;
use std::fs::File; use std::fs::File;
use std::io::{BufWriter, Read, Seek, Write}; use std::io::{BufWriter, Read, Seek, Write};
use std::path::{Path, PathBuf}; use std::path::{Path, PathBuf};
use anyhow::{Context, Result, bail}; use anyhow::{Context, Result, bail, ensure};
use memmap2::{Mmap, MmapOptions}; use memmap2::{Mmap, MmapOptions};
use senbei_crypto::hex_digest; use senbei_crypto::hex_digest;
use senbei_engine::android::{ExtractOptions, extract_stage2, is_protected_libil2cpp}; use senbei_engine::android::{ExtractOptions, extract_stage2, is_protected_libil2cpp};
@@ -103,16 +108,16 @@ pub fn file_content_identity(path: &Path) -> std::io::Result<String> {
/// implementation detail of the two-phase restore, not user-facing output). /// implementation detail of the two-phase restore, not user-facing output).
/// Returns the unwrapped embedded metadata blob when the restored image /// Returns the unwrapped embedded metadata blob when the restored image
/// carries one (see the module docs); the caller decides where to write it. /// carries one (see the module docs); the caller decides where to write it.
struct SoRestoreContext { ///
embedded_metadata: Option<Vec<u8>>, /// Module `0x0C` recovered here belongs to this library's application. Callers
method_index_module: Option<Vec<u8>>, /// that restore more than one application must keep those modules apart; this
} /// single-file entry point has only one application, so it keeps the module
/// only long enough to unwrap embedded metadata (which does not consume it).
fn restore_so_file_with_context( pub(crate) fn restore_so_with_engine(
input: &Path, input: &Path,
dest: &Path, dest: &Path,
verbose: bool, verbose: bool,
) -> Result<SoRestoreContext> { ) -> Result<context::RestoredLibrary> {
let temporary = tempfile::tempdir().context("create stage-2 workspace")?; let temporary = tempfile::tempdir().context("create stage-2 workspace")?;
let stage2_dir = temporary.path().join("stage2"); let stage2_dir = temporary.path().join("stage2");
let extraction = extract_stage2(&ExtractOptions::with_defaults( let extraction = extract_stage2(&ExtractOptions::with_defaults(
@@ -120,15 +125,33 @@ fn restore_so_file_with_context(
stage2_dir.clone(), stage2_dir.clone(),
)) ))
.context("extract stage-1/stage-2 payload")?; .context("extract stage-1/stage-2 payload")?;
let method_index_module = extraction // Every module 0x0C is collected. Two distinct images in one library are a
// conflict for that library's application, not a last-writer-wins update.
let mut modules = Vec::new();
for module in extraction
.module_registry .module_registry
.iter() .iter()
.find(|module| module.command_id == 0x0c) .filter(|module| module.command_id == 0x0c)
.map(|module| { {
std::fs::read(stage2_dir.join(&module.image_path)) let bytes = std::fs::read(stage2_dir.join(&module.image_path))
.with_context(|| format!("read decoded module 0x0C `{}`", module.image_path)) .with_context(|| format!("read decoded module 0x0C `{}`", module.image_path))?;
}) if modules
.transpose()?; .iter()
.all(|(_, existing): &(_, Vec<u8>)| existing != &bytes)
{
modules.push((module.image_path.clone(), bytes));
}
}
ensure!(
modules.len() <= 1,
"library `{}` contains {} distinct module 0x0C images",
input.display(),
modules.len()
);
let (module_image, method_index_module) = modules
.pop()
.map(|(image, bytes)| (Some(image), Some(bytes)))
.unwrap_or((None, None));
restore_libil2cpp(&RestoreOptions { restore_libil2cpp(&RestoreOptions {
input: input.to_path_buf(), input: input.to_path_buf(),
output: dest.to_path_buf(), output: dest.to_path_buf(),
@@ -139,30 +162,289 @@ fn restore_so_file_with_context(
verbose, verbose,
}) })
.context("restore protected library")?; .context("restore protected library")?;
let restored = let restored_file =
std::fs::read(dest).with_context(|| format!("read restored `{}`", dest.display()))?; File::open(dest).with_context(|| format!("read restored `{}`", dest.display()))?;
Ok(SoRestoreContext { let restored = map_read_only(&restored_file, dest)?;
Ok(context::RestoredLibrary {
embedded_metadata: senbei_metadata::android::extract_embedded_metadata(&restored), embedded_metadata: senbei_metadata::android::extract_embedded_metadata(&restored),
method_index_module, method_index_module,
module_image,
}) })
} }
pub fn restore_so_file(input: &Path, dest: &Path, verbose: bool) -> Result<Option<Vec<u8>>> { pub fn restore_so_file(input: &Path, dest: &Path, verbose: bool) -> Result<Option<Vec<u8>>> {
let mut method_index_module = None; Ok(context::EngineRestorer
restore_so_file_with_method_index_module(input, dest, verbose, &mut method_index_module) .restore(input, dest, verbose)?
.embedded_metadata)
} }
pub(crate) fn restore_so_file_with_method_index_module( /// Fresh run-scoped cache. Library hits reuse a restored image; metadata hits
/// are keyed by encrypted bytes and the module that recovered them.
pub(crate) fn run_cache() -> context::AndroidRunCache {
context::AndroidRunCache::default()
}
/// Assign loose libraries and metadata to application contexts.
pub(crate) fn loose_scopes(
libraries: &[PathBuf],
metadata: &[PathBuf],
) -> Vec<context::ScopedInput> {
context::group_loose_inputs(libraries, metadata)
}
/// One loose library restore, including the embedded metadata blob when the
/// rebuilt image carried one. `Duplicate` means an earlier successful restore
/// of the same encrypted bytes already populated this application.
#[derive(Debug)]
pub(crate) struct LooseLibraryOutcome {
pub status: EntryStatus,
/// Embedded metadata already written as part of this successful restore.
pub embedded: Option<PathBuf>,
}
pub(crate) fn restore_loose_library(
input: &Path, input: &Path,
dest: &Path, dest: &Path,
label: &str,
app: &context::AppKey,
verbose: bool, verbose: bool,
method_index_module: &mut Option<Vec<u8>>, cache: &mut context::AndroidRunCache,
) -> Result<Option<Vec<u8>>> { ) -> Result<LooseLibraryOutcome> {
let restored = restore_so_file_with_context(input, dest, verbose)?; restore_loose_library_with(
if let Some(module) = restored.method_index_module { input,
*method_index_module = Some(module); dest,
label,
app,
verbose,
cache,
&mut context::EngineRestorer,
)
}
fn restore_loose_library_with<R: context::LibraryRestorer>(
input: &Path,
dest: &Path,
label: &str,
app: &context::AppKey,
verbose: bool,
cache: &mut context::AndroidRunCache,
restorer: &mut R,
) -> Result<LooseLibraryOutcome> {
let identity = file_content_identity(input).with_context(|| format!("hash `{label}`"))?;
if let Some(cached) = cache.library(&identity).cloned() {
// Same encrypted library. Record its module on this application and
// write nothing: a duplicate has no output of its own.
record_module(cache, app, &cached, label);
return Ok(LooseLibraryOutcome {
status: EntryStatus::Duplicate,
embedded: None,
});
} }
Ok(restored.embedded_metadata) // A prior content-only success still needs context extraction. Restore it
// in a private workspace, never over a destination that must stay untouched.
let prior = cache.is_prior_success(&identity);
let workspace = prior.then(tempfile::tempdir).transpose().context("create duplicate context workspace")?;
let scratch = workspace.as_ref().map(|directory| directory.path().join("restored.so"));
let restore_dest = scratch.as_deref().unwrap_or(dest);
match restorer.restore(input, restore_dest, verbose) {
Ok(restored) => {
// The library file is already at `dest`. Embedded metadata is part
// of the same success: cache the module only after that write.
let mut embedded = None;
if !prior && let Some(blob) = &restored.embedded_metadata {
let meta_dest = embedded_metadata_dest(dest);
if let Err(error) = write_metadata_blob(&meta_dest, blob) {
return Ok(LooseLibraryOutcome {
status: EntryStatus::Failed(error),
embedded: None,
});
}
embedded = Some(meta_dest);
}
let cached = context::CachedLibrary {
method_index_module: restored.method_index_module,
module_image: restored.module_image,
};
record_module(cache, app, &cached, label);
cache.insert_library(identity, cached);
Ok(LooseLibraryOutcome {
status: if prior { EntryStatus::Duplicate } else { EntryStatus::Restored },
embedded,
})
}
Err(error) => Ok(LooseLibraryOutcome {
status: EntryStatus::Failed(error),
embedded: None,
}),
}
}
/// Metadata restored under one application context, plus the libraries whose
/// module was paired with it.
#[derive(Debug)]
pub(crate) struct LooseMetadataOutcome {
pub version: u32,
pub remapped: usize,
pub provenance: Vec<String>,
}
pub(crate) fn restore_loose_metadata(
input: &Path,
dest: &Path,
label: &str,
app: &context::AppKey,
verbose: bool,
cache: &mut context::AndroidRunCache,
) -> Result<LooseMetadataOutcome> {
let data = std::fs::read(input).with_context(|| format!("read `{label}`"))?;
let version = metadata_version(&data);
// Only raw v24 consumes module 0x0C. v29/v31/v39 and the structural path
// must not fail, and must not record a module they did not use, when the
// application's libraries disagree or have no module.
let (module, provenance) = if version == Some(24) {
let modules = cache.modules(app).cloned();
match context::binding_module(modules.as_ref()) {
Ok(module) => (
module.map(|bytes| bytes.to_vec()),
provenance_labels(&context::provenance_for(modules.as_ref(), None)),
),
Err(ambiguity) => {
let provenance = context::provenance_for(modules.as_ref(), Some(&ambiguity));
bail!(provenance_error(&provenance));
}
}
} else {
(None, Vec::new())
};
let module = module.as_deref();
// Loose metadata retains one output per path, as before. Do not keep all
// restored blobs in the folder cache just to avoid a repeated permutation.
let (out, report) =
restore_metadata_bytes_with_module(&data, module).with_context(|| label.to_owned())?;
if report.remapped > 0 {
write_metadata_blob(dest, &out)?;
if verbose { println!("Write to {}", dest.display()); }
}
Ok(LooseMetadataOutcome {
version: report.version,
remapped: report.remapped,
provenance,
})
}
/// Group loose `.apk` files that are a split set: same parent directory and
/// the same manifest package name plus version. Bundles are never grouped
/// with anything. Order inside a group does not matter; the caller restores
/// every library in the group before any of its metadata.
pub(crate) fn loose_apk_groups(packages: &[PathBuf]) -> Vec<Vec<PathBuf>> {
let mut groups: Vec<(context::AppKey, Vec<PathBuf>)> = Vec::new();
let mut alone = Vec::new();
for package in packages {
let extension = package.extension().and_then(|value| value.to_str());
let is_apk = extension.is_some_and(|value| value.eq_ignore_ascii_case("apk"));
if !is_apk {
alone.push(vec![package.clone()]);
continue;
}
let key = apk_manifest_key(package);
if let Some(key) = key {
if let Some((_, members)) = groups.iter_mut().find(|(existing, _)| existing == &key) {
members.push(package.clone());
} else {
groups.push((key, vec![package.clone()]));
}
} else {
alone.push(vec![package.clone()]);
}
}
let mut grouped: Vec<_> = groups.into_iter().map(|(_, members)| members).collect();
grouped.append(&mut alone);
grouped
}
fn apk_manifest_key(package: &Path) -> Option<context::AppKey> {
let file = std::fs::File::open(package).ok()?;
let mut archive = zip::ZipArchive::new(file).ok()?;
let parent = package
.parent()
.filter(|parent| !parent.as_os_str().is_empty())
.unwrap_or_else(|| Path::new("."));
let inventory = context::inventory_manifest_package(
&mut archive,
package,
parent,
&package.display().to_string(),
)
.ok()?;
match inventory.key {
context::AppKey::Manifest { .. } => Some(inventory.key),
_ => None,
}
}
/// Restore one package against the folder run's cache. Successful library
/// identities are published back through the cache itself; the public
/// [`restore_package`] wrapper remains for callers that only have a `HashSet`.
pub(crate) fn restore_package_cached(
package: &Path,
rel: &Path,
out_root: &Path,
cache: &mut context::AndroidRunCache,
verbose: bool,
) -> Result<Vec<EntryOutcome>> {
restore_package_with_restorer(
package,
rel,
out_root,
cache,
verbose,
&mut context::EngineRestorer,
)
}
/// One folder split-set: every library in the group, then every metadata blob.
/// A library-only pass records modules without writing metadata, so a metadata
/// blob in an earlier sibling cannot run before a library in a later one.
pub(crate) fn restore_apk_group(
packages: &[PathBuf],
root: &Path,
out_root: &Path,
cache: &mut context::AndroidRunCache,
verbose: bool,
) -> Result<Vec<EntryOutcome>> {
let mut outcomes = Vec::new();
for package in packages {
let rel = package.strip_prefix(root).unwrap_or(package);
outcomes.extend(restore_package_pass(
package,
rel,
out_root,
cache,
verbose,
&mut context::EngineRestorer,
PackagePass::Libraries,
)?);
}
for package in packages {
let rel = package.strip_prefix(root).unwrap_or(package);
outcomes.extend(restore_package_pass(
package,
rel,
out_root,
cache,
verbose,
&mut context::EngineRestorer,
PackagePass::Metadata,
)?);
}
Ok(outcomes)
}
#[derive(Clone, Copy, PartialEq, Eq)]
enum PackagePass {
All,
Libraries,
Metadata,
} }
/// Content identity for cross-source deduplication: the same library may /// Content identity for cross-source deduplication: the same library may
@@ -198,9 +480,13 @@ pub(crate) fn restore_metadata_bytes_with_module(
.get(4..8) .get(4..8)
.and_then(|bytes| <[u8; 4]>::try_from(bytes).ok()) .and_then(|bytes| <[u8; 4]>::try_from(bytes).ok())
.map(u32::from_le_bytes); .map(u32::from_le_bytes);
if version == Some(24) if version == Some(24) {
&& let Some(module) = method_index_module // v24.1 method indices come only from this application's module 0x0C.
{ // Falling through to the structural remap would report "unsupported
// version" and could be mistaken for a clean skip. No module is a
// failed binding, not an unsupported layout.
let module = method_index_module
.ok_or_else(|| anyhow::anyhow!("no module 0x0C for v24.1 metadata"))?;
let (out, report) = senbei_metadata::android::restore_method_indices_v24_1(data, module) let (out, report) = senbei_metadata::android::restore_method_indices_v24_1(data, module)
.map_err(anyhow::Error::new)?; .map_err(anyhow::Error::new)?;
return Ok(( return Ok((
@@ -252,6 +538,9 @@ pub struct EntryOutcome {
pub dest: PathBuf, pub dest: PathBuf,
pub kind: EntryKind, pub kind: EntryKind,
pub status: EntryStatus, pub status: EntryStatus,
/// Libraries whose module `0x0C` was paired with this metadata entry.
/// Empty for libraries and for metadata that had no paired module.
pub provenance: Vec<String>,
} }
#[derive(Debug, Clone, Copy, PartialEq, Eq)] #[derive(Debug, Clone, Copy, PartialEq, Eq)]
@@ -278,20 +567,12 @@ pub enum EntryStatus {
Failed(anyhow::Error), Failed(anyhow::Error),
} }
struct PackageRestoreContext<'a> { struct PackageRestoreState<'a, R> {
method_index_module: &'a mut Option<Vec<u8>>, restorer: &'a mut R,
cache: &'a mut context::AndroidRunCache,
verbose: bool, verbose: bool,
} }
impl<'a> PackageRestoreContext<'a> {
fn new(verbose: bool, method_index_module: &'a mut Option<Vec<u8>>) -> Self {
Self {
method_index_module,
verbose,
}
}
}
fn package_entry_priority(path: &Path) -> u8 { fn package_entry_priority(path: &Path) -> u8 {
if is_so_name(path) { 0 } else { 1 } if is_so_name(path) { 0 } else { 1 }
} }
@@ -310,24 +591,60 @@ pub fn restore_package(
seen: &mut HashSet<String>, seen: &mut HashSet<String>,
verbose: bool, verbose: bool,
) -> Result<Vec<EntryOutcome>> { ) -> Result<Vec<EntryOutcome>> {
let mut method_index_module = None; // `seen` is the public successful-content set shared with loose libraries.
restore_package_with_method_index_module( // Metadata recovery context does not fit in that set, so this wrapper keeps
// a run-scoped cache and only publishes content identities of entries that
// actually succeeded. A caller that passes identities restored by an earlier
// package still skips those bytes; module context for metadata is rebuilt
// inside this package (one package, or one manifest-scoped bundle, is one
// application boundary).
let mut cache = context::AndroidRunCache::default();
for identity in seen.iter().cloned() {
cache.insert_prior_success(identity);
}
let outcomes = restore_package_with_restorer(
package, package,
rel, rel,
out_root, out_root,
seen, &mut cache,
verbose, verbose,
&mut method_index_module, &mut context::EngineRestorer,
) );
// Publish successes even when the package returns `Err`. The caller's set
// is only replaced after that, so a panic still leaves the old successes.
let succeeded = cache.successful_content_identities();
seen.clear();
seen.extend(succeeded);
outcomes
} }
pub(crate) fn restore_package_with_method_index_module( pub(crate) fn restore_package_with_restorer<R: context::LibraryRestorer>(
package: &Path, package: &Path,
rel: &Path, rel: &Path,
out_root: &Path, out_root: &Path,
seen: &mut HashSet<String>, cache: &mut context::AndroidRunCache,
verbose: bool, verbose: bool,
method_index_module: &mut Option<Vec<u8>>, restorer: &mut R,
) -> Result<Vec<EntryOutcome>> {
restore_package_pass(
package,
rel,
out_root,
cache,
verbose,
restorer,
PackagePass::All,
)
}
fn restore_package_pass<R: context::LibraryRestorer>(
package: &Path,
rel: &Path,
out_root: &Path,
cache: &mut context::AndroidRunCache,
verbose: bool,
restorer: &mut R,
pass: PackagePass,
) -> Result<Vec<EntryOutcome>> { ) -> Result<Vec<EntryOutcome>> {
let bundle = package let bundle = package
.extension() .extension()
@@ -338,7 +655,11 @@ pub(crate) fn restore_package_with_method_index_module(
let mut archive = open_package(package)?; let mut archive = open_package(package)?;
let temporary = tempfile::tempdir().context("create package workspace")?; let temporary = tempfile::tempdir().context("create package workspace")?;
let mut outcomes = Vec::new(); let mut outcomes = Vec::new();
let mut context = PackageRestoreContext::new(verbose, method_index_module); let mut state = PackageRestoreState {
restorer,
cache,
verbose,
};
let mut direct = Vec::new(); let mut direct = Vec::new();
let mut nested = Vec::new(); let mut nested = Vec::new();
@@ -361,13 +682,35 @@ pub(crate) fn restore_package_with_method_index_module(
{ {
nested.push((index, name)); nested.push((index, name));
} }
} else { } else if is_android_entry_name(&name) {
if is_android_entry_name(&name) {
direct.push((index, name)); direct.push((index, name));
} }
} }
} // One direct package is one application inside its own package set (the
// package file). A plain manifest names it by package plus version; without
// both, the package is isolated. The ZIP file name is not a package id.
// Inventory already walked this central directory; entry extraction below
// reuses the same open and must not be charged as a second package.
let direct_inventory = if direct.is_empty() {
None
} else {
Some(context::inventory_manifest_package(
&mut archive,
package,
package
.parent()
.filter(|parent| !parent.as_os_str().is_empty())
.unwrap_or_else(|| Path::new(".")),
&package.display().to_string(),
)?)
};
direct.retain(|(_, name)| match pass {
PackagePass::Libraries => package_entry_priority(name) == 0,
PackagePass::Metadata => package_entry_priority(name) == 1,
PackagePass::All => true,
});
direct.sort_by_key(|(_, name)| package_entry_priority(name)); direct.sort_by_key(|(_, name)| package_entry_priority(name));
if let Some(inventory) = direct_inventory {
for (index, name) in direct { for (index, name) in direct {
let label = format!("{}::{}", rel.display(), name.display()); let label = format!("{}::{}", rel.display(), name.display());
let dest = out_root.join(rel).join(crate::job::out_name(&name)); let dest = out_root.join(rel).join(crate::job::out_name(&name));
@@ -377,16 +720,22 @@ pub(crate) fn restore_package_with_method_index_module(
&label, &label,
&dest, &dest,
&temporary, &temporary,
seen, &inventory.key,
&mut context, &mut state,
) )
.with_context(|| format!("extract `{label}`"))?; .with_context(|| format!("extract `{label}`"))?;
outcomes.append(&mut entry_outcomes); outcomes.append(&mut entry_outcomes);
} }
}
struct NestedPackage { struct NestedPackage {
label: PathBuf, label: PathBuf,
path: PathBuf, path: PathBuf,
base: PathBuf, base: PathBuf,
app: context::AppKey,
/// Manifest inventory, including the reopen key. Extraction opens
/// `path` again; `inventory.archive` is the pass a later budget must
/// not charge a second time.
inventory: context::PackageInventory,
entries: Vec<(usize, PathBuf)>, entries: Vec<(usize, PathBuf)>,
} }
@@ -396,6 +745,18 @@ pub(crate) fn restore_package_with_method_index_module(
let nested_path = extract_entry(&mut archive, index, &temporary, &nested_label) let nested_path = extract_entry(&mut archive, index, &temporary, &nested_label)
.with_context(|| format!("extract `{}`", nested_label.display()))?; .with_context(|| format!("extract `{}`", nested_label.display()))?;
let mut nested_archive = open_package(&nested_path)?; let mut nested_archive = open_package(&nested_path)?;
// Manifest first, before any library or metadata entry is chosen. Splits
// that name the same package and version, inside this bundle only, share
// one context. A split with no plain package name or no version is
// isolated and cannot borrow a sibling's module. `nested_path` is the
// reopen key: the central directory listed here is not charged again
// when entries are extracted below.
let inventory = context::inventory_manifest_package(
&mut nested_archive,
&nested_path,
package,
&nested_path.display().to_string(),
)?;
let mut entries = Vec::new(); let mut entries = Vec::new();
for nested_index in 0..nested_archive.len() { for nested_index in 0..nested_archive.len() {
let (entry_name, is_dir) = { let (entry_name, is_dir) = {
@@ -415,20 +776,40 @@ pub(crate) fn restore_package_with_method_index_module(
label: nested_label, label: nested_label,
path: nested_path, path: nested_path,
base: rel.join(name.with_extension("")), base: rel.join(name.with_extension("")),
app: inventory.key.clone(),
inventory,
entries, entries,
}); });
} }
// A bundle can put libil2cpp.so in an ABI split while metadata stays in // A bundle can put libil2cpp.so in an ABI split while metadata stays in
// base.apk. Share one context across the whole bundle and process all SOs // base.apk. Splits of the same manifest package share one context. All of
// before any metadata, regardless of which nested APK owns each entry. // that application's libraries are restored before its metadata, independent
for priority in [0_u8, 1_u8] { // of nested-APK order. Different package names never share a module.
let mut app_order: Vec<context::AppKey> = Vec::new();
for nested_package in &nested_packages { for nested_package in &nested_packages {
if !app_order.contains(&nested_package.app) {
app_order.push(nested_package.app.clone());
}
}
let priorities: &[u8] = match pass {
PackagePass::Libraries => &[0],
PackagePass::Metadata => &[1],
PackagePass::All => &[0, 1],
};
for app in &app_order {
for priority in priorities {
for nested_package in nested_packages.iter().filter(|nested| &nested.app == app) {
// Same file the inventory listed. Reopening is not a new package.
debug_assert_eq!(
nested_package.inventory.archive.reopen_key,
nested_package.path
);
let mut nested_archive = open_package(&nested_package.path)?; let mut nested_archive = open_package(&nested_package.path)?;
for (nested_index, entry_name) in nested_package for (nested_index, entry_name) in nested_package
.entries .entries
.iter() .iter()
.filter(|(_, entry_name)| package_entry_priority(entry_name) == priority) .filter(|(_, entry_name)| package_entry_priority(entry_name) == *priority)
{ {
let label = format!( let label = format!(
"{}::{}", "{}::{}",
@@ -444,119 +825,263 @@ pub(crate) fn restore_package_with_method_index_module(
&label, &label,
&dest, &dest,
&temporary, &temporary,
seen, app,
&mut context, &mut state,
) )
.with_context(|| format!("extract `{label}`"))?; .with_context(|| format!("extract `{label}`"))?;
outcomes.append(&mut entry_outcomes); outcomes.append(&mut entry_outcomes);
} }
} }
} }
}
Ok(outcomes) Ok(outcomes)
} }
/// Probe one extracted package entry and restore it when it is a target. /// Probe one extracted package entry and restore it when it is a target.
/// Returns one outcome per produced/consumed artifact: the entry itself, plus /// Returns one outcome per produced/consumed artifact: the entry itself, plus
/// an `EmbeddedMetadata` outcome when the restored library carried a blob. /// an `EmbeddedMetadata` outcome when the restored library carried a blob.
fn restore_package_entry<R: Read + Seek>( ///
/// `app` is the application this entry was inventoried into. Libraries record
/// their module there; metadata reads it back and refuses a conflicting or
/// missing binding before writing.
fn restore_package_entry<R: Read + Seek, T: context::LibraryRestorer>(
archive: &mut ZipArchive<R>, archive: &mut ZipArchive<R>,
index: usize, index: usize,
label: &str, label: &str,
dest: &Path, dest: &Path,
temporary: &tempfile::TempDir, temporary: &tempfile::TempDir,
seen: &mut HashSet<String>, app: &context::AppKey,
context: &mut PackageRestoreContext<'_>, state: &mut PackageRestoreState<'_, T>,
) -> Result<Vec<EntryOutcome>> { ) -> Result<Vec<EntryOutcome>> {
let entry_path = extract_entry(archive, index, temporary, Path::new(label))?; let entry_path = extract_entry(archive, index, temporary, Path::new(label))?;
let entry_file = let entry_file =
File::open(&entry_path).with_context(|| format!("open extracted `{label}`"))?; File::open(&entry_path).with_context(|| format!("open extracted `{label}`"))?;
let entry_data = map_read_only(&entry_file, &entry_path)?; let entry_data = map_read_only(&entry_file, &entry_path)?;
let is_so = is_elf64_aarch64(&entry_data) && is_protected_libil2cpp(&entry_data); let protected_override = state.restorer.protected_override(&entry_path);
let is_so = protected_override.is_some()
|| (is_elf64_aarch64(&entry_data) && is_protected_libil2cpp(&entry_data));
let is_meta = !is_so && senbei_metadata::is_metadata(&entry_data); let is_meta = !is_so && senbei_metadata::is_metadata(&entry_data);
let outcome = |kind, status| EntryOutcome { let outcome = |kind, status, provenance: Vec<String>| EntryOutcome {
label: label.to_owned(), label: label.to_owned(),
dest: dest.to_path_buf(), dest: dest.to_path_buf(),
kind, kind,
status, status,
provenance,
}; };
if !is_so && !is_meta { if !is_so && !is_meta {
return Ok(vec![outcome(EntryKind::So, EntryStatus::NotTarget)]); return Ok(vec![outcome(
EntryKind::So,
EntryStatus::NotTarget,
Vec::new(),
)]);
} }
let identity = content_identity(&entry_data); let identity = content_identity(&entry_data);
if seen.contains(&identity) { if is_so && state.cache.library(&identity).is_some() {
let kind = if is_so { if let Some(cached) = state.cache.library(&identity).cloned() {
EntryKind::So // Duplicate libraries still belong to the application that contains
} else { // them. The cached module is recorded here and no output is written.
EntryKind::Metadata { remapped: 0 } record_module(state.cache, app, &cached, label);
}; }
return Ok(vec![outcome(kind, EntryStatus::Duplicate)]); return Ok(vec![outcome(
EntryKind::So,
EntryStatus::Duplicate,
Vec::new(),
)]);
} }
if is_so { if is_so {
drop(entry_data); drop(entry_data);
drop(entry_file); drop(entry_file);
let prior = state.cache.is_prior_success(&identity);
let workspace = prior.then(|| tempfile::tempdir_in(temporary.path())).transpose()
.context("create duplicate context workspace")?;
let scratch = workspace.as_ref().map(|directory| directory.path().join("restored.so"));
let restore_dest = scratch.as_deref().unwrap_or(dest);
return Ok( return Ok(
match restore_so_file_with_context(&entry_path, dest, context.verbose) { match state.restorer.restore(&entry_path, restore_dest, state.verbose) {
Ok(restored) => { Ok(restored) => {
if let Some(module) = restored.method_index_module { // A prior content-only success is restored only in the
*context.method_index_module = Some(module); // private workspace to recover its module context.
let mut outcomes = Vec::new();
let mut embedded_failed = None;
if !prior {
outcomes.push(outcome(EntryKind::So, EntryStatus::Restored, Vec::new()));
} }
let mut outcomes = vec![outcome(EntryKind::So, EntryStatus::Restored)]; if !prior && let Some(blob) = &restored.embedded_metadata {
if let Some(blob) = restored.embedded_metadata {
let meta_dest = embedded_metadata_dest(dest); let meta_dest = embedded_metadata_dest(dest);
let status = match write_metadata_blob(&meta_dest, &blob) { match write_metadata_blob(&meta_dest, blob) {
Ok(()) => EntryStatus::Restored, Ok(()) => outcomes.push(EntryOutcome {
Err(error) => EntryStatus::Failed(error),
};
outcomes.push(EntryOutcome {
label: format!("{label} (embedded metadata)"), label: format!("{label} (embedded metadata)"),
dest: meta_dest, dest: meta_dest,
kind: EntryKind::EmbeddedMetadata, kind: EntryKind::EmbeddedMetadata,
status, status: EntryStatus::Restored,
}); provenance: Vec::new(),
}),
Err(error) => embedded_failed = Some(error),
} }
if outcomes
.iter()
.all(|outcome| !matches!(outcome.status, EntryStatus::Failed(_)))
{
seen.insert(identity);
} }
if let Some(error) = embedded_failed {
vec![outcome(
EntryKind::So,
EntryStatus::Failed(error),
Vec::new(),
)]
} else {
let cached = context::CachedLibrary {
method_index_module: restored.method_index_module.clone(),
module_image: restored.module_image.clone(),
};
record_module(state.cache, app, &cached, label);
state.cache.insert_library(identity, cached);
if prior {
vec![outcome(EntryKind::So, EntryStatus::Duplicate, Vec::new())]
} else {
outcomes outcomes
} }
Err(error) => vec![outcome(EntryKind::So, EntryStatus::Failed(error))], }
}
Err(error) => vec![outcome(
EntryKind::So,
EntryStatus::Failed(error),
Vec::new(),
)],
}, },
); );
} }
// Metadata entry: write only when the restore actually changed protected method fields — let version = metadata_version(&entry_data);
// a clean blob needs no copy (same contract as loose metadata files). // Module 0x0C is a v24.1 input only. Later layouts ignore it, so a missing
let kind_and_status = match restore_metadata_bytes_with_module( // or conflicting module does not block them and is not listed as paired.
&entry_data, let (module, provenance) = if version == Some(24) {
context.method_index_module.as_deref(), let modules = state.cache.modules(app).cloned();
) { match context::binding_module(modules.as_ref()) {
Ok(module) => (
module.map(|bytes| bytes.to_vec()),
context::provenance_for(modules.as_ref(), None),
),
Err(ambiguity) => {
let provenance = context::provenance_for(modules.as_ref(), Some(&ambiguity));
return Ok(vec![outcome(
EntryKind::Metadata { remapped: 0 },
EntryStatus::Failed(anyhow::anyhow!(provenance_error(&provenance))),
provenance_labels(&provenance),
)]);
}
}
} else {
(None, context::MetadataProvenance::default())
};
let module_bytes = module;
let module = module_bytes.as_deref();
let context_identity = context::metadata_context_identity(module);
if let Some(cached) = state.cache.metadata(&identity, &context_identity).cloned() {
// A repeated blob is not written again. The cached bytes stay available
// for a *different* recovery context; this context already succeeded.
let kind = match &cached {
context::CachedMetadata::Unchanged => EntryKind::Metadata { remapped: 0 },
context::CachedMetadata::Restored { remapped, .. } => EntryKind::Metadata {
remapped: *remapped,
},
};
return Ok(vec![outcome(
kind,
EntryStatus::Duplicate,
provenance_labels(&provenance),
)]);
}
// Metadata entry: write only when the restore actually changed protected
// method fields — a clean blob needs no copy (same contract as loose files).
// The binding was resolved before this write.
let kind_and_status = match restore_metadata_bytes_with_module(&entry_data, module) {
Ok((out, report)) if report.remapped > 0 => { Ok((out, report)) if report.remapped > 0 => {
let kind = EntryKind::Metadata { let kind = EntryKind::Metadata {
remapped: report.remapped, remapped: report.remapped,
}; };
match write_metadata_blob(dest, &out) { match write_metadata_blob(dest, &out) {
Ok(()) => (kind, EntryStatus::Restored), Ok(()) => {
state.cache.insert_metadata(
identity,
context_identity,
context::CachedMetadata::Restored {
remapped: report.remapped,
},
);
(kind, EntryStatus::Restored)
}
Err(error) => (kind, EntryStatus::Failed(error)), Err(error) => (kind, EntryStatus::Failed(error)),
} }
} }
Ok(_) => (EntryKind::Metadata { remapped: 0 }, EntryStatus::Unchanged), Ok(_) => {
state.cache.insert_metadata(
identity,
context_identity,
context::CachedMetadata::Unchanged,
);
(EntryKind::Metadata { remapped: 0 }, EntryStatus::Unchanged)
}
Err(error) => ( Err(error) => (
EntryKind::Metadata { remapped: 0 }, EntryKind::Metadata { remapped: 0 },
EntryStatus::Failed(error), EntryStatus::Failed(error),
), ),
}; };
if matches!( Ok(vec![outcome(
kind_and_status.0,
kind_and_status.1, kind_and_status.1,
EntryStatus::Restored | EntryStatus::Unchanged provenance_labels(&provenance),
) { )])
seen.insert(identity); }
fn record_module(
cache: &mut context::AndroidRunCache,
app: &context::AppKey,
restored: &context::CachedLibrary,
label: &str,
) {
if let Some(module) = &restored.method_index_module {
cache.modules_mut(app).record(
module,
context::ModuleSource {
label: label.to_owned(),
image: restored.module_image.clone(),
},
);
} }
Ok(vec![outcome(kind_and_status.0, kind_and_status.1)]) }
fn metadata_version(data: &[u8]) -> Option<u32> {
data.get(4..8)
.and_then(|bytes| <[u8; 4]>::try_from(bytes).ok())
.map(u32::from_le_bytes)
}
fn provenance_labels(provenance: &context::MetadataProvenance) -> Vec<String> {
provenance
.sources
.iter()
.map(|source| match &source.image {
Some(image) => format!("{} ({image})", source.label),
None => source.label.clone(),
})
.collect()
}
fn provenance_error(provenance: &context::MetadataProvenance) -> String {
let why = provenance
.note
.clone()
.unwrap_or_else(|| "ambiguous module 0x0C binding".to_owned());
if provenance.sources.is_empty() {
return why;
}
let sources = provenance
.sources
.iter()
.map(|source| source.label.as_str())
.collect::<Vec<_>>()
.join(", ");
format!("{why}; modules from {sources}")
} }
/// Output path for a metadata blob unwrapped from a restored library: next to /// Output path for a metadata blob unwrapped from a restored library: next to
File diff suppressed because it is too large. Load diff
+142 -84
View File
@@ -213,45 +213,50 @@ pub fn run_folder_opts(
bar.inc(1); bar.inc(1);
} }
// Android pass: protected AArch64 libraries and app packages. Loose `.so` // Android pass. Loose libraries restore before packages so an identical
// files restore first so the cross-source dedup keeps them over a copy // library keeps the loose destination (loose beats `.apk` beats bundle).
// inside a package (loose beats `.apk` beats `.apks`/`.xapk` bundle). // Module 0x0C stays inside the application that produced it: an extracted
let mut android_seen = std::collections::HashSet::new(); // `lib/<abi>/` tree shares with that tree's `assets/` metadata only, and a
let mut android_method_index_module = None; // package shares only with splits of the same package name and version.
// Hashing a protected library costs a full read, so only pay it when a let mut android_cache = crate::android::run_cache();
// duplicate source can actually exist in this run. let loose = crate::android::loose_scopes(&scan.android_so, metas);
let android_dedup = scan.android_so.len() > 1 || !scan.android_packages.is_empty(); for scoped in loose.iter().filter(|scoped| {
for input in &scan.android_so { scan.android_so
let rel = rel_in_tree(root, input); .iter()
.any(|library| library == &scoped.path)
}) {
let rel = rel_in_tree(root, &scoped.path);
let dest = out_root.join(out_name(&rel)); let dest = out_root.join(out_name(&rel));
// Failed restores must not hide an identical input encountered later. let input_owned = scoped.path.clone();
// Unreadable here is fine: the restore reports the same error. let dest_owned = dest.clone();
let identity = android_dedup let key = scoped.key.clone();
.then(|| crate::android::file_content_identity(input).ok()) let label = rel.display().to_string();
.flatten(); let result = senbei_engine::catch_unwind_quiet(|| {
if identity crate::android::restore_loose_library(
.as_ref() &input_owned,
.is_some_and(|id| android_seen.contains(id)) &dest_owned,
{ &label,
&key,
verbose_steps,
&mut android_cache,
)
});
match result {
Ok(Ok(outcome)) => match outcome.status {
crate::android::EntryStatus::Duplicate => {
s.skipped += 1; s.skipped += 1;
if let Some(log) = &log { if let Some(log) = &log {
log.step(&format!("SKIP {rel:?}: duplicate of an earlier target")); log.step(&format!("SKIP {rel:?}: duplicate of an earlier target"));
} }
bar.inc(1);
continue;
} }
let input_owned = input.clone(); crate::android::EntryStatus::Failed(e) => {
let dest_owned = dest.clone(); s.errors += 1;
let result = senbei_engine::catch_unwind_quiet(|| { crate::ui::err(&bar, suppress_file_lines, &rel, &e);
crate::android::restore_so_file_with_method_index_module( if let Some(log) = &log {
&input_owned, log.step(&format!("ERR {rel:?}: {e:#}"));
&dest_owned, }
verbose_steps, }
&mut android_method_index_module, crate::android::EntryStatus::Restored => {
)
});
match result {
Ok(Ok(embedded)) => {
s.unpacked += 1; s.unpacked += 1;
crate::ui::ok_label( crate::ui::ok_label(
&bar, &bar,
@@ -263,14 +268,7 @@ pub fn run_folder_opts(
if let Some(log) = &log { if let Some(log) = &log {
log.step(&format!("OK {rel:?} -> {dest:?} (Android SO)")); log.step(&format!("OK {rel:?} -> {dest:?} (Android SO)"));
} }
let embedded_result = write_embedded_metadata(embedded, &dest); if let Some(meta_dest) = outcome.embedded {
if embedded_result.is_ok()
&& let Some(identity) = identity
{
android_seen.insert(identity);
}
match embedded_result {
Ok(Some(meta_dest)) => {
s.metadata += 1; s.metadata += 1;
crate::ui::ok_label( crate::ui::ok_label(
&bar, &bar,
@@ -283,16 +281,11 @@ pub fn run_folder_opts(
log.step(&format!("META {rel:?} (embedded) -> {meta_dest:?}")); log.step(&format!("META {rel:?} (embedded) -> {meta_dest:?}"));
} }
} }
Ok(None) => {}
Err(e) => {
s.errors += 1;
crate::ui::err(&bar, suppress_file_lines, &rel, &e);
if let Some(log) = &log {
log.step(&format!("ERR {rel:?}: embedded metadata: {e:#}"));
}
}
} }
crate::android::EntryStatus::NotTarget | crate::android::EntryStatus::Unchanged => {
s.skipped += 1;
} }
},
Ok(Err(e)) => { Ok(Err(e)) => {
s.errors += 1; s.errors += 1;
crate::ui::err(&bar, suppress_file_lines, &rel, &e); crate::ui::err(&bar, suppress_file_lines, &rel, &e);
@@ -314,31 +307,32 @@ pub fn run_folder_opts(
} }
bar.inc(1); bar.inc(1);
} }
for package in &scan.android_packages { // Loose sibling APKs that name the same package and version are one split
let rel = rel_in_tree(root, package); // set. Their libraries run before their metadata because each package
s.packages += 1; // restores its own libraries first and the shared cache collects modules
let package_owned = package.clone(); // across the group. Bundles stay one package each.
let rel_owned = rel.clone().into_owned(); let package_groups = crate::android::loose_apk_groups(&scan.android_packages);
for group in &package_groups {
if group.len() > 1 {
s.packages += group.len();
let group_owned = group.clone();
let root_owned = root.to_path_buf();
let out_root_owned = out_root.clone(); let out_root_owned = out_root.clone();
let mut seen_taken = std::mem::take(&mut android_seen);
let result = senbei_engine::catch_unwind_quiet(|| { let result = senbei_engine::catch_unwind_quiet(|| {
let outcomes = crate::android::restore_package_with_method_index_module( crate::android::restore_apk_group(
&package_owned, &group_owned,
&rel_owned, &root_owned,
&out_root_owned, &out_root_owned,
&mut seen_taken, &mut android_cache,
verbose_steps, verbose_steps,
&mut android_method_index_module, )
);
(outcomes, seen_taken)
}); });
let rel = rel_in_tree(root, &group[0]);
match result { match result {
Ok((Ok(outcomes), seen_back)) => { Ok(Ok(outcomes)) => {
android_seen = seen_back;
apply_package_outcomes(outcomes, &mut s, &bar, suppress_file_lines, &log); apply_package_outcomes(outcomes, &mut s, &bar, suppress_file_lines, &log);
} }
Ok((Err(e), seen_back)) => { Ok(Err(e)) => {
android_seen = seen_back;
s.errors += 1; s.errors += 1;
crate::ui::err(&bar, suppress_file_lines, &rel, &e); crate::ui::err(&bar, suppress_file_lines, &rel, &e);
if let Some(log) = &log { if let Some(log) = &log {
@@ -346,8 +340,49 @@ pub fn run_folder_opts(
} }
} }
Err(panic) => { Err(panic) => {
// The dedup set may be in an unknown state after a panic; a let e = anyhow::anyhow!("unexpected panic: {}", panic_payload(&panic));
// re-scan costs a duplicate restore at worst, never corruption. s.errors += 1;
crate::ui::err(&bar, suppress_file_lines, &rel, &e);
if let Some(log) = &log {
log.step(&format!(
"ERR {rel:?}: panic during package restore: {}",
panic_payload(&panic)
));
}
}
}
bar.inc(group.len() as u64);
continue;
}
for package in group {
let rel = rel_in_tree(root, package);
s.packages += 1;
let package_owned = package.clone();
let rel_owned = rel.clone().into_owned();
let out_root_owned = out_root.clone();
let result = senbei_engine::catch_unwind_quiet(|| {
crate::android::restore_package_cached(
&package_owned,
&rel_owned,
&out_root_owned,
&mut android_cache,
verbose_steps,
)
});
match result {
Ok(Ok(outcomes)) => {
apply_package_outcomes(outcomes, &mut s, &bar, suppress_file_lines, &log);
}
Ok(Err(e)) => {
s.errors += 1;
crate::ui::err(&bar, suppress_file_lines, &rel, &e);
if let Some(log) = &log {
log.step(&format!("ERR {rel:?}: {e:#}"));
}
}
Err(panic) => {
// A panic may leave a partial module recorded. Keeping the cache
// preserves successful dedup; a conflict still fails closed.
let e = anyhow::anyhow!("unexpected panic: {}", panic_payload(&panic)); let e = anyhow::anyhow!("unexpected panic: {}", panic_payload(&panic));
s.errors += 1; s.errors += 1;
crate::ui::err(&bar, suppress_file_lines, &rel, &e); crate::ui::err(&bar, suppress_file_lines, &rel, &e);
@@ -361,24 +396,31 @@ pub fn run_folder_opts(
} }
bar.inc(1); bar.inc(1);
} }
}
// il2cpp metadata pass. Crackproof's `-GMD` option obfuscates the method // il2cpp metadata pass. A loose blob is paired only with the application
// tokens in `global-metadata.dat`; de-obfuscate any we find so the unpacked // assigned when the tree was grouped. v24.1 with a missing or conflicting
// il2cpp game assembly resolves methods instead of indexing its per-module // module writes nothing. Other versions still restore when the application
// tables out of bounds (see [`senbei_metadata`]). This is additive to the // has no conflicting module. The paired libraries are recorded on the log
// Crackproof module unpack above — the metadata blob is not itself a // line.
// Crackproof file. for scoped in loose
for meta in metas.iter() { .iter()
let rel = rel_in_tree(root, meta); .filter(|scoped| metas.iter().any(|meta| meta == &scoped.path))
{
let rel = rel_in_tree(root, &scoped.path);
let dest = out_root.join(out_name(&rel)); let dest = out_root.join(out_name(&rel));
let meta_owned = meta.clone(); let meta_owned = scoped.path.clone();
let dest_owned = dest.clone(); let dest_owned = dest.clone();
let key = scoped.key.clone();
let label = rel.display().to_string();
let result = senbei_engine::catch_unwind_quiet(|| { let result = senbei_engine::catch_unwind_quiet(|| {
deobfuscate_metadata_to_with_module( crate::android::restore_loose_metadata(
&meta_owned, &meta_owned,
&dest_owned, &dest_owned,
&label,
&key,
verbose_steps, verbose_steps,
android_method_index_module.as_deref(), &mut android_cache,
) )
}); });
match result { match result {
@@ -387,8 +429,10 @@ pub fn run_folder_opts(
crate::ui::metadata(&bar, suppress_file_lines, &rel, report.remapped, &dest); crate::ui::metadata(&bar, suppress_file_lines, &rel, report.remapped, &dest);
if let Some(log) = &log { if let Some(log) = &log {
log.step(&format!( log.step(&format!(
"META {rel:?} -> {dest:?}: v{} remapped {} method fields", "META {rel:?} -> {dest:?}: v{} remapped {} method fields{}",
report.version, report.remapped report.version,
report.remapped,
provenance_suffix(&report.provenance)
)); ));
} }
} }
@@ -397,8 +441,9 @@ pub fn run_folder_opts(
Ok(Ok(report)) => { Ok(Ok(report)) => {
if let Some(log) = &log { if let Some(log) = &log {
log.step(&format!( log.step(&format!(
"META {rel:?}: v{} already de-obfuscated", "META {rel:?}: v{} already de-obfuscated{}",
report.version report.version,
provenance_suffix(&report.provenance)
)); ));
} }
} }
@@ -709,6 +754,14 @@ pub fn out_name(input: &Path) -> PathBuf {
} }
} }
fn provenance_suffix(sources: &[String]) -> String {
if sources.is_empty() {
String::new()
} else {
format!("; module 0x0C from {}", sources.join(", "))
}
}
/// Extract a printable message from a caught panic payload. /// Extract a printable message from a caught panic payload.
fn panic_payload(panic: &(dyn std::any::Any + Send)) -> String { fn panic_payload(panic: &(dyn std::any::Any + Send)) -> String {
if let Some(s) = panic.downcast_ref::<&str>() { if let Some(s) = panic.downcast_ref::<&str>() {
@@ -832,7 +885,12 @@ fn apply_package_outcomes(
} }
} }
if let Some(log) = log { if let Some(log) = log {
log.step(&format!("OK {} -> {:?}", outcome.label, outcome.dest)); log.step(&format!(
"OK {} -> {:?}{}",
outcome.label,
outcome.dest,
provenance_suffix(&outcome.provenance)
));
} }
} }
EntryStatus::Duplicate | EntryStatus::NotTarget | EntryStatus::Unchanged => { EntryStatus::Duplicate | EntryStatus::NotTarget | EntryStatus::Unchanged => {