Momoko-Ayase/Senbei
1
1
Fork 0
mirror of https://github.com/Momoko-Ayase/Senbei.git synced 2026-09-19 03:57:59 -04:00
Code Issues Activity

fix(scan): stream Android package targets

Browse Source
This commit is contained in:
bfloat16
2026-09-06 22:25:41 +08:00
parent d436a200ba
commit 776d246065
9 changed files with 123 additions and 208 deletions
Download Patch File Download Diff File Expand all files Collapse all files
Cargo.lock
Generated
+1 -19
View File
@@ -2,12 +2,6 @@
# It is not intended for manual editing.
version = 4
[[package]]
name = "adler2"
version = "2.0.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "320119579fcad9c21884f5c4861d16174d0e06250625266f50fe6898340abefa"
[[package]]
name = "aes"
version = "0.9.3"
@@ -158,8 +152,6 @@ version = "1.1.10"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6e634e2e0ebac1ee034020da1ca582e17ffe4e0f5e985823721e168928136dcb"
dependencies = [
"crc32fast",
"miniz_oxide",
"zlib-rs",
]
@@ -306,16 +298,6 @@ dependencies = [
"libc",
]
[[package]]
name = "miniz_oxide"
version = "0.9.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b63fbc4a50860e98e7b2aa7804ded1db5cbc3aff9193adaff57a6931bf7c4b4c"
dependencies = [
"adler2",
"simd-adler32",
]
[[package]]
name = "once_cell"
version = "1.21.4"
@@ -464,9 +446,9 @@ name = "senbei-io"
version = "1.2.0"
dependencies = [
"anyhow",
"flate2",
"indicatif",
"libc",
"memmap2",
"owo-colors",
"senbei-engine",
"senbei-metadata",
Cargo.toml
-1
View File
@@ -23,7 +23,6 @@ license = "AGPL-3.0-only"
[workspace.dependencies]
aes = "0.9"
anyhow = "1"
flate2 = "1"
goblin = "0.10"
indicatif = "0.18"
libc = "0.2"
docs/development.md
+1 -1
View File
@@ -23,7 +23,7 @@ For an Android package, use one command at a time because a protected `.so` can
- `DD8_SHIFT` overrides the PE page-XOR shift; `99` skips that stage.
- `SEL_DIAG` prints PE layout-selector diagnostics.
- `SENBEI_THREADS` caps deterministic block fan-out; `1` forces the sequential reference path.
- `SENBEI_SCAN_ALL` enables the explicit scan-all mode for selected target names.
- `SENBEI_SCAN_ALL` enables probing selected target names below the size floor; it never enables arbitrary filenames.
- `SENBEI_ANDROID_SAMPLES` overrides the Android sample corpus location.
## Conventions
senbei-cli/src/main.rs
+2 -3
View File
@@ -104,8 +104,7 @@ fn print_help() {
\x20 these"
);
println!(
" --scan-all probe every file in a folder, including ones the scan\n\
\x20 pre-filter skips (under 4128 bytes, extensionless,\n\
\x20 or a bulk-asset extension). Much slower on large trees."
" --scan-all probe selected .exe/.dll/.so/metadata names below the\n\
\x20 size floor; other filenames remain excluded."
);
}
senbei-io/Cargo.toml
+1 -1
View File
@@ -7,8 +7,8 @@ description = "Filesystem, scanning, logging, and CLI orchestration for Senbei"
[dependencies]
anyhow.workspace = true
flate2.workspace = true
indicatif.workspace = true
memmap2.workspace = true
owo-colors.workspace = true
senbei-engine.workspace = true
senbei-metadata.workspace = true
senbei-io/src/android/mod.rs
+55 -36
View File
@@ -16,11 +16,12 @@
//! app (wasm) never touches them.
use std::collections::HashSet;
use std::io::{BufWriter, Write};
use std::fs::File;
use std::io::{BufWriter, Read, Seek, Write};
use std::path::{Path, PathBuf};
use anyhow::{Context, Result, bail};
use flate2::read::DeflateDecoder;
use memmap2::{Mmap, MmapOptions};
use senbei_engine::android::{ExtractOptions, extract_stage2, is_protected_libil2cpp};
use senbei_engine::android::{RestoreOptions, restore_libil2cpp};
use sha2::{Digest, Sha256};
@@ -30,8 +31,8 @@ use zip::ZipArchive;
pub const METADATA_FILE_NAME: &str = "global-metadata.dat";
/// Package extensions recognised as Android app packages. Packages are
/// *containers*: membership is decided by extension plus the zip magic, while
/// every file pulled out of one is still content-probed like a loose file.
/// *containers*: membership is decided by extension plus the ZIP magic, while
/// only `.so` and `global-metadata.dat` entries are read.
const PACKAGE_EXTENSIONS: [&str; 3] = ["apk", "apks", "xapk"];
/// Whether `prefix` (the first bytes of a file) is an ELF64/AArch64 image.
@@ -65,12 +66,23 @@ pub fn is_app_package(path: &Path, prefix: &[u8]) -> bool {
/// section-header table at the end); call only after [`is_elf64_aarch64`]
/// has matched a prefix.
pub fn is_protected_so_file(path: &Path) -> bool {
let Ok(bytes) = std::fs::read(path) else {
let Ok(file) = File::open(path) else {
return false;
};
let Ok(bytes) = map_read_only(&file, path) else {
return false;
};
is_elf64_aarch64(&bytes) && is_protected_libil2cpp(&bytes)
}
pub fn file_content_identity(path: &Path) -> std::io::Result<String> {
let file = File::open(path)?;
// SAFETY: the file remains open for the mapping lifetime and the mapping
// is read-only.
let bytes = unsafe { MmapOptions::new().map(&file)? };
Ok(content_identity(&bytes))
}
/// Restore one protected `.so` to `dest`.
///
/// The stage-2 module set is extracted into a temporary workspace (it is an
@@ -238,19 +250,24 @@ pub fn restore_package(
}
}
}
drop(archive);
for (index, name) in direct {
let label = format!("{}::{}", rel.display(), name.display());
let dest = out_root.join(rel).join(crate::job::out_name(&name));
let mut entry_outcomes =
restore_package_entry(package, index, &label, &dest, &temporary, seen, verbose)
let mut entry_outcomes = restore_package_entry(
&mut archive,
index,
&label,
&dest,
&temporary,
seen,
verbose,
)
.with_context(|| format!("extract `{label}`"))?;
outcomes.append(&mut entry_outcomes);
}
for (index, name) in nested {
let nested_label = rel.join(&name);
let nested_path = extract_entry(package, index, &temporary, &nested_label)
let nested_path = extract_entry(&mut archive, index, &temporary, &nested_label)
.with_context(|| format!("extract `{}`", nested_label.display()))?;
let mut nested_archive = open_package(&nested_path)?;
let mut entries = Vec::new();
@@ -268,7 +285,6 @@ pub fn restore_package(
}
}
}
drop(nested_archive);
// Keep the nested package's stem in the output layout so two splits
// carrying same-named entries cannot collide.
let base = rel.join(name.with_extension(""));
@@ -276,7 +292,7 @@ pub fn restore_package(
let label = format!("{}::{}", nested_label.display(), entry_name.display());
let dest = out_root.join(&base).join(crate::job::out_name(&entry_name));
let mut entry_outcomes = restore_package_entry(
&nested_path,
&mut nested_archive,
nested_index,
&label,
&dest,
@@ -294,8 +310,8 @@ pub fn restore_package(
/// Probe one extracted package entry and restore it when it is a target.
/// Returns one outcome per produced/consumed artifact: the entry itself, plus
/// an `EmbeddedMetadata` outcome when the restored library carried a blob.
fn restore_package_entry(
package: &Path,
fn restore_package_entry<R: Read + Seek>(
archive: &mut ZipArchive<R>,
index: usize,
label: &str,
dest: &Path,
@@ -303,11 +319,13 @@ fn restore_package_entry(
seen: &mut HashSet<String>,
verbose: bool,
) -> Result<Vec<EntryOutcome>> {
let entry_path = extract_entry(package, index, temporary, Path::new(label))?;
let data = std::fs::read(&entry_path).with_context(|| format!("read extracted `{label}`"))?;
let entry_path = extract_entry(archive, index, temporary, Path::new(label))?;
let entry_file =
File::open(&entry_path).with_context(|| format!("open extracted `{label}`"))?;
let entry_data = map_read_only(&entry_file, &entry_path)?;
let is_so = is_elf64_aarch64(&data) && is_protected_libil2cpp(&data);
let is_meta = !is_so && senbei_metadata::is_metadata(&data);
let is_so = is_elf64_aarch64(&entry_data) && is_protected_libil2cpp(&entry_data);
let is_meta = !is_so && senbei_metadata::is_metadata(&entry_data);
let outcome = |kind, status| EntryOutcome {
label: label.to_owned(),
dest: dest.to_path_buf(),
@@ -317,7 +335,7 @@ fn restore_package_entry(
if !is_so && !is_meta {
return Ok(vec![outcome(EntryKind::So, EntryStatus::NotTarget)]);
}
if !seen.insert(content_identity(&data)) {
if !seen.insert(content_identity(&entry_data)) {
let kind = if is_so {
EntryKind::So
} else {
@@ -327,7 +345,8 @@ fn restore_package_entry(
}
if is_so {
drop(data);
drop(entry_data);
drop(entry_file);
return Ok(match restore_so_file(&entry_path, dest, verbose) {
Ok(embedded) => {
let mut outcomes = vec![outcome(EntryKind::So, EntryStatus::Restored)];
@@ -352,7 +371,7 @@ fn restore_package_entry(
// Metadata entry: write only when the restore actually changed tokens —
// a clean blob needs no copy (same contract as loose metadata files).
let kind_and_status = match restore_metadata_bytes(&data) {
let kind_and_status = match restore_metadata_bytes(&entry_data) {
Ok((out, report)) if report.remapped > 0 => {
let kind = EntryKind::Metadata {
remapped: report.remapped,
@@ -396,31 +415,24 @@ fn open_package(path: &Path) -> Result<ZipArchive<std::fs::File>> {
ZipArchive::new(file).with_context(|| format!("read package `{}`", path.display()))
}
/// Extract one package entry to the temporary workspace, streaming stored
/// entries and inflating deflated ones by hand so compression-method
/// surprises fail loudly instead of producing a truncated file.
fn extract_entry(
package: &Path,
/// Stream one package entry to a temporary, seekable file. The Android engine
/// needs random access to ELF section tables, while the ZIP reader itself is
/// consumed directly without creating an in-memory compressed or decompressed
/// copy.
fn extract_entry<R: Read + Seek>(
archive: &mut ZipArchive<R>,
index: usize,
temporary: &tempfile::TempDir,
label: &Path,
) -> Result<PathBuf> {
let mut archive = open_package(package)?;
let mut entry = archive.by_index_raw(index)?;
let mut entry = archive.by_index(index)?;
let key = format!("{}-{index:08x}", label.display());
// `:` appears in `package::entry` labels and is invalid in Windows file
// names; sanitize every path-ish separator.
let destination = temporary.path().join(key.replace(['\\', '/', ':'], "_"));
let output_size = entry.size();
let mut output = BufWriter::new(std::fs::File::create(&destination)?);
let written = match entry.compression() {
zip::CompressionMethod::Stored => std::io::copy(&mut entry, &mut output)?,
zip::CompressionMethod::Deflated => {
let mut decoder = DeflateDecoder::new(&mut entry);
std::io::copy(&mut decoder, &mut output)?
}
method => bail!("unsupported compression method {method:?} in entry `{key}`"),
};
let written = std::io::copy(&mut entry, &mut output)?;
output.flush()?;
if written != output_size {
bail!(
@@ -430,6 +442,13 @@ fn extract_entry(
}
Ok(destination)
}
fn map_read_only(file: &File, path: &Path) -> Result<Mmap> {
// SAFETY: the file descriptor remains open for the returned mapping's
// lifetime, and this mapping is read-only.
unsafe { MmapOptions::new().map(file) }
.with_context(|| format!("map extracted `{}`", path.display()))
}
/// Lowercase hex of a digest output (sha2 0.11's `Array` no longer formats as
/// hex directly).
fn hex_digest(data: &[u8]) -> String {
senbei-io/src/job.rs
+6 -11
View File
@@ -403,11 +403,8 @@ pub fn run_folder_v(
/// Like [`run_folder_v`], but with the scan pre-filter explicitly controlled.
///
/// When `scan_all` is true every regular file under `root` is opened and
/// content-probed, instead of skipping ones the free directory metadata already
/// rules out (extensionless, too small to hold a Crackproof key table, or a
/// bulk-asset extension). See [`crate::scan::find_targets_opts`] — exhaustive
/// scanning is dramatically slower on asset-heavy trees.
/// When `scan_all` is true selected target names below the minimum size are
/// also opened and content-probed. Other filenames are never opened.
pub fn run_folder_opts(
root: &Path,
out_dir: Option<&Path>,
@@ -549,8 +546,8 @@ pub fn run_folder_opts(
let dest = out_root.join(out_name(&rel));
// Unreadable here is fine: the restore reports the same error.
if android_dedup
&& let Ok(bytes) = std::fs::read(input)
&& !android_seen.insert(crate::android::content_identity(&bytes))
&& let Ok(identity) = crate::android::file_content_identity(input)
&& !android_seen.insert(identity)
{
s.skipped += 1;
if let Some(log) = &log {
@@ -816,10 +813,8 @@ pub fn run_file_v(
// library probe needs the whole file (its payload section is found through
// the section-header table at the end), while a package is a container
// handled entry-by-entry. Anything else falls through to the PE pipeline.
let is_android_so = crate::android::is_elf64_aarch64(&prefix)
&& std::fs::read(input)
.map(|bytes| senbei_engine::android::is_protected_libil2cpp(&bytes))
.unwrap_or(false);
let is_android_so =
crate::android::is_elf64_aarch64(&prefix) && crate::android::is_protected_so_file(input);
let is_android_package = !is_android_so && crate::android::is_app_package(input, &prefix);
if is_meta {
senbei-io/src/scan.rs
+55 -116
View File
@@ -1,4 +1,6 @@
use memmap2::MmapOptions;
use senbei_engine::detect;
use std::fs::File;
use std::io::Read;
use std::path::{Path, PathBuf};
use walkdir::WalkDir;
@@ -77,78 +79,6 @@ pub(crate) fn is_android_entry_name(path: &Path) -> bool {
.is_some_and(|ext| ext.eq_ignore_ascii_case("so"))
}
/// File extensions that are bulk data by construction and can never be a target.
///
/// Set `SENBEI_SCAN_ALL=1` (or pass `--scan-all`) to probe every file regardless.
const DENY_EXT: &[&str] = &[
// Unity and other engine asset containers
"ab",
"bundle",
"unity3d",
"manifest",
"resource",
"ress",
"assets",
"sharedassets",
// audio / video / image / font
"acb",
"awb",
"usm",
"wav",
"ogg",
"mp3",
"mp4",
"avi",
"png",
"jpg",
"jpeg",
"bmp",
"gif",
"tga",
"dds",
"svg",
"ttf",
"otf",
// text, markup, config, logs
"xml",
"json",
"txt",
"csv",
"md",
"toml",
"ini",
"yml",
"yaml",
"log",
"html",
"htm",
"css",
"aspx",
"browser",
"config",
"sig",
"map",
"pdb",
// rhythm-game chart/score data
"ma2",
"sr",
];
/// Whether `path` can be skipped from its name alone.
fn denied_name(path: &Path) -> bool {
let Some(ext) = path.extension() else {
return true;
};
let Some(ext) = ext.to_str() else {
return false;
};
// Extensions are ASCII in practice; compare case-insensitively without
// allocating for the overwhelmingly common non-match.
DENY_EXT
.iter()
.any(|d| d.len() == ext.len() && d.eq_ignore_ascii_case(ext))
}
/// Content classification of a single file.
#[derive(Clone, Copy, PartialEq, Eq)]
enum Class {
@@ -197,9 +127,9 @@ pub struct ScanResult {
/// So the only lever is **probing fewer files**, which is what the target-name
/// filter and [`MIN_SIZE`] do — both decided before any file is opened.
///
/// The surviving probes (open + short read + magic test) are fanned out across
/// worker threads. Directory traversal itself stays serial (one cheap `readdir`
/// pass, no file opens) because it feeds the parallel probe.
/// The selected probes (open + short read + magic test) are fanned out across
/// worker threads. Directory traversal itself stays serial because it only
/// collects names and sizes before the parallel probe.
///
/// Thread count follows [`senbei_engine::thread_cap`] (honoring
/// `SENBEI_THREADS`, `1` = fully sequential). Output order is independent of
@@ -227,8 +157,7 @@ pub struct ScanStats {
}
/// [`find_targets`], but with the pre-filter explicitly controlled. When
/// `scan_all` is true every regular file is probed, restoring the exhaustive
/// (and on asset-heavy trees, far slower) behavior.
/// `scan_all` is true selected target names below [`MIN_SIZE`] are also probed.
pub fn find_targets_opts(root: &Path, scan_all: bool) -> ScanResult {
// Phase 1: serial traversal collecting regular-file paths only. No file is
// opened here; `readdir` is fast relative to the content probe that follows,
@@ -275,11 +204,6 @@ pub fn find_targets_opts(root: &Path, scan_all: bool) -> ScanResult {
continue;
}
if !scan_all {
// Name checks come first so extensionless asset chunks never
// trigger even an explicit metadata query.
if denied_name(entry.path()) {
continue;
}
// Skip on directory metadata alone — never open these.
let too_small = entry
.metadata()
@@ -354,9 +278,9 @@ fn is_reparse_point(_e: &walkdir::DirEntry) -> bool {
false
}
/// Whether the scan pre-filter is disabled via `SENBEI_SCAN_ALL`. Any value
/// other than `0`/empty turns exhaustive scanning on. The `--scan-all` flag is
/// ORed with this.
/// Whether the size pre-filter is disabled via `SENBEI_SCAN_ALL`. Any value
/// other than `0`/empty enables probing small selected target names. It never
/// expands the platform filename boundary.
pub fn scan_all_env() -> bool {
match std::env::var("SENBEI_SCAN_ALL") {
Ok(v) => !matches!(v.trim(), "" | "0"),
@@ -381,8 +305,7 @@ pub fn scan_all_env() -> bool {
/// The Android library probe needs more than the prefix: the protection
/// payload lives in a section found via the section-header table at the *end*
/// of the file, so an ELF64/AArch64 prefix triggers a full-file read. Only
/// aarch64 images pay for it — a handful of `.so` files per app tree, against
/// tens of thousands of assets the free name/size checks already rejected.
/// selected `.so` images pay for it.
fn classify(path: &Path) -> Option<Class> {
let head = read_prefix(path, DETECT_PREFIX)?;
let r = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| {
@@ -405,7 +328,12 @@ fn classify(path: &Path) -> Option<Class> {
.and_then(|ext| ext.to_str())
.is_some_and(|ext| ext.eq_ignore_ascii_case("so"))
&& crate::android::is_elf64_aarch64(&head)
&& std::fs::read(path)
&& File::open(path)
.and_then(|file| {
// SAFETY: the file remains open for the mapping lifetime
// and the mapping is read-only.
unsafe { MmapOptions::new().map(&file) }
})
.map(|bytes| senbei_engine::android::is_protected_libil2cpp(&bytes))
.unwrap_or(false)
{
@@ -430,33 +358,30 @@ mod tests {
use super::*;
#[test]
fn denies_bulk_asset_extensions_case_insensitively() {
for p in ["a.ab", "a.XML", "a.Acb", "a.ma2", "a.manifest", "a.PNG"] {
assert!(denied_name(Path::new(p)), "{p} should be denied");
fn candidate_names_are_platform_specific() {
for p in [
"daemon.exe",
"GameLib.DLL",
"libil2cpp.so",
"global-metadata.dat",
] {
assert!(
is_metadata_name(Path::new(p)) || is_target_extension(Path::new(p)),
"{p} should be a candidate"
);
}
}
#[test]
fn denies_extensionless_files() {
for p in ["asset", "level0", "0123456789abcdef"] {
assert!(denied_name(Path::new(p)), "{p} should be denied");
}
}
#[test]
fn never_denies_what_a_target_can_be_named() {
// Unknown extensions must still be probed. This keeps the filter a
// narrow deny-list rather than an executable-extension allow-list.
for p in [
"app.exe.bak",
"managed.dll.bak",
"daemon.exe",
"GameLib.dll",
"global-metadata.dat",
"a.so",
"a.bin",
"libil2cpp.so.bak",
"global-metadata.bin",
"asset",
"a.ab",
] {
assert!(!denied_name(Path::new(p)), "{p} must still be probed");
assert!(
!is_metadata_name(Path::new(p)) && !is_target_extension(Path::new(p)),
"{p} must not be a candidate"
);
}
}
@@ -475,10 +400,10 @@ mod tests {
assert!(exhaustive.metadata.is_empty());
}
/// A file below the Crackproof key-table bound is skipped without being
/// opened, but a large non-asset file is still probed.
/// A selected file below the Crackproof key-table bound is skipped without
/// being opened, while `scan_all` probes it.
#[test]
fn prefilter_skips_small_and_denied_files_only() {
fn prefilter_skips_small_selected_files_only() {
let td = tempfile::tempdir().unwrap();
let root = td.path();
std::fs::write(root.join("tiny.dll"), vec![0u8; 100]).unwrap();
@@ -486,15 +411,15 @@ mod tests {
std::fs::write(root.join("plain.dll"), vec![0u8; 100_000]).unwrap();
// None of them are Crackproof, so both modes find nothing; the point is
// that the filtered walk does not panic and honors `scan_all`.
// that only the selected names are considered and `scan_all` controls
// the size floor.
let scan = find_targets_opts(root, false);
assert!(scan.crackproof.is_empty() && scan.metadata.is_empty());
let scan = find_targets_opts(root, true);
assert!(scan.crackproof.is_empty() && scan.metadata.is_empty());
}
/// An il2cpp metadata blob is found by the filtered scan: `.dat` is not on
/// the deny-list and a real one is far above `MIN_SIZE`.
/// An exact `global-metadata.dat` name is found by the filtered scan.
#[test]
fn finds_metadata_through_the_prefilter() {
let td = tempfile::tempdir().unwrap();
@@ -549,4 +474,18 @@ mod tests {
assert_eq!(scan.stats.skipped, 1, "only the stub was probed");
assert!(is_windows_companion(&root.join("app.exe._")));
}
#[test]
fn scan_all_keeps_the_platform_name_boundary() {
let td = tempfile::tempdir().unwrap();
let root = td.path();
let mut metadata = vec![0_u8; MIN_SIZE as usize];
metadata[..4].copy_from_slice(&0xFAB1_1BAFu32.to_le_bytes());
std::fs::write(root.join("renamed.bin"), &metadata).unwrap();
std::fs::write(root.join("global-metadata.dat"), &metadata).unwrap();
let scan = find_targets_opts(root, true);
assert_eq!(scan.metadata.len(), 1);
assert!(scan.metadata[0].ends_with("global-metadata.dat"));
}
}
senbei-wasm/Cargo.lock
Generated
+1 -19
View File
@@ -2,12 +2,6 @@
# It is not intended for manual editing.
version = 4
[[package]]
name = "adler2"
version = "2.0.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "320119579fcad9c21884f5c4861d16174d0e06250625266f50fe6898340abefa"
[[package]]
name = "aes"
version = "0.9.3"
@@ -168,8 +162,6 @@ version = "1.1.10"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6e634e2e0ebac1ee034020da1ca582e17ffe4e0f5e985823721e168928136dcb"
dependencies = [
"crc32fast",
"miniz_oxide",
"zlib-rs",
]
@@ -316,16 +308,6 @@ dependencies = [
"libc",
]
[[package]]
name = "miniz_oxide"
version = "0.9.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b63fbc4a50860e98e7b2aa7804ded1db5cbc3aff9193adaff57a6931bf7c4b4c"
dependencies = [
"adler2",
"simd-adler32",
]
[[package]]
name = "once_cell"
version = "1.21.4"
@@ -455,9 +437,9 @@ name = "senbei-io"
version = "1.2.0"
dependencies = [
"anyhow",
"flate2",
"indicatif",
"libc",
"memmap2",
"owo-colors",
"senbei-engine",
"senbei-metadata",