mirror of
https://github.com/Momoko-Ayase/Senbei.git
synced 2026-09-19 03:57:59 -04:00
fix(scan): stream Android package targets
This commit is contained in:
Generated
+1
-19
@@ -2,12 +2,6 @@
|
|||||||
# It is not intended for manual editing.
|
# It is not intended for manual editing.
|
||||||
version = 4
|
version = 4
|
||||||
|
|
||||||
[[package]]
|
|
||||||
name = "adler2"
|
|
||||||
version = "2.0.1"
|
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
||||||
checksum = "320119579fcad9c21884f5c4861d16174d0e06250625266f50fe6898340abefa"
|
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "aes"
|
name = "aes"
|
||||||
version = "0.9.3"
|
version = "0.9.3"
|
||||||
@@ -158,8 +152,6 @@ version = "1.1.10"
|
|||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
checksum = "6e634e2e0ebac1ee034020da1ca582e17ffe4e0f5e985823721e168928136dcb"
|
checksum = "6e634e2e0ebac1ee034020da1ca582e17ffe4e0f5e985823721e168928136dcb"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"crc32fast",
|
|
||||||
"miniz_oxide",
|
|
||||||
"zlib-rs",
|
"zlib-rs",
|
||||||
]
|
]
|
||||||
|
|
||||||
@@ -306,16 +298,6 @@ dependencies = [
|
|||||||
"libc",
|
"libc",
|
||||||
]
|
]
|
||||||
|
|
||||||
[[package]]
|
|
||||||
name = "miniz_oxide"
|
|
||||||
version = "0.9.1"
|
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
||||||
checksum = "b63fbc4a50860e98e7b2aa7804ded1db5cbc3aff9193adaff57a6931bf7c4b4c"
|
|
||||||
dependencies = [
|
|
||||||
"adler2",
|
|
||||||
"simd-adler32",
|
|
||||||
]
|
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "once_cell"
|
name = "once_cell"
|
||||||
version = "1.21.4"
|
version = "1.21.4"
|
||||||
@@ -464,9 +446,9 @@ name = "senbei-io"
|
|||||||
version = "1.2.0"
|
version = "1.2.0"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"anyhow",
|
"anyhow",
|
||||||
"flate2",
|
|
||||||
"indicatif",
|
"indicatif",
|
||||||
"libc",
|
"libc",
|
||||||
|
"memmap2",
|
||||||
"owo-colors",
|
"owo-colors",
|
||||||
"senbei-engine",
|
"senbei-engine",
|
||||||
"senbei-metadata",
|
"senbei-metadata",
|
||||||
|
|||||||
@@ -23,7 +23,6 @@ license = "AGPL-3.0-only"
|
|||||||
[workspace.dependencies]
|
[workspace.dependencies]
|
||||||
aes = "0.9"
|
aes = "0.9"
|
||||||
anyhow = "1"
|
anyhow = "1"
|
||||||
flate2 = "1"
|
|
||||||
goblin = "0.10"
|
goblin = "0.10"
|
||||||
indicatif = "0.18"
|
indicatif = "0.18"
|
||||||
libc = "0.2"
|
libc = "0.2"
|
||||||
|
|||||||
+1
-1
@@ -23,7 +23,7 @@ For an Android package, use one command at a time because a protected `.so` can
|
|||||||
- `DD8_SHIFT` overrides the PE page-XOR shift; `99` skips that stage.
|
- `DD8_SHIFT` overrides the PE page-XOR shift; `99` skips that stage.
|
||||||
- `SEL_DIAG` prints PE layout-selector diagnostics.
|
- `SEL_DIAG` prints PE layout-selector diagnostics.
|
||||||
- `SENBEI_THREADS` caps deterministic block fan-out; `1` forces the sequential reference path.
|
- `SENBEI_THREADS` caps deterministic block fan-out; `1` forces the sequential reference path.
|
||||||
- `SENBEI_SCAN_ALL` enables the explicit scan-all mode for selected target names.
|
- `SENBEI_SCAN_ALL` enables probing selected target names below the size floor; it never enables arbitrary filenames.
|
||||||
- `SENBEI_ANDROID_SAMPLES` overrides the Android sample corpus location.
|
- `SENBEI_ANDROID_SAMPLES` overrides the Android sample corpus location.
|
||||||
|
|
||||||
## Conventions
|
## Conventions
|
||||||
|
|||||||
@@ -104,8 +104,7 @@ fn print_help() {
|
|||||||
\x20 these"
|
\x20 these"
|
||||||
);
|
);
|
||||||
println!(
|
println!(
|
||||||
" --scan-all probe every file in a folder, including ones the scan\n\
|
" --scan-all probe selected .exe/.dll/.so/metadata names below the\n\
|
||||||
\x20 pre-filter skips (under 4128 bytes, extensionless,\n\
|
\x20 size floor; other filenames remain excluded."
|
||||||
\x20 or a bulk-asset extension). Much slower on large trees."
|
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -7,8 +7,8 @@ description = "Filesystem, scanning, logging, and CLI orchestration for Senbei"
|
|||||||
|
|
||||||
[dependencies]
|
[dependencies]
|
||||||
anyhow.workspace = true
|
anyhow.workspace = true
|
||||||
flate2.workspace = true
|
|
||||||
indicatif.workspace = true
|
indicatif.workspace = true
|
||||||
|
memmap2.workspace = true
|
||||||
owo-colors.workspace = true
|
owo-colors.workspace = true
|
||||||
senbei-engine.workspace = true
|
senbei-engine.workspace = true
|
||||||
senbei-metadata.workspace = true
|
senbei-metadata.workspace = true
|
||||||
|
|||||||
@@ -16,11 +16,12 @@
|
|||||||
//! app (wasm) never touches them.
|
//! app (wasm) never touches them.
|
||||||
|
|
||||||
use std::collections::HashSet;
|
use std::collections::HashSet;
|
||||||
use std::io::{BufWriter, Write};
|
use std::fs::File;
|
||||||
|
use std::io::{BufWriter, Read, Seek, Write};
|
||||||
use std::path::{Path, PathBuf};
|
use std::path::{Path, PathBuf};
|
||||||
|
|
||||||
use anyhow::{Context, Result, bail};
|
use anyhow::{Context, Result, bail};
|
||||||
use flate2::read::DeflateDecoder;
|
use memmap2::{Mmap, MmapOptions};
|
||||||
use senbei_engine::android::{ExtractOptions, extract_stage2, is_protected_libil2cpp};
|
use senbei_engine::android::{ExtractOptions, extract_stage2, is_protected_libil2cpp};
|
||||||
use senbei_engine::android::{RestoreOptions, restore_libil2cpp};
|
use senbei_engine::android::{RestoreOptions, restore_libil2cpp};
|
||||||
use sha2::{Digest, Sha256};
|
use sha2::{Digest, Sha256};
|
||||||
@@ -30,8 +31,8 @@ use zip::ZipArchive;
|
|||||||
pub const METADATA_FILE_NAME: &str = "global-metadata.dat";
|
pub const METADATA_FILE_NAME: &str = "global-metadata.dat";
|
||||||
|
|
||||||
/// Package extensions recognised as Android app packages. Packages are
|
/// Package extensions recognised as Android app packages. Packages are
|
||||||
/// *containers*: membership is decided by extension plus the zip magic, while
|
/// *containers*: membership is decided by extension plus the ZIP magic, while
|
||||||
/// every file pulled out of one is still content-probed like a loose file.
|
/// only `.so` and `global-metadata.dat` entries are read.
|
||||||
const PACKAGE_EXTENSIONS: [&str; 3] = ["apk", "apks", "xapk"];
|
const PACKAGE_EXTENSIONS: [&str; 3] = ["apk", "apks", "xapk"];
|
||||||
|
|
||||||
/// Whether `prefix` (the first bytes of a file) is an ELF64/AArch64 image.
|
/// Whether `prefix` (the first bytes of a file) is an ELF64/AArch64 image.
|
||||||
@@ -65,12 +66,23 @@ pub fn is_app_package(path: &Path, prefix: &[u8]) -> bool {
|
|||||||
/// section-header table at the end); call only after [`is_elf64_aarch64`]
|
/// section-header table at the end); call only after [`is_elf64_aarch64`]
|
||||||
/// has matched a prefix.
|
/// has matched a prefix.
|
||||||
pub fn is_protected_so_file(path: &Path) -> bool {
|
pub fn is_protected_so_file(path: &Path) -> bool {
|
||||||
let Ok(bytes) = std::fs::read(path) else {
|
let Ok(file) = File::open(path) else {
|
||||||
|
return false;
|
||||||
|
};
|
||||||
|
let Ok(bytes) = map_read_only(&file, path) else {
|
||||||
return false;
|
return false;
|
||||||
};
|
};
|
||||||
is_elf64_aarch64(&bytes) && is_protected_libil2cpp(&bytes)
|
is_elf64_aarch64(&bytes) && is_protected_libil2cpp(&bytes)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
pub fn file_content_identity(path: &Path) -> std::io::Result<String> {
|
||||||
|
let file = File::open(path)?;
|
||||||
|
// SAFETY: the file remains open for the mapping lifetime and the mapping
|
||||||
|
// is read-only.
|
||||||
|
let bytes = unsafe { MmapOptions::new().map(&file)? };
|
||||||
|
Ok(content_identity(&bytes))
|
||||||
|
}
|
||||||
|
|
||||||
/// Restore one protected `.so` to `dest`.
|
/// Restore one protected `.so` to `dest`.
|
||||||
///
|
///
|
||||||
/// The stage-2 module set is extracted into a temporary workspace (it is an
|
/// The stage-2 module set is extracted into a temporary workspace (it is an
|
||||||
@@ -238,19 +250,24 @@ pub fn restore_package(
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
drop(archive);
|
|
||||||
|
|
||||||
for (index, name) in direct {
|
for (index, name) in direct {
|
||||||
let label = format!("{}::{}", rel.display(), name.display());
|
let label = format!("{}::{}", rel.display(), name.display());
|
||||||
let dest = out_root.join(rel).join(crate::job::out_name(&name));
|
let dest = out_root.join(rel).join(crate::job::out_name(&name));
|
||||||
let mut entry_outcomes =
|
let mut entry_outcomes = restore_package_entry(
|
||||||
restore_package_entry(package, index, &label, &dest, &temporary, seen, verbose)
|
&mut archive,
|
||||||
|
index,
|
||||||
|
&label,
|
||||||
|
&dest,
|
||||||
|
&temporary,
|
||||||
|
seen,
|
||||||
|
verbose,
|
||||||
|
)
|
||||||
.with_context(|| format!("extract `{label}`"))?;
|
.with_context(|| format!("extract `{label}`"))?;
|
||||||
outcomes.append(&mut entry_outcomes);
|
outcomes.append(&mut entry_outcomes);
|
||||||
}
|
}
|
||||||
for (index, name) in nested {
|
for (index, name) in nested {
|
||||||
let nested_label = rel.join(&name);
|
let nested_label = rel.join(&name);
|
||||||
let nested_path = extract_entry(package, index, &temporary, &nested_label)
|
let nested_path = extract_entry(&mut archive, index, &temporary, &nested_label)
|
||||||
.with_context(|| format!("extract `{}`", nested_label.display()))?;
|
.with_context(|| format!("extract `{}`", nested_label.display()))?;
|
||||||
let mut nested_archive = open_package(&nested_path)?;
|
let mut nested_archive = open_package(&nested_path)?;
|
||||||
let mut entries = Vec::new();
|
let mut entries = Vec::new();
|
||||||
@@ -268,7 +285,6 @@ pub fn restore_package(
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
drop(nested_archive);
|
|
||||||
// Keep the nested package's stem in the output layout so two splits
|
// Keep the nested package's stem in the output layout so two splits
|
||||||
// carrying same-named entries cannot collide.
|
// carrying same-named entries cannot collide.
|
||||||
let base = rel.join(name.with_extension(""));
|
let base = rel.join(name.with_extension(""));
|
||||||
@@ -276,7 +292,7 @@ pub fn restore_package(
|
|||||||
let label = format!("{}::{}", nested_label.display(), entry_name.display());
|
let label = format!("{}::{}", nested_label.display(), entry_name.display());
|
||||||
let dest = out_root.join(&base).join(crate::job::out_name(&entry_name));
|
let dest = out_root.join(&base).join(crate::job::out_name(&entry_name));
|
||||||
let mut entry_outcomes = restore_package_entry(
|
let mut entry_outcomes = restore_package_entry(
|
||||||
&nested_path,
|
&mut nested_archive,
|
||||||
nested_index,
|
nested_index,
|
||||||
&label,
|
&label,
|
||||||
&dest,
|
&dest,
|
||||||
@@ -294,8 +310,8 @@ pub fn restore_package(
|
|||||||
/// Probe one extracted package entry and restore it when it is a target.
|
/// Probe one extracted package entry and restore it when it is a target.
|
||||||
/// Returns one outcome per produced/consumed artifact: the entry itself, plus
|
/// Returns one outcome per produced/consumed artifact: the entry itself, plus
|
||||||
/// an `EmbeddedMetadata` outcome when the restored library carried a blob.
|
/// an `EmbeddedMetadata` outcome when the restored library carried a blob.
|
||||||
fn restore_package_entry(
|
fn restore_package_entry<R: Read + Seek>(
|
||||||
package: &Path,
|
archive: &mut ZipArchive<R>,
|
||||||
index: usize,
|
index: usize,
|
||||||
label: &str,
|
label: &str,
|
||||||
dest: &Path,
|
dest: &Path,
|
||||||
@@ -303,11 +319,13 @@ fn restore_package_entry(
|
|||||||
seen: &mut HashSet<String>,
|
seen: &mut HashSet<String>,
|
||||||
verbose: bool,
|
verbose: bool,
|
||||||
) -> Result<Vec<EntryOutcome>> {
|
) -> Result<Vec<EntryOutcome>> {
|
||||||
let entry_path = extract_entry(package, index, temporary, Path::new(label))?;
|
let entry_path = extract_entry(archive, index, temporary, Path::new(label))?;
|
||||||
let data = std::fs::read(&entry_path).with_context(|| format!("read extracted `{label}`"))?;
|
let entry_file =
|
||||||
|
File::open(&entry_path).with_context(|| format!("open extracted `{label}`"))?;
|
||||||
|
let entry_data = map_read_only(&entry_file, &entry_path)?;
|
||||||
|
|
||||||
let is_so = is_elf64_aarch64(&data) && is_protected_libil2cpp(&data);
|
let is_so = is_elf64_aarch64(&entry_data) && is_protected_libil2cpp(&entry_data);
|
||||||
let is_meta = !is_so && senbei_metadata::is_metadata(&data);
|
let is_meta = !is_so && senbei_metadata::is_metadata(&entry_data);
|
||||||
let outcome = |kind, status| EntryOutcome {
|
let outcome = |kind, status| EntryOutcome {
|
||||||
label: label.to_owned(),
|
label: label.to_owned(),
|
||||||
dest: dest.to_path_buf(),
|
dest: dest.to_path_buf(),
|
||||||
@@ -317,7 +335,7 @@ fn restore_package_entry(
|
|||||||
if !is_so && !is_meta {
|
if !is_so && !is_meta {
|
||||||
return Ok(vec![outcome(EntryKind::So, EntryStatus::NotTarget)]);
|
return Ok(vec![outcome(EntryKind::So, EntryStatus::NotTarget)]);
|
||||||
}
|
}
|
||||||
if !seen.insert(content_identity(&data)) {
|
if !seen.insert(content_identity(&entry_data)) {
|
||||||
let kind = if is_so {
|
let kind = if is_so {
|
||||||
EntryKind::So
|
EntryKind::So
|
||||||
} else {
|
} else {
|
||||||
@@ -327,7 +345,8 @@ fn restore_package_entry(
|
|||||||
}
|
}
|
||||||
|
|
||||||
if is_so {
|
if is_so {
|
||||||
drop(data);
|
drop(entry_data);
|
||||||
|
drop(entry_file);
|
||||||
return Ok(match restore_so_file(&entry_path, dest, verbose) {
|
return Ok(match restore_so_file(&entry_path, dest, verbose) {
|
||||||
Ok(embedded) => {
|
Ok(embedded) => {
|
||||||
let mut outcomes = vec![outcome(EntryKind::So, EntryStatus::Restored)];
|
let mut outcomes = vec![outcome(EntryKind::So, EntryStatus::Restored)];
|
||||||
@@ -352,7 +371,7 @@ fn restore_package_entry(
|
|||||||
|
|
||||||
// Metadata entry: write only when the restore actually changed tokens —
|
// Metadata entry: write only when the restore actually changed tokens —
|
||||||
// a clean blob needs no copy (same contract as loose metadata files).
|
// a clean blob needs no copy (same contract as loose metadata files).
|
||||||
let kind_and_status = match restore_metadata_bytes(&data) {
|
let kind_and_status = match restore_metadata_bytes(&entry_data) {
|
||||||
Ok((out, report)) if report.remapped > 0 => {
|
Ok((out, report)) if report.remapped > 0 => {
|
||||||
let kind = EntryKind::Metadata {
|
let kind = EntryKind::Metadata {
|
||||||
remapped: report.remapped,
|
remapped: report.remapped,
|
||||||
@@ -396,31 +415,24 @@ fn open_package(path: &Path) -> Result<ZipArchive<std::fs::File>> {
|
|||||||
ZipArchive::new(file).with_context(|| format!("read package `{}`", path.display()))
|
ZipArchive::new(file).with_context(|| format!("read package `{}`", path.display()))
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Extract one package entry to the temporary workspace, streaming stored
|
/// Stream one package entry to a temporary, seekable file. The Android engine
|
||||||
/// entries and inflating deflated ones by hand so compression-method
|
/// needs random access to ELF section tables, while the ZIP reader itself is
|
||||||
/// surprises fail loudly instead of producing a truncated file.
|
/// consumed directly without creating an in-memory compressed or decompressed
|
||||||
fn extract_entry(
|
/// copy.
|
||||||
package: &Path,
|
fn extract_entry<R: Read + Seek>(
|
||||||
|
archive: &mut ZipArchive<R>,
|
||||||
index: usize,
|
index: usize,
|
||||||
temporary: &tempfile::TempDir,
|
temporary: &tempfile::TempDir,
|
||||||
label: &Path,
|
label: &Path,
|
||||||
) -> Result<PathBuf> {
|
) -> Result<PathBuf> {
|
||||||
let mut archive = open_package(package)?;
|
let mut entry = archive.by_index(index)?;
|
||||||
let mut entry = archive.by_index_raw(index)?;
|
|
||||||
let key = format!("{}-{index:08x}", label.display());
|
let key = format!("{}-{index:08x}", label.display());
|
||||||
// `:` appears in `package::entry` labels and is invalid in Windows file
|
// `:` appears in `package::entry` labels and is invalid in Windows file
|
||||||
// names; sanitize every path-ish separator.
|
// names; sanitize every path-ish separator.
|
||||||
let destination = temporary.path().join(key.replace(['\\', '/', ':'], "_"));
|
let destination = temporary.path().join(key.replace(['\\', '/', ':'], "_"));
|
||||||
let output_size = entry.size();
|
let output_size = entry.size();
|
||||||
let mut output = BufWriter::new(std::fs::File::create(&destination)?);
|
let mut output = BufWriter::new(std::fs::File::create(&destination)?);
|
||||||
let written = match entry.compression() {
|
let written = std::io::copy(&mut entry, &mut output)?;
|
||||||
zip::CompressionMethod::Stored => std::io::copy(&mut entry, &mut output)?,
|
|
||||||
zip::CompressionMethod::Deflated => {
|
|
||||||
let mut decoder = DeflateDecoder::new(&mut entry);
|
|
||||||
std::io::copy(&mut decoder, &mut output)?
|
|
||||||
}
|
|
||||||
method => bail!("unsupported compression method {method:?} in entry `{key}`"),
|
|
||||||
};
|
|
||||||
output.flush()?;
|
output.flush()?;
|
||||||
if written != output_size {
|
if written != output_size {
|
||||||
bail!(
|
bail!(
|
||||||
@@ -430,6 +442,13 @@ fn extract_entry(
|
|||||||
}
|
}
|
||||||
Ok(destination)
|
Ok(destination)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
fn map_read_only(file: &File, path: &Path) -> Result<Mmap> {
|
||||||
|
// SAFETY: the file descriptor remains open for the returned mapping's
|
||||||
|
// lifetime, and this mapping is read-only.
|
||||||
|
unsafe { MmapOptions::new().map(file) }
|
||||||
|
.with_context(|| format!("map extracted `{}`", path.display()))
|
||||||
|
}
|
||||||
/// Lowercase hex of a digest output (sha2 0.11's `Array` no longer formats as
|
/// Lowercase hex of a digest output (sha2 0.11's `Array` no longer formats as
|
||||||
/// hex directly).
|
/// hex directly).
|
||||||
fn hex_digest(data: &[u8]) -> String {
|
fn hex_digest(data: &[u8]) -> String {
|
||||||
|
|||||||
+6
-11
@@ -403,11 +403,8 @@ pub fn run_folder_v(
|
|||||||
|
|
||||||
/// Like [`run_folder_v`], but with the scan pre-filter explicitly controlled.
|
/// Like [`run_folder_v`], but with the scan pre-filter explicitly controlled.
|
||||||
///
|
///
|
||||||
/// When `scan_all` is true every regular file under `root` is opened and
|
/// When `scan_all` is true selected target names below the minimum size are
|
||||||
/// content-probed, instead of skipping ones the free directory metadata already
|
/// also opened and content-probed. Other filenames are never opened.
|
||||||
/// rules out (extensionless, too small to hold a Crackproof key table, or a
|
|
||||||
/// bulk-asset extension). See [`crate::scan::find_targets_opts`] — exhaustive
|
|
||||||
/// scanning is dramatically slower on asset-heavy trees.
|
|
||||||
pub fn run_folder_opts(
|
pub fn run_folder_opts(
|
||||||
root: &Path,
|
root: &Path,
|
||||||
out_dir: Option<&Path>,
|
out_dir: Option<&Path>,
|
||||||
@@ -549,8 +546,8 @@ pub fn run_folder_opts(
|
|||||||
let dest = out_root.join(out_name(&rel));
|
let dest = out_root.join(out_name(&rel));
|
||||||
// Unreadable here is fine: the restore reports the same error.
|
// Unreadable here is fine: the restore reports the same error.
|
||||||
if android_dedup
|
if android_dedup
|
||||||
&& let Ok(bytes) = std::fs::read(input)
|
&& let Ok(identity) = crate::android::file_content_identity(input)
|
||||||
&& !android_seen.insert(crate::android::content_identity(&bytes))
|
&& !android_seen.insert(identity)
|
||||||
{
|
{
|
||||||
s.skipped += 1;
|
s.skipped += 1;
|
||||||
if let Some(log) = &log {
|
if let Some(log) = &log {
|
||||||
@@ -816,10 +813,8 @@ pub fn run_file_v(
|
|||||||
// library probe needs the whole file (its payload section is found through
|
// library probe needs the whole file (its payload section is found through
|
||||||
// the section-header table at the end), while a package is a container
|
// the section-header table at the end), while a package is a container
|
||||||
// handled entry-by-entry. Anything else falls through to the PE pipeline.
|
// handled entry-by-entry. Anything else falls through to the PE pipeline.
|
||||||
let is_android_so = crate::android::is_elf64_aarch64(&prefix)
|
let is_android_so =
|
||||||
&& std::fs::read(input)
|
crate::android::is_elf64_aarch64(&prefix) && crate::android::is_protected_so_file(input);
|
||||||
.map(|bytes| senbei_engine::android::is_protected_libil2cpp(&bytes))
|
|
||||||
.unwrap_or(false);
|
|
||||||
let is_android_package = !is_android_so && crate::android::is_app_package(input, &prefix);
|
let is_android_package = !is_android_so && crate::android::is_app_package(input, &prefix);
|
||||||
|
|
||||||
if is_meta {
|
if is_meta {
|
||||||
|
|||||||
+55
-116
@@ -1,4 +1,6 @@
|
|||||||
|
use memmap2::MmapOptions;
|
||||||
use senbei_engine::detect;
|
use senbei_engine::detect;
|
||||||
|
use std::fs::File;
|
||||||
use std::io::Read;
|
use std::io::Read;
|
||||||
use std::path::{Path, PathBuf};
|
use std::path::{Path, PathBuf};
|
||||||
use walkdir::WalkDir;
|
use walkdir::WalkDir;
|
||||||
@@ -77,78 +79,6 @@ pub(crate) fn is_android_entry_name(path: &Path) -> bool {
|
|||||||
.is_some_and(|ext| ext.eq_ignore_ascii_case("so"))
|
.is_some_and(|ext| ext.eq_ignore_ascii_case("so"))
|
||||||
}
|
}
|
||||||
|
|
||||||
/// File extensions that are bulk data by construction and can never be a target.
|
|
||||||
///
|
|
||||||
/// Set `SENBEI_SCAN_ALL=1` (or pass `--scan-all`) to probe every file regardless.
|
|
||||||
const DENY_EXT: &[&str] = &[
|
|
||||||
// Unity and other engine asset containers
|
|
||||||
"ab",
|
|
||||||
"bundle",
|
|
||||||
"unity3d",
|
|
||||||
"manifest",
|
|
||||||
"resource",
|
|
||||||
"ress",
|
|
||||||
"assets",
|
|
||||||
"sharedassets",
|
|
||||||
// audio / video / image / font
|
|
||||||
"acb",
|
|
||||||
"awb",
|
|
||||||
"usm",
|
|
||||||
"wav",
|
|
||||||
"ogg",
|
|
||||||
"mp3",
|
|
||||||
"mp4",
|
|
||||||
"avi",
|
|
||||||
"png",
|
|
||||||
"jpg",
|
|
||||||
"jpeg",
|
|
||||||
"bmp",
|
|
||||||
"gif",
|
|
||||||
"tga",
|
|
||||||
"dds",
|
|
||||||
"svg",
|
|
||||||
"ttf",
|
|
||||||
"otf",
|
|
||||||
// text, markup, config, logs
|
|
||||||
"xml",
|
|
||||||
"json",
|
|
||||||
"txt",
|
|
||||||
"csv",
|
|
||||||
"md",
|
|
||||||
"toml",
|
|
||||||
"ini",
|
|
||||||
"yml",
|
|
||||||
"yaml",
|
|
||||||
"log",
|
|
||||||
"html",
|
|
||||||
"htm",
|
|
||||||
"css",
|
|
||||||
"aspx",
|
|
||||||
"browser",
|
|
||||||
"config",
|
|
||||||
"sig",
|
|
||||||
"map",
|
|
||||||
"pdb",
|
|
||||||
// rhythm-game chart/score data
|
|
||||||
"ma2",
|
|
||||||
"sr",
|
|
||||||
];
|
|
||||||
|
|
||||||
/// Whether `path` can be skipped from its name alone.
|
|
||||||
fn denied_name(path: &Path) -> bool {
|
|
||||||
let Some(ext) = path.extension() else {
|
|
||||||
return true;
|
|
||||||
};
|
|
||||||
let Some(ext) = ext.to_str() else {
|
|
||||||
return false;
|
|
||||||
};
|
|
||||||
// Extensions are ASCII in practice; compare case-insensitively without
|
|
||||||
// allocating for the overwhelmingly common non-match.
|
|
||||||
DENY_EXT
|
|
||||||
.iter()
|
|
||||||
.any(|d| d.len() == ext.len() && d.eq_ignore_ascii_case(ext))
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Content classification of a single file.
|
/// Content classification of a single file.
|
||||||
#[derive(Clone, Copy, PartialEq, Eq)]
|
#[derive(Clone, Copy, PartialEq, Eq)]
|
||||||
enum Class {
|
enum Class {
|
||||||
@@ -197,9 +127,9 @@ pub struct ScanResult {
|
|||||||
/// So the only lever is **probing fewer files**, which is what the target-name
|
/// So the only lever is **probing fewer files**, which is what the target-name
|
||||||
/// filter and [`MIN_SIZE`] do — both decided before any file is opened.
|
/// filter and [`MIN_SIZE`] do — both decided before any file is opened.
|
||||||
///
|
///
|
||||||
/// The surviving probes (open + short read + magic test) are fanned out across
|
/// The selected probes (open + short read + magic test) are fanned out across
|
||||||
/// worker threads. Directory traversal itself stays serial (one cheap `readdir`
|
/// worker threads. Directory traversal itself stays serial because it only
|
||||||
/// pass, no file opens) because it feeds the parallel probe.
|
/// collects names and sizes before the parallel probe.
|
||||||
///
|
///
|
||||||
/// Thread count follows [`senbei_engine::thread_cap`] (honoring
|
/// Thread count follows [`senbei_engine::thread_cap`] (honoring
|
||||||
/// `SENBEI_THREADS`, `1` = fully sequential). Output order is independent of
|
/// `SENBEI_THREADS`, `1` = fully sequential). Output order is independent of
|
||||||
@@ -227,8 +157,7 @@ pub struct ScanStats {
|
|||||||
}
|
}
|
||||||
|
|
||||||
/// [`find_targets`], but with the pre-filter explicitly controlled. When
|
/// [`find_targets`], but with the pre-filter explicitly controlled. When
|
||||||
/// `scan_all` is true every regular file is probed, restoring the exhaustive
|
/// `scan_all` is true selected target names below [`MIN_SIZE`] are also probed.
|
||||||
/// (and on asset-heavy trees, far slower) behavior.
|
|
||||||
pub fn find_targets_opts(root: &Path, scan_all: bool) -> ScanResult {
|
pub fn find_targets_opts(root: &Path, scan_all: bool) -> ScanResult {
|
||||||
// Phase 1: serial traversal collecting regular-file paths only. No file is
|
// Phase 1: serial traversal collecting regular-file paths only. No file is
|
||||||
// opened here; `readdir` is fast relative to the content probe that follows,
|
// opened here; `readdir` is fast relative to the content probe that follows,
|
||||||
@@ -275,11 +204,6 @@ pub fn find_targets_opts(root: &Path, scan_all: bool) -> ScanResult {
|
|||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
if !scan_all {
|
if !scan_all {
|
||||||
// Name checks come first so extensionless asset chunks never
|
|
||||||
// trigger even an explicit metadata query.
|
|
||||||
if denied_name(entry.path()) {
|
|
||||||
continue;
|
|
||||||
}
|
|
||||||
// Skip on directory metadata alone — never open these.
|
// Skip on directory metadata alone — never open these.
|
||||||
let too_small = entry
|
let too_small = entry
|
||||||
.metadata()
|
.metadata()
|
||||||
@@ -354,9 +278,9 @@ fn is_reparse_point(_e: &walkdir::DirEntry) -> bool {
|
|||||||
false
|
false
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Whether the scan pre-filter is disabled via `SENBEI_SCAN_ALL`. Any value
|
/// Whether the size pre-filter is disabled via `SENBEI_SCAN_ALL`. Any value
|
||||||
/// other than `0`/empty turns exhaustive scanning on. The `--scan-all` flag is
|
/// other than `0`/empty enables probing small selected target names. It never
|
||||||
/// ORed with this.
|
/// expands the platform filename boundary.
|
||||||
pub fn scan_all_env() -> bool {
|
pub fn scan_all_env() -> bool {
|
||||||
match std::env::var("SENBEI_SCAN_ALL") {
|
match std::env::var("SENBEI_SCAN_ALL") {
|
||||||
Ok(v) => !matches!(v.trim(), "" | "0"),
|
Ok(v) => !matches!(v.trim(), "" | "0"),
|
||||||
@@ -381,8 +305,7 @@ pub fn scan_all_env() -> bool {
|
|||||||
/// The Android library probe needs more than the prefix: the protection
|
/// The Android library probe needs more than the prefix: the protection
|
||||||
/// payload lives in a section found via the section-header table at the *end*
|
/// payload lives in a section found via the section-header table at the *end*
|
||||||
/// of the file, so an ELF64/AArch64 prefix triggers a full-file read. Only
|
/// of the file, so an ELF64/AArch64 prefix triggers a full-file read. Only
|
||||||
/// aarch64 images pay for it — a handful of `.so` files per app tree, against
|
/// selected `.so` images pay for it.
|
||||||
/// tens of thousands of assets the free name/size checks already rejected.
|
|
||||||
fn classify(path: &Path) -> Option<Class> {
|
fn classify(path: &Path) -> Option<Class> {
|
||||||
let head = read_prefix(path, DETECT_PREFIX)?;
|
let head = read_prefix(path, DETECT_PREFIX)?;
|
||||||
let r = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| {
|
let r = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| {
|
||||||
@@ -405,7 +328,12 @@ fn classify(path: &Path) -> Option<Class> {
|
|||||||
.and_then(|ext| ext.to_str())
|
.and_then(|ext| ext.to_str())
|
||||||
.is_some_and(|ext| ext.eq_ignore_ascii_case("so"))
|
.is_some_and(|ext| ext.eq_ignore_ascii_case("so"))
|
||||||
&& crate::android::is_elf64_aarch64(&head)
|
&& crate::android::is_elf64_aarch64(&head)
|
||||||
&& std::fs::read(path)
|
&& File::open(path)
|
||||||
|
.and_then(|file| {
|
||||||
|
// SAFETY: the file remains open for the mapping lifetime
|
||||||
|
// and the mapping is read-only.
|
||||||
|
unsafe { MmapOptions::new().map(&file) }
|
||||||
|
})
|
||||||
.map(|bytes| senbei_engine::android::is_protected_libil2cpp(&bytes))
|
.map(|bytes| senbei_engine::android::is_protected_libil2cpp(&bytes))
|
||||||
.unwrap_or(false)
|
.unwrap_or(false)
|
||||||
{
|
{
|
||||||
@@ -430,33 +358,30 @@ mod tests {
|
|||||||
use super::*;
|
use super::*;
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn denies_bulk_asset_extensions_case_insensitively() {
|
fn candidate_names_are_platform_specific() {
|
||||||
for p in ["a.ab", "a.XML", "a.Acb", "a.ma2", "a.manifest", "a.PNG"] {
|
for p in [
|
||||||
assert!(denied_name(Path::new(p)), "{p} should be denied");
|
"daemon.exe",
|
||||||
|
"GameLib.DLL",
|
||||||
|
"libil2cpp.so",
|
||||||
|
"global-metadata.dat",
|
||||||
|
] {
|
||||||
|
assert!(
|
||||||
|
is_metadata_name(Path::new(p)) || is_target_extension(Path::new(p)),
|
||||||
|
"{p} should be a candidate"
|
||||||
|
);
|
||||||
}
|
}
|
||||||
}
|
|
||||||
|
|
||||||
#[test]
|
|
||||||
fn denies_extensionless_files() {
|
|
||||||
for p in ["asset", "level0", "0123456789abcdef"] {
|
|
||||||
assert!(denied_name(Path::new(p)), "{p} should be denied");
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
#[test]
|
|
||||||
fn never_denies_what_a_target_can_be_named() {
|
|
||||||
// Unknown extensions must still be probed. This keeps the filter a
|
|
||||||
// narrow deny-list rather than an executable-extension allow-list.
|
|
||||||
for p in [
|
for p in [
|
||||||
"app.exe.bak",
|
"app.exe.bak",
|
||||||
"managed.dll.bak",
|
"managed.dll.bak",
|
||||||
"daemon.exe",
|
"libil2cpp.so.bak",
|
||||||
"GameLib.dll",
|
"global-metadata.bin",
|
||||||
"global-metadata.dat",
|
"asset",
|
||||||
"a.so",
|
"a.ab",
|
||||||
"a.bin",
|
|
||||||
] {
|
] {
|
||||||
assert!(!denied_name(Path::new(p)), "{p} must still be probed");
|
assert!(
|
||||||
|
!is_metadata_name(Path::new(p)) && !is_target_extension(Path::new(p)),
|
||||||
|
"{p} must not be a candidate"
|
||||||
|
);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -475,10 +400,10 @@ mod tests {
|
|||||||
assert!(exhaustive.metadata.is_empty());
|
assert!(exhaustive.metadata.is_empty());
|
||||||
}
|
}
|
||||||
|
|
||||||
/// A file below the Crackproof key-table bound is skipped without being
|
/// A selected file below the Crackproof key-table bound is skipped without
|
||||||
/// opened, but a large non-asset file is still probed.
|
/// being opened, while `scan_all` probes it.
|
||||||
#[test]
|
#[test]
|
||||||
fn prefilter_skips_small_and_denied_files_only() {
|
fn prefilter_skips_small_selected_files_only() {
|
||||||
let td = tempfile::tempdir().unwrap();
|
let td = tempfile::tempdir().unwrap();
|
||||||
let root = td.path();
|
let root = td.path();
|
||||||
std::fs::write(root.join("tiny.dll"), vec![0u8; 100]).unwrap();
|
std::fs::write(root.join("tiny.dll"), vec![0u8; 100]).unwrap();
|
||||||
@@ -486,15 +411,15 @@ mod tests {
|
|||||||
std::fs::write(root.join("plain.dll"), vec![0u8; 100_000]).unwrap();
|
std::fs::write(root.join("plain.dll"), vec![0u8; 100_000]).unwrap();
|
||||||
|
|
||||||
// None of them are Crackproof, so both modes find nothing; the point is
|
// None of them are Crackproof, so both modes find nothing; the point is
|
||||||
// that the filtered walk does not panic and honors `scan_all`.
|
// that only the selected names are considered and `scan_all` controls
|
||||||
|
// the size floor.
|
||||||
let scan = find_targets_opts(root, false);
|
let scan = find_targets_opts(root, false);
|
||||||
assert!(scan.crackproof.is_empty() && scan.metadata.is_empty());
|
assert!(scan.crackproof.is_empty() && scan.metadata.is_empty());
|
||||||
let scan = find_targets_opts(root, true);
|
let scan = find_targets_opts(root, true);
|
||||||
assert!(scan.crackproof.is_empty() && scan.metadata.is_empty());
|
assert!(scan.crackproof.is_empty() && scan.metadata.is_empty());
|
||||||
}
|
}
|
||||||
|
|
||||||
/// An il2cpp metadata blob is found by the filtered scan: `.dat` is not on
|
/// An exact `global-metadata.dat` name is found by the filtered scan.
|
||||||
/// the deny-list and a real one is far above `MIN_SIZE`.
|
|
||||||
#[test]
|
#[test]
|
||||||
fn finds_metadata_through_the_prefilter() {
|
fn finds_metadata_through_the_prefilter() {
|
||||||
let td = tempfile::tempdir().unwrap();
|
let td = tempfile::tempdir().unwrap();
|
||||||
@@ -549,4 +474,18 @@ mod tests {
|
|||||||
assert_eq!(scan.stats.skipped, 1, "only the stub was probed");
|
assert_eq!(scan.stats.skipped, 1, "only the stub was probed");
|
||||||
assert!(is_windows_companion(&root.join("app.exe._")));
|
assert!(is_windows_companion(&root.join("app.exe._")));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn scan_all_keeps_the_platform_name_boundary() {
|
||||||
|
let td = tempfile::tempdir().unwrap();
|
||||||
|
let root = td.path();
|
||||||
|
let mut metadata = vec![0_u8; MIN_SIZE as usize];
|
||||||
|
metadata[..4].copy_from_slice(&0xFAB1_1BAFu32.to_le_bytes());
|
||||||
|
std::fs::write(root.join("renamed.bin"), &metadata).unwrap();
|
||||||
|
std::fs::write(root.join("global-metadata.dat"), &metadata).unwrap();
|
||||||
|
|
||||||
|
let scan = find_targets_opts(root, true);
|
||||||
|
assert_eq!(scan.metadata.len(), 1);
|
||||||
|
assert!(scan.metadata[0].ends_with("global-metadata.dat"));
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
Generated
+1
-19
@@ -2,12 +2,6 @@
|
|||||||
# It is not intended for manual editing.
|
# It is not intended for manual editing.
|
||||||
version = 4
|
version = 4
|
||||||
|
|
||||||
[[package]]
|
|
||||||
name = "adler2"
|
|
||||||
version = "2.0.1"
|
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
||||||
checksum = "320119579fcad9c21884f5c4861d16174d0e06250625266f50fe6898340abefa"
|
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "aes"
|
name = "aes"
|
||||||
version = "0.9.3"
|
version = "0.9.3"
|
||||||
@@ -168,8 +162,6 @@ version = "1.1.10"
|
|||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
checksum = "6e634e2e0ebac1ee034020da1ca582e17ffe4e0f5e985823721e168928136dcb"
|
checksum = "6e634e2e0ebac1ee034020da1ca582e17ffe4e0f5e985823721e168928136dcb"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"crc32fast",
|
|
||||||
"miniz_oxide",
|
|
||||||
"zlib-rs",
|
"zlib-rs",
|
||||||
]
|
]
|
||||||
|
|
||||||
@@ -316,16 +308,6 @@ dependencies = [
|
|||||||
"libc",
|
"libc",
|
||||||
]
|
]
|
||||||
|
|
||||||
[[package]]
|
|
||||||
name = "miniz_oxide"
|
|
||||||
version = "0.9.1"
|
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
||||||
checksum = "b63fbc4a50860e98e7b2aa7804ded1db5cbc3aff9193adaff57a6931bf7c4b4c"
|
|
||||||
dependencies = [
|
|
||||||
"adler2",
|
|
||||||
"simd-adler32",
|
|
||||||
]
|
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "once_cell"
|
name = "once_cell"
|
||||||
version = "1.21.4"
|
version = "1.21.4"
|
||||||
@@ -455,9 +437,9 @@ name = "senbei-io"
|
|||||||
version = "1.2.0"
|
version = "1.2.0"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"anyhow",
|
"anyhow",
|
||||||
"flate2",
|
|
||||||
"indicatif",
|
"indicatif",
|
||||||
"libc",
|
"libc",
|
||||||
|
"memmap2",
|
||||||
"owo-colors",
|
"owo-colors",
|
||||||
"senbei-engine",
|
"senbei-engine",
|
||||||
"senbei-metadata",
|
"senbei-metadata",
|
||||||
|
|||||||
Reference in New Issue
Block a user