mokurin000/sdgb-utils-rs
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

perf: cache for players scraping

Browse Source
This commit is contained in:
mokurin000
2025-07-30 19:06:17 +08:00
parent 19a0d53624
commit 5ad0135deb
8 changed files with 136 additions and 16 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
.gitignore vendored
View File

@@ -1,3 +1,4 @@
/target
/*.txt
/players.redb

47
Cargo.lock generated
View File

@@ -97,6 +97,26 @@ dependencies = [
"windows-targets 0.52.6",
]
[[package]]
name = "bincode"
version = "2.0.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "36eaf5d7b090263e8150820482d5d93cd964a81e4019913c972f4edcc6edb740"
dependencies = [
"bincode_derive",
"serde",
"unty",
]
[[package]]
name = "bincode_derive"
version = "2.0.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "bf95709a440f45e986983918d0e8a1f30a9b1df04918fc828670606804ac3c09"
dependencies = [
"virtue",
]
[[package]]
name = "bitflags"
version = "2.9.1"
@@ -553,7 +573,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "778e2ac28f6c47af28e4907f13ffd1e1ddbd400980a9abd7c8df189bf578a5ad"
dependencies = [
"libc",
"windows-sys 0.59.0",
"windows-sys 0.60.2",
]
[[package]]
@@ -1237,6 +1257,15 @@ dependencies = [
"getrandom 0.3.3",
]
[[package]]
name = "redb"
version = "2.6.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "fef838cd981b5c46e9e91e20e4623e43b29b5c251eb245b34da0cbd2da09ab27"
dependencies = [
"libc",
]
[[package]]
name = "ref-cast"
version = "1.0.24"
@@ -1302,7 +1331,7 @@ dependencies = [
"errno",
"libc",
"linux-raw-sys",
"windows-sys 0.59.0",
"windows-sys 0.60.2",
]
[[package]]
@@ -1377,6 +1406,7 @@ name = "sdgb-api"
version = "0.1.0"
dependencies = [
"aes",
"bincode",
"cbc",
"chrono",
"cipher",
@@ -1401,6 +1431,7 @@ dependencies = [
"futures-util",
"nyquest-preset",
"palc",
"redb",
"sdgb-api",
"serde_json",
"snafu",
@@ -1753,6 +1784,12 @@ version = "0.9.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8ecb6da28b8a351d773b68d5825ac39017e680750f980f3a1a85cd8dd28a47c1"
[[package]]
name = "unty"
version = "0.0.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6d49784317cd0d1ee7ec5c716dd598ec5b4483ea832a2dced265471cc0f690ae"
[[package]]
name = "vcpkg"
version = "0.2.15"
@@ -1765,6 +1802,12 @@ version = "0.9.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a"
[[package]]
name = "virtue"
version = "0.0.18"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "051eb1abcf10076295e815102942cc58f9d5e3b4560e46e53c21e8ff6f3af7b1"
[[package]]
name = "wasi"
version = "0.11.1+wasi-snapshot-preview1"

4
sdgb-api/Cargo.toml
View File

@@ -6,9 +6,10 @@ edition = "2024"
license = "GPL-3.0"
[features]
default = ["compio"]
default = ["compio", "bincode"]
compio = ["dep:compio"]
tokio = ["dep:tokio"]
bincode = ["dep:bincode"]
[dependencies]
snafu = { workspace = true }
@@ -36,3 +37,4 @@ flate2 = "1.1.2"
cbc = "0.1.2"
aes = "0.8.4"
cipher = { version = "0.4.4", features = ["block-padding"] }
bincode = { version = "2.0.1", optional = true }

2
sdgb-api/src/lib.rs
View File

@@ -4,3 +4,5 @@ pub mod title;
mod error;
pub use error::ApiError;
pub use bincode;

5
sdgb-api/src/title/model/get_user_preview_api/mod.rs
View File

@@ -1,5 +1,6 @@
use std::fmt::Display;
use bincode::{Decode, Encode};
use serde::{Deserialize, Serialize};
#[derive(Serialize)]
@@ -8,10 +9,10 @@ pub struct GetUserPreviewApi {
pub user_id: u32,
}
#[derive(Debug, Clone, Serialize, Deserialize)]
#[derive(Debug, Clone, Serialize, Deserialize, Encode, Decode)]
#[serde(rename_all = "camelCase")]
pub struct GetUserPreviewApiResp {
pub user_id: i64,
pub user_id: u32,
pub user_name: String,
pub is_login: bool,
pub last_rom_version: String,

8
sdgb-cli/Cargo.toml
View File

@@ -6,12 +6,15 @@ authors = ["mokurin000"]
description = "CLI tool for SDGB protocol"
[features]
default = ["compio"]
default = ["compio", "cache"]
compio = ["dep:compio", "sdgb-api/compio"]
tokio = ["dep:tokio", "sdgb-api/tokio"]
cache = ["dep:redb"]
[dependencies]
sdgb-api = { workspace = true }
sdgb-api = { workspace = true, features = ["bincode"] }
snafu = { workspace = true }
serde_json = { workspace = true }
@@ -27,3 +30,4 @@ spdlog-rs = { version = "0.4.3", default-features = false, features = [
"release-level-info",
] }
futures-util = "0.3.31"
redb = { version = "2.6.1", optional = true }

25
sdgb-cli/src/cache/mod.rs vendored Normal file
View File

@@ -0,0 +1,25 @@
use std::sync::LazyLock;
use redb::{Table, TableDefinition, WriteTransaction};
static DATABASE: LazyLock<redb::Database> = LazyLock::new(|| {
redb::Database::builder()
.create("players.redb")
.expect("failed to open database")
});
const DIFINITION: TableDefinition<'_, u32, Vec<u8>> = redb::TableDefinition::new("players");
pub fn write_txn() -> Result<WriteTransaction, redb::Error> {
Ok(DATABASE.begin_write()?)
}
pub fn open_table(write: &WriteTransaction) -> Result<Table<'_, u32, Vec<u8>>, redb::Error> {
Ok(write.open_table(DIFINITION)?)
}
pub fn init_db() -> Result<(), redb::Error> {
let write_txn = DATABASE.begin_write()?;
write_txn.open_table(DIFINITION)?;
write_txn.commit()?;
Ok(())
}

58
sdgb-cli/src/main.rs
View File

@@ -6,6 +6,7 @@ use std::{
use futures_util::StreamExt;
use nyquest_preset::nyquest::ClientBuilder;
use palc::Parser;
use sdgb_api::{
ApiError,
all_net::QRCode,
@@ -23,6 +24,8 @@ use spdlog::{error, info, warn};
use crate::commands::Cli;
#[cfg(feature = "cache")]
mod cache;
mod commands;
#[cfg_attr(feature = "compio", compio::main)]
@@ -102,9 +105,33 @@ async fn main() -> Result<(), Box<dyn snafu::Error>> {
user_ids.push(user_id);
}
let users = futures_util::stream::iter(user_ids)
#[cfg(feature = "cache")]
let _ = cache::init_db();
#[cfg(feature = "cache")]
let write = cache::write_txn()?;
#[cfg(feature = "cache")]
let config = sdgb_api::bincode::config::Configuration::<
sdgb_api::bincode::config::LittleEndian,
>::default()
.with_no_limit();
let players = futures_util::stream::iter(user_ids)
.map(async |user_id| {
info!("preview: {user_id}");
#[cfg(feature = "cache")]
{
use redb::ReadableTable;
use sdgb_api::bincode::borrow_decode_from_slice;
let cache_table = cache::open_table(&write)?;
let data = cache_table.get(user_id)?;
if let Some(data) = data {
let decoded: (GetUserPreviewApiResp, _) =
borrow_decode_from_slice(&data.value(), config)?;
return Ok(decoded.0);
}
}
let resp = Sdgb1_50::request::<_, GetUserPreviewApiResp>(
&client,
APIMethod::GetUserPreviewApi,
@@ -113,15 +140,30 @@ async fn main() -> Result<(), Box<dyn snafu::Error>> {
)
.await;
if let Err(e) = &resp {
if matches!(e, ApiError::JSON { .. }) {
match &resp {
Ok(resp) => {
info!("preview: {user_id} succeed");
#[cfg(feature = "cache")]
{
use sdgb_api::bincode::encode_to_vec;
if let Ok(mut table) = cache::open_table(&write)
&& let Ok(encoded) = encode_to_vec(resp, config)
{
_ = table.insert(resp.user_id, encoded);
}
}
}
Err(ApiError::JSON { .. }) => {
warn!("account unregistered: {user_id}");
} else {
}
Err(e) => {
error!("preview failed: {e}");
}
}
resp
Result::<_, Box<dyn snafu::Error>>::Ok(resp?)
})
.buffer_unordered(20)
.filter_map(async |r| r.ok())
@@ -132,8 +174,8 @@ async fn main() -> Result<(), Box<dyn snafu::Error>> {
.write(true)
.truncate(true)
.create(true)
.open("users.json")?;
serde_json::to_writer_pretty(output, &users)?;
.open("players.json")?;
serde_json::to_writer_pretty(output, &players)?;
}
}