perf: cache for players scraping

.gitignore

@@ -1,3 +1,4 @@
 /target
 
 /*.txt
+/players.redb

Cargo.lock

@@ -97,6 +97,26 @@ dependencies = [
  "windows-targets 0.52.6",
 ]
 
+[[package]]
+name = "bincode"
+version = "2.0.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "36eaf5d7b090263e8150820482d5d93cd964a81e4019913c972f4edcc6edb740"
+dependencies = [
+ "bincode_derive",
+ "serde",
+ "unty",
+]
+
+[[package]]
+name = "bincode_derive"
+version = "2.0.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "bf95709a440f45e986983918d0e8a1f30a9b1df04918fc828670606804ac3c09"
+dependencies = [
+ "virtue",
+]
+
 [[package]]
 name = "bitflags"
 version = "2.9.1"
@@ -553,7 +573,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "778e2ac28f6c47af28e4907f13ffd1e1ddbd400980a9abd7c8df189bf578a5ad"
 dependencies = [
  "libc",
- "windows-sys 0.59.0",
+ "windows-sys 0.60.2",
 ]
 
 [[package]]
@@ -1237,6 +1257,15 @@ dependencies = [
  "getrandom 0.3.3",
 ]
 
+[[package]]
+name = "redb"
+version = "2.6.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "fef838cd981b5c46e9e91e20e4623e43b29b5c251eb245b34da0cbd2da09ab27"
+dependencies = [
+ "libc",
+]
+
 [[package]]
 name = "ref-cast"
 version = "1.0.24"
@@ -1302,7 +1331,7 @@ dependencies = [
  "errno",
  "libc",
  "linux-raw-sys",
- "windows-sys 0.59.0",
+ "windows-sys 0.60.2",
 ]
 
 [[package]]
@@ -1377,6 +1406,7 @@ name = "sdgb-api"
 version = "0.1.0"
 dependencies = [
  "aes",
+ "bincode",
  "cbc",
  "chrono",
  "cipher",
@@ -1401,6 +1431,7 @@ dependencies = [
  "futures-util",
  "nyquest-preset",
  "palc",
+ "redb",
  "sdgb-api",
  "serde_json",
  "snafu",
@@ -1753,6 +1784,12 @@ version = "0.9.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "8ecb6da28b8a351d773b68d5825ac39017e680750f980f3a1a85cd8dd28a47c1"
 
+[[package]]
+name = "unty"
+version = "0.0.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "6d49784317cd0d1ee7ec5c716dd598ec5b4483ea832a2dced265471cc0f690ae"
+
 [[package]]
 name = "vcpkg"
 version = "0.2.15"
@@ -1765,6 +1802,12 @@ version = "0.9.5"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a"
 
+[[package]]
+name = "virtue"
+version = "0.0.18"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "051eb1abcf10076295e815102942cc58f9d5e3b4560e46e53c21e8ff6f3af7b1"
+
 [[package]]
 name = "wasi"
 version = "0.11.1+wasi-snapshot-preview1"

sdgb-api/Cargo.toml

@@ -6,9 +6,10 @@ edition = "2024"
 license = "GPL-3.0"
 
 [features]
-default = ["compio"]
+default = ["compio", "bincode"]
 compio = ["dep:compio"]
 tokio = ["dep:tokio"]
+bincode = ["dep:bincode"]
 
 [dependencies]
 snafu = { workspace = true }
@@ -36,3 +37,4 @@ flate2 = "1.1.2"
 cbc = "0.1.2"
 aes = "0.8.4"
 cipher = { version = "0.4.4", features = ["block-padding"] }
+bincode = { version = "2.0.1", optional = true }

sdgb-api/src/lib.rs

@@ -4,3 +4,5 @@ pub mod title;
 
 mod error;
 pub use error::ApiError;
+
+pub use bincode;

sdgb-api/src/title/model/get_user_preview_api/mod.rs

@@ -1,5 +1,6 @@
 use std::fmt::Display;
 
+use bincode::{Decode, Encode};
 use serde::{Deserialize, Serialize};
 
 #[derive(Serialize)]
@@ -8,10 +9,10 @@ pub struct GetUserPreviewApi {
     pub user_id: u32,
 }
 
-#[derive(Debug, Clone, Serialize, Deserialize)]
+#[derive(Debug, Clone, Serialize, Deserialize, Encode, Decode)]
 #[serde(rename_all = "camelCase")]
 pub struct GetUserPreviewApiResp {
-    pub user_id: i64,
+    pub user_id: u32,
     pub user_name: String,
     pub is_login: bool,
     pub last_rom_version: String,

sdgb-cli/Cargo.toml

@@ -6,12 +6,15 @@ authors = ["mokurin000"]
 description = "CLI tool for SDGB protocol"
 
 [features]
-default = ["compio"]
+default = ["compio", "cache"]
+
 compio = ["dep:compio", "sdgb-api/compio"]
 tokio = ["dep:tokio", "sdgb-api/tokio"]
 
+cache = ["dep:redb"]
+
 [dependencies]
-sdgb-api = { workspace = true }
+sdgb-api = { workspace = true, features = ["bincode"] }
 
 snafu = { workspace = true }
 serde_json = { workspace = true }
@@ -27,3 +30,4 @@ spdlog-rs = { version = "0.4.3", default-features = false, features = [
     "release-level-info",
 ] }
 futures-util = "0.3.31"
+redb = { version = "2.6.1", optional = true }

sdgb-cli/src/cache/mod.rs

@@ -0,0 +1,25 @@
+use std::sync::LazyLock;
+
+use redb::{Table, TableDefinition, WriteTransaction};
+
+static DATABASE: LazyLock<redb::Database> = LazyLock::new(|| {
+    redb::Database::builder()
+        .create("players.redb")
+        .expect("failed to open database")
+});
+const DIFINITION: TableDefinition<'_, u32, Vec<u8>> = redb::TableDefinition::new("players");
+
+pub fn write_txn() -> Result<WriteTransaction, redb::Error> {
+    Ok(DATABASE.begin_write()?)
+}
+
+pub fn open_table(write: &WriteTransaction) -> Result<Table<'_, u32, Vec<u8>>, redb::Error> {
+    Ok(write.open_table(DIFINITION)?)
+}
+
+pub fn init_db() -> Result<(), redb::Error> {
+    let write_txn = DATABASE.begin_write()?;
+    write_txn.open_table(DIFINITION)?;
+    write_txn.commit()?;
+    Ok(())
+}

sdgb-cli/src/main.rs

@@ -6,6 +6,7 @@ use std::{
 use futures_util::StreamExt;
 use nyquest_preset::nyquest::ClientBuilder;
 use palc::Parser;
+
 use sdgb_api::{
     ApiError,
     all_net::QRCode,
@@ -23,6 +24,8 @@ use spdlog::{error, info, warn};
 
 use crate::commands::Cli;
 
+#[cfg(feature = "cache")]
+mod cache;
 mod commands;
 
 #[cfg_attr(feature = "compio", compio::main)]
@@ -102,9 +105,33 @@ async fn main() -> Result<(), Box<dyn snafu::Error>> {
                 user_ids.push(user_id);
             }
 
-            let users = futures_util::stream::iter(user_ids)
+            #[cfg(feature = "cache")]
+            let _ = cache::init_db();
+            #[cfg(feature = "cache")]
+            let write = cache::write_txn()?;
+            #[cfg(feature = "cache")]
+            let config = sdgb_api::bincode::config::Configuration::<
+                sdgb_api::bincode::config::LittleEndian,
+            >::default()
+            .with_no_limit();
+
+            let players = futures_util::stream::iter(user_ids)
                 .map(async |user_id| {
-                    info!("preview: {user_id}");
+                    #[cfg(feature = "cache")]
+                    {
+                        use redb::ReadableTable;
+                        use sdgb_api::bincode::borrow_decode_from_slice;
+
+                        let cache_table = cache::open_table(&write)?;
+                        let data = cache_table.get(user_id)?;
+                        if let Some(data) = data {
+                            let decoded: (GetUserPreviewApiResp, _) =
+                                borrow_decode_from_slice(&data.value(), config)?;
+
+                            return Ok(decoded.0);
+                        }
+                    }
+
                     let resp = Sdgb1_50::request::<_, GetUserPreviewApiResp>(
                         &client,
                         APIMethod::GetUserPreviewApi,
@@ -113,15 +140,30 @@ async fn main() -> Result<(), Box<dyn snafu::Error>> {
                     )
                     .await;
 
-                    if let Err(e) = &resp {
+                    match &resp {
-                        if matches!(e, ApiError::JSON { .. }) {
+                        Ok(resp) => {
+                            info!("preview: {user_id} succeed");
+
+                            #[cfg(feature = "cache")]
+                            {
+                                use sdgb_api::bincode::encode_to_vec;
+
+                                if let Ok(mut table) = cache::open_table(&write)
+                                    && let Ok(encoded) = encode_to_vec(resp, config)
+                                {
+                                    _ = table.insert(resp.user_id, encoded);
+                                }
+                            }
+                        }
+                        Err(ApiError::JSON { .. }) => {
                             warn!("account unregistered: {user_id}");
-                        } else {
+                        }
+                        Err(e) => {
                             error!("preview failed: {e}");
                         }
                     }
 
-                    resp
+                    Result::<_, Box<dyn snafu::Error>>::Ok(resp?)
                 })
                 .buffer_unordered(20)
                 .filter_map(async |r| r.ok())
@@ -132,8 +174,8 @@ async fn main() -> Result<(), Box<dyn snafu::Error>> {
                 .write(true)
                 .truncate(true)
                 .create(true)
-                .open("users.json")?;
+                .open("players.json")?;
-            serde_json::to_writer_pretty(output, &users)?;
+            serde_json::to_writer_pretty(output, &players)?;
         }
     }