refactor: dump database to json

Cargo.toml

@@ -1,6 +1,7 @@
 [workspace]
 members = ["sdgb-api", "sdgb-cli"]
 resolver = "3"
+default-members = ["sdgb-cli"]
 
 [workspace.dependencies]
 sdgb-api = { path = "./sdgb-api", default-features = false }
@@ -15,6 +16,7 @@ serde_json = "1.0.141"
 strum = { version = "0.27.2", features = ["derive"] }
 tokio = { version = "1", features = ["rt-multi-thread"] }
 compio = { version = "0.15.0", features = ["runtime"] }
+redb = "2.6.1"
 
 [profile.release]
 lto = true

sdgb-api/src/lib.rs

@@ -6,3 +6,6 @@ mod error;
 pub use error::ApiError;
 
 pub use bincode;
+
+#[cfg(all(feature = "compio", feature = "tokio"))]
+compile_error!("you must not enable both `compio` and `tokio`");

sdgb-api/src/title/mod.rs

@@ -74,6 +74,9 @@ pub trait MaiVersionExt: MaiVersion {
         #[cfg(feature = "tokio")]
         use tokio::task::spawn_blocking;
 
+        #[cfg(all(not(feature = "compio"), not(feature = "tokio")))]
+        compile_error!("you must enable one of `compio` or `tokio`");
+
         async {
             let req = spawn_blocking(move || Self::api_call(api, agent_extra, data))
                 .await

sdgb-cli/Cargo.toml

@@ -6,26 +6,26 @@ authors = ["mokurin000"]
 description = "CLI tool for SDGB protocol"
 
 [features]
-default = ["compio", "cache"]
+default = ["compio", "fetchall"]
 
 compio = ["dep:compio", "sdgb-api/compio"]
 tokio = ["dep:tokio", "sdgb-api/tokio"]
 
-cache = ["dep:redb"]
+fetchall = ["dep:redb", "dep:futures-util"]
 
 [dependencies]
 sdgb-api = { workspace = true, features = ["bincode"] }
-spdlog-rs = { workspace = true }
 
+spdlog-rs = { workspace = true }
 snafu = { workspace = true }
 serde_json = { workspace = true }
 strum = { workspace = true }
+redb = { workspace = true, optional = true }
 tokio = { workspace = true, features = ["macros"], optional = true }
 compio = { workspace = true, features = ["macros"], optional = true }
 
 nyquest-preset = { version = "0.2.0", features = ["async"] }
 
 palc = { version = "0.0.1", features = ["derive"] }
-futures-util = "0.3.31"
+futures-util = { version = "0.3.31", optional = true }
-redb = { version = "2.6.1", optional = true }
 ctrlc = { version = "3.4.7", features = ["termination"] }

sdgb-cli/src/cache/mod.rs

@@ -1,6 +1,6 @@
 use std::sync::LazyLock;
 
-use redb::{Table, TableDefinition, WriteTransaction};
+use redb::{ReadTransaction, Table, TableDefinition, WriteTransaction};
 
 static DATABASE: LazyLock<redb::Database> = LazyLock::new(|| {
     redb::Database::builder()
@@ -17,6 +17,16 @@ pub fn open_table(write: &WriteTransaction) -> Result<Table<'_, u32, Vec<u8>>, r
     Ok(write.open_table(DIFINITION)?)
 }
 
+pub fn read_txn() -> Result<ReadTransaction, redb::Error> {
+    Ok(DATABASE.begin_read()?)
+}
+
+pub fn open_table_read(
+    read: &ReadTransaction,
+) -> Result<redb::ReadOnlyTable<u32, Vec<u8>>, redb::Error> {
+    Ok(read.open_table(DIFINITION)?)
+}
+
 pub fn init_db() -> Result<(), redb::Error> {
     let write_txn = DATABASE.begin_write()?;
     write_txn.open_table(DIFINITION)?;

sdgb-cli/src/commands.rs

@@ -41,10 +41,13 @@ pub enum Commands {
         user_id: u32,
     },
 
+    #[cfg(feature = "fetchall")]
     ListAllUser {
         #[arg(short, long, default_value_t = 5)]
         concurrency: usize,
     },
+    #[cfg(feature = "fetchall")]
+    ListAllUserDump {},
 
     Logout {
         #[arg(short, long)]

sdgb-cli/src/main.rs

@@ -1,16 +1,10 @@
-use std::{
+use std::sync::atomic::{AtomicBool, Ordering};
-    fs::OpenOptions,
-    io::{self, BufRead},
-    sync::atomic::{AtomicBool, Ordering},
-};
 
-use futures_util::StreamExt;
 use nyquest_preset::nyquest::ClientBuilder;
 use palc::Parser;
 use spdlog::{Level, LevelFilter::MoreSevereEqual};
 
 use sdgb_api::{
-    ApiError,
     all_net::QRCode,
     auth_lite::{SDGB, SDHJ, delivery_raw},
     title::{
@@ -26,7 +20,7 @@ use spdlog::{error, info, warn};
 
 use crate::{commands::Cli, utils::login_action};
 
-#[cfg(feature = "cache")]
+#[cfg(feature = "fetchall")]
 mod cache;
 mod commands;
 mod utils;
@@ -108,7 +102,13 @@ async fn main() -> Result<(), Box<dyn snafu::Error>> {
             println!("{}", String::from_utf8_lossy(&resp));
         }
 
+        #[cfg(feature = "fetchall")]
         commands::Commands::ListAllUser { concurrency } => {
+            use futures_util::StreamExt;
+            use redb::ReadableTable;
+            use sdgb_api::bincode::borrow_decode_from_slice;
+            use std::io::{self, BufRead};
+
             let mut stdin = io::stdin().lock();
             let mut buf = String::new();
             let mut user_ids = Vec::new();
@@ -123,23 +123,15 @@ async fn main() -> Result<(), Box<dyn snafu::Error>> {
                 user_ids.push(user_id);
             }
 
-            #[cfg(feature = "cache")]
             let _ = cache::init_db();
-            #[cfg(feature = "cache")]
             let write = cache::write_txn()?;
-            #[cfg(feature = "cache")]
             let config = sdgb_api::bincode::config::Configuration::<
                 sdgb_api::bincode::config::LittleEndian,
             >::default()
             .with_no_limit();
 
-            let players = futures_util::stream::iter(user_ids)
+            let _ = futures_util::stream::iter(user_ids)
                 .map(async |user_id| {
-                    #[cfg(feature = "cache")]
-                    {
-                        use redb::ReadableTable;
-                        use sdgb_api::bincode::borrow_decode_from_slice;
-
                     let cache_table = cache::open_table(&write)?;
                     let data = cache_table.get(user_id)?;
                     if let Some(data) = data {
@@ -148,7 +140,6 @@ async fn main() -> Result<(), Box<dyn snafu::Error>> {
 
                         return Ok(decoded.0);
                     }
-                    }
 
                     if EARLY_QUIT.load(Ordering::Relaxed) {
                         return Err("early skip due to ctrl-c")?;
@@ -166,8 +157,6 @@ async fn main() -> Result<(), Box<dyn snafu::Error>> {
                         Ok(resp) => {
                             info!("preview: {user_id} succeed");
 
-                            #[cfg(feature = "cache")]
-                            {
                             use sdgb_api::bincode::encode_to_vec;
 
                             if let Ok(mut table) = cache::open_table(&write)
@@ -176,8 +165,7 @@ async fn main() -> Result<(), Box<dyn snafu::Error>> {
                                 _ = table.insert(resp.user_id, encoded);
                             }
                         }
-                        }
+                        Err(sdgb_api::ApiError::JSON { .. }) => {
-                        Err(ApiError::JSON { .. }) => {
                             warn!("account unregistered: {user_id}");
                         }
                         Err(e) => {
@@ -192,20 +180,40 @@ async fn main() -> Result<(), Box<dyn snafu::Error>> {
                 .collect::<Vec<_>>()
                 .await;
 
-            #[cfg(feature = "cache")]
             let _ = write.commit();
+        }
+        #[cfg(feature = "fetchall")]
+        commands::Commands::ListAllUserDump { .. } => {
+            use std::fs::OpenOptions;
 
-            if !EARLY_QUIT.load(Ordering::Relaxed) {
+            use redb::ReadableTable;
-                let output = OpenOptions::new()
+            use sdgb_api::bincode::{self, borrow_decode_from_slice};
-                    .write(true)
+
-                    .truncate(true)
+            use crate::cache::{open_table_read, read_txn};
+
+            let txn = read_txn()?;
+            let table = open_table_read(&txn)?;
+
+            let config = bincode::config::Configuration::<bincode::config::LittleEndian>::default()
+                .with_no_limit();
+
+            let user_ids = table
+                .iter()?
+                .flatten()
+                .map(|d| borrow_decode_from_slice(&d.1.value(), config))
+                .flatten()
+                .map(|(value, _)| value)
+                .collect::<Vec<GetUserPreviewApiResp>>();
+
+            let file = OpenOptions::new()
                 .create(true)
+                .truncate(true)
+                .write(true)
                 .open("players.json")?;
-                serde_json::to_writer_pretty(output, &players)?;
+            file.lock()?;
-            } else {
+            serde_json::to_writer(file, &user_ids)?;
-                info!("current progress: {}", players.len());
-            }
         }
+
         commands::Commands::Userdata { user_id } => {
             let action = async |_| match Sdgb1_50::request::<_, GetUserDataApiResp>(
                 &client,
@@ -228,3 +236,6 @@ async fn main() -> Result<(), Box<dyn snafu::Error>> {
 
     Ok(())
 }
+
+#[cfg(all(feature = "compio", feature = "tokio"))]
+compile_error!("you must not enable both `compio` and `tokio`");