mokurin000/sdgb-utils-rs
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

refactor: dump database to json

Browse Source
This commit is contained in:
mokurin000
2025-07-31 11:25:04 +08:00
parent 3ab53b426d
commit b72addd661
7 changed files with 84 additions and 52 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
Cargo.toml
View File

@@ -1,6 +1,7 @@
[workspace]
members = ["sdgb-api", "sdgb-cli"]
resolver = "3"
default-members = ["sdgb-cli"]
[workspace.dependencies]
sdgb-api = { path = "./sdgb-api", default-features = false }
@@ -15,6 +16,7 @@ serde_json = "1.0.141"
strum = { version = "0.27.2", features = ["derive"] }
tokio = { version = "1", features = ["rt-multi-thread"] }
compio = { version = "0.15.0", features = ["runtime"] }
redb = "2.6.1"
[profile.release]
lto = true

3
sdgb-api/src/lib.rs
View File

@@ -6,3 +6,6 @@ mod error;
pub use error::ApiError;
pub use bincode;
#[cfg(all(feature = "compio", feature = "tokio"))]
compile_error!("you must not enable both `compio` and `tokio`");

3
sdgb-api/src/title/mod.rs
View File

@@ -74,6 +74,9 @@ pub trait MaiVersionExt: MaiVersion {
#[cfg(feature = "tokio")]
use tokio::task::spawn_blocking;
#[cfg(all(not(feature = "compio"), not(feature = "tokio")))]
compile_error!("you must enable one of `compio` or `tokio`");
async {
let req = spawn_blocking(move || Self::api_call(api, agent_extra, data))
.await

10
sdgb-cli/Cargo.toml
View File

@@ -6,26 +6,26 @@ authors = ["mokurin000"]
description = "CLI tool for SDGB protocol"
[features]
default = ["compio", "cache"]
default = ["compio", "fetchall"]
compio = ["dep:compio", "sdgb-api/compio"]
tokio = ["dep:tokio", "sdgb-api/tokio"]
cache = ["dep:redb"]
fetchall = ["dep:redb", "dep:futures-util"]
[dependencies]
sdgb-api = { workspace = true, features = ["bincode"] }
spdlog-rs = { workspace = true }
spdlog-rs = { workspace = true }
snafu = { workspace = true }
serde_json = { workspace = true }
strum = { workspace = true }
redb = { workspace = true, optional = true }
tokio = { workspace = true, features = ["macros"], optional = true }
compio = { workspace = true, features = ["macros"], optional = true }
nyquest-preset = { version = "0.2.0", features = ["async"] }
palc = { version = "0.0.1", features = ["derive"] }
futures-util = "0.3.31"
redb = { version = "2.6.1", optional = true }
futures-util = { version = "0.3.31", optional = true }
ctrlc = { version = "3.4.7", features = ["termination"] }

12
sdgb-cli/src/cache/mod.rs vendored
View File

@@ -1,6 +1,6 @@
use std::sync::LazyLock;
use redb::{Table, TableDefinition, WriteTransaction};
use redb::{ReadTransaction, Table, TableDefinition, WriteTransaction};
static DATABASE: LazyLock<redb::Database> = LazyLock::new(|| {
redb::Database::builder()
@@ -17,6 +17,16 @@ pub fn open_table(write: &WriteTransaction) -> Result<Table<'_, u32, Vec<u8>>, r
Ok(write.open_table(DIFINITION)?)
}
pub fn read_txn() -> Result<ReadTransaction, redb::Error> {
Ok(DATABASE.begin_read()?)
}
pub fn open_table_read(
read: &ReadTransaction,
) -> Result<redb::ReadOnlyTable<u32, Vec<u8>>, redb::Error> {
Ok(read.open_table(DIFINITION)?)
}
pub fn init_db() -> Result<(), redb::Error> {
let write_txn = DATABASE.begin_write()?;
write_txn.open_table(DIFINITION)?;

3
sdgb-cli/src/commands.rs
View File

@@ -41,10 +41,13 @@ pub enum Commands {
user_id: u32,
},
#[cfg(feature = "fetchall")]
ListAllUser {
#[arg(short, long, default_value_t = 5)]
concurrency: usize,
},
#[cfg(feature = "fetchall")]
ListAllUserDump {},
Logout {
#[arg(short, long)]

73
sdgb-cli/src/main.rs
View File

@@ -1,16 +1,10 @@
use std::{
fs::OpenOptions,
io::{self, BufRead},
sync::atomic::{AtomicBool, Ordering},
};
use std::sync::atomic::{AtomicBool, Ordering};
use futures_util::StreamExt;
use nyquest_preset::nyquest::ClientBuilder;
use palc::Parser;
use spdlog::{Level, LevelFilter::MoreSevereEqual};
use sdgb_api::{
ApiError,
all_net::QRCode,
auth_lite::{SDGB, SDHJ, delivery_raw},
title::{
@@ -26,7 +20,7 @@ use spdlog::{error, info, warn};
use crate::{commands::Cli, utils::login_action};
#[cfg(feature = "cache")]
#[cfg(feature = "fetchall")]
mod cache;
mod commands;
mod utils;
@@ -108,7 +102,13 @@ async fn main() -> Result<(), Box<dyn snafu::Error>> {
println!("{}", String::from_utf8_lossy(&resp));
}
#[cfg(feature = "fetchall")]
commands::Commands::ListAllUser { concurrency } => {
use futures_util::StreamExt;
use redb::ReadableTable;
use sdgb_api::bincode::borrow_decode_from_slice;
use std::io::{self, BufRead};
let mut stdin = io::stdin().lock();
let mut buf = String::new();
let mut user_ids = Vec::new();
@@ -123,23 +123,15 @@ async fn main() -> Result<(), Box<dyn snafu::Error>> {
user_ids.push(user_id);
}
#[cfg(feature = "cache")]
let _ = cache::init_db();
#[cfg(feature = "cache")]
let write = cache::write_txn()?;
#[cfg(feature = "cache")]
let config = sdgb_api::bincode::config::Configuration::<
sdgb_api::bincode::config::LittleEndian,
>::default()
.with_no_limit();
let players = futures_util::stream::iter(user_ids)
let _ = futures_util::stream::iter(user_ids)
.map(async |user_id| {
#[cfg(feature = "cache")]
{
use redb::ReadableTable;
use sdgb_api::bincode::borrow_decode_from_slice;
let cache_table = cache::open_table(&write)?;
let data = cache_table.get(user_id)?;
if let Some(data) = data {
@@ -148,7 +140,6 @@ async fn main() -> Result<(), Box<dyn snafu::Error>> {
return Ok(decoded.0);
}
}
if EARLY_QUIT.load(Ordering::Relaxed) {
return Err("early skip due to ctrl-c")?;
@@ -166,8 +157,6 @@ async fn main() -> Result<(), Box<dyn snafu::Error>> {
Ok(resp) => {
info!("preview: {user_id} succeed");
#[cfg(feature = "cache")]
{
use sdgb_api::bincode::encode_to_vec;
if let Ok(mut table) = cache::open_table(&write)
@@ -176,8 +165,7 @@ async fn main() -> Result<(), Box<dyn snafu::Error>> {
_ = table.insert(resp.user_id, encoded);
}
}
}
Err(ApiError::JSON { .. }) => {
Err(sdgb_api::ApiError::JSON { .. }) => {
warn!("account unregistered: {user_id}");
}
Err(e) => {
@@ -192,20 +180,40 @@ async fn main() -> Result<(), Box<dyn snafu::Error>> {
.collect::<Vec<_>>()
.await;
#[cfg(feature = "cache")]
let _ = write.commit();
}
#[cfg(feature = "fetchall")]
commands::Commands::ListAllUserDump { .. } => {
use std::fs::OpenOptions;
if !EARLY_QUIT.load(Ordering::Relaxed) {
let output = OpenOptions::new()
.write(true)
.truncate(true)
use redb::ReadableTable;
use sdgb_api::bincode::{self, borrow_decode_from_slice};
use crate::cache::{open_table_read, read_txn};
let txn = read_txn()?;
let table = open_table_read(&txn)?;
let config = bincode::config::Configuration::<bincode::config::LittleEndian>::default()
.with_no_limit();
let user_ids = table
.iter()?
.flatten()
.map(|d| borrow_decode_from_slice(&d.1.value(), config))
.flatten()
.map(|(value, _)| value)
.collect::<Vec<GetUserPreviewApiResp>>();
let file = OpenOptions::new()
.create(true)
.truncate(true)
.write(true)
.open("players.json")?;
serde_json::to_writer_pretty(output, &players)?;
} else {
info!("current progress: {}", players.len());
}
file.lock()?;
serde_json::to_writer(file, &user_ids)?;
}
commands::Commands::Userdata { user_id } => {
let action = async |_| match Sdgb1_50::request::<_, GetUserDataApiResp>(
&client,
@@ -228,3 +236,6 @@ async fn main() -> Result<(), Box<dyn snafu::Error>> {
Ok(())
}
#[cfg(all(feature = "compio", feature = "tokio"))]
compile_error!("you must not enable both `compio` and `tokio`");