mokurin000/sdgb-utils-rs
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

perf: read transcition for exisiting check

Browse Source
This commit is contained in:
mokurin000
2025-07-31 11:54:01 +08:00
parent b72addd661
commit 57c29c8959
2 changed files with 38 additions and 25 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
.gitignore vendored
View File

@@ -3,4 +3,4 @@
/*.txt /*.txt
/players.redb /players.redb
/players.json /players.json*

29
sdgb-cli/src/main.rs
View File

@@ -39,8 +39,13 @@ async fn main() -> Result<(), Box<dyn snafu::Error>> {
} }
ctrlc::set_handler(|| { ctrlc::set_handler(|| {
if EARLY_QUIT.load(Ordering::Relaxed) {
error!("force-quit triggered!");
std::process::exit(1);
} else {
warn!("received early-quit request! will abort soon"); warn!("received early-quit request! will abort soon");
EARLY_QUIT.store(true, Ordering::Relaxed); EARLY_QUIT.store(true, Ordering::Relaxed);
}
})?; })?;
let Cli { command } = <Cli as Parser>::parse(); let Cli { command } = <Cli as Parser>::parse();
@@ -105,13 +110,13 @@ async fn main() -> Result<(), Box<dyn snafu::Error>> {
#[cfg(feature = "fetchall")] #[cfg(feature = "fetchall")]
commands::Commands::ListAllUser { concurrency } => { commands::Commands::ListAllUser { concurrency } => {
use futures_util::StreamExt; use futures_util::StreamExt;
use redb::ReadableTable;
use sdgb_api::bincode::borrow_decode_from_slice; use sdgb_api::bincode::borrow_decode_from_slice;
use std::io::{self, BufRead}; use std::io::{self, BufRead};
let mut user_ids = Vec::new();
{
let mut stdin = io::stdin().lock(); let mut stdin = io::stdin().lock();
let mut buf = String::new(); let mut buf = String::new();
let mut user_ids = Vec::new();
while stdin.read_line(&mut buf).is_ok_and(|size| size != 0) { while stdin.read_line(&mut buf).is_ok_and(|size| size != 0) {
if buf.is_empty() { if buf.is_empty() {
@@ -122,17 +127,22 @@ async fn main() -> Result<(), Box<dyn snafu::Error>> {
buf.clear(); buf.clear();
user_ids.push(user_id); user_ids.push(user_id);
} }
}
let _ = cache::init_db(); let _ = cache::init_db();
let read = cache::read_txn()?;
let write = cache::write_txn()?; let write = cache::write_txn()?;
let config = sdgb_api::bincode::config::Configuration::< let config = sdgb_api::bincode::config::Configuration::<
sdgb_api::bincode::config::LittleEndian, sdgb_api::bincode::config::LittleEndian,
>::default() >::default()
.with_no_limit(); .with_no_limit();
let _ = futures_util::stream::iter(user_ids) info!("number of user_id: {}", user_ids.len());
let collect = futures_util::stream::iter(user_ids)
.map(async |user_id| { .map(async |user_id| {
let cache_table = cache::open_table(&write)?; {
let cache_table = cache::open_table_read(&read)?;
let data = cache_table.get(user_id)?; let data = cache_table.get(user_id)?;
if let Some(data) = data { if let Some(data) = data {
let decoded: (GetUserPreviewApiResp, _) = let decoded: (GetUserPreviewApiResp, _) =
@@ -140,11 +150,14 @@ async fn main() -> Result<(), Box<dyn snafu::Error>> {
return Ok(decoded.0); return Ok(decoded.0);
} }
}
if EARLY_QUIT.load(Ordering::Relaxed) { if EARLY_QUIT.load(Ordering::Relaxed) {
return Err("early skip due to ctrl-c")?; return Err("early skip due to ctrl-c")?;
} }
info!("processing: {user_id}");
let resp = Sdgb1_50::request::<_, GetUserPreviewApiResp>( let resp = Sdgb1_50::request::<_, GetUserPreviewApiResp>(
&client, &client,
APIMethod::GetUserPreviewApi, APIMethod::GetUserPreviewApi,
@@ -155,8 +168,6 @@ async fn main() -> Result<(), Box<dyn snafu::Error>> {
match &resp { match &resp {
Ok(resp) => { Ok(resp) => {
info!("preview: {user_id} succeed");
use sdgb_api::bincode::encode_to_vec; use sdgb_api::bincode::encode_to_vec;
if let Ok(mut table) = cache::open_table(&write) if let Ok(mut table) = cache::open_table(&write)
@@ -179,12 +190,13 @@ async fn main() -> Result<(), Box<dyn snafu::Error>> {
.filter_map(async |r| r.ok()) .filter_map(async |r| r.ok())
.collect::<Vec<_>>() .collect::<Vec<_>>()
.await; .await;
drop(collect);
let _ = write.commit(); let _ = write.commit();
} }
#[cfg(feature = "fetchall")] #[cfg(feature = "fetchall")]
commands::Commands::ListAllUserDump { .. } => { commands::Commands::ListAllUserDump { .. } => {
use std::fs::OpenOptions; use std::{fs::OpenOptions, io::BufWriter};
use redb::ReadableTable; use redb::ReadableTable;
use sdgb_api::bincode::{self, borrow_decode_from_slice}; use sdgb_api::bincode::{self, borrow_decode_from_slice};
@@ -211,7 +223,8 @@ async fn main() -> Result<(), Box<dyn snafu::Error>> {
.write(true) .write(true)
.open("players.json")?; .open("players.json")?;
file.lock()?; file.lock()?;
serde_json::to_writer(file, &user_ids)?; let writer = BufWriter::new(file);
serde_json::to_writer(writer, &user_ids)?;
} }
commands::Commands::Userdata { user_id } => { commands::Commands::Userdata { user_id } => {