mokurin000/sdgb-utils-rs
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

perf: parquet based data export

Browse Source
This commit is contained in:
mokurin000
2025-08-04 01:49:43 +08:00
parent 89d8177180
commit 73e1046be9
10 changed files with 642 additions and 17 deletions
Download Patch File Download Diff File Expand all files Collapse all files

34
sdgb-cli/src/main.rs
View File

@@ -311,23 +311,35 @@ async fn main() -> Result<(), Box<dyn snafu::Error>> {
#[cfg(feature = "fetchall")]
Commands::ListAllUserDump {} => {
use crate::{cache::PLAYERS, utils::helpers::dump_cache};
use crate::{
cache::PLAYERS,
utils::helpers::{dump_parquet, read_cache},
};
dump_cache::<GetUserPreviewApiResp>("players.json", PLAYERS)?;
}
#[cfg(feature = "fetchall")]
Commands::ScrapeAllB50Dump {} => {
use crate::{cache::B50, utils::helpers::dump_cache};
dump_cache::<GetUserRatingApiResp>("b50.json", B50)?;
let players: Vec<GetUserPreviewApiResp> = read_cache(PLAYERS)?;
dump_parquet(players, "players.parquet")?;
}
#[cfg(feature = "fetchall")]
Commands::ScrapeAllRegionDump {} => {
use sdgb_api::title::model::GetUserRegionApiResp;
use crate::{
cache::REGIONS,
utils::helpers::{dump_parquet, read_cache},
};
use sdgb_api::title::model::{GetUserRegionApiResp, UserRegionFlatten};
use crate::{cache::REGIONS, utils::helpers::dump_cache};
let regions: Vec<GetUserRegionApiResp> = read_cache(REGIONS)?;
let regions_flat = regions
.into_iter()
.map(Vec::<UserRegionFlatten>::from)
.flatten()
.collect::<Vec<_>>();
dump_parquet(regions_flat, "regions.parquet")?;
}
#[cfg(feature = "fetchall")]
Commands::ScrapeAllB50Dump {} => {
use crate::{cache::B50, utils::helpers::dump_json};
dump_cache::<GetUserRegionApiResp>("region.json", REGIONS)?;
dump_json::<GetUserRatingApiResp>("b50.json", B50)?;
}
Commands::Userdata { user_id } => {