diff --git a/sdgb-api/src/title/model/get_user_music_api/mod.rs b/sdgb-api/src/title/model/get_user_music_api/mod.rs index c47d429..0393f0e 100644 --- a/sdgb-api/src/title/model/get_user_music_api/mod.rs +++ b/sdgb-api/src/title/model/get_user_music_api/mod.rs @@ -1,5 +1,7 @@ use std::fmt::Display; +use bincode::Decode; +use bincode::Encode; use music_db::query_music; use music_db::query_music_level; use serde::Deserialize; @@ -31,7 +33,7 @@ pub struct UserMusic { pub length: u32, } -#[derive(Default, Debug, Clone, PartialEq, Serialize, Deserialize)] +#[derive(Default, Debug, Clone, PartialEq, Serialize, Deserialize, Encode, Decode)] #[serde(rename_all = "camelCase")] pub struct UserMusicDetail { pub music_id: u32, diff --git a/sdgb-cli/src/cache/mod.rs b/sdgb-cli/src/cache/mod.rs index a184fed..697f02e 100644 --- a/sdgb-cli/src/cache/mod.rs +++ b/sdgb-cli/src/cache/mod.rs @@ -13,6 +13,7 @@ static DATABASE: LazyLock = LazyLock::new(|| { pub const PLAYERS: TableDefinition<'_, u32, Vec> = redb::TableDefinition::new("players"); pub const PLAYER_B50: TableDefinition<'_, u32, Vec> = redb::TableDefinition::new("b50"); +pub const PLAYER_RECORD: TableDefinition<'_, u32, Vec> = redb::TableDefinition::new("records"); pub fn write_txn() -> Result { Ok(DATABASE.begin_write()?) diff --git a/sdgb-cli/src/commands.rs b/sdgb-cli/src/commands.rs index 71580bd..2f13253 100644 --- a/sdgb-cli/src/commands.rs +++ b/sdgb-cli/src/commands.rs @@ -94,6 +94,17 @@ pub enum Commands { #[arg(long, default_value_t = 16500)] max_rating: i64, }, + #[cfg(feature = "fetchall")] + ScrapeAllRecord { + #[arg(short, long, default_value_t = 5)] + concurrency: usize, + + #[arg(long, default_value_t = 10000)] + min_rating: i64, + #[arg(long, default_value_t = 16400)] + max_rating: i64, + }, + #[cfg(feature = "fetchall")] ListAllUserDump {}, #[cfg(feature = "fetchall")] diff --git a/sdgb-cli/src/main.rs b/sdgb-cli/src/main.rs index 3317cb8..d727609 100644 --- a/sdgb-cli/src/main.rs +++ b/sdgb-cli/src/main.rs @@ -231,6 +231,29 @@ async fn main() -> Result<(), Box> { .await?; } + #[cfg(feature = "fetchall")] + Commands::ScrapeAllRecord { + concurrency, + min_rating, + max_rating, + } => { + use crate::{ + cache::{PLAYER_RECORD, PLAYERS}, + utils::helpers::{cached_concurrent_fetch_userfn, read_cache}, + }; + + let mut players: Vec = read_cache(PLAYERS)?; + players.retain(|p| p.player_rating >= min_rating && p.player_rating <= max_rating); + + cached_concurrent_fetch_userfn( + players.iter().map(|p| p.user_id).collect::>(), + &client, + concurrency, + PLAYER_RECORD, + get_user_all_music, + ) + .await?; + } #[cfg(feature = "fetchall")] Commands::ScrapeAllB50 { concurrency, diff --git a/sdgb-cli/src/utils/helpers/mod.rs b/sdgb-cli/src/utils/helpers/mod.rs index 4d52bd9..09d70d7 100644 --- a/sdgb-cli/src/utils/helpers/mod.rs +++ b/sdgb-cli/src/utils/helpers/mod.rs @@ -10,10 +10,7 @@ use serde::Serialize; use spdlog::{error, info}; use sdgb_api::title::MaiVersionExt; -use sdgb_api::title::{ - Sdgb1_50, - methods::{APIExt, HasUid}, -}; +use sdgb_api::title::{Sdgb1_50, methods::APIExt}; use sdgb_api::{ApiError, bincode}; use bincode::{BorrowDecode, Encode, borrow_decode_from_slice}; @@ -87,7 +84,7 @@ pub async fn cached_concurrent_fetch( ) -> Result<(), Box> where A::Payload: From, - A::Response: Encode + for<'a> BorrowDecode<'a, ()> + HasUid, + A::Response: Encode + for<'a> BorrowDecode<'a, ()>, { cached_concurrent_fetch_userfn( user_ids, @@ -109,7 +106,7 @@ pub async fn cached_concurrent_fetch_userfn( scrape: impl AsyncFn(&AsyncClient, u32) -> Result, ) -> Result<(), Box> where - R: Encode + for<'a> BorrowDecode<'a, ()> + HasUid, + R: Encode + for<'a> BorrowDecode<'a, ()>, { let _ = cache::init_db(); @@ -150,7 +147,7 @@ where if let Ok(mut table) = cache::open_table(&write, definition) && let Ok(encoded) = encode_to_vec(resp, config) { - _ = table.insert(resp.get_uid(), encoded); + _ = table.insert(user_id, encoded); } } Err(sdgb_api::ApiError::JSON { .. }) => {}