chore: scraping user data becomes impossible

Cargo.toml

@@ -20,7 +20,7 @@ serde = { version = "1.0.226", features = ["derive"] }
 serde_json = "1.0.145"
 strum = { version = "0.27.2", features = ["derive"] }
 tokio = { version = "1.47.1", features = ["rt-multi-thread"] }
-compio = { version = "0.16.0", features = ["runtime"] }
+compio = { version = "0.17.0", features = ["runtime"] }
 redb = "3.1.0"
 crabtime = { git = "https://github.com/wdanilo/crabtime.git", rev = "2ed856f5" }
 

sdgb-api/Cargo.toml

@@ -6,12 +6,9 @@ edition = "2024"
 license = "GPL-3.0"
 
 [features]
-default = ["compio", "bincode"]
+default = ["compio"]
 compio = ["dep:compio"]
 tokio = ["dep:tokio"]
-bincode = ["dep:bincode"]
-
-parquet = ['dep:parquet', 'dep:parquet_derive']
 
 [dependencies]
 snafu = { workspace = true }
@@ -45,7 +42,3 @@ flate2 = "1.1.2"
 cbc = { version = "0.1.2", features = ["alloc"] }
 aes = "0.8.4"
 cipher = { version = "0.4.4", features = ["block-padding"] }
-bincode = { version = "2.0.1", optional = true }
-
-parquet = { workspace = true, optional = true }
-parquet_derive = { workspace = true, optional = true }

sdgb-api/benches/enc_dec.rs

@@ -1,22 +0,0 @@
-#![feature(test)]
-extern crate test;
-
-use sdgb_api::title::{MaiVersionExt, Sdgb1_50};
-
-#[bench]
-pub fn sdgb_150_enc_short(b: &mut test::Bencher) {
-    b.iter(|| _ = Sdgb1_50::encode(b"Hello world"));
-}
-
-#[bench]
-pub fn sdgb_150_enc_4k(b: &mut test::Bencher) {
-    let data = [1u8; 4096];
-    b.iter(|| _ = Sdgb1_50::encode(data));
-}
-
-#[bench]
-pub fn sdgb_150_dec_4k(b: &mut test::Bencher) {
-    let data = [1u8; 4096];
-    let enc_data = Sdgb1_50::encode(data).unwrap();
-    b.iter(|| _ = Sdgb1_50::decode(enc_data.clone()));
-}

sdgb-api/src/all_net/model.rs

@@ -29,7 +29,7 @@ pub struct GetResponse {
 
 impl GetUserId {
     pub fn new(qr_code: impl Into<String>) -> Self {
-        let chip_id = "A63E-01E54389854".to_string();
+        let chip_id = "A63E-01C28055905".to_string();
 
         let timestamp = Utc::now()
             .with_timezone(&FixedOffset::east_opt(8 * 60 * 60).unwrap())

sdgb-api/src/lib.rs

@@ -7,8 +7,5 @@ pub mod helper;
 mod error;
 pub use error::ApiError;
 
-#[cfg(feature = "bincode")]
-pub use bincode;
-
 #[cfg(all(feature = "compio", feature = "tokio"))]
 compile_error!("you must not enable both `compio` and `tokio`");

sdgb-api/src/title/encryption/mod.rs

@@ -9,25 +9,7 @@ use flate2::write::{ZlibDecoder, ZlibEncoder};
 use spdlog::debug;
 
 use crate::error::ApiError;
-use crate::title::{MaiVersion, MaiVersionExt, Sdgb1_50, Sdgb1_53};
-
-impl MaiVersionExt for Sdgb1_50 {
-    fn decode(mut data: impl AsMut<[u8]>) -> Result<Vec<u8>, ApiError> {
-        if data.as_mut().is_empty() {
-            return Err(ApiError::EmptyResponse);
-        }
-
-        debug!("data size: {}", data.as_mut().len());
-        let decrypted = decrypt(&mut data, Self::AES_KEY, Self::AES_IV)?;
-        Ok(decompress(decrypted))
-    }
-
-    fn encode(data: impl AsRef<[u8]>) -> Result<Vec<u8>, ApiError> {
-        let compressed = compress(data)?;
-        let enc = encrypt(compressed, Self::AES_KEY, Self::AES_IV)?;
-        Ok(enc)
-    }
-}
+use crate::title::{MaiVersion, MaiVersionExt, Sdgb1_53};
 
 impl MaiVersionExt for Sdgb1_53 {
     fn decode(mut data: impl AsMut<[u8]>) -> Result<Vec<u8>, ApiError> {
@@ -93,101 +75,3 @@ fn decrypt<'ct>(
     let result = decryptor.decrypt_padded_mut::<Pkcs7>(data.as_mut())?;
     Ok(result)
 }
-
-#[cfg(test)]
-mod _tests {
-
-    use crate::title::{Sdgb1_50, encryption::*};
-
-    #[test]
-    fn test_ping_dec() -> Result<(), ApiError> {
-        let mut data = b"\x72\x5c\xa5\x55\x27\x14\x85\xd1\x64\xc8\x64\x5b\x6e\x5f\xd8\xe3\
-  \x3f\x36\x4c\x9a\x3b\xa5\xb0\x9e\x75\xae\x83\xee\xb3\xb9\x2a\x75"
-            .to_vec();
-        let decoded = Sdgb1_50::decode(&mut data)?;
-        assert_eq!(decoded, b"{\"result\":\"Pong\"}");
-        Ok(())
-    }
-
-    #[test]
-    fn test_sdgb_150_dec_enc() -> Result<(), ApiError> {
-        let data = [
-            161, 166, 3, 157, 202, 233, 151, 73, 40, 113, 186, 162, 177, 46, 118, 113, 98, 231, 67,
-            185, 246, 180, 109, 253, 1, 152, 0, 31, 81, 211, 28, 137, 95, 12, 110, 105, 181, 246,
-            177, 1, 45, 59, 182, 113, 56, 97, 56, 100, 34, 168, 27, 51, 228, 77, 192, 194, 248, 45,
-            118, 80, 21, 159, 37, 248, 54, 85, 94, 61, 48, 59, 117, 163, 161, 165, 206, 36, 23, 71,
-            73, 231, 214, 81, 82, 117, 115, 32, 122, 8, 161, 213, 252, 125, 35, 131, 144, 147, 74,
-            27, 138, 26, 133, 240, 73, 197, 25, 173, 213, 237, 216, 76, 101, 210, 202, 172, 216,
-            91, 83, 87, 243, 79, 143, 42, 149, 130, 210, 13, 63, 98, 198, 165, 122, 58, 254, 39,
-            150, 71, 155, 231, 55, 142, 5, 102, 253, 148, 191, 9, 212, 188, 69, 236, 60, 152, 13,
-            40, 111, 219, 162, 160, 34, 150, 211, 85, 190, 176, 137, 60, 25, 228, 218, 163, 240,
-            143, 44, 238, 77, 92, 12, 166, 209, 238, 100, 92, 98, 142, 10, 104, 213, 12, 89, 236,
-            114, 212, 222, 0, 237, 1, 208, 216, 114, 114, 71, 135, 21, 213, 61, 6, 162, 155, 119,
-            143, 70, 83, 136, 136, 136, 251, 94, 137, 244, 26, 125, 15, 132, 207, 60, 57, 105, 78,
-            177, 84, 85, 152, 183, 77, 67, 163, 61, 165, 144, 125, 255, 89, 108, 58, 137, 142, 9,
-            8, 54, 228, 34, 55, 124, 158, 83, 36,
-        ];
-        let dec = Sdgb1_50::decode(data)?;
-        assert_eq!(dec, r#"{"userId":10103750,"userName":"舞萌","isLogin":false,"lastGameId":null,"lastRomVersion":"1.01.00","lastDataVersion":"1.05.03","lastLoginDate":"1970-01-01 00:00:00","lastPlayDate":"1970-01-01 00:00:00","playerRating":1024,"nameplateId":0,"iconId":11,"trophyId":0,"isNetMember":1,"isInherit":false,"totalAwake":5,"dispRate":0,"dailyBonusDate":"1970-01-01 09:00:00","headPhoneVolume":null,"banState":0}"#.as_bytes());
-        let enc = Sdgb1_50::encode(dec)?;
-        assert_eq!(enc, data);
-        Ok(())
-    }
-
-    #[test]
-    fn test_user_data_dec() -> Result<(), ApiError> {
-        let data = [
-            112, 133, 192, 229, 116, 195, 219, 220, 56, 176, 98, 148, 246, 73, 179, 157, 181, 251,
-            9, 7, 190, 113, 101, 8, 144, 58, 23, 196, 16, 176, 78, 96, 106, 53, 191, 118, 86, 247,
-            50, 250, 168, 155, 164, 108, 7, 152, 251, 123, 186, 121, 113, 41, 104, 79, 29, 71, 47,
-            228, 214, 97, 223, 29, 27, 163, 159, 113, 82, 45, 29, 41, 176, 246, 33, 235, 22, 202,
-            1, 61, 133, 126, 8, 20, 9, 214, 153, 11, 203, 207, 5, 195, 129, 172, 70, 81, 58, 156,
-            240, 87, 203, 137, 110, 183, 245, 177, 210, 63, 231, 136, 82, 97, 201, 184, 236, 117,
-            89, 85, 22, 29, 2, 238, 250, 148, 158, 151, 139, 179, 9, 42, 47, 220, 88, 137, 135,
-            139, 57, 35, 0, 50, 123, 133, 103, 15, 87, 86, 208, 203, 235, 153, 214, 87, 236, 46,
-            78, 50, 72, 50, 8, 46, 30, 242, 44, 184, 72, 118, 51, 120, 99, 158, 247, 255, 168, 181,
-            119, 156, 214, 143, 253, 185, 21, 76, 117, 201, 38, 38, 79, 171, 8, 251, 90, 53, 59,
-            89, 30, 136, 69, 168, 57, 34, 115, 145, 159, 110, 182, 5, 126, 208, 202, 216, 92, 200,
-            168, 63, 114, 119, 129, 38, 139, 189, 101, 215, 102, 76, 29, 228, 219, 46, 79, 166,
-            127, 194, 60, 183, 169, 167, 210, 120, 77, 219, 58, 16, 231, 233, 189, 66, 215, 202,
-            28, 209, 59, 192, 141, 91, 65, 17, 187, 88, 189, 149, 139, 48, 237, 152, 161, 87, 120,
-            99, 2, 50, 12, 120, 179, 50, 235, 255, 223, 162, 216, 84, 13, 135, 196, 131, 121, 97,
-            171, 106, 240, 189, 112, 92, 41, 59, 204, 24, 72, 91, 14, 220, 249, 10, 166, 4, 254,
-            183, 194, 227, 53, 163, 35, 165, 253, 149, 83, 253, 191, 138, 236, 208, 146, 242, 31,
-            185, 152, 226, 100, 191, 2, 2, 82, 101, 141, 31, 71, 106, 2, 83, 1, 231, 140, 20, 16,
-            156, 171, 108, 109, 14, 93, 168, 203, 50, 20, 21, 142, 135, 97, 7, 80, 61, 110, 76,
-            152, 106, 231, 100, 78, 187, 28, 39, 191, 10, 206, 78, 127, 79, 247, 192, 164, 51, 237,
-            9, 63, 201, 7, 27, 81, 243, 88, 30, 244, 205, 57, 14, 126, 60, 61, 173, 21, 84, 15,
-            105, 38, 239, 249, 82, 202, 245, 219, 88, 195, 112, 113, 40, 60, 76, 10, 243, 232, 52,
-            27, 0, 84, 247, 85, 140, 99, 140, 165, 145, 140, 96, 55, 0, 174, 155, 241, 166, 252,
-            150, 87, 106, 42, 58, 33, 154, 222, 83, 69, 172, 226, 216, 108, 115, 203, 38, 133, 43,
-            171, 172, 78, 142, 70, 78, 186, 146, 24, 126, 203, 106, 221, 144, 17, 32, 42, 186, 125,
-            134, 186, 174, 214, 137, 212, 234, 202, 79, 241, 28, 222, 98, 83, 76, 254, 90, 210, 12,
-            141, 40, 191, 123, 143, 170, 154, 39, 137, 222, 224, 241, 61, 136, 184, 104, 106, 209,
-            184, 128, 30, 95, 36, 250, 163, 47, 82, 19, 121, 123, 134, 142, 31, 170, 23, 148, 20,
-            80, 157, 252, 103, 192, 204, 229, 10, 66, 84, 49, 21, 197, 110, 208, 202, 124, 217,
-            117, 19, 190, 241, 154, 178, 83, 37, 175, 209, 52, 228, 219, 137, 238, 146, 111, 228,
-            254, 89, 219, 49, 85, 30, 214, 162, 234, 138, 122, 9, 93, 164, 133, 136, 160, 75, 118,
-            87, 14, 170, 92, 109, 244, 40, 234, 40, 216, 72, 207, 81, 161, 252, 252, 0, 38, 206,
-            123, 212, 93, 252, 225, 205, 16, 5, 197, 59, 93, 100, 56, 93, 125, 214, 150, 133, 208,
-            12, 0, 226, 246, 94, 62, 235, 164, 48, 134, 205, 77, 14, 107, 162, 60, 23, 150, 47,
-            198, 5, 214, 125, 12, 150, 63, 128, 95, 237, 209, 55, 5, 11, 59, 187, 0, 254, 180, 226,
-            126, 88, 87, 172, 38, 169, 27, 25, 92, 204, 24, 103, 78, 226, 65, 163, 114, 16, 202,
-            31, 160, 182, 100, 226, 15, 64, 5, 71, 117, 237, 31, 145, 250, 97, 105, 103, 67, 243,
-            2, 208, 60, 72, 16, 199, 57, 170, 99, 151, 62, 100, 53, 23, 45, 123, 225, 170, 149, 65,
-            26, 142, 240, 82, 214, 88, 93, 100, 158, 84, 42, 5, 112, 165, 194, 1, 160, 149, 103,
-            238, 63, 75, 3, 134, 113, 197, 251, 251, 90, 34, 184, 248, 214, 183, 168, 135, 4, 169,
-            134, 194, 106, 83, 108, 176, 26, 159, 80, 143, 105, 111, 53, 4, 1, 240, 44, 240, 149,
-            118, 189, 208, 190, 235, 145, 166, 163, 231, 158, 219, 221, 208, 61, 158, 132, 39, 75,
-            235, 36, 199, 169, 34, 119, 150, 223, 74, 5, 107, 123, 132, 116, 97, 241, 53, 43, 238,
-            115, 189, 195, 124, 127, 172, 5, 109, 112, 149, 190, 19, 202, 253, 171, 53, 105, 123,
-            173, 50, 50, 145, 56, 232, 13, 169, 47, 60, 112, 35, 100, 205, 35, 142, 5, 198, 235,
-            206, 112, 145, 99, 21, 214, 1, 184, 57, 125, 87, 245, 204, 162, 167, 124, 18, 154, 49,
-            25, 144, 181, 58, 184, 212, 59, 252, 72, 167, 228, 60, 118, 113, 65, 50, 150, 235, 163,
-            121, 215, 82, 91, 100, 78, 54, 199, 238, 93, 21, 21, 29, 215, 18, 201, 205, 106, 211,
-            78, 141, 155,
-        ];
-        let _ = Sdgb1_50::decode(data)?;
-        Ok(())
-    }
-}

sdgb-api/src/title/helper/mod.rs

@@ -3,7 +3,7 @@ use nyquest::AsyncClient;
 use crate::{
     ApiError,
     title::{
-        MaiVersionExt as _, Sdgb1_50,
+        MaiVersionExt as _, Sdgb1_53,
         methods::APIMethod,
         model::{GetUserMusicApi, GetUserMusicApiResp},
     },
@@ -21,7 +21,7 @@ pub async fn get_user_all_music(
             next_index,
             user_music_list: mut new_list,
             ..
-        } = Sdgb1_50::request::<_, GetUserMusicApiResp>(
+        } = Sdgb1_53::request::<_, GetUserMusicApiResp>(
             &client,
             APIMethod::GetUserMusicApi,
             user_id,

sdgb-api/src/title/methods/has_uid/mod.rs

@@ -1,21 +0,0 @@
-use crate::title::methods::HasUid;
-
-#[crabtime::function]
-fn uid_get_impl(api_names: Vec<String>) {
-    for api_name in api_names {
-        crabtime::output!(
-            impl HasUid for crate::title::model::{{api_name}}Resp {
-                fn get_uid(&self) -> u32 {
-                    self.user_id
-                }
-            }
-        );
-    }
-}
-
-uid_get_impl!([
-    "GetUserDataApi",
-    "GetUserMusicApi",
-    "GetUserPreviewApi",
-    "GetUserRatingApi"
-]);

sdgb-api/src/title/methods/mod.rs

@@ -53,30 +53,25 @@ pub trait APIExt {
     type Response: for<'de> Deserialize<'de>;
 }
 
-pub trait HasUid {
-    fn get_uid(&self) -> u32;
-}
-
 mod api_ext;
-mod has_uid;
 pub use api_ext::*;
 
 #[cfg(test)]
 mod _test {
-    use crate::title::{MaiVersionExt, Sdgb1_50, methods::APIMethod};
+    use crate::title::{MaiVersionExt, Sdgb1_53, methods::APIMethod};
 
     #[test]
     fn test_obfuscate_1_50() {
         assert_eq!(
-            Sdgb1_50::api_hash(APIMethod::Ping),
+            Sdgb1_53::api_hash(APIMethod::Ping),
             "250b3482854e7697de7d8eb6ea1fabb1"
         );
         assert_eq!(
-            Sdgb1_50::api_hash(APIMethod::GetUserPreviewApi),
+            Sdgb1_53::api_hash(APIMethod::GetUserPreviewApi),
             "004cf848f96d393a5f2720101e30b93d"
         );
         assert_eq!(
-            Sdgb1_50::api_hash(APIMethod::GetUserDataApi),
+            Sdgb1_53::api_hash(APIMethod::GetUserDataApi),
             "3af1e5b298bb5b7379c94934b2e038c5"
         );
     }

sdgb-api/src/title/mod.rs

@@ -135,17 +135,8 @@ pub trait MaiVersionExt: MaiVersion {
     }
 }
 
-pub struct Sdgb1_50;
 pub struct Sdgb1_53;
 
-impl MaiVersion for Sdgb1_50 {
-    const AES_KEY: &[u8; 32] = b"a>32bVP7v<63BVLkY[xM>daZ1s9MBP<R";
-    const AES_IV: &[u8; 16] = b"d6xHIKq]1J]Dt^ue";
-    const OBFUSECATE_SUFFIX: &str = "MaimaiChnB44df8yT";
-
-    const VERSION: &str = "1.50";
-}
-
 impl MaiVersion for Sdgb1_53 {
     const AES_KEY: &[u8; 32] = b"o2U8F6<adcYl25f_qwx_n]5_qxRcbLN>";
     const AES_IV: &[u8; 16] = b"AL<G:k:X6Vu7@_U]";

sdgb-api/src/title/model/get_user_music_api/mod.rs

@@ -16,7 +16,6 @@ pub struct GetUserMusicApi {
 }
 
 #[derive(Default, Debug, Clone, PartialEq, Serialize, Deserialize)]
-#[cfg_attr(feature = "bincode", derive(bincode::Encode, bincode::Decode))]
 #[serde(rename_all = "camelCase")]
 pub struct GetUserMusicApiResp {
     pub user_id: u32,
@@ -25,7 +24,6 @@ pub struct GetUserMusicApiResp {
     pub user_music_list: Vec<UserMusic>,
 }
 
-#[cfg_attr(feature = "bincode", derive(bincode::Encode, bincode::Decode))]
 #[derive(Default, Debug, Clone, PartialEq, Serialize, Deserialize)]
 #[serde(rename_all = "camelCase")]
 pub struct UserMusic {
@@ -33,7 +31,6 @@ pub struct UserMusic {
     pub length: u32,
 }
 
-#[cfg_attr(feature = "bincode", derive(bincode::Encode, bincode::Decode))]
 #[derive(Default, Debug, Clone, PartialEq, Serialize, Deserialize)]
 #[serde(rename_all = "camelCase")]
 pub struct UserMusicDetail {
@@ -85,21 +82,6 @@ pub struct UserMusicDetail {
     pub ext_num2: i64,
 }
 
-#[cfg_attr(feature = "parquet", derive(parquet_derive::ParquetRecordWriter))]
-pub struct UserMusicDetailFlatten {
-    pub user_id: u32,
-    pub music_id: u32,
-    pub level: u8,
-    pub play_count: u32,
-    pub achievement: u32,
-    pub combo_status: u8,
-    pub sync_status: u8,
-    pub deluxscore_max: u16,
-    pub score_rank: u8,
-    pub ext_num1: u32,
-    pub ext_num2: u32,
-}
-
 impl Display for UserMusicDetail {
     fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
         if let Some(music_title) = query_music(self.music_id).map(|i| &i.name) {
@@ -149,35 +131,3 @@ impl Display for UserMusicDetail {
         Ok(())
     }
 }
-
-impl UserMusicDetailFlatten {
-    pub fn new(
-        user_id: u32,
-        UserMusicDetail {
-            music_id,
-            level,
-            play_count,
-            achievement,
-            combo_status,
-            sync_status,
-            deluxscore_max,
-            score_rank,
-            ext_num1,
-            ext_num2,
-        }: UserMusicDetail,
-    ) -> Self {
-        Self {
-            user_id,
-            music_id,
-            level: level as _,
-            sync_status: sync_status as _,
-            deluxscore_max: deluxscore_max as _,
-            score_rank: score_rank as _,
-            play_count: play_count as _,
-            achievement: achievement as _,
-            combo_status: combo_status as _,
-            ext_num1: ext_num1 as _,
-            ext_num2: ext_num2 as _,
-        }
-    }
-}

sdgb-api/src/title/model/get_user_preview_api/mod.rs

@@ -6,17 +6,27 @@ use serde::{Deserialize, Serialize};
 #[serde(rename_all = "camelCase")]
 pub struct GetUserPreviewApi {
     pub user_id: u32,
+    pub client_id: &'static str,
+    pub token: Option<String>,
 }
 
 impl From<u32> for GetUserPreviewApi {
     fn from(user_id: u32) -> Self {
-        Self { user_id }
+        Self {
+            user_id,
+            client_id: "A63E01C2805",
+            token: None,
+        }
+    }
+}
+
+impl Default for GetUserPreviewApi {
+    fn default() -> Self {
+        Self::from(0)
     }
 }
 
 #[derive(Debug, Clone, Serialize, Deserialize)]
-#[cfg_attr(feature = "bincode", derive(bincode::Encode, bincode::Decode))]
-#[cfg_attr(feature = "parquet", derive(parquet_derive::ParquetRecordWriter))]
 #[serde(rename_all = "camelCase")]
 pub struct GetUserPreviewApiResp {
     pub user_id: u32,

sdgb-api/src/title/model/get_user_rating_api/mod.rs

@@ -19,7 +19,6 @@ impl From<u32> for GetUserRatingApi {
     }
 }
 
-#[cfg_attr(feature = "bincode", derive(bincode::Encode, bincode::Decode))]
 #[derive(Debug, Serialize, Deserialize)]
 #[serde(rename_all = "camelCase")]
 pub struct GetUserRatingApiResp {
@@ -27,7 +26,6 @@ pub struct GetUserRatingApiResp {
     pub user_rating: UserRating,
 }
 
-#[cfg_attr(feature = "bincode", derive(bincode::Encode, bincode::Decode))]
 #[derive(Default, Debug, Clone, PartialEq, Serialize, Deserialize)]
 #[serde(rename_all = "camelCase")]
 pub struct UserRating {
@@ -44,7 +42,6 @@ pub struct UserRating {
     pub udemae: Udemae,
 }
 
-#[cfg_attr(feature = "bincode", derive(bincode::Encode, bincode::Decode))]
 #[derive(Default, Debug, Clone, PartialEq, Serialize, Deserialize)]
 #[serde(rename_all = "camelCase")]
 pub struct MusicRating {
@@ -66,18 +63,6 @@ pub struct MusicRating {
     pub achievement: i32,
 }
 
-#[cfg_attr(feature = "parquet", derive(parquet_derive::ParquetRecordWriter))]
-#[derive(Default, Debug, Clone, PartialEq)]
-pub struct MusicRatingFlatten {
-    pub user_id: u32,
-    pub music_id: u32,
-    pub level: u32,
-    pub rom_version: i64,
-    pub achievement: i32,
-    pub dx_rating: u32,
-}
-
-#[cfg_attr(feature = "bincode", derive(bincode::Encode, bincode::Decode))]
 #[derive(Default, Debug, Clone, PartialEq, Serialize, Deserialize)]
 #[serde(rename_all = "camelCase")]
 pub struct Udemae {

sdgb-api/src/title/model/get_user_region_api/mod.rs

@@ -12,44 +12,6 @@ impl From<u32> for GetUserRegionApi {
     }
 }
 
-impl From<GetUserRegionApiResp> for Vec<UserRegionFlatten> {
-    fn from(
-        GetUserRegionApiResp {
-            user_id,
-            user_region_list,
-            ..
-        }: GetUserRegionApiResp,
-    ) -> Self {
-        user_region_list
-            .into_iter()
-            .map(
-                |UserRegion {
-                     region_id,
-                     play_count,
-                     created,
-                 }| {
-                    UserRegionFlatten {
-                        user_id,
-                        region_id,
-                        play_count,
-                        created,
-                    }
-                },
-            )
-            .collect()
-    }
-}
-
-#[derive(Default, Debug, Clone, PartialEq)]
-#[cfg_attr(feature = "parquet", derive(parquet_derive::ParquetRecordWriter))]
-pub struct UserRegionFlatten {
-    pub user_id: u32,
-    pub region_id: u32,
-    pub play_count: i64,
-    pub created: String,
-}
-
-#[cfg_attr(feature = "bincode", derive(bincode::Encode, bincode::Decode))]
 #[derive(Default, Debug, Clone, PartialEq, Serialize, Deserialize)]
 #[serde(rename_all = "camelCase")]
 pub struct GetUserRegionApiResp {
@@ -58,7 +20,6 @@ pub struct GetUserRegionApiResp {
     pub user_region_list: Vec<UserRegion>,
 }
 
-#[cfg_attr(feature = "bincode", derive(bincode::Encode, bincode::Decode))]
 #[derive(Default, Debug, Clone, PartialEq, Serialize, Deserialize)]
 #[serde(rename_all = "camelCase")]
 pub struct UserRegion {

sdgb-api/src/title/model/mod.rs

@@ -18,20 +18,15 @@ pub use get_user_rating_api::{
     GetUserRatingApi,
     GetUserRatingApiResp, // api
     MusicRating,
-    MusicRatingFlatten,
     Udemae,
     UserRating,
 };
 
 mod get_user_music_api;
-pub use get_user_music_api::{
-    GetUserMusicApi, GetUserMusicApiResp, UserMusic, UserMusicDetail, UserMusicDetailFlatten,
-};
+pub use get_user_music_api::{GetUserMusicApi, GetUserMusicApiResp, UserMusic, UserMusicDetail};
 
 mod get_user_region_api;
-pub use get_user_region_api::{
-    GetUserRegionApi, GetUserRegionApiResp, UserRegion, UserRegionFlatten,
-};
+pub use get_user_region_api::{GetUserRegionApi, GetUserRegionApiResp, UserRegion};
 
 mod dxrating;
 pub use dxrating::{

sdgb-api/src/title/model/user_login_api/mod.rs

@@ -12,6 +12,10 @@ pub struct UserLoginApi {
     pub acsess_code: String,
     pub place_id: String,
     pub client_id: String,
+
+    /// QR Login 结果的 Token
+    pub token: Option<String>,
+
     /// false 的情况，二维码扫描后半小时可登录。
     ///
     /// true  的情况，可延长至二维码扫描后的两小时可登录。
@@ -26,6 +30,7 @@ pub struct UserLoginApiResp {
     /// - `1`: success
     /// - `100`: logged
     /// - `102`: QRCode expired
+    /// - `110`: KeyChip mismatch
     pub return_code: i32,
     /// format: yyyy-mm-dd HH:MM:SS
     pub last_login_date: Option<String>,
@@ -36,7 +41,7 @@ pub struct UserLoginApiResp {
 }
 
 impl UserLoginApi {
-    pub fn new(user_id: u32, is_continue: bool) -> Self {
+    pub fn new(user_id: u32, is_continue: bool, token: Option<String>) -> Self {
         let date_time = SystemTime::now()
             .duration_since(UNIX_EPOCH)
             .map(|t| t.as_secs())
@@ -44,16 +49,19 @@ impl UserLoginApi {
 
         info!("login unix timestamp: {date_time}");
 
-        // 爱玩星球焦作解放店
+        // 插电师北京王府井银泰店
         UserLoginApi {
             user_id,
             date_time,
-            region_id: 13,
             acsess_code: "".to_owned(),
-            place_id: 3223.to_string(),
-            is_continue,
             generic_flag: 0,
-            client_id: "A63E01E6170".into(),
+
+            token,
+            is_continue,
+
+            region_id: 1,
+            place_id: 1403.to_string(),
+            client_id: "A63E01C2805".into(),
         }
     }
 }
@@ -62,7 +70,7 @@ impl UserLoginApiResp {
     pub fn error(&self) -> Option<LoginError> {
         match self.return_code {
             1 => None,
-            100 => Some(LoginError::AlreadyLogged),
+            100 | 110 => Some(LoginError::AlreadyLogged),
             102 => Some(LoginError::QRCodeExpired),
             103 => Some(LoginError::AccountUnregistered),
             106 => Some(LoginError::KeychipMismatch),

sdgb-api/src/title/model/user_logout_api/mod.rs

@@ -1,5 +1,3 @@
-use std::time::{SystemTime, UNIX_EPOCH};
-
 use serde::{Deserialize, Serialize};
 
 #[derive(Debug, Clone, PartialEq, Serialize)]
@@ -13,7 +11,7 @@ pub struct UserLogoutApi {
     /// keychip without dash, 11 bytes
     pub client_id: String,
     /// Unix timestamp
-    pub date_time: u64,
+    pub login_date_time: u64,
     #[serde(rename = "type")]
     pub type_: i64,
 }
@@ -27,19 +25,15 @@ pub struct UserLogoutApiResp {
 impl Default for UserLogoutApi {
     fn default() -> Self {
         let user_id = 0;
-        let date_time = SystemTime::now()
-            .duration_since(UNIX_EPOCH)
-            .map(|t| t.as_secs())
-            .unwrap_or_default();
-
         Self {
             user_id,
-            date_time,
             region_id: 22,
             place_id: 3490,
-            client_id: "A63E01E9564".into(),
+            client_id: "A63E01C2805".into(),
             type_: 1,
             access_code: "",
+
+            login_date_time: 0,
         }
     }
 }

sdgb-cli/Cargo.toml

@@ -6,21 +6,13 @@ authors = ["mokurin000"]
 description = "CLI tool for SDGB protocol"
 
 [features]
-default = ["compio", "fetchall"]
+default = ["compio"]
 
 compio = ["dep:compio", "sdgb-api/compio"]
 tokio = ["dep:tokio", "sdgb-api/tokio"]
 
-fetchall = [
-    "dep:redb",
-    "dep:futures-util",
-    "dep:parquet",
-    "dep:music-db",
-    "sdgb-api/parquet",
-]
-
 [dependencies]
-sdgb-api = { workspace = true, features = ["bincode"] }
+sdgb-api = { workspace = true, features = [] }
 music-db = { workspace = true, optional = true }
 
 # (de)serialization
@@ -32,10 +24,6 @@ strum = { workspace = true }
 spdlog-rs = { workspace = true }
 snafu = { workspace = true }
 
-
-# kv database
-redb = { workspace = true, optional = true }
-
 # async runtime
 tokio = { workspace = true, features = ["macros"], optional = true }
 compio = { workspace = true, features = ["macros"], optional = true }
@@ -49,7 +37,5 @@ ctrlc = { version = "3.4.7", features = ["termination"] }
 # magic macro
 crabtime = { workspace = true }
 
-parquet = { workspace = true, optional = true }
-
 [build-dependencies]
 version_check = "0.9.5"

sdgb-cli/src/cache/mod.rs

@@ -1,64 +0,0 @@
-use std::sync::LazyLock;
-
-use redb::{ReadTransaction, ReadableDatabase as _, Table, TableDefinition, WriteTransaction};
-
-static DATABASE: LazyLock<redb::Database> = LazyLock::new(|| {
-    let mut db = redb::Database::builder()
-        .create("players.redb")
-        .expect("failed to open database");
-    _ = db.compact();
-    db
-});
-
-pub fn write_txn() -> Result<WriteTransaction, redb::Error> {
-    Ok(DATABASE.begin_write()?)
-}
-
-pub fn read_txn() -> Result<ReadTransaction, redb::Error> {
-    Ok(DATABASE.begin_read()?)
-}
-
-pub fn open_table<'a>(
-    write: &'a WriteTransaction,
-    definition: TableDefinition<'_, u32, Vec<u8>>,
-) -> Result<Table<'a, u32, Vec<u8>>, redb::Error> {
-    Ok(write.open_table(definition)?)
-}
-
-pub fn open_table_ro(
-    read: &ReadTransaction,
-    definition: TableDefinition<'_, u32, Vec<u8>>,
-) -> Result<redb::ReadOnlyTable<u32, Vec<u8>>, redb::Error> {
-    Ok(read.open_table(definition)?)
-}
-
-#[crabtime::function]
-fn table_definitions_impl(tables: Vec<String>) {
-    let mut defs: Vec<String> = Vec::new();
-
-    for table in tables {
-        let definition = table.to_uppercase();
-        let table_name = format!("\"{table}\"");
-
-        crabtime::output!(
-            pub const {{definition}}: TableDefinition<'_, u32, Vec<u8>> = redb::TableDefinition::new({{table_name}});
-        );
-
-        defs.push(format!("write_txn.open_table({definition})?;"));
-    }
-
-    let init_statements = defs.join("\n");
-
-    crabtime::output!(
-        pub fn init_db() -> Result<(), redb::Error> {
-            let write_txn = DATABASE.begin_write()?;
-            {
-                { init_statements }
-            }
-            write_txn.commit()?;
-            Ok(())
-        }
-    );
-}
-
-table_definitions_impl!(["players", "b50", "records", "regions"]);

sdgb-cli/src/commands.rs

@@ -31,7 +31,7 @@ pub enum Commands {
 
     /// Retrieve update package of SDGB
     AuthLite {
-        #[arg(short, long, default_value = "1.50")]
+        #[arg(short, long, default_value = "1.53")]
         title_ver: String,
         #[arg(long, default_value = "SDGB")]
         variant: AuthLiteVariant,
@@ -43,6 +43,8 @@ pub enum Commands {
     Preview {
         #[arg(short, long)]
         user_id: u32,
+        #[arg(short, long)]
+        token: Option<String>,
     },
     /// Get B35 + B15 play records
     Rating {
@@ -79,57 +81,10 @@ pub enum Commands {
         user_id: u32,
         #[arg(long)]
         skip_login: bool,
+        #[arg(short, long)]
+        token: Option<String>,
     },
 
-    /// Scrape all user, read possible id from stdin
-    #[cfg(feature = "fetchall")]
-    ListAllUser {
-        #[arg(short, long, default_value_t = 5)]
-        concurrency: usize,
-    },
-    #[cfg(feature = "fetchall")]
-    /// Scrape B50 data
-    ScrapeAllB50 {
-        #[arg(short, long, default_value_t = 5)]
-        concurrency: usize,
-
-        #[arg(long, default_value_t = 1000)]
-        min_rating: i64,
-        #[arg(long, default_value_t = 16500)]
-        max_rating: i64,
-    },
-    /// Scrape Region data
-    #[cfg(feature = "fetchall")]
-    ScrapeAllRegion {
-        #[arg(short, long, default_value_t = 5)]
-        concurrency: usize,
-
-        #[arg(long, default_value_t = 1000)]
-        min_rating: i64,
-        #[arg(long, default_value_t = 16500)]
-        max_rating: i64,
-    },
-    /// Scrape all player record
-    #[cfg(feature = "fetchall")]
-    ScrapeAllRecord {
-        #[arg(short, long, default_value_t = 5)]
-        concurrency: usize,
-
-        #[arg(long, default_value_t = 10000)]
-        min_rating: i64,
-        #[arg(long, default_value_t = 16400)]
-        max_rating: i64,
-    },
-
-    #[cfg(feature = "fetchall")]
-    ListAllUserDump {},
-    #[cfg(feature = "fetchall")]
-    ScrapeAllB50Dump {},
-    #[cfg(feature = "fetchall")]
-    ScrapeAllRegionDump {},
-    #[cfg(feature = "fetchall")]
-    ScrapeAllRecordDump {},
-
     Logout {
         #[arg(short, long)]
         user_id: u32,

sdgb-cli/src/main.rs

@@ -14,7 +14,7 @@ use sdgb_api::{
     all_net::{GetResponse, QRCode},
     auth_lite::{SDGB, SDHJ, delivery_raw},
     title::{
-        MaiVersionExt, Sdgb1_50, Sdgb1_53,
+        MaiVersionExt, Sdgb1_53,
         helper::get_user_all_music,
         methods::APIMethod,
         model::{
@@ -32,8 +32,6 @@ use crate::{
     utils::{human_readable_display, json_display, login_action},
 };
 
-#[cfg(feature = "fetchall")]
-mod cache;
 mod commands;
 mod utils;
 
@@ -118,7 +116,7 @@ async fn main() -> Result<(), Box<dyn snafu::Error>> {
             }
         }
         Commands::Rating { user_id, format } => {
-            let rating: GetUserRatingApiResp = Sdgb1_50::request(
+            let rating: GetUserRatingApiResp = Sdgb1_53::request(
                 &client,
                 APIMethod::GetUserRatingApi,
                 user_id,
@@ -147,13 +145,13 @@ async fn main() -> Result<(), Box<dyn snafu::Error>> {
             }
         }
         Commands::Logout { user_id, timestamp } => {
-            let logout: UserLogoutApiResp = Sdgb1_50::request(
+            let logout: UserLogoutApiResp = Sdgb1_53::request(
                 &client,
                 APIMethod::UserLogoutApi,
                 user_id,
                 UserLogoutApi {
                     user_id,
-                    date_time: timestamp,
+                    login_date_time: timestamp,
                     ..Default::default()
                 },
             )
@@ -165,34 +163,38 @@ async fn main() -> Result<(), Box<dyn snafu::Error>> {
                 json_display(logout)?;
             }
         }
-        Commands::Preview { user_id } => {
-            let preview: GetUserPreviewApiResp = Sdgb1_50::request(
+        Commands::Preview { user_id, token } => {
+            let preview: GetUserPreviewApiResp = Sdgb1_53::request(
                 &client,
                 APIMethod::GetUserPreviewApi,
                 user_id,
-                GetUserPreviewApi { user_id },
+                GetUserPreviewApi {
+                    user_id,
+                    client_id: "A63E01C2805",
+                    token,
+                },
             )
             .await?;
 
             human_readable_display(preview, human_readable)?;
         }
         Commands::Ping => {
+            for _ in 0..10 {
                 let time = SystemTime::now();
-            let decoded: PingResp =
-                Sdgb1_50::request(&client, APIMethod::Ping, "", Ping {}).await?;
-            info!(
-                "sdgb 1.50 resp: {decoded}, {}ms",
-                time.elapsed().unwrap_or_default().as_millis()
-            );
-
-            let time = SystemTime::now();
-            let decoded: PingResp =
-                Sdgb1_53::request(&client, APIMethod::Ping, "", Ping {}).await?;
+                match Sdgb1_53::request::<_, PingResp>(&client, APIMethod::Ping, "", Ping {}).await
+                {
+                    Ok(decoded) => {
                         info!(
                             "sdgb 1.53 resp: {decoded}, {}ms",
                             time.elapsed().unwrap_or_default().as_millis()
                         );
                     }
+                    Err(e) => {
+                        error!("sdgb 1.53 error: {e}");
+                    }
+                }
+            }
+        }
         Commands::QRLogin { ref qrcode_content } => {
             let qrcode = QRCode { qrcode_content };
             let resp = qrcode.login(&client).await;
@@ -217,221 +219,12 @@ async fn main() -> Result<(), Box<dyn snafu::Error>> {
             println!("{}", String::from_utf8_lossy(&resp));
         }
 
-        #[cfg(feature = "fetchall")]
-        Commands::ListAllUser { concurrency } => {
-            use crate::{cache::PLAYERS, utils::helpers::cached_concurrent_fetch};
-            use sdgb_api::title::methods::GetUserPreviewApiExt;
-            use std::io::BufRead as _;
-
-            let mut user_ids = Vec::new();
-            {
-                let mut stdin = std::io::stdin().lock();
-                let mut buf = String::new();
-
-                while stdin.read_line(&mut buf).is_ok_and(|size| size != 0) {
-                    if buf.is_empty() {
-                        continue;
-                    }
-
-                    let user_id: u32 = buf.trim().parse()?;
-                    buf.clear();
-                    user_ids.push(user_id);
-                }
-            }
-
-            cached_concurrent_fetch::<GetUserPreviewApiExt>(
-                user_ids,
-                &client,
-                concurrency,
-                PLAYERS,
-            )
-            .await?;
-        }
-
-        #[cfg(feature = "fetchall")]
-        Commands::ScrapeAllRecord {
-            concurrency,
-            min_rating,
-            max_rating,
-        } => {
-            use crate::{
-                cache::{PLAYERS, RECORDS},
-                utils::helpers::{cached_concurrent_fetch_userfn, read_cache},
-            };
-
-            let players: Vec<GetUserPreviewApiResp> = read_cache(PLAYERS)?;
-            cached_concurrent_fetch_userfn(
-                players
-                    .iter()
-                    .filter(|p| p.player_rating >= min_rating && p.player_rating <= max_rating)
-                    .map(|p| p.user_id)
-                    .collect::<Vec<u32>>(),
-                &client,
-                concurrency,
-                RECORDS,
-                get_user_all_music,
-            )
-            .await?;
-        }
-        #[cfg(feature = "fetchall")]
-        Commands::ScrapeAllB50 {
-            concurrency,
-            min_rating,
-            max_rating,
-        } => {
-            use sdgb_api::title::methods::GetUserRatingApiExt;
-
-            use crate::{
-                cache::{B50, PLAYERS},
-                utils::helpers::{cached_concurrent_fetch, read_cache},
-            };
-
-            let players: Vec<GetUserPreviewApiResp> = read_cache(PLAYERS)?;
-            cached_concurrent_fetch::<GetUserRatingApiExt>(
-                players
-                    .iter()
-                    .filter(|p| p.player_rating >= min_rating && p.player_rating <= max_rating)
-                    .map(|p| p.user_id)
-                    .collect::<Vec<u32>>(),
-                &client,
-                concurrency,
-                B50,
-            )
-            .await?;
-        }
-        #[cfg(feature = "fetchall")]
-        Commands::ScrapeAllRegion {
-            concurrency,
-            min_rating,
-            max_rating,
-        } => {
-            use sdgb_api::title::methods::GetUserRegionApiExt;
-
-            use crate::{
-                cache::{PLAYERS, REGIONS},
-                utils::helpers::{cached_concurrent_fetch, read_cache},
-            };
-
-            let players: Vec<GetUserPreviewApiResp> = read_cache(PLAYERS)?;
-            cached_concurrent_fetch::<GetUserRegionApiExt>(
-                players
-                    .iter()
-                    .filter(|p| p.player_rating >= min_rating && p.player_rating <= max_rating)
-                    .map(|p| p.user_id)
-                    .collect::<Vec<u32>>(),
-                &client,
-                concurrency,
-                REGIONS,
-            )
-            .await?;
-        }
-
-        #[cfg(feature = "fetchall")]
-        Commands::ListAllUserDump {} => {
-            use crate::{
-                cache::PLAYERS,
-                utils::helpers::{dump_parquet, read_cache},
-            };
-
-            let players: Vec<GetUserPreviewApiResp> = read_cache(PLAYERS)?;
-            dump_parquet(players, "players.parquet")?;
-        }
-        #[cfg(feature = "fetchall")]
-        Commands::ScrapeAllRegionDump {} => {
-            use crate::{
-                cache::REGIONS,
-                utils::helpers::{dump_parquet, read_cache},
-            };
-            use sdgb_api::title::model::{GetUserRegionApiResp, UserRegionFlatten};
-
-            let regions: Vec<GetUserRegionApiResp> = read_cache(REGIONS)?;
-            let regions_flat = regions
-                .into_iter()
-                .map(Vec::<UserRegionFlatten>::from)
-                .flatten()
-                .collect::<Vec<_>>();
-            dump_parquet(regions_flat, "regions.parquet")?;
-        }
-        #[cfg(feature = "fetchall")]
-        Commands::ScrapeAllRecordDump {} => {
-            use crate::{
-                cache::RECORDS,
-                utils::helpers::{dump_parquet, read_cache},
-            };
-            use sdgb_api::title::model::GetUserMusicApiResp;
-            use sdgb_api::title::model::UserMusicDetailFlatten;
-
-            let records: Vec<GetUserMusicApiResp> = read_cache(RECORDS)?;
-            dump_parquet(
-                records
-                    .into_iter()
-                    .map(|resp| {
-                        resp.user_music_list
-                            .into_iter()
-                            .map(|music| music.user_music_detail_list)
-                            .flatten()
-                            .map(move |detail| UserMusicDetailFlatten::new(resp.user_id, detail))
-                    })
-                    .flatten()
-                    .collect::<Vec<UserMusicDetailFlatten>>(),
-                "records.parquet",
-            )?;
-        }
-        #[cfg(feature = "fetchall")]
-        Commands::ScrapeAllB50Dump {} => {
-            use sdgb_api::title::model::{MusicRating, MusicRatingFlatten};
-
-            use crate::{
-                cache::B50,
-                utils::helpers::{dump_parquet, read_cache},
-            };
-
-            let records: Vec<GetUserRatingApiResp> = read_cache(B50)?;
-            dump_parquet::<MusicRatingFlatten>(
-                records
-                    .into_iter()
-                    .map(
-                        |GetUserRatingApiResp {
-                             user_id,
-                             user_rating,
-                         }| {
-                            user_rating
-                                .rating_list
-                                .into_iter()
-                                .chain(user_rating.next_rating_list)
-                                .filter_map(
-                                    move |MusicRating {
-                                              music_id,
-                                              level,
-                                              rom_version,
-                                              achievement,
-                                          }| {
-                                        let (_rank, dx_rating) =
-                                            music_db::query_music_level(music_id, level)?
-                                                .dx_rating(achievement);
-                                        Some(MusicRatingFlatten {
-                                            user_id,
-                                            music_id,
-                                            level,
-                                            rom_version,
-                                            achievement,
-                                            dx_rating,
-                                        })
-                                    },
-                                )
-                        },
-                    )
-                    .flatten()
-                    .collect::<Vec<_>>(),
-                "b50.parquet",
-            )?;
-        }
-
         Commands::Userdata {
             user_id,
             skip_login,
+            token,
         } => {
-            let action = async |_| match Sdgb1_50::request::<_, GetUserDataApiResp>(
+            let action = async |_| match Sdgb1_53::request::<_, GetUserDataApiResp>(
                 &client,
                 APIMethod::GetUserDataApi,
                 user_id,
@@ -450,7 +243,7 @@ async fn main() -> Result<(), Box<dyn snafu::Error>> {
             if skip_login {
                 action(UserLoginApiResp::default()).await;
             } else {
-                login_action(&client, user_id, action).await?;
+                login_action(&client, user_id, token, action).await?;
             }
         }
     }

sdgb-cli/src/utils/helpers/mod.rs

@@ -1,183 +0,0 @@
-use std::sync::Arc;
-use std::{fs::OpenOptions, io::BufWriter};
-use std::{path::Path, sync::atomic::Ordering};
-
-use futures_util::StreamExt;
-use nyquest_preset::nyquest::AsyncClient;
-
-use parquet::basic::BrotliLevel;
-use parquet::file::properties::WriterProperties;
-use parquet::file::writer::SerializedFileWriter;
-use parquet::record::RecordWriter;
-use redb::ReadableTable;
-use redb::TableDefinition;
-use spdlog::{error, info};
-
-use sdgb_api::title::MaiVersionExt;
-use sdgb_api::title::{Sdgb1_50, methods::APIExt};
-use sdgb_api::{ApiError, bincode};
-
-use bincode::{BorrowDecode, Encode, borrow_decode_from_slice};
-
-use crate::{EARLY_QUIT, cache};
-
-#[allow(unused)]
-pub fn read_cache_keys(
-    definition: TableDefinition<'_, u32, Vec<u8>>,
-) -> Result<Vec<u32>, Box<dyn snafu::Error>> {
-    let txn = cache::read_txn()?;
-    let table = cache::open_table_ro(&txn, definition)?;
-
-    Ok(table
-        .iter()?
-        .flatten()
-        .map(|(value, _)| value.value())
-        .collect::<Vec<u32>>())
-}
-
-pub fn read_cache<D>(
-    definition: TableDefinition<'_, u32, Vec<u8>>,
-) -> Result<Vec<D>, Box<dyn snafu::Error>>
-where
-    D: for<'d> BorrowDecode<'d, ()>,
-{
-    let txn = cache::read_txn()?;
-    let table = cache::open_table_ro(&txn, definition)?;
-
-    let config =
-        bincode::config::Configuration::<bincode::config::LittleEndian>::default().with_no_limit();
-
-    Ok(table
-        .iter()?
-        .flatten()
-        .map(|d| borrow_decode_from_slice(&d.1.value(), config))
-        .flatten()
-        .map(|(value, _)| value)
-        .collect::<Vec<D>>())
-}
-
-pub fn dump_parquet<D>(
-    data: impl Into<Vec<D>>,
-    output_path: impl AsRef<Path>,
-) -> Result<(), Box<dyn snafu::Error>>
-where
-    for<'a> &'a [D]: RecordWriter<D>,
-{
-    let data = data.into();
-    let file = OpenOptions::new()
-        .create(true)
-        .truncate(true)
-        .write(true)
-        .open(output_path)?;
-
-    #[cfg(file_lock_ready)]
-    file.try_lock()?;
-
-    let writer = BufWriter::new(file);
-    let schema = data.as_slice().schema()?;
-    let props = Arc::new(
-        WriterProperties::builder()
-            .set_compression(parquet::basic::Compression::BROTLI(BrotliLevel::try_new(
-                6,
-            )?))
-            .build(),
-    );
-
-    let mut writer = SerializedFileWriter::new(writer, schema, props).unwrap();
-    let mut row_group = writer.next_row_group().unwrap();
-
-    data.as_slice().write_to_row_group(&mut row_group)?;
-    row_group.close()?;
-
-    writer.close().unwrap();
-    info!("dumped {} records", data.len());
-
-    Ok(())
-}
-
-pub async fn cached_concurrent_fetch<A: APIExt>(
-    user_ids: impl Into<Vec<u32>>,
-    client: &AsyncClient,
-    concurrency: usize,
-    definition: TableDefinition<'_, u32, Vec<u8>>,
-) -> Result<(), Box<dyn snafu::Error>>
-where
-    A::Payload: From<u32>,
-    A::Response: Encode + for<'a> BorrowDecode<'a, ()>,
-{
-    cached_concurrent_fetch_userfn(
-        user_ids,
-        client,
-        concurrency,
-        definition,
-        async |client, user_id| {
-            Sdgb1_50::request_ext::<A>(client, A::Payload::from(user_id), user_id).await
-        },
-    )
-    .await
-}
-
-pub async fn cached_concurrent_fetch_userfn<R>(
-    user_ids: impl Into<Vec<u32>>,
-    client: &AsyncClient,
-    concurrency: usize,
-    definition: TableDefinition<'_, u32, Vec<u8>>,
-    scrape: impl AsyncFn(&AsyncClient, u32) -> Result<R, ApiError>,
-) -> Result<(), Box<dyn snafu::Error>>
-where
-    R: Encode + for<'a> BorrowDecode<'a, ()>,
-{
-    let _ = cache::init_db();
-
-    let user_ids = user_ids.into();
-    let read = cache::read_txn()?;
-    let write = cache::write_txn()?;
-    let config = sdgb_api::bincode::config::Configuration::<
-            sdgb_api::bincode::config::LittleEndian,
-        >::default()
-        .with_no_limit();
-
-    info!("number of user_id: {}", user_ids.len());
-
-    let collect = futures_util::stream::iter(user_ids)
-        .map(async |user_id| {
-            {
-                let cache_table = cache::open_table_ro(&read, definition)?;
-                let data = cache_table.get(user_id)?;
-                if data.is_some() {
-                    return Ok(());
-                }
-            }
-
-            if EARLY_QUIT.load(Ordering::Relaxed) {
-                return Err("early skip due to ctrl-c")?;
-            }
-
-            let resp = scrape(&client, user_id).await;
-            match &resp {
-                Ok(resp) => {
-                    use sdgb_api::bincode::encode_to_vec;
-
-                    if let Ok(mut table) = cache::open_table(&write, definition)
-                        && let Ok(encoded) = encode_to_vec(resp, config)
-                    {
-                        info!("encode length for {user_id}: {}", encoded.len());
-                        _ = table.insert(user_id, encoded);
-                    }
-                }
-                Err(sdgb_api::ApiError::JSON { .. }) => {}
-                Err(e) => {
-                    error!("fetch failed: {e}");
-                }
-            }
-
-            Result::<_, Box<dyn snafu::Error>>::Ok(())
-        })
-        .buffer_unordered(concurrency) // slower to avoid being banned
-        .collect::<Vec<_>>()
-        .await;
-    drop(collect);
-
-    let _ = write.commit();
-    Ok(())
-}

sdgb-cli/src/utils/mod.rs

@@ -4,7 +4,7 @@ use nyquest_preset::nyquest::AsyncClient;
 use sdgb_api::{
     ApiError,
     title::{
-        MaiVersionExt as _, Sdgb1_50,
+        MaiVersionExt as _, Sdgb1_53,
         methods::APIMethod,
         model::{UserLoginApi, UserLoginApiResp, UserLogoutApi, UserLogoutApiResp},
     },
@@ -15,13 +15,14 @@ use spdlog::info;
 pub async fn login_action<R>(
     client: &AsyncClient,
     user_id: u32,
+    token: Option<String>,
     action: impl AsyncFnOnce(UserLoginApiResp) -> R,
 ) -> Result<R, ApiError> {
-    let login = UserLoginApi::new(user_id, true);
-    let date_time = login.date_time;
+    let login = UserLoginApi::new(user_id, true, token);
+    let login_date_time = login.date_time;
 
     let login_resp: UserLoginApiResp =
-        Sdgb1_50::request(&client, APIMethod::UserLoginApi, user_id, login).await?;
+        Sdgb1_53::request(&client, APIMethod::UserLoginApi, user_id, login).await?;
 
     match login_resp.error() {
         None => info!("login succeed"),
@@ -30,13 +31,13 @@ pub async fn login_action<R>(
 
     let return_data = action(login_resp).await;
 
-    let logout_resp = Sdgb1_50::request::<_, UserLogoutApiResp>(
+    let logout_resp = Sdgb1_53::request::<_, UserLogoutApiResp>(
         &client,
         APIMethod::UserLogoutApi,
         user_id,
         UserLogoutApi {
             user_id,
-            date_time,
+            login_date_time,
             ..Default::default()
         },
     )
@@ -64,6 +65,3 @@ pub fn human_readable_display(
 
     Ok(())
 }
-
-#[cfg(feature = "fetchall")]
-pub mod helpers;