chore: remove pcrt function

C-style shit syntax

.gitignore

@@ -1 +1,19 @@
 /target
+
+/*.txt
+
+/players.redb*
+
+/*.json*
+/players*.parquet
+/region*.parquet
+/records*.parquet
+
+/.python-version
+/uv.lock
+
+/.venv
+
+*.pyc
+
+/*html

Cargo.toml

@@ -1,16 +1,30 @@
 [workspace]
-members = ["sdgb-api", "sdgb-cli"]
+members = ["music_db", "sdgb-api", "sdgb-cli"]
 resolver = "3"
+default-members = ["sdgb-cli"]
 
 [workspace.dependencies]
-sdgb-api = { path = "./sdgb-api" }
+music-db = { path = "./music_db", default-features = false }
+sdgb-api = { path = "./sdgb-api", default-features = false }
+
+spdlog-rs = { version = "0.4.3", default-features = false, features = [
+    "level-debug",
+    "release-level-info",
+] }
 
 snafu = { version = "0.8.6", features = ["backtrace", "rust_1_81"] }
+serde = { version = "1.0.219", features = ["derive"] }
 serde_json = "1.0.141"
 strum = { version = "0.27.2", features = ["derive"] }
+tokio = { version = "1.47.1", features = ["rt-multi-thread"] }
+compio = { version = "0.15.0", features = ["runtime"] }
+redb = "3.0.0"
+crabtime = { git = "https://github.com/wdanilo/crabtime.git", rev = "2ed856f5" }
+
+parquet = "56.0.0"
 
 [profile.release]
-lto = true
+lto = "thin"
 strip = true
-codegen-units = 1
+codegen-units = 4
 panic = "abort"

README.md

@@ -0,0 +1,5 @@
+# sdgb-utils-rs
+
+- SBGA 舞萌DX API 文档参考
+- “裸” cli 工具，没多少人性化功能
+- 暂时不完整开放，留在私仓

music_db/Cargo.toml

@@ -0,0 +1,19 @@
+[package]
+name = "music-db"
+version = "0.1.0"
+edition = "2024"
+
+[dependencies]
+rustc-hash = "2.1.1"
+rust_decimal = { version = "1.37.2", default-features = false, features = [
+    "serde-with-arbitrary-precision",
+    "macros",
+] }
+
+serde = { workspace = true }
+serde_json = { workspace = true }
+spdlog-rs = { workspace = true, optional = true }
+
+[features]
+default = ["log"]
+log = ["dep:spdlog-rs"]

music_db/src/lib.rs

@@ -0,0 +1,117 @@
+use std::sync::LazyLock;
+
+use rust_decimal::{Decimal, dec, serde::DecimalFromString};
+use rustc_hash::FxHashMap;
+use serde::Deserialize;
+
+#[derive(Deserialize)]
+#[serde(rename_all = "camelCase")]
+pub struct MusicInfo {
+    pub id: u32,
+    pub name: String,
+    pub version: i64,
+    pub levels: Vec<Level>,
+}
+
+#[derive(Deserialize)]
+#[serde(rename_all = "camelCase")]
+pub struct Level {
+    /// 0, 1, 2, 3, 4, 5
+    pub level: u32,
+    /// for example: "13.7"
+    pub difficulty: DecimalFromString,
+}
+
+type MusicDB = FxHashMap<u32, MusicInfo>;
+
+pub fn preload_db() {
+    _ = &*MUSIC_DB;
+}
+
+pub fn query_music(music_id: u32) -> Option<&'static MusicInfo> {
+    MUSIC_DB.as_ref()?.get(&music_id)
+}
+pub fn query_music_level(music_id: u32, level: u32) -> Option<&'static Level> {
+    MUSIC_DB
+        .as_ref()?
+        .get(&music_id)?
+        .levels
+        .iter()
+        .find(|d| d.level == level)
+}
+
+pub static MUSIC_DB: LazyLock<Option<MusicDB>> = LazyLock::new(|| {
+    let db: Vec<MusicInfo> = serde_json::from_slice(include_bytes!("musicDB.json"))
+        .inspect_err(|_e| {
+            #[cfg(feature = "log")]
+            spdlog::warn!("failed to load musicDB: {_e}")
+        })
+        .ok()?;
+
+    Some(db.into_iter().map(|entry| (entry.id, entry)).collect())
+});
+
+impl Level {
+    /// achievement: xxx.xxxx% * 10000
+    ///
+    /// This will **NOT** ignore utage level, you can calculate a in-theory DX Rating.
+    ///
+    /// On invalid input, it returns 0.
+    pub fn dx_rating(&self, achievement: i32) -> (&'static str, u32) {
+        let achievement = achievement.min(1005000); // SSS+ case
+        let (rank, _, factor) = RANKS
+            .into_iter()
+            .rev()
+            .find(|&(_, threshold, _)| threshold <= achievement)
+            .unwrap(); // save here, due to zero threshold
+        let difficulty_rank: Decimal = self.difficulty.value;
+        let achievement = Decimal::new(achievement as _, 4);
+
+        #[cfg(feature = "log")]
+        spdlog::info!("factor: {factor}, achievement: {achievement}");
+
+        // when ach > 100.5%, calculate as 100.5%
+        let rating: u32 = (factor * difficulty_rank * achievement)
+            .floor()
+            .try_into()
+            .unwrap_or_default();
+        (rank, rating)
+    }
+}
+
+const RANKS: [(&'static str, i32, Decimal); 23] = [
+    ("D", 0, dec!(0.0)),
+    ("D", 100000, dec!(0.016)),
+    ("D", 200000, dec!(0.032)),
+    ("D", 300000, dec!(0.048)),
+    ("D", 400000, dec!(0.064)),
+    ("C", 500000, dec!(0.080)),
+    ("B", 600000, dec!(0.096)),
+    ("BB", 700000, dec!(0.112)),
+    ("BBB", 750000, dec!(0.120)),
+    ("BBB", 799999, dec!(0.128)),
+    ("A", 800000, dec!(0.136)),
+    ("AA", 900000, dec!(0.152)),
+    ("AAA", 940000, dec!(0.168)),
+    ("AAA", 969999, dec!(0.176)),
+    ("S", 970000, dec!(0.200)),
+    ("S+", 980000, dec!(0.203)),
+    ("S+", 989999, dec!(0.206)),
+    ("SS", 990000, dec!(0.208)),
+    ("SS+", 995000, dec!(0.211)),
+    ("SS+", 999999, dec!(0.214)),
+    ("SSS", 1000000, dec!(0.216)),
+    ("SSS", 1004999, dec!(0.222)),
+    ("SSS+", 1005000, dec!(0.224)),
+];
+
+#[cfg(test)]
+mod tests {
+    use crate::query_music_level;
+
+    #[test]
+    fn test_rating_calculate() {
+        let level = query_music_level(11696, 3).expect("not found");
+        assert_eq!(level.dx_rating(953184), ("AAA", 184));
+    }
+}

pyproject.toml

@@ -0,0 +1,14 @@
+[project]
+name = "sdgb-utils-rs"
+version = "0.1.0"
+description = "Add your description here"
+readme = "README.md"
+requires-python = ">=3.12"
+dependencies = [
+    "diskcache>=5.6.3",
+    "loguru>=0.7.3",
+    "orjson>=3.11.1",
+    "polars>=1.32.0",
+    "polars-hash>=0.5.4",
+    "pyecharts>=2.0.8",
+]

sdgb-api/Cargo.toml

@@ -5,10 +5,28 @@ edition = "2024"
 
 license = "GPL-3.0"
 
+[features]
+default = ["compio", "bincode"]
+compio = ["dep:compio"]
+tokio = ["dep:tokio"]
+bincode = ["dep:bincode"]
+
+parquet = ['dep:parquet', 'dep:parquet_derive']
+
 [dependencies]
 snafu = { workspace = true }
 serde_json = { workspace = true }
 strum = { workspace = true }
+tokio = { workspace = true, optional = true }
+compio = { workspace = true, optional = true }
+spdlog-rs = { workspace = true }
+music-db = { workspace = true }
+
+# (de)serialization
+serde = { workspace = true }
+
+# magic macro
+crabtime = { workspace = true }
 
 # hashing
 digest = "0.10.7"
@@ -19,13 +37,15 @@ md5 = "0.8.0"
 chrono = "0.4.41"
 
 # network request
-nyquest = { version = "0.2.0", features = ["async", "json"] }
+nyquest = { version = "0.3.0", features = ["async", "json"] }
 
-# (de)serialization
-serde = { version = "1.0.219", features = ["derive"] }
 
 # compression / encryption
 flate2 = "1.1.2"
-cbc = "0.1.2"
+cbc = { version = "0.1.2", features = ["alloc"] }
 aes = "0.8.4"
 cipher = { version = "0.4.4", features = ["block-padding"] }
+bincode = { version = "2.0.1", optional = true }
+
+parquet = { version = "56.0.0", optional = true }
+parquet_derive = { version = "56.0.0", optional = true }

sdgb-api/benches/enc_dec.rs

@@ -0,0 +1,22 @@
+#![feature(test)]
+extern crate test;
+
+use sdgb_api::title::{MaiVersionExt, Sdgb1_50};
+
+#[bench]
+pub fn sdgb_150_enc_short(b: &mut test::Bencher) {
+    b.iter(|| _ = Sdgb1_50::encode(b"Hello world"));
+}
+
+#[bench]
+pub fn sdgb_150_enc_4k(b: &mut test::Bencher) {
+    let data = [1u8; 4096];
+    b.iter(|| _ = Sdgb1_50::encode(data));
+}
+
+#[bench]
+pub fn sdgb_150_dec_4k(b: &mut test::Bencher) {
+    let data = [1u8; 4096];
+    let enc_data = Sdgb1_50::encode(data).unwrap();
+    b.iter(|| _ = Sdgb1_50::decode(enc_data.clone()));
+}

sdgb-api/src/all_net/mod.rs

@@ -4,12 +4,13 @@ use nyquest::{AsyncClient, Body, Request, header::USER_AGENT};
 
 mod model;
 use model::{GetResponse, GetUserId};
+use serde::Serialize;
 
 pub struct QRCode<'a> {
     pub qrcode_content: &'a str,
 }
 
-#[derive(Debug, snafu::Snafu)]
+#[derive(Debug, snafu::Snafu, Serialize)]
 pub enum QRLoginError {
     #[snafu(display("QRCode expired [10mins]"))]
     QRCodeExpired10,
@@ -23,13 +24,17 @@ pub enum QRLoginError {
     #[snafu(context(false))]
     #[snafu(display("request error: {source}"))]
     NyquestError {
+        #[serde(skip)]
         source: nyquest::Error,
+        #[serde(skip)]
         backtrace: Backtrace,
     },
 
     #[snafu(context(false))]
     JSONError {
+        #[serde(skip)]
         source: serde_json::error::Error,
+        #[serde(skip)]
         backtrace: Backtrace,
     },
 }
@@ -53,7 +58,7 @@ impl QRCode<'_> {
             2 => Err(QRLoginError::QRCodeExpired10),
             1 => Err(QRLoginError::QRCodeExpired30),
             50 => Err(QRLoginError::BadSingature),
-            error_kind @ _ => Err(QRLoginError::Unknown { error_kind }),
+            error_kind => Err(QRLoginError::Unknown { error_kind }),
         }
     }
 }

sdgb-api/src/error.rs

@@ -1,26 +1,46 @@
 use aes::cipher::{block_padding::UnpadError, inout::PadError};
 use snafu::Snafu;
 
+use crate::title::model::LoginError;
+
 #[derive(Debug, Snafu)]
 pub enum ApiError {
+    JoinError,
+
     #[snafu(display("api returned nothing!"))]
     EmptyResponse,
     #[snafu(display("encrypt data: {error}"))]
-    Pad { error: PadError },
+    Pad {
+        error: PadError,
+    },
     #[snafu(display("decrypt data: {error}"))]
-    Unpad { error: UnpadError },
+    Unpad {
+        error: UnpadError,
+    },
 
     #[snafu(display("io error: {source}"))]
     #[snafu(context(false))]
-    IO { source: std::io::Error },
+    IO {
+        source: std::io::Error,
+    },
 
     #[snafu(display("json error: {source}"))]
     #[snafu(context(false))]
-    JSON { source: serde_json::Error },
+    JSON {
+        source: serde_json::Error,
+    },
 
     #[snafu(display("request error: {source}"))]
     #[snafu(context(false))]
-    Request { source: nyquest::Error },
+    Request {
+        source: nyquest::Error,
+    },
+
+    #[snafu(display("login error: {source}"))]
+    #[snafu(context(false))]
+    Login {
+        source: LoginError,
+    },
 }
 
 impl From<UnpadError> for ApiError {

sdgb-api/src/helper/mod.rs

@@ -0,0 +1,11 @@
+pub fn level_name(level: u32) -> &'static str {
+    match level {
+        0 => "BASIC",
+        1 => "ADVANCED",
+        2 => "EXPERT",
+        3 => "MASTER",
+        4 => "RE: MASTER",
+        5 => "UTAGE",
+        _ => "Unknown",
+    }
+}

sdgb-api/src/lib.rs

@@ -2,5 +2,13 @@ pub mod all_net;
 pub mod auth_lite;
 pub mod title;
 
+pub mod helper;
+
 mod error;
 pub use error::ApiError;
+
+#[cfg(feature = "bincode")]
+pub use bincode;
+
+#[cfg(all(feature = "compio", feature = "tokio"))]
+compile_error!("you must not enable both `compio` and `tokio`");

sdgb-api/src/title/encryption/mod.rs

@@ -1,44 +1,15 @@
-use std::io::{Read, Write as _};
+use std::io::Write as _;
 
 use aes::cipher::{
     BlockDecryptMut, BlockEncryptMut, BlockSizeUser, KeyIvInit, block_padding::Pkcs7,
 };
-
 use digest::generic_array::GenericArray;
-use flate2::write::ZlibEncoder;
+use flate2::Compression;
-use flate2::{Compression, read::ZlibDecoder};
+use flate2::write::{ZlibDecoder, ZlibEncoder};
+use spdlog::debug;
 
 use crate::error::ApiError;
-use crate::title::{MaiVersion, MaiVersionExt, Sdgb1_40, Sdgb1_50};
+use crate::title::{MaiVersion, MaiVersionExt, Sdgb1_50};
-
-impl MaiVersionExt for Sdgb1_40 {
-    fn decode(mut data: impl AsMut<[u8]>) -> Result<Vec<u8>, ApiError> {
-        let mut decompressed = decompress(data.as_mut());
-        if decompressed.is_empty() {
-            return Err(ApiError::EmptyResponse);
-        }
-
-        let orig_len = decompressed.len();
-
-        let remain = 16 - decompressed.len() % 16;
-
-        if
-        // weird but nessacary for Rust Pkcs7
-        remain != 16 {
-            decompressed.resize(remain + orig_len, remain as _);
-        }
-
-        let unpad_size = decrypt(&mut decompressed, Self::AES_KEY, Self::AES_IV)?.len();
-        decompressed.truncate(unpad_size);
-        Ok(decompressed)
-    }
-
-    fn encode(data: impl AsRef<[u8]>) -> Result<Vec<u8>, ApiError> {
-        let enc = encrypt(data, Self::AES_KEY, Self::AES_IV)?;
-        let compressed = compress(enc)?;
-        Ok(compressed)
-    }
-}
 
 impl MaiVersionExt for Sdgb1_50 {
     fn decode(mut data: impl AsMut<[u8]>) -> Result<Vec<u8>, ApiError> {
@@ -46,9 +17,9 @@ impl MaiVersionExt for Sdgb1_50 {
             return Err(ApiError::EmptyResponse);
         }
 
+        debug!("data size: {}", data.as_mut().len());
         let decrypted = decrypt(&mut data, Self::AES_KEY, Self::AES_IV)?;
-        let decompressed = decompress(decrypted);
+        Ok(decompress(decrypted))
-        Ok(decompressed)
     }
 
     fn encode(data: impl AsRef<[u8]>) -> Result<Vec<u8>, ApiError> {
@@ -72,8 +43,9 @@ fn compress(data: impl AsRef<[u8]>) -> Result<Vec<u8>, ApiError> {
 
 fn decompress(data: impl AsRef<[u8]>) -> Vec<u8> {
     let mut buf = Vec::with_capacity(data.as_ref().len() * 2);
-    let mut decode = ZlibDecoder::new(data.as_ref());
+    let mut decode = ZlibDecoder::new(&mut buf);
-    _ = decode.read_to_end(&mut buf);
+    _ = decode.write_all(data.as_ref());
+    _ = decode.finish();
     buf
 }
 
@@ -109,21 +81,6 @@ mod _tests {
 
     use crate::title::{Sdgb1_50, encryption::*};
 
-    #[test]
-    fn test_sdgb_140_dec_enc() -> Result<(), ApiError> {
-        let data = [
-            120_u8, 156, 171, 77, 91, 233, 184, 108, 2, 71, 125, 142, 118, 135, 112, 181, 85, 217,
-            239, 243, 159, 153, 248, 98, 159, 185, 63, 43, 173, 106, 221, 115, 104, 105, 221, 107,
-            0, 241, 176, 16, 37,
-        ];
-
-        let dec = Sdgb1_40::decode(data)?;
-        assert_eq!(dec, br#"{"result":"Pong"}"#);
-        let enc = Sdgb1_40::encode(dec)?;
-        assert_eq!(enc, data);
-        Ok(())
-    }
-
     #[test]
     fn test_sdgb_150_dec_enc() -> Result<(), ApiError> {
         let data = [
@@ -148,4 +105,62 @@ mod _tests {
         assert_eq!(enc, data);
         Ok(())
     }
+
+    // FIXME: user data decryption
+    #[test]
+    fn test_user_data_dec() -> Result<(), ApiError> {
+        let data = [
+            112, 133, 192, 229, 116, 195, 219, 220, 56, 176, 98, 148, 246, 73, 179, 157, 181, 251,
+            9, 7, 190, 113, 101, 8, 144, 58, 23, 196, 16, 176, 78, 96, 106, 53, 191, 118, 86, 247,
+            50, 250, 168, 155, 164, 108, 7, 152, 251, 123, 186, 121, 113, 41, 104, 79, 29, 71, 47,
+            228, 214, 97, 223, 29, 27, 163, 159, 113, 82, 45, 29, 41, 176, 246, 33, 235, 22, 202,
+            1, 61, 133, 126, 8, 20, 9, 214, 153, 11, 203, 207, 5, 195, 129, 172, 70, 81, 58, 156,
+            240, 87, 203, 137, 110, 183, 245, 177, 210, 63, 231, 136, 82, 97, 201, 184, 236, 117,
+            89, 85, 22, 29, 2, 238, 250, 148, 158, 151, 139, 179, 9, 42, 47, 220, 88, 137, 135,
+            139, 57, 35, 0, 50, 123, 133, 103, 15, 87, 86, 208, 203, 235, 153, 214, 87, 236, 46,
+            78, 50, 72, 50, 8, 46, 30, 242, 44, 184, 72, 118, 51, 120, 99, 158, 247, 255, 168, 181,
+            119, 156, 214, 143, 253, 185, 21, 76, 117, 201, 38, 38, 79, 171, 8, 251, 90, 53, 59,
+            89, 30, 136, 69, 168, 57, 34, 115, 145, 159, 110, 182, 5, 126, 208, 202, 216, 92, 200,
+            168, 63, 114, 119, 129, 38, 139, 189, 101, 215, 102, 76, 29, 228, 219, 46, 79, 166,
+            127, 194, 60, 183, 169, 167, 210, 120, 77, 219, 58, 16, 231, 233, 189, 66, 215, 202,
+            28, 209, 59, 192, 141, 91, 65, 17, 187, 88, 189, 149, 139, 48, 237, 152, 161, 87, 120,
+            99, 2, 50, 12, 120, 179, 50, 235, 255, 223, 162, 216, 84, 13, 135, 196, 131, 121, 97,
+            171, 106, 240, 189, 112, 92, 41, 59, 204, 24, 72, 91, 14, 220, 249, 10, 166, 4, 254,
+            183, 194, 227, 53, 163, 35, 165, 253, 149, 83, 253, 191, 138, 236, 208, 146, 242, 31,
+            185, 152, 226, 100, 191, 2, 2, 82, 101, 141, 31, 71, 106, 2, 83, 1, 231, 140, 20, 16,
+            156, 171, 108, 109, 14, 93, 168, 203, 50, 20, 21, 142, 135, 97, 7, 80, 61, 110, 76,
+            152, 106, 231, 100, 78, 187, 28, 39, 191, 10, 206, 78, 127, 79, 247, 192, 164, 51, 237,
+            9, 63, 201, 7, 27, 81, 243, 88, 30, 244, 205, 57, 14, 126, 60, 61, 173, 21, 84, 15,
+            105, 38, 239, 249, 82, 202, 245, 219, 88, 195, 112, 113, 40, 60, 76, 10, 243, 232, 52,
+            27, 0, 84, 247, 85, 140, 99, 140, 165, 145, 140, 96, 55, 0, 174, 155, 241, 166, 252,
+            150, 87, 106, 42, 58, 33, 154, 222, 83, 69, 172, 226, 216, 108, 115, 203, 38, 133, 43,
+            171, 172, 78, 142, 70, 78, 186, 146, 24, 126, 203, 106, 221, 144, 17, 32, 42, 186, 125,
+            134, 186, 174, 214, 137, 212, 234, 202, 79, 241, 28, 222, 98, 83, 76, 254, 90, 210, 12,
+            141, 40, 191, 123, 143, 170, 154, 39, 137, 222, 224, 241, 61, 136, 184, 104, 106, 209,
+            184, 128, 30, 95, 36, 250, 163, 47, 82, 19, 121, 123, 134, 142, 31, 170, 23, 148, 20,
+            80, 157, 252, 103, 192, 204, 229, 10, 66, 84, 49, 21, 197, 110, 208, 202, 124, 217,
+            117, 19, 190, 241, 154, 178, 83, 37, 175, 209, 52, 228, 219, 137, 238, 146, 111, 228,
+            254, 89, 219, 49, 85, 30, 214, 162, 234, 138, 122, 9, 93, 164, 133, 136, 160, 75, 118,
+            87, 14, 170, 92, 109, 244, 40, 234, 40, 216, 72, 207, 81, 161, 252, 252, 0, 38, 206,
+            123, 212, 93, 252, 225, 205, 16, 5, 197, 59, 93, 100, 56, 93, 125, 214, 150, 133, 208,
+            12, 0, 226, 246, 94, 62, 235, 164, 48, 134, 205, 77, 14, 107, 162, 60, 23, 150, 47,
+            198, 5, 214, 125, 12, 150, 63, 128, 95, 237, 209, 55, 5, 11, 59, 187, 0, 254, 180, 226,
+            126, 88, 87, 172, 38, 169, 27, 25, 92, 204, 24, 103, 78, 226, 65, 163, 114, 16, 202,
+            31, 160, 182, 100, 226, 15, 64, 5, 71, 117, 237, 31, 145, 250, 97, 105, 103, 67, 243,
+            2, 208, 60, 72, 16, 199, 57, 170, 99, 151, 62, 100, 53, 23, 45, 123, 225, 170, 149, 65,
+            26, 142, 240, 82, 214, 88, 93, 100, 158, 84, 42, 5, 112, 165, 194, 1, 160, 149, 103,
+            238, 63, 75, 3, 134, 113, 197, 251, 251, 90, 34, 184, 248, 214, 183, 168, 135, 4, 169,
+            134, 194, 106, 83, 108, 176, 26, 159, 80, 143, 105, 111, 53, 4, 1, 240, 44, 240, 149,
+            118, 189, 208, 190, 235, 145, 166, 163, 231, 158, 219, 221, 208, 61, 158, 132, 39, 75,
+            235, 36, 199, 169, 34, 119, 150, 223, 74, 5, 107, 123, 132, 116, 97, 241, 53, 43, 238,
+            115, 189, 195, 124, 127, 172, 5, 109, 112, 149, 190, 19, 202, 253, 171, 53, 105, 123,
+            173, 50, 50, 145, 56, 232, 13, 169, 47, 60, 112, 35, 100, 205, 35, 142, 5, 198, 235,
+            206, 112, 145, 99, 21, 214, 1, 184, 57, 125, 87, 245, 204, 162, 167, 124, 18, 154, 49,
+            25, 144, 181, 58, 184, 212, 59, 252, 72, 167, 228, 60, 118, 113, 65, 50, 150, 235, 163,
+            121, 215, 82, 91, 100, 78, 54, 199, 238, 93, 21, 21, 29, 215, 18, 201, 205, 106, 211,
+            78, 141, 155,
+        ];
+        let _ = Sdgb1_50::decode(data)?;
+        Ok(())
+    }
 }

sdgb-api/src/title/helper/mod.rs

@@ -0,0 +1,55 @@
+use nyquest::AsyncClient;
+
+use crate::{
+    ApiError,
+    title::{
+        MaiVersionExt as _, Sdgb1_50,
+        methods::APIMethod,
+        model::{GetUserMusicApi, GetUserMusicApiResp},
+    },
+};
+
+pub async fn get_user_all_music(
+    client: &AsyncClient,
+    user_id: u32,
+) -> Result<GetUserMusicApiResp, ApiError> {
+    let mut user_music_list = Vec::new();
+    let mut index = None;
+
+    loop {
+        let GetUserMusicApiResp {
+            next_index,
+            user_music_list: mut new_list,
+            ..
+        } = Sdgb1_50::request::<_, GetUserMusicApiResp>(
+            &client,
+            APIMethod::GetUserMusicApi,
+            user_id,
+            GetUserMusicApi {
+                user_id,
+                next_index: index.unwrap_or_default(),
+                max_count: 2000,
+            },
+        )
+        .await?;
+
+        if new_list.is_empty() {
+            break;
+        }
+
+        user_music_list.append(&mut new_list);
+
+        if next_index == 0 {
+            break;
+        }
+
+        index = Some(next_index);
+    }
+
+    Ok(GetUserMusicApiResp {
+        user_id,
+        next_index: 0,
+        length: user_music_list.len() as _,
+        user_music_list,
+    })
+}

sdgb-api/src/title/methods/api_ext/mod.rs

@@ -0,0 +1,27 @@
+use crate::title::methods::{APIExt, APIMethod};
+
+#[crabtime::function]
+fn api_implement(api_names: Vec<String>) {
+    for api_name in api_names {
+        crabtime::output!(
+            pub struct {{api_name}}Ext;
+
+            impl APIExt for {{api_name}}Ext {
+                const METHOD: APIMethod = APIMethod::{{api_name}};
+                type Payload = crate::title::model::{{api_name}};
+                type Response  = crate::title::model::{{api_name}}Resp;
+            }
+        );
+    }
+}
+
+api_implement!([
+    "Ping",
+    "UserLoginApi",
+    "UserLogoutApi",
+    "GetUserDataApi",
+    "GetUserPreviewApi",
+    "GetUserRatingApi",
+    "GetUserMusicApi",
+    "GetUserRegionApi",
+]);

sdgb-api/src/title/methods/has_uid/mod.rs

@@ -0,0 +1,21 @@
+use crate::title::methods::HasUid;
+
+#[crabtime::function]
+fn uid_get_impl(api_names: Vec<String>) {
+    for api_name in api_names {
+        crabtime::output!(
+            impl HasUid for crate::title::model::{{api_name}}Resp {
+                fn get_uid(&self) -> u32 {
+                    self.user_id
+                }
+            }
+        );
+    }
+}
+
+uid_get_impl!([
+    "GetUserDataApi",
+    "GetUserMusicApi",
+    "GetUserPreviewApi",
+    "GetUserRatingApi"
+]);

sdgb-api/src/title/methods/mod.rs

@@ -1,3 +1,5 @@
+use serde::{Deserialize, Serialize};
+
 #[derive(strum::IntoStaticStr)]
 pub enum APIMethod {
     GetGameChargeApi,
@@ -45,6 +47,20 @@ pub enum APIMethod {
     UserLogoutApi,
 }
 
+pub trait APIExt {
+    const METHOD: APIMethod;
+    type Payload: Serialize + Send + 'static;
+    type Response: for<'de> Deserialize<'de>;
+}
+
+pub trait HasUid {
+    fn get_uid(&self) -> u32;
+}
+
+mod api_ext;
+mod has_uid;
+pub use api_ext::*;
+
 #[cfg(test)]
 mod _test {
     use crate::title::{MaiVersionExt, Sdgb1_50, methods::APIMethod};
@@ -59,5 +75,9 @@ mod _test {
             Sdgb1_50::api_hash(APIMethod::GetUserPreviewApi),
             "004cf848f96d393a5f2720101e30b93d"
         );
+        assert_eq!(
+            Sdgb1_50::api_hash(APIMethod::GetUserDataApi),
+            "3af1e5b298bb5b7379c94934b2e038c5"
+        );
     }
 }

sdgb-api/src/title/mod.rs

@@ -1,11 +1,13 @@
 use std::fmt::Display;
 
-use crate::title::methods::APIMethod;
+use crate::title::methods::{APIExt, APIMethod};
 
 pub mod encryption;
 pub mod methods;
 pub mod model;
 
+pub mod helper;
+
 use super::ApiError;
 
 use nyquest::{
@@ -14,6 +16,7 @@ use nyquest::{
     header::{ACCEPT_ENCODING, CONTENT_ENCODING, EXPECT, USER_AGENT},
 };
 use serde::{Deserialize, Serialize};
+use spdlog::debug;
 
 pub trait MaiVersion {
     const AES_KEY: &[u8; 32];
@@ -37,11 +40,7 @@ pub trait MaiVersionExt: MaiVersion {
         format!("{digest:x}")
     }
 
-    fn api_request<D>(
+    fn api_call<D>(api: APIMethod, agent_extra: impl Display, data: D) -> Result<Request, ApiError>
-        api: APIMethod,
-        agent_extra: impl Display,
-        data: D,
-    ) -> Result<Request, ApiError>
     where
         D: Serialize,
     {
@@ -66,16 +65,31 @@ pub trait MaiVersionExt: MaiVersion {
     fn request_raw<D>(
         client: &AsyncClient,
         api: APIMethod,
-        agent_extra: impl Display,
+        agent_extra: impl Display + Send + 'static,
         data: D,
     ) -> impl Future<Output = Result<Vec<u8>, ApiError>>
     where
-        D: Serialize,
+        D: Serialize + Send + 'static,
     {
+        #[cfg(feature = "compio")]
+        use compio::runtime::spawn_blocking;
+        #[cfg(feature = "tokio")]
+        use tokio::task::spawn_blocking;
+
+        #[cfg(all(not(feature = "compio"), not(feature = "tokio")))]
+        compile_error!("you must enable one of `compio` or `tokio`");
+
         async {
-            let req = Self::api_request(api, agent_extra, data)?;
+            let req = spawn_blocking(move || Self::api_call(api, agent_extra, data))
+                .await
+                .map_err(|_| ApiError::JoinError)??;
             let data = client.request(req).await?.bytes().await?;
-            let decoded = Self::decode(data)?;
+
+            debug!("received: {data:?}");
+
+            let decoded = spawn_blocking(move || Self::decode(data))
+                .await
+                .map_err(|_| ApiError::JoinError)??;
             Ok(decoded)
         }
     }
@@ -83,11 +97,11 @@ pub trait MaiVersionExt: MaiVersion {
     fn request<D, R>(
         client: &AsyncClient,
         api: APIMethod,
-        agent_extra: impl Display,
+        agent_extra: impl Display + Send + 'static,
         data: D,
     ) -> impl Future<Output = Result<R, ApiError>>
     where
-        D: Serialize,
+        D: Serialize + Send + 'static,
         R: for<'a> Deserialize<'a>,
     {
         async {
@@ -95,18 +109,18 @@ pub trait MaiVersionExt: MaiVersion {
             Ok(serde_json::from_slice(&raw_data)?)
         }
     }
+
+    fn request_ext<M: APIExt>(
+        client: &AsyncClient,
+        data: M::Payload,
+        agent_extra: impl Display + Send + 'static,
+    ) -> impl Future<Output = Result<M::Response, ApiError>> {
+        Self::request(client, M::METHOD, agent_extra, data)
+    }
 }
 
-pub struct Sdgb1_40;
 pub struct Sdgb1_50;
 
-impl MaiVersion for Sdgb1_40 {
-    const AES_KEY: &[u8; 32] = b"n7bx6:@Fg_:2;5E89Phy7AyIcpxEQ:R@";
-    const AES_IV: &[u8; 16] = b";;KjR1C3hgB1ovXa";
-    const OBFUSECATE_SUFFIX: &str = "MaimaiChnBEs2D5vW";
-
-    const VERSION: &str = "1.40";
-}
 impl MaiVersion for Sdgb1_50 {
     const AES_KEY: &[u8; 32] = b"a>32bVP7v<63BVLkY[xM>daZ1s9MBP<R";
     const AES_IV: &[u8; 16] = b"d6xHIKq]1J]Dt^ue";

sdgb-api/src/title/model/dxrating/conversion/mod.rs

@@ -0,0 +1,98 @@
+use music_db::query_music;
+
+use crate::title::model::{
+    dxrating::{DxCalculatedEntries, DxLevelName, DxMusicRecord, DxSheetId},
+    get_user_music_api::UserMusicDetail,
+    get_user_rating_api::{MusicRating, UserRating},
+};
+
+impl DxCalculatedEntries {
+    pub fn from_user_rating_lossy(rating: &UserRating) -> DxCalculatedEntries {
+        let b35 = rating
+            .rating_list
+            .iter()
+            .map(DxMusicRecord::try_from)
+            .flatten()
+            .collect();
+        let b15 = rating
+            .new_rating_list
+            .iter()
+            .map(DxMusicRecord::try_from)
+            .flatten()
+            .collect();
+
+        DxCalculatedEntries { b35, b15 }
+    }
+}
+
+impl TryFrom<u32> for DxLevelName {
+    type Error = ConversionError;
+
+    fn try_from(level: u32) -> Result<Self, Self::Error> {
+        Self::from_repr(level).ok_or(ConversionError::UnknownDifficulty { level })
+    }
+}
+
+impl TryFrom<&UserMusicDetail> for DxMusicRecord {
+    type Error = ConversionError;
+
+    fn try_from(
+        &UserMusicDetail {
+            music_id,
+            level,
+            achievement,
+            ..
+        }: &UserMusicDetail,
+    ) -> Result<Self, Self::Error> {
+        let music_title = query_music(music_id)
+            .map(|info| info.name.clone())
+            .ok_or(ConversionError::MusicNotInDB)?;
+
+        Ok(Self {
+            sheet_id: DxSheetId {
+                music_title,
+                level: DxLevelName::try_from(level)?,
+                dx_version: music_id >= 10000,
+            },
+            achievement_rate: (achievement as f64) / 10000.0,
+        })
+    }
+}
+
+impl TryFrom<&MusicRating> for DxMusicRecord {
+    type Error = ConversionError;
+
+    fn try_from(
+        &MusicRating {
+            music_id,
+            level,
+            achievement,
+            ..
+        }: &MusicRating,
+    ) -> Result<Self, Self::Error> {
+        let music_title = query_music(music_id)
+            .map(|info| info.name.clone())
+            .ok_or(ConversionError::MusicNotInDB)?;
+
+        Ok(Self {
+            sheet_id: DxSheetId {
+                music_title,
+                level: DxLevelName::try_from(level)?,
+                dx_version: music_id >= 10000,
+            },
+            achievement_rate: (achievement as f64) / 10000.0,
+        })
+    }
+}
+
+#[derive(Debug, snafu::Snafu)]
+pub enum ConversionError {
+    #[snafu(display("Music was not found in database"))]
+    MusicNotInDB,
+
+    #[snafu(display("Utage difficulty was disallowed"))]
+    UtageDifficulty,
+
+    #[snafu(display("Unknown difficulty: {level}"))]
+    UnknownDifficulty { level: u32 },
+}

sdgb-api/src/title/model/dxrating/mod.rs

@@ -0,0 +1,95 @@
+use serde::Serialize;
+
+/// Full payload for image generate api
+#[derive(Debug, Clone, PartialEq, Serialize)]
+#[serde(rename_all = "camelCase")]
+pub struct DxRatingNet {
+    pub calculated_entries: DxCalculatedEntries,
+
+    pub version: DataVersion,
+    /// use `_generic`
+    pub region: &'static str,
+}
+
+/// Export/Import format
+#[derive(Debug, Clone, PartialEq, Serialize)]
+#[serde(rename_all = "camelCase")]
+pub struct DxCalculatedEntries {
+    pub b35: Vec<DxMusicRecord>,
+    pub b15: Vec<DxMusicRecord>,
+}
+
+/// full music record
+#[derive(Debug, Clone, PartialEq, Serialize)]
+#[serde(rename_all = "camelCase")]
+pub struct DxMusicRecord {
+    pub sheet_id: DxSheetId,
+    pub achievement_rate: f64,
+}
+
+#[derive(Debug, Clone, PartialEq)]
+pub struct DxSheetId {
+    pub music_title: String,
+    pub dx_version: bool,
+    pub level: DxLevelName,
+}
+
+#[derive(Debug, Clone, Copy, PartialEq, strum::IntoStaticStr, strum::FromRepr)]
+#[strum(serialize_all = "lowercase")]
+#[repr(u32)]
+pub enum DxLevelName {
+    Basic,
+    Advanced,
+    Expert,
+    Master,
+    ReMaster,
+}
+
+#[derive(Debug, Clone, Copy, PartialEq)]
+pub enum DataVersion {
+    Buddies,
+    BuddiesPlus,
+    Prism,
+    PrismPlus,
+}
+
+impl Serialize for DataVersion {
+    fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>
+    where
+        S: serde::Serializer,
+    {
+        serializer.serialize_str(match self {
+            DataVersion::Buddies => "BUDDiES",
+            DataVersion::BuddiesPlus => "BUDDiES PLUS",
+            DataVersion::Prism => "PRiSM",
+            DataVersion::PrismPlus => "PRiSM PLUS",
+        })
+    }
+}
+
+impl Serialize for DxSheetId {
+    fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>
+    where
+        S: serde::Serializer,
+    {
+        serializer.serialize_str(&self.to_string())
+    }
+}
+
+impl ToString for DxSheetId {
+    fn to_string(&self) -> String {
+        let mut output = self.music_title.clone();
+
+        if self.dx_version {
+            output += "__dxrt__dx__dxrt__"
+        } else {
+            output += "__dxrt__std__dxrt__"
+        }
+
+        output += self.level.into();
+
+        output
+    }
+}
+
+mod conversion;

sdgb-api/src/title/model/get_user_data_api/mod.rs

@@ -0,0 +1,153 @@
+use serde::Deserialize;
+use serde::Serialize;
+
+#[derive(Default, Debug, Clone, PartialEq, Serialize, Deserialize)]
+#[serde(rename_all = "camelCase")]
+pub struct GetUserDataApi {
+    pub user_id: u32,
+}
+
+#[derive(Default, Debug, Clone, PartialEq, Serialize, Deserialize)]
+#[serde(rename_all = "camelCase")]
+pub struct GetUserDataApiResp {
+    pub user_id: u32,
+    pub user_data: UserData,
+    pub ban_state: u32,
+}
+
+#[derive(Default, Debug, Clone, PartialEq, Serialize, Deserialize)]
+#[serde(rename_all = "camelCase")]
+pub struct UserData {
+    /// your username, in full-width character
+    pub user_name: String,
+    /// should be `1`
+    pub is_net_member: i64,
+    /// maimile, 舞里程
+    pub point: i64,
+    /// 总获取过舞里程
+    pub total_point: i64,
+    /// DX RATING
+    pub player_rating: i64,
+    pub music_rating: i64,
+    /// B35
+    pub player_old_rating: i64,
+    /// B15
+    pub player_new_rating: i64,
+    /// highest total dx rating
+    pub highest_rating: i64,
+    pub grade_rating: i64,
+    /// 级别认定
+    pub grade_rank: i64,
+    /// 段位认定
+    pub course_rank: i64,
+    /// 友人对战阶级
+    pub class_rank: i64,
+    pub nameplate_id: i64,
+    pub frame_id: i64,
+    pub icon_id: i64,
+    pub trophy_id: i64,
+    pub plate_id: i64,
+    pub title_id: i64,
+    pub partner_id: i64,
+    pub chara_slot: Vec<i64>,
+    pub chara_lock_slot: Vec<i64>,
+    pub content_bit: i64,
+    pub select_map_id: i64,
+    /// 总游玩次数
+    pub play_count: i64,
+    /// 当前版本游玩次数
+    pub current_play_count: i64,
+    pub play_vs_count: i64,
+    pub play_sync_count: i64,
+    pub win_count: i64,
+    pub help_count: i64,
+    pub combo_count: i64,
+    /// 总 DX 分数
+    pub total_deluxscore: i64,
+    /// 绿 DX 分数
+    pub total_basic_deluxscore: i64,
+    /// 黄 DX 分数
+    pub total_advanced_deluxscore: i64,
+    /// 红 DX 分数
+    pub total_expert_deluxscore: i64,
+    /// 紫 DX 分数
+    pub total_master_deluxscore: i64,
+    /// 白 DX 分数
+    pub total_re_master_deluxscore: i64,
+    /// 总 Sync
+    pub total_sync: i64,
+    pub total_basic_sync: i64,
+    pub total_advanced_sync: i64,
+    pub total_expert_sync: i64,
+    pub total_master_sync: i64,
+    pub total_re_master_sync: i64,
+    pub total_achievement: i64,
+    pub total_basic_achievement: i64,
+    pub total_advanced_achievement: i64,
+    pub total_expert_achievement: i64,
+    pub total_master_achievement: i64,
+    pub total_re_master_achievement: i64,
+    pub event_watched_date: String,
+    /// 最后游玩ROM版本
+    pub last_rom_version: String,
+    /// 最后游玩数据版本
+    pub last_data_version: String,
+    /// 上次登陆日期
+    pub last_login_date: String,
+    /// 上次游玩结束日期
+    pub last_play_date: String,
+    /// 上次双人登陆日期
+    pub last_pair_login_date: String,
+    /// 上次免费游玩日期
+    pub last_trial_play_date: String,
+    pub last_play_credit: i64,
+    pub last_play_mode: i64,
+
+    /// 上次游玩位置 ID
+    pub last_place_id: i64,
+    /// 上次游玩位置
+    pub last_place_name: Option<String>,
+
+    pub last_all_net_id: i64,
+    /// 上次游玩地区 ID
+    pub last_region_id: i64,
+    /// 上次游玩地区
+    pub last_region_name: String,
+    pub last_country_code: String,
+    #[serde(rename = "lastSelectEMoney")]
+    pub last_select_emoney: i64,
+    /// 上次功能票
+    pub last_select_ticket: i64,
+    /// 上次挑战段位
+    pub last_select_course: i64,
+    /// 上次段位计数
+    pub last_count_course: i64,
+    pub first_game_id: String,
+    /// 首次游玩游戏版本
+    pub first_rom_version: String,
+    /// 首次游玩数据版本
+    pub first_data_version: String,
+    /// 首次游玩
+    pub first_play_date: String,
+    pub compatible_cm_version: String,
+    /// 总觉醒数
+    pub total_awake: i64,
+    pub daily_bonus_date: String,
+    pub daily_course_bonus_date: String,
+    pub map_stock: i64,
+    pub rename_credit: i64,
+    pub friend_regist_skip: i64,
+    pub cm_last_emoney_credit: i64,
+    pub cm_last_emoney_brand: i64,
+
+    /// 访问密码（国区无）
+    pub access_code: Option<String>,
+    /// 好友代码（国区无）
+    pub friend_code: Option<u32>,
+    /// 上次游玩 ID
+    pub last_game_id: Option<u32>,
+    /// 上次登入狗号
+    pub last_client_id: Option<String>,
+    /// 时间戳
+    pub date_time: Option<u64>,
+}

sdgb-api/src/title/model/get_user_music_api/mod.rs

@@ -0,0 +1,183 @@
+use std::fmt::Display;
+
+use music_db::query_music;
+use music_db::query_music_level;
+use serde::Deserialize;
+use serde::Serialize;
+
+use crate::helper::level_name;
+
+#[derive(Default, Debug, Clone, PartialEq, Serialize, Deserialize)]
+#[serde(rename_all = "camelCase")]
+pub struct GetUserMusicApi {
+    pub user_id: u32,
+    pub next_index: u32,
+    pub max_count: u32,
+}
+
+#[derive(Default, Debug, Clone, PartialEq, Serialize, Deserialize)]
+#[cfg_attr(feature = "bincode", derive(bincode::Encode, bincode::Decode))]
+#[serde(rename_all = "camelCase")]
+pub struct GetUserMusicApiResp {
+    pub user_id: u32,
+    pub length: u32,
+    pub next_index: u32,
+    pub user_music_list: Vec<UserMusic>,
+}
+
+#[cfg_attr(feature = "bincode", derive(bincode::Encode, bincode::Decode))]
+#[derive(Default, Debug, Clone, PartialEq, Serialize, Deserialize)]
+#[serde(rename_all = "camelCase")]
+pub struct UserMusic {
+    pub user_music_detail_list: Vec<UserMusicDetail>,
+    pub length: u32,
+}
+
+#[cfg_attr(feature = "bincode", derive(bincode::Encode, bincode::Decode))]
+#[derive(Default, Debug, Clone, PartialEq, Serialize, Deserialize)]
+#[serde(rename_all = "camelCase")]
+pub struct UserMusicDetail {
+    pub music_id: u32,
+    pub level: u32,
+    pub play_count: i64,
+    /// 达成率
+    pub achievement: i64,
+
+    /// Full Combo
+    ///
+    /// - 0: None
+    /// - 1: Full Combo
+    /// - 2: Full Combo+
+    /// - 3: All Perfect
+    /// - 4: All Perfect+
+    pub combo_status: i64,
+
+    /// Full Sync
+    ///
+    /// - 0: None
+    /// - 1: FullSync
+    /// - 2: FullSync+
+    /// - 3: FullSync DX
+    /// - 4: Full Sync DX+
+    /// - 5: SYNC
+    pub sync_status: i64,
+
+    /// DX 分数
+    pub deluxscore_max: i64,
+    /// - D = 0,
+    /// - C = 1,
+    /// - B = 2,
+    /// - BB = 3,
+    /// - BBB = 4,
+    /// - A = 5,
+    /// - AA = 6,
+    /// - AAA = 7,
+    /// - S = 8,
+    /// - S_PLUS = 9,
+    /// - SS = 10,
+    /// - SS_PLUS = 11,
+    /// - SSS = 12,
+    /// - SSS_PLUS = 13
+    pub score_rank: i64,
+
+    /// 理论次数
+    pub ext_num1: i64,
+    pub ext_num2: i64,
+}
+
+#[cfg_attr(feature = "parquet", derive(parquet_derive::ParquetRecordWriter))]
+pub struct UserMusicDetailFlatten {
+    pub user_id: u32,
+    pub music_id: u32,
+    pub level: u8,
+    pub play_count: u32,
+    pub achievement: u32,
+    pub combo_status: u8,
+    pub sync_status: u8,
+    pub deluxscore_max: u16,
+    pub score_rank: u8,
+    pub ext_num1: u32,
+    pub ext_num2: u32,
+}
+
+impl Display for UserMusicDetail {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        if let Some(music_title) = query_music(self.music_id).map(|i| &i.name) {
+            f.write_fmt(format_args!("曲目名称: \t{music_title}\n"))?;
+        }
+        f.write_fmt(format_args!("难度名称: \t{}\n", level_name(self.level)))?;
+        f.write_fmt(format_args!("游玩次数: \t{}\n", self.play_count))?;
+        f.write_fmt(format_args!(
+            "达成率: \t{}.{:04}%\n",
+            self.achievement / 10000,
+            self.achievement % 10000
+        ))?;
+
+        f.write_fmt(format_args!(
+            "达成状态: \t{}\n",
+            match self.combo_status {
+                0 => "无",
+                1 => "Full Combo",
+                2 => "Full Combo+",
+                3 => "All Perfect",
+                4 => "All Perfect+",
+                _ => "未知",
+            }
+        ))?;
+
+        f.write_fmt(format_args!(
+            "同步状态: \t{}\n",
+            match self.sync_status {
+                0 => "无",
+                1 => "Full Sync",
+                2 => "Full Sync+",
+                3 => "Full Sync DX",
+                4 => "Full Sync DX+",
+                5 => "SYNC", // 一起玩过
+                _ => "未知",
+            }
+        ))?;
+
+        f.write_fmt(format_args!("DX 分数: \t{}\n", self.deluxscore_max))?;
+
+        if let Some(level) = query_music_level(self.music_id, self.level) {
+            let (rank, rating) = level.dx_rating(self.achievement as _);
+            f.write_fmt(format_args!("DX RATING: \t{rating}\n"))?;
+            f.write_fmt(format_args!("RANK: \t{rank}"))?;
+        }
+
+        Ok(())
+    }
+}
+
+impl UserMusicDetailFlatten {
+    pub fn new(
+        user_id: u32,
+        UserMusicDetail {
+            music_id,
+            level,
+            play_count,
+            achievement,
+            combo_status,
+            sync_status,
+            deluxscore_max,
+            score_rank,
+            ext_num1,
+            ext_num2,
+        }: UserMusicDetail,
+    ) -> Self {
+        Self {
+            user_id,
+            music_id,
+            level: level as _,
+            sync_status: sync_status as _,
+            deluxscore_max: deluxscore_max as _,
+            score_rank: score_rank as _,
+            play_count: play_count as _,
+            achievement: achievement as _,
+            combo_status: combo_status as _,
+            ext_num1: ext_num1 as _,
+            ext_num2: ext_num2 as _,
+        }
+    }
+}

sdgb-api/src/title/model/get_user_preview_api/mod.rs

@@ -8,10 +8,18 @@ pub struct GetUserPreviewApi {
     pub user_id: u32,
 }
 
+impl From<u32> for GetUserPreviewApi {
+    fn from(user_id: u32) -> Self {
+        Self { user_id }
+    }
+}
+
 #[derive(Debug, Clone, Serialize, Deserialize)]
+#[cfg_attr(feature = "bincode", derive(bincode::Encode, bincode::Decode))]
+#[cfg_attr(feature = "parquet", derive(parquet_derive::ParquetRecordWriter))]
 #[serde(rename_all = "camelCase")]
 pub struct GetUserPreviewApiResp {
-    pub user_id: i64,
+    pub user_id: u32,
     pub user_name: String,
     pub is_login: bool,
     pub last_rom_version: String,
@@ -44,7 +52,6 @@ impl Display for GetUserPreviewApiResp {
         f.write_fmt(format_args!("DX Rating: {}\n", self.player_rating))?;
         f.write_fmt(format_args!("牌子: {}\n", self.nameplate_id))?;
         f.write_fmt(format_args!("图标: {}\n", self.icon_id))?;
-        f.write_fmt(format_args!("trophy: {}\n", self.trophy_id))?;
         f.write_fmt(format_args!("Net成员: {}\n", self.is_net_member))?;
         f.write_fmt(format_args!("继承账号: {}\n", self.is_inherit))?;
         f.write_fmt(format_args!("总觉醒: {}\n", self.total_awake))?;

sdgb-api/src/title/model/get_user_rating_api/mod.rs

@@ -0,0 +1,191 @@
+use std::fmt::Display;
+
+use music_db::query_music;
+use music_db::query_music_level;
+use serde::Deserialize;
+use serde::Serialize;
+
+use crate::helper::level_name;
+
+#[derive(Default, Debug, Clone, PartialEq, Serialize, Deserialize)]
+#[serde(rename_all = "camelCase")]
+pub struct GetUserRatingApi {
+    pub user_id: u32,
+}
+
+impl From<u32> for GetUserRatingApi {
+    fn from(user_id: u32) -> Self {
+        Self { user_id }
+    }
+}
+
+#[cfg_attr(feature = "bincode", derive(bincode::Encode, bincode::Decode))]
+#[derive(Debug, Serialize, Deserialize)]
+#[serde(rename_all = "camelCase")]
+pub struct GetUserRatingApiResp {
+    pub user_id: u32,
+    pub user_rating: UserRating,
+}
+
+#[cfg_attr(feature = "bincode", derive(bincode::Encode, bincode::Decode))]
+#[derive(Default, Debug, Clone, PartialEq, Serialize, Deserialize)]
+#[serde(rename_all = "camelCase")]
+pub struct UserRating {
+    /// total rating, now it's 0
+    pub rating: i64,
+    /// b35
+    pub rating_list: Vec<MusicRating>,
+    /// b15
+    pub new_rating_list: Vec<MusicRating>,
+    /// 候补 b35
+    pub next_rating_list: Vec<MusicRating>,
+    /// 候补 b15
+    pub next_new_rating_list: Vec<MusicRating>,
+    pub udemae: Udemae,
+}
+
+#[cfg_attr(feature = "bincode", derive(bincode::Encode, bincode::Decode))]
+#[derive(Default, Debug, Clone, PartialEq, Serialize, Deserialize)]
+#[serde(rename_all = "camelCase")]
+pub struct MusicRating {
+    /// Maimai music id
+    pub music_id: u32,
+    /// - 0: BASIC
+    /// - 1: ADVANCED
+    /// - 2: EXPERT
+    /// - 3: MASTER
+    /// - 4: RE: MASTER
+    /// - 5: Utage 宴会场
+    pub level: u32,
+    /// 歌曲 ROM 版本
+    ///
+    /// - `1mmpp` -> `1.mm.pp`
+    /// - `2mmpp` -> `1.mm.pp` DX
+    pub rom_version: i64,
+    /// 达成率 * 10000 的整数
+    pub achievement: i32,
+}
+
+#[cfg_attr(feature = "bincode", derive(bincode::Encode, bincode::Decode))]
+#[derive(Default, Debug, Clone, PartialEq, Serialize, Deserialize)]
+#[serde(rename_all = "camelCase")]
+pub struct Udemae {
+    pub max_lose_num: i64,
+    pub npc_total_win_num: i64,
+    pub npc_total_lose_num: i64,
+    pub npc_max_win_num: i64,
+    pub npc_max_lose_num: i64,
+    pub npc_win_num: i64,
+    pub npc_lose_num: i64,
+    pub rate: i64,
+    pub class_value: i64,
+    pub max_rate: i64,
+    pub max_class_value: i64,
+    pub total_win_num: i64,
+    pub total_lose_num: i64,
+    pub max_win_num: i64,
+    pub win_num: i64,
+    pub lose_num: i64,
+    #[serde(rename = "MaxLoseNum")]
+    pub max_lose_num2: i64,
+    #[serde(rename = "NpcTotalWinNum")]
+    pub npc_total_win_num2: i64,
+    #[serde(rename = "NpcTotalLoseNum")]
+    pub npc_total_lose_num2: i64,
+    #[serde(rename = "NpcMaxWinNum")]
+    pub npc_max_win_num2: i64,
+    #[serde(rename = "NpcMaxLoseNum")]
+    pub npc_max_lose_num2: i64,
+    #[serde(rename = "NpcWinNum")]
+    pub npc_win_num2: i64,
+    #[serde(rename = "NpcLoseNum")]
+    pub npc_lose_num2: i64,
+}
+
+impl Display for GetUserRatingApiResp {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        let b35 = &self.user_rating.rating_list;
+        let b15 = &self.user_rating.new_rating_list;
+        f.write_fmt(format_args!("用户ID: {}\n", self.user_id))?;
+
+        f.write_str("\n---------  B35  ---------\n")?;
+        for music in b35 {
+            f.write_fmt(format_args!("{music}\n---\n"))?;
+        }
+
+        f.write_str("\n---------  B15  ---------\n")?;
+        for music in b15 {
+            f.write_fmt(format_args!("{music}\n---\n"))?;
+        }
+
+        let b35_rating: u32 = b35.iter().filter_map(|m| m.dx_rating()).sum();
+        let b15_rating: u32 = b15.iter().filter_map(|m| m.dx_rating()).sum();
+
+        f.write_str("\n--------- Total ---------\n")?;
+        f.write_fmt(format_args!("B35 Rating: {b35_rating}\n"))?;
+        f.write_fmt(format_args!("B15 Rating: {b15_rating}\n"))?;
+        f.write_fmt(format_args!("总 Rating: {}\n", b35_rating + b15_rating))?;
+
+        Ok(())
+    }
+}
+
+impl Display for MusicRating {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        f.write_fmt(format_args!("歌曲ID: \t{}\n", self.music_id))?;
+
+        if let Some(title) = self.music_title() {
+            f.write_fmt(format_args!("曲目标题: \t{title}\n"))?;
+        }
+
+        f.write_fmt(format_args!(
+            "谱面版本: \t{}\n",
+            match (self.music_id / 10000) % 10 {
+                0 => "SD",
+                1 => "DX",
+                _ => "宴",
+            }
+        ))?;
+
+        f.write_fmt(format_args!("游玩难度: \t{}\n", level_name(self.level)))?;
+        f.write_fmt(format_args!(
+            "达成率: \t{}.{:04}%\n",
+            self.achievement / 10000,
+            self.achievement % 10000
+        ))?;
+
+        if self.rom_version >= 20000 {
+            f.write_fmt(format_args!(
+                "谱面版本: \tDX 1.{:02}.{:02}\n",
+                (self.rom_version / 100) % 100,
+                self.rom_version % 100,
+            ))?;
+        } else {
+            f.write_fmt(format_args!(
+                "谱面版本: \tSD 1.{:02}.{:02}\n",
+                (self.rom_version / 100) % 100,
+                self.rom_version % 100,
+            ))?;
+        }
+
+        if let Some(dx_rating) = self.dx_rating() {
+            f.write_fmt(format_args!("DX RATING: \t{dx_rating}"))?;
+        }
+
+        Ok(())
+    }
+}
+
+impl MusicRating {
+    pub fn music_title(&self) -> Option<String> {
+        Some(query_music(self.music_id).as_ref()?.name.clone())
+    }
+
+    pub fn dx_rating(&self) -> Option<u32> {
+        Some(
+            query_music_level(self.music_id, self.level)?
+                .dx_rating(self.achievement)
+                .1,
+        )
+    }
+}

sdgb-api/src/title/model/get_user_region_api/mod.rs

@@ -0,0 +1,68 @@
+use serde::{Deserialize, Serialize};
+
+#[derive(Default, Debug, Clone, PartialEq, Serialize, Deserialize)]
+#[serde(rename_all = "camelCase")]
+pub struct GetUserRegionApi {
+    pub user_id: u32,
+}
+
+impl From<u32> for GetUserRegionApi {
+    fn from(user_id: u32) -> Self {
+        Self { user_id }
+    }
+}
+
+impl From<GetUserRegionApiResp> for Vec<UserRegionFlatten> {
+    fn from(
+        GetUserRegionApiResp {
+            user_id,
+            user_region_list,
+            ..
+        }: GetUserRegionApiResp,
+    ) -> Self {
+        user_region_list
+            .into_iter()
+            .map(
+                |UserRegion {
+                     region_id,
+                     play_count,
+                     created,
+                 }| {
+                    UserRegionFlatten {
+                        user_id,
+                        region_id,
+                        play_count,
+                        created,
+                    }
+                },
+            )
+            .collect()
+    }
+}
+
+#[derive(Default, Debug, Clone, PartialEq)]
+#[cfg_attr(feature = "parquet", derive(parquet_derive::ParquetRecordWriter))]
+pub struct UserRegionFlatten {
+    pub user_id: u32,
+    pub region_id: u32,
+    pub play_count: i64,
+    pub created: String,
+}
+
+#[cfg_attr(feature = "bincode", derive(bincode::Encode, bincode::Decode))]
+#[derive(Default, Debug, Clone, PartialEq, Serialize, Deserialize)]
+#[serde(rename_all = "camelCase")]
+pub struct GetUserRegionApiResp {
+    pub user_id: u32,
+    pub length: i64,
+    pub user_region_list: Vec<UserRegion>,
+}
+
+#[cfg_attr(feature = "bincode", derive(bincode::Encode, bincode::Decode))]
+#[derive(Default, Debug, Clone, PartialEq, Serialize, Deserialize)]
+#[serde(rename_all = "camelCase")]
+pub struct UserRegion {
+    pub region_id: u32,
+    pub play_count: i64,
+    pub created: String,
+}

sdgb-api/src/title/model/mod.rs

@@ -6,3 +6,38 @@ pub use get_user_preview_api::{GetUserPreviewApi, GetUserPreviewApiResp};
 
 mod user_logout_api;
 pub use user_logout_api::{UserLogoutApi, UserLogoutApiResp};
+
+mod user_login_api;
+pub use user_login_api::{LoginError, UserLoginApi, UserLoginApiResp};
+
+mod get_user_data_api;
+pub use get_user_data_api::{GetUserDataApi, GetUserDataApiResp, UserData};
+
+mod get_user_rating_api;
+pub use get_user_rating_api::{
+    GetUserRatingApi,
+    GetUserRatingApiResp, // api
+    MusicRating,
+    Udemae,
+    UserRating,
+};
+
+mod get_user_music_api;
+pub use get_user_music_api::{
+    GetUserMusicApi, GetUserMusicApiResp, UserMusic, UserMusicDetail, UserMusicDetailFlatten,
+};
+
+mod get_user_region_api;
+pub use get_user_region_api::{
+    GetUserRegionApi, GetUserRegionApiResp, UserRegion, UserRegionFlatten,
+};
+
+mod dxrating;
+pub use dxrating::{
+    DataVersion,
+    DxCalculatedEntries, // entries
+    DxLevelName,         // level name
+    DxMusicRecord,
+    DxRatingNet,
+    DxSheetId,
+};

sdgb-api/src/title/model/user_login_api/mod.rs

@@ -0,0 +1,84 @@
+use std::time::{SystemTime, UNIX_EPOCH};
+
+use serde::{Deserialize, Serialize};
+use spdlog::info;
+
+#[derive(Default, Debug, Clone, PartialEq, Serialize, Deserialize)]
+#[serde(rename_all = "camelCase")]
+pub struct UserLoginApi {
+    pub user_id: u32,
+    pub region_id: u32,
+    pub date_time: u64,
+    pub acsess_code: String,
+    pub place_id: String,
+    pub client_id: String,
+    /// false 的情况，二维码扫描后半小时可登录。
+    ///
+    /// true  的情况，可延长至二维码扫描后的两小时可登录。
+    pub is_continue: bool,
+    /// fixed to 0
+    pub generic_flag: u8,
+}
+
+#[derive(Default, Debug, Clone, PartialEq, Serialize, Deserialize)]
+#[serde(rename_all = "camelCase")]
+pub struct UserLoginApiResp {
+    /// - `1`: success
+    /// - `100`: logged
+    /// - `102`: QRCode expired
+    pub return_code: i32,
+    /// format: yyyy-mm-dd HH:MM:SS
+    pub last_login_date: Option<String>,
+    pub login_count: Option<u64>,
+    pub consecutive_login_count: Option<u64>,
+    /// needed for some operation
+    pub login_id: Option<u64>,
+}
+
+impl UserLoginApi {
+    pub fn new(user_id: u32, is_continue: bool) -> Self {
+        let date_time = SystemTime::now()
+            .duration_since(UNIX_EPOCH)
+            .map(|t| t.as_secs())
+            .unwrap_or_default();
+
+        info!("login unix timestamp: {date_time}");
+
+        // 爱玩星球焦作解放店
+        UserLoginApi {
+            user_id,
+            date_time,
+            region_id: 13,
+            acsess_code: "".to_owned(),
+            place_id: 3223.to_string(),
+            is_continue,
+            generic_flag: 0,
+            client_id: "A63E01E6170".into(),
+        }
+    }
+}
+
+impl UserLoginApiResp {
+    pub fn error(&self) -> Option<LoginError> {
+        match self.return_code {
+            1 => None,
+            100 => Some(LoginError::AlreadyLogged),
+            102 => Some(LoginError::QRCodeExpired),
+            103 => Some(LoginError::AccountUnregistered),
+            error => Some(LoginError::Unknown { error }),
+        }
+    }
+}
+
+#[derive(Debug, snafu::Snafu)]
+pub enum LoginError {
+    #[snafu(display("QRCode was expired"))]
+    QRCodeExpired,
+    #[snafu(display("You did not logout last session"))]
+    AlreadyLogged,
+    #[snafu(display("userId does not exist"))]
+    AccountUnregistered,
+
+    #[snafu(display("Unknown error: {error}"))]
+    Unknown { error: i32 },
+}

sdgb-api/src/title/model/user_logout_api/mod.rs

@@ -18,7 +18,7 @@ pub struct UserLogoutApi {
     pub type_: i64,
 }
 
-#[derive(Default, Debug, Clone, PartialEq, Deserialize)]
+#[derive(Default, Debug, Clone, PartialEq, Deserialize, Serialize)]
 #[serde(rename_all = "camelCase")]
 pub struct UserLogoutApiResp {
     pub return_code: i64,

sdgb-cli/Cargo.toml

@@ -5,20 +5,44 @@ edition = "2024"
 authors = ["mokurin000"]
 description = "CLI tool for SDGB protocol"
 
+[features]
+default = ["compio", "fetchall"]
+
+compio = ["dep:compio", "sdgb-api/compio"]
+tokio = ["dep:tokio", "sdgb-api/tokio"]
+
+fetchall = ["dep:redb", "dep:futures-util", "dep:parquet", "sdgb-api/parquet"]
+
 [dependencies]
-snafu = { workspace = true }
+sdgb-api = { workspace = true, features = ["bincode"] }
-sdgb-api = { workspace = true }
+
+# (de)serialization
+serde = { workspace = true }
 serde_json = { workspace = true }
 strum = { workspace = true }
 
-nyquest-preset = { version = "0.2.0", features = ["async"] }
+# logging / errors
-compio = { version = "0.15.0", default-features = false, features = [
+spdlog-rs = { workspace = true }
-    "runtime",
+snafu = { workspace = true }
-    "macros",
+
-] }
+
+# kv database
+redb = { workspace = true, optional = true }
+
+# async runtime
+tokio = { workspace = true, features = ["macros"], optional = true }
+compio = { workspace = true, features = ["macros"], optional = true }
+
+nyquest-preset = { version = "0.3.0", features = ["async"] }
 
 palc = { version = "0.0.1", features = ["derive"] }
-spdlog-rs = { version = "0.4.3", default-features = false, features = [
+futures-util = { version = "0.3.31", optional = true }
-    "level-info",
+ctrlc = { version = "3.4.7", features = ["termination"] }
-    "release-level-info",
+
-] }
+# magic macro
+crabtime = { workspace = true }
+
+parquet = { workspace = true, optional = true }
+
+[build-dependencies]
+version_check = "0.9.5"

sdgb-cli/build.rs

@@ -0,0 +1,7 @@
+fn main() {
+    println!("cargo::rustc-check-cfg=cfg(file_lock_ready)");
+
+    if version_check::is_min_version("1.89") == Some(true) {
+        println!("cargo:rustc-cfg=file_lock_ready");
+    }
+}

sdgb-cli/src/cache/mod.rs

@@ -0,0 +1,64 @@
+use std::sync::LazyLock;
+
+use redb::{ReadTransaction, ReadableDatabase as _, Table, TableDefinition, WriteTransaction};
+
+static DATABASE: LazyLock<redb::Database> = LazyLock::new(|| {
+    let mut db = redb::Database::builder()
+        .create("players.redb")
+        .expect("failed to open database");
+    _ = db.compact();
+    db
+});
+
+pub fn write_txn() -> Result<WriteTransaction, redb::Error> {
+    Ok(DATABASE.begin_write()?)
+}
+
+pub fn read_txn() -> Result<ReadTransaction, redb::Error> {
+    Ok(DATABASE.begin_read()?)
+}
+
+pub fn open_table<'a>(
+    write: &'a WriteTransaction,
+    definition: TableDefinition<'_, u32, Vec<u8>>,
+) -> Result<Table<'a, u32, Vec<u8>>, redb::Error> {
+    Ok(write.open_table(definition)?)
+}
+
+pub fn open_table_ro(
+    read: &ReadTransaction,
+    definition: TableDefinition<'_, u32, Vec<u8>>,
+) -> Result<redb::ReadOnlyTable<u32, Vec<u8>>, redb::Error> {
+    Ok(read.open_table(definition)?)
+}
+
+#[crabtime::function]
+fn table_definitions_impl(tables: Vec<String>) {
+    let mut defs: Vec<String> = Vec::new();
+
+    for table in tables {
+        let definition = table.to_uppercase();
+        let table_name = format!("\"{table}\"");
+
+        crabtime::output!(
+            pub const {{definition}}: TableDefinition<'_, u32, Vec<u8>> = redb::TableDefinition::new({{table_name}});
+        );
+
+        defs.push(format!("write_txn.open_table({definition})?;"));
+    }
+
+    let init_statements = defs.join("\n");
+
+    crabtime::output!(
+        pub fn init_db() -> Result<(), redb::Error> {
+            let write_txn = DATABASE.begin_write()?;
+            {
+                { init_statements }
+            }
+            write_txn.commit()?;
+            Ok(())
+        }
+    );
+}
+
+table_definitions_impl!(["players", "b50", "records", "regions"]);

sdgb-cli/src/commands.rs

@@ -5,6 +5,11 @@ use strum::EnumString;
 #[derive(Parser)]
 #[command(about = "SDGB api tool", long_about = env!("CARGO_PKG_DESCRIPTION"))]
 pub struct Cli {
+    /// Try to generate machine readable format.
+    ///
+    /// You must specify this for `-f, --format` to take effect.
+    #[arg(short = 'M', long)]
+    pub machine_readable: bool,
     #[command(subcommand)]
     pub command: Commands,
 }
@@ -24,6 +29,7 @@ pub enum Commands {
         qrcode_content: String,
     },
 
+    /// Retrieve update package of SDGB
     AuthLite {
         #[arg(short, long, default_value = "1.50")]
         title_ver: String,
@@ -31,13 +37,117 @@ pub enum Commands {
         variant: AuthLiteVariant,
     },
 
+    /// Test delay to SDGB server
     Ping,
+    /// Get basic info
     Preview {
         #[arg(short, long)]
         user_id: u32,
     },
+    /// Get B35 + B15 play records
+    Rating {
+        #[arg(short, long)]
+        user_id: u32,
+
+        /// JSON format.
+        ///
+        /// `origin`: official json response
+        ///
+        /// `dx_rating_net`: DxRatingNet Format
+        #[arg(short, long, default_value_t = RatingFormat::default())]
+        format: RatingFormat,
+    },
+    /// Get all play records
+    MusicDetail {
+        #[arg(short, long)]
+        user_id: u32,
+
+        /// JSON format.
+        ///
+        /// `origin`: official json response
+        ///
+        /// `dx_rating_net`: DxRatingNet Format
+        #[arg(short, long, default_value_t = RatingFormat::default())]
+        format: RatingFormat,
+    },
+
+    /// Retrieve full userdata
+    ///
+    /// WARNING: This requires to login & logout your account
+    Userdata {
+        #[arg(short, long)]
+        user_id: u32,
+        #[arg(long)]
+        skip_login: bool,
+    },
+
+    #[cfg(feature = "fetchall")]
+    ListAllUser {
+        #[arg(short, long, default_value_t = 5)]
+        concurrency: usize,
+    },
+    #[cfg(feature = "fetchall")]
+    ScrapeAllB50 {
+        #[arg(short, long, default_value_t = 5)]
+        concurrency: usize,
+
+        #[arg(long, default_value_t = 1000)]
+        min_rating: i64,
+        #[arg(long, default_value_t = 16500)]
+        max_rating: i64,
+    },
+    #[cfg(feature = "fetchall")]
+    ScrapeAllRegion {
+        #[arg(short, long, default_value_t = 5)]
+        concurrency: usize,
+
+        #[arg(long, default_value_t = 1000)]
+        min_rating: i64,
+        #[arg(long, default_value_t = 16500)]
+        max_rating: i64,
+    },
+    #[cfg(feature = "fetchall")]
+    ScrapeAllRecord {
+        #[arg(short, long, default_value_t = 5)]
+        concurrency: usize,
+
+        #[arg(long, default_value_t = 10000)]
+        min_rating: i64,
+        #[arg(long, default_value_t = 16400)]
+        max_rating: i64,
+    },
+
+    #[cfg(feature = "fetchall")]
+    ListAllUserDump {},
+    #[cfg(feature = "fetchall")]
+    ScrapeAllB50Dump {},
+    #[cfg(feature = "fetchall")]
+    ScrapeAllRegionDump {},
+    #[cfg(feature = "fetchall")]
+    ScrapeAllRecordDump {},
+
     Logout {
         #[arg(short, long)]
         user_id: u32,
+        /// Second-precision login unix timestamp, must be the same as on `login`
+        ///
+        /// For official arcades, it's commonly the time `amdaemon.exe` starts
+        ///
+        /// For unofficial clients, it depends.
+        #[arg(short, long)]
+        timestamp: u64,
     },
 }
+
+#[derive(Debug, Default, EnumString)]
+#[strum(serialize_all = "snake_case")]
+pub enum RatingFormat {
+    #[default]
+    /// Official API response
+    Origin,
+
+    /// dxrating.net format
+    DxRatingNet,
+    /// dxrating.net image gen payload
+    DxRatingPayload,
+}

sdgb-cli/src/main.rs

@@ -1,45 +1,171 @@
+use std::{
+    sync::{
+        Arc,
+        atomic::{AtomicBool, Ordering},
+    },
+    time::SystemTime,
+};
+
 use nyquest_preset::nyquest::ClientBuilder;
 use palc::Parser;
+use spdlog::{Level, LevelFilter::MoreSevereEqual, sink::StdStreamSink, terminal_style::StyleMode};
+
 use sdgb_api::{
     all_net::QRCode,
     auth_lite::{SDGB, SDHJ, delivery_raw},
     title::{
-        MaiVersionExt, Sdgb1_40, Sdgb1_50,
+        MaiVersionExt, Sdgb1_50,
+        helper::get_user_all_music,
         methods::APIMethod,
         model::{
-            GetUserPreviewApi, GetUserPreviewApiResp, Ping, PingResp, UserLogoutApi,
+            DataVersion, DxCalculatedEntries, DxMusicRecord, DxRatingNet, GetUserDataApi,
+            GetUserDataApiResp, GetUserPreviewApi, GetUserPreviewApiResp, GetUserRatingApi,
+            GetUserRatingApiResp, Ping, PingResp, UserLoginApiResp, UserLogoutApi,
             UserLogoutApiResp,
         },
     },
 };
-use spdlog::{error, info};
+use spdlog::{error, info, warn};
 
-use crate::commands::Cli;
+use crate::{
+    commands::{Cli, Commands, RatingFormat},
+    utils::{human_readable_display, json_display, login_action},
+};
 
+#[cfg(feature = "fetchall")]
+mod cache;
 mod commands;
+mod utils;
 
-#[compio::main]
+static EARLY_QUIT: AtomicBool = AtomicBool::new(false);
+
+#[cfg_attr(feature = "compio", compio::main)]
+#[cfg_attr(feature = "tokio", tokio::main)]
 async fn main() -> Result<(), Box<dyn snafu::Error>> {
     nyquest_preset::register();
-    let cmd = <Cli as Parser>::parse();
+
+    let logger = spdlog::default_logger().fork_with(|log| {
+        log.set_level_filter(MoreSevereEqual(if cfg!(debug_assertions) {
+            Level::Debug
+        } else {
+            Level::Info
+        }));
+        let sink = StdStreamSink::builder()
+            .stderr()
+            .style_mode(StyleMode::Always)
+            .build()?;
+        *log.sinks_mut() = vec![Arc::new(sink)];
+        Ok(())
+    })?;
+    spdlog::swap_default_logger(logger);
+
+    ctrlc::set_handler(|| {
+        if EARLY_QUIT.load(Ordering::Relaxed) {
+            error!("force-quit triggered!");
+            std::process::exit(1);
+        } else {
+            warn!("received early-quit request! will abort soon");
+            EARLY_QUIT.store(true, Ordering::Relaxed);
+        }
+    })?;
+
+    let Cli {
+        command,
+        machine_readable,
+    } = <Cli as Parser>::parse();
+    let human_readable = !machine_readable;
 
     let client = ClientBuilder::default().build_async().await?;
 
-    match cmd.command {
+    // TODO: refactor via enum_dispatch
-        commands::Commands::Logout { user_id } => {
+    match command {
+        Commands::MusicDetail { user_id, format } => {
+            let music_detail = get_user_all_music(&client, user_id).await?;
+            let details = music_detail
+                .user_music_list
+                .iter()
+                .map(|m| &m.user_music_detail_list)
+                .flatten();
+
+            match (human_readable, format) {
+                (true, _) => {
+                    let mut count = 0;
+                    for detail in details {
+                        println!("{detail}");
+                        println!("----------");
+                        count += 1;
+                    }
+
+                    println!("共查询到 {count} 条记录！");
+                }
+                (false, RatingFormat::Origin) => json_display(music_detail)?,
+                (false, RatingFormat::DxRatingNet) => {
+                    let dx_export = Vec::from_iter(
+                        details
+                            .map(|music| {
+                                DxMusicRecord::try_from(music).inspect_err(|e| {
+                                    warn!("failed to process {}: {e}", music.music_id)
+                                })
+                            })
+                            .flatten(),
+                    );
+                    json_display(dx_export)?;
+                }
+                (_, format) => {
+                    error!("{format:?} was not supported yet");
+                    json_display(())?;
+                }
+            }
+        }
+        Commands::Rating { user_id, format } => {
+            let rating: GetUserRatingApiResp = Sdgb1_50::request(
+                &client,
+                APIMethod::GetUserRatingApi,
+                user_id,
+                GetUserRatingApi { user_id },
+            )
+            .await?;
+
+            match (human_readable, format) {
+                (true, _) => println!("{rating}"),
+                (false, RatingFormat::Origin) => json_display(rating)?,
+                (false, RatingFormat::DxRatingNet) => {
+                    let mut data = DxCalculatedEntries::from_user_rating_lossy(&rating.user_rating);
+                    let mut records = data.b35;
+                    records.append(&mut data.b15);
+                    json_display(records)?;
+                }
+                (false, RatingFormat::DxRatingPayload) => {
+                    let data = DxCalculatedEntries::from_user_rating_lossy(&rating.user_rating);
+                    let payload = DxRatingNet {
+                        calculated_entries: data,
+                        version: DataVersion::Prism,
+                        region: "_generic",
+                    };
+                    json_display(payload)?;
+                }
+            }
+        }
+        Commands::Logout { user_id, timestamp } => {
             let logout: UserLogoutApiResp = Sdgb1_50::request(
                 &client,
                 APIMethod::UserLogoutApi,
                 user_id,
                 UserLogoutApi {
                     user_id,
+                    date_time: timestamp,
                     ..Default::default()
                 },
             )
             .await?;
-            println!("{logout:?}");
+
+            if human_readable {
+                println!("啥都木有");
+            } else {
+                json_display(logout)?;
             }
-        commands::Commands::Preview { user_id } => {
+        }
+        Commands::Preview { user_id } => {
             let preview: GetUserPreviewApiResp = Sdgb1_50::request(
                 &client,
                 APIMethod::GetUserPreviewApi,
@@ -48,37 +174,230 @@ async fn main() -> Result<(), Box<dyn snafu::Error>> {
             )
             .await?;
 
-            println!("{preview}");
+            human_readable_display(preview, human_readable)?;
         }
-        commands::Commands::Ping => {
+        Commands::Ping => {
-            let decoded: PingResp = Sdgb1_40::request(
+            let time = SystemTime::now();
-                &client,
-                APIMethod::Ping,
-                "",
-                Ping {}, // note: must not be `Ping`, or serde_json serializes to nothing
-            )
-            .await?;
-            info!("sdgb 1.40 resp: {decoded}");
             let decoded: PingResp =
                 Sdgb1_50::request(&client, APIMethod::Ping, "", Ping {}).await?;
-            info!("sdgb 1.50 resp: {decoded}");
+            info!(
+                "sdgb 1.50 resp: {decoded}, {}ms",
+                time.elapsed().unwrap_or_default().as_millis()
+            );
         }
-        commands::Commands::QRLogin { ref qrcode_content } => {
+        Commands::QRLogin { ref qrcode_content } => {
             let qrcode = QRCode { qrcode_content };
-            match qrcode.login(&client).await {
+            let resp = qrcode.login(&client).await;
+
+            match &resp {
                 Ok(user_id) => info!("login succeed: {user_id}"),
                 Err(e) => error!("login failed: {e}"),
             }
+
+            if !human_readable {
+                json_display(resp)?;
+            }
         }
 
-        commands::Commands::AuthLite { title_ver, variant } => {
+        Commands::AuthLite { title_ver, variant } => {
             let resp = match variant {
                 commands::AuthLiteVariant::SDGB => delivery_raw::<SDGB>(&client, title_ver).await?,
                 commands::AuthLiteVariant::SDHJ => delivery_raw::<SDHJ>(&client, title_ver).await?,
             };
             println!("{}", String::from_utf8_lossy(&resp));
         }
+
+        #[cfg(feature = "fetchall")]
+        Commands::ListAllUser { concurrency } => {
+            use crate::{cache::PLAYERS, utils::helpers::cached_concurrent_fetch};
+            use sdgb_api::title::methods::GetUserPreviewApiExt;
+            use std::io::BufRead as _;
+
+            let mut user_ids = Vec::new();
+            {
+                let mut stdin = std::io::stdin().lock();
+                let mut buf = String::new();
+
+                while stdin.read_line(&mut buf).is_ok_and(|size| size != 0) {
+                    if buf.is_empty() {
+                        continue;
+                    }
+
+                    let user_id: u32 = buf.trim().parse()?;
+                    buf.clear();
+                    user_ids.push(user_id);
+                }
+            }
+
+            cached_concurrent_fetch::<GetUserPreviewApiExt>(
+                user_ids,
+                &client,
+                concurrency,
+                PLAYERS,
+            )
+            .await?;
+        }
+
+        #[cfg(feature = "fetchall")]
+        Commands::ScrapeAllRecord {
+            concurrency,
+            min_rating,
+            max_rating,
+        } => {
+            use crate::{
+                cache::{PLAYERS, RECORDS},
+                utils::helpers::{cached_concurrent_fetch_userfn, read_cache},
+            };
+
+            let mut players: Vec<GetUserPreviewApiResp> = read_cache(PLAYERS)?;
+            players.retain(|p| p.player_rating >= min_rating && p.player_rating <= max_rating);
+
+            cached_concurrent_fetch_userfn(
+                players.iter().map(|p| p.user_id).collect::<Vec<u32>>(),
+                &client,
+                concurrency,
+                RECORDS,
+                get_user_all_music,
+            )
+            .await?;
+        }
+        #[cfg(feature = "fetchall")]
+        Commands::ScrapeAllB50 {
+            concurrency,
+            min_rating,
+            max_rating,
+        } => {
+            use sdgb_api::title::methods::GetUserRatingApiExt;
+
+            use crate::{
+                cache::{B50, PLAYERS},
+                utils::helpers::{cached_concurrent_fetch, read_cache},
+            };
+
+            let mut players: Vec<GetUserPreviewApiResp> = read_cache(PLAYERS)?;
+            players.retain(|p| p.player_rating >= min_rating && p.player_rating <= max_rating);
+
+            cached_concurrent_fetch::<GetUserRatingApiExt>(
+                players.iter().map(|p| p.user_id).collect::<Vec<u32>>(),
+                &client,
+                concurrency,
+                B50,
+            )
+            .await?;
+        }
+        #[cfg(feature = "fetchall")]
+        Commands::ScrapeAllRegion {
+            concurrency,
+            min_rating,
+            max_rating,
+        } => {
+            use sdgb_api::title::methods::GetUserRegionApiExt;
+
+            use crate::{
+                cache::{PLAYERS, REGIONS},
+                utils::helpers::{cached_concurrent_fetch, read_cache},
+            };
+
+            let mut players: Vec<GetUserPreviewApiResp> = read_cache(PLAYERS)?;
+            players.retain(|p| p.player_rating >= min_rating && p.player_rating <= max_rating);
+
+            cached_concurrent_fetch::<GetUserRegionApiExt>(
+                players.iter().map(|p| p.user_id).collect::<Vec<u32>>(),
+                &client,
+                concurrency,
+                REGIONS,
+            )
+            .await?;
+        }
+
+        #[cfg(feature = "fetchall")]
+        Commands::ListAllUserDump {} => {
+            use crate::{
+                cache::PLAYERS,
+                utils::helpers::{dump_parquet, read_cache},
+            };
+
+            let players: Vec<GetUserPreviewApiResp> = read_cache(PLAYERS)?;
+            dump_parquet(players, "players.parquet")?;
+        }
+        #[cfg(feature = "fetchall")]
+        Commands::ScrapeAllRegionDump {} => {
+            use crate::{
+                cache::REGIONS,
+                utils::helpers::{dump_parquet, read_cache},
+            };
+            use sdgb_api::title::model::{GetUserRegionApiResp, UserRegionFlatten};
+
+            let regions: Vec<GetUserRegionApiResp> = read_cache(REGIONS)?;
+            let regions_flat = regions
+                .into_iter()
+                .map(Vec::<UserRegionFlatten>::from)
+                .flatten()
+                .collect::<Vec<_>>();
+            dump_parquet(regions_flat, "regions.parquet")?;
+        }
+        #[cfg(feature = "fetchall")]
+        Commands::ScrapeAllRecordDump {} => {
+            use crate::{
+                cache::RECORDS,
+                utils::helpers::{dump_parquet, read_cache},
+            };
+            use sdgb_api::title::model::GetUserMusicApiResp;
+            use sdgb_api::title::model::UserMusicDetailFlatten;
+
+            let records: Vec<GetUserMusicApiResp> = read_cache(RECORDS)?;
+            dump_parquet(
+                records
+                    .into_iter()
+                    .map(|resp| {
+                        resp.user_music_list
+                            .into_iter()
+                            .map(|music| music.user_music_detail_list)
+                            .flatten()
+                            .map(move |detail| UserMusicDetailFlatten::new(resp.user_id, detail))
+                    })
+                    .flatten()
+                    .collect::<Vec<UserMusicDetailFlatten>>(),
+                "records.parquet",
+            )?;
+        }
+        #[cfg(feature = "fetchall")]
+        Commands::ScrapeAllB50Dump {} => {
+            use crate::{cache::B50, utils::helpers::dump_json};
+
+            dump_json::<GetUserRatingApiResp>("b50.json", B50)?;
+        }
+
+        Commands::Userdata {
+            user_id,
+            skip_login,
+        } => {
+            let action = async |_| match Sdgb1_50::request::<_, GetUserDataApiResp>(
+                &client,
+                APIMethod::GetUserDataApi,
+                user_id,
+                GetUserDataApi { user_id },
+            )
+            .await
+            {
+                Ok(udata) => {
+                    info!("{udata:#?}");
+                }
+                Err(e) => {
+                    error!("failed to get userdata: {e}");
+                }
+            };
+            // userdata does not require loginResult
+            if skip_login {
+                action(UserLoginApiResp::default()).await;
+            } else {
+                login_action(&client, user_id, action).await?;
+            }
+        }
     }
 
     Ok(())
 }
+
+#[cfg(all(feature = "compio", feature = "tokio"))]
+compile_error!("you must not enable both `compio` and `tokio`");

sdgb-cli/src/utils/helpers/mod.rs

@@ -0,0 +1,213 @@
+use std::sync::Arc;
+use std::{fs::OpenOptions, io::BufWriter};
+use std::{path::Path, sync::atomic::Ordering};
+
+use futures_util::StreamExt;
+use nyquest_preset::nyquest::AsyncClient;
+
+use parquet::basic::BrotliLevel;
+use parquet::file::properties::WriterProperties;
+use parquet::file::writer::SerializedFileWriter;
+use parquet::record::RecordWriter;
+use redb::ReadableTable;
+use redb::TableDefinition;
+use serde::Serialize;
+use spdlog::{error, info};
+
+use sdgb_api::title::MaiVersionExt;
+use sdgb_api::title::{Sdgb1_50, methods::APIExt};
+use sdgb_api::{ApiError, bincode};
+
+use bincode::{BorrowDecode, Encode, borrow_decode_from_slice};
+
+use crate::{EARLY_QUIT, cache};
+
+#[allow(unused)]
+pub fn read_cache_keys(
+    definition: TableDefinition<'_, u32, Vec<u8>>,
+) -> Result<Vec<u32>, Box<dyn snafu::Error>> {
+    let txn = cache::read_txn()?;
+    let table = cache::open_table_ro(&txn, definition)?;
+
+    Ok(table
+        .iter()?
+        .flatten()
+        .map(|(value, _)| value.value())
+        .collect::<Vec<u32>>())
+}
+
+pub fn read_cache<D>(
+    definition: TableDefinition<'_, u32, Vec<u8>>,
+) -> Result<Vec<D>, Box<dyn snafu::Error>>
+where
+    D: for<'d> BorrowDecode<'d, ()>,
+{
+    let txn = cache::read_txn()?;
+    let table = cache::open_table_ro(&txn, definition)?;
+
+    let config =
+        bincode::config::Configuration::<bincode::config::LittleEndian>::default().with_no_limit();
+
+    Ok(table
+        .iter()?
+        .flatten()
+        .map(|d| borrow_decode_from_slice(&d.1.value(), config))
+        .flatten()
+        .map(|(value, _)| value)
+        .collect::<Vec<D>>())
+}
+
+pub fn dump_parquet<D>(
+    data: impl Into<Vec<D>>,
+    output_path: impl AsRef<Path>,
+) -> Result<(), Box<dyn snafu::Error>>
+where
+    for<'a> &'a [D]: RecordWriter<D>,
+{
+    let data = data.into();
+    let file = OpenOptions::new()
+        .create(true)
+        .truncate(true)
+        .write(true)
+        .open(output_path)?;
+
+    #[cfg(file_lock_ready)]
+    file.try_lock()?;
+
+    let writer = BufWriter::new(file);
+    let schema = data.as_slice().schema()?;
+    let props = Arc::new(
+        WriterProperties::builder()
+            .set_compression(parquet::basic::Compression::BROTLI(BrotliLevel::try_new(
+                6,
+            )?))
+            .build(),
+    );
+
+    let mut writer = SerializedFileWriter::new(writer, schema, props).unwrap();
+    let mut row_group = writer.next_row_group().unwrap();
+
+    data.as_slice().write_to_row_group(&mut row_group)?;
+    row_group.close()?;
+
+    writer.close().unwrap();
+    info!("dumped {} records", data.len());
+
+    Ok(())
+}
+
+pub fn dump_json<D>(
+    output_path: impl AsRef<Path>,
+    definition: TableDefinition<'_, u32, Vec<u8>>,
+) -> Result<(), Box<dyn snafu::Error>>
+where
+    D: for<'d> BorrowDecode<'d, ()> + Serialize,
+{
+    let file = OpenOptions::new()
+        .create(true)
+        .truncate(true)
+        .write(true)
+        .open(output_path)?;
+
+    #[cfg(file_lock_ready)]
+    file.try_lock()?;
+
+    let data = read_cache::<D>(definition)?;
+    let writer = BufWriter::new(file);
+    serde_json::to_writer(writer, &data)?;
+    info!("dumped {} records", data.len());
+
+    Ok(())
+}
+
+pub async fn cached_concurrent_fetch<A: APIExt>(
+    user_ids: impl Into<Vec<u32>>,
+    client: &AsyncClient,
+    concurrency: usize,
+    definition: TableDefinition<'_, u32, Vec<u8>>,
+) -> Result<(), Box<dyn snafu::Error>>
+where
+    A::Payload: From<u32>,
+    A::Response: Encode + for<'a> BorrowDecode<'a, ()>,
+{
+    cached_concurrent_fetch_userfn(
+        user_ids,
+        client,
+        concurrency,
+        definition,
+        async |client, user_id| {
+            Sdgb1_50::request_ext::<A>(client, A::Payload::from(user_id), user_id).await
+        },
+    )
+    .await
+}
+
+pub async fn cached_concurrent_fetch_userfn<R>(
+    user_ids: impl Into<Vec<u32>>,
+    client: &AsyncClient,
+    concurrency: usize,
+    definition: TableDefinition<'_, u32, Vec<u8>>,
+    scrape: impl AsyncFn(&AsyncClient, u32) -> Result<R, ApiError>,
+) -> Result<(), Box<dyn snafu::Error>>
+where
+    R: Encode + for<'a> BorrowDecode<'a, ()>,
+{
+    let _ = cache::init_db();
+
+    let user_ids = user_ids.into();
+    let read = cache::read_txn()?;
+    let write = cache::write_txn()?;
+    let config = sdgb_api::bincode::config::Configuration::<
+            sdgb_api::bincode::config::LittleEndian,
+        >::default()
+        .with_no_limit();
+
+    info!("number of user_id: {}", user_ids.len());
+
+    let collect = futures_util::stream::iter(user_ids)
+        .map(async |user_id| {
+            {
+                let cache_table = cache::open_table_ro(&read, definition)?;
+                let data = cache_table.get(user_id)?;
+                if let Some(data) = data {
+                    let decoded: (R, _) = borrow_decode_from_slice(&data.value(), config)?;
+
+                    return Ok(decoded.0);
+                }
+            }
+
+            if EARLY_QUIT.load(Ordering::Relaxed) {
+                return Err("early skip due to ctrl-c")?;
+            }
+
+            let resp = scrape(&client, user_id).await;
+
+            match &resp {
+                Ok(resp) => {
+                    use sdgb_api::bincode::encode_to_vec;
+
+                    info!("fetched: {user_id}");
+
+                    if let Ok(mut table) = cache::open_table(&write, definition)
+                        && let Ok(encoded) = encode_to_vec(resp, config)
+                    {
+                        _ = table.insert(user_id, encoded);
+                    }
+                }
+                Err(sdgb_api::ApiError::JSON { .. }) => {}
+                Err(e) => {
+                    error!("fetch failed: {e}");
+                }
+            }
+
+            Result::<_, Box<dyn snafu::Error>>::Ok(resp?)
+        })
+        .buffer_unordered(concurrency) // slower to avoid being banned
+        .filter_map(async |r| r.ok())
+        .collect::<Vec<_>>()
+        .await;
+    drop(collect);
+
+    let _ = write.commit();
+    Ok(())
+}

sdgb-cli/src/utils/mod.rs

@@ -0,0 +1,69 @@
+use std::{fmt::Display, io::stdout};
+
+use nyquest_preset::nyquest::AsyncClient;
+use sdgb_api::{
+    ApiError,
+    title::{
+        MaiVersionExt as _, Sdgb1_50,
+        methods::APIMethod,
+        model::{UserLoginApi, UserLoginApiResp, UserLogoutApi, UserLogoutApiResp},
+    },
+};
+use serde::Serialize;
+use spdlog::info;
+
+pub async fn login_action<R>(
+    client: &AsyncClient,
+    user_id: u32,
+    action: impl AsyncFnOnce(UserLoginApiResp) -> R,
+) -> Result<R, ApiError> {
+    let login = UserLoginApi::new(user_id, true);
+    let date_time = login.date_time;
+
+    let login_resp: UserLoginApiResp =
+        Sdgb1_50::request(&client, APIMethod::UserLoginApi, user_id, login).await?;
+
+    match login_resp.error() {
+        None => info!("login succeed"),
+        Some(e) => return Err(e)?,
+    }
+
+    let return_data = action(login_resp).await;
+
+    let logout_resp = Sdgb1_50::request::<_, UserLogoutApiResp>(
+        &client,
+        APIMethod::UserLogoutApi,
+        user_id,
+        UserLogoutApi {
+            user_id,
+            date_time,
+            ..Default::default()
+        },
+    )
+    .await;
+
+    info!("logout: {logout_resp:?}");
+    Ok(return_data)
+}
+
+pub fn json_display(value: impl Serialize) -> Result<(), Box<dyn snafu::Error>> {
+    let lock = stdout().lock();
+    serde_json::to_writer_pretty(lock, &value)?;
+    Ok(())
+}
+
+pub fn human_readable_display(
+    value: impl Display + Serialize,
+    human_readable: bool,
+) -> Result<(), Box<dyn snafu::Error>> {
+    if human_readable {
+        println!("{value}");
+    } else {
+        json_display(value)?;
+    }
+
+    Ok(())
+}
+
+#[cfg(feature = "fetchall")]
+pub mod helpers;

utils/export_b50.py

@@ -0,0 +1,106 @@
+import orjson as json
+from typing import Callable
+from datetime import datetime
+from decimal import Decimal
+
+from helpers import dx_rating, find_level, query_music_db, salted_hash_userid
+
+
+def clean_b50(b50: dict[str, str | dict]):
+    urating: dict[str, list[dict[str, int]]] = b50["userRating"]
+
+    def add_rating(entry: dict[str, int]):
+        """
+        ```
+        {
+            "musicId": 11638,
+            "level": 2,
+            "romVersion": 24005,
+            "achievement": 988145
+        }
+        ```
+        - level: EXPERT
+        - ver: DX, 1.40.05
+        - ach: 98.8145%
+        """
+
+        entry["musicTitle"] = None
+        entry["difficulty"] = None
+        entry["dxRating"] = 0
+
+        music_info = query_music_db(entry["musicId"])
+
+        if music_info is None:
+            return
+
+        entry["musicTitle"] = music_info["name"]
+        levels = find_level(music_info, entry["level"])
+
+        if not levels:
+            return
+
+        level: dict[str, str | int] = levels.pop()
+        difficulty = level["difficulty"]
+
+        entry["difficulty"] = difficulty
+        entry["dxRating"] = dx_rating(
+            difficulty=Decimal(difficulty),
+            achievement=entry["achievement"],
+        )
+
+    for b35 in urating["ratingList"]:
+        add_rating(b35)
+    for b15 in urating["newRatingList"]:
+        add_rating(b15)
+
+    urating["rating"] = sum(
+        map(
+            lambda lst: sum(map(lambda entry: entry["dxRating"], urating[lst])),
+            ["ratingList", "newRatingList"],
+        )
+    )
+
+
+def record_time(*, _: list[datetime] = []):
+    last_time = _
+    if not last_time:
+        last_time.append(datetime.now())
+    else:
+        new = datetime.now()
+        diff = (new - last_time.pop()).total_seconds()
+        last_time.append(new)
+        return diff
+
+
+def process(
+    clean_fields: Callable[[dict], None],
+    input_file: str,
+    output_file: str,
+):
+    record_time()
+    with open(input_file, "rb") as f:
+        data = json.loads(f.read())
+    print(f"loaded, cost {record_time():.2f}s")
+
+    for entry in data:
+        entry["userId"] = salted_hash_userid(entry["userId"])
+        clean_fields(entry)
+    print(f"processed, cost {record_time():.2f}s")
+
+    with open(output_file, "wb") as f:
+        f.write(json.dumps(data))
+    print(f"written out, cost {record_time():.2f}s")
+
+    return data
+
+
+def main():
+    process(
+        clean_b50,
+        "b50.json",
+        "b50_pub.json",
+    )
+
+
+if __name__ == "__main__":
+    main()

utils/get_last_uid.py

@@ -0,0 +1,11 @@
+import orjson as json
+
+
+def main():
+    with open("players.json", "r", encoding="utf-8") as f:
+        d: list[dict[str, int | str]] = json.loads(f.read())
+    print(d[-1]["userId"])
+
+
+if __name__ == "__main__":
+    main()

utils/hash_userid.py

@@ -0,0 +1,13 @@
+from sys import argv
+import polars as pl
+import polars_hash as pl_hash
+
+file = argv[1]
+
+pl.scan_parquet(file).with_columns(
+    pl.col("user_id").cast(pl.String).add("Lt2N5xgjJOqRsT5qVt7wWYw6SqOPZDI7")
+).with_columns(
+    pl_hash.col("user_id").chash.sha2_256().str.head(16)
+).collect().write_parquet(
+    file.replace(".parquet", "_pub.parquet"), compression="zstd", compression_level=15
+)

utils/helpers.py

@@ -0,0 +1,105 @@
+from decimal import Decimal, getcontext
+import hashlib
+
+import orjson as json
+from diskcache import Cache
+
+getcontext().prec = 28
+
+CACHE = Cache("target")
+
+
+def salted_hash_userid(user_id: int):
+    hex = CACHE.get(user_id)
+    if hex is not None:
+        return hex
+
+    SALT = b"Lt2N5xgjJOqRsT5qVt7wWYw6SqOPZDI7"
+
+    hash_uid = hashlib.sha256(f"{user_id}".encode("utf-8") + SALT)
+    result = hash_uid.hexdigest()[:16]
+
+    CACHE.add(user_id, result)
+    return result
+
+
+def dx_rating(difficulty: Decimal, achievement: int) -> int:
+    # Constants
+    SSS_PLUS_THRESHOLD = Decimal("100.5")
+    SSS_PLUS_FACTOR = Decimal("0.224")
+    SSS_PRO_THRESHOLD = Decimal("100.4999")
+    SSS_PRO_FACTOR = Decimal("0.222")
+    SSS_THRESHOLD = Decimal("100.0")
+    SSS_FACTOR = Decimal("0.216")
+    SS_PLUS_PRO_THRESHOLD = Decimal("99.9999")
+    SS_PLUS_PRO_FACTOR = Decimal("0.214")
+    SS_PLUS_THRESHOLD = Decimal("99.5")
+    SS_PLUS_FACTOR = Decimal("0.211")
+    SS_THRESHOLD = Decimal("99.0")
+    SS_FACTOR = Decimal("0.208")
+    S_PLUS_PRO_THRESHOLD = Decimal("98.9999")
+    S_PLUS_PRO_FACTOR = Decimal("0.206")
+    S_PLUS_THRESHOLD = Decimal("98.0")
+    S_PLUS_FACTOR = Decimal("0.203")
+    S_THRESHOLD = Decimal("97.0")
+    S_FACTOR = Decimal("0.2")
+    AAA_PRO_THRESHOLD = Decimal("96.9999")
+    AAA_PRO_FACTOR = Decimal("0.176")
+    AAA_THRESHOLD = Decimal("94.0")
+    AAA_FACTOR = Decimal("0.168")
+    AA_THRESHOLD = Decimal("90.0")
+    AA_FACTOR = Decimal("0.152")
+    A_THRESHOLD = Decimal("80.0")
+    A_FACTOR = Decimal("0.136")
+
+    ach = Decimal(achievement) / Decimal("10000")
+    if ach > Decimal("101.0") or ach < A_THRESHOLD:
+        return 0
+    if ach >= SSS_PLUS_THRESHOLD:
+        factor = SSS_PLUS_FACTOR
+        ach = Decimal("100.5")
+    elif ach >= SSS_PRO_THRESHOLD:
+        factor = SSS_PRO_FACTOR
+    elif ach >= SSS_THRESHOLD:
+        factor = SSS_FACTOR
+    elif ach >= SS_PLUS_PRO_THRESHOLD:
+        factor = SS_PLUS_PRO_FACTOR
+    elif ach >= SS_PLUS_THRESHOLD:
+        factor = SS_PLUS_FACTOR
+    elif ach >= SS_THRESHOLD:
+        factor = SS_FACTOR
+    elif ach >= S_PLUS_PRO_THRESHOLD:
+        factor = S_PLUS_PRO_FACTOR
+    elif ach >= S_PLUS_THRESHOLD:
+        factor = S_PLUS_FACTOR
+    elif ach >= S_THRESHOLD:
+        factor = S_FACTOR
+    elif ach >= AAA_PRO_THRESHOLD:
+        factor = AAA_PRO_FACTOR
+    elif ach >= AAA_THRESHOLD:
+        factor = AAA_FACTOR
+    elif ach >= AA_THRESHOLD:
+        factor = AA_FACTOR
+    elif ach >= A_THRESHOLD:
+        factor = A_FACTOR
+    else:
+        return 0
+    result = (factor * difficulty * ach).quantize(Decimal("1."), rounding="ROUND_FLOOR")
+    return int(result)
+
+
+with open("musicDB.json", "r", encoding="utf-8") as f:
+    MUSIC_DB = json.loads(f.read())
+
+MUSIC_DB = {entry["id"]: entry for entry in MUSIC_DB}
+
+
+def query_music_db(music_id: int):
+    music_info = MUSIC_DB.get(music_id)
+    if music_info is None:
+        return
+    return music_info
+
+
+def find_level(music_info: dict, level_id: int):
+    return [level for level in music_info["levels"] if level["level"] == level_id]

utils/music_db_dump.py

@@ -0,0 +1,128 @@
+# forked from maimaiDX-Api
+import json
+import xml.dom.minidom as minidom
+from pathlib import Path
+
+ONLY_REMOVED = True
+
+
+def makeMusicDBJson():
+    """
+    从 HDD 的文件来生成 music_db.json
+    推荐的是如果要国服用 那就用国际服的文件来生成
+    免得国服每次更新还要重新生成太麻烦
+    """
+    # 记得改
+    A000_DIR = Path(
+        "C:/MaimaiDX/SDEZ-1.56-B/Standard/Package/Sinmai_Data/StreamingAssets/A000"
+    )
+    OPTION_DIR = Path("C:/MaimaiDX/SDGA-1.50-G/NoMovieData/StreamingAssets")
+
+    music_db: list[dict[str, str | int | list[dict[str, str | int]]]] = []
+    DEST_PATH = Path("./musicDB.json")
+
+    dup_count = 0
+    music_ids = set()
+
+    music_folders = [f for f in (A000_DIR / "music").iterdir() if f.is_dir()]
+    for option_dir in OPTION_DIR.iterdir():
+        # only removed ones
+        if ONLY_REMOVED and option_dir.name != "A100":
+            continue
+
+        if (option_dir / "music").exists():
+            music_folders.extend(
+                [f for f in (option_dir / "music").iterdir() if f.is_dir()]
+            )
+
+    for folder in music_folders:
+        xml_path = folder / "Music.xml"
+        if xml_path.exists():
+            xml = minidom.parse(xml_path.as_posix())
+            data = xml.getElementsByTagName("MusicData")[0]
+            music_id = int(
+                data.getElementsByTagName("name")[0]
+                .getElementsByTagName("id")[0]
+                .firstChild.data
+            )
+            music_name = (
+                data.getElementsByTagName("name")[0]
+                .getElementsByTagName("str")[0]
+                .firstChild.data
+            )
+            music_version = (
+                data.getElementsByTagName("AddVersion")[0]
+                .getElementsByTagName("id")[0]
+                .firstChild.data
+            )
+
+            def handle_note(note: minidom.Element):
+                if (
+                    "false"
+                    == note.getElementsByTagName("isEnable")
+                    .pop()
+                    .firstChild.data.lower()
+                ):
+                    return
+
+                if music_id >= 100000:
+                    level = 5
+                else:
+                    level = int(
+                        note.getElementsByTagName("file")
+                        .pop()
+                        .getElementsByTagName("path")
+                        .pop()
+                        .firstChild.data[-5]
+                    )
+                difficulty_int = (
+                    note.getElementsByTagName("level").pop().firstChild.data
+                )
+                difficulty_dec = (
+                    note.getElementsByTagName("levelDecimal").pop().firstChild.data
+                )
+                difficulty = f"{difficulty_int}.{difficulty_dec}"
+                return level, difficulty
+
+            music_levels = [
+                {"level": level, "difficulty": difficulty}
+                for level, difficulty in filter(
+                    lambda d: d is not None,
+                    (
+                        handle_note(note)
+                        for note in data.getElementsByTagName("notesData")[
+                            0
+                        ].getElementsByTagName("Notes")
+                    ),
+                )
+            ]
+
+            if music_id not in music_ids:
+                music_ids.add(music_id)
+                music_db.append(
+                    {
+                        "id": music_id,
+                        "name": music_name,
+                        "version": int(music_version),
+                        "levels": music_levels,
+                    }
+                )
+            else:
+                # e.g. SDEZ-only song
+                dup_count += 1
+
+    print(f"Found {len(music_db)} music data")
+    print(f"Found {dup_count} duplications")
+
+    music_db = sorted(
+        music_db,
+        key=lambda m: m["id"],
+    )
+
+    with open(DEST_PATH, "w", encoding="utf-8") as f:
+        json.dump(music_db, f, ensure_ascii=False, indent=4)
+
+
+if __name__ == "__main__":
+    makeMusicDBJson()
+    print("Done.")