diff --git a/Cargo.lock b/Cargo.lock index 6314311..7c135c2 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -28,35 +28,6 @@ dependencies = [ "cpufeatures", ] -[[package]] -name = "ahash" -version = "0.8.12" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5a15f179cd60c4584b8a8c596927aadc462e27f2ca70c04e0071964a73ba7a75" -dependencies = [ - "cfg-if", - "const-random", - "getrandom 0.3.4", - "once_cell", - "version_check", - "zerocopy", -] - -[[package]] -name = "alloc-no-stdlib" -version = "2.0.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cc7bb162ec39d46ab1ca8c77bf72e890535becd1751bb45f64c597edb4c8c6b3" - -[[package]] -name = "alloc-stdlib" -version = "0.2.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "94fb8275041c72129eb51b7d0322c29b8387a0386127718b096429201a5d6ece" -dependencies = [ - "alloc-no-stdlib", -] - [[package]] name = "android_system_properties" version = "0.1.5" @@ -81,132 +52,12 @@ version = "0.7.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7c02d123df017efcdfbd739ef81735b36c5ba83ec3c59c80a9d7ecc718f92e50" -[[package]] -name = "arrow-array" -version = "57.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a23eaff85a44e9fa914660fb0d0bb00b79c4a3d888b5334adb3ea4330c84f002" -dependencies = [ - "ahash", - "arrow-buffer", - "arrow-data", - "arrow-schema", - "chrono", - "half", - "hashbrown", - "num-complex", - "num-integer", - "num-traits", -] - -[[package]] -name = "arrow-buffer" -version = "57.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a2819d893750cb3380ab31ebdc8c68874dd4429f90fd09180f3c93538bd21626" -dependencies = [ - "bytes", - "half", - "num-bigint", - "num-traits", -] - -[[package]] -name = "arrow-cast" -version = "57.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e3d131abb183f80c450d4591dc784f8d7750c50c6e2bc3fcaad148afc8361271" -dependencies = [ - "arrow-array", - "arrow-buffer", - "arrow-data", - "arrow-ord", - "arrow-schema", - "arrow-select", - "atoi", - "base64", - "chrono", - "half", - "lexical-core", - "num-traits", - "ryu", -] - -[[package]] -name = "arrow-data" -version = "57.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "05738f3d42cb922b9096f7786f606fcb8669260c2640df8490533bb2fa38c9d3" -dependencies = [ - "arrow-buffer", - "arrow-schema", - "half", - "num-integer", - "num-traits", -] - -[[package]] -name = "arrow-ipc" -version = "57.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3d09446e8076c4b3f235603d9ea7c5494e73d441b01cd61fb33d7254c11964b3" -dependencies = [ - "arrow-array", - "arrow-buffer", - "arrow-data", - "arrow-schema", - "arrow-select", - "flatbuffers", -] - -[[package]] -name = "arrow-ord" -version = "57.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cbc94fc7adec5d1ba9e8cd1b1e8d6f72423b33fe978bf1f46d970fafab787521" -dependencies = [ - "arrow-array", - "arrow-buffer", - "arrow-data", - "arrow-schema", - "arrow-select", -] - -[[package]] -name = "arrow-schema" -version = "57.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d27609cd7dd45f006abae27995c2729ef6f4b9361cde1ddd019dc31a5aa017e0" - -[[package]] -name = "arrow-select" -version = "57.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ae980d021879ea119dd6e2a13912d81e64abed372d53163e804dfe84639d8010" -dependencies = [ - "ahash", - "arrow-array", - "arrow-buffer", - "arrow-data", - "arrow-schema", - "num-traits", -] - [[package]] name = "async-task" version = "4.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8b75356056920673b02621b35afd0f7dda9306d03c79a30f5c56c44cf256e3de" -[[package]] -name = "atoi" -version = "2.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f28d99ec8bfea296261ca1af174f24225171fea9664ba9003cbebee704810528" -dependencies = [ - "num-traits", -] - [[package]] name = "atomic" version = "0.6.1" @@ -237,32 +88,6 @@ dependencies = [ "windows-link 0.2.1", ] -[[package]] -name = "base64" -version = "0.22.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6" - -[[package]] -name = "bincode" -version = "2.0.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "36eaf5d7b090263e8150820482d5d93cd964a81e4019913c972f4edcc6edb740" -dependencies = [ - "bincode_derive", - "serde", - "unty", -] - -[[package]] -name = "bincode_derive" -version = "2.0.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bf95709a440f45e986983918d0e8a1f30a9b1df04918fc828670606804ac3c09" -dependencies = [ - "virtue", -] - [[package]] name = "bitflags" version = "2.10.0" @@ -296,27 +121,6 @@ dependencies = [ "objc2", ] -[[package]] -name = "brotli" -version = "8.0.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4bd8b9603c7aa97359dbd97ecf258968c95f3adddd6db2f7e7a5bef101c84560" -dependencies = [ - "alloc-no-stdlib", - "alloc-stdlib", - "brotli-decompressor", -] - -[[package]] -name = "brotli-decompressor" -version = "5.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "874bb8112abecc98cbd6d81ea4fa7e94fb9449648c93cc89aa40c81c24d7de03" -dependencies = [ - "alloc-no-stdlib", - "alloc-stdlib", -] - [[package]] name = "bumpalo" version = "3.19.1" @@ -343,12 +147,6 @@ dependencies = [ "syn", ] -[[package]] -name = "byteorder" -version = "1.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b" - [[package]] name = "bytes" version = "1.11.0" @@ -366,13 +164,11 @@ dependencies = [ [[package]] name = "cc" -version = "1.2.51" +version = "1.2.53" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7a0aeaff4ff1a90589618835a598e545176939b97874f7abc7851caa0618f203" +checksum = "755d2fce177175ffca841e9a06afdb2c4ab0f593d53b4dee48147dfaade85932" dependencies = [ "find-msvc-tools", - "jobserver", - "libc", "shlex", ] @@ -390,9 +186,9 @@ checksum = "613afe47fcd5fac7ccf1db93babcb082c5994d996f20b8b159f2ad1658eb5724" [[package]] name = "chrono" -version = "0.4.42" +version = "0.4.43" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "145052bdd345b87320e369255277e3fb5152762ad123a901ef5c262dd38fe8d2" +checksum = "fac4744fb15ae8337dc853fee7fb3f4e48c0fbaa23d0afe49c447b4fab126118" dependencies = [ "iana-time-zone", "js-sys", @@ -569,26 +365,6 @@ version = "0.9.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c2459377285ad874054d797f3ccebf984978aa39129f6eafde5cdc8315b612f8" -[[package]] -name = "const-random" -version = "0.1.18" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "87e00182fe74b066627d63b85fd550ac2998d4b0bd86bfed477a0ae4c7c71359" -dependencies = [ - "const-random-macro", -] - -[[package]] -name = "const-random-macro" -version = "0.1.16" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f9d839f2a20b0aee515dc581a6172f2321f96cab76c1a38a4c584a194955390e" -dependencies = [ - "getrandom 0.2.16", - "once_cell", - "tiny-keccak", -] - [[package]] name = "core-foundation-sys" version = "0.8.7" @@ -659,12 +435,6 @@ version = "0.8.21" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d0a5c400df2834b80a4c3327b3aad3a4c4cd4de0629063962b03235697506a28" -[[package]] -name = "crunchy" -version = "0.2.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "460fbee9c2c2f33933d720630a6a0bac33ba7053db5344fac858d4b8952d77d5" - [[package]] name = "crypto-common" version = "0.1.7" @@ -703,9 +473,9 @@ dependencies = [ [[package]] name = "curl-sys" -version = "0.4.84+curl-8.17.0" +version = "0.4.85+curl-8.18.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "abc4294dc41b882eaff37973c2ec3ae203d0091341ee68fbadd1d06e0c18a73b" +checksum = "c0efa6142b5ecc05f6d3eaa39e6af4888b9d3939273fb592c92b7088a8cf3fdb" dependencies = [ "cc", "libc", @@ -780,28 +550,17 @@ dependencies = [ [[package]] name = "find-msvc-tools" -version = "0.1.6" +version = "0.1.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "645cbb3a84e60b7531617d5ae4e57f7e27308f6445f5abf653209ea76dec8dff" - -[[package]] -name = "flatbuffers" -version = "25.12.19" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "35f6839d7b3b98adde531effaf34f0c2badc6f4735d26fe74709d8e513a96ef3" -dependencies = [ - "bitflags", - "rustc_version", -] +checksum = "8591b0bcc8a98a64310a2fae1bb3e9b8564dd10e381e6e28010fde8e8e8568db" [[package]] name = "flate2" -version = "1.1.5" +version = "1.1.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bfe33edd8e85a12a67454e37f8c75e730830d83e313556ab9ebf9ee7fbeb3bfb" +checksum = "b375d6465b98090a5f25b1c7703f3859783755aa9a80433b36e0379a3ec2f369" dependencies = [ "crc32fast", - "libz-rs-sys", "miniz_oxide", ] @@ -885,47 +644,12 @@ dependencies = [ "version_check", ] -[[package]] -name = "getrandom" -version = "0.2.16" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "335ff9f135e4384c8150d6f27c6daed433577f86b4750418338c01a1a2528592" -dependencies = [ - "cfg-if", - "libc", - "wasi", -] - -[[package]] -name = "getrandom" -version = "0.3.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "899def5c37c4fd7b2664648c28120ecec138e4d395b459e5ca34f9cce2dd77fd" -dependencies = [ - "cfg-if", - "libc", - "r-efi", - "wasip2", -] - [[package]] name = "gimli" version = "0.32.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e629b9b98ef3dd8afe6ca2bd0f89306cec16d43d907889945bc5d6687f2f13c7" -[[package]] -name = "half" -version = "2.7.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6ea2d84b969582b4b1864a92dc5d27cd2b77b622a8d79306834f1be5ba20d84b" -dependencies = [ - "cfg-if", - "crunchy", - "num-traits", - "zerocopy", -] - [[package]] name = "hashbrown" version = "0.16.1" @@ -979,9 +703,9 @@ dependencies = [ [[package]] name = "indexmap" -version = "2.12.1" +version = "2.13.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0ad4bb2b565bca0645f4d68c5c9af97fba094e9791da685bf83cb5f3ce74acf2" +checksum = "7714e70437a7dc3ac8eb7e6f8df75fd8eb422675fc7678aff7364301092b1017" dependencies = [ "equivalent", "hashbrown", @@ -997,12 +721,6 @@ dependencies = [ "generic-array", ] -[[package]] -name = "integer-encoding" -version = "3.0.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8bb03732005da905c88227371639bf1ad885cc712789c011c31c5fb3ab3ccf02" - [[package]] name = "io-uring" version = "0.7.11" @@ -1042,103 +760,21 @@ version = "1.0.17" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "92ecc6618181def0457392ccd0ee51198e065e016d1d527a7ac1b6dc7c1f09d2" -[[package]] -name = "jobserver" -version = "0.1.34" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9afb3de4395d6b3e67a780b6de64b51c978ecf11cb9a462c66be7d4ca9039d33" -dependencies = [ - "getrandom 0.3.4", - "libc", -] - [[package]] name = "js-sys" -version = "0.3.83" +version = "0.3.85" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "464a3709c7f55f1f721e5389aa6ea4e3bc6aba669353300af094b29ffbdde1d8" +checksum = "8c942ebf8e95485ca0d52d97da7c5a2c387d0e7f0ba4c35e93bfcaee045955b3" dependencies = [ "once_cell", "wasm-bindgen", ] -[[package]] -name = "lexical-core" -version = "1.0.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7d8d125a277f807e55a77304455eb7b1cb52f2b18c143b60e766c120bd64a594" -dependencies = [ - "lexical-parse-float", - "lexical-parse-integer", - "lexical-util", - "lexical-write-float", - "lexical-write-integer", -] - -[[package]] -name = "lexical-parse-float" -version = "1.0.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "52a9f232fbd6f550bc0137dcb5f99ab674071ac2d690ac69704593cb4abbea56" -dependencies = [ - "lexical-parse-integer", - "lexical-util", -] - -[[package]] -name = "lexical-parse-integer" -version = "1.0.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9a7a039f8fb9c19c996cd7b2fcce303c1b2874fe1aca544edc85c4a5f8489b34" -dependencies = [ - "lexical-util", -] - -[[package]] -name = "lexical-util" -version = "1.0.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2604dd126bb14f13fb5d1bd6a66155079cb9fa655b37f875b3a742c705dbed17" - -[[package]] -name = "lexical-write-float" -version = "1.0.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "50c438c87c013188d415fbabbb1dceb44249ab81664efbd31b14ae55dabb6361" -dependencies = [ - "lexical-util", - "lexical-write-integer", -] - -[[package]] -name = "lexical-write-integer" -version = "1.0.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "409851a618475d2d5796377cad353802345cba92c867d9fbcde9cf4eac4e14df" -dependencies = [ - "lexical-util", -] - [[package]] name = "libc" -version = "0.2.179" +version = "0.2.180" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c5a2d376baa530d1238d133232d15e239abad80d05838b4b59354e5268af431f" - -[[package]] -name = "libm" -version = "0.2.15" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f9fbbcab51052fe104eb5e5d351cf728d30a5be1fe14d9be8a3b097481fb97de" - -[[package]] -name = "libz-rs-sys" -version = "0.5.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c10501e7805cee23da17c7790e59df2870c0d4043ec6d03f67d31e2b53e77415" -dependencies = [ - "zlib-rs", -] +checksum = "bcc35a38544a891a5f7c865aca548a982ccb3b8650a5b06d0fd33a10283c56fc" [[package]] name = "libz-sys" @@ -1173,15 +809,6 @@ version = "0.4.29" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5e5032e24019045c762d3c0f28f5b6b8bbf38563a65908389bf7978758920897" -[[package]] -name = "lz4_flex" -version = "0.12.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ab6473172471198271ff72e9379150e9dfd70d8e533e0752a27e515b48dd375e" -dependencies = [ - "twox-hash", -] - [[package]] name = "md5" version = "0.8.0" @@ -1236,34 +863,6 @@ dependencies = [ "memchr", ] -[[package]] -name = "num-bigint" -version = "0.4.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a5e44f723f1133c9deac646763579fdb3ac745e418f2a7af9cd0c431da1f20b9" -dependencies = [ - "num-integer", - "num-traits", -] - -[[package]] -name = "num-complex" -version = "0.4.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "73f88a1307638156682bada9d7604135552957b7818057dcef22705b4d509495" -dependencies = [ - "num-traits", -] - -[[package]] -name = "num-integer" -version = "0.1.46" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7969661fd2958a5cb096e56c8e1ad0444ac2bbcd0061bd28660485a44879858f" -dependencies = [ - "num-traits", -] - [[package]] name = "num-traits" version = "0.2.19" @@ -1271,7 +870,6 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "071dfc062690e90b734c0b2273ce72ad0ffa95f0c74596bc250dcfd960262841" dependencies = [ "autocfg", - "libm", ] [[package]] @@ -1450,15 +1048,6 @@ dependencies = [ "vcpkg", ] -[[package]] -name = "ordered-float" -version = "2.10.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "68f19d67e5a2795c94e73e0bb1cc1a7edeb2e28efd39e2e1c9b7a40c1108b11c" -dependencies = [ - "num-traits", -] - [[package]] name = "os_pipe" version = "1.2.3" @@ -1521,52 +1110,6 @@ dependencies = [ "windows-link 0.2.1", ] -[[package]] -name = "parquet" -version = "57.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "be3e4f6d320dd92bfa7d612e265d7d08bba0a240bab86af3425e1d255a511d89" -dependencies = [ - "ahash", - "arrow-array", - "arrow-buffer", - "arrow-cast", - "arrow-data", - "arrow-ipc", - "arrow-schema", - "arrow-select", - "base64", - "brotli", - "bytes", - "chrono", - "flate2", - "half", - "hashbrown", - "lz4_flex", - "num-bigint", - "num-integer", - "num-traits", - "paste", - "seq-macro", - "simdutf8", - "snap", - "thrift", - "twox-hash", - "zstd", -] - -[[package]] -name = "parquet_derive" -version = "57.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "777a89bd0515e7948516ce6d3f41c16b3a32de71f7eb99c5fdb55456b695b227" -dependencies = [ - "parquet", - "proc-macro2", - "quote", - "syn", -] - [[package]] name = "paste" version = "1.0.15" @@ -1616,37 +1159,22 @@ dependencies = [ [[package]] name = "proc-macro2" -version = "1.0.104" +version = "1.0.106" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9695f8df41bb4f3d222c95a67532365f569318332d03d5f3f67f37b20e6ebdf0" +checksum = "8fd00f0bb2e90d81d1044c2b32617f68fcb9fa3bb7640c23e9c748e53fb30934" dependencies = [ "unicode-ident", ] [[package]] name = "quote" -version = "1.0.42" +version = "1.0.43" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a338cc41d27e6cc6dce6cefc13a0729dfbb81c262b1f519331575dd80ef3067f" +checksum = "dc74d9a594b72ae6656596548f56f667211f8a97b3d4c3d467150794690dc40a" dependencies = [ "proc-macro2", ] -[[package]] -name = "r-efi" -version = "5.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "69cdb34c158ceb288df11e18b4bd39de994f6657d83847bdffdbd7f346754b0f" - -[[package]] -name = "redb" -version = "3.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ae323eb086579a3769daa2c753bb96deb95993c534711e0dbe881b5192906a06" -dependencies = [ - "libc", -] - [[package]] name = "redox_syscall" version = "0.5.18" @@ -1678,22 +1206,23 @@ dependencies = [ [[package]] name = "rust_decimal" -version = "1.39.0" +version = "1.40.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "35affe401787a9bd846712274d97654355d21b2a2c092a3139aabe31e9022282" +checksum = "61f703d19852dbf87cbc513643fa81428361eb6940f1ac14fd58155d295a3eb0" dependencies = [ "arrayvec", "num-traits", "rust_decimal_macros", + "ryu", "serde", "serde_json", ] [[package]] name = "rust_decimal_macros" -version = "1.39.0" +version = "1.40.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ae8c0cb48f413ebe24dc2d148788e0efbe09ba3e011d9277162f2eaf8e1069a3" +checksum = "74a5a6f027e892c7a035c6fddb50435a1fbf5a734ffc0c2a9fed4d0221440519" dependencies = [ "quote", "syn", @@ -1701,9 +1230,9 @@ dependencies = [ [[package]] name = "rustc-demangle" -version = "0.1.26" +version = "0.1.27" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "56f7d92ca342cea22a06f2121d944b4fd82af56988c270852495420f961d4ace" +checksum = "b50b8869d9fc858ce7266cce0194bd74df58b9d0e3f6df3a9fc8eb470d95c09d" [[package]] name = "rustc-hash" @@ -1771,7 +1300,6 @@ name = "sdgb-api" version = "0.1.0" dependencies = [ "aes", - "bincode", "cbc", "chrono", "cipher", @@ -1783,8 +1311,6 @@ dependencies = [ "md5", "music-db", "nyquest", - "parquet", - "parquet_derive", "serde", "serde_json", "snafu", @@ -1804,8 +1330,6 @@ dependencies = [ "music-db", "nyquest-preset", "palc", - "parquet", - "redb", "sdgb-api", "serde", "serde_json", @@ -1822,12 +1346,6 @@ version = "1.0.27" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d767eb0aabc880b29956c35734170f26ed551a859dbd361d140cdbeca61ab1e2" -[[package]] -name = "seq-macro" -version = "0.3.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1bc711410fbe7399f390ca1c3b60ad0f53f80e95c5eb935e52268a0e2cd49acc" - [[package]] name = "serde" version = "1.0.228" @@ -1878,9 +1396,9 @@ dependencies = [ [[package]] name = "serde_json" -version = "1.0.148" +version = "1.0.149" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3084b546a1dd6289475996f182a22aba973866ea8e8b02c51d9f46b1336a22da" +checksum = "83fc039473c5595ace860d8c4fafa220ff474b3fc6bfdb4293327f1a37e94d86" dependencies = [ "itoa", "memchr", @@ -1910,12 +1428,6 @@ version = "0.3.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e320a6c5ad31d271ad523dcf3ad13e2767ad8b1cb8f047f75a8aeaf8da139da2" -[[package]] -name = "simdutf8" -version = "0.1.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e3a9fe34e3e7a50316060351f37187a3f546bce95496156754b601a5fa71b76e" - [[package]] name = "slab" version = "0.4.11" @@ -1950,12 +1462,6 @@ dependencies = [ "syn", ] -[[package]] -name = "snap" -version = "1.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1b6b67fb9a61334225b5b790716f609cd58395f895b3fe8b328786812a40bc3b" - [[package]] name = "socket2" version = "0.6.1" @@ -2122,9 +1628,9 @@ dependencies = [ [[package]] name = "syn" -version = "2.0.113" +version = "2.0.114" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "678faa00651c9eb72dd2020cbdf275d92eccb2400d568e419efdd64838145cb4" +checksum = "d4d107df263a3013ef9b1879b0df87d706ff80f65a86ea879bd9c31f9b307c2a" dependencies = [ "proc-macro2", "quote", @@ -2133,44 +1639,24 @@ dependencies = [ [[package]] name = "thiserror" -version = "2.0.17" +version = "2.0.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f63587ca0f12b72a0600bcba1d40081f830876000bb46dd2337a3051618f4fc8" +checksum = "4288b5bcbc7920c07a1149a35cf9590a2aa808e0bc1eafaade0b80947865fbc4" dependencies = [ "thiserror-impl", ] [[package]] name = "thiserror-impl" -version = "2.0.17" +version = "2.0.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3ff15c8ecd7de3849db632e14d18d2571fa09dfc5ed93479bc4485c7a517c913" +checksum = "ebc4ee7f67670e9b64d05fa4253e753e016c6c95ff35b89b7941d6b856dec1d5" dependencies = [ "proc-macro2", "quote", "syn", ] -[[package]] -name = "thrift" -version = "0.17.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7e54bc85fc7faa8bc175c4bab5b92ba8d9a3ce893d0e9f42cc455c8ab16a9e09" -dependencies = [ - "byteorder", - "integer-encoding", - "ordered-float", -] - -[[package]] -name = "tiny-keccak" -version = "2.0.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2c9d3793400a45f954c52e73d068316d76b6f4e36977e3fcebb13a2721e80237" -dependencies = [ - "crunchy", -] - [[package]] name = "tokio" version = "1.49.0" @@ -2279,12 +1765,6 @@ version = "0.1.36" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "db97caf9d906fbde555dd62fa95ddba9eecfd14cb388e4f491a66d74cd5fb79a" -[[package]] -name = "twox-hash" -version = "2.1.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9ea3136b675547379c4bd395ca6b938e5ad3c3d20fad76e7fe85f9e0d011419c" - [[package]] name = "typeid" version = "1.0.3" @@ -2303,12 +1783,6 @@ version = "1.0.22" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9312f7c4f6ff9069b165498234ce8be658059c6728633667c526e27dc2cf1df5" -[[package]] -name = "unty" -version = "0.0.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6d49784317cd0d1ee7ec5c716dd598ec5b4483ea832a2dced265471cc0f690ae" - [[package]] name = "value-bag" version = "1.12.0" @@ -2358,32 +1832,11 @@ version = "0.9.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" -[[package]] -name = "virtue" -version = "0.0.18" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "051eb1abcf10076295e815102942cc58f9d5e3b4560e46e53c21e8ff6f3af7b1" - -[[package]] -name = "wasi" -version = "0.11.1+wasi-snapshot-preview1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ccf3ec651a847eb01de73ccad15eb7d99f80485de043efb2f370cd654f4ea44b" - -[[package]] -name = "wasip2" -version = "1.0.1+wasi-0.2.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0562428422c63773dad2c345a1882263bbf4d65cf3f42e90921f787ef5ad58e7" -dependencies = [ - "wit-bindgen", -] - [[package]] name = "wasm-bindgen" -version = "0.2.106" +version = "0.2.108" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0d759f433fa64a2d763d1340820e46e111a7a5ab75f993d1852d70b03dbb80fd" +checksum = "64024a30ec1e37399cf85a7ffefebdb72205ca1c972291c51512360d90bd8566" dependencies = [ "cfg-if", "once_cell", @@ -2394,9 +1847,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro" -version = "0.2.106" +version = "0.2.108" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "48cb0d2638f8baedbc542ed444afc0644a29166f1595371af4fecf8ce1e7eeb3" +checksum = "008b239d9c740232e71bd39e8ef6429d27097518b6b30bdf9086833bd5b6d608" dependencies = [ "quote", "wasm-bindgen-macro-support", @@ -2404,9 +1857,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro-support" -version = "0.2.106" +version = "0.2.108" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cefb59d5cd5f92d9dcf80e4683949f15ca4b511f4ac0a6e14d4e1ac60c6ecd40" +checksum = "5256bae2d58f54820e6490f9839c49780dff84c65aeab9e772f15d5f0e913a55" dependencies = [ "bumpalo", "proc-macro2", @@ -2417,9 +1870,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-shared" -version = "0.2.106" +version = "0.2.108" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cbc538057e648b67f72a982e708d485b2efa771e1ac05fec311f9f63e5800db4" +checksum = "1f01b580c9ac74c8d8f0c0e4afb04eeef2acf145458e52c03845ee9cd23e3d12" dependencies = [ "unicode-ident", ] @@ -2765,68 +2218,8 @@ dependencies = [ "memchr", ] -[[package]] -name = "wit-bindgen" -version = "0.46.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f17a85883d4e6d00e8a97c586de764dabcc06133f7f1d55dce5cdc070ad7fe59" - -[[package]] -name = "zerocopy" -version = "0.8.31" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fd74ec98b9250adb3ca554bdde269adf631549f51d8a8f8f0a10b50f1cb298c3" -dependencies = [ - "zerocopy-derive", -] - -[[package]] -name = "zerocopy-derive" -version = "0.8.31" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d8a8d209fdf45cf5138cbb5a506f6b52522a25afccc534d1475dad8e31105c6a" -dependencies = [ - "proc-macro2", - "quote", - "syn", -] - -[[package]] -name = "zlib-rs" -version = "0.5.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "40990edd51aae2c2b6907af74ffb635029d5788228222c4bb811e9351c0caad3" - [[package]] name = "zmij" -version = "1.0.10" +version = "1.0.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "30e0d8dffbae3d840f64bda38e28391faef673a7b5a6017840f2a106c8145868" - -[[package]] -name = "zstd" -version = "0.13.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e91ee311a569c327171651566e07972200e76fcfe2242a4fa446149a3881c08a" -dependencies = [ - "zstd-safe", -] - -[[package]] -name = "zstd-safe" -version = "7.2.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8f49c4d5f0abb602a93fb8736af2a4f4dd9512e36f7f570d66e65ff867ed3b9d" -dependencies = [ - "zstd-sys", -] - -[[package]] -name = "zstd-sys" -version = "2.0.16+zstd.1.5.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "91e19ebc2adc8f83e43039e79776e3fda8ca919132d68a1fed6a5faca2683748" -dependencies = [ - "cc", - "pkg-config", -] +checksum = "dfcd145825aace48cff44a8844de64bf75feec3080e0aa5cdbde72961ae51a65" diff --git a/sdgb-api/Cargo.toml b/sdgb-api/Cargo.toml index cd5a881..efa7ff5 100644 --- a/sdgb-api/Cargo.toml +++ b/sdgb-api/Cargo.toml @@ -6,12 +6,9 @@ edition = "2024" license = "GPL-3.0" [features] -default = ["compio", "bincode"] +default = ["compio"] compio = ["dep:compio"] tokio = ["dep:tokio"] -bincode = ["dep:bincode"] - -parquet = ['dep:parquet', 'dep:parquet_derive'] [dependencies] snafu = { workspace = true } @@ -45,7 +42,3 @@ flate2 = "1.1.2" cbc = { version = "0.1.2", features = ["alloc"] } aes = "0.8.4" cipher = { version = "0.4.4", features = ["block-padding"] } -bincode = { version = "2.0.1", optional = true } - -parquet = { workspace = true, optional = true } -parquet_derive = { workspace = true, optional = true } diff --git a/sdgb-api/src/lib.rs b/sdgb-api/src/lib.rs index fd471aa..40691e3 100644 --- a/sdgb-api/src/lib.rs +++ b/sdgb-api/src/lib.rs @@ -7,8 +7,5 @@ pub mod helper; mod error; pub use error::ApiError; -#[cfg(feature = "bincode")] -pub use bincode; - #[cfg(all(feature = "compio", feature = "tokio"))] compile_error!("you must not enable both `compio` and `tokio`"); diff --git a/sdgb-api/src/title/model/get_user_music_api/mod.rs b/sdgb-api/src/title/model/get_user_music_api/mod.rs index a10fa3e..7297540 100644 --- a/sdgb-api/src/title/model/get_user_music_api/mod.rs +++ b/sdgb-api/src/title/model/get_user_music_api/mod.rs @@ -16,7 +16,6 @@ pub struct GetUserMusicApi { } #[derive(Default, Debug, Clone, PartialEq, Serialize, Deserialize)] -#[cfg_attr(feature = "bincode", derive(bincode::Encode, bincode::Decode))] #[serde(rename_all = "camelCase")] pub struct GetUserMusicApiResp { pub user_id: u32, @@ -25,7 +24,6 @@ pub struct GetUserMusicApiResp { pub user_music_list: Vec, } -#[cfg_attr(feature = "bincode", derive(bincode::Encode, bincode::Decode))] #[derive(Default, Debug, Clone, PartialEq, Serialize, Deserialize)] #[serde(rename_all = "camelCase")] pub struct UserMusic { @@ -33,7 +31,6 @@ pub struct UserMusic { pub length: u32, } -#[cfg_attr(feature = "bincode", derive(bincode::Encode, bincode::Decode))] #[derive(Default, Debug, Clone, PartialEq, Serialize, Deserialize)] #[serde(rename_all = "camelCase")] pub struct UserMusicDetail { @@ -85,21 +82,6 @@ pub struct UserMusicDetail { pub ext_num2: i64, } -#[cfg_attr(feature = "parquet", derive(parquet_derive::ParquetRecordWriter))] -pub struct UserMusicDetailFlatten { - pub user_id: u32, - pub music_id: u32, - pub level: u8, - pub play_count: u32, - pub achievement: u32, - pub combo_status: u8, - pub sync_status: u8, - pub deluxscore_max: u16, - pub score_rank: u8, - pub ext_num1: u32, - pub ext_num2: u32, -} - impl Display for UserMusicDetail { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { if let Some(music_title) = query_music(self.music_id).map(|i| &i.name) { @@ -149,35 +131,3 @@ impl Display for UserMusicDetail { Ok(()) } } - -impl UserMusicDetailFlatten { - pub fn new( - user_id: u32, - UserMusicDetail { - music_id, - level, - play_count, - achievement, - combo_status, - sync_status, - deluxscore_max, - score_rank, - ext_num1, - ext_num2, - }: UserMusicDetail, - ) -> Self { - Self { - user_id, - music_id, - level: level as _, - sync_status: sync_status as _, - deluxscore_max: deluxscore_max as _, - score_rank: score_rank as _, - play_count: play_count as _, - achievement: achievement as _, - combo_status: combo_status as _, - ext_num1: ext_num1 as _, - ext_num2: ext_num2 as _, - } - } -} diff --git a/sdgb-api/src/title/model/get_user_preview_api/mod.rs b/sdgb-api/src/title/model/get_user_preview_api/mod.rs index 495a9ae..b389ab6 100644 --- a/sdgb-api/src/title/model/get_user_preview_api/mod.rs +++ b/sdgb-api/src/title/model/get_user_preview_api/mod.rs @@ -27,8 +27,6 @@ impl Default for GetUserPreviewApi { } #[derive(Debug, Clone, Serialize, Deserialize)] -#[cfg_attr(feature = "bincode", derive(bincode::Encode, bincode::Decode))] -#[cfg_attr(feature = "parquet", derive(parquet_derive::ParquetRecordWriter))] #[serde(rename_all = "camelCase")] pub struct GetUserPreviewApiResp { pub user_id: u32, diff --git a/sdgb-api/src/title/model/get_user_rating_api/mod.rs b/sdgb-api/src/title/model/get_user_rating_api/mod.rs index cb7fbd7..29f61fd 100644 --- a/sdgb-api/src/title/model/get_user_rating_api/mod.rs +++ b/sdgb-api/src/title/model/get_user_rating_api/mod.rs @@ -19,7 +19,6 @@ impl From for GetUserRatingApi { } } -#[cfg_attr(feature = "bincode", derive(bincode::Encode, bincode::Decode))] #[derive(Debug, Serialize, Deserialize)] #[serde(rename_all = "camelCase")] pub struct GetUserRatingApiResp { @@ -27,7 +26,6 @@ pub struct GetUserRatingApiResp { pub user_rating: UserRating, } -#[cfg_attr(feature = "bincode", derive(bincode::Encode, bincode::Decode))] #[derive(Default, Debug, Clone, PartialEq, Serialize, Deserialize)] #[serde(rename_all = "camelCase")] pub struct UserRating { @@ -44,7 +42,6 @@ pub struct UserRating { pub udemae: Udemae, } -#[cfg_attr(feature = "bincode", derive(bincode::Encode, bincode::Decode))] #[derive(Default, Debug, Clone, PartialEq, Serialize, Deserialize)] #[serde(rename_all = "camelCase")] pub struct MusicRating { @@ -66,18 +63,6 @@ pub struct MusicRating { pub achievement: i32, } -#[cfg_attr(feature = "parquet", derive(parquet_derive::ParquetRecordWriter))] -#[derive(Default, Debug, Clone, PartialEq)] -pub struct MusicRatingFlatten { - pub user_id: u32, - pub music_id: u32, - pub level: u32, - pub rom_version: i64, - pub achievement: i32, - pub dx_rating: u32, -} - -#[cfg_attr(feature = "bincode", derive(bincode::Encode, bincode::Decode))] #[derive(Default, Debug, Clone, PartialEq, Serialize, Deserialize)] #[serde(rename_all = "camelCase")] pub struct Udemae { diff --git a/sdgb-api/src/title/model/get_user_region_api/mod.rs b/sdgb-api/src/title/model/get_user_region_api/mod.rs index 5656c5e..30d2ac0 100644 --- a/sdgb-api/src/title/model/get_user_region_api/mod.rs +++ b/sdgb-api/src/title/model/get_user_region_api/mod.rs @@ -12,44 +12,6 @@ impl From for GetUserRegionApi { } } -impl From for Vec { - fn from( - GetUserRegionApiResp { - user_id, - user_region_list, - .. - }: GetUserRegionApiResp, - ) -> Self { - user_region_list - .into_iter() - .map( - |UserRegion { - region_id, - play_count, - created, - }| { - UserRegionFlatten { - user_id, - region_id, - play_count, - created, - } - }, - ) - .collect() - } -} - -#[derive(Default, Debug, Clone, PartialEq)] -#[cfg_attr(feature = "parquet", derive(parquet_derive::ParquetRecordWriter))] -pub struct UserRegionFlatten { - pub user_id: u32, - pub region_id: u32, - pub play_count: i64, - pub created: String, -} - -#[cfg_attr(feature = "bincode", derive(bincode::Encode, bincode::Decode))] #[derive(Default, Debug, Clone, PartialEq, Serialize, Deserialize)] #[serde(rename_all = "camelCase")] pub struct GetUserRegionApiResp { @@ -58,7 +20,6 @@ pub struct GetUserRegionApiResp { pub user_region_list: Vec, } -#[cfg_attr(feature = "bincode", derive(bincode::Encode, bincode::Decode))] #[derive(Default, Debug, Clone, PartialEq, Serialize, Deserialize)] #[serde(rename_all = "camelCase")] pub struct UserRegion { diff --git a/sdgb-api/src/title/model/mod.rs b/sdgb-api/src/title/model/mod.rs index c009a9e..079d20a 100644 --- a/sdgb-api/src/title/model/mod.rs +++ b/sdgb-api/src/title/model/mod.rs @@ -18,20 +18,15 @@ pub use get_user_rating_api::{ GetUserRatingApi, GetUserRatingApiResp, // api MusicRating, - MusicRatingFlatten, Udemae, UserRating, }; mod get_user_music_api; -pub use get_user_music_api::{ - GetUserMusicApi, GetUserMusicApiResp, UserMusic, UserMusicDetail, UserMusicDetailFlatten, -}; +pub use get_user_music_api::{GetUserMusicApi, GetUserMusicApiResp, UserMusic, UserMusicDetail}; mod get_user_region_api; -pub use get_user_region_api::{ - GetUserRegionApi, GetUserRegionApiResp, UserRegion, UserRegionFlatten, -}; +pub use get_user_region_api::{GetUserRegionApi, GetUserRegionApiResp, UserRegion}; mod dxrating; pub use dxrating::{ diff --git a/sdgb-api/src/title/model/user_logout_api/mod.rs b/sdgb-api/src/title/model/user_logout_api/mod.rs index d3c176c..b2ff062 100644 --- a/sdgb-api/src/title/model/user_logout_api/mod.rs +++ b/sdgb-api/src/title/model/user_logout_api/mod.rs @@ -32,7 +32,7 @@ impl Default for UserLogoutApi { client_id: "A63E01C2805".into(), type_: 1, access_code: "", - + login_date_time: 0, } } diff --git a/sdgb-cli/Cargo.toml b/sdgb-cli/Cargo.toml index dc60712..7fb342e 100644 --- a/sdgb-cli/Cargo.toml +++ b/sdgb-cli/Cargo.toml @@ -6,21 +6,13 @@ authors = ["mokurin000"] description = "CLI tool for SDGB protocol" [features] -default = ["compio", "fetchall"] +default = ["compio"] compio = ["dep:compio", "sdgb-api/compio"] tokio = ["dep:tokio", "sdgb-api/tokio"] -fetchall = [ - "dep:redb", - "dep:futures-util", - "dep:parquet", - "dep:music-db", - "sdgb-api/parquet", -] - [dependencies] -sdgb-api = { workspace = true, features = ["bincode"] } +sdgb-api = { workspace = true, features = [] } music-db = { workspace = true, optional = true } # (de)serialization @@ -32,10 +24,6 @@ strum = { workspace = true } spdlog-rs = { workspace = true } snafu = { workspace = true } - -# kv database -redb = { workspace = true, optional = true } - # async runtime tokio = { workspace = true, features = ["macros"], optional = true } compio = { workspace = true, features = ["macros"], optional = true } @@ -49,7 +37,5 @@ ctrlc = { version = "3.4.7", features = ["termination"] } # magic macro crabtime = { workspace = true } -parquet = { workspace = true, optional = true } - [build-dependencies] version_check = "0.9.5" diff --git a/sdgb-cli/src/cache/mod.rs b/sdgb-cli/src/cache/mod.rs deleted file mode 100644 index db70296..0000000 --- a/sdgb-cli/src/cache/mod.rs +++ /dev/null @@ -1,64 +0,0 @@ -use std::sync::LazyLock; - -use redb::{ReadTransaction, ReadableDatabase as _, Table, TableDefinition, WriteTransaction}; - -static DATABASE: LazyLock = LazyLock::new(|| { - let mut db = redb::Database::builder() - .create("players.redb") - .expect("failed to open database"); - _ = db.compact(); - db -}); - -pub fn write_txn() -> Result { - Ok(DATABASE.begin_write()?) -} - -pub fn read_txn() -> Result { - Ok(DATABASE.begin_read()?) -} - -pub fn open_table<'a>( - write: &'a WriteTransaction, - definition: TableDefinition<'_, u32, Vec>, -) -> Result>, redb::Error> { - Ok(write.open_table(definition)?) -} - -pub fn open_table_ro( - read: &ReadTransaction, - definition: TableDefinition<'_, u32, Vec>, -) -> Result>, redb::Error> { - Ok(read.open_table(definition)?) -} - -#[crabtime::function] -fn table_definitions_impl(tables: Vec) { - let mut defs: Vec = Vec::new(); - - for table in tables { - let definition = table.to_uppercase(); - let table_name = format!("\"{table}\""); - - crabtime::output!( - pub const {{definition}}: TableDefinition<'_, u32, Vec> = redb::TableDefinition::new({{table_name}}); - ); - - defs.push(format!("write_txn.open_table({definition})?;")); - } - - let init_statements = defs.join("\n"); - - crabtime::output!( - pub fn init_db() -> Result<(), redb::Error> { - let write_txn = DATABASE.begin_write()?; - { - { init_statements } - } - write_txn.commit()?; - Ok(()) - } - ); -} - -table_definitions_impl!(["players", "b50", "records", "regions"]); diff --git a/sdgb-cli/src/commands.rs b/sdgb-cli/src/commands.rs index c8c105b..6c00426 100644 --- a/sdgb-cli/src/commands.rs +++ b/sdgb-cli/src/commands.rs @@ -85,55 +85,6 @@ pub enum Commands { token: Option, }, - /// Scrape all user, read possible id from stdin - #[cfg(feature = "fetchall")] - ListAllUser { - #[arg(short, long, default_value_t = 5)] - concurrency: usize, - }, - #[cfg(feature = "fetchall")] - /// Scrape B50 data - ScrapeAllB50 { - #[arg(short, long, default_value_t = 5)] - concurrency: usize, - - #[arg(long, default_value_t = 1000)] - min_rating: i64, - #[arg(long, default_value_t = 16500)] - max_rating: i64, - }, - /// Scrape Region data - #[cfg(feature = "fetchall")] - ScrapeAllRegion { - #[arg(short, long, default_value_t = 5)] - concurrency: usize, - - #[arg(long, default_value_t = 1000)] - min_rating: i64, - #[arg(long, default_value_t = 16500)] - max_rating: i64, - }, - /// Scrape all player record - #[cfg(feature = "fetchall")] - ScrapeAllRecord { - #[arg(short, long, default_value_t = 5)] - concurrency: usize, - - #[arg(long, default_value_t = 10000)] - min_rating: i64, - #[arg(long, default_value_t = 16400)] - max_rating: i64, - }, - - #[cfg(feature = "fetchall")] - ListAllUserDump {}, - #[cfg(feature = "fetchall")] - ScrapeAllB50Dump {}, - #[cfg(feature = "fetchall")] - ScrapeAllRegionDump {}, - #[cfg(feature = "fetchall")] - ScrapeAllRecordDump {}, - Logout { #[arg(short, long)] user_id: u32, diff --git a/sdgb-cli/src/main.rs b/sdgb-cli/src/main.rs index 093db23..8693c19 100644 --- a/sdgb-cli/src/main.rs +++ b/sdgb-cli/src/main.rs @@ -32,8 +32,6 @@ use crate::{ utils::{human_readable_display, json_display, login_action}, }; -#[cfg(feature = "fetchall")] -mod cache; mod commands; mod utils; @@ -221,216 +219,6 @@ async fn main() -> Result<(), Box> { println!("{}", String::from_utf8_lossy(&resp)); } - #[cfg(feature = "fetchall")] - Commands::ListAllUser { concurrency } => { - use crate::{cache::PLAYERS, utils::helpers::cached_concurrent_fetch}; - use sdgb_api::title::methods::GetUserPreviewApiExt; - use std::io::BufRead as _; - - let mut user_ids = Vec::new(); - { - let mut stdin = std::io::stdin().lock(); - let mut buf = String::new(); - - while stdin.read_line(&mut buf).is_ok_and(|size| size != 0) { - if buf.is_empty() { - continue; - } - - let user_id: u32 = buf.trim().parse()?; - buf.clear(); - user_ids.push(user_id); - } - } - - cached_concurrent_fetch::( - user_ids, - &client, - concurrency, - PLAYERS, - ) - .await?; - } - - #[cfg(feature = "fetchall")] - Commands::ScrapeAllRecord { - concurrency, - min_rating, - max_rating, - } => { - use crate::{ - cache::{PLAYERS, RECORDS}, - utils::helpers::{cached_concurrent_fetch_userfn, read_cache}, - }; - - let players: Vec = read_cache(PLAYERS)?; - cached_concurrent_fetch_userfn( - players - .iter() - .filter(|p| p.player_rating >= min_rating && p.player_rating <= max_rating) - .map(|p| p.user_id) - .collect::>(), - &client, - concurrency, - RECORDS, - get_user_all_music, - ) - .await?; - } - #[cfg(feature = "fetchall")] - Commands::ScrapeAllB50 { - concurrency, - min_rating, - max_rating, - } => { - use sdgb_api::title::methods::GetUserRatingApiExt; - - use crate::{ - cache::{B50, PLAYERS}, - utils::helpers::{cached_concurrent_fetch, read_cache}, - }; - - let players: Vec = read_cache(PLAYERS)?; - cached_concurrent_fetch::( - players - .iter() - .filter(|p| p.player_rating >= min_rating && p.player_rating <= max_rating) - .map(|p| p.user_id) - .collect::>(), - &client, - concurrency, - B50, - ) - .await?; - } - #[cfg(feature = "fetchall")] - Commands::ScrapeAllRegion { - concurrency, - min_rating, - max_rating, - } => { - use sdgb_api::title::methods::GetUserRegionApiExt; - - use crate::{ - cache::{PLAYERS, REGIONS}, - utils::helpers::{cached_concurrent_fetch, read_cache}, - }; - - let players: Vec = read_cache(PLAYERS)?; - cached_concurrent_fetch::( - players - .iter() - .filter(|p| p.player_rating >= min_rating && p.player_rating <= max_rating) - .map(|p| p.user_id) - .collect::>(), - &client, - concurrency, - REGIONS, - ) - .await?; - } - - #[cfg(feature = "fetchall")] - Commands::ListAllUserDump {} => { - use crate::{ - cache::PLAYERS, - utils::helpers::{dump_parquet, read_cache}, - }; - - let players: Vec = read_cache(PLAYERS)?; - dump_parquet(players, "players.parquet")?; - } - #[cfg(feature = "fetchall")] - Commands::ScrapeAllRegionDump {} => { - use crate::{ - cache::REGIONS, - utils::helpers::{dump_parquet, read_cache}, - }; - use sdgb_api::title::model::{GetUserRegionApiResp, UserRegionFlatten}; - - let regions: Vec = read_cache(REGIONS)?; - let regions_flat = regions - .into_iter() - .map(Vec::::from) - .flatten() - .collect::>(); - dump_parquet(regions_flat, "regions.parquet")?; - } - #[cfg(feature = "fetchall")] - Commands::ScrapeAllRecordDump {} => { - use crate::{ - cache::RECORDS, - utils::helpers::{dump_parquet, read_cache}, - }; - use sdgb_api::title::model::GetUserMusicApiResp; - use sdgb_api::title::model::UserMusicDetailFlatten; - - let records: Vec = read_cache(RECORDS)?; - dump_parquet( - records - .into_iter() - .map(|resp| { - resp.user_music_list - .into_iter() - .map(|music| music.user_music_detail_list) - .flatten() - .map(move |detail| UserMusicDetailFlatten::new(resp.user_id, detail)) - }) - .flatten() - .collect::>(), - "records.parquet", - )?; - } - #[cfg(feature = "fetchall")] - Commands::ScrapeAllB50Dump {} => { - use sdgb_api::title::model::{MusicRating, MusicRatingFlatten}; - - use crate::{ - cache::B50, - utils::helpers::{dump_parquet, read_cache}, - }; - - let records: Vec = read_cache(B50)?; - dump_parquet::( - records - .into_iter() - .map( - |GetUserRatingApiResp { - user_id, - user_rating, - }| { - user_rating - .rating_list - .into_iter() - .chain(user_rating.next_rating_list) - .filter_map( - move |MusicRating { - music_id, - level, - rom_version, - achievement, - }| { - let (_rank, dx_rating) = - music_db::query_music_level(music_id, level)? - .dx_rating(achievement); - Some(MusicRatingFlatten { - user_id, - music_id, - level, - rom_version, - achievement, - dx_rating, - }) - }, - ) - }, - ) - .flatten() - .collect::>(), - "b50.parquet", - )?; - } - Commands::Userdata { user_id, skip_login, diff --git a/sdgb-cli/src/utils/helpers/mod.rs b/sdgb-cli/src/utils/helpers/mod.rs deleted file mode 100644 index 1b52bd2..0000000 --- a/sdgb-cli/src/utils/helpers/mod.rs +++ /dev/null @@ -1,183 +0,0 @@ -use std::sync::Arc; -use std::{fs::OpenOptions, io::BufWriter}; -use std::{path::Path, sync::atomic::Ordering}; - -use futures_util::StreamExt; -use nyquest_preset::nyquest::AsyncClient; - -use parquet::basic::BrotliLevel; -use parquet::file::properties::WriterProperties; -use parquet::file::writer::SerializedFileWriter; -use parquet::record::RecordWriter; -use redb::ReadableTable; -use redb::TableDefinition; -use spdlog::{error, info}; - -use sdgb_api::title::MaiVersionExt; -use sdgb_api::title::{Sdgb1_53, methods::APIExt}; -use sdgb_api::{ApiError, bincode}; - -use bincode::{BorrowDecode, Encode, borrow_decode_from_slice}; - -use crate::{EARLY_QUIT, cache}; - -#[allow(unused)] -pub fn read_cache_keys( - definition: TableDefinition<'_, u32, Vec>, -) -> Result, Box> { - let txn = cache::read_txn()?; - let table = cache::open_table_ro(&txn, definition)?; - - Ok(table - .iter()? - .flatten() - .map(|(value, _)| value.value()) - .collect::>()) -} - -pub fn read_cache( - definition: TableDefinition<'_, u32, Vec>, -) -> Result, Box> -where - D: for<'d> BorrowDecode<'d, ()>, -{ - let txn = cache::read_txn()?; - let table = cache::open_table_ro(&txn, definition)?; - - let config = - bincode::config::Configuration::::default().with_no_limit(); - - Ok(table - .iter()? - .flatten() - .map(|d| borrow_decode_from_slice(&d.1.value(), config)) - .flatten() - .map(|(value, _)| value) - .collect::>()) -} - -pub fn dump_parquet( - data: impl Into>, - output_path: impl AsRef, -) -> Result<(), Box> -where - for<'a> &'a [D]: RecordWriter, -{ - let data = data.into(); - let file = OpenOptions::new() - .create(true) - .truncate(true) - .write(true) - .open(output_path)?; - - #[cfg(file_lock_ready)] - file.try_lock()?; - - let writer = BufWriter::new(file); - let schema = data.as_slice().schema()?; - let props = Arc::new( - WriterProperties::builder() - .set_compression(parquet::basic::Compression::BROTLI(BrotliLevel::try_new( - 6, - )?)) - .build(), - ); - - let mut writer = SerializedFileWriter::new(writer, schema, props).unwrap(); - let mut row_group = writer.next_row_group().unwrap(); - - data.as_slice().write_to_row_group(&mut row_group)?; - row_group.close()?; - - writer.close().unwrap(); - info!("dumped {} records", data.len()); - - Ok(()) -} - -pub async fn cached_concurrent_fetch( - user_ids: impl Into>, - client: &AsyncClient, - concurrency: usize, - definition: TableDefinition<'_, u32, Vec>, -) -> Result<(), Box> -where - A::Payload: From, - A::Response: Encode + for<'a> BorrowDecode<'a, ()>, -{ - cached_concurrent_fetch_userfn( - user_ids, - client, - concurrency, - definition, - async |client, user_id| { - Sdgb1_53::request_ext::(client, A::Payload::from(user_id), user_id).await - }, - ) - .await -} - -pub async fn cached_concurrent_fetch_userfn( - user_ids: impl Into>, - client: &AsyncClient, - concurrency: usize, - definition: TableDefinition<'_, u32, Vec>, - scrape: impl AsyncFn(&AsyncClient, u32) -> Result, -) -> Result<(), Box> -where - R: Encode + for<'a> BorrowDecode<'a, ()>, -{ - let _ = cache::init_db(); - - let user_ids = user_ids.into(); - let read = cache::read_txn()?; - let write = cache::write_txn()?; - let config = sdgb_api::bincode::config::Configuration::< - sdgb_api::bincode::config::LittleEndian, - >::default() - .with_no_limit(); - - info!("number of user_id: {}", user_ids.len()); - - let collect = futures_util::stream::iter(user_ids) - .map(async |user_id| { - { - let cache_table = cache::open_table_ro(&read, definition)?; - let data = cache_table.get(user_id)?; - if data.is_some() { - return Ok(()); - } - } - - if EARLY_QUIT.load(Ordering::Relaxed) { - return Err("early skip due to ctrl-c")?; - } - - let resp = scrape(&client, user_id).await; - match &resp { - Ok(resp) => { - use sdgb_api::bincode::encode_to_vec; - - if let Ok(mut table) = cache::open_table(&write, definition) - && let Ok(encoded) = encode_to_vec(resp, config) - { - info!("encode length for {user_id}: {}", encoded.len()); - _ = table.insert(user_id, encoded); - } - } - Err(sdgb_api::ApiError::JSON { .. }) => {} - Err(e) => { - error!("fetch failed: {e}"); - } - } - - Result::<_, Box>::Ok(()) - }) - .buffer_unordered(concurrency) // slower to avoid being banned - .collect::>() - .await; - drop(collect); - - let _ = write.commit(); - Ok(()) -} diff --git a/sdgb-cli/src/utils/mod.rs b/sdgb-cli/src/utils/mod.rs index 2319ab9..7c6c9b8 100644 --- a/sdgb-cli/src/utils/mod.rs +++ b/sdgb-cli/src/utils/mod.rs @@ -65,6 +65,3 @@ pub fn human_readable_display( Ok(()) } - -#[cfg(feature = "fetchall")] -pub mod helpers;