perf: drop json output support

sdgb-api/src/title/model/get_user_rating_api/mod.rs

@@ -66,6 +66,16 @@ pub struct MusicRating {
     pub achievement: i32,
 }
 
+#[cfg_attr(feature = "parquet", derive(parquet_derive::ParquetRecordWriter))]
+#[derive(Default, Debug, Clone, PartialEq)]
+pub struct MusicRatingFlatten {
+    pub user_id: u32,
+    pub music_id: u32,
+    pub level: u32,
+    pub rom_version: i64,
+    pub achievement: i32,
+}
+
 #[cfg_attr(feature = "bincode", derive(bincode::Encode, bincode::Decode))]
 #[derive(Default, Debug, Clone, PartialEq, Serialize, Deserialize)]
 #[serde(rename_all = "camelCase")]

sdgb-api/src/title/model/mod.rs

@@ -18,6 +18,7 @@ pub use get_user_rating_api::{
     GetUserRatingApi,
     GetUserRatingApiResp, // api
     MusicRating,
+    MusicRatingFlatten,
     Udemae,
     UserRating,
 };

sdgb-cli/src/main.rs

@@ -363,9 +363,48 @@ async fn main() -> Result<(), Box<dyn snafu::Error>> {
         }
         #[cfg(feature = "fetchall")]
         Commands::ScrapeAllB50Dump {} => {
-            use crate::{cache::B50, utils::helpers::dump_json};
+            use sdgb_api::title::model::{MusicRating, MusicRatingFlatten};
 
-            dump_json::<GetUserRatingApiResp>("b50.json", B50)?;
+            use crate::{
+                cache::B50,
+                utils::helpers::{dump_parquet, read_cache},
+            };
+
+            let records: Vec<GetUserRatingApiResp> = read_cache(B50)?;
+            dump_parquet::<MusicRatingFlatten>(
+                records
+                    .into_iter()
+                    .map(
+                        |GetUserRatingApiResp {
+                             user_id,
+                             user_rating,
+                         }| {
+                            user_rating
+                                .rating_list
+                                .into_iter()
+                                .chain(user_rating.next_rating_list)
+                                .map(
+                                    move |MusicRating {
+                                              music_id,
+                                              level,
+                                              rom_version,
+                                              achievement,
+                                          }| {
+                                        MusicRatingFlatten {
+                                            user_id,
+                                            music_id,
+                                            level,
+                                            rom_version,
+                                            achievement,
+                                        }
+                                    },
+                                )
+                        },
+                    )
+                    .flatten()
+                    .collect::<Vec<_>>(),
+                "b50.parquet",
+            )?;
         }
 
         Commands::Userdata {

sdgb-cli/src/utils/helpers/mod.rs

@@ -11,7 +11,6 @@ use parquet::file::writer::SerializedFileWriter;
 use parquet::record::RecordWriter;
 use redb::ReadableTable;
 use redb::TableDefinition;
-use serde::Serialize;
 use spdlog::{error, info};
 
 use sdgb_api::title::MaiVersionExt;
@@ -96,30 +95,6 @@ where
     Ok(())
 }
 
-pub fn dump_json<D>(
-    output_path: impl AsRef<Path>,
-    definition: TableDefinition<'_, u32, Vec<u8>>,
-) -> Result<(), Box<dyn snafu::Error>>
-where
-    D: for<'d> BorrowDecode<'d, ()> + Serialize,
-{
-    let file = OpenOptions::new()
-        .create(true)
-        .truncate(true)
-        .write(true)
-        .open(output_path)?;
-
-    #[cfg(file_lock_ready)]
-    file.try_lock()?;
-
-    let data = read_cache::<D>(definition)?;
-    let writer = BufWriter::new(file);
-    serde_json::to_writer_pretty(writer, &data)?;
-    info!("dumped {} records", data.len());
-
-    Ok(())
-}
-
 pub async fn cached_concurrent_fetch<A: APIExt>(
     user_ids: impl Into<Vec<u32>>,
     client: &AsyncClient,