perf: migrate to orjson for performance

This commit is contained in:
mokurin000
2025-08-03 10:32:29 +08:00
parent 0e40282e87
commit a60e65e110
3 changed files with 21 additions and 9 deletions

View File

@@ -1,4 +1,4 @@
import json
import orjson as json
import hashlib
from typing import Callable
from datetime import datetime
@@ -7,7 +7,7 @@ from decimal import Decimal, getcontext
SALT = b"Lt2N5xgjJOqRsT5qVt7wWYw6SqOPZDI7"
with open("musicDB.json", "r", encoding="utf-8") as f:
music_db = json.load(f)
music_db = json.loads(f.read())
music_db = {entry["id"]: entry for entry in music_db}
@@ -169,19 +169,17 @@ def process(
output_file: str,
):
record_time()
with open(input_file, "r", encoding="utf-8") as f:
data = json.load(f)
with open(input_file, "rb") as f:
data = json.loads(f.read())
print(f"loaded, cost {record_time():.2f}s")
record_time()
for entry in data:
salted_hash_userid(entry)
clean_fields(entry)
print(f"processed, cost {record_time():.2f}s")
record_time()
with open(output_file, "w", encoding="utf-8") as f:
json.dump(data, f, ensure_ascii=False)
with open(output_file, "wb") as f:
f.write(json.dumps(data))
print(f"written out, cost {record_time():.2f}s")
return data