perf: migrate to orjson for performance
This commit is contained in:
@@ -1,4 +1,4 @@
|
||||
import json
|
||||
import orjson as json
|
||||
import hashlib
|
||||
from typing import Callable
|
||||
from datetime import datetime
|
||||
@@ -7,7 +7,7 @@ from decimal import Decimal, getcontext
|
||||
SALT = b"Lt2N5xgjJOqRsT5qVt7wWYw6SqOPZDI7"
|
||||
|
||||
with open("musicDB.json", "r", encoding="utf-8") as f:
|
||||
music_db = json.load(f)
|
||||
music_db = json.loads(f.read())
|
||||
|
||||
music_db = {entry["id"]: entry for entry in music_db}
|
||||
|
||||
@@ -169,19 +169,17 @@ def process(
|
||||
output_file: str,
|
||||
):
|
||||
record_time()
|
||||
with open(input_file, "r", encoding="utf-8") as f:
|
||||
data = json.load(f)
|
||||
with open(input_file, "rb") as f:
|
||||
data = json.loads(f.read())
|
||||
print(f"loaded, cost {record_time():.2f}s")
|
||||
|
||||
record_time()
|
||||
for entry in data:
|
||||
salted_hash_userid(entry)
|
||||
clean_fields(entry)
|
||||
print(f"processed, cost {record_time():.2f}s")
|
||||
|
||||
record_time()
|
||||
with open(output_file, "w", encoding="utf-8") as f:
|
||||
json.dump(data, f, ensure_ascii=False)
|
||||
with open(output_file, "wb") as f:
|
||||
f.write(json.dumps(data))
|
||||
print(f"written out, cost {record_time():.2f}s")
|
||||
|
||||
return data
|
||||
|
||||
Reference in New Issue
Block a user