Compare commits

...

1 Commits

Author SHA1 Message Date
Willi Ballenthin
f83fb74502 helpers: move msgspect imports within functions
so that some environments can avoid installing msgspec (like PyPy)
2024-11-07 12:16:19 +00:00

View File

@@ -19,7 +19,6 @@ from pathlib import Path
from zipfile import ZipFile from zipfile import ZipFile
from datetime import datetime from datetime import datetime
import msgspec.json
from rich.console import Console from rich.console import Console
from rich.progress import ( from rich.progress import (
Task, Task,
@@ -154,6 +153,7 @@ def stdout_redirector(stream):
def load_json_from_path(json_path: Path): def load_json_from_path(json_path: Path):
import msgspec.json
with gzip.open(json_path, "r") as compressed_report: with gzip.open(json_path, "r") as compressed_report:
try: try:
report_json = compressed_report.read() report_json = compressed_report.read()
@@ -165,6 +165,7 @@ def load_json_from_path(json_path: Path):
def decode_json_lines(fd: Union[BinaryIO, gzip.GzipFile]): def decode_json_lines(fd: Union[BinaryIO, gzip.GzipFile]):
import msgspec.json
for line in fd: for line in fd:
try: try:
line_s = line.strip().decode() line_s = line.strip().decode()
@@ -186,6 +187,7 @@ def load_jsonl_from_path(jsonl_path: Path) -> Iterator[Dict]:
def load_one_jsonl_from_path(jsonl_path: Path): def load_one_jsonl_from_path(jsonl_path: Path):
# this loads one json line to avoid the overhead of loading the entire file # this loads one json line to avoid the overhead of loading the entire file
import msgspec.json
try: try:
with gzip.open(jsonl_path, "rb") as f: with gzip.open(jsonl_path, "rb") as f:
line = next(iter(f)) line = next(iter(f))