Files
spark/tools/py/dump_log.py
T

273 lines
8.1 KiB
Python

import argparse
import os
import re
import sqlite3
from io import TextIOWrapper
from typing import Iterable, Iterator, Optional
level_ordinals = {
"Diag": 1,
"Info": 2,
"Warn": 3,
"Error": 4,
"Fatal": 5,
}
log_msg_start_re = re.compile('^\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}.\d{6}\|')
def _get_arg_parser():
parser = argparse.ArgumentParser(
description="Builds the Spark server (A.K.A sparkd, the daemon)")
parser.add_argument("--pattern", help="Regex pattern to filter logs")
parser.add_argument("--file", help="The log file to dump")
parser.add_argument(
"--category",
help=
"The category (subsystem) to grep (e.g., Gen, Http, Md, OrderMan etc.)"
)
parser.add_argument("--level",
help="The minimum log level to include",
choices=["Diag", "Info", "Warn", "Error", "Fatal"])
parser.add_argument(
"--source-file",
help="Source file(s) whose logs to include (includes all by default)")
parser.add_argument("--line-num",
help="The line number in the source file(s) to grep")
parser.add_argument("--count",
help="Count the number of log messages",
action="store_true",
dest="count")
parser.add_argument(
"--count-only",
help="Count the number of log messages without printing them",
action="store_true",
dest="count_only")
parser.add_argument(
"--count-all",
help="Count the number of log messages in each category",
action="store_true",
dest="count_all")
parser.add_argument(
"--dump-db",
help="Dump logs to a sqlite3 db (named <--file>.sqlite)",
action="store_true")
parser.add_argument("--dump-db-to",
help="Dump logs to a specific sqlite3 db",
metavar="SQLITE_FILE")
parser.add_argument("--dump-db-table-to",
help="Dump logs to a specific table in a sqlite3 db",
metavar="SQLITE_TABLE")
parser.add_argument("-y, --yes",
help="Overwrite existing db without asking",
action="store_true",
dest="yes")
parser.set_defaults(
pattern=None,
file=None,
level=0,
category=None,
count=False,
count_only=False,
dump_db=False,
dump_db_to=None,
)
return parser
class ChunkIterator:
def __init__(self, source: Iterator, chunk_size: int):
self.source = source
self.chunk_size = chunk_size
def __iter__(self):
return self
def __next__(self):
chunk = []
try:
while len(chunk) < self.chunk_size:
chunk.append(self.source.__next__())
except StopIteration:
pass
if chunk:
return chunk
raise StopIteration()
class LogRecord:
def __init__(self, msg: str):
self.full_msg = msg
self.parts = []
field_start = 0
for _ in range(0, 9):
field_end = msg.find("|", field_start)
field = msg[field_start:field_end].strip()
field_start = field_end + 1
self.parts.append(field)
self.parts.append(msg[field_end + 1:].rstrip())
@property
def ts(self):
return self.parts[0]
@property
def ts_nanos(self):
return self.parts[1]
@property
def category(self):
return self.parts[2]
@property
def level(self):
return self.parts[3]
@property
def thread_id(self):
return self.parts[4]
@property
def thread_name(self):
return self.parts[5]
@property
def pid(self):
return self.parts[6]
@property
def src_file(self):
i = self.parts[7].find(":")
return self.parts[7][:i]
@property
def line_num(self):
i = self.parts[7].find(":")
return self.parts[7][i + 1:]
@property
def deployment_type(self):
return self.parts[8]
@property
def msg(self):
return self.parts[9]
def __str__(self):
return self.full_msg.rstrip()
class LogRecordIterator:
def __init__(self, log_file: TextIOWrapper):
self.log_file = log_file
self.count = 0
self.curr_log = None
self.next_ln = None
self.next_log_start = None
def __iter__(self):
return self
def _read_until_next_log(self):
curr_log = self.next_log_start
next_ln = self.log_file.readline()
while next_ln and not log_msg_start_re.match(next_ln):
curr_log += next_ln
next_ln = self.log_file.readline()
self.next_log_start = next_ln
return LogRecord(curr_log)
def __next__(self):
if self.next_log_start is None:
self.next_log_start = self.log_file.readline()
if not self.next_log_start:
self.log_file.close()
raise StopIteration
return self._read_until_next_log()
class LogFilterCriteria:
def __init__(
self,
level: Optional[int | str] = None,
category: Optional[str] = None,
source_file: Optional[str] = None,
line_num: Optional[int] = None,
pattern: Optional[str] = None,
):
self.level = level_ordinals[level] if isinstance(level, str) else level
self.category = category
self.source_file = source_file
self.line_num = line_num
self.pattern_re = re.compile(pattern) if pattern else None
def include(self, log_rec: LogRecord):
if level_ordinals[log_rec.level] < self.level:
return False
if self.source_file:
if self.source_file != log_rec.src_file:
return False
if self.line_num and self.line_num != log_rec.line_num:
return False
if self.category and self.category != log_rec.category:
return False
if (self.pattern_re and not self.pattern_re.match(log_rec.msg)):
return False
return True
def dump_to_db(log_recs: Iterable[LogRecord], db_out: str, table_name: str):
con = sqlite3.connect(db_out)
cur = con.cursor()
columns = [
"ts TEXT", "ts_nanos INTEGER", "category TEXT", "level INTEGER",
"thread_id INTEGER", "thread_name TEXT", "pid INTEGER",
"src_file TEXT", "line_num TEXT", "deployment_type TEXT", "msg TEXT"
]
columns_expr = ", ".join(columns)
cur.execute(f"CREATE TABLE IF NOT EXISTS {table_name}({columns_expr})")
insert_expr = f"INSERT INTO {table_name} VALUES ({', '.join(['?' for _ in columns])})"
chunks = ChunkIterator([(lr.ts, lr.ts_nanos, lr.category, lr.level,
lr.thread_id, lr.thread_name, lr.pid, lr.src_file,
lr.line_num, lr.deployment_type, lr.msg.rstrip())
for lr in log_recs].__iter__(), 256)
for chunk in chunks:
cur.executemany(insert_expr, chunk)
print(f"Commiting sqlite transactions")
con.commit()
def _main():
parser = _get_arg_parser()
args = parser.parse_args()
dump_db = args.dump_db or args.dump_db_to or args.dump_db_table_to
log_filter = LogFilterCriteria(
level=args.level,
category=args.category,
source_file=args.source_file,
line_num=args.line_num,
pattern=args.pattern,
)
with open(args.file) as f:
log_recs = filter(log_filter.include, LogRecordIterator(f))
if dump_db:
db_file = args.dump_db_to or f"{args.file}.sqlite"
if os.path.exists(db_file):
if not args.yes:
raise ValueError(f"'{db_file}' already exists")
os.remove(db_file)
db_table = args.dump_db_table_to or "log"
dump_to_db(log_recs, db_file, db_table)
else:
n = 0
for lr in log_recs:
if not args.count_only:
print(lr)
n += 1
if args.count or args.count_only:
print(n)
if __name__ == "__main__":
_main()