273 lines
8.1 KiB
Python
273 lines
8.1 KiB
Python
import argparse
|
|
import os
|
|
import re
|
|
import sqlite3
|
|
|
|
from io import TextIOWrapper
|
|
from typing import Iterable, Iterator, Optional
|
|
|
|
level_ordinals = {
|
|
"Diag": 1,
|
|
"Info": 2,
|
|
"Warn": 3,
|
|
"Error": 4,
|
|
"Fatal": 5,
|
|
}
|
|
log_msg_start_re = re.compile('^\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}.\d{6}\|')
|
|
|
|
|
|
def _get_arg_parser():
|
|
parser = argparse.ArgumentParser(
|
|
description="Builds the Spark server (A.K.A sparkd, the daemon)")
|
|
parser.add_argument("--pattern", help="Regex pattern to filter logs")
|
|
parser.add_argument("--file", help="The log file to dump")
|
|
parser.add_argument(
|
|
"--category",
|
|
help=
|
|
"The category (subsystem) to grep (e.g., Gen, Http, Md, OrderMan etc.)"
|
|
)
|
|
parser.add_argument("--level",
|
|
help="The minimum log level to include",
|
|
choices=["Diag", "Info", "Warn", "Error", "Fatal"])
|
|
parser.add_argument(
|
|
"--source-file",
|
|
help="Source file(s) whose logs to include (includes all by default)")
|
|
parser.add_argument("--line-num",
|
|
help="The line number in the source file(s) to grep")
|
|
parser.add_argument("--count",
|
|
help="Count the number of log messages",
|
|
action="store_true",
|
|
dest="count")
|
|
parser.add_argument(
|
|
"--count-only",
|
|
help="Count the number of log messages without printing them",
|
|
action="store_true",
|
|
dest="count_only")
|
|
parser.add_argument(
|
|
"--count-all",
|
|
help="Count the number of log messages in each category",
|
|
action="store_true",
|
|
dest="count_all")
|
|
parser.add_argument(
|
|
"--dump-db",
|
|
help="Dump logs to a sqlite3 db (named <--file>.sqlite)",
|
|
action="store_true")
|
|
parser.add_argument("--dump-db-to",
|
|
help="Dump logs to a specific sqlite3 db",
|
|
metavar="SQLITE_FILE")
|
|
parser.add_argument("--dump-db-table-to",
|
|
help="Dump logs to a specific table in a sqlite3 db",
|
|
metavar="SQLITE_TABLE")
|
|
parser.add_argument("-y, --yes",
|
|
help="Overwrite existing db without asking",
|
|
action="store_true",
|
|
dest="yes")
|
|
parser.set_defaults(
|
|
pattern=None,
|
|
file=None,
|
|
level=0,
|
|
category=None,
|
|
count=False,
|
|
count_only=False,
|
|
dump_db=False,
|
|
dump_db_to=None,
|
|
)
|
|
return parser
|
|
|
|
|
|
class ChunkIterator:
|
|
def __init__(self, source: Iterator, chunk_size: int):
|
|
self.source = source
|
|
self.chunk_size = chunk_size
|
|
|
|
def __iter__(self):
|
|
return self
|
|
|
|
def __next__(self):
|
|
chunk = []
|
|
try:
|
|
while len(chunk) < self.chunk_size:
|
|
chunk.append(self.source.__next__())
|
|
except StopIteration:
|
|
pass
|
|
if chunk:
|
|
return chunk
|
|
raise StopIteration()
|
|
|
|
|
|
class LogRecord:
|
|
def __init__(self, msg: str):
|
|
self.full_msg = msg
|
|
self.parts = []
|
|
field_start = 0
|
|
for _ in range(0, 9):
|
|
field_end = msg.find("|", field_start)
|
|
field = msg[field_start:field_end].strip()
|
|
field_start = field_end + 1
|
|
self.parts.append(field)
|
|
self.parts.append(msg[field_end + 1:].rstrip())
|
|
|
|
@property
|
|
def ts(self):
|
|
return self.parts[0]
|
|
|
|
@property
|
|
def ts_nanos(self):
|
|
return self.parts[1]
|
|
|
|
@property
|
|
def category(self):
|
|
return self.parts[2]
|
|
|
|
@property
|
|
def level(self):
|
|
return self.parts[3]
|
|
|
|
@property
|
|
def thread_id(self):
|
|
return self.parts[4]
|
|
|
|
@property
|
|
def thread_name(self):
|
|
return self.parts[5]
|
|
|
|
@property
|
|
def pid(self):
|
|
return self.parts[6]
|
|
|
|
@property
|
|
def src_file(self):
|
|
i = self.parts[7].find(":")
|
|
return self.parts[7][:i]
|
|
|
|
@property
|
|
def line_num(self):
|
|
i = self.parts[7].find(":")
|
|
return self.parts[7][i + 1:]
|
|
|
|
@property
|
|
def deployment_type(self):
|
|
return self.parts[8]
|
|
|
|
@property
|
|
def msg(self):
|
|
return self.parts[9]
|
|
|
|
def __str__(self):
|
|
return self.full_msg.rstrip()
|
|
|
|
|
|
class LogRecordIterator:
|
|
def __init__(self, log_file: TextIOWrapper):
|
|
self.log_file = log_file
|
|
self.count = 0
|
|
self.curr_log = None
|
|
self.next_ln = None
|
|
self.next_log_start = None
|
|
|
|
def __iter__(self):
|
|
return self
|
|
|
|
def _read_until_next_log(self):
|
|
curr_log = self.next_log_start
|
|
next_ln = self.log_file.readline()
|
|
while next_ln and not log_msg_start_re.match(next_ln):
|
|
curr_log += next_ln
|
|
next_ln = self.log_file.readline()
|
|
self.next_log_start = next_ln
|
|
return LogRecord(curr_log)
|
|
|
|
def __next__(self):
|
|
if self.next_log_start is None:
|
|
self.next_log_start = self.log_file.readline()
|
|
if not self.next_log_start:
|
|
self.log_file.close()
|
|
raise StopIteration
|
|
return self._read_until_next_log()
|
|
|
|
|
|
class LogFilterCriteria:
|
|
def __init__(
|
|
self,
|
|
level: Optional[int | str] = None,
|
|
category: Optional[str] = None,
|
|
source_file: Optional[str] = None,
|
|
line_num: Optional[int] = None,
|
|
pattern: Optional[str] = None,
|
|
):
|
|
self.level = level_ordinals[level] if isinstance(level, str) else level
|
|
self.category = category
|
|
self.source_file = source_file
|
|
self.line_num = line_num
|
|
self.pattern_re = re.compile(pattern) if pattern else None
|
|
|
|
def include(self, log_rec: LogRecord):
|
|
if level_ordinals[log_rec.level] < self.level:
|
|
return False
|
|
if self.source_file:
|
|
if self.source_file != log_rec.src_file:
|
|
return False
|
|
if self.line_num and self.line_num != log_rec.line_num:
|
|
return False
|
|
if self.category and self.category != log_rec.category:
|
|
return False
|
|
if (self.pattern_re and not self.pattern_re.match(log_rec.msg)):
|
|
return False
|
|
return True
|
|
|
|
|
|
def dump_to_db(log_recs: Iterable[LogRecord], db_out: str, table_name: str):
|
|
con = sqlite3.connect(db_out)
|
|
cur = con.cursor()
|
|
columns = [
|
|
"ts TEXT", "ts_nanos INTEGER", "category TEXT", "level INTEGER",
|
|
"thread_id INTEGER", "thread_name TEXT", "pid INTEGER",
|
|
"src_file TEXT", "line_num TEXT", "deployment_type TEXT", "msg TEXT"
|
|
]
|
|
columns_expr = ", ".join(columns)
|
|
cur.execute(f"CREATE TABLE IF NOT EXISTS {table_name}({columns_expr})")
|
|
insert_expr = f"INSERT INTO {table_name} VALUES ({', '.join(['?' for _ in columns])})"
|
|
chunks = ChunkIterator([(lr.ts, lr.ts_nanos, lr.category, lr.level,
|
|
lr.thread_id, lr.thread_name, lr.pid, lr.src_file,
|
|
lr.line_num, lr.deployment_type, lr.msg.rstrip())
|
|
for lr in log_recs].__iter__(), 256)
|
|
for chunk in chunks:
|
|
cur.executemany(insert_expr, chunk)
|
|
print(f"Commiting sqlite transactions")
|
|
con.commit()
|
|
|
|
|
|
def _main():
|
|
parser = _get_arg_parser()
|
|
args = parser.parse_args()
|
|
dump_db = args.dump_db or args.dump_db_to or args.dump_db_table_to
|
|
log_filter = LogFilterCriteria(
|
|
level=args.level,
|
|
category=args.category,
|
|
source_file=args.source_file,
|
|
line_num=args.line_num,
|
|
pattern=args.pattern,
|
|
)
|
|
with open(args.file) as f:
|
|
log_recs = filter(log_filter.include, LogRecordIterator(f))
|
|
if dump_db:
|
|
db_file = args.dump_db_to or f"{args.file}.sqlite"
|
|
if os.path.exists(db_file):
|
|
if not args.yes:
|
|
raise ValueError(f"'{db_file}' already exists")
|
|
os.remove(db_file)
|
|
db_table = args.dump_db_table_to or "log"
|
|
dump_to_db(log_recs, db_file, db_table)
|
|
else:
|
|
n = 0
|
|
for lr in log_recs:
|
|
if not args.count_only:
|
|
print(lr)
|
|
n += 1
|
|
if args.count or args.count_only:
|
|
print(n)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
_main()
|