from __future__ import annotations import platform import random import subprocess import time import unittest from pathlib import Path from xmlrpc.client import ServerProxy def is_port_in_use(port: int) -> bool: import socket with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s: return s.connect_ex(("127.0.0.1", port)) == 0 class LogParser(object): process: subprocess.Popen client: ServerProxy def __init__(self, binpath: str) -> None: for _ in range(10): port = random.randint(10000, 65535) if is_port_in_use(port): continue self.process = subprocess.Popen([binpath, str(port)]) self.client = ServerProxy(f"http://127.0.0.1:{port}/RPC2") return None raise RuntimeError("can't find a free port") def __enter__(self) -> LogParser: time.sleep(1) return self def __exit__(self, *args) -> None: self.client.__exit__() self.process.terminate() self.process.wait() def parse( self, component: str, line: str ) -> tuple[str, str, str, str, str, str, str, list[dict[str, str]]]: xs = self.client.LogService.Parse(component, line.encode()) return ( xs[:6] + (xs[6].data.decode(),) + ([dict(Name=x["Name"], Value=x["Value"].data.decode()) for x in xs[7]],) ) def id(self, component: str, line: str) -> str: return self.client.LogService.ID(component, line.encode()) class TestParser(unittest.TestCase): def test_parse_log(self) -> None: cases = [ ( "tidb", '[2022/02/15 20:18:51.360 +08:00] [INFO] [server.go:247] ["server is running MySQL protocol"] [addr=0.0.0.0:4000]', ( "10011", "server is running MySQL protocol", "INFO", "2022-02-15T20:18:51+08:00", "server.go", "247", "server is running MySQL protocol", [{"Name": "addr", "Value": "0.0.0.0:4000"}], ), ), ( "tidb", '[2022/02/15 20:18:51.248 +08:00] [INFO] [main.go:349] ["disable Prometheus push client"]', ( "10008", "disable Prometheus push client", "INFO", "2022-02-15T20:18:51+08:00", "main.go", "349", "disable Prometheus push client", [], ), ), ( "pd", '[2022/02/15 20:53:16.009 +08:00] [INFO] [etcd.go:576] ["serving peer traffic"] [address=127.0.0.1:2380]', ( "30111", "serving peer traffic", "INFO", "2022-02-15T20:53:16+08:00", "etcd.go", "576", "serving peer traffic", [{"Name": "address", "Value": "127.0.0.1:2380"}], ), ), ( "tidb", '[2021/12/29 15:15:38.167 +08:00] [WARN] [session.go:539] [sql] [conn=869] [label=general] [error="[kv:9007]Write conflict, txnStartTS=430115949945815044, conflictStartTS=430115949945815046, conflictCommitTS=0, key=t\ufffd\u0000\u0000\u0000\u0000\u0000\u0007M_r\ufffd\u0000\u0000\u0000\u0002H\ufffd> [try again later]"] [txn="Txn{state=invalid}"]', ( "10069", "sql", "WARN", "2021-12-29T15:15:38+08:00", "session.go", "539", "sql", [ {"Name": "conn", "Value": "869"}, {"Name": "label", "Value": "general"}, { "Name": "error", "Value": "[kv:9007]Write conflict, txnStartTS=430115949945815044, conflictStartTS=430115949945815046, conflictCommitTS=0, key=t\ufffd\u0000\u0000\u0000\u0000\u0000\u0007M_r\ufffd\u0000\u0000\u0000\u0002H\ufffd> [try again later]", }, {"Name": "txn", "Value": "Txn{state=invalid}"}, ], ), ), ] with LogParser( Path(__file__).with_name(f"tidb-log-parser-{platform.system().lower()}") ) as p: for case in cases: self.assertEqual(p.parse(case[0], case[1]), case[2]) self.assertEqual(p.id(case[0], case[1]), case[2][0]) if __name__ == "__main__": unittest.main()