From 78fb6516f639046de409c5f0cbf501ee8cd2972f Mon Sep 17 00:00:00 2001 From: tcely Date: Sun, 17 May 2026 21:19:56 -0400 Subject: [PATCH] Add hat-syslog patches to Dockerfile --- Dockerfile | 4 + patches/hat/syslog/encoder.py | 217 ++++++++++++++++++++++++++++ patches/hat/syslog/older_formats.py | 111 ++++++++++++++ 3 files changed, 332 insertions(+) create mode 100644 patches/hat/syslog/encoder.py create mode 100644 patches/hat/syslog/older_formats.py diff --git a/Dockerfile b/Dockerfile index 170c7bea..c6a977cb 100644 --- a/Dockerfile +++ b/Dockerfile @@ -720,6 +720,10 @@ RUN --mount=type=tmpfs,target=/cache \ # Copy root COPY config/root / +# patch hat-syslog +COPY patches/hat/ \ + /usr/local/lib/python3/dist-packages/hat/ + # patch yt_dlp COPY patches/yt_dlp/ \ /usr/local/lib/python3/dist-packages/yt_dlp/ diff --git a/patches/hat/syslog/encoder.py b/patches/hat/syslog/encoder.py new file mode 100644 index 00000000..b9fc6596 --- /dev/null +++ b/patches/hat/syslog/encoder.py @@ -0,0 +1,217 @@ +"""Syslog message encoder/decoder""" + +import datetime +import re + +from hat import json + +from hat.syslog import common +from hat.syslog import older_formats + + +def msg_to_str(msg: common.Msg) -> str: + """Create string representation of message according to RFC 5424""" + buff = [ + f'<{msg.facility.value * 8 + msg.severity.value}>{msg.version}', + _timestamp_to_str(msg.timestamp), + msg.hostname if msg.hostname else '-', + msg.app_name if msg.app_name else '-', + msg.procid if msg.procid else '-', + msg.msgid if msg.msgid else '-', + _data_to_str(msg.data)] + if msg.msg: + buff.append('BOM' + msg.msg) + return ' '.join(buff) + + +def msg_from_str(msg_str: str) -> common.Msg: + """Parse message string formatted according to RFC 5424""" + match = _msg_pattern.fullmatch(msg_str) + if match is None: + return older_formats.msg_from_rfc3164_str(msg_str) + match = match.groupdict() + prival = int(match['prival']) + return common.Msg( + facility=common.Facility(prival // 8), + severity=common.Severity(prival % 8), + version=int(match['version']), + timestamp=_parse_timestamp(match['timestamp']), + hostname=None if match['hostname'] == '-' else match['hostname'], + app_name=None if match['app_name'] == '-' else match['app_name'], + procid=None if match['procid'] == '-' else match['procid'], + msgid=None if match['msgid'] == '-' else match['msgid'], + data=_parse_data(match['data']), + msg=(match['msg'][3:] if match['msg'] and match['msg'][:3] == 'BOM' + else match['msg'])) + + +def msg_to_json(msg: common.Msg) -> json.Data: + """Convert message to json serializable data""" + return {'facility': msg.facility.name, + 'severity': msg.severity.name, + 'version': msg.version, + 'timestamp': msg.timestamp, + 'hostname': msg.hostname, + 'app_name': msg.app_name, + 'procid': msg.procid, + 'msgid': msg.msgid, + 'data': msg.data, + 'msg': msg.msg} + + +def msg_from_json(data: json.Data) -> common.Msg: + """Convert json serializable data to message""" + return common.Msg(facility=common.Facility[data['facility']], + severity=common.Severity[data['severity']], + version=data['version'], + timestamp=data['timestamp'], + hostname=data['hostname'], + app_name=data['app_name'], + procid=data['procid'], + msgid=data['msgid'], + data=data['data'], + msg=data['msg']) + + +_msg_pattern = re.compile(r''' + < (?P \d+) > + (?P \d+) + \ (?P - | + [^ ]+) + \ (?P - | + [^ ]+) + \ (?P - | + [^ ]+) + \ (?P - | + [^ ]+) + \ (?P - | + [^ ]+) + \ (?P - | + (\[ + ((\\(\\\\)*\]) | + [^\]])* + \])+) + (\ (?P .*))? +''', re.X | re.DOTALL) + +_timestamp_pattern = re.compile(r''' + (?P \d{4}) + - + (?P \d{2}) + - + (?P \d{2}) + T + (?P \d{2}) + : + (?P \d{2}) + : + (?P \d{2}) + (\. (?P \d+))? + ((?P Z) | + ((?P \+ | + -) + (?P \d{2}) + : + (?P \d{2}))) +''', re.X | re.DOTALL) + +_data_pattern = re.compile(r''' + \[ + (?P [^ \]]+) + (?P ((\\(\\\\)*\]) | + [^\]])*) + \] + (?P .*) +''', re.X | re.DOTALL) + +_param_pattern = re.compile(r''' + \ (?P [^=\]]+) + =" + (?P ((\\\\) | + (\\") | + (\\\]) | + [^"\]\\])*) + " + (?P .*) +''', re.X | re.DOTALL) + +_escape_pattern = re.compile(r'''((\\\\)|(\\")|(\\]))''') + + +def _timestamp_to_str(timestamp): + if not timestamp: + return '-' + return datetime.datetime.fromtimestamp( + timestamp, datetime.timezone.utc).replace( + tzinfo=None).isoformat() + 'Z' + + +def _data_to_str(data_json): + data = json.decode(data_json) if data_json else None + if not data: + return '-' + return ''.join(f'[{sd_id}{_param_to_str(param)}]' + for sd_id, param in data.items()) + + +def _param_to_str(param): + if not param: + return '' + return ' ' + ' '.join(f'{k}="{_escape_value(v)}"' + for k, v in param.items()) + + +def _parse_timestamp(timestamp_str): + if timestamp_str == '-': + return + match = _timestamp_pattern.fullmatch(timestamp_str).groupdict() + return datetime.datetime( + year=int(match['year']), + month=int(match['month']), + day=int(match['day']), + hour=int(match['hour']), + minute=int(match['minute']), + second=int(match['second']), + microsecond=(int(int(match['fraction']) * + pow(10, 6 - len(match['fraction']))) + if match['fraction'] else None), + tzinfo=(datetime.timezone.utc if match['tz_utc'] else + datetime.timezone(datetime.timedelta( + hours=((1 if match['tz_sign'] == '+' else -1) * + int(match['tz_hour'])), + minutes=int(match['tz_hour']))))).timestamp() + + +def _parse_data(data_str): + if data_str == '-': + return + data = {} + while data_str: + match = _data_pattern.fullmatch(data_str).groupdict() + data[match['id']] = _parse_param(match['param']) + data_str = match['rest'] + data_json = json.encode(data) + return data_json + + +def _parse_param(param_str): + param = {} + while param_str: + match = _param_pattern.fullmatch(param_str).groupdict() + param[match['name']] = _unescape_value(match['value']) + param_str = match['rest'] + return param + + +def _escape_value(value): + return value.replace('\\', '\\\\').replace('"', '\\"').replace(']', '\\]') + + +def _unescape_value(value): + return re.sub(_escape_pattern, _unescape_value_char, value) + + +def _unescape_value_char(match): + return {r'\\': '\\', + r'\"': r'"', + r'\]': r']'}[match.group(0)] diff --git a/patches/hat/syslog/older_formats.py b/patches/hat/syslog/older_formats.py new file mode 100644 index 00000000..d25c0ea5 --- /dev/null +++ b/patches/hat/syslog/older_formats.py @@ -0,0 +1,111 @@ +import re +import socket +from datetime import datetime +from hat.syslog import common + + +KNOWN_HOSTNAME = socket.gethostname() +RE_HOSTNAME = re.escape(KNOWN_HOSTNAME) + +VALID_MONTHS = ( + 'Jan', 'Feb', 'Mar', 'Apr', 'May', 'Jun', + 'Jul', 'Aug', 'Sep', 'Oct', 'Nov', 'Dec', +) + +S = ' ' +EOL = r'\r?\n?' +PRI = '<(?P0|[1-9][0-9]?|1[0-8][0-9]|19[0-1])>' +MONTH = f'(?P{"|".join(map(re.escape, VALID_MONTHS))})' +DAY = f'(?P{S}[1-9]|[1-2][0-9]|3[0-1])' +HOUR = '(?P[0-1][0-9]|2[0-3])' +MINUTE = ':(?P[0-5][0-9])' +SECOND = ':(?P[0-5][0-9])' +HOST_STRICT = f'(?:{S}{RE_HOSTNAME}(?={S}))?' +NOT_HOST = f'(?!{RE_HOSTNAME})' +PID = r'(?:[\[](?P[0-9]+)[\]])' +TAG_PID = f'(?P.+?){PID}?:' +MSG_BODY = '(?P(?s:.)+)' + +formats = [ + # Generic with optional hostname and PID: + # Begins with a partial (5-15) ctime() date string. + # Does not accept hostnames other than this one. + # Remote logs that do not include a hostname are accepted. + {'parts': ( + PRI, MONTH, S, DAY, S, HOUR, MINUTE, SECOND, + HOST_STRICT, S, NOT_HOST, TAG_PID, S, + MSG_BODY, EOL, + )}, + # Support for `gunicorn` logs: + # No date or hostname. + # Also, non-standard PID placement. + {'parts': ( + PRI, + '(?P.+?):', S, PID, S, + MSG_BODY, EOL, + )}, +] +for _dict in formats: + _dict['regex'] = re.compile(''.join(_dict['parts'])) + + +def msg_from_rfc3164_str(msg_str: str) -> common.Msg: + """RFC 3164 parser. Raises ValueError on any deviation.""" + now = datetime.now() + + for _dict in formats: + _format_ = _dict['regex'] + match_obj = _format_.fullmatch(msg_str) + if match_obj is not None: + break + + if match_obj is None: + raise ValueError(f'No formats matched: {msg_str.encode()!r}') + + m = match_obj.groupdict() + + if 'month' in m: + day_val = m['day'].replace(' ', '0') + time_str = f'{m["hour"]}:{m["minute"]}:{m["second"]}' + ts_str = f'{now.year} {m["month"]} {day_val} {time_str}' + + dt = datetime.strptime(ts_str, '%Y %b %d %H:%M:%S') + # The skew should be zero when logging from the same host. + if now < dt: + dt = dt.replace(year=now.year - 1) + else: + # The matched format did not include the date and time. + dt = now + + prival = int(str(m['prival']), 10) + procid = m.get('procid', None) + tag_str = m['app_name'] + tag_ends_with_brackets = ( + ']' == tag_str[-1] and + tag_str.rsplit('[')[-1][:-1] and + tag_str[-1] != tag_str.rsplit('[')[-1][:-1] + ) + if procid is None and tag_ends_with_brackets: + procid = tag_str.rsplit('[')[-1][:-1] + if procid is not None: + try: + _pid = int(str(procid), 10) + if 0 >= _pid: + raise ValueError('too low') + elif 4_194_304 < _pid: # read from /proc instead? + raise ValueError('too high') + except Exception as e: + raise ValueError(f'Invalid process ID: {e}') + + return common.Msg( + facility=common.Facility(prival // 8), + severity=common.Severity(prival % 8), + version=None, + timestamp=dt.timestamp(), + hostname=KNOWN_HOSTNAME, + app_name=m['app_name'], + procid=m.get('procid', None), + msgid=None, + data=None, + msg=m['msg'] + )