Add hat-syslog patches to Dockerfile
This commit is contained in:
tcely
@@ -720,6 +720,10 @@ RUN --mount=type=tmpfs,target=/cache \
|
||||
# Copy root
|
||||
COPY config/root /
|
||||
|
||||
# patch hat-syslog
|
||||
COPY patches/hat/ \
|
||||
/usr/local/lib/python3/dist-packages/hat/
|
||||
|
||||
# patch yt_dlp
|
||||
COPY patches/yt_dlp/ \
|
||||
/usr/local/lib/python3/dist-packages/yt_dlp/
|
||||
|
||||
217
patches/hat/syslog/encoder.py
Normal file
217
patches/hat/syslog/encoder.py
Normal file
@@ -0,0 +1,217 @@
|
||||
"""Syslog message encoder/decoder"""
|
||||
|
||||
import datetime
|
||||
import re
|
||||
|
||||
from hat import json
|
||||
|
||||
from hat.syslog import common
|
||||
from hat.syslog import older_formats
|
||||
|
||||
|
||||
def msg_to_str(msg: common.Msg) -> str:
|
||||
"""Create string representation of message according to RFC 5424"""
|
||||
buff = [
|
||||
f'<{msg.facility.value * 8 + msg.severity.value}>{msg.version}',
|
||||
_timestamp_to_str(msg.timestamp),
|
||||
msg.hostname if msg.hostname else '-',
|
||||
msg.app_name if msg.app_name else '-',
|
||||
msg.procid if msg.procid else '-',
|
||||
msg.msgid if msg.msgid else '-',
|
||||
_data_to_str(msg.data)]
|
||||
if msg.msg:
|
||||
buff.append('BOM' + msg.msg)
|
||||
return ' '.join(buff)
|
||||
|
||||
|
||||
def msg_from_str(msg_str: str) -> common.Msg:
|
||||
"""Parse message string formatted according to RFC 5424"""
|
||||
match = _msg_pattern.fullmatch(msg_str)
|
||||
if match is None:
|
||||
return older_formats.msg_from_rfc3164_str(msg_str)
|
||||
match = match.groupdict()
|
||||
prival = int(match['prival'])
|
||||
return common.Msg(
|
||||
facility=common.Facility(prival // 8),
|
||||
severity=common.Severity(prival % 8),
|
||||
version=int(match['version']),
|
||||
timestamp=_parse_timestamp(match['timestamp']),
|
||||
hostname=None if match['hostname'] == '-' else match['hostname'],
|
||||
app_name=None if match['app_name'] == '-' else match['app_name'],
|
||||
procid=None if match['procid'] == '-' else match['procid'],
|
||||
msgid=None if match['msgid'] == '-' else match['msgid'],
|
||||
data=_parse_data(match['data']),
|
||||
msg=(match['msg'][3:] if match['msg'] and match['msg'][:3] == 'BOM'
|
||||
else match['msg']))
|
||||
|
||||
|
||||
def msg_to_json(msg: common.Msg) -> json.Data:
|
||||
"""Convert message to json serializable data"""
|
||||
return {'facility': msg.facility.name,
|
||||
'severity': msg.severity.name,
|
||||
'version': msg.version,
|
||||
'timestamp': msg.timestamp,
|
||||
'hostname': msg.hostname,
|
||||
'app_name': msg.app_name,
|
||||
'procid': msg.procid,
|
||||
'msgid': msg.msgid,
|
||||
'data': msg.data,
|
||||
'msg': msg.msg}
|
||||
|
||||
|
||||
def msg_from_json(data: json.Data) -> common.Msg:
|
||||
"""Convert json serializable data to message"""
|
||||
return common.Msg(facility=common.Facility[data['facility']],
|
||||
severity=common.Severity[data['severity']],
|
||||
version=data['version'],
|
||||
timestamp=data['timestamp'],
|
||||
hostname=data['hostname'],
|
||||
app_name=data['app_name'],
|
||||
procid=data['procid'],
|
||||
msgid=data['msgid'],
|
||||
data=data['data'],
|
||||
msg=data['msg'])
|
||||
|
||||
|
||||
_msg_pattern = re.compile(r'''
|
||||
< (?P<prival> \d+) >
|
||||
(?P<version> \d+)
|
||||
\ (?P<timestamp> - |
|
||||
[^ ]+)
|
||||
\ (?P<hostname> - |
|
||||
[^ ]+)
|
||||
\ (?P<app_name> - |
|
||||
[^ ]+)
|
||||
\ (?P<procid> - |
|
||||
[^ ]+)
|
||||
\ (?P<msgid> - |
|
||||
[^ ]+)
|
||||
\ (?P<data> - |
|
||||
(\[
|
||||
((\\(\\\\)*\]) |
|
||||
[^\]])*
|
||||
\])+)
|
||||
(\ (?P<msg> .*))?
|
||||
''', re.X | re.DOTALL)
|
||||
|
||||
_timestamp_pattern = re.compile(r'''
|
||||
(?P<year> \d{4})
|
||||
-
|
||||
(?P<month> \d{2})
|
||||
-
|
||||
(?P<day> \d{2})
|
||||
T
|
||||
(?P<hour> \d{2})
|
||||
:
|
||||
(?P<minute> \d{2})
|
||||
:
|
||||
(?P<second> \d{2})
|
||||
(\. (?P<fraction> \d+))?
|
||||
((?P<tz_utc> Z) |
|
||||
((?P<tz_sign> \+ |
|
||||
-)
|
||||
(?P<tz_hour> \d{2})
|
||||
:
|
||||
(?P<tz_minute> \d{2})))
|
||||
''', re.X | re.DOTALL)
|
||||
|
||||
_data_pattern = re.compile(r'''
|
||||
\[
|
||||
(?P<id> [^ \]]+)
|
||||
(?P<param> ((\\(\\\\)*\]) |
|
||||
[^\]])*)
|
||||
\]
|
||||
(?P<rest> .*)
|
||||
''', re.X | re.DOTALL)
|
||||
|
||||
_param_pattern = re.compile(r'''
|
||||
\ (?P<name> [^=\]]+)
|
||||
="
|
||||
(?P<value> ((\\\\) |
|
||||
(\\") |
|
||||
(\\\]) |
|
||||
[^"\]\\])*)
|
||||
"
|
||||
(?P<rest> .*)
|
||||
''', re.X | re.DOTALL)
|
||||
|
||||
_escape_pattern = re.compile(r'''((\\\\)|(\\")|(\\]))''')
|
||||
|
||||
|
||||
def _timestamp_to_str(timestamp):
|
||||
if not timestamp:
|
||||
return '-'
|
||||
return datetime.datetime.fromtimestamp(
|
||||
timestamp, datetime.timezone.utc).replace(
|
||||
tzinfo=None).isoformat() + 'Z'
|
||||
|
||||
|
||||
def _data_to_str(data_json):
|
||||
data = json.decode(data_json) if data_json else None
|
||||
if not data:
|
||||
return '-'
|
||||
return ''.join(f'[{sd_id}{_param_to_str(param)}]'
|
||||
for sd_id, param in data.items())
|
||||
|
||||
|
||||
def _param_to_str(param):
|
||||
if not param:
|
||||
return ''
|
||||
return ' ' + ' '.join(f'{k}="{_escape_value(v)}"'
|
||||
for k, v in param.items())
|
||||
|
||||
|
||||
def _parse_timestamp(timestamp_str):
|
||||
if timestamp_str == '-':
|
||||
return
|
||||
match = _timestamp_pattern.fullmatch(timestamp_str).groupdict()
|
||||
return datetime.datetime(
|
||||
year=int(match['year']),
|
||||
month=int(match['month']),
|
||||
day=int(match['day']),
|
||||
hour=int(match['hour']),
|
||||
minute=int(match['minute']),
|
||||
second=int(match['second']),
|
||||
microsecond=(int(int(match['fraction']) *
|
||||
pow(10, 6 - len(match['fraction'])))
|
||||
if match['fraction'] else None),
|
||||
tzinfo=(datetime.timezone.utc if match['tz_utc'] else
|
||||
datetime.timezone(datetime.timedelta(
|
||||
hours=((1 if match['tz_sign'] == '+' else -1) *
|
||||
int(match['tz_hour'])),
|
||||
minutes=int(match['tz_hour']))))).timestamp()
|
||||
|
||||
|
||||
def _parse_data(data_str):
|
||||
if data_str == '-':
|
||||
return
|
||||
data = {}
|
||||
while data_str:
|
||||
match = _data_pattern.fullmatch(data_str).groupdict()
|
||||
data[match['id']] = _parse_param(match['param'])
|
||||
data_str = match['rest']
|
||||
data_json = json.encode(data)
|
||||
return data_json
|
||||
|
||||
|
||||
def _parse_param(param_str):
|
||||
param = {}
|
||||
while param_str:
|
||||
match = _param_pattern.fullmatch(param_str).groupdict()
|
||||
param[match['name']] = _unescape_value(match['value'])
|
||||
param_str = match['rest']
|
||||
return param
|
||||
|
||||
|
||||
def _escape_value(value):
|
||||
return value.replace('\\', '\\\\').replace('"', '\\"').replace(']', '\\]')
|
||||
|
||||
|
||||
def _unescape_value(value):
|
||||
return re.sub(_escape_pattern, _unescape_value_char, value)
|
||||
|
||||
|
||||
def _unescape_value_char(match):
|
||||
return {r'\\': '\\',
|
||||
r'\"': r'"',
|
||||
r'\]': r']'}[match.group(0)]
|
||||
111
patches/hat/syslog/older_formats.py
Normal file
111
patches/hat/syslog/older_formats.py
Normal file
@@ -0,0 +1,111 @@
|
||||
import re
|
||||
import socket
|
||||
from datetime import datetime
|
||||
from hat.syslog import common
|
||||
|
||||
|
||||
KNOWN_HOSTNAME = socket.gethostname()
|
||||
RE_HOSTNAME = re.escape(KNOWN_HOSTNAME)
|
||||
|
||||
VALID_MONTHS = (
|
||||
'Jan', 'Feb', 'Mar', 'Apr', 'May', 'Jun',
|
||||
'Jul', 'Aug', 'Sep', 'Oct', 'Nov', 'Dec',
|
||||
)
|
||||
|
||||
S = ' '
|
||||
EOL = r'\r?\n?'
|
||||
PRI = '<(?P<prival>0|[1-9][0-9]?|1[0-8][0-9]|19[0-1])>'
|
||||
MONTH = f'(?P<month>{"|".join(map(re.escape, VALID_MONTHS))})'
|
||||
DAY = f'(?P<day>{S}[1-9]|[1-2][0-9]|3[0-1])'
|
||||
HOUR = '(?P<hour>[0-1][0-9]|2[0-3])'
|
||||
MINUTE = ':(?P<minute>[0-5][0-9])'
|
||||
SECOND = ':(?P<second>[0-5][0-9])'
|
||||
HOST_STRICT = f'(?:{S}{RE_HOSTNAME}(?={S}))?'
|
||||
NOT_HOST = f'(?!{RE_HOSTNAME})'
|
||||
PID = r'(?:[\[](?P<procid>[0-9]+)[\]])'
|
||||
TAG_PID = f'(?P<app_name>.+?){PID}?:'
|
||||
MSG_BODY = '(?P<msg>(?s:.)+)'
|
||||
|
||||
formats = [
|
||||
# Generic with optional hostname and PID:
|
||||
# Begins with a partial (5-15) ctime() date string.
|
||||
# Does not accept hostnames other than this one.
|
||||
# Remote logs that do not include a hostname are accepted.
|
||||
{'parts': (
|
||||
PRI, MONTH, S, DAY, S, HOUR, MINUTE, SECOND,
|
||||
HOST_STRICT, S, NOT_HOST, TAG_PID, S,
|
||||
MSG_BODY, EOL,
|
||||
)},
|
||||
# Support for `gunicorn` logs:
|
||||
# No date or hostname.
|
||||
# Also, non-standard PID placement.
|
||||
{'parts': (
|
||||
PRI,
|
||||
'(?P<app_name>.+?):', S, PID, S,
|
||||
MSG_BODY, EOL,
|
||||
)},
|
||||
]
|
||||
for _dict in formats:
|
||||
_dict['regex'] = re.compile(''.join(_dict['parts']))
|
||||
|
||||
|
||||
def msg_from_rfc3164_str(msg_str: str) -> common.Msg:
|
||||
"""RFC 3164 parser. Raises ValueError on any deviation."""
|
||||
now = datetime.now()
|
||||
|
||||
for _dict in formats:
|
||||
_format_ = _dict['regex']
|
||||
match_obj = _format_.fullmatch(msg_str)
|
||||
if match_obj is not None:
|
||||
break
|
||||
|
||||
if match_obj is None:
|
||||
raise ValueError(f'No formats matched: {msg_str.encode()!r}')
|
||||
|
||||
m = match_obj.groupdict()
|
||||
|
||||
if 'month' in m:
|
||||
day_val = m['day'].replace(' ', '0')
|
||||
time_str = f'{m["hour"]}:{m["minute"]}:{m["second"]}'
|
||||
ts_str = f'{now.year} {m["month"]} {day_val} {time_str}'
|
||||
|
||||
dt = datetime.strptime(ts_str, '%Y %b %d %H:%M:%S')
|
||||
# The skew should be zero when logging from the same host.
|
||||
if now < dt:
|
||||
dt = dt.replace(year=now.year - 1)
|
||||
else:
|
||||
# The matched format did not include the date and time.
|
||||
dt = now
|
||||
|
||||
prival = int(str(m['prival']), 10)
|
||||
procid = m.get('procid', None)
|
||||
tag_str = m['app_name']
|
||||
tag_ends_with_brackets = (
|
||||
']' == tag_str[-1] and
|
||||
tag_str.rsplit('[')[-1][:-1] and
|
||||
tag_str[-1] != tag_str.rsplit('[')[-1][:-1]
|
||||
)
|
||||
if procid is None and tag_ends_with_brackets:
|
||||
procid = tag_str.rsplit('[')[-1][:-1]
|
||||
if procid is not None:
|
||||
try:
|
||||
_pid = int(str(procid), 10)
|
||||
if 0 >= _pid:
|
||||
raise ValueError('too low')
|
||||
elif 4_194_304 < _pid: # read from /proc instead?
|
||||
raise ValueError('too high')
|
||||
except Exception as e:
|
||||
raise ValueError(f'Invalid process ID: {e}')
|
||||
|
||||
return common.Msg(
|
||||
facility=common.Facility(prival // 8),
|
||||
severity=common.Severity(prival % 8),
|
||||
version=None,
|
||||
timestamp=dt.timestamp(),
|
||||
hostname=KNOWN_HOSTNAME,
|
||||
app_name=m['app_name'],
|
||||
procid=m.get('procid', None),
|
||||
msgid=None,
|
||||
data=None,
|
||||
msg=m['msg']
|
||||
)
|
||||
Reference in New Issue
Block a user