Index

lognestmonster / 6177dd6

A general-purpose single-header C logging library and parser for event-based logs. (Incomplete)

Download | Refs | Commit Log | Files | README | LICENSE

Latest Commit

{#}TimeHashSubjectAuthor#(+)(-)GPG?
9407 Sep 2019 22:046177dd6updatesJosh Stockin187N

Blob @ lognestmonster / parser / read.py

application/x-python6370 bytesdownload raw
1# lognestmonster Copyright (c) 2019 Joshua 'joshuas3' Stockin
2# <https://github.com/JoshuaS3/lognestmonster/>.
3
4
5# This file is part of lognestmonster.
6
7# lognestmonster is free software: you can redistribute it and/or modify
8# it under the terms of the GNU General Public License as published by
9# the Free Software Foundation, either version 3 of the License, or
10# (at your option) any later version.
11
12# lognestmonster is distributed in the hope that it will be useful,
13# but WITHOUT ANY WARRANTY; without even the implied warranty of
14# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
15# GNU General Public License for more details.
16
17# You should have received a copy of the GNU General Public License
18# along with lognestmonster. If not, see <https://www.gnu.org/licenses/>.
19
20import struct
21import os
22
23STATEMENT_START = 0
24STATEMENT_END = 1
25EVENT_START = 2
26EVENT_END = 3
27
28VERBOSITY_LEVELS = {
29 0: "INIT",
30 1: "DEBUG",
31 2: "VERBOSE",
32 3: "VERYVERBOSE",
33 4: "WARNING",
34 5: "ERROR"
35}
36
37def ulonglong(bytestr):
38 return struct.unpack("@Q", bytestr)[0]
39def uchar(charv):
40 return struct.unpack("@B", charv)[0]
41def ushort(shortv):
42 return struct.unpack("@H", shortv)[0]
43
44
45class EventProto:
46 parent = None
47 pushed = [False]
48 def __init__(self):
49 self.parent = None
50 self.pushed = [False]
51
52class Reader:
53 fd = None
54
55 top_level = []
56
57 current_event = None
58
59 event_count = 0
60 statement_count = 0
61
62 seekable = True
63 file_size = 0
64 position = 0
65
66 version = 0
67 timestamp = 0
68
69 bad_bytes = 0
70
71 filter_time_start = -1
72 filter_time_end = -1
73 filter_verbosity = -1
74 filter_tag = -1
75
76 update_callbacks = []
77
78 def __init__(self, fd, seekable=True):
79 self.fd = fd
80
81 self.top_level = []
82
83 self.current_event = None
84
85 self.event_count = 0
86 self.statement_count = 0
87
88 self.seekable = seekable
89 self.file_size = 0
90 self.position = 0
91
92 self.version = 0
93 self.timestamp = 0
94
95 self.bad_bytes = 0
96
97 self.filter_time_start = -1
98 self.filter_time_end = -1
99 self.filter_verbosity = -1
100 self.filter_tag = -1
101
102 self.update_callbacks = []
103
104 def update(self):
105 for callback in self.update_callbacks:
106 callback()
107
108 def onupdate(self, callback):
109 self.update_callbacks.append(callback)
110
111 def size(self):
112 self.fd.seek(0, os.SEEK_END) # go to end of file and get position
113 newsize = self.fd.tell()
114 self.fd.seek(self.position) # return to previous position
115
116 is_diff = self.file_size is not newsize
117 self.file_size = newsize
118 return is_diff
119
120 def pos(self):
121 p = self.fd.tell()
122 self.position = p
123 return p
124
125 def seek(self, position):
126 self.position = position
127 self.fd.seek(self.position)
128
129 def read(self, byte_count):
130 return self.fd.read(byte_count)
131
132 def fetch_item(self, position):
133 previouspos = self.pos()
134 self.seek(position)
135 timestamp = ulonglong(self.read(8))
136 verbosity = uchar(self.read(1))
137 tag_size = uchar(self.read(1))
138 tag = self.read(tag_size).decode("utf-8")
139 message_size = ushort(self.read(2))
140 message = self.read(message_size).decode("utf-8")
141 self.seek(previouspos)
142 return (timestamp, verbosity, tag_size, tag, message_size, message)
143
144 def parse_block(self, in_byte):
145 seekable = self.seekable
146 if in_byte == STATEMENT_START: # the byte indicates a statement's start, begin interpreting
147 if seekable:
148 this_position = self.pos() # identify and save the seeker position of this statement
149
150 try:
151 timestamp = ulonglong(self.read(8))
152 verbosity = uchar(self.read(1))
153 tag_size = uchar(self.read(1))
154 tag = self.read(tag_size).decode("utf-8")
155
156 append = True
157
158 if self.filter_time_start is not -1 and append:
159 append = timestamp > self.filter_time_start
160
161 if self.filter_time_end is not -1 and append:
162 append = timestamp < self.filter_time_end
163
164 if self.filter_verbosity is not -1 and append:
165 append = verbosity in self.filter_verbosity
166
167 if self.filter_tag is not -1 and append:
168 append = tag == self.filter_tag
169
170 message_size = ushort(self.read(2))
171 if seekable:
172 self.read(message_size)
173 else:
174 message = self.read(message_size).decode("utf-8")
175
176 if seekable:
177 while uchar(self.read(1)) is not STATEMENT_END and self.pos() < self.file_size:
178 self.bad_bytes += 1
179 else:
180 while uchar(self.read(1)) is not STATEMENT_END:
181 self.bad_bytes += 1
182
183 if append == True:
184 self.statement_count += 1
185 #self.update()
186 if self.current_event is not None:
187 if seekable:
188 self.current_event.pushed.append(this_position)
189 else:
190 self.current_event.pushed.append((timestamp, verbosity, tag_size, tag, message_size, message))
191 else:
192 if seekable:
193 self.top_level.append(this_position)
194 else:
195 self.top_level.append((timestamp, verbosity, tag_size, tag, message_size, message))
196 except:
197 return -1
198
199 elif in_byte == EVENT_START: # the byte indicates an event's start, create an event
200 new_event = EventProto()
201 if self.current_event is not None: # we're already inside an event, set the new event's parent to match
202 new_event.parent = self.current_event
203 self.current_event = new_event
204
205
206 elif in_byte == EVENT_END: # end of event
207 if self.current_event is not None:
208 if len(self.current_event.pushed) > 1:
209 self.event_count += 1
210 self.update()
211 if self.current_event.parent is not None:
212 self.current_event.parent.pushed.append(self.current_event)
213 self.current_event = self.current_event.parent
214 else:
215 self.top_level.append(self.current_event)
216 self.current_event = None
217 else: # event is empty
218 if self.current_event.parent is not None:
219 self.current_event = self.current_event.parent
220 else:
221 self.current_event = None
222 else:
223 self.bad_bytes += 1 # event doesn't exist, bad byte
224 return -1
225
226 else: # unknown byte
227 self.bad_bytes += 1
228 return -1
229
230 def scan(self): # scan for events and statements from self.position to the end of file
231 if self.position == 0 and self.seekable: # if it's the start of the file, grab version and timestamp
232 self.version = uchar(self.read(1))
233 self.timestamp = ulonglong(self.read(8))
234
235 current_event = None
236
237 if self.pos() < self.file_size: # if the seeker is before EOF
238 while self.pos() < self.file_size: # while the seeker is before EOF
239 in_byte = uchar(self.read(1)) # read 1 byte
240 self.parse_block(in_byte) # parse block based on byte read
241
242