Index

lognestmonster / 0faeedc

A general-purpose single-header C logging library and parser for event-based logs. (Incomplete)

Download | Refs | Commit Log | Files | README | LICENSE

Latest Commit

{#}TimeHashSubjectAuthor#(+)(-)GPG?
9001 Sep 2019 21:000faeedcupdate log reading/parsingJosh Stockin113132N

Blob @ lognestmonster / parser / read.py

application/x-python5491 bytesdownload raw
1# lognestmonster Copyright (c) 2019 Joshua 'joshuas3' Stockin
2# <https://github.com/JoshuaS3/lognestmonster/>.
3
4
5# This file is part of lognestmonster.
6
7# lognestmonster is free software: you can redistribute it and/or modify
8# it under the terms of the GNU General Public License as published by
9# the Free Software Foundation, either version 3 of the License, or
10# (at your option) any later version.
11
12# lognestmonster is distributed in the hope that it will be useful,
13# but WITHOUT ANY WARRANTY; without even the implied warranty of
14# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
15# GNU General Public License for more details.
16
17# You should have received a copy of the GNU General Public License
18# along with lognestmonster. If not, see <https://www.gnu.org/licenses/>.
19
20import struct
21import os
22import resource
23import time
24import sys
25
26STATEMENT_START = 0
27STATEMENT_END = 1
28EVENT_START = 2
29EVENT_END = 3
30
31BUFFER_SIZE = 4096
32
33VERBOSITY_LEVELS = {
34 0: "INIT",
35 1: "DEBUG",
36 2: "VERBOSE",
37 3: "VERYVERBOSE",
38 4: "WARNING",
39 5: "ERROR"
40}
41
42def ulonglong(bytestr):
43 return struct.unpack("@Q", bytestr)[0]
44def uchar(charv):
45 return struct.unpack("@B", charv)[0]
46def ushort(shortv):
47 return struct.unpack("@H", shortv)[0]
48
49
50class EventProto:
51 parent = None
52 pushed = []
53 def __init__(self):
54 self.parent = None
55 self.pushed = []
56
57class Reader:
58 fd = None
59
60 version = 0
61 timestamp = 0
62
63 top_level = []
64
65 event_count = 0
66 statement_count = 0
67
68 file_size = 0
69 position = 0
70 bad_bytes = 0
71
72 filter_time_start = -1
73 filter_time_end = -1
74 filter_verbosity = -1
75 filter_tag = -1
76
77 buf = b""
78 bufp = BUFFER_SIZE
79
80 def __init__(self, fd):
81 self.fd = fd
82
83 self.version = 0
84 self.timestamp = 0
85
86 self.top_level = []
87
88 self.event_count = 0
89 self.statement_count = 0
90
91 self.file_size = 0
92 self.position = 0
93 self.bad_bytes = 0
94
95 self.filter_time_start = -1
96 self.filter_time_end = -1
97 self.filter_verbosity = -1
98 self.filter_tag = -1
99
100 self.buf = b""
101 self.bufp = BUFFER_SIZE
102
103 self.size()
104 self.scan()
105
106 def size(self):
107 self.fd.seek(0, os.SEEK_END) # go to end of file and get position
108 newsize = self.fd.tell()
109 self.fd.seek(self.position) # return to previous position
110
111 is_diff = self.file_size is not newsize
112 self.file_size = newsize
113 return is_diff
114
115 def pos(self):
116 self.position = self.fd.tell()
117 return self.position
118
119 def seek(self, position):
120 self.position = position
121 self.fd.seek(self.position)
122
123 def read(self, byte_count):
124 data = self.fd.read(byte_count)
125 if len(data) == byte_count:
126 return data
127 else:
128 return False
129
130 def scan(self): # scan for events and statements from self.position to the end of file
131 print()
132 print("beginning file scan")
133 print("file size: {0}".format(self.file_size))
134 print()
135
136 s = time.time()
137
138 if self.position == 0: # if it's the start of the file, grab version and timestamp
139 self.version = uchar(self.read(1))
140 self.timestamp = ulonglong(self.read(8))
141
142 current_statement = None
143 current_event = None
144
145 if self.position < self.file_size: # if the seeker is before EOF
146 while self.position < self.file_size: # while the seeker is before EOF
147 in_byte = uchar(self.read(1)) # read 1 byte
148
149 if in_byte == STATEMENT_START: # the byte indicates a statement's start, begin interpreting
150 self.statement_count += 1
151
152 new_statement = self.position
153
154 timestamp = ulonglong(self.read(8))
155 verbosity = uchar(self.read(1))
156
157 tag_size = uchar(self.read(1))
158 tag = self.read(tag_size).decode("utf-8")
159
160 append = True
161 if self.filter_time_start is not -1:
162 if timestamp < self.filter_time_start: append = False
163 if self.filter_time_end is not -1:
164 if timestamp > self.filter_time_end: append = False
165 if self.filter_verbosity is not -1:
166 if verbosity is not self.filter_verbosity: append = False
167 if self.filter_tag is not -1:
168 if tag is not self.filter_tag: append = False
169
170 message_size = ushort(self.read(2))
171 self.seek(self.pos() + message_size) # ignore the message
172
173 while uchar(self.read(1)) is not STATEMENT_END:
174 self.bad_bytes += 1
175
176 if append:
177 if current_event is not None:
178 current_event.pushed.append(new_statement)
179 else:
180 self.top_level.append(new_statement)
181
182 elif in_byte == EVENT_START: # the byte indicates an event's start, create an event
183 self.event_count += 1
184 new_event = EventProto()
185 if current_event is not None: # if an event exists, push the new event to it
186 new_event.parent = current_event
187 current_event.pushed.append(new_event)
188 current_event = new_event
189
190
191 elif in_byte == EVENT_END: # the byte indicates an event's end, close event if exists
192 if current_event is not None: # if an event exists
193 if current_event.parent is not None:
194 current_event = current_event.parent # if the event has a parent, set the parent to current
195 else:
196 self.top_level.append(current_event) # event has no parent, it's a top-level log item
197 current_event = None
198
199 else:
200 self.bad_bytes += 1
201
202 self.pos() # update seeker position for next byte (if not EOF)
203
204 print()
205 print("finished reading, {0} bad bytes".format(self.bad_bytes))
206 print()
207 print("version {0}".format(self.version))
208 print("timestamp {0}".format(self.timestamp))
209 print("event count {0}".format(self.event_count))
210 print("statement count {0}".format(self.statement_count))
211 print("time: {0}".format(time.time() - s))
212
213