[PATCH] log rendering in real time in audit-viewer
Xeniya Muratova
muratova at itsirius.su
Wed Mar 25 13:21:31 UTC 2015
This code extends audit-viewer _ParserEventSource to produce UpdatableEventSource (unprivileged).
I have not added this option to SourceDialog, so to see how updating works command line call may be used (python main.py -p -s /path/to/log)
diff -u -X ex or_src/event_source.py oav/src/event_source.py
--- or_src/event_source.py 2009-06-09 23:10:41.000000000 +0400
+++ oav/src/event_source.py 2015-03-25 16:09:59.933965077 +0300
@@ -19,7 +19,7 @@
import datetime
import os.path
import re
-
+import os
import auparse
__all__ = ('ClientEventSource', 'ClientWithRotatedEventSource',
@@ -95,7 +95,7 @@
'''A source of audit events, reading from an auparse parser.'''
def read_events(self, filters, wanted_fields, want_other_fields,
- keep_raw_records):
+ keep_raw_records, direction = None, tab = None):
'''Return a sequence of audit events read from parser.
Use filters to select events. Store wanted_fields in event.fields, the
@@ -265,6 +265,106 @@
def _create_parser(self):
return auparse.AuParser(auparse.AUSOURCE_BUFFER, self.str)
+class UpdatableEventSourceReader():
+
+ '''A separate reader of audit events, created for each tab of main window.
+ To be used with UpdatableEventSource.
+
+ '''
+
+ def __init__(self, event_source, tab):
+ self.event_source = event_source
+ self.tab = tab
+ self.chunk_size = event_source.chunk_size
+ self.up_file = open(event_source.base_file)
+ self.bottom_file = open(event_source.base_file)
+ self.up_file.seek(0, 2)
+ self.up_pos = self.down_pos = self.up_file.tell()
+
+ def read_down(self):
+ ''' read older lines from file starting from self.down_pos'''
+ if os.path.exists(self.bottom_file.name):
+ if os.stat(self.bottom_file.name).st_ino != os.fstat(self.bottom_file.fileno()).st_ino:
+ self.bottom_file.close()
+ self.tab.want_read_down = False
+ return ''
+ if self.down_pos <= self.event_source.avg_line_length * self.chunk_size:
+ self.bottom_file.seek(self.down_pos)
+ lines = self.bottom_file.read(self.down_pos)
+ try:
+ files = os.listdir(os.path.dirname(self.event_source.base_file))
+ files = sorted(files, key = lambda x : os.stat(x).st_mtime)
+ filename = self.bottom_file.name
+ self.bottom_file.close()
+ self.bottom_file = open(files[files.index(filename) + 1])
+ except:
+ self.tab.want_read_down = False
+ return ''
+ else:
+ self.bottom_file.seek(self.down_pos - self.event_source.avg_line_length * self.chunk_size)
+ lines = self.bottom_file.read(self.event_source.avg_line_length * self.chunk_size)
+ lines = lines[lines.find('\n') + 1:]
+ self.down_pos -= len(lines)
+ return lines
+
+ def read_up(self):
+ '''try to read new lines if there any after the previous read'''
+ if os.path.exists(self.up_file.name):
+ if os.stat(self.up_file.name).st_ino != os.fstat(self.up_file.fileno()).st_ino:
+ self.up_file.close()
+ self.up_file = open(self.event_source.base_file)
+ self.up_pos = 0
+ else:
+ self.up_file.close()
+ self.tab.want_read_up = False
+ return ''
+ self.up_file.seek(0, 2)
+ file_end_pos = self.up_file.tell()
+ if self.up_pos != file_end_pos:
+ self.up_file.seek(self.up_pos)
+ if file_end_pos - self.up_pos > self.event_source.avg_line_length*self.chunk_size*5:
+ lines = self.up_file.read(self.event_source.avg_line_length*self.chunk_size)
+ file_end_pos = self.up_file.tell()
+ else:
+ lines = self.up_file.read()
+ if lines.endswith('\n'):
+ self.up_pos = file_end_pos
+ else:
+ lines = lines[:lines.rfind('\n') + 1]
+ self.up_pos += len(lines)
+ return lines
+ return ''
+
+
+class UpdatableEventSource(_ParserEventSource):
+
+ def __init__(self, base_file, chunk_size = 50, avg_line_length = 74):
+ self.chunk_size = chunk_size
+ self.avg_line_length = avg_line_length
+ self.readers = {}
+ self.base_file = base_file
+ self.current_lines = ''
+
+ def add_tab(self, tab):
+ self.readers[tab] = UpdatableEventSourceReader(self, tab)
+
+ def remove_tab(self, tab):
+ del self.readers[tab]
+
+ def read_events(self, filters, wanted_fields, want_other_fields,
+ keep_raw_records, direction, tab):
+ if direction == 'up':
+ self.current_lines = self.readers[tab].read_up()
+ else:
+ self.current_lines = self.readers[tab].read_down()
+ return _ParserEventSource.read_events(self, filters, wanted_fields, want_other_fields,
+ keep_raw_records)
+
+
+ def _create_parser(self):
+ return auparse.AuParser(auparse.AUSOURCE_BUFFER, self.current_lines)
+
+
def check_expression(expr):
'''Check expr.
diff -u -X ex or_src/list_tab.py oav/src/list_tab.py
--- or_src/list_tab.py 2009-12-19 10:00:00.000000000 +0300
+++ oav/src/list_tab.py 2015-03-25 15:42:07.797941743 +0300
@@ -28,6 +28,7 @@
from search_entry import SearchEntry
from tab import Tab
import util
+import event_source
__all__ = ('ListTab')
@@ -130,7 +131,7 @@
date_column_label = '__audit_viewer_date'
__list_number = 1
- def __init__(self, filters, main_window, will_refresh = False):
+ def __init__(self, filters, main_window, will_refresh = True):
Tab.__init__(self, filters, main_window, 'list_vbox')
# date_column_label == event date, None == all other columns
@@ -157,6 +158,13 @@
util.connect_and_run(self.selection, 'changed',
self.__selection_changed)
+ if isinstance(self.main_window.event_source, event_source.UpdatableEventSource):
+ self.main_window.event_source.add_tab(self)
+ self.want_read_up = True
+ self.want_read_down = True
+ self.__refresh_dont_read_events = False
+ self.refresh(True, 'up', True)
+ return
self.__refresh_dont_read_events = will_refresh
self.refresh()
self.__refresh_dont_read_events = False
@@ -191,20 +199,21 @@
% (util.filename_to_utf8(filename),
e.strerror))
- def refresh(self):
- event_sequence = self.__refresh_get_event_sequence()
+ def refresh(self, updatable = False, direction = None, init = False):
+ event_sequence = self.__refresh_get_event_sequence(direction, self)
if event_sequence is None:
return
-
- if self.filters:
- t = _(', ').join(f.ui_text() for f in self.filters)
- else:
- t = _('None')
- self.list_filter_label.set_text(t)
- self.__refresh_update_tree_view()
+ if not updatable or init:
+ if self.filters:
+ t = _(', ').join(f.ui_text() for f in self.filters)
+ else:
+ t = _('None')
+ self.list_filter_label.set_text(t)
+ self.__refresh_update_tree_view()
events = self.__refresh_collect_events(event_sequence)
- self.__refresh_update_store(events)
+ self.__refresh_update_store(events, updatable, direction)
+
def report_on_view(self):
self.main_window.new_report_tab(self.filters)
@@ -462,7 +471,7 @@
for record in event.records
for (key, value) in record.fields]))
- def __refresh_get_event_sequence(self):
+ def __refresh_get_event_sequence(self, direction = None, tab = None):
'''Return an event sequence (as if from self.main_window.read_events()).
Return None on error.
@@ -480,7 +489,7 @@
elif title is not self.date_column_label:
wanted_fields.add(title)
return self.main_window.read_events(self.filters, wanted_fields,
- want_other_fields, True)
+ want_other_fields, True, direction, tab)
def __refresh_update_tree_view(self):
'''Update self.list_tree_view for current configuration.
@@ -560,7 +569,32 @@
events.sort(key = lambda event: event[0], reverse = self.sort_reverse)
return events
- def __refresh_update_store(self, events):
+ def __insert_row(self, event, direction):
+ ''' insert new row with event into self.store preserving sort order'''
+ it = None
+ if not self.sort_by:
+ if self.sort_reverse:
+ if direction == 'up':
+ it = self.store.insert(0, event[1])
+ else:
+ it = self.store.append(event[1])
+ else:
+ if direction == 'down':
+ it = self.store.insert(0, event[1])
+ else:
+ it = self.store.append(event[1])
+ else:
+ sort_field = self.__field_columns.index(self.sort_by) + 1
+ for i in range(len(self.store)):
+ if event[1][sort_field] <= self.store[i][sort_field] and self.sort_reverse or \
+ event[1][sort_field] > self.store[i][sort_field] and not self.sort_reverse:
+ it = self.store.insert(i, event[1])
+ break
+ if not it:
+ it = self.store.append(event[1])
+ return it
+
+ def __refresh_update_store(self, events, updatable = False, direction = None):
'''Update self.store and related data.
events is the result of self.__refresh_collect_events().
@@ -571,11 +605,15 @@
key = pos.event_key
l = positions_for_event_key.setdefault(key, [])
l.append(pos)
- self.store.clear()
+ if not updatable:
+ self.store.clear()
if (self.text_filter is None and
len(positions_for_event_key) == 0): # Fast path
for event in events:
- self.store.append(event[1])
+ if not updatable:
+ self.store.append(event[1])
+ else:
+ self.__insert_row(event, direction)
else:
event_to_it = {}
text_filter = self.text_filter
@@ -604,7 +642,10 @@
or (self.__other_column_event_text(event_tuple[0]).
find(self.text_filter) == -1))):
continue
- it = self.store.append(event_tuple)
+ if not updatable:
+ it = self.store.append(event_tuple)
+ else:
+ it = self.__insert_row(event, direction)
event_id = event_tuple[0].id
key = (event_id.serial, event_id.sec, event_id.milli)
if key in positions_for_event_key:
diff -u -X ex or_src/main.py oav/src/main.py
--- or_src/main.py 2008-06-26 00:17:59.000000000 +0400
+++ oav/src/main.py 2015-03-25 15:31:29.663932132 +0300
@@ -29,6 +29,7 @@
from main_window import MainWindow
import settings
import util
+import event_source
_ = gettext.gettext
@@ -48,12 +49,20 @@
help = _('do not attempt to start the privileged backend '
'for reading system audit logs'))
parser.set_defaults(unprivileged = False)
+ parser.add_option('-p', '--updatable', action = 'store_true',
+ dest = 'updatable',
+ help = _('read new lines from log '))
+ parser.set_defaults(updatable = False)
+ parser.add_option('-s', '--source', type = 'string',
+ dest = 'source',
+ help = _('path to log file '))
(options, args) = parser.parse_args()
gnome.init(settings.gettext_domain, settings.version)
gtk.glade.bindtextdomain(settings.gettext_domain, settings.localedir)
gtk.glade.textdomain(settings.gettext_domain)
+ ev_source = None
if options.unprivileged:
cl = None
else:
@@ -66,7 +75,9 @@
sys.exit(1)
except client.ClientNotAvailableError:
cl = None
+ if options.updatable:
+ ev_source = event_source.UpdatableEventSource(options.source)
- w = MainWindow(cl)
+ w = MainWindow(cl, ev_source)
if w.setup_initial_window(args):
gtk.main()
diff -u -X ex or_src/main_window.py oav/src/main_window.py
--- or_src/main_window.py 2008-08-19 14:38:16.000000000 +0400
+++ oav/src/main_window.py 2015-03-23 19:13:04.399266459 +0300
@@ -135,6 +135,8 @@
'''
try:
+ if isinstance(self.event_source, event_source.UpdatableEventSource):
+ self.updater = gobject.idle_add(self.__refresh_all_tabs, True)
if isinstance(self.event_source, event_source.EmptyEventSource):
self.__event_error_report_only_one_push()
if self.client is not None:
@@ -246,7 +248,7 @@
return (filename, extension)
def read_events(self, filters, wanted_fields, want_other_fields,
- keep_raw_records):
+ keep_raw_records, direction = None, tab = None):
'''Read audit events.
Return a sequence of events, or None on error (without throwing
@@ -262,7 +264,7 @@
try:
return self.event_source.read_events(filters, wanted_fields,
want_other_fields,
- keep_raw_records)
+ keep_raw_records, direction, tab)
except IOError, e:
if (self.__event_error_report_only_one_depth == 0 or
not self.__event_error_reported):
@@ -381,16 +383,24 @@
'''End a region in which only one error message should be reported.'''
self.__event_error_report_only_one_depth -= 1
- def __refresh_all_tabs(self):
+ def __refresh_all_tabs(self, updatable = False):
'''Refresh all tabs, taking care to report errors only once.'''
self.__event_error_report_only_one_push()
try:
- for page_num in xrange(self.main_notebook.get_n_pages()):
- tab = self.__tab_objects[self.main_notebook
- .get_nth_page(page_num)]
- tab.refresh()
+ if not updatable:
+ for page_num in xrange(self.main_notebook.get_n_pages()):
+ tab = self.__tab_objects[self.main_notebook
+ .get_nth_page(page_num)]
+ tab.refresh()
+ else:
+ for page_num in xrange(self.main_notebook.get_n_pages()):
+ tab = self.__tab_objects[self.main_notebook
+ .get_nth_page(page_num)]
+ tab.refresh(True, 'up')
+ tab.refresh(True, 'down')
finally:
self.__event_error_report_only_one_pop()
+ return True
def __menu_new_list_activate(self, *_):
self.new_list_tab([])
----- Исходное сообщение -----
От: "mitr" <mitr at redhat.com>
Кому: "Xeniya Muratova" <muratova at itsirius.su>
Копия: "linux-audit" <linux-audit at redhat.com>
Отправленные: Среда, 4 Март 2015 г 20:50:53
Тема: Re: log rendering in real time in audit-viewer
Hello,
> Hello Miloslav, and all the guys!
>
> We use audit-viewer for events monitoring.
> Unfortunately, if some log is rather big it takes to much time for
> audit-viewer to parse and render it.
> Besides, we need to render log updates in real time, i.e. when a new line
> appears in a log, it should appear in a viewer too.
> Can you suggest the better way to extend audit-viewer to meet these
> requirements?
Well, write the code? Something like inotify could be useful. There isn’t any hidden switch to enable these features, if that is what you are asking.
As for performance, I may have missed something but I think I have squeezed as much as can be done with Python; improving performance further would very likely require a C extension.
(audit-viewer is a PyGtk2 application, and at I’m afraid I don’t currently have plans to port it to GTK+3/gobject-introspection or do any other non-trivial work on the project, at least in the near term.)
Mirek
More information about the Linux-audit
mailing list