From 352a88dccefed70c303e139dfd05027b6bd12060 Mon Sep 17 00:00:00 2001 From: gryf Date: Sat, 26 Nov 2016 15:10:19 +0100 Subject: [PATCH] Implementation of TextReporter --- slack_backup/reporters.py | 216 ++++++++++++++++++++++++++++++++++---- tests/test_reporter.py | 51 +++++++++ 2 files changed, 246 insertions(+), 21 deletions(-) create mode 100644 tests/test_reporter.py diff --git a/slack_backup/reporters.py b/slack_backup/reporters.py index 6113fb4..5591a4d 100644 --- a/slack_backup/reporters.py +++ b/slack_backup/reporters.py @@ -5,8 +5,11 @@ Reporters module. There are several classes for specific format reporting, and also some of the slack conversation/convention parsers. """ +from __future__ import absolute_import, division, print_function import os +import errno import logging +import re from slack_backup import objects as o from slack_backup import utils @@ -18,9 +21,51 @@ class Reporter(object): def __init__(self, args, query): self.out = args.output + self.theme = args.theme self.q = query + self.types = {"channel_join": self._msg_join, + "channel_leave": self._msg_leave, + "channel_topic": self._msg_topic, + "file_share": self._msg_file, + "me_message": self._msg_me} + self.symbols = {'plain': {'join': '->', + 'leave': '<-', + 'me': '*', + 'file': '-', + 'topic': '+', + 'separator': '|'}, + 'unicode': {'join': '⮊', + 'leave': '⮈', + 'me': '🟊', + 'file': '📂', + 'topic': '🟅', + 'separator': '│'}} self.channels = self._get_channels(args.channels) + self.users = self.q(o.User).all() + self._re_first_idnick = re.compile(r'^(?P' + r'<@(?PU[A-Z,0-9]+)\|.+>)') + self._re_first_id = re.compile('^(?P' + '<@(?PU[A-Z,0-9]+)>)') + self._re_idnick = re.compile(r'.*(?P' + r'<@(?PU[A-Z,0-9]+)\|.+>)') + self._re_id = re.compile('.*(?P<@(?PU[A-Z,0-9]+)>)') + + def generate(self): + """Generate raport it's a dummmy one - for use with none reporter""" + return + + def get_log_path(self, name): + """Return relative log file name """ + return os.path.join(self.out, name + self.ext) + + def write_msg(self, message, log): + """Write message to file""" + raise NotImplementedError() + + def _get_symbol(self, item): + """Return appropriate item depending on the selected theme""" + return self.symbols[self.theme][item] def _get_channels(self, selected_channels): """ @@ -36,19 +81,29 @@ class Reporter(object): if channel.name in selected_channels: result.append(channel) - return result - - def generate(self): - """Generate raport it's a dummmy one - for use with none reporter""" + def _msg_join(self, msg, text): + """return formatter for join""" return - def get_log_path(self, name): - """Return relative log file name """ - return os.path.join(self.out, name + self.ext) + def _msg_leave(self, msg, text): + """return formatter for leave""" + return - def write_msg(self, message, log): - """Write message to file""" - raise NotImplementedError() + def _msg_topic(self, msg, text): + """return formatter for set topic""" + return + + def _msg_me(self, msg, text): + """return formatter for /me""" + return + + def _msg_file(self, msg, text): + """return formatter for /me""" + return + + def _filter_slackid(self, text): + """filter out all of the id from slack""" + return class TextReporter(Reporter): @@ -56,13 +111,9 @@ class TextReporter(Reporter): ext = '.log' def __init__(self, args, query): - super().__init__(args, query) + super(TextReporter, self).__init__(args, query) utils.makedirs(self.out) - self._line = "" - - def _prepare_line(self, channel): - users = [m.user for m in channel.messages] - users = set([u.name for u in users]) + self._max_len = 0 return @@ -70,7 +121,12 @@ class TextReporter(Reporter): """Generate raport""" for channel in self.channels: log_path = self.get_log_path(channel.name) - self._prepare_line(channel) + self._set_max_len(channel) + try: + os.unlink(log_path) + except IOError as err: + if err.errno != errno.ENOENT: + raise for message in self.q(o.Message).\ filter(o.Message.channel == channel).\ order_by(o.Message.ts).all(): @@ -81,15 +137,133 @@ class TextReporter(Reporter): with open(log, "a") as fobj: fobj.write(self._format_message(message)) + def _set_max_len(self, channel): + """calculate max_len for sepcified channel""" + users = [m.user for m in channel.messages] + users = set([u.name for u in users]) + + self._max_len = 0 + for user_name in users: + if len(user_name) > self._max_len: + self._max_len = len(user_name) + def _format_message(self, msg): """ Check what kind of message we are dealing with and do appropriate formatting """ - return msg.text - return (msg.datetime().strftime("%Y-%m-%d %H:%M:%S"), - msg.user.name, - msg.text) + msg_txt = self._filter_slackid(msg.text) + msg_txt = self._fix_newlines(msg_txt) + formatter = self.types.get(msg.type, self._msg) + if not msg_txt.strip(): + logging.info("Skipping message from `%s' since it's empty", + msg.user.name) + return '' + + return formatter(msg, msg_txt) + + def _msg_join(self, msg, text): + """return formatter for join""" + data = {'date': msg.datetime().strftime("%Y-%m-%d %H:%M:%S"), + 'msg': text, + 'max_len': self._max_len, + 'separator': self._get_symbol('separator'), + 'nick': self._get_symbol('join')} + return '{date} {nick:>{max_len}} {separator} {msg}\n'.format(**data) + + def _msg_leave(self, msg, text): + """return formatter for leave""" + data = {'date': msg.datetime().strftime("%Y-%m-%d %H:%M:%S"), + 'msg': text, + 'max_len': self._max_len, + 'separator': self._get_symbol('separator'), + 'nick': self._get_symbol('leave')} + return '{date} {nick:>{max_len}} {separator} {msg}\n'.format(**data) + + def _msg_topic(self, msg, text): + """return formatter for set topic""" + data = {'date': msg.datetime().strftime("%Y-%m-%d %H:%M:%S"), + 'msg': text, + 'max_len': self._max_len, + 'separator': self._get_symbol('separator'), + 'char': self._get_symbol('topic')} + return '{date} {char:>{max_len}} {separator} {msg}\n'.format(**data) + + def _msg_me(self, msg, text): + """return formatter for /me""" + data = {'date': msg.datetime().strftime("%Y-%m-%d %H:%M:%S"), + 'msg': text, + 'max_len': self._max_len, + 'nick': msg.user.name, + 'separator': self._get_symbol('separator'), + 'char': self._get_symbol('me')} + return '{date} {char:>{max_len}} {separator} {nick} {msg}\n'.\ + format(**data) + + def _msg_file(self, msg, text): + """return formatter for file""" + groups = self._re_first_idnick.match(msg.text).groupdict() + text = msg.text.replace(groups['replace'], '') + filename = msg.file.filepath + if filename: + filename = os.path.relpath(msg.file.filepath, start=self.out) + else: + filename = msg.file.url + + if not filename: + logging.warning("Dude, we have a file object, but nothing has " + "found. Name of the file object is `i%s'", + msg.file.name) + filename = msg.file.name + + text = self._filter_slackid(text) + text = self._fix_newlines(text) + + data = {'date': msg.datetime().strftime("%Y-%m-%d %H:%M:%S"), + 'msg': text, + 'max_len': self._max_len, + 'separator': self._get_symbol('separator'), + 'filename': filename, + 'nick': msg.user.name, + 'char': self._get_symbol('file')} + return ('{date} {char:>{max_len}} {separator} {nick} ' + 'shared file "{filename}"{msg}\n'.format(**data)) + + def _msg(self, msg, text): + """return formatter for /me""" + data = {'date': msg.datetime().strftime("%Y-%m-%d %H:%M:%S"), + 'msg': text, + 'max_len': self._max_len, + 'separator': self._get_symbol('separator'), + 'nick': msg.user.name} + return '{date} {nick:>{max_len}} {separator} {msg}\n'.format(**data) + + def _filter_slackid(self, text): + """filter out all of the id from slack""" + for pat in (self._re_first_idnick, self._re_first_id): + while pat.search(text): + groups = pat.search(text).groupdict('slackid') + user = [u for u in self.users + if u.slackid == groups['slackid']][0] + text = text.replace(groups['replace'], user.name + ":") + + for pat in (self._re_idnick, self._re_id): + while pat.search(text): + groups = pat.search(text).groupdict('slackid') + user = [u for u in self.users + if u.slackid == groups['slackid']][0] + text = text.replace(groups['replace'], user.name) + + return text + + def _fix_newlines(self, text): + """Shift text with new lines to the right with separator""" + shift = 19 # length of the date + shift += 1 # separator space + shift += self._max_len # length reserved for the nicks + shift += 1 # separator space + return text.replace('\n', '\n' + shift * ' ' + + self._get_symbol('separator') + ' ') def get_reporter(args, query): diff --git a/tests/test_reporter.py b/tests/test_reporter.py new file mode 100644 index 0000000..139366d --- /dev/null +++ b/tests/test_reporter.py @@ -0,0 +1,51 @@ +# -*- coding: utf-8 -*- + +from unittest import TestCase +try: + from unittest.mock import MagicMock +except ImportError: + from mock import MagicMock + +from slack_backup import reporters as r + + +class FakeUser(object): + def __init__(self, slackid, name): + self.slackid = slackid + self.name = name + + +class TestReporter(TestCase): + + def setUp(self): + + users = [FakeUser('U111AAAAA', 'user1'), + FakeUser('U111BBBBB', 'user2'), + FakeUser('U111CCCCC', 'funky_username1'), + FakeUser('U111DDDDD', 'user-name°')] + + args = MagicMock() + args.output = 'logs' + query1 = MagicMock() + query1.all = MagicMock(return_value=users) + query = MagicMock(return_value=query1) + + self.reporter = r.TextReporter(args, query) + + def test_regexp(self): + text = 'Cras vestibulum <@U111AAAAA|user1> erat ultrices neque.' + self.assertEqual(self.reporter._filter_slackid(text), + 'Cras vestibulum user1 erat ultrices neque.') + + text = ('Cras vestibulum <@U111AAAAA|user1> erat ultrices ' + '<@U111AAAAA|user1> neque.') + self.assertEqual(self.reporter._filter_slackid(text), + 'Cras vestibulum user1 erat ultrices user1 neque.') + + text = ('<@U111BBBBB|user2>Praesent vel enim sed eros luctus ' + 'imperdiet.\nMauris neque ante, <@U111DDDDD> placerat at, ' + 'mollis vitae, faucibus quis, <@U111CCCCC>leo. Ut feugiat.') + +# Praesent vel enim sed eros luctus imperdiet. Vivamus urna quam, congue +# vulputate, convallis non, cursus cursus, risus. Quisque aliquet. Donec +# vulputate egestas elit. Morbi dictum, sem sit amet aliquam.