From 93b0bc2dd75f5cb5ae72ed4fbf1b9b513c849a82 Mon Sep 17 00:00:00 2001 From: gryf Date: Mon, 16 Jul 2018 16:11:23 +0200 Subject: [PATCH] Added raw-dir option This option can be used to debug slack API responses - if set, slack-backup will dump all of the data as a JSON in provided direcotry. --- README.rst | 4 ++++ slack_backup/client.py | 31 +++++++++++++++++++++++++++++++ slack_backup/command.py | 2 ++ slack_backup/config.py | 5 +++-- tests/test_config.py | 9 ++++++--- 5 files changed, 46 insertions(+), 5 deletions(-) diff --git a/README.rst b/README.rst index 3652b6d..da7b84b 100644 --- a/README.rst +++ b/README.rst @@ -90,6 +90,10 @@ where: created, but you'll (obviously) lost all the records. Besides the db file, assets directory might be created for downloadable items. +You can also specify directory, where pure response JSONs from Slack API will +be stored by using ``-r/--raw-dir`` or by providing it in config file in +``fetch`` section. This might be useful for debugging purposes. + There is one more switch to take into consideration - ``-f/--url_file_to_attachment`` which influence the way how external file share would be treated. First of all, what is *external* file share from slack diff --git a/slack_backup/client.py b/slack_backup/client.py index a147e42..837fdc3 100644 --- a/slack_backup/client.py +++ b/slack_backup/client.py @@ -1,6 +1,7 @@ """ Create backup for certain date for specified channel in slack """ +from datetime import datetime import getpass import json import logging @@ -23,6 +24,8 @@ class Client(object): This class is intended to provide an interface for getting, storing and querying data fetched out using Slack API. """ + RAW = '%Y%m%d%H%M%S_{name}.json' + def __init__(self, args): if 'token' in args: self.slack = slackclient.SlackClient(args.token) @@ -44,6 +47,13 @@ class Client(object): self.selected_channels = args.channels self.q = self.session.query + self._raw_fname = None + if 'raw_dir' in args and args.raw_dir: + if not os.path.exists(args.raw_dir): + os.mkdir(args.raw_dir) + fpath = os.path.join(args.raw_dir, self.RAW) + self._raw_fname = datetime.now().strftime(fpath) + if 'format' in args: self.reporter = reporters.get_reporter(args, self.q) @@ -72,6 +82,10 @@ class Client(object): if not result: return + if self._raw_fname: + with open(self._raw_fname.format("channels"), "w") as fobj: + fobj.write(json.dumps(result)) + for data in result: channel = self.q(o.Channel).\ filter(o.Channel.slackid == data['id']).one_or_none() @@ -91,6 +105,10 @@ class Client(object): if not result: return + if self._raw_fname: + with open(self._raw_fname.format("users"), "w") as fobj: + fobj.write(json.dumps(result)) + for user_data in result: user = self.q(o.User).\ filter(o.User.slackid == user_data['id']).one_or_none() @@ -134,9 +152,11 @@ class Client(object): # starting from first January 1970. latest = latest and latest.ts or 1 + result = [] while True: logging.debug("Fetching another portion of messages") messages, latest = self._channels_history(channel, latest) + result.extend(messages) for msg in messages: self._create_message(msg, channel) @@ -144,6 +164,11 @@ class Client(object): if latest is None: break + if self._raw_fname: + with open(self._raw_fname.format("channel-" + channel.name), + "w") as fobj: + fobj.write(json.dumps(result)) + self.session.commit() def generate_history(self): @@ -184,6 +209,12 @@ class Client(object): user.real_name = result['bot']['name'] self.session.add(user) self.session.flush() + + if self._raw_fname: + with open(self._raw_fname.format('bot-' + user.slackid), + "w") as fobj: + fobj.write(json.dumps(result)) + return user logging.exception('Failed on data: %s', pprint.pformat(data)) diff --git a/slack_backup/command.py b/slack_backup/command.py index 33ae62b..54750bd 100644 --- a/slack_backup/command.py +++ b/slack_backup/command.py @@ -94,6 +94,8 @@ def main(): help='Path to the database file.') fetch.add_argument('-i', '--config', default=None, help='Use specific config file.') + fetch.add_argument('-r', '--raw-dir', default=None, + help='Write raw responses to provided directory.') fetch.add_argument('-f', '--url_file_to_attachment', default=False, action='store_true', help='Treat shared files (but not uploaded to the ' diff --git a/slack_backup/config.py b/slack_backup/config.py index 55f196f..993a710 100644 --- a/slack_backup/config.py +++ b/slack_backup/config.py @@ -17,7 +17,7 @@ class Config(object): sections = {'common': ['channels', 'database', 'quiet', 'verbose'], 'fetch': ['user', 'password', 'team', 'token', - 'url_file_to_attachment'], + 'url_file_to_attachment', 'raw_dir'], 'generate': ['output', 'format', 'theme']} def __init__(self): @@ -38,7 +38,8 @@ class Config(object): 'output': None, 'format': None, 'theme': None, - 'url_file_to_attachment': False} + 'url_file_to_attachment': False, + 'raw_dir': None} # This message supposed to be displayed in INFO level. During the time # of running the code where it should be displayed there is no # complete information about logging level. Displaying message is diff --git a/tests/test_config.py b/tests/test_config.py index 1c7e51b..f75ef0f 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -64,7 +64,8 @@ class TestConfig(unittest.TestCase): 'password': None, 'team': None, 'token': None, - 'url_file_to_attachment': False}) + 'url_file_to_attachment': False, + 'raw_dir': None}) args = argparse.Namespace() args.config = self.confname @@ -99,7 +100,8 @@ class TestConfig(unittest.TestCase): 'token': 'xxxx-1111111111-' '222222222222-333333333333-' 'r4nd0ms7uff', - 'url_file_to_attachment': False}) + 'url_file_to_attachment': False, + 'raw_dir': None}) # override some conf options with commandline args = argparse.Namespace() @@ -127,4 +129,5 @@ class TestConfig(unittest.TestCase): 'password': 'ultricies', 'team': '', 'token': 'the token', - 'url_file_to_attachment': False}) + 'url_file_to_attachment': False, + 'raw_dir': None})