Added raw-dir option

This option can be used to debug slack API responses - if set,
slack-backup will dump all of the data as a JSON in provided direcotry.

README.rst

@@ -90,6 +90,10 @@ where:
   created, but you'll (obviously) lost all the records. Besides the db file,
   assets directory might be created for downloadable items.
 
+You can also specify directory, where pure response JSONs from Slack API will 
+be stored by using ``-r/--raw-dir`` or by providing it in config file in 
+``fetch`` section. This might be useful for debugging purposes.
+
 There is one more switch to take into consideration -
 ``-f/--url_file_to_attachment`` which influence the way how external file
 share would be treated. First of all, what is *external* file share from slack

slack_backup/client.py

@@ -1,6 +1,7 @@
 """
 Create backup for certain date for specified channel in slack
 """
+from datetime import datetime
 import getpass
 import json
 import logging
@@ -23,6 +24,8 @@ class Client(object):
     This class is intended to provide an interface for getting, storing and
     querying data fetched out using Slack API.
     """
+    RAW = '%Y%m%d%H%M%S_{name}.json'
+
     def __init__(self, args):
         if 'token' in args:
             self.slack = slackclient.SlackClient(args.token)
@@ -44,6 +47,13 @@ class Client(object):
         self.selected_channels = args.channels
         self.q = self.session.query
 
+        self._raw_fname = None
+        if 'raw_dir' in args and args.raw_dir:
+            if not os.path.exists(args.raw_dir):
+                os.mkdir(args.raw_dir)
+            fpath = os.path.join(args.raw_dir, self.RAW)
+            self._raw_fname = datetime.now().strftime(fpath)
+
         if 'format' in args:
             self.reporter = reporters.get_reporter(args, self.q)
 
@@ -72,6 +82,10 @@ class Client(object):
         if not result:
             return
 
+        if self._raw_fname:
+            with open(self._raw_fname.format("channels"), "w") as fobj:
+                fobj.write(json.dumps(result))
+
         for data in result:
             channel = self.q(o.Channel).\
                 filter(o.Channel.slackid == data['id']).one_or_none()
@@ -91,6 +105,10 @@ class Client(object):
         if not result:
             return
 
+        if self._raw_fname:
+            with open(self._raw_fname.format("users"), "w") as fobj:
+                fobj.write(json.dumps(result))
+
         for user_data in result:
             user = self.q(o.User).\
                 filter(o.User.slackid == user_data['id']).one_or_none()
@@ -134,9 +152,11 @@ class Client(object):
             # starting from first January 1970.
             latest = latest and latest.ts or 1
 
+            result = []
             while True:
                 logging.debug("Fetching another portion of messages")
                 messages, latest = self._channels_history(channel, latest)
+                result.extend(messages)
 
                 for msg in messages:
                     self._create_message(msg, channel)
@@ -144,6 +164,11 @@ class Client(object):
                 if latest is None:
                     break
 
+            if self._raw_fname:
+                with open(self._raw_fname.format("channel-" + channel.name),
+                          "w") as fobj:
+                    fobj.write(json.dumps(result))
+
         self.session.commit()
 
     def generate_history(self):
@@ -184,6 +209,12 @@ class Client(object):
             user.real_name = result['bot']['name']
             self.session.add(user)
             self.session.flush()
+
+            if self._raw_fname:
+                with open(self._raw_fname.format('bot-' + user.slackid),
+                          "w") as fobj:
+                    fobj.write(json.dumps(result))
+
             return user
 
         logging.exception('Failed on data: %s', pprint.pformat(data))

slack_backup/command.py

@@ -94,6 +94,8 @@ def main():
                        help='Path to the database file.')
     fetch.add_argument('-i', '--config', default=None,
                        help='Use specific config file.')
+    fetch.add_argument('-r', '--raw-dir', default=None,
+                       help='Write raw responses to provided directory.')
     fetch.add_argument('-f', '--url_file_to_attachment', default=False,
                        action='store_true',
                        help='Treat shared files (but not uploaded to the '

slack_backup/config.py

@@ -17,7 +17,7 @@ class Config(object):
 
     sections = {'common': ['channels', 'database', 'quiet', 'verbose'],
                 'fetch': ['user', 'password', 'team', 'token',
-                          'url_file_to_attachment'],
+                          'url_file_to_attachment', 'raw_dir'],
                 'generate': ['output', 'format', 'theme']}
 
     def __init__(self):
@@ -38,7 +38,8 @@ class Config(object):
                          'output': None,
                          'format': None,
                          'theme': None,
-                         'url_file_to_attachment': False}
+                         'url_file_to_attachment': False,
+                         'raw_dir': None}
         # This message supposed to be displayed in INFO level. During the time
         # of running the code where it should be displayed there is no
         # complete information about logging level. Displaying message is

tests/test_config.py

@@ -64,7 +64,8 @@ class TestConfig(unittest.TestCase):
                                           'password': None,
                                           'team': None,
                                           'token': None,
-                                          'url_file_to_attachment': False})
+                                          'url_file_to_attachment': False,
+                                          'raw_dir': None})
 
         args = argparse.Namespace()
         args.config = self.confname
@@ -99,7 +100,8 @@ class TestConfig(unittest.TestCase):
                                           'token': 'xxxx-1111111111-'
                                                    '222222222222-333333333333-'
                                                    'r4nd0ms7uff',
-                                          'url_file_to_attachment': False})
+                                          'url_file_to_attachment': False,
+                                          'raw_dir': None})
 
         # override some conf options with commandline
         args = argparse.Namespace()
@@ -127,4 +129,5 @@ class TestConfig(unittest.TestCase):
                                           'password': 'ultricies',
                                           'team': '',
                                           'token': 'the token',
-                                          'url_file_to_attachment': False})
+                                          'url_file_to_attachment': False,
+                                          'raw_dir': None})