Added raw-dir option

This option can be used to debug slack API responses - if set,
slack-backup will dump all of the data as a JSON in provided direcotry.

slack_backup/client.py

@@ -1,6 +1,7 @@
 """
 Create backup for certain date for specified channel in slack
 """
+from datetime import datetime
 import getpass
 import json
 import logging
@@ -23,6 +24,8 @@ class Client(object):
     This class is intended to provide an interface for getting, storing and
     querying data fetched out using Slack API.
     """
+    RAW = '%Y%m%d%H%M%S_{name}.json'
+
     def __init__(self, args):
         if 'token' in args:
             self.slack = slackclient.SlackClient(args.token)
@@ -44,6 +47,13 @@ class Client(object):
         self.selected_channels = args.channels
         self.q = self.session.query
 
+        self._raw_fname = None
+        if 'raw_dir' in args and args.raw_dir:
+            if not os.path.exists(args.raw_dir):
+                os.mkdir(args.raw_dir)
+            fpath = os.path.join(args.raw_dir, self.RAW)
+            self._raw_fname = datetime.now().strftime(fpath)
+
         if 'format' in args:
             self.reporter = reporters.get_reporter(args, self.q)
 
@@ -72,6 +82,10 @@ class Client(object):
         if not result:
             return
 
+        if self._raw_fname:
+            with open(self._raw_fname.format("channels"), "w") as fobj:
+                fobj.write(json.dumps(result))
+
         for data in result:
             channel = self.q(o.Channel).\
                 filter(o.Channel.slackid == data['id']).one_or_none()
@@ -91,6 +105,10 @@ class Client(object):
         if not result:
             return
 
+        if self._raw_fname:
+            with open(self._raw_fname.format("users"), "w") as fobj:
+                fobj.write(json.dumps(result))
+
         for user_data in result:
             user = self.q(o.User).\
                 filter(o.User.slackid == user_data['id']).one_or_none()
@@ -134,9 +152,11 @@ class Client(object):
             # starting from first January 1970.
             latest = latest and latest.ts or 1
 
+            result = []
             while True:
                 logging.debug("Fetching another portion of messages")
                 messages, latest = self._channels_history(channel, latest)
+                result.extend(messages)
 
                 for msg in messages:
                     self._create_message(msg, channel)
@@ -144,6 +164,11 @@ class Client(object):
                 if latest is None:
                     break
 
+            if self._raw_fname:
+                with open(self._raw_fname.format("channel-" + channel.name),
+                          "w") as fobj:
+                    fobj.write(json.dumps(result))
+
         self.session.commit()
 
     def generate_history(self):
@@ -184,6 +209,12 @@ class Client(object):
             user.real_name = result['bot']['name']
             self.session.add(user)
             self.session.flush()
+
+            if self._raw_fname:
+                with open(self._raw_fname.format('bot-' + user.slackid),
+                          "w") as fobj:
+                    fobj.write(json.dumps(result))
+
             return user
 
         logging.exception('Failed on data: %s', pprint.pformat(data))

slack_backup/command.py

@@ -94,6 +94,8 @@ def main():
                        help='Path to the database file.')
     fetch.add_argument('-i', '--config', default=None,
                        help='Use specific config file.')
+    fetch.add_argument('-r', '--raw-dir', default=None,
+                       help='Write raw responses to provided directory.')
     fetch.add_argument('-f', '--url_file_to_attachment', default=False,
                        action='store_true',
                        help='Treat shared files (but not uploaded to the '

slack_backup/config.py

@@ -17,7 +17,7 @@ class Config(object):
 
     sections = {'common': ['channels', 'database', 'quiet', 'verbose'],
                 'fetch': ['user', 'password', 'team', 'token',
-                          'url_file_to_attachment'],
+                          'url_file_to_attachment', 'raw_dir'],
                 'generate': ['output', 'format', 'theme']}
 
     def __init__(self):
@@ -38,7 +38,8 @@ class Config(object):
                          'output': None,
                          'format': None,
                          'theme': None,
-                         'url_file_to_attachment': False}
+                         'url_file_to_attachment': False,
+                         'raw_dir': None}
         # This message supposed to be displayed in INFO level. During the time
         # of running the code where it should be displayed there is no
         # complete information about logging level. Displaying message is