Added reporters module

slack_backup/client.py

@@ -4,12 +4,14 @@ Create backup for certain date for specified channel in slack
 from datetime import datetime
 import getpass
 import logging
+import os
 
 import slackclient
 
 from slack_backup import db
 from slack_backup import objects as o
 from slack_backup import download
+from slack_backup import reporters
 
 
 class Client(object):
@@ -18,10 +20,8 @@ class Client(object):
     querying data fetched out using Slack API.
     """
     def __init__(self, args):
+        if 'token' in args:
             self.slack = slackclient.SlackClient(args.token)
-        self.engine = db.connect(args.database)
-        self.session = db.Session()
-        self.selected_channels = args.channels
             self.user = args.user
             self.password = args.password
             if not self.user and not self.password:
@@ -33,8 +33,15 @@ class Client(object):
             elif self.user and not self.password:
                 self.password = getpass.getpass(prompt='Provide password for '
                                                 'your slack account: ')
+            dbpath = self._get_asset_dir(args.database)
+            self.downloader = download.Download(args, dbpath)
+        self.engine = db.connect(args.database)
+        self.session = db.Session()
+        self.selected_channels = args.channels
         self.q = self.session.query
-        self.downloader = download.Download(args)
+
+        if 'format' in args:
+            self.reporter = reporters.get_reporter(args, self.q)
 
     def update(self):
         """
@@ -117,6 +124,7 @@ class Client(object):
             latest = latest and latest.ts or 1
 
             while True:
+                logging.debug("Fetching another portion of messages")
                 messages, latest = self._channels_history(channel, latest)
 
                 for msg in messages:
@@ -131,13 +139,13 @@ class Client(object):
         """
         Return a history accumulated in DB into desired format. Special format
         """
+        self.reporter.generate()
 
     def _create_message(self, data, channel):
         """
         Create message with corresponding possible metadata, like reactions,
         files etc.
         """
-        logging.info("Fetching messages for channel %s", channel.name)
         message = o.Message(data)
         message.user = self.q(o.User).\
             filter(o.User.slackid == data['user']).one()
@@ -173,8 +181,11 @@ class Client(object):
             message.is_starred = True
 
         if is_external:
+            logging.debug("Found external file `%s'", data['url_private'])
             message.file.url = data['url_private']
         else:
+            logging.debug("Found internal file `%s'",
+                          data['url_private_download'])
             priv_url = data['url_private_download']
             message.file.filepath = self.downloader.download(priv_url, 'file')
 
@@ -228,6 +239,17 @@ class Client(object):
         channel.topic = self._get_create_obj(data['topic'], o.Topic, channel)
         self.session.flush()
 
+    def _get_asset_dir(self, database):
+        """
+        Get absolute assets directory using sqlite database path as a
+        reference.
+        """
+        if not database:
+            return 'assets'
+
+        path = os.path.dirname(os.path.abspath(database))
+        return os.path.join(path, 'assets')
+
     def _channels_list(self):
         """
         Get channel list using Slack API. Return list of channel data or None

slack_backup/download.py

@@ -4,10 +4,11 @@ to local ones, so that sophisticated writers can make a use of it
 """
 import logging
 import os
-import errno
 
 import requests
 
+from slack_backup import utils
+
 
 class NotAuthorizedError(requests.HTTPError):
     pass
@@ -16,15 +17,15 @@ class NotAuthorizedError(requests.HTTPError):
 class Download(object):
     """Download class for taking care of Slack internally uploaded files"""
 
-    def __init__(self, args):
+    def __init__(self, args, assets_dir):
         self.session = requests.session()
         self.team = args.team
         self.user = args.user
         self.password = args.password
-        self.assets_dir = args.assets
+        self.assets_dir = assets_dir
         self._files = os.path.join(self.assets_dir, 'files')
         self._images = os.path.join(self.assets_dir, 'images')
-        self._do_download = False
+        self._authorized = False
         self._hier_created = False
         self.cookies = {}
 
@@ -32,8 +33,6 @@ class Download(object):
         """
         Download asset, return local path to it
         """
-        if not self._do_download:
-            return
 
         if not self._hier_created:
             self._create_assets_dir()
@@ -44,12 +43,8 @@ class Download(object):
         return fname
 
     def _create_assets_dir(self):
-        for name in ('images', 'files'):
+        for path in (self._files, self._images):
-            try:
+            utils.makedirs(path)
-                os.makedirs(os.path.join(self.assets_dir, name))
-            except OSError as err:
-                if err.errno != errno.EEXIST:
-                    raise
 
         self._hier_created = True
 
@@ -59,6 +54,11 @@ class Download(object):
         typemap = {'avatar': self._images,
                    'file': self._files}
 
+        if filetype == 'file' and not self._authorized:
+            logging.info("There was no (valid) credentials passed, therefore "
+                         "file `%s' cannot be downloaded", url)
+            return
+
         splitted = url.split('/')
 
         if len(splitted) == 7 and 'slack.com' in splitted[2]:
@@ -72,12 +72,8 @@ class Download(object):
         path = typemap[filetype]
 
         if part:
-            try:
+            utils.makedirs(os.path.join(path, part))
             path = os.path.join(path, part)
-                os.makedirs(path)
-            except OSError as err:
-                if err.errno != errno.EEXIST:
-                    raise
 
         path = os.path.join(path, fname)
         count = 1
@@ -91,11 +87,18 @@ class Download(object):
     def _download(self, url, local):
         """Download file"""
 
+        try:
             res = self.session.get(url, stream=True)
+        except requests.exceptions.RequestException as exc:
+            logging.error('Request for %s failed. Reported reason: %s',
+                          url, exc.__doc__)
+            raise
+
         with open(local, 'wb') as fobj:
             for chunk in res.iter_content(chunk_size=5120):
                 if chunk:
                     fobj.write(chunk)
+        logging.debug("Downloaded `%s' to `'%s'", url, local)
 
     def authorize(self):
         """
@@ -126,4 +129,4 @@ class Download(object):
                 ('a-' + self.cookies['a']) in self.cookies):
             logging.error('Failed to login into Slack app')
         else:
-            self._do_download = True
+            self._authorized = True

slack_backup/reporters.py

@@ -0,0 +1,77 @@
+"""
+Reporters module.
+
+There are several classes for specific format reporting, and also some of the
+slack conversation/convention parsers.
+"""
+import os
+import logging
+
+from slack_backup import objects as o
+
+
+class Reporter(object):
+    """Base reporter class"""
+    ext = ''
+
+    def __init__(self, args, query):
+        self.out = args.output
+        self.q = query
+
+        self.channels = self._get_channels(args.channels)
+
+    def _get_channels(self, selected_channels):
+        """
+        Retrive channels from db and return those which names matched from
+        selected_channels list
+        """
+        result = []
+        all_channels = self.q(o.Channel).all()
+        if not selected_channels:
+            return all_channels
+
+        for channel in all_channels:
+            if channel.name in selected_channels:
+                result.append(channel)
+
+        return channel
+
+    def generate(self):
+        """Generate raport it's a dummmy one - for use with none reporter"""
+        return
+
+    def get_log_path(self, name):
+        """Return relative log file name """
+        return os.path.join(self.out, name + self.ext)
+
+    def write_msg(self, message, log):
+        """Write message to file"""
+        raise NotImplementedError()
+
+
+class TextReporter(Reporter):
+    """Text aka IRC reporter"""
+    ext = '.log'
+
+    def generate(self):
+        """Generate raport"""
+        for channel in self.channels:
+            log_path = self.get_log_path(channel.name)
+            for message in self.q(o.Message).\
+                    filter(o.Message.channel == channel).\
+                    order_by(o.Message.ts).all():
+                self.write_msg(message, log_path)
+
+    def write_msg(self, message, log):
+        """Write message to file"""
+
+
+def get_reporter(args, query):
+    """Return object of right reporter class"""
+    reporters = {'text': TextReporter}
+
+    klass = reporters.get(args.format, Reporter)
+    if klass.__name__ == 'Reporter':
+        logging.warning('None, or wrong (%s) formatter selected, falling to'
+                        ' None Reporter', args.format)
+    return klass(args, query)