diff --git a/slack_backup/client.py b/slack_backup/client.py index 228a355..8ebbd54 100644 --- a/slack_backup/client.py +++ b/slack_backup/client.py @@ -4,12 +4,14 @@ Create backup for certain date for specified channel in slack from datetime import datetime import getpass import logging +import os import slackclient from slack_backup import db from slack_backup import objects as o from slack_backup import download +from slack_backup import reporters class Client(object): @@ -18,23 +20,28 @@ class Client(object): querying data fetched out using Slack API. """ def __init__(self, args): - self.slack = slackclient.SlackClient(args.token) + if 'token' in args: + self.slack = slackclient.SlackClient(args.token) + self.user = args.user + self.password = args.password + if not self.user and not self.password: + logging.warning('No media will be downloaded, due to not ' + 'providing credentials for a slack account') + elif not self.user and self.password: + logging.warning('No media will be downloaded, due to not ' + 'providing username for a slack account') + elif self.user and not self.password: + self.password = getpass.getpass(prompt='Provide password for ' + 'your slack account: ') + dbpath = self._get_asset_dir(args.database) + self.downloader = download.Download(args, dbpath) self.engine = db.connect(args.database) self.session = db.Session() self.selected_channels = args.channels - self.user = args.user - self.password = args.password - if not self.user and not self.password: - logging.warning('No media will be downloaded, due to not ' - 'providing credentials for a slack account') - elif not self.user and self.password: - logging.warning('No media will be downloaded, due to not ' - 'providing username for a slack account') - elif self.user and not self.password: - self.password = getpass.getpass(prompt='Provide password for ' - 'your slack account: ') self.q = self.session.query - self.downloader = download.Download(args) + + if 'format' in args: + self.reporter = reporters.get_reporter(args, self.q) def update(self): """ @@ -117,6 +124,7 @@ class Client(object): latest = latest and latest.ts or 1 while True: + logging.debug("Fetching another portion of messages") messages, latest = self._channels_history(channel, latest) for msg in messages: @@ -131,13 +139,13 @@ class Client(object): """ Return a history accumulated in DB into desired format. Special format """ + self.reporter.generate() def _create_message(self, data, channel): """ Create message with corresponding possible metadata, like reactions, files etc. """ - logging.info("Fetching messages for channel %s", channel.name) message = o.Message(data) message.user = self.q(o.User).\ filter(o.User.slackid == data['user']).one() @@ -173,8 +181,11 @@ class Client(object): message.is_starred = True if is_external: + logging.debug("Found external file `%s'", data['url_private']) message.file.url = data['url_private'] else: + logging.debug("Found internal file `%s'", + data['url_private_download']) priv_url = data['url_private_download'] message.file.filepath = self.downloader.download(priv_url, 'file') @@ -228,6 +239,17 @@ class Client(object): channel.topic = self._get_create_obj(data['topic'], o.Topic, channel) self.session.flush() + def _get_asset_dir(self, database): + """ + Get absolute assets directory using sqlite database path as a + reference. + """ + if not database: + return 'assets' + + path = os.path.dirname(os.path.abspath(database)) + return os.path.join(path, 'assets') + def _channels_list(self): """ Get channel list using Slack API. Return list of channel data or None diff --git a/slack_backup/download.py b/slack_backup/download.py index f4e9a93..4c2d376 100644 --- a/slack_backup/download.py +++ b/slack_backup/download.py @@ -4,10 +4,11 @@ to local ones, so that sophisticated writers can make a use of it """ import logging import os -import errno import requests +from slack_backup import utils + class NotAuthorizedError(requests.HTTPError): pass @@ -16,15 +17,15 @@ class NotAuthorizedError(requests.HTTPError): class Download(object): """Download class for taking care of Slack internally uploaded files""" - def __init__(self, args): + def __init__(self, args, assets_dir): self.session = requests.session() self.team = args.team self.user = args.user self.password = args.password - self.assets_dir = args.assets + self.assets_dir = assets_dir self._files = os.path.join(self.assets_dir, 'files') self._images = os.path.join(self.assets_dir, 'images') - self._do_download = False + self._authorized = False self._hier_created = False self.cookies = {} @@ -32,8 +33,6 @@ class Download(object): """ Download asset, return local path to it """ - if not self._do_download: - return if not self._hier_created: self._create_assets_dir() @@ -44,12 +43,8 @@ class Download(object): return fname def _create_assets_dir(self): - for name in ('images', 'files'): - try: - os.makedirs(os.path.join(self.assets_dir, name)) - except OSError as err: - if err.errno != errno.EEXIST: - raise + for path in (self._files, self._images): + utils.makedirs(path) self._hier_created = True @@ -59,6 +54,11 @@ class Download(object): typemap = {'avatar': self._images, 'file': self._files} + if filetype == 'file' and not self._authorized: + logging.info("There was no (valid) credentials passed, therefore " + "file `%s' cannot be downloaded", url) + return + splitted = url.split('/') if len(splitted) == 7 and 'slack.com' in splitted[2]: @@ -72,12 +72,8 @@ class Download(object): path = typemap[filetype] if part: - try: - path = os.path.join(path, part) - os.makedirs(path) - except OSError as err: - if err.errno != errno.EEXIST: - raise + utils.makedirs(os.path.join(path, part)) + path = os.path.join(path, part) path = os.path.join(path, fname) count = 1 @@ -91,11 +87,18 @@ class Download(object): def _download(self, url, local): """Download file""" - res = self.session.get(url, stream=True) + try: + res = self.session.get(url, stream=True) + except requests.exceptions.RequestException as exc: + logging.error('Request for %s failed. Reported reason: %s', + url, exc.__doc__) + raise + with open(local, 'wb') as fobj: for chunk in res.iter_content(chunk_size=5120): if chunk: fobj.write(chunk) + logging.debug("Downloaded `%s' to `'%s'", url, local) def authorize(self): """ @@ -126,4 +129,4 @@ class Download(object): ('a-' + self.cookies['a']) in self.cookies): logging.error('Failed to login into Slack app') else: - self._do_download = True + self._authorized = True diff --git a/slack_backup/reporters.py b/slack_backup/reporters.py new file mode 100644 index 0000000..bbf29f3 --- /dev/null +++ b/slack_backup/reporters.py @@ -0,0 +1,77 @@ +""" +Reporters module. + +There are several classes for specific format reporting, and also some of the +slack conversation/convention parsers. +""" +import os +import logging + +from slack_backup import objects as o + + +class Reporter(object): + """Base reporter class""" + ext = '' + + def __init__(self, args, query): + self.out = args.output + self.q = query + + self.channels = self._get_channels(args.channels) + + def _get_channels(self, selected_channels): + """ + Retrive channels from db and return those which names matched from + selected_channels list + """ + result = [] + all_channels = self.q(o.Channel).all() + if not selected_channels: + return all_channels + + for channel in all_channels: + if channel.name in selected_channels: + result.append(channel) + + return channel + + def generate(self): + """Generate raport it's a dummmy one - for use with none reporter""" + return + + def get_log_path(self, name): + """Return relative log file name """ + return os.path.join(self.out, name + self.ext) + + def write_msg(self, message, log): + """Write message to file""" + raise NotImplementedError() + + +class TextReporter(Reporter): + """Text aka IRC reporter""" + ext = '.log' + + def generate(self): + """Generate raport""" + for channel in self.channels: + log_path = self.get_log_path(channel.name) + for message in self.q(o.Message).\ + filter(o.Message.channel == channel).\ + order_by(o.Message.ts).all(): + self.write_msg(message, log_path) + + def write_msg(self, message, log): + """Write message to file""" + + +def get_reporter(args, query): + """Return object of right reporter class""" + reporters = {'text': TextReporter} + + klass = reporters.get(args.format, Reporter) + if klass.__name__ == 'Reporter': + logging.warning('None, or wrong (%s) formatter selected, falling to' + ' None Reporter', args.format) + return klass(args, query)