Version bump to 0.7

Uploaded cookie check for logins

.travis.yml

@@ -1,8 +1,6 @@
 language: python
 env:
-- TOXENV=py27
 - TOXENV=py34
-- TOXENV=py27-flake8
 - TOXENV=py34-flake8
 install: pip install tox
 script: tox

MANIFEST.in

@@ -0,0 +1,2 @@
+include README.rst
+include config_example

README.rst

@@ -4,15 +4,18 @@ Slack backup
 .. image:: https://travis-ci.org/gryf/slack-backup.svg?branch=master
     :target: https://travis-ci.org/gryf/slack-backup
 
-This simple project which aim is to collect conversations from Slack using its
-API and optionally user account information, and provides convenient way to
-represent as a log.
+.. image:: https://img.shields.io/pypi/v/slack-backup.svg
+    :target: https://pypi.python.org/pypi/slack-backup
+
+The project aim is to collect conversations from Slack using its API and
+optionally user account information, and provides convenient way to represent
+as a log.
 
 Requirements
 ------------
 
-This project is written in Python 2.7, and 3.4+, although version 2.7, which
-should work, wasn't tested as extensively as it should be.
+This project is written in Python 3, 3.4 to be precise, although it may work on
+earlier version of Python3. Sorry no support for Python2.
 
 Other than that, required packages are as follows:
 
@@ -65,7 +68,7 @@ typical session:
    (myenv)user@localhost ~/mylogs $ slack-backup fetch \
    --token xxxx-1111111111-222222222222-333333333333-r4nd0ms7uff \
    --user some@email.address.org --password secret --team myteam \
-   -qqq -d mydatabase.sqlite
+   -qq -d mydatabase.sqlite
 
 where:
 
@@ -111,6 +114,87 @@ where:
 The rest of the options (``-d`` and ``-v``) have same meaning as in ``fetch``
 command.
 
+See help for the ``slack-backup`` command for complete list of options.
+
+Configuration
+-------------
+
+For convenience, you can place all of needed options into configuration file
+(aka .ini), which all options (with their defaults) will look like:
+
+.. code:: ini
+
+   [common]
+   channels =
+   database =
+   quiet = 0
+   verbose = 0
+
+   [generate]
+   output =
+   format = text
+   theme = plain
+
+   [fetch]
+   user =
+   password =
+   team =
+   token =
+
+Note, that you don't have to put every option. To illustrate ``fetch`` example
+from above, here is a corresponding config file:
+
+.. code:: ini
+
+   [common]
+   database = mydatabase.sqlite
+   quiet = 2
+
+   [fetch]
+   user = some@email.address.org
+   password = secret
+   team = myteam
+   token = xxxx-1111111111-222222222222-333333333333-r4nd0ms7uff
+
+Note, that only ``[common]`` and ``[fetch]`` sections are provided, so it is
+enough to invoke ``slack-backup`` command as:
+
+.. code:: shell-session
+
+   (myenv)user@localhost ~/mylogs $ slack-backup fetch
+
+There are couple of places, where configuration file would be searched for, in
+particular order:
+
+* file provided via argument ``-i`` or ``--config``
+* ``slack-backup.ini`` in current directory
+* ``$XDG_CONFIG_HOME/slack-backup.ini``, where ``$XDG_CONFIG_HOME`` usually
+  defaults to ``$HOME/.config``
+
+Details
+-------
+
+During first run, database with provided name is generated. For ease of use
+sqlite database is used, although it is easy to switch the engine, since there
+is an ORM (SQLAlchemy) used.
+
+Slack users, channels and messages are mapped to SQLAlchemy models, as well as
+other information, like:
+
+- user profiles
+- channel topic
+- channel purpose
+- message reactions
+- message attachments
+- and files
+
+Channels and users are always synchronized in every run, so every modification
+to the user or channels are overwriting old data. During first run, all messages
+are retrieved for all/selected channels. Every other run will only fetch those
+messages, which are older then newest message in the database - so that we don't
+loose any old messages, which might be automatically removed from Slack servers.
+The drawback of this behaviour is that all past messages which was altered in
+the meantime will not be updated.
 
 License
 -------

scripts/slack-backup

@@ -1,108 +1,10 @@
 #!/usr/bin/env python3
 # -*- coding: utf-8 -*-
 """
-Create backup for certain date for specified channel in slack
+Execute commands for slack-backup
 """
-import argparse
-import logging
-
-from slack_backup import client
-
-
-def setup_logger(args):
-    """Setup logger format and level"""
-
-    level = logging.WARNING
-
-    if args.quiet:
-        level = logging.ERROR
-        if args.quiet > 1:
-            level = logging.CRITICAL
-
-    if args.verbose:
-        level = logging.INFO
-        if args.verbose > 1:
-            level = logging.DEBUG
-
-    logging.basicConfig(level=level,
-                        format="%(asctime)s %(levelname)s: %(message)s")
-
-def generate_raport(args):
-    """Generate logs"""
-    slack = client.Client(args)
-    slack.generate_history()
-
-def fetch_data(args):
-    """Fetch and store data"""
-    slack = client.Client(args)
-    slack.update()
-
-
-def main():
-    """Main function"""
-    parser = argparse.ArgumentParser()
-
-    subparser = parser.add_subparsers(dest='parser')
-    subparser.required = True
-
-    fetch = subparser.add_parser('fetch', help='Update local db with Slack'
-                                 ' data')
-    fetch.add_argument('-t', '--token', required=True, help='Slack token - '
-                       'a string, which can be generated/obtained via '
-                       'https://api.slack.com/docs/oauth-test-tokens page.')
-    fetch.add_argument('-u', '--user', default='', help='Username for your '
-                       'Slack account')
-    fetch.add_argument('-p', '--password', default='', help='Password for your '
-                       'Slack account.')
-    fetch.add_argument('-e', '--team', required=True, help='Team name, which '
-                       'is part of slack url, for example: if url is '
-                       '"https://team.slack.com" than "team" is a name of '
-                       'the team.')
-    fetch.add_argument('-v', '--verbose', help='Be verbose. Adding more "v"'
-                       'will increase verbosity', action="count", default=0)
-    fetch.add_argument('-q', '--quiet', help='Be quiet. Adding more "q"'
-                       'will decrease verbosity', action="count", default=0)
-    fetch.add_argument('-c', '--channels', default=[], nargs='+',
-                       help='List of channels to perform actions on. '
-                       'Default is all channels.')
-
-    fetch.add_argument('-d', '--database', default='',
-                       help='Path to the database file.')
-
-    fetch.set_defaults(func=fetch_data)
-
-    generate = subparser.add_parser('generate', help='Generate logs out of '
-                                    'data in provided database')
-    generate.add_argument('-o', '--output', default='logs', help="Output "
-                          "directory for store logs. All logs are organised "
-                          "per channel. By default it's `logs' directory")
-    generate.add_argument('-f', '--format', default='none',
-                          choices=('text', 'none'),
-                          help='Output format. Default is none; only database '
-                          'is updated by latest messages for all/selected '
-                          'channels.')
-    generate.add_argument('-t', '--theme', default='plain',
-                          choices=('plain', 'unicode'),
-                          help='Choose theme for text output. It doesn\'t '
-                          'affect other output formats.')
-
-    generate.add_argument('-v', '--verbose', help='Be verbose. Adding more "v"'
-                          'will increase verbosity', action="count", default=0)
-    generate.add_argument('-q', '--quiet', help='Be quiet. Adding more "q"'
-                          'will decrease verbosity', action="count", default=0)
-    generate.add_argument('-c', '--channels', default=[], nargs='+',
-                          help='List of channels to perform actions on. '
-                          'Default is all channels.')
-
-    generate.add_argument('-d', '--database', default='',
-                          help='Path to the database file.')
-
-    generate.set_defaults(func=generate_raport)
-
-    args = parser.parse_args()
-    setup_logger(args)
-    args.func(args)
+from slack_backup import command
 
 
 if __name__ == "__main__":
-    main()
+    command.main()

setup.py

@@ -2,12 +2,15 @@
 """
 Setup for the slack-backup project
 """
-from distutils.core import setup
+try:
+    from setuptools import setup
+except ImportError:
+    from distutils.core import setup
 
 
 setup(name="slack-backup",
       packages=["slack_backup"],
-      version="0.1",
+      version="0.7",
       description="Make copy of slack converstaions",
       author="Roman Dobosz",
       author_email="gryf73@gmail.com",
@@ -18,7 +21,7 @@ setup(name="slack-backup",
       scripts=["scripts/slack-backup"],
       classifiers=["Programming Language :: Python :: 3",
                    "Programming Language :: Python :: 3.4",
-                   "Development Status :: 2 - Pre-Alpha",
+                   "Development Status :: 4 - Beta",
                    "Environment :: Console",
                    "Intended Audience :: End Users/Desktop",
                    "License :: OSI Approved :: BSD License",

slack_backup/client.py

@@ -3,6 +3,7 @@ Create backup for certain date for specified channel in slack
 """
 from datetime import datetime
 import getpass
+import json
 import logging
 import os
 
@@ -112,6 +113,7 @@ class Client(object):
             channels = all_channels
 
         for channel in channels:
+            logging.info("Getting messages for channel `%s'", channel.name)
             latest = self.q(o.Message).\
                 filter(o.Message.channel == channel).\
                 order_by(o.Message.ts.desc()).first()
@@ -146,10 +148,27 @@ class Client(object):
         Create message with corresponding possible metadata, like reactions,
         files etc.
         """
+        if data['type'] != 'message':
+            logging.info("Skipping message of type `%s'.", data['type'])
+            return
+
+        logging.debug('Message data: %s', json.dumps(data))
+
+        try:
+            user = self.q(o.User).\
+                filter(o.User.slackid == data['user']).one()
+        except KeyError:
+            user = self.q(o.User).\
+                    filter(o.User.slackid == data['comment']['user']).one()
+
+        if not data['text'].strip():
+            logging.info("Skipping message from `%s' since it's empty",
+                         user.name)
+            return
+
         message = o.Message(data)
-        message.user = self.q(o.User).\
-            filter(o.User.slackid == data['user']).one()
         message.channel = channel
+        message.user = user
 
         if data.get('is_starred'):
             message.is_starred = True
@@ -248,7 +267,7 @@ class Client(object):
         if not database:
             return 'assets'
 
-        path = os.path.dirname(os.path.abspath(database))
+        path = os.path.dirname(database)
         return os.path.join(path, 'assets')
 
     def _channels_list(self):

slack_backup/command.py

@@ -0,0 +1,141 @@
+"""
+Create backup for certain date for specified channel in slack
+"""
+import argparse
+import logging
+import platform
+
+from slack_backup import client
+from slack_backup import config
+
+
+def setup_logger(args):
+    """Setup logger format and level"""
+
+    if platform.system() != "Windows":
+        # hack to have colors in terminal
+        logging.addLevelName(logging.DEBUG,
+                             "\033[1;30m%s\033[1;0m" %
+                             logging.getLevelName(logging.DEBUG))
+
+        logging.addLevelName(logging.INFO,
+                             "\033[1;32m%s\033[1;0m" %
+                             logging.getLevelName(logging.INFO))
+
+        logging.addLevelName(logging.WARNING,
+                             "\033[1;33m%s\033[1;0m" %
+                             logging.getLevelName(logging.WARNING))
+
+        logging.addLevelName(logging.ERROR,
+                             "\033[1;31m%s\033[1;0m" %
+                             logging.getLevelName(logging.ERROR))
+
+        logging.addLevelName(logging.CRITICAL,
+                             "\033[7;31m%s\033[1;0m" %
+                             logging.getLevelName(logging.CRITICAL))
+
+    level = logging.WARNING
+
+    if args.quiet:
+        level = logging.ERROR
+        if args.quiet > 1:
+            level = logging.CRITICAL
+
+    if args.verbose:
+        level = logging.INFO
+        if args.verbose > 1:
+            level = logging.DEBUG
+
+    logging.basicConfig(level=level,
+                        format="%(asctime)s %(levelname)s: %(message)s")
+
+
+def generate_raport(args):
+    """Generate logs"""
+    slack = client.Client(args)
+    slack.generate_history()
+
+
+def fetch_data(args):
+    """Fetch and store data"""
+    slack = client.Client(args)
+    slack.update()
+
+
+def main():
+    """Main function"""
+    parser = argparse.ArgumentParser()
+
+    subparser = parser.add_subparsers(dest='parser')
+    subparser.required = True
+
+    fetch = subparser.add_parser('fetch', help='Update local db with Slack'
+                                 ' data')
+    fetch.add_argument('-t', '--token', default=None, help='Slack token - '
+                       'a string, which can be generated/obtained via '
+                       'https://api.slack.com/docs/oauth-test-tokens page.')
+    fetch.add_argument('-u', '--user', default=None, help='Username for your '
+                       'Slack account')
+    fetch.add_argument('-p', '--password', default=None, help='Password for '
+                       'your Slack account.')
+    fetch.add_argument('-e', '--team', default=None, help='Team name, which '
+                       'is part of slack url, for example: if url is '
+                       '"https://team.slack.com" than "team" is a name of '
+                       'the team.')
+    fetch.add_argument('-v', '--verbose', help='Be verbose. Adding more "v" '
+                       'will increase verbosity', action="count",
+                       default=None)
+    fetch.add_argument('-q', '--quiet', help='Be quiet. Adding more "q" will'
+                       ' decrease verbosity', action="count", default=None)
+    fetch.add_argument('-c', '--channels', default=None, nargs='+',
+                       help='List of channels to perform actions on. '
+                       'Default is all channels.')
+
+    fetch.add_argument('-d', '--database', default=None,
+                       help='Path to the database file.')
+
+    fetch.add_argument('-i', '--config', default=None,
+                       help='Use specific config file.')
+
+    fetch.set_defaults(func=fetch_data)
+
+    generate = subparser.add_parser('generate', help='Generate logs out of '
+                                    'data in provided database')
+    generate.add_argument('-o', '--output', default=None, help="Output "
+                          "directory for store logs. All logs are organised "
+                          "per channel. By default it's `logs' directory")
+    generate.add_argument('-f', '--format', default=None,
+                          choices=('text', 'none'),
+                          help='Output format. Default is none; only database '
+                          'is updated by latest messages for all/selected '
+                          'channels.')
+    generate.add_argument('-t', '--theme', default=None,
+                          choices=('plain', 'unicode'),
+                          help='Choose theme for text output. It doesn\'t '
+                          'affect other output formats.')
+
+    generate.add_argument('-v', '--verbose', help='Be verbose. Adding more '
+                          '"v" will increase verbosity', action="count",
+                          default=None)
+    generate.add_argument('-q', '--quiet', help='Be quiet. Adding more "q" '
+                          'will decrease verbosity', action="count",
+                          default=None)
+    generate.add_argument('-c', '--channels', default=[], nargs='+',
+                          help='List of channels to perform actions on. '
+                          'Default is all channels.')
+
+    generate.add_argument('-d', '--database', default=None,
+                          help='Path to the database file.')
+
+    generate.add_argument('-i', '--config', default=None,
+                          help='Use specific config file.')
+
+    generate.set_defaults(func=generate_raport)
+
+    args = parser.parse_args()
+    cfg = config.Config()
+    msg = cfg.update(args)
+    setup_logger(args)
+    logging.info(msg)
+
+    args.func(args)

slack_backup/config.py

@@ -0,0 +1,112 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+
+"""
+Configuration module for slack-backup
+"""
+import json
+import os
+import configparser
+
+
+class Config(object):
+    """Configuration keeper"""
+
+    ints = ['verbose', 'quiet']
+
+    sections = {'common': ['channels', 'database', 'quiet', 'verbose'],
+                'fetch': ['user', 'password', 'team', 'token'],
+                'generate': ['output', 'format', 'theme']}
+
+    def __init__(self):
+        """
+        Init. Read config, if exists, and update passed argument parser
+        object.
+        """
+
+        self.cp = configparser.ConfigParser()
+        self._options = {'channels': [],
+                         'database': None,
+                         'quiet': 0,
+                         'verbose': 0,
+                         'user': None,
+                         'password': None,
+                         'team': None,
+                         'token': None,
+                         'output': None,
+                         'format': None,
+                         'theme': None}
+        # This message supposed to be displayed in INFO level. During the time
+        # of running the code where it should be displayed there is no
+        # complete information about logging level. Displaying message is
+        # dependent on the a) config file, b) argument from commandline. Let's
+        # resolve if user want to have that information or not after merging
+        # those two sources. If user do not want to see any message in INFO
+        # level, we shouldn't do so.
+        self.msg = ''
+
+    def update(self, args):
+        self.load_config(args)
+        self.parse_loaded_options()
+        self.update_args(args)
+        return self.msg
+
+    def load_config(self, args):
+
+        path = ''
+        if hasattr(args, 'config') and args.config:
+            path = args.config
+
+        locations = [path,
+                     './slack-backup.ini',
+                     os.path.expandvars('$XDG_CONFIG_HOME/slack-backup.ini'),
+                     os.path.expandvars('$HOME/.config/slack-backup.ini')]
+
+        for location in locations:
+            if os.path.exists(location):
+                self.cp.read(location)
+                self.msg = 'Found configuration file: %s' % location
+                break
+        else:
+            self.msg = 'No configuration file found'
+
+    def parse_loaded_options(self):
+
+        for section in self.cp.sections():
+            if section not in self.sections:
+                continue
+
+            for option in self.sections[section]:
+                if option in self.ints:
+                    val = self.cp.getint(section, option, fallback=0)
+                elif option == 'channels':
+                    val = self.cp.get(section, option, fallback='[]')
+                    val = json.loads(val)
+                else:
+                    val = self.cp.get(section, option, fallback=None)
+
+                self._options[option] = val
+
+    def update_args(self, args):
+        if 'parser' not in args:
+            # it doesn't make sense to update args, since no action was
+            # choosen
+            return
+
+        # special case, re-set information for verbose/quiet options
+        if 'verbose' in args and args.verbose is not None:
+            self._options['verbose'] = args.verbose
+            if self._options['quiet'] is not None:
+                self._options['quiet'] = 0
+        if 'quiet' in args and args.quiet is not None:
+            self._options['quiet'] = args.quiet
+            if self._options['verbose'] is not None:
+                self._options['verbose'] = 0
+
+        for sec_id in (args.parser, 'common'):
+            for option in self.sections[sec_id]:
+                if option in args:
+                    if getattr(args, option) is not None:
+                        continue
+
+                setattr(args, option, self._options[option])

slack_backup/download.py

@@ -2,14 +2,45 @@
 Module for download files, store them in local filesystem and convert the URLs
 to local ones, so that sophisticated writers can make a use of it
 """
+import functools
 import logging
 import os
+import shutil
 
 import requests
 
 from slack_backup import utils
 
 
+def retry(count):
+    """
+    Decorator for a case, when there is some network hiccup, or slack servers
+    are too busy to respond or on connection timeout. Parameter count says how
+    many times it should try to perform request.
+    """
+    def wrapper(func):
+        @functools.wraps(func)
+        def inner(obj, *args, **kwargs):
+            counter = count
+
+            while counter:
+                counter -= 1
+                try:
+                    return func(obj, *args, **kwargs)
+                except requests.exceptions.RequestException as exc:
+                    if not counter:
+                        logging.error('Request for %s failed. Reported '
+                                      'reason: %s', args[0], exc.__doc__)
+                        raise
+                    logging.warning('Request for %s failed. Reported '
+                                    'reason: %s. Retrying.', args[0],
+                                    exc.__doc__)
+                    # Renew the session before retry
+                    obj.authorize()
+        return inner
+    return wrapper
+
+
 class NotAuthorizedError(requests.HTTPError):
     pass
 
@@ -18,7 +49,7 @@ class Download(object):
     """Download class for taking care of Slack internally uploaded files"""
 
     def __init__(self, args, assets_dir):
-        self.session = requests.session()
+        self.session = None
         self.team = args.team
         self.user = args.user
         self.password = args.password
@@ -37,10 +68,26 @@ class Download(object):
         if not self._hier_created:
             self._create_assets_dir()
 
-        fname = self.prepare_filepath(url, filetype)
+        filepath = self.get_filepath(url, filetype)
+        temp_file = utils.get_temp_name()
 
-        self._download(url, fname)
-        return fname
+        self._download(url, temp_file)
+
+        if filepath and os.path.exists(filepath) and filetype != 'avatar':
+            if not utils.same_files(filepath, temp_file):
+                logging.warning("File `%s' already exist, renamed to `%s'",
+                                filepath,
+                                self.calculate_new_filename(filepath,
+                                                            filetype))
+                filepath = self.calculate_new_filename(filepath, filetype)
+                shutil.move(temp_file, filepath)
+            else:
+                logging.debug("File `%s' already exist, skipping", filepath)
+                os.unlink(filepath)
+        else:
+            shutil.move(temp_file, filepath)
+
+        return filepath
 
     def _create_assets_dir(self):
         for path in (self._files, self._images):
@@ -48,20 +95,21 @@ class Download(object):
 
         self._hier_created = True
 
-    def prepare_filepath(self, url, filetype):
-        """Prepare directory where to download file into"""
+    def get_filepath(self, url, filetype):
+        """Get full path and filename for the file"""
 
         typemap = {'avatar': self._images,
                    'file': self._files}
 
         if filetype == 'file' and not self._authorized:
-            logging.info("There was no (valid) credentials passed, therefore "
-                         "file `%s' cannot be downloaded", url)
+            logging.warning("There was no (valid) credentials passed, "
+                            "therefore file `%s' cannot be downloaded", url)
             return
 
         splitted = url.split('/')
 
-        if len(splitted) == 7 and 'slack.com' in splitted[2]:
+        if len(splitted) == 7 and ('slack.com' in splitted[2] or
+                                   'slack-edge.com' in splitted[2]):
             part = url.split('/')[-3]
             fname = url.split('/')[-1]
         else:
@@ -75,24 +123,24 @@ class Download(object):
             utils.makedirs(os.path.join(path, part))
             path = os.path.join(path, part)
 
-        path = os.path.join(path, fname)
+        return os.path.join(path, fname)
+
+    def calculate_new_filename(self, path, filetype):
         count = 1
 
-        while os.path.exists(path):
-            base, ext = os.path.splitext(path)
-            path = base + "%0.3d" % count + ext
+        while filetype != 'avatar' and os.path.exists(path):
+            if count == 1:
+                base, ext = os.path.splitext(path)
+            path = base + ".%0.3d" % count + ext
+            count += 1
 
         return path
 
+    @retry(3)
     def _download(self, url, local):
         """Download file"""
 
-        try:
-            res = self.session.get(url, stream=True)
-        except requests.exceptions.RequestException as exc:
-            logging.error('Request for %s failed. Reported reason: %s',
-                          url, exc.__doc__)
-            raise
+        res = self.session.get(url, stream=True)
 
         with open(local, 'wb') as fobj:
             for chunk in res.iter_content(chunk_size=5120):
@@ -104,7 +152,9 @@ class Download(object):
         """
         Authenticate and gather session for Slack
         """
+        self.session = requests.session()  # new session
         res = self.session.get('https://%s.slack.com/' % self.team)
+
         if not all((self.team, self.password, self.user)):
             logging.warning('There is neither username, password or team name'
                             ' provided. Downloading will not be performed.')
@@ -112,7 +162,7 @@ class Download(object):
 
         crumb = ''
         for line in res.text.split('\n'):
-            if 'crumb' in line:
+            if 'crumb' in line and 'value' in line:
                 crumb = line.split('value=')[1].split('"')[1]
                 break
         else:
@@ -125,8 +175,7 @@ class Download(object):
                                  'password': self.password,
                                  'signin': 1})
         self.cookies = requests.utils.dict_from_cookiejar(self.session.cookies)
-        if not ('a' in self.cookies and 'b' in self.cookies and
-                ('a-' + self.cookies['a']) in self.cookies):
+        if not ('d' in self.cookies and 'd-s' in self.cookies):
             logging.error('Failed to login into Slack app')
         else:
             self._authorized = True

slack_backup/reporters.py

@@ -1,18 +1,18 @@
-# -*- coding: utf-8 -*-
 """
 Reporters module.
 
 There are several classes for specific format reporting, and also some of the
 slack conversation/convention parsers.
 """
-from __future__ import absolute_import, division, print_function
 import os
 import errno
+import html.parser
 import logging
 import re
 
 from slack_backup import objects as o
 from slack_backup import utils
+from slack_backup import emoji
 
 
 class Reporter(object):
@@ -40,16 +40,18 @@ class Reporter(object):
                                     'file': '📂',
                                     'topic': '🟅',
                                     'separator': '│'}}
+        self.emoji = emoji.EMOJI.get(args.theme, {})
 
         self.channels = self._get_channels(args.channels)
         self.users = self.q(o.User).all()
-        self._re_first_idnick = re.compile(r'^(?P<replace>'
-                                           r'<@(?P<slackid>U[A-Z,0-9]+)\|.+>)')
-        self._re_first_id = re.compile('^(?P<replace>'
-                                       '<@(?P<slackid>U[A-Z,0-9]+)>)')
-        self._re_idnick = re.compile(r'.*(?P<replace>'
-                                     r'<@(?P<slackid>U[A-Z,0-9]+)\|.+>)')
-        self._re_id = re.compile('.*(?P<replace><@(?P<slackid>U[A-Z,0-9]+)>)')
+        self._slackid_pat = [re.compile(r'^(?P<replace>'
+                                        r'<@(?P<slackid>U[A-Z,0-9]+)\|.+>)'),
+                             re.compile('^(?P<replace>'
+                                        '<@(?P<slackid>U[A-Z,0-9]+)>)'),
+                             re.compile(r'.*(?P<replace>'
+                                        r'<@(?P<slackid>U[A-Z,0-9]+)\|.+>)'),
+                             re.compile('.*(?P<replace><@(?P<slackid>'
+                                        'U[A-Z,0-9]+)>)')]
 
     def generate(self):
         """Generate raport it's a dummmy one - for use with none reporter"""
@@ -154,11 +156,9 @@ class TextReporter(Reporter):
         """
         msg_txt = self._filter_slackid(msg.text)
         msg_txt = self._fix_newlines(msg_txt)
+        for emoticon in self.emoji:
+            msg_txt = msg_txt.replace(emoticon, self.emoji[emoticon])
         formatter = self.types.get(msg.type, self._msg)
-        if not msg_txt.strip():
-            logging.info("Skipping message from `%s' since it's empty",
-                         msg.user.name)
-            return ''
 
         return formatter(msg, msg_txt)
 
@@ -202,7 +202,7 @@ class TextReporter(Reporter):
 
     def _msg_file(self, msg, text):
         """return formatter for file"""
-        groups = self._re_first_idnick.match(msg.text).groupdict()
+        groups = self._slackid_pat[0].match(msg.text).groupdict()
         text = msg.text.replace(groups['replace'], '')
         filename = msg.file.filepath
         if filename:
@@ -211,14 +211,17 @@ class TextReporter(Reporter):
             filename = msg.file.url
 
         if not filename:
-            logging.warning("Dude, we have a file object, but nothing has "
-                            "found. Name of the file object is `i%s'",
-                            msg.file.name)
+            logging.warning("There is a file object, but without filename."
+                            "Name of the file object is `%s'", msg.file.name)
             filename = msg.file.name
 
         text = self._filter_slackid(text)
+        text = self._remove_entities(text)
         text = self._fix_newlines(text)
 
+        for emoticon in self.emoji:
+            text = text.replace(emoticon, self.emoji[emoticon])
+
         data = {'date': msg.datetime().strftime("%Y-%m-%d %H:%M:%S"),
                 'msg': text,
                 'max_len': self._max_len,
@@ -230,24 +233,40 @@ class TextReporter(Reporter):
                 'shared file "{filename}"{msg}\n'.format(**data))
 
     def _msg(self, msg, text):
-        """return formatter for /me"""
+        """return formatter for all other message types"""
+
         data = {'date': msg.datetime().strftime("%Y-%m-%d %H:%M:%S"),
                 'msg': text,
                 'max_len': self._max_len,
                 'separator': self._get_symbol('separator'),
                 'nick': msg.user.name}
-        return '{date} {nick:>{max_len}} {separator} {msg}\n'.format(**data)
+        result = '{date} {nick:>{max_len}} {separator} {msg}\n'.format(**data)
+
+        if msg.attachments:
+            for att in msg.attachments:
+                if att.title:
+                    att_text = "\n" + att.title + '\n'
+                else:
+                    att_text = "\n" + self._fix_newlines(att.fallback) + '\n'
+
+                if att.text:
+                    att_text += att.text
+
+                att_text = self._fix_newlines(att_text)
+                # remove first newline
+                att_text = att_text[1:]
+
+                result += att_text + '\n'
+
+        return result
+
+    def _remove_entities(self, text):
+        """replace html entites into appropriate chars"""
+        return html.parser.HTMLParser().unescape(text)
 
     def _filter_slackid(self, text):
         """filter out all of the id from slack"""
-        for pat in (self._re_first_idnick, self._re_first_id):
-            while pat.search(text):
-                groups = pat.search(text).groupdict('slackid')
-                user = [u for u in self.users
-                        if u.slackid == groups['slackid']][0]
-                text = text.replace(groups['replace'], user.name + ":")
-
-        for pat in (self._re_idnick, self._re_id):
+        for pat in self._slackid_pat:
             while pat.search(text):
                 groups = pat.search(text).groupdict('slackid')
                 user = [u for u in self.users

slack_backup/utils.py

@@ -4,6 +4,8 @@ Some utils functions. Jsut to not copypaste the code around
 import errno
 import os
 import logging
+import tempfile
+import hashlib
 
 
 def makedirs(path):
@@ -19,3 +21,24 @@ def makedirs(path):
                 logging.error("Cannot create `%s'. There is some file on the "
                               "way; cannot proceed.", path)
                 raise
+
+
+def get_temp_name():
+    """Return temporary file name"""
+    fdesc, fname = tempfile.mkstemp()
+    os.close(fdesc)
+    return fname
+
+
+def same_files(file1, file2):
+    """
+    Compare files by calculating hash for each of them. Return True if hash is
+    identical, False otherwise
+    """
+    with open(file1, 'rb') as fobj:
+        hash1 = hashlib.sha256(fobj.read())
+
+    with open(file2, 'rb') as fobj:
+        hash2 = hashlib.sha256(fobj.read())
+
+    return hash1.hexdigest() == hash2.hexdigest()

tests/test_client.py

@@ -1,8 +1,5 @@
 from unittest import TestCase
-try:
-    from unittest.mock import MagicMock
-except ImportError:
-    from mock import MagicMock
+from unittest import mock
 
 from slack_backup import client
 from slack_backup import objects as o
@@ -327,7 +324,10 @@ MSGS = {'messages': [{"type": "message",
                               "<https://esm64.slack.com/files/name2/F3405RRB5/"
                               "screenshot.png|Screenshot.png>",
                       "ts": "1478107371.000052",
-                      "upload": True}],
+                      "upload": True},
+                     {'type': 'something else',
+                      'ts': '1502003415232.000001',
+                      "wibblr": True}],
         "ok": True,
         "latest": "1479501075.000020",
         "has_more": True}
@@ -365,34 +365,34 @@ class TestApiCalls(TestCase):
 
     def test_channels_list(self):
         cl = client.Client(FakeArgs())
-        cl.slack.api_call = MagicMock(return_value=CHANNELS)
+        cl.slack.api_call = mock.MagicMock(return_value=CHANNELS)
         channels = cl._channels_list()
         self.assertListEqual(CHANNELS['channels'], channels)
 
     def test_users_list(self):
         cl = client.Client(FakeArgs())
-        cl.slack.api_call = MagicMock(return_value=USERS)
+        cl.slack.api_call = mock.MagicMock(return_value=USERS)
         users = cl._users_list()
         self.assertListEqual(USERS['members'], users)
 
     def test_channels_history(self):
         cl = client.Client(FakeArgs())
 
-        cl.slack.api_call = MagicMock(return_value=USERS)
-        cl.downloader._download = MagicMock(return_value=None)
+        cl.slack.api_call = mock.MagicMock(return_value=USERS)
+        cl.downloader._download = mock.MagicMock(return_value=None)
         cl.update_users()
 
-        cl.slack.api_call = MagicMock(return_value=CHANNELS)
+        cl.slack.api_call = mock.MagicMock(return_value=CHANNELS)
         cl.update_channels()
 
-        cl.slack.api_call = MagicMock()
+        cl.slack.api_call = mock.MagicMock()
         cl.slack.api_call.side_effect = [MSGS, MSG2, MSG3]
 
         channel = cl.q(o.Channel).filter(o.Channel.slackid ==
                                          "C00000001").one()
 
         msg, ts = cl._channels_history(channel, 0)
-        self.assertEqual(len(msg), 5)
+        self.assertEqual(len(msg), 6)
         self.assertEqual(ts, '1479501074.000032')
 
         msg, ts = cl._channels_history(channel, ts)
@@ -408,8 +408,8 @@ class TestClient(TestCase):
 
     def test_update_users(self):
         cl = client.Client(FakeArgs())
-        cl.slack.api_call = MagicMock(return_value=USERS)
-        cl.downloader._download = MagicMock(return_value=None)
+        cl.slack.api_call = mock.MagicMock(return_value=USERS)
+        cl.downloader._download = mock.MagicMock(return_value=None)
         cl.update_users()
         users = cl.session.query(o.User).all()
         self.assertEqual(len(users), 4)
@@ -429,19 +429,22 @@ class TestMessage(TestCase):
         args.channels = ['general']
 
         self.cl = client.Client(args)
-        self.cl.downloader.authorize = MagicMock()
-        self.cl.slack.api_call = MagicMock(return_value=USERS)
-        self.cl.downloader._download = MagicMock(return_value=None)
+        self.cl.downloader.authorize = mock.MagicMock()
+        self.cl.slack.api_call = mock.MagicMock(return_value=USERS)
+        self.cl.downloader._download = mock.MagicMock(return_value=None)
         self.cl.update_users()
 
-        self.cl.slack.api_call = MagicMock(return_value=CHANNELS)
+        self.cl.slack.api_call = mock.MagicMock(return_value=CHANNELS)
         self.cl.update_channels()
 
-        self.cl.slack.api_call = MagicMock()
+        self.cl.slack.api_call = mock.MagicMock()
 
-    def test_update_history(self):
+    @mock.patch('slack_backup.download.Download.download')
+    def test_update_history(self, download):
 
-        self.cl.downloader._download = MagicMock(return_value=None)
+        download.return_value = 'foo'
+
+        self.cl.downloader._download = mock.MagicMock(return_value=None)
         self.cl.slack.api_call.side_effect = [MSGS, MSG3]
         self.cl.update_history()
         self.assertEqual(len(self.cl.q(o.Message).all()), 5)

tests/test_config.py

@@ -0,0 +1,127 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+
+import argparse
+import tempfile
+import os
+import unittest
+
+from slack_backup import config
+
+
+CONF = """\
+[common]
+channels=["one","two", "three"]
+database=dbfname.sqlite
+quiet=1
+verbose=2
+
+[generate]
+output=logs
+format=text
+theme=plain
+
+[fetch]
+user=someuser@address.com
+password=secret
+team=myteam
+token=xxxx-1111111111-222222222222-333333333333-r4nd0ms7uff
+"""
+
+
+class TestConfig(unittest.TestCase):
+
+    def setUp(self):
+
+        fd, self.confname = tempfile.mkstemp()
+        os.close(fd)
+
+        with open(self.confname, 'w') as fobj:
+            fobj.write(CONF)
+
+    def tearDown(self):
+        os.unlink(self.confname)
+
+    def test_config(self):
+
+        self.assertTrue(os.path.exists(self.confname))
+        self.assertTrue(os.path.isfile(self.confname))
+
+        args = argparse.Namespace()
+        args.config = None
+        args.parser = 'fetch'
+        args.verbose = 2
+
+        conf = config.Config()
+        conf.update(args)
+        self.assertDictEqual(vars(args), {'config': None,
+                                          'parser': 'fetch',
+                                          'verbose': 2,
+                                          'quiet': 0,
+                                          'channels': [],
+                                          'database': None,
+                                          'user': None,
+                                          'password': None,
+                                          'team': None,
+                                          'token': None})
+
+        args = argparse.Namespace()
+        args.config = self.confname
+        args.parser = 'fetch'
+        args.verbose = 2
+        args.quiet = None
+        args.channels = None
+        args.database = None
+        args.user = None
+        args.password = None
+        args.team = None
+        args.token = None
+
+        conf = config.Config()
+        conf.update(args)
+
+        self.assertEqual(conf._options['verbose'], 2)
+        self.assertListEqual(conf._options['channels'],
+                             ['one', 'two', 'three'])
+        self.assertEqual(conf._options['database'], 'dbfname.sqlite')
+        self.assertEqual(conf._options['user'], 'someuser@address.com')
+
+        self.assertDictEqual(vars(args), {'config': self.confname,
+                                          'parser': 'fetch',
+                                          'verbose': 2,
+                                          'quiet': 0,
+                                          'channels': ['one', 'two', 'three'],
+                                          'database': 'dbfname.sqlite',
+                                          'user': 'someuser@address.com',
+                                          'password': 'secret',
+                                          'team': 'myteam',
+                                          'token': 'xxxx-1111111111-'
+                                                   '222222222222-333333333333-'
+                                                   'r4nd0ms7uff'})
+
+        # override some conf options with commandline
+        args = argparse.Namespace()
+        args.config = self.confname
+        args.parser = 'fetch'
+        args.verbose = None
+        args.quiet = 2
+        args.channels = ['foo']
+        args.database = None
+        args.user = 'joe'
+        args.password = 'ultricies'
+        args.team = ''
+        args.token = 'the token'
+
+        conf = config.Config()
+        conf.update(args)
+
+        self.assertDictEqual(vars(args), {'config': self.confname,
+                                          'parser': 'fetch',
+                                          'verbose': 0,
+                                          'quiet': 2,
+                                          'channels': ['foo'],
+                                          'database': 'dbfname.sqlite',
+                                          'user': 'joe',
+                                          'password': 'ultricies',
+                                          'team': '',
+                                          'token': 'the token'})

tests/test_reporter.py

@@ -1,10 +1,5 @@
-# -*- coding: utf-8 -*-
-
 from unittest import TestCase
-try:
-    from unittest.mock import MagicMock
-except ImportError:
-    from mock import MagicMock
+from unittest.mock import MagicMock
 
 from slack_backup import reporters as r
 
@@ -45,7 +40,3 @@ class TestReporter(TestCase):
         text = ('<@U111BBBBB|user2>Praesent vel enim sed eros luctus '
                 'imperdiet.\nMauris neque ante, <@U111DDDDD> placerat at, '
                 'mollis vitae, faucibus quis, <@U111CCCCC>leo. Ut feugiat.')
-
-# Praesent vel enim sed eros luctus imperdiet.  Vivamus urna quam, congue
-# vulputate, convallis non, cursus cursus, risus. Quisque aliquet. Donec
-# vulputate egestas elit. Morbi dictum, sem sit amet aliquam.

tox.ini

@@ -1,5 +1,5 @@
 [tox]
-envlist = py27,py34,{py27,py34}-flake8
+envlist = py34,py34-flake8
 
 usedevelop = True
 
@@ -10,18 +10,6 @@ commands = py.test --cov=slack_backup --cov-report=term-missing
 deps = -r{toxinidir}/requirements.txt
        -r{toxinidir}/test-requirements.txt
 
-[testenv:py27]
-usedevelop={[testenv]usedevelop}
-setenv = {[testenv]setenv}
-commands = {[testenv]commands}
-deps = {[testenv]deps}
-        mock
-
-[testenv:py27-flake8]
-basepython = python2.7
-deps = flake8
-commands = flake8 {posargs}
-
 [testenv:py34-flake8]
 basepython = python3.4
 deps = flake8