Version bump

Till now, if we download certain files (like those attached to the
conversation) and we already have the file with the same name, number in
format '%03d' was added just before extension. That way there could be
possibility, that the very same file will be downloaded and stored
multiple times, like:

file.png
file.001.png
file.002.png
...

This commit prevents that by adding comparison between files we already
have and file which is downloaded from slack. Adding another file with
additional number will only have place when stored file and downloaded
have different content.

MANIFEST.in

@@ -0,0 +1,2 @@
+include README.rst
+include config_example

README.rst

@@ -68,7 +68,7 @@ typical session:
    (myenv)user@localhost ~/mylogs $ slack-backup fetch \
    --token xxxx-1111111111-222222222222-333333333333-r4nd0ms7uff \
    --user some@email.address.org --password secret --team myteam \
-   -qqq -d mydatabase.sqlite
+   -qq -d mydatabase.sqlite
 
 where:
 
@@ -116,6 +116,60 @@ command.
 
 See help for the ``slack-backup`` command for complete list of options.
 
+Configuration
+-------------
+
+For convenience, you can place all of needed options into configuration file
+(aka .ini), which all options (with their defaults) will look like:
+
+.. code:: ini
+
+   [common]
+   channels =
+   database =
+   quiet = 0
+   verbose = 0
+
+   [generate]
+   output =
+   format = text
+   theme = plain
+
+   [fetch]
+   user =
+   password =
+   team =
+   token =
+
+Note, that you don't have to put every option. To illustrate ``fetch`` example
+from above, here is a corresponding config file:
+
+.. code:: ini
+
+   [common]
+   database = mydatabase.sqlite
+   quiet = 2
+
+   [fetch]
+   user = some@email.address.org
+   password = secret
+   team = myteam
+   token = xxxx-1111111111-222222222222-333333333333-r4nd0ms7uff
+
+Note, that only ``[common]`` and ``[fetch]`` sections are provided, so it is
+enough to invoke ``slack-backup`` command as:
+
+.. code:: shell-session
+
+   (myenv)user@localhost ~/mylogs $ slack-backup fetch
+
+There are couple of places, where configuration file would be searched for, in
+particular order:
+
+* file provided via argument ``-i`` or ``--config``
+* ``slack-backup.ini`` in current directory
+* ``$XDG_CONFIG_HOME/slack-backup.ini``, where ``$XDG_CONFIG_HOME`` usually
+  defaults to ``$HOME/.config``
 
 Details
 -------

scripts/slack-backup

@@ -1,108 +1,10 @@
 #!/usr/bin/env python3
 # -*- coding: utf-8 -*-
 """
-Create backup for certain date for specified channel in slack
+Execute commands for slack-backup
 """
-import argparse
-import logging
-
-from slack_backup import client
-
-
-def setup_logger(args):
-    """Setup logger format and level"""
-
-    level = logging.WARNING
-
-    if args.quiet:
-        level = logging.ERROR
-        if args.quiet > 1:
-            level = logging.CRITICAL
-
-    if args.verbose:
-        level = logging.INFO
-        if args.verbose > 1:
-            level = logging.DEBUG
-
-    logging.basicConfig(level=level,
-                        format="%(asctime)s %(levelname)s: %(message)s")
-
-def generate_raport(args):
-    """Generate logs"""
-    slack = client.Client(args)
-    slack.generate_history()
-
-def fetch_data(args):
-    """Fetch and store data"""
-    slack = client.Client(args)
-    slack.update()
-
-
-def main():
-    """Main function"""
-    parser = argparse.ArgumentParser()
-
-    subparser = parser.add_subparsers(dest='parser')
-    subparser.required = True
-
-    fetch = subparser.add_parser('fetch', help='Update local db with Slack'
-                                 ' data')
-    fetch.add_argument('-t', '--token', required=True, help='Slack token - '
-                       'a string, which can be generated/obtained via '
-                       'https://api.slack.com/docs/oauth-test-tokens page.')
-    fetch.add_argument('-u', '--user', default='', help='Username for your '
-                       'Slack account')
-    fetch.add_argument('-p', '--password', default='', help='Password for your '
-                       'Slack account.')
-    fetch.add_argument('-e', '--team', required=True, help='Team name, which '
-                       'is part of slack url, for example: if url is '
-                       '"https://team.slack.com" than "team" is a name of '
-                       'the team.')
-    fetch.add_argument('-v', '--verbose', help='Be verbose. Adding more "v"'
-                       'will increase verbosity', action="count", default=0)
-    fetch.add_argument('-q', '--quiet', help='Be quiet. Adding more "q"'
-                       'will decrease verbosity', action="count", default=0)
-    fetch.add_argument('-c', '--channels', default=[], nargs='+',
-                       help='List of channels to perform actions on. '
-                       'Default is all channels.')
-
-    fetch.add_argument('-d', '--database', default='',
-                       help='Path to the database file.')
-
-    fetch.set_defaults(func=fetch_data)
-
-    generate = subparser.add_parser('generate', help='Generate logs out of '
-                                    'data in provided database')
-    generate.add_argument('-o', '--output', default='logs', help="Output "
-                          "directory for store logs. All logs are organised "
-                          "per channel. By default it's `logs' directory")
-    generate.add_argument('-f', '--format', default='none',
-                          choices=('text', 'none'),
-                          help='Output format. Default is none; only database '
-                          'is updated by latest messages for all/selected '
-                          'channels.')
-    generate.add_argument('-t', '--theme', default='plain',
-                          choices=('plain', 'unicode'),
-                          help='Choose theme for text output. It doesn\'t '
-                          'affect other output formats.')
-
-    generate.add_argument('-v', '--verbose', help='Be verbose. Adding more "v"'
-                          'will increase verbosity', action="count", default=0)
-    generate.add_argument('-q', '--quiet', help='Be quiet. Adding more "q"'
-                          'will decrease verbosity', action="count", default=0)
-    generate.add_argument('-c', '--channels', default=[], nargs='+',
-                          help='List of channels to perform actions on. '
-                          'Default is all channels.')
-
-    generate.add_argument('-d', '--database', default='',
-                          help='Path to the database file.')
-
-    generate.set_defaults(func=generate_raport)
-
-    args = parser.parse_args()
-    setup_logger(args)
-    args.func(args)
+from slack_backup import command
 
 
 if __name__ == "__main__":
-    main()
+    command.main()

setup.py

@@ -10,7 +10,7 @@ except ImportError:
 
 setup(name="slack-backup",
       packages=["slack_backup"],
-      version="0.3",
+      version="0.6",
       description="Make copy of slack converstaions",
       author="Roman Dobosz",
       author_email="gryf73@gmail.com",

slack_backup/client.py

@@ -3,6 +3,7 @@ Create backup for certain date for specified channel in slack
 """
 from datetime import datetime
 import getpass
+import json
 import logging
 import os
 
@@ -112,6 +113,7 @@ class Client(object):
             channels = all_channels
 
         for channel in channels:
+            logging.info("Getting messages for channel `%s'", channel.name)
             latest = self.q(o.Message).\
                 filter(o.Message.channel == channel).\
                 order_by(o.Message.ts.desc()).first()
@@ -146,10 +148,20 @@ class Client(object):
         Create message with corresponding possible metadata, like reactions,
         files etc.
         """
-        user = self.q(o.User).\
-            filter(o.User.slackid == data['user']).one()
+        if data['type'] != 'message':
+            logging.info("Skipping message of type `%s'.", data['type'])
+            return
 
-        if data['type'] == 'message' and not data['text'].strip():
+        logging.debug('Message data: %s', json.dumps(data))
+
+        try:
+            user = self.q(o.User).\
+                filter(o.User.slackid == data['user']).one()
+        except KeyError:
+            user = self.q(o.User).\
+                    filter(o.User.slackid == data['comment']['user']).one()
+
+        if not data['text'].strip():
             logging.info("Skipping message from `%s' since it's empty",
                          user.name)
             return
@@ -255,7 +267,7 @@ class Client(object):
         if not database:
             return 'assets'
 
-        path = os.path.dirname(os.path.abspath(database))
+        path = os.path.dirname(database)
         return os.path.join(path, 'assets')
 
     def _channels_list(self):

slack_backup/command.py

@@ -0,0 +1,141 @@
+"""
+Create backup for certain date for specified channel in slack
+"""
+import argparse
+import logging
+import platform
+
+from slack_backup import client
+from slack_backup import config
+
+
+def setup_logger(args):
+    """Setup logger format and level"""
+
+    if platform.system() != "Windows":
+        # hack to have colors in terminal
+        logging.addLevelName(logging.DEBUG,
+                             "\033[1;30m%s\033[1;0m" %
+                             logging.getLevelName(logging.DEBUG))
+
+        logging.addLevelName(logging.INFO,
+                             "\033[1;32m%s\033[1;0m" %
+                             logging.getLevelName(logging.INFO))
+
+        logging.addLevelName(logging.WARNING,
+                             "\033[1;33m%s\033[1;0m" %
+                             logging.getLevelName(logging.WARNING))
+
+        logging.addLevelName(logging.ERROR,
+                             "\033[1;31m%s\033[1;0m" %
+                             logging.getLevelName(logging.ERROR))
+
+        logging.addLevelName(logging.CRITICAL,
+                             "\033[7;31m%s\033[1;0m" %
+                             logging.getLevelName(logging.CRITICAL))
+
+    level = logging.WARNING
+
+    if args.quiet:
+        level = logging.ERROR
+        if args.quiet > 1:
+            level = logging.CRITICAL
+
+    if args.verbose:
+        level = logging.INFO
+        if args.verbose > 1:
+            level = logging.DEBUG
+
+    logging.basicConfig(level=level,
+                        format="%(asctime)s %(levelname)s: %(message)s")
+
+
+def generate_raport(args):
+    """Generate logs"""
+    slack = client.Client(args)
+    slack.generate_history()
+
+
+def fetch_data(args):
+    """Fetch and store data"""
+    slack = client.Client(args)
+    slack.update()
+
+
+def main():
+    """Main function"""
+    parser = argparse.ArgumentParser()
+
+    subparser = parser.add_subparsers(dest='parser')
+    subparser.required = True
+
+    fetch = subparser.add_parser('fetch', help='Update local db with Slack'
+                                 ' data')
+    fetch.add_argument('-t', '--token', default=None, help='Slack token - '
+                       'a string, which can be generated/obtained via '
+                       'https://api.slack.com/docs/oauth-test-tokens page.')
+    fetch.add_argument('-u', '--user', default=None, help='Username for your '
+                       'Slack account')
+    fetch.add_argument('-p', '--password', default=None, help='Password for '
+                       'your Slack account.')
+    fetch.add_argument('-e', '--team', default=None, help='Team name, which '
+                       'is part of slack url, for example: if url is '
+                       '"https://team.slack.com" than "team" is a name of '
+                       'the team.')
+    fetch.add_argument('-v', '--verbose', help='Be verbose. Adding more "v" '
+                       'will increase verbosity', action="count",
+                       default=None)
+    fetch.add_argument('-q', '--quiet', help='Be quiet. Adding more "q" will'
+                       ' decrease verbosity', action="count", default=None)
+    fetch.add_argument('-c', '--channels', default=None, nargs='+',
+                       help='List of channels to perform actions on. '
+                       'Default is all channels.')
+
+    fetch.add_argument('-d', '--database', default=None,
+                       help='Path to the database file.')
+
+    fetch.add_argument('-i', '--config', default=None,
+                       help='Use specific config file.')
+
+    fetch.set_defaults(func=fetch_data)
+
+    generate = subparser.add_parser('generate', help='Generate logs out of '
+                                    'data in provided database')
+    generate.add_argument('-o', '--output', default=None, help="Output "
+                          "directory for store logs. All logs are organised "
+                          "per channel. By default it's `logs' directory")
+    generate.add_argument('-f', '--format', default=None,
+                          choices=('text', 'none'),
+                          help='Output format. Default is none; only database '
+                          'is updated by latest messages for all/selected '
+                          'channels.')
+    generate.add_argument('-t', '--theme', default=None,
+                          choices=('plain', 'unicode'),
+                          help='Choose theme for text output. It doesn\'t '
+                          'affect other output formats.')
+
+    generate.add_argument('-v', '--verbose', help='Be verbose. Adding more '
+                          '"v" will increase verbosity', action="count",
+                          default=None)
+    generate.add_argument('-q', '--quiet', help='Be quiet. Adding more "q" '
+                          'will decrease verbosity', action="count",
+                          default=None)
+    generate.add_argument('-c', '--channels', default=[], nargs='+',
+                          help='List of channels to perform actions on. '
+                          'Default is all channels.')
+
+    generate.add_argument('-d', '--database', default=None,
+                          help='Path to the database file.')
+
+    generate.add_argument('-i', '--config', default=None,
+                          help='Use specific config file.')
+
+    generate.set_defaults(func=generate_raport)
+
+    args = parser.parse_args()
+    cfg = config.Config()
+    msg = cfg.update(args)
+    setup_logger(args)
+    logging.info(msg)
+
+    args.func(args)

slack_backup/config.py

@@ -0,0 +1,112 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+
+"""
+Configuration module for slack-backup
+"""
+import json
+import os
+import configparser
+
+
+class Config(object):
+    """Configuration keeper"""
+
+    ints = ['verbose', 'quiet']
+
+    sections = {'common': ['channels', 'database', 'quiet', 'verbose'],
+                'fetch': ['user', 'password', 'team', 'token'],
+                'generate': ['output', 'format', 'theme']}
+
+    def __init__(self):
+        """
+        Init. Read config, if exists, and update passed argument parser
+        object.
+        """
+
+        self.cp = configparser.ConfigParser()
+        self._options = {'channels': [],
+                         'database': None,
+                         'quiet': 0,
+                         'verbose': 0,
+                         'user': None,
+                         'password': None,
+                         'team': None,
+                         'token': None,
+                         'output': None,
+                         'format': None,
+                         'theme': None}
+        # This message supposed to be displayed in INFO level. During the time
+        # of running the code where it should be displayed there is no
+        # complete information about logging level. Displaying message is
+        # dependent on the a) config file, b) argument from commandline. Let's
+        # resolve if user want to have that information or not after merging
+        # those two sources. If user do not want to see any message in INFO
+        # level, we shouldn't do so.
+        self.msg = ''
+
+    def update(self, args):
+        self.load_config(args)
+        self.parse_loaded_options()
+        self.update_args(args)
+        return self.msg
+
+    def load_config(self, args):
+
+        path = ''
+        if hasattr(args, 'config') and args.config:
+            path = args.config
+
+        locations = [path,
+                     './slack-backup.ini',
+                     os.path.expandvars('$XDG_CONFIG_HOME/slack-backup.ini'),
+                     os.path.expandvars('$HOME/.config/slack-backup.ini')]
+
+        for location in locations:
+            if os.path.exists(location):
+                self.cp.read(location)
+                self.msg = 'Found configuration file: %s' % location
+                break
+        else:
+            self.msg = 'No configuration file found'
+
+    def parse_loaded_options(self):
+
+        for section in self.cp.sections():
+            if section not in self.sections:
+                continue
+
+            for option in self.sections[section]:
+                if option in self.ints:
+                    val = self.cp.getint(section, option, fallback=0)
+                elif option == 'channels':
+                    val = self.cp.get(section, option, fallback='[]')
+                    val = json.loads(val)
+                else:
+                    val = self.cp.get(section, option, fallback=None)
+
+                self._options[option] = val
+
+    def update_args(self, args):
+        if 'parser' not in args:
+            # it doesn't make sense to update args, since no action was
+            # choosen
+            return
+
+        # special case, re-set information for verbose/quiet options
+        if 'verbose' in args and args.verbose is not None:
+            self._options['verbose'] = args.verbose
+            if self._options['quiet'] is not None:
+                self._options['quiet'] = 0
+        if 'quiet' in args and args.quiet is not None:
+            self._options['quiet'] = args.quiet
+            if self._options['verbose'] is not None:
+                self._options['verbose'] = 0
+
+        for sec_id in (args.parser, 'common'):
+            for option in self.sections[sec_id]:
+                if option in args:
+                    if getattr(args, option) is not None:
+                        continue
+
+                setattr(args, option, self._options[option])

slack_backup/download.py

@@ -2,14 +2,45 @@
 Module for download files, store them in local filesystem and convert the URLs
 to local ones, so that sophisticated writers can make a use of it
 """
+import functools
 import logging
 import os
+import shutil
 
 import requests
 
 from slack_backup import utils
 
 
+def retry(count):
+    """
+    Decorator for a case, when there is some network hiccup, or slack servers
+    are too busy to respond or on connection timeout. Parameter count says how
+    many times it should try to perform request.
+    """
+    def wrapper(func):
+        @functools.wraps(func)
+        def inner(obj, *args, **kwargs):
+            counter = count
+
+            while counter:
+                counter -= 1
+                try:
+                    return func(obj, *args, **kwargs)
+                except requests.exceptions.RequestException as exc:
+                    if not counter:
+                        logging.error('Request for %s failed. Reported '
+                                      'reason: %s', args[0], exc.__doc__)
+                        raise
+                    logging.warning('Request for %s failed. Reported '
+                                    'reason: %s. Retrying.', args[0],
+                                    exc.__doc__)
+                    # Renew the session before retry
+                    obj.authorize()
+        return inner
+    return wrapper
+
+
 class NotAuthorizedError(requests.HTTPError):
     pass
 
@@ -18,7 +49,7 @@ class Download(object):
     """Download class for taking care of Slack internally uploaded files"""
 
     def __init__(self, args, assets_dir):
-        self.session = requests.session()
+        self.session = None
         self.team = args.team
         self.user = args.user
         self.password = args.password
@@ -37,10 +68,26 @@ class Download(object):
         if not self._hier_created:
             self._create_assets_dir()
 
-        fname = self.prepare_filepath(url, filetype)
+        filepath = self.get_filepath(url, filetype)
+        temp_file = utils.get_temp_name()
 
-        self._download(url, fname)
-        return fname
+        self._download(url, temp_file)
+
+        if filepath and os.path.exists(filepath) and filetype != 'avatar':
+            if not utils.same_files(filepath, temp_file):
+                logging.warning("File `%s' already exist, renamed to `%s'",
+                                filepath,
+                                self.calculate_new_filename(filepath,
+                                                            filetype))
+                filepath = self.calculate_new_filename(filepath, filetype)
+                shutil.move(temp_file, filepath)
+            else:
+                logging.debug("File `%s' already exist, skipping", filepath)
+                os.unlink(filepath)
+        else:
+            shutil.move(temp_file, filepath)
+
+        return filepath
 
     def _create_assets_dir(self):
         for path in (self._files, self._images):
@@ -48,20 +95,21 @@ class Download(object):
 
         self._hier_created = True
 
-    def prepare_filepath(self, url, filetype):
-        """Prepare directory where to download file into"""
+    def get_filepath(self, url, filetype):
+        """Get full path and filename for the file"""
 
         typemap = {'avatar': self._images,
                    'file': self._files}
 
         if filetype == 'file' and not self._authorized:
-            logging.info("There was no (valid) credentials passed, therefore "
-                         "file `%s' cannot be downloaded", url)
+            logging.warning("There was no (valid) credentials passed, "
+                            "therefore file `%s' cannot be downloaded", url)
             return
 
         splitted = url.split('/')
 
-        if len(splitted) == 7 and 'slack.com' in splitted[2]:
+        if len(splitted) == 7 and ('slack.com' in splitted[2] or
+                                   'slack-edge.com' in splitted[2]):
             part = url.split('/')[-3]
             fname = url.split('/')[-1]
         else:
@@ -75,24 +123,24 @@ class Download(object):
             utils.makedirs(os.path.join(path, part))
             path = os.path.join(path, part)
 
-        path = os.path.join(path, fname)
+        return os.path.join(path, fname)
+
+    def calculate_new_filename(self, path, filetype):
         count = 1
 
         while filetype != 'avatar' and os.path.exists(path):
-            base, ext = os.path.splitext(path)
-            path = base + "%0.3d" % count + ext
+            if count == 1:
+                base, ext = os.path.splitext(path)
+            path = base + ".%0.3d" % count + ext
+            count += 1
 
         return path
 
+    @retry(3)
     def _download(self, url, local):
         """Download file"""
 
-        try:
-            res = self.session.get(url, stream=True)
-        except requests.exceptions.RequestException as exc:
-            logging.error('Request for %s failed. Reported reason: %s',
-                          url, exc.__doc__)
-            raise
+        res = self.session.get(url, stream=True)
 
         with open(local, 'wb') as fobj:
             for chunk in res.iter_content(chunk_size=5120):
@@ -104,7 +152,9 @@ class Download(object):
         """
         Authenticate and gather session for Slack
         """
+        self.session = requests.session()  # new session
         res = self.session.get('https://%s.slack.com/' % self.team)
+
         if not all((self.team, self.password, self.user)):
             logging.warning('There is neither username, password or team name'
                             ' provided. Downloading will not be performed.')
@@ -112,7 +162,7 @@ class Download(object):
 
         crumb = ''
         for line in res.text.split('\n'):
-            if 'crumb' in line:
+            if 'crumb' in line and 'value' in line:
                 crumb = line.split('value=')[1].split('"')[1]
                 break
         else:

slack_backup/utils.py

@@ -4,6 +4,8 @@ Some utils functions. Jsut to not copypaste the code around
 import errno
 import os
 import logging
+import tempfile
+import hashlib
 
 
 def makedirs(path):
@@ -19,3 +21,24 @@ def makedirs(path):
                 logging.error("Cannot create `%s'. There is some file on the "
                               "way; cannot proceed.", path)
                 raise
+
+
+def get_temp_name():
+    """Return temporary file name"""
+    fdesc, fname = tempfile.mkstemp()
+    os.close(fdesc)
+    return fname
+
+
+def same_files(file1, file2):
+    """
+    Compare files by calculating hash for each of them. Return True if hash is
+    identical, False otherwise
+    """
+    with open(file1, 'rb') as fobj:
+        hash1 = hashlib.sha256(fobj.read())
+
+    with open(file2, 'rb') as fobj:
+        hash2 = hashlib.sha256(fobj.read())
+
+    return hash1.hexdigest() == hash2.hexdigest()

tests/test_client.py

@@ -1,5 +1,5 @@
 from unittest import TestCase
-from unittest.mock import MagicMock
+from unittest import mock
 
 from slack_backup import client
 from slack_backup import objects as o
@@ -324,7 +324,10 @@ MSGS = {'messages': [{"type": "message",
                               "<https://esm64.slack.com/files/name2/F3405RRB5/"
                               "screenshot.png|Screenshot.png>",
                       "ts": "1478107371.000052",
-                      "upload": True}],
+                      "upload": True},
+                     {'type': 'something else',
+                      'ts': '1502003415232.000001',
+                      "wibblr": True}],
         "ok": True,
         "latest": "1479501075.000020",
         "has_more": True}
@@ -362,34 +365,34 @@ class TestApiCalls(TestCase):
 
     def test_channels_list(self):
         cl = client.Client(FakeArgs())
-        cl.slack.api_call = MagicMock(return_value=CHANNELS)
+        cl.slack.api_call = mock.MagicMock(return_value=CHANNELS)
         channels = cl._channels_list()
         self.assertListEqual(CHANNELS['channels'], channels)
 
     def test_users_list(self):
         cl = client.Client(FakeArgs())
-        cl.slack.api_call = MagicMock(return_value=USERS)
+        cl.slack.api_call = mock.MagicMock(return_value=USERS)
         users = cl._users_list()
         self.assertListEqual(USERS['members'], users)
 
     def test_channels_history(self):
         cl = client.Client(FakeArgs())
 
-        cl.slack.api_call = MagicMock(return_value=USERS)
-        cl.downloader._download = MagicMock(return_value=None)
+        cl.slack.api_call = mock.MagicMock(return_value=USERS)
+        cl.downloader._download = mock.MagicMock(return_value=None)
         cl.update_users()
 
-        cl.slack.api_call = MagicMock(return_value=CHANNELS)
+        cl.slack.api_call = mock.MagicMock(return_value=CHANNELS)
         cl.update_channels()
 
-        cl.slack.api_call = MagicMock()
+        cl.slack.api_call = mock.MagicMock()
         cl.slack.api_call.side_effect = [MSGS, MSG2, MSG3]
 
         channel = cl.q(o.Channel).filter(o.Channel.slackid ==
                                          "C00000001").one()
 
         msg, ts = cl._channels_history(channel, 0)
-        self.assertEqual(len(msg), 5)
+        self.assertEqual(len(msg), 6)
         self.assertEqual(ts, '1479501074.000032')
 
         msg, ts = cl._channels_history(channel, ts)
@@ -405,8 +408,8 @@ class TestClient(TestCase):
 
     def test_update_users(self):
         cl = client.Client(FakeArgs())
-        cl.slack.api_call = MagicMock(return_value=USERS)
-        cl.downloader._download = MagicMock(return_value=None)
+        cl.slack.api_call = mock.MagicMock(return_value=USERS)
+        cl.downloader._download = mock.MagicMock(return_value=None)
         cl.update_users()
         users = cl.session.query(o.User).all()
         self.assertEqual(len(users), 4)
@@ -426,19 +429,22 @@ class TestMessage(TestCase):
         args.channels = ['general']
 
         self.cl = client.Client(args)
-        self.cl.downloader.authorize = MagicMock()
-        self.cl.slack.api_call = MagicMock(return_value=USERS)
-        self.cl.downloader._download = MagicMock(return_value=None)
+        self.cl.downloader.authorize = mock.MagicMock()
+        self.cl.slack.api_call = mock.MagicMock(return_value=USERS)
+        self.cl.downloader._download = mock.MagicMock(return_value=None)
         self.cl.update_users()
 
-        self.cl.slack.api_call = MagicMock(return_value=CHANNELS)
+        self.cl.slack.api_call = mock.MagicMock(return_value=CHANNELS)
         self.cl.update_channels()
 
-        self.cl.slack.api_call = MagicMock()
+        self.cl.slack.api_call = mock.MagicMock()
 
-    def test_update_history(self):
+    @mock.patch('slack_backup.download.Download.download')
+    def test_update_history(self, download):
 
-        self.cl.downloader._download = MagicMock(return_value=None)
+        download.return_value = 'foo'
+
+        self.cl.downloader._download = mock.MagicMock(return_value=None)
         self.cl.slack.api_call.side_effect = [MSGS, MSG3]
         self.cl.update_history()
         self.assertEqual(len(self.cl.q(o.Message).all()), 5)

tests/test_config.py

@@ -0,0 +1,127 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+
+import argparse
+import tempfile
+import os
+import unittest
+
+from slack_backup import config
+
+
+CONF = """\
+[common]
+channels=["one","two", "three"]
+database=dbfname.sqlite
+quiet=1
+verbose=2
+
+[generate]
+output=logs
+format=text
+theme=plain
+
+[fetch]
+user=someuser@address.com
+password=secret
+team=myteam
+token=xxxx-1111111111-222222222222-333333333333-r4nd0ms7uff
+"""
+
+
+class TestConfig(unittest.TestCase):
+
+    def setUp(self):
+
+        fd, self.confname = tempfile.mkstemp()
+        os.close(fd)
+
+        with open(self.confname, 'w') as fobj:
+            fobj.write(CONF)
+
+    def tearDown(self):
+        os.unlink(self.confname)
+
+    def test_config(self):
+
+        self.assertTrue(os.path.exists(self.confname))
+        self.assertTrue(os.path.isfile(self.confname))
+
+        args = argparse.Namespace()
+        args.config = None
+        args.parser = 'fetch'
+        args.verbose = 2
+
+        conf = config.Config()
+        conf.update(args)
+        self.assertDictEqual(vars(args), {'config': None,
+                                          'parser': 'fetch',
+                                          'verbose': 2,
+                                          'quiet': 0,
+                                          'channels': [],
+                                          'database': None,
+                                          'user': None,
+                                          'password': None,
+                                          'team': None,
+                                          'token': None})
+
+        args = argparse.Namespace()
+        args.config = self.confname
+        args.parser = 'fetch'
+        args.verbose = 2
+        args.quiet = None
+        args.channels = None
+        args.database = None
+        args.user = None
+        args.password = None
+        args.team = None
+        args.token = None
+
+        conf = config.Config()
+        conf.update(args)
+
+        self.assertEqual(conf._options['verbose'], 2)
+        self.assertListEqual(conf._options['channels'],
+                             ['one', 'two', 'three'])
+        self.assertEqual(conf._options['database'], 'dbfname.sqlite')
+        self.assertEqual(conf._options['user'], 'someuser@address.com')
+
+        self.assertDictEqual(vars(args), {'config': self.confname,
+                                          'parser': 'fetch',
+                                          'verbose': 2,
+                                          'quiet': 0,
+                                          'channels': ['one', 'two', 'three'],
+                                          'database': 'dbfname.sqlite',
+                                          'user': 'someuser@address.com',
+                                          'password': 'secret',
+                                          'team': 'myteam',
+                                          'token': 'xxxx-1111111111-'
+                                                   '222222222222-333333333333-'
+                                                   'r4nd0ms7uff'})
+
+        # override some conf options with commandline
+        args = argparse.Namespace()
+        args.config = self.confname
+        args.parser = 'fetch'
+        args.verbose = None
+        args.quiet = 2
+        args.channels = ['foo']
+        args.database = None
+        args.user = 'joe'
+        args.password = 'ultricies'
+        args.team = ''
+        args.token = 'the token'
+
+        conf = config.Config()
+        conf.update(args)
+
+        self.assertDictEqual(vars(args), {'config': self.confname,
+                                          'parser': 'fetch',
+                                          'verbose': 0,
+                                          'quiet': 2,
+                                          'channels': ['foo'],
+                                          'database': 'dbfname.sqlite',
+                                          'user': 'joe',
+                                          'password': 'ultricies',
+                                          'team': '',
+                                          'token': 'the token'})