Fix the tests.

For some reason, database key was treated differently in configparser object
in python 3.4.2. In Python 3.4.5 everything is fine. Fixed the defaults to
make sure all string options are treated equally.

.travis.yml

@@ -1,8 +1,6 @@
 language: python
 env:
-- TOXENV=py27
 - TOXENV=py34
-- TOXENV=py27-flake8
 - TOXENV=py34-flake8
 install: pip install tox
 script: tox

MANIFEST.in

@@ -0,0 +1,2 @@
+include README.rst
+include config_example

README.rst

@@ -4,15 +4,18 @@ Slack backup
 .. image:: https://travis-ci.org/gryf/slack-backup.svg?branch=master
     :target: https://travis-ci.org/gryf/slack-backup
 
-This simple project which aim is to collect conversations from Slack using its
+.. image:: https://img.shields.io/pypi/v/slack-backup.svg
-API and optionally user account information, and provides convenient way to
+    :target: https://pypi.python.org/pypi/slack-backup
-represent as a log.
+
+The project aim is to collect conversations from Slack using its API and
+optionally user account information, and provides convenient way to represent
+as a log.
 
 Requirements
 ------------
 
-This project is written in Python 2.7, and 3.4+, although version 2.7, which
+This project is written in Python 3, 3.4 to be precise, although it may work on
-should work, wasn't tested as extensively as it should be.
+earlier version of Python3. Sorry no support for Python2.
 
 Other than that, required packages are as follows:
 
@@ -65,7 +68,7 @@ typical session:
    (myenv)user@localhost ~/mylogs $ slack-backup fetch \
    --token xxxx-1111111111-222222222222-333333333333-r4nd0ms7uff \
    --user some@email.address.org --password secret --team myteam \
-   -qqq -d mydatabase.sqlite
+   -qq -d mydatabase.sqlite
 
 where:
 
@@ -111,6 +114,87 @@ where:
 The rest of the options (``-d`` and ``-v``) have same meaning as in ``fetch``
 command.
 
+See help for the ``slack-backup`` command for complete list of options.
+
+Configuration
+-------------
+
+For convenience, you can place all of needed options into configuration file
+(aka .ini), which all options (with their defaults) will look like:
+
+.. code:: ini
+
+   [common]
+   channels =
+   database =
+   quiet = 0
+   verbose = 0
+
+   [generate]
+   output =
+   format = text
+   theme = plain
+
+   [fetch]
+   user =
+   password =
+   team =
+   token =
+
+Note, that you don't have to put every option. To illustrate ``fetch`` example
+from above, here is a corresponding config file:
+
+.. code:: ini
+
+   [common]
+   database = mydatabase.sqlite
+   quiet = 2
+
+   [fetch]
+   user = some@email.address.org
+   password = secret
+   team = myteam
+   token = xxxx-1111111111-222222222222-333333333333-r4nd0ms7uff
+
+Note, that only ``[common]`` and ``[fetch]`` sections are provided, so it is
+enough to invoke ``slack-backup`` command as:
+
+.. code:: shell-session
+
+   (myenv)user@localhost ~/mylogs $ slack-backup fetch
+
+There are couple of places, where configuration file would be searched for, in
+particular order:
+
+* file provided via argument ``-i`` or ``--config``
+* ``slack-backup.ini`` in current directory
+* ``$XDG_CONFIG_HOME/slack-backup.ini``, where ``$XDG_CONFIG_HOME`` usually
+  defaults to ``$HOME/.config``
+
+Details
+-------
+
+During first run, database with provided name is generated. For ease of use
+sqlite database is used, although it is easy to switch the engine, since there
+is an ORM (SQLAlchemy) used.
+
+Slack users, channels and messages are mapped to SQLAlchemy models, as well as
+other information, like:
+
+- user profiles
+- channel topic
+- channel purpose
+- message reactions
+- message attachments
+- and files
+
+Channels and users are always synchronized in every run, so every modification
+to the user or channels are overwriting old data. During first run, all messages
+are retrieved for all/selected channels. Every other run will only fetch those
+messages, which are older then newest message in the database - so that we don't
+loose any old messages, which might be automatically removed from Slack servers.
+The drawback of this behaviour is that all past messages which was altered in
+the meantime will not be updated.
 
 License
 -------

scripts/slack-backup

@@ -7,6 +7,7 @@ import argparse
 import logging
 
 from slack_backup import client
+from slack_backup import config
 
 
 def setup_logger(args):
@@ -47,60 +48,73 @@ def main():
 
     fetch = subparser.add_parser('fetch', help='Update local db with Slack'
                                  ' data')
-    fetch.add_argument('-t', '--token', required=True, help='Slack token - '
+    fetch.add_argument('-t', '--token', default=None, help='Slack token - '
                        'a string, which can be generated/obtained via '
                        'https://api.slack.com/docs/oauth-test-tokens page.')
-    fetch.add_argument('-u', '--user', default='', help='Username for your '
+    fetch.add_argument('-u', '--user', default=None, help='Username for your '
                        'Slack account')
-    fetch.add_argument('-p', '--password', default='', help='Password for your '
+    fetch.add_argument('-p', '--password', default=None, help='Password for '
-                       'Slack account.')
+                       'your Slack account.')
-    fetch.add_argument('-e', '--team', required=True, help='Team name, which '
+    fetch.add_argument('-e', '--team', default=None, help='Team name, which '
                        'is part of slack url, for example: if url is '
                        '"https://team.slack.com" than "team" is a name of '
                        'the team.')
-    fetch.add_argument('-v', '--verbose', help='Be verbose. Adding more "v"'
+    fetch.add_argument('-v', '--verbose', help='Be verbose. Adding more "v" '
-                       'will increase verbosity', action="count", default=0)
+                       'will increase verbosity', action="count",
-    fetch.add_argument('-q', '--quiet', help='Be quiet. Adding more "q"'
+                       default=None)
-                       'will decrease verbosity', action="count", default=0)
+    fetch.add_argument('-q', '--quiet', help='Be quiet. Adding more "q" will'
-    fetch.add_argument('-c', '--channels', default=[], nargs='+',
+                       ' decrease verbosity', action="count", default=None)
+    fetch.add_argument('-c', '--channels', default=None, nargs='+',
                        help='List of channels to perform actions on. '
                        'Default is all channels.')
 
-    fetch.add_argument('-d', '--database', default='',
+    fetch.add_argument('-d', '--database', default=None,
                        help='Path to the database file.')
 
+    fetch.add_argument('-i', '--config', default=None,
+                       help='Use specific config file.')
+
     fetch.set_defaults(func=fetch_data)
 
     generate = subparser.add_parser('generate', help='Generate logs out of '
                                     'data in provided database')
-    generate.add_argument('-o', '--output', default='logs', help="Output "
+    generate.add_argument('-o', '--output', default=None, help="Output "
                           "directory for store logs. All logs are organised "
                           "per channel. By default it's `logs' directory")
-    generate.add_argument('-f', '--format', default='none',
+    generate.add_argument('-f', '--format', default=None,
                           choices=('text', 'none'),
                           help='Output format. Default is none; only database '
                           'is updated by latest messages for all/selected '
                           'channels.')
-    generate.add_argument('-t', '--theme', default='plain',
+    generate.add_argument('-t', '--theme', default=None,
                           choices=('plain', 'unicode'),
                           help='Choose theme for text output. It doesn\'t '
                           'affect other output formats.')
 
-    generate.add_argument('-v', '--verbose', help='Be verbose. Adding more "v"'
+    generate.add_argument('-v', '--verbose', help='Be verbose. Adding more '
-                          'will increase verbosity', action="count", default=0)
+                          '"v" will increase verbosity', action="count",
-    generate.add_argument('-q', '--quiet', help='Be quiet. Adding more "q"'
+                          default=None)
-                          'will decrease verbosity', action="count", default=0)
+    generate.add_argument('-q', '--quiet', help='Be quiet. Adding more "q" '
+                          'will decrease verbosity', action="count",
+                          default=None)
     generate.add_argument('-c', '--channels', default=[], nargs='+',
                           help='List of channels to perform actions on. '
                           'Default is all channels.')
 
-    generate.add_argument('-d', '--database', default='',
+    generate.add_argument('-d', '--database', default=None,
                           help='Path to the database file.')
 
+    generate.add_argument('-i', '--config', default=None,
+                          help='Use specific config file.')
+
     generate.set_defaults(func=generate_raport)
 
     args = parser.parse_args()
+    cfg = config.Config()
+    msg = cfg.update(args)
     setup_logger(args)
+    logging.info(msg)
+
     args.func(args)
 
 

setup.py

@@ -2,12 +2,15 @@
 """
 Setup for the slack-backup project
 """
-from distutils.core import setup
+try:
+    from setuptools import setup
+except ImportError:
+    from distutils.core import setup
 
 
 setup(name="slack-backup",
       packages=["slack_backup"],
-      version="0.1",
+      version="0.4.2",
       description="Make copy of slack converstaions",
       author="Roman Dobosz",
       author_email="gryf73@gmail.com",
@@ -18,7 +21,7 @@ setup(name="slack-backup",
       scripts=["scripts/slack-backup"],
       classifiers=["Programming Language :: Python :: 3",
                    "Programming Language :: Python :: 3.4",
-                   "Development Status :: 2 - Pre-Alpha",
+                   "Development Status :: 3 - Alpha",
                    "Environment :: Console",
                    "Intended Audience :: End Users/Desktop",
                    "License :: OSI Approved :: BSD License",

slack_backup/client.py

@@ -146,10 +146,17 @@ class Client(object):
         Create message with corresponding possible metadata, like reactions,
         files etc.
         """
-        message = o.Message(data)
+        user = self.q(o.User).\
-        message.user = self.q(o.User).\
             filter(o.User.slackid == data['user']).one()
+
+        if data['type'] == 'message' and not data['text'].strip():
+            logging.info("Skipping message from `%s' since it's empty",
+                         user.name)
+            return
+
+        message = o.Message(data)
         message.channel = channel
+        message.user = user
 
         if data.get('is_starred'):
             message.is_starred = True

slack_backup/config.py

@@ -0,0 +1,112 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+
+"""
+Configuration module for slack-backup
+"""
+import json
+import os
+import configparser
+
+
+class Config(object):
+    """Configuration keeper"""
+
+    ints = ['verbose', 'quiet']
+
+    sections = {'common': ['channels', 'database', 'quiet', 'verbose'],
+                'fetch': ['user', 'password', 'team', 'token'],
+                'generate': ['output', 'format', 'theme']}
+
+    def __init__(self):
+        """
+        Init. Read config, if exists, and update passed argument parser
+        object.
+        """
+
+        self.cp = configparser.ConfigParser()
+        self._options = {'channels': [],
+                         'database': None,
+                         'quiet': 0,
+                         'verbose': 0,
+                         'user': None,
+                         'password': None,
+                         'team': None,
+                         'token': None,
+                         'output': None,
+                         'format': None,
+                         'theme': None}
+        # This message supposed to be displayed in INFO level. During the time
+        # of running the code where it should be displayed there is no
+        # complete information about logging level. Displaying message is
+        # dependent on the a) config file, b) argument from commandline. Let's
+        # resolve if user want to have that information or not after merging
+        # those two sources. If user do not want to see any message in INFO
+        # level, we shouldn't do so.
+        self.msg = ''
+
+    def update(self, args):
+        self.load_config(args)
+        self.parse_loaded_options()
+        self.update_args(args)
+        return self.msg
+
+    def load_config(self, args):
+
+        path = ''
+        if hasattr(args, 'config') and args.config:
+            path = args.config
+
+        locations = [path,
+                     './slack-backup.conf',
+                     os.path.expandvars('$XDG_CONFIG_HOME/slack-backup.ini'),
+                     os.path.expandvars('$HOME/.config/slack-backup.ini')]
+
+        for location in locations:
+            if os.path.exists(location):
+                self.cp.read(location)
+                self.msg = 'Found configuration file: %s' % location
+                break
+        else:
+            self.msg = 'No configuration file found'
+
+    def parse_loaded_options(self):
+
+        for section in self.cp.sections():
+            if section not in self.sections:
+                continue
+
+            for option in self.sections[section]:
+                if option in self.ints:
+                    val = self.cp.getint(section, option, fallback=0)
+                elif option == 'channels':
+                    val = self.cp.get(section, option, fallback='[]')
+                    val = json.loads(val)
+                else:
+                    val = self.cp.get(section, option, fallback=None)
+
+                self._options[option] = val
+
+    def update_args(self, args):
+        if 'parser' not in args:
+            # it doesn't make sense to update args, since no action was
+            # choosen
+            return
+
+        # special case, re-set information for verbose/quiet options
+        if 'verbose' in args and args.verbose is not None:
+            self._options['verbose'] = args.verbose
+            if self._options['quiet'] is not None:
+                self._options['quiet'] = 0
+        if 'quiet' in args and args.quiet is not None:
+            self._options['quiet'] = args.quiet
+            if self._options['verbose'] is not None:
+                self._options['verbose'] = 0
+
+        for sec_id in (args.parser, 'common'):
+            for option in self.sections[sec_id]:
+                if option in args:
+                    if getattr(args, option) is not None:
+                        continue
+
+                setattr(args, option, self._options[option])

slack_backup/download.py

@@ -78,7 +78,7 @@ class Download(object):
         path = os.path.join(path, fname)
         count = 1
 
-        while os.path.exists(path):
+        while filetype != 'avatar' and os.path.exists(path):
             base, ext = os.path.splitext(path)
             path = base + "%0.3d" % count + ext
 

slack_backup/reporters.py

@@ -1,18 +1,18 @@
-# -*- coding: utf-8 -*-
 """
 Reporters module.
 
 There are several classes for specific format reporting, and also some of the
 slack conversation/convention parsers.
 """
-from __future__ import absolute_import, division, print_function
 import os
 import errno
+import html.parser
 import logging
 import re
 
 from slack_backup import objects as o
 from slack_backup import utils
+from slack_backup import emoji
 
 
 class Reporter(object):
@@ -40,16 +40,18 @@ class Reporter(object):
                                     'file': '📂',
                                     'topic': '🟅',
                                     'separator': '│'}}
+        self.emoji = emoji.EMOJI.get(args.theme, {})
 
         self.channels = self._get_channels(args.channels)
         self.users = self.q(o.User).all()
-        self._re_first_idnick = re.compile(r'^(?P<replace>'
+        self._slackid_pat = [re.compile(r'^(?P<replace>'
-                                           r'<@(?P<slackid>U[A-Z,0-9]+)\|.+>)')
+                                        r'<@(?P<slackid>U[A-Z,0-9]+)\|.+>)'),
-        self._re_first_id = re.compile('^(?P<replace>'
+                             re.compile('^(?P<replace>'
-                                       '<@(?P<slackid>U[A-Z,0-9]+)>)')
+                                        '<@(?P<slackid>U[A-Z,0-9]+)>)'),
-        self._re_idnick = re.compile(r'.*(?P<replace>'
+                             re.compile(r'.*(?P<replace>'
-                                     r'<@(?P<slackid>U[A-Z,0-9]+)\|.+>)')
+                                        r'<@(?P<slackid>U[A-Z,0-9]+)\|.+>)'),
-        self._re_id = re.compile('.*(?P<replace><@(?P<slackid>U[A-Z,0-9]+)>)')
+                             re.compile('.*(?P<replace><@(?P<slackid>'
+                                        'U[A-Z,0-9]+)>)')]
 
     def generate(self):
         """Generate raport it's a dummmy one - for use with none reporter"""
@@ -154,11 +156,9 @@ class TextReporter(Reporter):
         """
         msg_txt = self._filter_slackid(msg.text)
         msg_txt = self._fix_newlines(msg_txt)
+        for emoticon in self.emoji:
+            msg_txt = msg_txt.replace(emoticon, self.emoji[emoticon])
         formatter = self.types.get(msg.type, self._msg)
-        if not msg_txt.strip():
-            logging.info("Skipping message from `%s' since it's empty",
-                         msg.user.name)
-            return ''
 
         return formatter(msg, msg_txt)
 
@@ -202,7 +202,7 @@ class TextReporter(Reporter):
 
     def _msg_file(self, msg, text):
         """return formatter for file"""
-        groups = self._re_first_idnick.match(msg.text).groupdict()
+        groups = self._slackid_pat[0].match(msg.text).groupdict()
         text = msg.text.replace(groups['replace'], '')
         filename = msg.file.filepath
         if filename:
@@ -211,14 +211,17 @@ class TextReporter(Reporter):
             filename = msg.file.url
 
         if not filename:
-            logging.warning("Dude, we have a file object, but nothing has "
+            logging.warning("There is a file object, but without filename."
-                            "found. Name of the file object is `i%s'",
+                            "Name of the file object is `%s'", msg.file.name)
-                            msg.file.name)
             filename = msg.file.name
 
         text = self._filter_slackid(text)
+        text = self._remove_entities(text)
         text = self._fix_newlines(text)
 
+        for emoticon in self.emoji:
+            text = text.replace(emoticon, self.emoji[emoticon])
+
         data = {'date': msg.datetime().strftime("%Y-%m-%d %H:%M:%S"),
                 'msg': text,
                 'max_len': self._max_len,
@@ -230,24 +233,40 @@ class TextReporter(Reporter):
                 'shared file "{filename}"{msg}\n'.format(**data))
 
     def _msg(self, msg, text):
-        """return formatter for /me"""
+        """return formatter for all other message types"""
+
         data = {'date': msg.datetime().strftime("%Y-%m-%d %H:%M:%S"),
                 'msg': text,
                 'max_len': self._max_len,
                 'separator': self._get_symbol('separator'),
                 'nick': msg.user.name}
-        return '{date} {nick:>{max_len}} {separator} {msg}\n'.format(**data)
+        result = '{date} {nick:>{max_len}} {separator} {msg}\n'.format(**data)
+
+        if msg.attachments:
+            for att in msg.attachments:
+                if att.title:
+                    att_text = "\n" + att.title + '\n'
+                else:
+                    att_text = "\n" + self._fix_newlines(att.fallback) + '\n'
+
+                if att.text:
+                    att_text += att.text
+
+                att_text = self._fix_newlines(att_text)
+                # remove first newline
+                att_text = att_text[1:]
+
+                result += att_text + '\n'
+
+        return result
+
+    def _remove_entities(self, text):
+        """replace html entites into appropriate chars"""
+        return html.parser.HTMLParser().unescape(text)
 
     def _filter_slackid(self, text):
         """filter out all of the id from slack"""
-        for pat in (self._re_first_idnick, self._re_first_id):
+        for pat in self._slackid_pat:
-            while pat.search(text):
-                groups = pat.search(text).groupdict('slackid')
-                user = [u for u in self.users
-                        if u.slackid == groups['slackid']][0]
-                text = text.replace(groups['replace'], user.name + ":")
-
-        for pat in (self._re_idnick, self._re_id):
             while pat.search(text):
                 groups = pat.search(text).groupdict('slackid')
                 user = [u for u in self.users

tests/test_client.py

@@ -1,8 +1,5 @@
 from unittest import TestCase
-try:
+from unittest.mock import MagicMock
-    from unittest.mock import MagicMock
-except ImportError:
-    from mock import MagicMock
 
 from slack_backup import client
 from slack_backup import objects as o

tests/test_config.py

@@ -0,0 +1,127 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+
+import argparse
+import tempfile
+import os
+import unittest
+
+from slack_backup import config
+
+
+CONF = """\
+[common]
+channels=["one","two", "three"]
+database=dbfname.sqlite
+quiet=1
+verbose=2
+
+[generate]
+output=logs
+format=text
+theme=plain
+
+[fetch]
+user=someuser@address.com
+password=secret
+team=myteam
+token=xxxx-1111111111-222222222222-333333333333-r4nd0ms7uff
+"""
+
+
+class TestConfig(unittest.TestCase):
+
+    def setUp(self):
+
+        fd, self.confname = tempfile.mkstemp()
+        os.close(fd)
+
+        with open(self.confname, 'w') as fobj:
+            fobj.write(CONF)
+
+    def tearDown(self):
+        os.unlink(self.confname)
+
+    def test_config(self):
+
+        self.assertTrue(os.path.exists(self.confname))
+        self.assertTrue(os.path.isfile(self.confname))
+
+        args = argparse.Namespace()
+        args.config = None
+        args.parser = 'fetch'
+        args.verbose = 2
+
+        conf = config.Config()
+        conf.update(args)
+        self.assertDictEqual(vars(args), {'config': None,
+                                          'parser': 'fetch',
+                                          'verbose': 2,
+                                          'quiet': 0,
+                                          'channels': [],
+                                          'database': None,
+                                          'user': None,
+                                          'password': None,
+                                          'team': None,
+                                          'token': None})
+
+        args = argparse.Namespace()
+        args.config = self.confname
+        args.parser = 'fetch'
+        args.verbose = 2
+        args.quiet = None
+        args.channels = None
+        args.database = None
+        args.user = None
+        args.password = None
+        args.team = None
+        args.token = None
+
+        conf = config.Config()
+        conf.update(args)
+
+        self.assertEqual(conf._options['verbose'], 2)
+        self.assertListEqual(conf._options['channels'],
+                             ['one', 'two', 'three'])
+        self.assertEqual(conf._options['database'], 'dbfname.sqlite')
+        self.assertEqual(conf._options['user'], 'someuser@address.com')
+
+        self.assertDictEqual(vars(args), {'config': self.confname,
+                                          'parser': 'fetch',
+                                          'verbose': 2,
+                                          'quiet': 0,
+                                          'channels': ['one', 'two', 'three'],
+                                          'database': 'dbfname.sqlite',
+                                          'user': 'someuser@address.com',
+                                          'password': 'secret',
+                                          'team': 'myteam',
+                                          'token': 'xxxx-1111111111-'
+                                                   '222222222222-333333333333-'
+                                                   'r4nd0ms7uff'})
+
+        # override some conf options with commandline
+        args = argparse.Namespace()
+        args.config = self.confname
+        args.parser = 'fetch'
+        args.verbose = None
+        args.quiet = 2
+        args.channels = ['foo']
+        args.database = None
+        args.user = 'joe'
+        args.password = 'ultricies'
+        args.team = ''
+        args.token = 'the token'
+
+        conf = config.Config()
+        conf.update(args)
+
+        self.assertDictEqual(vars(args), {'config': self.confname,
+                                          'parser': 'fetch',
+                                          'verbose': 0,
+                                          'quiet': 2,
+                                          'channels': ['foo'],
+                                          'database': 'dbfname.sqlite',
+                                          'user': 'joe',
+                                          'password': 'ultricies',
+                                          'team': '',
+                                          'token': 'the token'})

tests/test_reporter.py

@@ -1,10 +1,5 @@
-# -*- coding: utf-8 -*-
-
 from unittest import TestCase
-try:
+from unittest.mock import MagicMock
-    from unittest.mock import MagicMock
-except ImportError:
-    from mock import MagicMock
 
 from slack_backup import reporters as r
 
@@ -45,7 +40,3 @@ class TestReporter(TestCase):
         text = ('<@U111BBBBB|user2>Praesent vel enim sed eros luctus '
                 'imperdiet.\nMauris neque ante, <@U111DDDDD> placerat at, '
                 'mollis vitae, faucibus quis, <@U111CCCCC>leo. Ut feugiat.')
-
-# Praesent vel enim sed eros luctus imperdiet.  Vivamus urna quam, congue
-# vulputate, convallis non, cursus cursus, risus. Quisque aliquet. Donec
-# vulputate egestas elit. Morbi dictum, sem sit amet aliquam.

tox.ini

@@ -1,5 +1,5 @@
 [tox]
-envlist = py27,py34,{py27,py34}-flake8
+envlist = py34,py34-flake8
 
 usedevelop = True
 
@@ -10,18 +10,6 @@ commands = py.test --cov=slack_backup --cov-report=term-missing
 deps = -r{toxinidir}/requirements.txt
        -r{toxinidir}/test-requirements.txt
 
-[testenv:py27]
-usedevelop={[testenv]usedevelop}
-setenv = {[testenv]setenv}
-commands = {[testenv]commands}
-deps = {[testenv]deps}
-        mock
-
-[testenv:py27-flake8]
-basepython = python2.7
-deps = flake8
-commands = flake8 {posargs}
-
 [testenv:py34-flake8]
 basepython = python3.4
 deps = flake8