Fix for handling messages of different types than 'message'

If comment is sent by the user, different structure of the data is sent.
First of all, the type of this message is "message", but it contain
dictionary under 'comment' key, which can be confusing, which contain
needed data (like user id). For this kind of messages, in case of lack
of 'user' in main dict, dict['comment']['user'] will be used for getting
user identifier, while dict['text'] remains as a message text.

MANIFEST.in

@@ -0,0 +1,2 @@
+include README.rst
+include config_example

README.rst

@@ -68,7 +68,7 @@ typical session:
    (myenv)user@localhost ~/mylogs $ slack-backup fetch \
    --token xxxx-1111111111-222222222222-333333333333-r4nd0ms7uff \
    --user some@email.address.org --password secret --team myteam \
-   -qqq -d mydatabase.sqlite
+   -qq -d mydatabase.sqlite
 
 where:
 
@@ -116,6 +116,60 @@ command.
 
 See help for the ``slack-backup`` command for complete list of options.
 
+Configuration
+-------------
+
+For convenience, you can place all of needed options into configuration file
+(aka .ini), which all options (with their defaults) will look like:
+
+.. code:: ini
+
+   [common]
+   channels =
+   database =
+   quiet = 0
+   verbose = 0
+
+   [generate]
+   output =
+   format = text
+   theme = plain
+
+   [fetch]
+   user =
+   password =
+   team =
+   token =
+
+Note, that you don't have to put every option. To illustrate ``fetch`` example
+from above, here is a corresponding config file:
+
+.. code:: ini
+
+   [common]
+   database = mydatabase.sqlite
+   quiet = 2
+
+   [fetch]
+   user = some@email.address.org
+   password = secret
+   team = myteam
+   token = xxxx-1111111111-222222222222-333333333333-r4nd0ms7uff
+
+Note, that only ``[common]`` and ``[fetch]`` sections are provided, so it is
+enough to invoke ``slack-backup`` command as:
+
+.. code:: shell-session
+
+   (myenv)user@localhost ~/mylogs $ slack-backup fetch
+
+There are couple of places, where configuration file would be searched for, in
+particular order:
+
+* file provided via argument ``-i`` or ``--config``
+* ``slack-backup.ini`` in current directory
+* ``$XDG_CONFIG_HOME/slack-backup.ini``, where ``$XDG_CONFIG_HOME`` usually
+  defaults to ``$HOME/.config``
 
 Details
 -------

scripts/slack-backup

@@ -1,108 +1,10 @@
 #!/usr/bin/env python3
 # -*- coding: utf-8 -*-
 """
-Create backup for certain date for specified channel in slack
+Execute commands for slack-backup
 """
-import argparse
+from slack_backup import command
-import logging
-
-from slack_backup import client
-
-
-def setup_logger(args):
-    """Setup logger format and level"""
-
-    level = logging.WARNING
-
-    if args.quiet:
-        level = logging.ERROR
-        if args.quiet > 1:
-            level = logging.CRITICAL
-
-    if args.verbose:
-        level = logging.INFO
-        if args.verbose > 1:
-            level = logging.DEBUG
-
-    logging.basicConfig(level=level,
-                        format="%(asctime)s %(levelname)s: %(message)s")
-
-def generate_raport(args):
-    """Generate logs"""
-    slack = client.Client(args)
-    slack.generate_history()
-
-def fetch_data(args):
-    """Fetch and store data"""
-    slack = client.Client(args)
-    slack.update()
-
-
-def main():
-    """Main function"""
-    parser = argparse.ArgumentParser()
-
-    subparser = parser.add_subparsers(dest='parser')
-    subparser.required = True
-
-    fetch = subparser.add_parser('fetch', help='Update local db with Slack'
-                                 ' data')
-    fetch.add_argument('-t', '--token', required=True, help='Slack token - '
-                       'a string, which can be generated/obtained via '
-                       'https://api.slack.com/docs/oauth-test-tokens page.')
-    fetch.add_argument('-u', '--user', default='', help='Username for your '
-                       'Slack account')
-    fetch.add_argument('-p', '--password', default='', help='Password for your '
-                       'Slack account.')
-    fetch.add_argument('-e', '--team', required=True, help='Team name, which '
-                       'is part of slack url, for example: if url is '
-                       '"https://team.slack.com" than "team" is a name of '
-                       'the team.')
-    fetch.add_argument('-v', '--verbose', help='Be verbose. Adding more "v"'
-                       'will increase verbosity', action="count", default=0)
-    fetch.add_argument('-q', '--quiet', help='Be quiet. Adding more "q"'
-                       'will decrease verbosity', action="count", default=0)
-    fetch.add_argument('-c', '--channels', default=[], nargs='+',
-                       help='List of channels to perform actions on. '
-                       'Default is all channels.')
-
-    fetch.add_argument('-d', '--database', default='',
-                       help='Path to the database file.')
-
-    fetch.set_defaults(func=fetch_data)
-
-    generate = subparser.add_parser('generate', help='Generate logs out of '
-                                    'data in provided database')
-    generate.add_argument('-o', '--output', default='logs', help="Output "
-                          "directory for store logs. All logs are organised "
-                          "per channel. By default it's `logs' directory")
-    generate.add_argument('-f', '--format', default='none',
-                          choices=('text', 'none'),
-                          help='Output format. Default is none; only database '
-                          'is updated by latest messages for all/selected '
-                          'channels.')
-    generate.add_argument('-t', '--theme', default='plain',
-                          choices=('plain', 'unicode'),
-                          help='Choose theme for text output. It doesn\'t '
-                          'affect other output formats.')
-
-    generate.add_argument('-v', '--verbose', help='Be verbose. Adding more "v"'
-                          'will increase verbosity', action="count", default=0)
-    generate.add_argument('-q', '--quiet', help='Be quiet. Adding more "q"'
-                          'will decrease verbosity', action="count", default=0)
-    generate.add_argument('-c', '--channels', default=[], nargs='+',
-                          help='List of channels to perform actions on. '
-                          'Default is all channels.')
-
-    generate.add_argument('-d', '--database', default='',
-                          help='Path to the database file.')
-
-    generate.set_defaults(func=generate_raport)
-
-    args = parser.parse_args()
-    setup_logger(args)
-    args.func(args)
 
 
 if __name__ == "__main__":
-    main()
+    command.main()

setup.py

@@ -10,7 +10,7 @@ except ImportError:
 
 setup(name="slack-backup",
       packages=["slack_backup"],
-      version="0.3",
+      version="0.5",
       description="Make copy of slack converstaions",
       author="Roman Dobosz",
       author_email="gryf73@gmail.com",

slack_backup/client.py

@@ -112,6 +112,7 @@ class Client(object):
             channels = all_channels
 
         for channel in channels:
+            logging.info("Getting messages for channel `%s'", channel.name)
             latest = self.q(o.Message).\
                 filter(o.Message.channel == channel).\
                 order_by(o.Message.ts.desc()).first()
@@ -146,10 +147,18 @@ class Client(object):
         Create message with corresponding possible metadata, like reactions,
         files etc.
         """
-        user = self.q(o.User).\
+        if data['type'] != 'message':
-            filter(o.User.slackid == data['user']).one()
+            logging.info("Skipping message of type `%s'.", data['type'])
+            return
 
-        if data['type'] == 'message' and not data['text'].strip():
+        try:
+            user = self.q(o.User).\
+                filter(o.User.slackid == data['user']).one()
+        except KeyError:
+            user = self.q(o.User).\
+                    filter(o.User.slackid == data['comment']['user']).one()
+
+        if not data['text'].strip():
             logging.info("Skipping message from `%s' since it's empty",
                          user.name)
             return
@@ -255,7 +264,7 @@ class Client(object):
         if not database:
             return 'assets'
 
-        path = os.path.dirname(os.path.abspath(database))
+        path = os.path.dirname(database)
         return os.path.join(path, 'assets')
 
     def _channels_list(self):

slack_backup/command.py

@@ -0,0 +1,118 @@
+"""
+Create backup for certain date for specified channel in slack
+"""
+import argparse
+import logging
+
+from slack_backup import client
+from slack_backup import config
+
+
+def setup_logger(args):
+    """Setup logger format and level"""
+
+    level = logging.WARNING
+
+    if args.quiet:
+        level = logging.ERROR
+        if args.quiet > 1:
+            level = logging.CRITICAL
+
+    if args.verbose:
+        level = logging.INFO
+        if args.verbose > 1:
+            level = logging.DEBUG
+
+    logging.basicConfig(level=level,
+                        format="%(asctime)s %(levelname)s: %(message)s")
+
+
+def generate_raport(args):
+    """Generate logs"""
+    slack = client.Client(args)
+    slack.generate_history()
+
+
+def fetch_data(args):
+    """Fetch and store data"""
+    slack = client.Client(args)
+    slack.update()
+
+
+def main():
+    """Main function"""
+    parser = argparse.ArgumentParser()
+
+    subparser = parser.add_subparsers(dest='parser')
+    subparser.required = True
+
+    fetch = subparser.add_parser('fetch', help='Update local db with Slack'
+                                 ' data')
+    fetch.add_argument('-t', '--token', default=None, help='Slack token - '
+                       'a string, which can be generated/obtained via '
+                       'https://api.slack.com/docs/oauth-test-tokens page.')
+    fetch.add_argument('-u', '--user', default=None, help='Username for your '
+                       'Slack account')
+    fetch.add_argument('-p', '--password', default=None, help='Password for '
+                       'your Slack account.')
+    fetch.add_argument('-e', '--team', default=None, help='Team name, which '
+                       'is part of slack url, for example: if url is '
+                       '"https://team.slack.com" than "team" is a name of '
+                       'the team.')
+    fetch.add_argument('-v', '--verbose', help='Be verbose. Adding more "v" '
+                       'will increase verbosity', action="count",
+                       default=None)
+    fetch.add_argument('-q', '--quiet', help='Be quiet. Adding more "q" will'
+                       ' decrease verbosity', action="count", default=None)
+    fetch.add_argument('-c', '--channels', default=None, nargs='+',
+                       help='List of channels to perform actions on. '
+                       'Default is all channels.')
+
+    fetch.add_argument('-d', '--database', default=None,
+                       help='Path to the database file.')
+
+    fetch.add_argument('-i', '--config', default=None,
+                       help='Use specific config file.')
+
+    fetch.set_defaults(func=fetch_data)
+
+    generate = subparser.add_parser('generate', help='Generate logs out of '
+                                    'data in provided database')
+    generate.add_argument('-o', '--output', default=None, help="Output "
+                          "directory for store logs. All logs are organised "
+                          "per channel. By default it's `logs' directory")
+    generate.add_argument('-f', '--format', default=None,
+                          choices=('text', 'none'),
+                          help='Output format. Default is none; only database '
+                          'is updated by latest messages for all/selected '
+                          'channels.')
+    generate.add_argument('-t', '--theme', default=None,
+                          choices=('plain', 'unicode'),
+                          help='Choose theme for text output. It doesn\'t '
+                          'affect other output formats.')
+
+    generate.add_argument('-v', '--verbose', help='Be verbose. Adding more '
+                          '"v" will increase verbosity', action="count",
+                          default=None)
+    generate.add_argument('-q', '--quiet', help='Be quiet. Adding more "q" '
+                          'will decrease verbosity', action="count",
+                          default=None)
+    generate.add_argument('-c', '--channels', default=[], nargs='+',
+                          help='List of channels to perform actions on. '
+                          'Default is all channels.')
+
+    generate.add_argument('-d', '--database', default=None,
+                          help='Path to the database file.')
+
+    generate.add_argument('-i', '--config', default=None,
+                          help='Use specific config file.')
+
+    generate.set_defaults(func=generate_raport)
+
+    args = parser.parse_args()
+    cfg = config.Config()
+    msg = cfg.update(args)
+    setup_logger(args)
+    logging.info(msg)
+
+    args.func(args)

slack_backup/config.py

@@ -0,0 +1,112 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+
+"""
+Configuration module for slack-backup
+"""
+import json
+import os
+import configparser
+
+
+class Config(object):
+    """Configuration keeper"""
+
+    ints = ['verbose', 'quiet']
+
+    sections = {'common': ['channels', 'database', 'quiet', 'verbose'],
+                'fetch': ['user', 'password', 'team', 'token'],
+                'generate': ['output', 'format', 'theme']}
+
+    def __init__(self):
+        """
+        Init. Read config, if exists, and update passed argument parser
+        object.
+        """
+
+        self.cp = configparser.ConfigParser()
+        self._options = {'channels': [],
+                         'database': None,
+                         'quiet': 0,
+                         'verbose': 0,
+                         'user': None,
+                         'password': None,
+                         'team': None,
+                         'token': None,
+                         'output': None,
+                         'format': None,
+                         'theme': None}
+        # This message supposed to be displayed in INFO level. During the time
+        # of running the code where it should be displayed there is no
+        # complete information about logging level. Displaying message is
+        # dependent on the a) config file, b) argument from commandline. Let's
+        # resolve if user want to have that information or not after merging
+        # those two sources. If user do not want to see any message in INFO
+        # level, we shouldn't do so.
+        self.msg = ''
+
+    def update(self, args):
+        self.load_config(args)
+        self.parse_loaded_options()
+        self.update_args(args)
+        return self.msg
+
+    def load_config(self, args):
+
+        path = ''
+        if hasattr(args, 'config') and args.config:
+            path = args.config
+
+        locations = [path,
+                     './slack-backup.ini',
+                     os.path.expandvars('$XDG_CONFIG_HOME/slack-backup.ini'),
+                     os.path.expandvars('$HOME/.config/slack-backup.ini')]
+
+        for location in locations:
+            if os.path.exists(location):
+                self.cp.read(location)
+                self.msg = 'Found configuration file: %s' % location
+                break
+        else:
+            self.msg = 'No configuration file found'
+
+    def parse_loaded_options(self):
+
+        for section in self.cp.sections():
+            if section not in self.sections:
+                continue
+
+            for option in self.sections[section]:
+                if option in self.ints:
+                    val = self.cp.getint(section, option, fallback=0)
+                elif option == 'channels':
+                    val = self.cp.get(section, option, fallback='[]')
+                    val = json.loads(val)
+                else:
+                    val = self.cp.get(section, option, fallback=None)
+
+                self._options[option] = val
+
+    def update_args(self, args):
+        if 'parser' not in args:
+            # it doesn't make sense to update args, since no action was
+            # choosen
+            return
+
+        # special case, re-set information for verbose/quiet options
+        if 'verbose' in args and args.verbose is not None:
+            self._options['verbose'] = args.verbose
+            if self._options['quiet'] is not None:
+                self._options['quiet'] = 0
+        if 'quiet' in args and args.quiet is not None:
+            self._options['quiet'] = args.quiet
+            if self._options['verbose'] is not None:
+                self._options['verbose'] = 0
+
+        for sec_id in (args.parser, 'common'):
+            for option in self.sections[sec_id]:
+                if option in args:
+                    if getattr(args, option) is not None:
+                        continue
+
+                setattr(args, option, self._options[option])

slack_backup/download.py

@@ -55,8 +55,8 @@ class Download(object):
                    'file': self._files}
 
         if filetype == 'file' and not self._authorized:
-            logging.info("There was no (valid) credentials passed, therefore "
+            logging.warning("There was no (valid) credentials passed, "
-                         "file `%s' cannot be downloaded", url)
+                            "therefore file `%s' cannot be downloaded", url)
             return
 
         splitted = url.split('/')
@@ -79,8 +79,10 @@ class Download(object):
         count = 1
 
         while filetype != 'avatar' and os.path.exists(path):
-            base, ext = os.path.splitext(path)
+            if count == 1:
-            path = base + "%0.3d" % count + ext
+                base, ext = os.path.splitext(path)
+            path = base + ".%0.3d" % count + ext
+            count += 1
 
         return path
 

tests/test_client.py

@@ -324,7 +324,10 @@ MSGS = {'messages': [{"type": "message",
                               "<https://esm64.slack.com/files/name2/F3405RRB5/"
                               "screenshot.png|Screenshot.png>",
                       "ts": "1478107371.000052",
-                      "upload": True}],
+                      "upload": True},
+                     {'type': 'something else',
+                      'ts': '1502003415232.000001',
+                      "wibblr": True}],
         "ok": True,
         "latest": "1479501075.000020",
         "has_more": True}
@@ -389,7 +392,7 @@ class TestApiCalls(TestCase):
                                          "C00000001").one()
 
         msg, ts = cl._channels_history(channel, 0)
-        self.assertEqual(len(msg), 5)
+        self.assertEqual(len(msg), 6)
         self.assertEqual(ts, '1479501074.000032')
 
         msg, ts = cl._channels_history(channel, ts)

tests/test_config.py

@@ -0,0 +1,127 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+
+import argparse
+import tempfile
+import os
+import unittest
+
+from slack_backup import config
+
+
+CONF = """\
+[common]
+channels=["one","two", "three"]
+database=dbfname.sqlite
+quiet=1
+verbose=2
+
+[generate]
+output=logs
+format=text
+theme=plain
+
+[fetch]
+user=someuser@address.com
+password=secret
+team=myteam
+token=xxxx-1111111111-222222222222-333333333333-r4nd0ms7uff
+"""
+
+
+class TestConfig(unittest.TestCase):
+
+    def setUp(self):
+
+        fd, self.confname = tempfile.mkstemp()
+        os.close(fd)
+
+        with open(self.confname, 'w') as fobj:
+            fobj.write(CONF)
+
+    def tearDown(self):
+        os.unlink(self.confname)
+
+    def test_config(self):
+
+        self.assertTrue(os.path.exists(self.confname))
+        self.assertTrue(os.path.isfile(self.confname))
+
+        args = argparse.Namespace()
+        args.config = None
+        args.parser = 'fetch'
+        args.verbose = 2
+
+        conf = config.Config()
+        conf.update(args)
+        self.assertDictEqual(vars(args), {'config': None,
+                                          'parser': 'fetch',
+                                          'verbose': 2,
+                                          'quiet': 0,
+                                          'channels': [],
+                                          'database': None,
+                                          'user': None,
+                                          'password': None,
+                                          'team': None,
+                                          'token': None})
+
+        args = argparse.Namespace()
+        args.config = self.confname
+        args.parser = 'fetch'
+        args.verbose = 2
+        args.quiet = None
+        args.channels = None
+        args.database = None
+        args.user = None
+        args.password = None
+        args.team = None
+        args.token = None
+
+        conf = config.Config()
+        conf.update(args)
+
+        self.assertEqual(conf._options['verbose'], 2)
+        self.assertListEqual(conf._options['channels'],
+                             ['one', 'two', 'three'])
+        self.assertEqual(conf._options['database'], 'dbfname.sqlite')
+        self.assertEqual(conf._options['user'], 'someuser@address.com')
+
+        self.assertDictEqual(vars(args), {'config': self.confname,
+                                          'parser': 'fetch',
+                                          'verbose': 2,
+                                          'quiet': 0,
+                                          'channels': ['one', 'two', 'three'],
+                                          'database': 'dbfname.sqlite',
+                                          'user': 'someuser@address.com',
+                                          'password': 'secret',
+                                          'team': 'myteam',
+                                          'token': 'xxxx-1111111111-'
+                                                   '222222222222-333333333333-'
+                                                   'r4nd0ms7uff'})
+
+        # override some conf options with commandline
+        args = argparse.Namespace()
+        args.config = self.confname
+        args.parser = 'fetch'
+        args.verbose = None
+        args.quiet = 2
+        args.channels = ['foo']
+        args.database = None
+        args.user = 'joe'
+        args.password = 'ultricies'
+        args.team = ''
+        args.token = 'the token'
+
+        conf = config.Config()
+        conf.update(args)
+
+        self.assertDictEqual(vars(args), {'config': self.confname,
+                                          'parser': 'fetch',
+                                          'verbose': 0,
+                                          'quiet': 2,
+                                          'channels': ['foo'],
+                                          'database': 'dbfname.sqlite',
+                                          'user': 'joe',
+                                          'password': 'ultricies',
+                                          'team': '',
+                                          'token': 'the token'})