Version bump to 0.7

Uploaded cookie check for logins

setup.py

@@ -10,7 +10,7 @@ except ImportError:
 
 setup(name="slack-backup",
       packages=["slack_backup"],
-      version="0.4.4",
+      version="0.7",
       description="Make copy of slack converstaions",
       author="Roman Dobosz",
       author_email="gryf73@gmail.com",
@@ -21,7 +21,7 @@ setup(name="slack-backup",
       scripts=["scripts/slack-backup"],
       classifiers=["Programming Language :: Python :: 3",
                    "Programming Language :: Python :: 3.4",
-                   "Development Status :: 3 - Alpha",
+                   "Development Status :: 4 - Beta",
                    "Environment :: Console",
                    "Intended Audience :: End Users/Desktop",
                    "License :: OSI Approved :: BSD License",

slack_backup/client.py

@@ -3,6 +3,7 @@ Create backup for certain date for specified channel in slack
 """
 from datetime import datetime
 import getpass
+import json
 import logging
 import os
 
@@ -112,6 +113,7 @@ class Client(object):
             channels = all_channels
 
         for channel in channels:
+            logging.info("Getting messages for channel `%s'", channel.name)
             latest = self.q(o.Message).\
                 filter(o.Message.channel == channel).\
                 order_by(o.Message.ts.desc()).first()
@@ -146,10 +148,20 @@ class Client(object):
         Create message with corresponding possible metadata, like reactions,
         files etc.
         """
-        user = self.q(o.User).\
+        if data['type'] != 'message':
-            filter(o.User.slackid == data['user']).one()
+            logging.info("Skipping message of type `%s'.", data['type'])
+            return
 
-        if data['type'] == 'message' and not data['text'].strip():
+        logging.debug('Message data: %s', json.dumps(data))
+
+        try:
+            user = self.q(o.User).\
+                filter(o.User.slackid == data['user']).one()
+        except KeyError:
+            user = self.q(o.User).\
+                    filter(o.User.slackid == data['comment']['user']).one()
+
+        if not data['text'].strip():
             logging.info("Skipping message from `%s' since it's empty",
                          user.name)
             return
@@ -255,7 +267,7 @@ class Client(object):
         if not database:
             return 'assets'
 
-        path = os.path.dirname(os.path.abspath(database))
+        path = os.path.dirname(database)
         return os.path.join(path, 'assets')
 
     def _channels_list(self):

slack_backup/command.py

@@ -3,6 +3,7 @@ Create backup for certain date for specified channel in slack
 """
 import argparse
 import logging
+import platform
 
 from slack_backup import client
 from slack_backup import config
@@ -11,6 +12,28 @@ from slack_backup import config
 def setup_logger(args):
     """Setup logger format and level"""
 
+    if platform.system() != "Windows":
+        # hack to have colors in terminal
+        logging.addLevelName(logging.DEBUG,
+                             "\033[1;30m%s\033[1;0m" %
+                             logging.getLevelName(logging.DEBUG))
+
+        logging.addLevelName(logging.INFO,
+                             "\033[1;32m%s\033[1;0m" %
+                             logging.getLevelName(logging.INFO))
+
+        logging.addLevelName(logging.WARNING,
+                             "\033[1;33m%s\033[1;0m" %
+                             logging.getLevelName(logging.WARNING))
+
+        logging.addLevelName(logging.ERROR,
+                             "\033[1;31m%s\033[1;0m" %
+                             logging.getLevelName(logging.ERROR))
+
+        logging.addLevelName(logging.CRITICAL,
+                             "\033[7;31m%s\033[1;0m" %
+                             logging.getLevelName(logging.CRITICAL))
+
     level = logging.WARNING
 
     if args.quiet:

slack_backup/download.py

@@ -2,14 +2,45 @@
 Module for download files, store them in local filesystem and convert the URLs
 to local ones, so that sophisticated writers can make a use of it
 """
+import functools
 import logging
 import os
+import shutil
 
 import requests
 
 from slack_backup import utils
 
 
+def retry(count):
+    """
+    Decorator for a case, when there is some network hiccup, or slack servers
+    are too busy to respond or on connection timeout. Parameter count says how
+    many times it should try to perform request.
+    """
+    def wrapper(func):
+        @functools.wraps(func)
+        def inner(obj, *args, **kwargs):
+            counter = count
+
+            while counter:
+                counter -= 1
+                try:
+                    return func(obj, *args, **kwargs)
+                except requests.exceptions.RequestException as exc:
+                    if not counter:
+                        logging.error('Request for %s failed. Reported '
+                                      'reason: %s', args[0], exc.__doc__)
+                        raise
+                    logging.warning('Request for %s failed. Reported '
+                                    'reason: %s. Retrying.', args[0],
+                                    exc.__doc__)
+                    # Renew the session before retry
+                    obj.authorize()
+        return inner
+    return wrapper
+
+
 class NotAuthorizedError(requests.HTTPError):
     pass
 
@@ -18,7 +49,7 @@ class Download(object):
     """Download class for taking care of Slack internally uploaded files"""
 
     def __init__(self, args, assets_dir):
-        self.session = requests.session()
+        self.session = None
         self.team = args.team
         self.user = args.user
         self.password = args.password
@@ -37,10 +68,26 @@ class Download(object):
         if not self._hier_created:
             self._create_assets_dir()
 
-        fname = self.prepare_filepath(url, filetype)
+        filepath = self.get_filepath(url, filetype)
+        temp_file = utils.get_temp_name()
 
-        self._download(url, fname)
+        self._download(url, temp_file)
-        return fname
+
+        if filepath and os.path.exists(filepath) and filetype != 'avatar':
+            if not utils.same_files(filepath, temp_file):
+                logging.warning("File `%s' already exist, renamed to `%s'",
+                                filepath,
+                                self.calculate_new_filename(filepath,
+                                                            filetype))
+                filepath = self.calculate_new_filename(filepath, filetype)
+                shutil.move(temp_file, filepath)
+            else:
+                logging.debug("File `%s' already exist, skipping", filepath)
+                os.unlink(filepath)
+        else:
+            shutil.move(temp_file, filepath)
+
+        return filepath
 
     def _create_assets_dir(self):
         for path in (self._files, self._images):
@@ -48,8 +95,8 @@ class Download(object):
 
         self._hier_created = True
 
-    def prepare_filepath(self, url, filetype):
+    def get_filepath(self, url, filetype):
-        """Prepare directory where to download file into"""
+        """Get full path and filename for the file"""
 
         typemap = {'avatar': self._images,
                    'file': self._files}
@@ -61,7 +108,8 @@ class Download(object):
 
         splitted = url.split('/')
 
-        if len(splitted) == 7 and 'slack.com' in splitted[2]:
+        if len(splitted) == 7 and ('slack.com' in splitted[2] or
+                                   'slack-edge.com' in splitted[2]):
             part = url.split('/')[-3]
             fname = url.split('/')[-1]
         else:
@@ -75,24 +123,24 @@ class Download(object):
             utils.makedirs(os.path.join(path, part))
             path = os.path.join(path, part)
 
-        path = os.path.join(path, fname)
+        return os.path.join(path, fname)
+
+    def calculate_new_filename(self, path, filetype):
         count = 1
 
         while filetype != 'avatar' and os.path.exists(path):
-            base, ext = os.path.splitext(path)
+            if count == 1:
-            path = base + "%0.3d" % count + ext
+                base, ext = os.path.splitext(path)
+            path = base + ".%0.3d" % count + ext
+            count += 1
 
         return path
 
+    @retry(3)
     def _download(self, url, local):
         """Download file"""
 
-        try:
+        res = self.session.get(url, stream=True)
-            res = self.session.get(url, stream=True)
-        except requests.exceptions.RequestException as exc:
-            logging.error('Request for %s failed. Reported reason: %s',
-                          url, exc.__doc__)
-            raise
 
         with open(local, 'wb') as fobj:
             for chunk in res.iter_content(chunk_size=5120):
@@ -104,7 +152,9 @@ class Download(object):
         """
         Authenticate and gather session for Slack
         """
+        self.session = requests.session()  # new session
         res = self.session.get('https://%s.slack.com/' % self.team)
+
         if not all((self.team, self.password, self.user)):
             logging.warning('There is neither username, password or team name'
                             ' provided. Downloading will not be performed.')
@@ -112,7 +162,7 @@ class Download(object):
 
         crumb = ''
         for line in res.text.split('\n'):
-            if 'crumb' in line:
+            if 'crumb' in line and 'value' in line:
                 crumb = line.split('value=')[1].split('"')[1]
                 break
         else:
@@ -125,8 +175,7 @@ class Download(object):
                                  'password': self.password,
                                  'signin': 1})
         self.cookies = requests.utils.dict_from_cookiejar(self.session.cookies)
-        if not ('a' in self.cookies and 'b' in self.cookies and
+        if not ('d' in self.cookies and 'd-s' in self.cookies):
-                ('a-' + self.cookies['a']) in self.cookies):
             logging.error('Failed to login into Slack app')
         else:
             self._authorized = True

slack_backup/utils.py

@@ -4,6 +4,8 @@ Some utils functions. Jsut to not copypaste the code around
 import errno
 import os
 import logging
+import tempfile
+import hashlib
 
 
 def makedirs(path):
@@ -19,3 +21,24 @@ def makedirs(path):
                 logging.error("Cannot create `%s'. There is some file on the "
                               "way; cannot proceed.", path)
                 raise
+
+
+def get_temp_name():
+    """Return temporary file name"""
+    fdesc, fname = tempfile.mkstemp()
+    os.close(fdesc)
+    return fname
+
+
+def same_files(file1, file2):
+    """
+    Compare files by calculating hash for each of them. Return True if hash is
+    identical, False otherwise
+    """
+    with open(file1, 'rb') as fobj:
+        hash1 = hashlib.sha256(fobj.read())
+
+    with open(file2, 'rb') as fobj:
+        hash2 = hashlib.sha256(fobj.read())
+
+    return hash1.hexdigest() == hash2.hexdigest()

tests/test_client.py

@@ -1,5 +1,5 @@
 from unittest import TestCase
-from unittest.mock import MagicMock
+from unittest import mock
 
 from slack_backup import client
 from slack_backup import objects as o
@@ -324,7 +324,10 @@ MSGS = {'messages': [{"type": "message",
                               "<https://esm64.slack.com/files/name2/F3405RRB5/"
                               "screenshot.png|Screenshot.png>",
                       "ts": "1478107371.000052",
-                      "upload": True}],
+                      "upload": True},
+                     {'type': 'something else',
+                      'ts': '1502003415232.000001',
+                      "wibblr": True}],
         "ok": True,
         "latest": "1479501075.000020",
         "has_more": True}
@@ -362,34 +365,34 @@ class TestApiCalls(TestCase):
 
     def test_channels_list(self):
         cl = client.Client(FakeArgs())
-        cl.slack.api_call = MagicMock(return_value=CHANNELS)
+        cl.slack.api_call = mock.MagicMock(return_value=CHANNELS)
         channels = cl._channels_list()
         self.assertListEqual(CHANNELS['channels'], channels)
 
     def test_users_list(self):
         cl = client.Client(FakeArgs())
-        cl.slack.api_call = MagicMock(return_value=USERS)
+        cl.slack.api_call = mock.MagicMock(return_value=USERS)
         users = cl._users_list()
         self.assertListEqual(USERS['members'], users)
 
     def test_channels_history(self):
         cl = client.Client(FakeArgs())
 
-        cl.slack.api_call = MagicMock(return_value=USERS)
+        cl.slack.api_call = mock.MagicMock(return_value=USERS)
-        cl.downloader._download = MagicMock(return_value=None)
+        cl.downloader._download = mock.MagicMock(return_value=None)
         cl.update_users()
 
-        cl.slack.api_call = MagicMock(return_value=CHANNELS)
+        cl.slack.api_call = mock.MagicMock(return_value=CHANNELS)
         cl.update_channels()
 
-        cl.slack.api_call = MagicMock()
+        cl.slack.api_call = mock.MagicMock()
         cl.slack.api_call.side_effect = [MSGS, MSG2, MSG3]
 
         channel = cl.q(o.Channel).filter(o.Channel.slackid ==
                                          "C00000001").one()
 
         msg, ts = cl._channels_history(channel, 0)
-        self.assertEqual(len(msg), 5)
+        self.assertEqual(len(msg), 6)
         self.assertEqual(ts, '1479501074.000032')
 
         msg, ts = cl._channels_history(channel, ts)
@@ -405,8 +408,8 @@ class TestClient(TestCase):
 
     def test_update_users(self):
         cl = client.Client(FakeArgs())
-        cl.slack.api_call = MagicMock(return_value=USERS)
+        cl.slack.api_call = mock.MagicMock(return_value=USERS)
-        cl.downloader._download = MagicMock(return_value=None)
+        cl.downloader._download = mock.MagicMock(return_value=None)
         cl.update_users()
         users = cl.session.query(o.User).all()
         self.assertEqual(len(users), 4)
@@ -426,19 +429,22 @@ class TestMessage(TestCase):
         args.channels = ['general']
 
         self.cl = client.Client(args)
-        self.cl.downloader.authorize = MagicMock()
+        self.cl.downloader.authorize = mock.MagicMock()
-        self.cl.slack.api_call = MagicMock(return_value=USERS)
+        self.cl.slack.api_call = mock.MagicMock(return_value=USERS)
-        self.cl.downloader._download = MagicMock(return_value=None)
+        self.cl.downloader._download = mock.MagicMock(return_value=None)
         self.cl.update_users()
 
-        self.cl.slack.api_call = MagicMock(return_value=CHANNELS)
+        self.cl.slack.api_call = mock.MagicMock(return_value=CHANNELS)
         self.cl.update_channels()
 
-        self.cl.slack.api_call = MagicMock()
+        self.cl.slack.api_call = mock.MagicMock()
 
-    def test_update_history(self):
+    @mock.patch('slack_backup.download.Download.download')
+    def test_update_history(self, download):
 
-        self.cl.downloader._download = MagicMock(return_value=None)
+        download.return_value = 'foo'
+
+        self.cl.downloader._download = mock.MagicMock(return_value=None)
         self.cl.slack.api_call.side_effect = [MSGS, MSG3]
         self.cl.update_history()
         self.assertEqual(len(self.cl.q(o.Message).all()), 5)