diff --git a/README.rst b/README.rst index e212b49..3652b6d 100644 --- a/README.rst +++ b/README.rst @@ -90,6 +90,28 @@ where: created, but you'll (obviously) lost all the records. Besides the db file, assets directory might be created for downloadable items. +There is one more switch to take into consideration - +``-f/--url_file_to_attachment`` which influence the way how external file +share would be treated. First of all, what is *external* file share from slack +point of view, one could ask. Slack have some sort of integration with Goolgle +services, like Googla Drive, which provide slack users to create or "upload" +files from Google Drive. "Upload", since no uploading actually takes place, +and only URL is provided for such "uploads". By default `slack-backup` will +create a file which is prefixed ``manual_download_`` which will contain url and +destination path to the file, where user should manual download file to. +Example file contents: + +.. code:: + + http://foo.bar.com/some/file --> assets/files/83340cbe-fee2-4d2e-bdb1-cace9c82e6d4 + http://foo.bar.com/some/other/file --> assets/files/8a4c873c-1864-4f1b-b515-bbef119f33a3 + http://docs/google.com/some/gdoc/file --> assets/files/ec8752bc-0bf8-4743-a8bd-9756107ab386 + +By setting ``url_file_to_attachment`` flag (or making it set to true in config +file) such "uploads" would be internally converted into Slack "attachment", +which internally is an object to store external links, so there is no need for +user interaction. + During DB creation, all available messages are stored in the database. On the next run, ``fetch`` would only take those records, which are older from currently oldest in DB. So that it will only fetch a subset of the overall of @@ -136,6 +158,7 @@ For convenience, you can place all of needed options into configuration file theme = plain [fetch] + url_file_to_attachment = false user = password = team = diff --git a/slack_backup/client.py b/slack_backup/client.py index 4f952e5..a147e42 100644 --- a/slack_backup/client.py +++ b/slack_backup/client.py @@ -6,6 +6,7 @@ import json import logging import os import pprint +import uuid import slackclient import sqlalchemy.orm.exc @@ -46,6 +47,13 @@ class Client(object): if 'format' in args: self.reporter = reporters.get_reporter(args, self.q) + self._url_file_to_attachment = args.url_file_to_attachment + + self._dlpath = utils.get_temp_name(dir=os.path.curdir, + prefix='manual_download_', + unlink=True) + self._dldata = [] + def update(self): """ Perform an update, store data to db @@ -54,6 +62,7 @@ class Client(object): self.update_users() self.update_channels() self.update_history() + self._finalize() def update_channels(self): """Fetch and update channel list with current state in db""" @@ -210,19 +219,31 @@ class Client(object): message.reactions.append(o.Reaction(reaction_data)) if data.get('subtype') == 'file_share': - self._file_data(message, data['file'], data['file']['is_external']) + if (self._url_file_to_attachment and + data['file'].get('is_external')): + fdata = data['file'] + # change message type from file_share to default + message.type = '' + message.text = (message.text.split('shared a file:')[0] + + 'shared a file: ') + logging.debug("Found external file `%s'. Saving as " + "attachment.", fdata['url_private']) + self._att_data(message, [{'title': fdata['name'], + 'text': fdata['url_private'], + 'fallback': ''}]) + else: + self._file_data(message, data['file']) elif data.get('subtype') == 'pinned_item': if data.get('attachments'): self._att_data(message, data['attachments']) elif data.get('item'): - self._file_data(message, data['item'], - data['item']['is_external']) + self._file_data(message, data['item']) elif data.get('attachments'): self._att_data(message, data['attachments']) self.session.add(message) - def _file_data(self, message, data, is_external=True): + def _file_data(self, message, data): """ Process file data. Could be either represented as 'file' object or 'item' object in case of pinned items @@ -231,8 +252,15 @@ class Client(object): if data.get('is_starred'): message.is_starred = True - if is_external: - logging.debug("Found external file `%s'", data['url_private']) + if data.get('is_external'): + # Create a link and corresponding file name for manual download + fname = str(uuid.uuid4()) + message.file.filepath = self.downloader.get_filepath(fname, 'file') + logging.info("Please, manually download an external file from " + "URL `%s' to `%s'", data['url_private'], + message.file.filepath) + self._dldata.append('%s --> %s\n' % (data['url_private'], + message.file.filepath)) message.file.url = data['url_private'] else: logging.debug("Found internal file `%s'", @@ -354,3 +382,14 @@ class Client(object): return result['messages'], None return [], None + + def _finalize(self): + """Create misc files if necessary - like manual donwload""" + if not self._dldata: + return + + with open(self._dlpath, "a") as fobj: + fobj.write(''.join(self._dldata)) + logging.warning("Manual action required! Download all the files " + "listed in `%s' and each of them save as file listed " + "right after `-->' sign", self._dlpath) diff --git a/slack_backup/command.py b/slack_backup/command.py index 1bffb85..33ae62b 100644 --- a/slack_backup/command.py +++ b/slack_backup/command.py @@ -94,10 +94,10 @@ def main(): help='Path to the database file.') fetch.add_argument('-i', '--config', default=None, help='Use specific config file.') - fetch.add_argument('-f', '--url_file_to_attachement', default=False, + fetch.add_argument('-f', '--url_file_to_attachment', default=False, action='store_true', help='Treat shared files (but not uploaded to the ' - 'Slack servers) as attachement. By default there will ' + 'Slack servers) as attachment. By default there will ' 'be file created in current directory with url and ' 'path to the filename under which it would be ' 'registered in the DB.') diff --git a/slack_backup/config.py b/slack_backup/config.py index 0871df1..55f196f 100644 --- a/slack_backup/config.py +++ b/slack_backup/config.py @@ -13,11 +13,11 @@ class Config(object): """Configuration keeper""" ints = ['verbose', 'quiet'] - bools = ['url_file_to_attachement'] + bools = ['url_file_to_attachment'] - sections = {'common': ['channels', 'database', 'quiet', 'verbose', - 'url_file_to_attachement'], - 'fetch': ['user', 'password', 'team', 'token'], + sections = {'common': ['channels', 'database', 'quiet', 'verbose'], + 'fetch': ['user', 'password', 'team', 'token', + 'url_file_to_attachment'], 'generate': ['output', 'format', 'theme']} def __init__(self): @@ -38,7 +38,7 @@ class Config(object): 'output': None, 'format': None, 'theme': None, - 'url_file_to_attachement': False} + 'url_file_to_attachment': False} # This message supposed to be displayed in INFO level. During the time # of running the code where it should be displayed there is no # complete information about logging level. Displaying message is diff --git a/slack_backup/reporters.py b/slack_backup/reporters.py index 433b68a..5ede715 100644 --- a/slack_backup/reporters.py +++ b/slack_backup/reporters.py @@ -403,7 +403,7 @@ class StaticHtmlReporter(Reporter): 'nick': msg.user.name} link = '{title}' - attachement_msg = [] + attachment_msg = [] if msg.attachments: for att in msg.attachments: @@ -429,9 +429,9 @@ class StaticHtmlReporter(Reporter): link.format(**match)) else: att_text = att.fallback - attachement_msg.append(att_text) + attachment_msg.append(att_text) - data['msg'] += '
'.join(attachement_msg) + data['msg'] += '
'.join(attachment_msg) return data diff --git a/slack_backup/utils.py b/slack_backup/utils.py index 37f1838..c2c51b1 100644 --- a/slack_backup/utils.py +++ b/slack_backup/utils.py @@ -24,10 +24,12 @@ def makedirs(path): raise -def get_temp_name(): +def get_temp_name(suffix='', prefix='tmp', dir=None, unlink=False): """Return temporary file name""" - fdesc, fname = tempfile.mkstemp() + fdesc, fname = tempfile.mkstemp(suffix=suffix, prefix=prefix, dir=dir) os.close(fdesc) + if unlink: + os.unlink(fname) return fname diff --git a/tests/test_client.py b/tests/test_client.py index d85eed3..32d449f 100644 --- a/tests/test_client.py +++ b/tests/test_client.py @@ -1,4 +1,5 @@ -from unittest import TestCase +import copy +import unittest from unittest import mock from slack_backup import client @@ -348,6 +349,234 @@ MSG3 = {"ok": True, "has_more": False, "is_limited": False} +STARRED = {"ok": True, + "oldest": "1479505026.000003", + "messages": [], + "has_more": False, + "is_limited": False} + +SHARED = {"type": "message", + "subtype": "file_share", + "text": "<@UAAAAAAAA> shared a file: ", + "file": { + "id": "F6ABMB0Ja", + "created": 1479147929, + "timestamp": 1479147929, + "name": "att name", + "title": "some_spreadsheet", + "mimetype": "application/vnd.google-apps." + "spreadsheet", + "filetype": "gsheet", + "pretty_type": "GDocs Spreadsheet", + "user": 'UAAAAAAAA', + "editable": False, + "size": 666, + "mode": "external", + "is_external": True, + "external_type": "gdrive", + "is_public": True, + "public_url_shared": False, + "display_as_bot": False, + "username": "", + "url_private": "https://docs.google.com/" + "spreadsheets/d/name%20lastname/" + "edit?usp=drivesdk", + # removed useless thumb_* definition + "image_exif_rotation": 1, + "original_w": 1024, + "original_h": 1449, + "permalink": "https://bla.slack.com/files/" + "name%20lastname/F7ARMB4JU/17", + "channels": ["C00000001"], + "groups": [], + "ims": [], + "comments_count": 0, + "has_rich_preview": True + }, + "user": 'UAAAAAAAA', + "upload": False, + "display_as_bot": False, + "username": "name lastname", + "bot_id": None, + "ts": "1479147929.000043"} + +PINNED = {'attachments': [{'fallback': 'blah', + 'id': 1, + 'image_bytes': 5, + 'image_height': 2, + 'image_url': 'http://fake.com/i.png', + 'image_width': 1, + 'original_url': 'http://fake.com/fake', + 'service_icon': 'http://fake.com/favicon.ico', + 'service_name': 'fake service', + 'text': 'the text', + 'title': 'Fake service title', + 'title_link': 'http://fake.com/fake'}], + 'item_type': 'C', + 'subtype': 'pinned_item', + 'text': '<@UAAAAAAAA> pinned a message to this channel.', + 'ts': '1479147929.000043', + 'type': 'message', + 'user': 'UAAAAAAAA'} + +EXTERNAL_DATA = {"bot_id": None, + "display_as_bot": False, + "file": { + "channels": [ + "xxx" + ], + "id": "F7ARMB4JU", + "created": 1506819447, + "timestamp": 1506819447, + "name": "17", + "title": "PT Card Count 9/30/17", + "mimetype": "application/" + "vnd.google-apps.spreadsheet", + "filetype": "gsheet", + "pretty_type": "GDocs Spreadsheet", + "user": "xxx", + "editable": False, + "size": 37583, + "mode": "external", + "is_external": True, + "external_type": "gdrive", + "is_public": True, + "public_url_shared": False, + "display_as_bot": False, + "username": "", + "url_private": "https://docs.google.com/" + "spreadsheets/d/xxx/edit?" + "usp=drivesdk", + "thumb_64": "https://files.slack.com/files-tmb/" + "xxx-F7ARMB4JU-xxx/17_64.png", + "thumb_80": "https://files.slack.com/files-tmb/" + "xxx-F7ARMB4JU-xxx/17_80.png", + "thumb_360": "https://files.slack.com/files-tmb" + "/xxx-F7ARMB4JU-xxx/17_360.png", + "thumb_360_w": 254, + "thumb_360_h": 360, + "thumb_480": "https://files.slack.com/files-tmb" + "/xxx-F7ARMB4JU-xxx/17_480.png", + "thumb_480_w": 339, + "thumb_480_h": 480, + "thumb_160": "https://files.slack.com/files-tmb" + "/xxx-F7ARMB4JU-xxx/17_160.png", + "thumb_720": "https://files.slack.com/files-tmb" + "/xxx-F7ARMB4JU-xxx/17_720.png", + "thumb_720_w": 509, + "thumb_720_h": 720, + "thumb_800": "https://files.slack.com/files-tmb" + "/xxx-F7ARMB4JU-xxx/17_800.png", + "thumb_800_w": 800, + "thumb_800_h": 1132, + "thumb_960": "https://files.slack.com/files-tmb" + "/xxx-F7ARMB4JU-xxx/17_960.png", + "thumb_960_w": 678, + "thumb_960_h": 960, + "thumb_1024": "https://files.slack.com/files-tmb" + "/xxx-F7ARMB4JU-xxx/17_1024.png", + "thumb_1024_w": 724, + "thumb_1024_h": 1024, + "image_exif_rotation": 1, + "original_w": 1024, + "original_h": 1449, + "permalink": "https://xxx.slack.com/files/xxx/" + "F7ARMB4JU/17", + "groups": [], + "ims": [], + "comments_count": 0, + "has_rich_preview": True}, + "user": "xxx", + "upload": False, + "username": "xxx"} + +INTERNAL_DATA = {"bot_id": None, + "display_as_bot": False, + "file": {"channels": ["zzz"], + "comments_count": 1, + "created": 1524724681, + "display_as_bot": False, + "editable": False, + "external_type": "", + "filetype": "jpg", + "groups": [], + "id": "id", + "image_exif_rotation": 1, + "ims": [], + "initial_comment": {"comment": "bla", + "created": 1524724681, + "id": "yyy", + "is_intro": True, + "timestamp": 1524724681, + "user": "UAAAAAAAA"}, + "is_external": False, + "is_public": True, + "mimetype": "image/jpeg", + "mode": "hosted", + "name": "img.jpg", + "original_h": 768, + "original_w": 1080, + "permalink": "https://fake.slack.com/files/" + "UAAAAAAAA/id/img.jpg", + "permalink_public": "https://slack-files.com/" + "TXXXXXX-id-3de9b969d2", + "pretty_type": "JPEG", + "public_url_shared": False, + "reactions": [{"count": 1, + "name": "thinking_face", + "users": ["U6P9KEALW"]}], + "size": 491335, + "thumb_1024": "https://files.slack.com/files-tmb/" + "TXXXXXX-id-123/img_1024.jpg", + "thumb_1024_h": 728, + "thumb_1024_w": 1024, + "thumb_160": "https://files.slack.com/files-tmb/" + "TXXXXXX-id-123/img_160.jpg", + "thumb_360": "https://files.slack.com/files-tmb/" + "TXXXXXX-id-123/img_360.jpg", + "thumb_360_h": 256, + "thumb_360_w": 360, + "thumb_480": "https://files.slack.com/files-tmb/" + "TXXXXXX-id-123/img_480.jpg", + "thumb_480_h": 341, + "thumb_480_w": 480, + "thumb_64": "https://files.slack.com/files-tmb/" + "TXXXXXX-id-123/img_64.jpg", + "thumb_720": "https://files.slack.com/files-tmb/" + "TXXXXXX-id-123/img_720.jpg", + "thumb_720_h": 512, + "thumb_720_w": 720, + "thumb_80": "https://files.slack.com/files-tmb/" + "TXXXXXX-id-123/img_80.jpg", + "thumb_800": "https://files.slack.com/files-tmb/" + "TXXXXXX-id-123/img_800.jpg", + "thumb_800_h": 569, + "thumb_800_w": 800, + "thumb_960": "https://files.slack.com/files-tmb/" + "TXXXXXX-id-123/img_960.jpg", + "thumb_960_h": 683, + "thumb_960_w": 960, + "timestamp": 1524724681, + "title": "img.jpg", + "url_private": "https://files.slack.com/files-pri/" + "TXXXXXX-id/img.jpg", + "url_private_download": "https://files.slack.com/" + "files-pri/TXXXXXX-id/" + "download/img.jpg", + "user": "UAAAAAAAA", + "username": ""}, + "subtype": "file_share", + "text": "<@UAAAAAAAA> uploaded a file: and " + "commented: bla", + "ts": "1524724685.000201", + "type": "message", + "upload": True, + "user": "UAAAAAAAA", + "username": "bob"} + class FakeArgs(object): token = 'token_string' @@ -356,12 +585,13 @@ class FakeArgs(object): team = 'fake_team' database = None channels = None + url_file_to_attachment = False def __contains__(self, key): return hasattr(self, key) -class TestApiCalls(TestCase): +class TestApiCalls(unittest.TestCase): def test_channels_list(self): cl = client.Client(FakeArgs()) @@ -404,7 +634,7 @@ class TestApiCalls(TestCase): self.assertIsNone(ts) -class TestClient(TestCase): +class TestClient(unittest.TestCase): def test_update_users(self): cl = client.Client(FakeArgs()) @@ -422,7 +652,7 @@ class TestClient(TestCase): self.assertEqual(users[0].slackid, 'UAAAAAAAA') -class TestMessage(TestCase): +class TestMessage(unittest.TestCase): def setUp(self): args = FakeArgs() @@ -453,3 +683,174 @@ class TestMessage(TestCase): self.cl.update_history() self.assertEqual(len(self.cl.q(o.Message).all()), 6) + + +class TestCreateMessage(unittest.TestCase): + + @mock.patch('slack_backup.client.Client._file_data') + @mock.patch('slack_backup.client.Client._get_user') + def test_empty_message(self, gu, fd): + cl = client.Client(FakeArgs()) + cl.downloader._download = mock.MagicMock(return_value='aa') + cl.session = mock.MagicMock() + channel = o.Channel({'name': 'test', 'id': 'C00000001'}) + + cl._create_message({'type': 'message', 'text': ''}, channel) + cl.session.add.assert_not_called() + + @mock.patch('slack_backup.client.Client._file_data') + @mock.patch('slack_backup.client.Client._get_user') + def test_message_with_reaction(self, gu, fd): + cl = client.Client(FakeArgs()) + cl.downloader._download = mock.MagicMock(return_value='aa') + cl.session = mock.MagicMock() + channel = o.Channel({'name': 'test', 'id': 'C00000001'}) + + cl._create_message(MSGS['messages'][1], channel) + + msg = cl.session.add.call_args[0][0] + self.assertEqual(len(msg.attachments), 1) + self.assertEqual(len(msg.reactions), 1) + self.assertEqual(msg.reactions[0].name, '+1') + self.assertFalse(msg.is_starred) + + @mock.patch('slack_backup.client.Client._get_user') + def test_starred_item(self, gu): + cl = client.Client(FakeArgs()) + cl.downloader._download = mock.MagicMock(return_value='aa') + cl.session = mock.MagicMock() + channel = o.Channel({'name': 'test', 'id': 'C00000001'}) + + data = {"type": "message", + "user": "UAAAAAAAA", + "text": "test", + "ts": "1479501074.000032", + "is_starred": True} + cl._create_message(data, channel) + + msg = cl.session.add.call_args[0][0] + self.assertEqual(len(msg.attachments), 0) + self.assertEqual(msg.text, 'test') + self.assertEqual(msg.type, '') + self.assertTrue(msg.is_starred) + + @mock.patch('slack_backup.client.Client._file_data') + @mock.patch('slack_backup.client.Client._get_user') + def test_external_file_upload(self, gu, fd): + cl = client.Client(FakeArgs()) + cl.downloader._download = mock.MagicMock(return_value='aa') + cl.session = mock.MagicMock() + channel = o.Channel({'name': 'test', 'id': 'C00000001'}) + + cl._create_message(SHARED, channel) + + msg = cl.session.add.call_args[0][0] + self.assertEqual(len(msg.attachments), 0) + self.assertTrue('shared a file' in msg.text) + self.assertFalse(msg.is_starred) + self.assertEqual(msg.type, 'file_share') + fd.assert_called_once_with(msg, SHARED['file']) + + @mock.patch('slack_backup.client.Client._get_user') + def test_external_file_upload_as_attachment(self, gu): + cl = client.Client(FakeArgs()) + cl.downloader._download = mock.MagicMock(return_value='aa') + cl.session = mock.MagicMock() + cl._url_file_to_attachment = True + channel = o.Channel({'name': 'test', 'id': 'C00000001'}) + + cl._create_message(SHARED, channel) + + msg = cl.session.add.call_args[0][0] + self.assertEqual(len(msg.attachments), 1) + self.assertTrue('shared a file' in msg.text) + self.assertFalse(msg.is_starred) + + @mock.patch('slack_backup.client.Client._file_data') + @mock.patch('slack_backup.client.Client._get_user') + def test_pinned_message_with_attachments(self, gu, fd): + cl = client.Client(FakeArgs()) + cl.downloader._download = mock.MagicMock(return_value='aa') + cl.session = mock.MagicMock() + cl._url_file_to_attachment = True + channel = o.Channel({'name': 'test', 'id': 'C00000001'}) + + cl._create_message(PINNED, channel) + + msg = cl.session.add.call_args[0][0] + self.assertEqual(len(msg.attachments), 1) + self.assertEqual(msg.text, '<@UAAAAAAAA> pinned a message to this ' + 'channel.') + self.assertEqual(msg.type, 'pinned_item') + self.assertEqual(msg.attachments[0].text, 'the text') + self.assertEqual(msg.attachments[0].title, 'Fake service title') + + +class TestFileShare(unittest.TestCase): + + @mock.patch('slack_backup.download.Download.download') + @mock.patch('slack_backup.utils.makedirs') + def test_file_data(self, md, dl): + dl.side_effect = ['some_path'] + + url = INTERNAL_DATA['file']['url_private_download'] + + cl = client.Client(FakeArgs()) + cl.downloader._download = mock.MagicMock(return_value=url) + cl.session = mock.MagicMock() + + msg = o.Message() + cl._file_data(msg, INTERNAL_DATA['file']) + + self.assertIsNotNone(msg.file) + self.assertEqual(msg.file.filepath, 'some_path') + + @mock.patch('slack_backup.download.Download.download') + @mock.patch('slack_backup.utils.makedirs') + def test_starred_file_data(self, md, dl): + dl.side_effect = ['some_path'] + + data = copy.deepcopy(INTERNAL_DATA) + data['file']['is_starred'] = True + url = data['file']['url_private_download'] + + cl = client.Client(FakeArgs()) + cl.downloader._download = mock.MagicMock(return_value=url) + cl.session = mock.MagicMock() + + msg = o.Message() + cl._file_data(msg, data['file']) + + self.assertTrue(msg.is_starred) + + @mock.patch('uuid.uuid4') + @mock.patch('slack_backup.download.Download.download') + @mock.patch('slack_backup.utils.makedirs') + def test_external_file_data(self, md, dl, uuid): + uuid.side_effect = ['aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeeeee'] + dl.side_effect = ['some_path'] + + # url = EXTERNAL_DATA['file']['url_private'] + + cl = client.Client(FakeArgs()) + cl.session = mock.MagicMock() + + # pretend, that we are authorized + cl.downloader._authorized = True + + msg = o.Message() + expexted_line = ('https://docs.google.com/spreadsheets/d/xxx/edit?' + 'usp=drivesdk --> ' + 'assets/files/aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeeeee\n') + + cl._file_data(msg, EXTERNAL_DATA['file']) + file_ = cl.session.add.call_args[0][0] + self.assertEqual(cl._dldata, [expexted_line]) + self.assertEqual(file_.filepath, + 'assets/files/aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeeeee') + + dl.assert_not_called() + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_config.py b/tests/test_config.py index 0fc8c5b..1c7e51b 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -64,7 +64,7 @@ class TestConfig(unittest.TestCase): 'password': None, 'team': None, 'token': None, - 'url_file_to_attachement': False}) + 'url_file_to_attachment': False}) args = argparse.Namespace() args.config = self.confname @@ -99,7 +99,7 @@ class TestConfig(unittest.TestCase): 'token': 'xxxx-1111111111-' '222222222222-333333333333-' 'r4nd0ms7uff', - 'url_file_to_attachement': False}) + 'url_file_to_attachment': False}) # override some conf options with commandline args = argparse.Namespace() @@ -127,4 +127,4 @@ class TestConfig(unittest.TestCase): 'password': 'ultricies', 'team': '', 'token': 'the token', - 'url_file_to_attachement': False}) + 'url_file_to_attachment': False})