diff --git a/scripts/slack-backup b/scripts/slack-backup index 2c81c56..4c52fe2 100755 --- a/scripts/slack-backup +++ b/scripts/slack-backup @@ -31,6 +31,8 @@ def main(): args = parser.parse_args() slack = client.Client(args.token, args.database) + slack.update_users() + slack.update_channels() slack.update_history(selected_channels=args.channels) diff --git a/slack_backup/client.py b/slack_backup/client.py index ac8a196..e40d693 100644 --- a/slack_backup/client.py +++ b/slack_backup/client.py @@ -7,7 +7,7 @@ from datetime import datetime import slackclient from slack_backup import db -from slack_backup.objects import User, Channel, Purpose, Topic +from slack_backup import objects as o class Client(object): @@ -17,57 +17,26 @@ class Client(object): self.session = db.Session() self.q = self.session.query - def update_history(self, selected_channels=None, from_date=0): + def update_channels(self): + """Fetch and update channel list with current state in db""" + result = self._channels_list() - self._update_users() - self._update_channels() + if not result: + return - channels = self._get_channel_list() - if selected_channels: - selected_channels = [c for c in channels - if c.name in selected_channels] - else: - selected_channels = channels + for channel_data in result['channels']: + channel = self.q(o.Channel).\ + filter(o.Channel.slackid == channel_data['id']).one_or_none() - for channel in selected_channels: - history = [] - latest = 0 + if not channel: + channel = o.Channel() + self.session.add(channel) - while True: - messages, latest = self._get_channel_history(channel, latest) - # TODO: merge messages witihn a channel - if latest is None: - break - for msg in messages: - history.append(msg) + self._update_channel(channel, channel_data) - self.session.close() - return history + self.session.commit() - def _get_channel_history(self, channel, latest='now'): - result = self.slack.api_call("channels.history", - channel=channel.slackid, count=1000, - latest=latest) - - if not result.get("ok"): - logging.error(result['error']) - return None, None - - if result['messages']: - return result['messages'], result['messages'][-1]['ts'] - else: - return result['messages'], None - - def _get_channel_list(self): - result = self.slack.api_call("channels.list") - - if not result.get("ok"): - logging.error(result['error']) - return None - - return [Channel(chan) for chan in result['channels']] - - def _update_users(self): + def update_users(self): """Fetch and update user list with current state in db""" result = self.slack.api_call("users.list", presence=0) @@ -76,33 +45,75 @@ class Client(object): return for user_data in result['members']: - user = self.q(User).\ - filter(User.slackid == user_data['id']).one_or_none() + user = self.q(o.User).\ + filter(o.User.slackid == user_data['id']).one_or_none() if user: user.update(user_data) else: - user = User(user_data) + user = o.User(user_data) self.session.add(user) self.session.flush() self.session.commit() + def update_history(self, selected_channels=None): + """ + Get the latest or all messages out of optionally selected channels + """ + + channels = self.q(o.Channel).all() + if selected_channels: + selected_channels = [c for c in channels + if c.name in selected_channels] + else: + selected_channels = channels + + for channel in selected_channels: + latest = self.q(o.Message).\ + filter(o.Message.channel == channel).\ + order_by(o.Message.ts.desc()).first() + latest = latest and latest.ts or 0 + + while True: + messages, latest = self._channels_history(channel, latest) + + for msg in messages: + self._create_message(msg, channel) + + if latest is None: + break + + self.session.commit() + + def _create_message(self, data, channel): + message = o.Message(data) + message.user = self.q(o.User).\ + filter(o.User.slackid == data['user']).one() + message.channel = channel + + if 'reactions' in data: + for reaction_data in data['reactions']: + o.Message.reactions.append(o.Reaction(reaction_data)) + + self.session.add(o.Message) + def _get_create_obj(self, data, classobj, channel): """ Return object if exist in appropriate table (Topic or Purpose), compared to the data provided, create it otherwise. """ - user = self.q(User).filter(User.slackid == - data['creator']).one_or_none() - if not user: + if not data['value']: return + user = self.q(o.User).filter(o.User.slackid == + data['creator']).one_or_none() + obj = self.q(classobj).\ filter(classobj.last_set == datetime.fromtimestamp(data['last_set'])).\ filter(classobj.value == data['value']).\ - filter(classobj.creator_id == user.id).one_or_none() + filter(classobj.creator == user).one_or_none() if not obj: # break channel relation @@ -118,30 +129,63 @@ class Client(object): return obj def _update_channel(self, channel, data): + """Update a channel with provided data""" channel.update(data) - channel.user = self.q(User).filter(User.slackid == - data['created']).one_or_none() - channel.purpose = self._get_create_obj(data['purpose'], Purpose, + channel.user = self.q(o.User).filter(o.User.slackid == + data['creator']).one_or_none() + channel.purpose = self._get_create_obj(data['purpose'], o.Purpose, channel) - channel.topic = self._get_create_obj(data['topic'], Topic, channel) + channel.topic = self._get_create_obj(data['topic'], o.Topic, channel) self.session.flush() - def _update_channels(self): - """Fetch and update user list with current state in db""" - result = self.slack.api_call("channels.list", presence=0) + def _channels_list(self): + """ + Get channel list using Slack API. Return list of channel data or None + in case of error. + """ + result = self.slack.api_call("channels.list") if not result.get("ok"): logging.error(result['error']) return None - for channel_data in result['channels']: - channel = self.q(Channel).filter(Channel.slackid == - channel_data['id']).one_or_none() + return result['channels'] - if not channel: - channel = Channel() - self.session.add(channel) + def _users_list(self): + """ + Get users list using Slack API. Return list of channel data or None + in case of error. + """ + result = self.slack.api_call("users.list", presence=0) - self._update_channel(channel, channel_data) + if not result.get("ok"): + logging.error(result['error']) + return None - self.session.commit() + return result['channels'] + + def _channels_history(self, channel, latest): + """ + Get list of messages using Slack API. Return tuple containing: + - list of messages data and returned timestramp if has_more is set + to true, + - list of messages data and None if has_more is set to false, + - empty list and None if there is no messages + """ + result = self.slack.api_call("channels.history", + channel=channel.slackid, count=1000, + oldest=latest) + + if not result.get("ok"): + logging.error(result['error']) + return None, None + + if result['messages']: + if result['has_more']: + # TODO: this one might be not true, if API will return + # messages not sorted by timestamp in descending order + return result['messages'], result['messages'][0]['ts'] + else: + return result['messages'], None + + return [], None diff --git a/slack_backup/objects.py b/slack_backup/objects.py index 554648b..19b2e9a 100644 --- a/slack_backup/objects.py +++ b/slack_backup/objects.py @@ -3,7 +3,7 @@ Convinient object mapping from slack API reponses """ from datetime import datetime -from sqlalchemy import Column, Integer, Text, Boolean, ForeignKey +from sqlalchemy import Column, Integer, Text, Boolean, ForeignKey, Float from sqlalchemy import DateTime from sqlalchemy.orm import relationship @@ -81,6 +81,7 @@ class Channel(Base): purpose = relationship("Purpose", uselist=False, back_populates="channel") topic = relationship("Topic", uselist=False, back_populates="channel") + messages = relationship("Message", back_populates="channel") def __init__(self, data_dict=None): self.update(data_dict) @@ -151,9 +152,12 @@ class User(Base): slackid = Column(Text) channels = relationship("Channel", back_populates="creator") + messages = relationship("Message", back_populates="user") profile = relationship("UserProfile", uselist=False, back_populates="user") purposes = relationship("Purpose", back_populates="creator") topics = relationship("Topic", back_populates="creator") + reaction_id = Column(Integer, ForeignKey("reactions.id")) + reaction = relationship("Reaction", back_populates="users") def __init__(self, data_dict=None): self.update(data_dict) @@ -178,17 +182,53 @@ class User(Base): return u'%s, %s %s' % (self.__class__.__name__, self.id, self.name) -class Reactions(object): +class Reaction(Base): + __tablename__ = "reactions" + + id = Column(Integer, primary_key=True) + name = Column(Text) + + users = relationship("User", back_populates="reaction") + + message_id = Column(Integer, ForeignKey("messages.id"), nullable=True) + message = relationship("Message", back_populates="reactions") + def __init__(self, data_dict=None): + self.update(data_dict) + + def update(self, data_dict): data_dict = data_dict or {} + self.name = data_dict.get('name', '') + + +class Message(Base): + __tablename__ = "messages" + + id = Column(Integer, primary_key=True) + ts = Column(Float, index=True) + text = Column(Text) + type = Column(Text) + + user_id = Column(Integer, ForeignKey("users.id"), nullable=True, + index=True) + user = relationship("User", back_populates="messages") + + channel_id = Column(Integer, ForeignKey("channels.id"), nullable=True, + index=True) + channel = relationship("Channel", back_populates="messages") + + reactions = relationship("Reaction", back_populates="message") -class Messages(object): def __init__(self, data_dict=None): + self.update(data_dict) + + def datetime(self): + return datetime.fromtimestamp(self.ts) + + def update(self, data_dict): data_dict = data_dict or {} - self.ts = data_dict.get('ts', '') - self.user_id = data_dict.get('user', '') - self.type = data_dict.get('type', '') + self.ts = float(data_dict.get('ts', 0)) self.text = data_dict.get('text', '') - self.reactions = Reactions(data_dict.get('reactions', '')) + self.type = data_dict.get('subtype', '')