Added message model, moving updates of users and channels as public methods

scripts/slack-backup

@@ -31,6 +31,8 @@ def main():
     args = parser.parse_args()
 
     slack = client.Client(args.token, args.database)
+    slack.update_users()
+    slack.update_channels()
     slack.update_history(selected_channels=args.channels)
 
 

slack_backup/client.py

@@ -7,7 +7,7 @@ from datetime import datetime
 import slackclient
 
 from slack_backup import db
-from slack_backup.objects import User, Channel, Purpose, Topic
+from slack_backup import objects as o
 
 
 class Client(object):
@@ -17,57 +17,26 @@ class Client(object):
         self.session = db.Session()
         self.q = self.session.query
 
-    def update_history(self, selected_channels=None, from_date=0):
+    def update_channels(self):
+        """Fetch and update channel list with current state in db"""
+        result = self._channels_list()
 
-        self._update_users()
+        if not result:
-        self._update_channels()
+            return
 
-        channels = self._get_channel_list()
+        for channel_data in result['channels']:
-        if selected_channels:
+            channel = self.q(o.Channel).\
-            selected_channels = [c for c in channels
+                filter(o.Channel.slackid == channel_data['id']).one_or_none()
-                                 if c.name in selected_channels]
-        else:
-            selected_channels = channels
 
-        for channel in selected_channels:
+            if not channel:
-            history = []
+                channel = o.Channel()
-            latest = 0
+                self.session.add(channel)
 
-            while True:
+            self._update_channel(channel, channel_data)
-                messages, latest = self._get_channel_history(channel, latest)
-                # TODO: merge messages witihn a channel
-                if latest is None:
-                    break
-                for msg in messages:
-                    history.append(msg)
 
-        self.session.close()
+        self.session.commit()
-        return history
 
-    def _get_channel_history(self, channel, latest='now'):
+    def update_users(self):
-        result = self.slack.api_call("channels.history",
-                                     channel=channel.slackid, count=1000,
-                                     latest=latest)
-
-        if not result.get("ok"):
-            logging.error(result['error'])
-            return None, None
-
-        if result['messages']:
-            return result['messages'], result['messages'][-1]['ts']
-        else:
-            return result['messages'], None
-
-    def _get_channel_list(self):
-        result = self.slack.api_call("channels.list")
-
-        if not result.get("ok"):
-            logging.error(result['error'])
-            return None
-
-        return [Channel(chan) for chan in result['channels']]
-
-    def _update_users(self):
         """Fetch and update user list with current state in db"""
         result = self.slack.api_call("users.list", presence=0)
 
@@ -76,33 +45,75 @@ class Client(object):
             return
 
         for user_data in result['members']:
-            user = self.q(User).\
+            user = self.q(o.User).\
-                filter(User.slackid == user_data['id']).one_or_none()
+                filter(o.User.slackid == user_data['id']).one_or_none()
 
             if user:
                 user.update(user_data)
             else:
-                user = User(user_data)
+                user = o.User(user_data)
                 self.session.add(user)
                 self.session.flush()
 
         self.session.commit()
 
+    def update_history(self, selected_channels=None):
+        """
+        Get the latest or all messages out of optionally selected channels
+        """
+
+        channels = self.q(o.Channel).all()
+        if selected_channels:
+            selected_channels = [c for c in channels
+                                 if c.name in selected_channels]
+        else:
+            selected_channels = channels
+
+        for channel in selected_channels:
+            latest = self.q(o.Message).\
+                filter(o.Message.channel == channel).\
+                order_by(o.Message.ts.desc()).first()
+            latest = latest and latest.ts or 0
+
+            while True:
+                messages, latest = self._channels_history(channel, latest)
+
+                for msg in messages:
+                    self._create_message(msg, channel)
+
+                if latest is None:
+                    break
+
+        self.session.commit()
+
+    def _create_message(self, data, channel):
+        message = o.Message(data)
+        message.user = self.q(o.User).\
+            filter(o.User.slackid == data['user']).one()
+        message.channel = channel
+
+        if 'reactions' in data:
+            for reaction_data in data['reactions']:
+                o.Message.reactions.append(o.Reaction(reaction_data))
+
+        self.session.add(o.Message)
+
     def _get_create_obj(self, data, classobj, channel):
         """
         Return object if exist in appropriate table (Topic or Purpose),
         compared to the data provided, create it otherwise.
         """
-        user = self.q(User).filter(User.slackid ==
+        if not data['value']:
-                                   data['creator']).one_or_none()
-        if not user:
             return
 
+        user = self.q(o.User).filter(o.User.slackid ==
+                                     data['creator']).one_or_none()
+
         obj = self.q(classobj).\
             filter(classobj.last_set ==
                    datetime.fromtimestamp(data['last_set'])).\
             filter(classobj.value == data['value']).\
-            filter(classobj.creator_id == user.id).one_or_none()
+            filter(classobj.creator == user).one_or_none()
 
         if not obj:
             # break channel relation
@@ -118,30 +129,63 @@ class Client(object):
         return obj
 
     def _update_channel(self, channel, data):
+        """Update a channel with provided data"""
         channel.update(data)
-        channel.user = self.q(User).filter(User.slackid ==
+        channel.user = self.q(o.User).filter(o.User.slackid ==
-                                           data['created']).one_or_none()
+                                             data['creator']).one_or_none()
-        channel.purpose = self._get_create_obj(data['purpose'], Purpose,
+        channel.purpose = self._get_create_obj(data['purpose'], o.Purpose,
                                                channel)
-        channel.topic = self._get_create_obj(data['topic'], Topic, channel)
+        channel.topic = self._get_create_obj(data['topic'], o.Topic, channel)
         self.session.flush()
 
-    def _update_channels(self):
+    def _channels_list(self):
-        """Fetch and update user list with current state in db"""
+        """
-        result = self.slack.api_call("channels.list", presence=0)
+        Get channel list using Slack API. Return list of channel data or None
+        in case of error.
+        """
+        result = self.slack.api_call("channels.list")
 
         if not result.get("ok"):
             logging.error(result['error'])
             return None
 
-        for channel_data in result['channels']:
+        return result['channels']
-            channel = self.q(Channel).filter(Channel.slackid ==
-                                             channel_data['id']).one_or_none()
 
-            if not channel:
+    def _users_list(self):
-                channel = Channel()
+        """
-                self.session.add(channel)
+        Get users list using Slack API. Return list of channel data or None
+        in case of error.
+        """
+        result = self.slack.api_call("users.list", presence=0)
 
-            self._update_channel(channel, channel_data)
+        if not result.get("ok"):
+            logging.error(result['error'])
+            return None
 
-        self.session.commit()
+        return result['channels']
+
+    def _channels_history(self, channel, latest):
+        """
+        Get list of messages using Slack API. Return tuple containing:
+         - list of messages data and returned timestramp if has_more is set
+           to true,
+         - list of messages data and None if has_more is set to false,
+         - empty list and None if there is no messages
+        """
+        result = self.slack.api_call("channels.history",
+                                     channel=channel.slackid, count=1000,
+                                     oldest=latest)
+
+        if not result.get("ok"):
+            logging.error(result['error'])
+            return None, None
+
+        if result['messages']:
+            if result['has_more']:
+                # TODO: this one might be not true, if API will return
+                # messages not sorted by timestamp in descending order
+                return result['messages'], result['messages'][0]['ts']
+            else:
+                return result['messages'], None
+
+        return [], None

slack_backup/objects.py

@@ -3,7 +3,7 @@ Convinient object mapping from slack API reponses
 """
 from datetime import datetime
 
-from sqlalchemy import Column, Integer, Text, Boolean, ForeignKey
+from sqlalchemy import Column, Integer, Text, Boolean, ForeignKey, Float
 from sqlalchemy import DateTime
 from sqlalchemy.orm import relationship
 
@@ -81,6 +81,7 @@ class Channel(Base):
 
     purpose = relationship("Purpose", uselist=False, back_populates="channel")
     topic = relationship("Topic", uselist=False, back_populates="channel")
+    messages = relationship("Message", back_populates="channel")
 
     def __init__(self, data_dict=None):
         self.update(data_dict)
@@ -151,9 +152,12 @@ class User(Base):
     slackid = Column(Text)
 
     channels = relationship("Channel", back_populates="creator")
+    messages = relationship("Message", back_populates="user")
     profile = relationship("UserProfile", uselist=False, back_populates="user")
     purposes = relationship("Purpose", back_populates="creator")
     topics = relationship("Topic", back_populates="creator")
+    reaction_id = Column(Integer, ForeignKey("reactions.id"))
+    reaction = relationship("Reaction", back_populates="users")
 
     def __init__(self, data_dict=None):
         self.update(data_dict)
@@ -178,17 +182,53 @@ class User(Base):
         return u'%s, %s %s' % (self.__class__.__name__, self.id, self.name)
 
 
-class Reactions(object):
+class Reaction(Base):
+    __tablename__ = "reactions"
+
+    id = Column(Integer, primary_key=True)
+    name = Column(Text)
+
+    users = relationship("User", back_populates="reaction")
+
+    message_id = Column(Integer, ForeignKey("messages.id"), nullable=True)
+    message = relationship("Message", back_populates="reactions")
+
     def __init__(self, data_dict=None):
+        self.update(data_dict)
+
+    def update(self, data_dict):
         data_dict = data_dict or {}
 
+        self.name = data_dict.get('name', '')
+
+
+class Message(Base):
+    __tablename__ = "messages"
+
+    id = Column(Integer, primary_key=True)
+    ts = Column(Float, index=True)
+    text = Column(Text)
+    type = Column(Text)
+
+    user_id = Column(Integer, ForeignKey("users.id"), nullable=True,
+                     index=True)
+    user = relationship("User", back_populates="messages")
+
+    channel_id = Column(Integer, ForeignKey("channels.id"), nullable=True,
+                        index=True)
+    channel = relationship("Channel", back_populates="messages")
+
+    reactions = relationship("Reaction", back_populates="message")
 
-class Messages(object):
     def __init__(self, data_dict=None):
+        self.update(data_dict)
+
+    def datetime(self):
+        return datetime.fromtimestamp(self.ts)
+
+    def update(self, data_dict):
         data_dict = data_dict or {}
 
-        self.ts = data_dict.get('ts', '')
+        self.ts = float(data_dict.get('ts', 0))
-        self.user_id = data_dict.get('user', '')
-        self.type = data_dict.get('type', '')
         self.text = data_dict.get('text', '')
-        self.reactions = Reactions(data_dict.get('reactions', ''))
+        self.type = data_dict.get('subtype', '')