Added logic for Image and Thumbnail db objects. Added scan functionality

convert_1.x_to_2.x.py

@@ -1,3 +1,4 @@
+#!/usr/bin/env python
 """
     Project: pyGTKtalog
     Description: convert db created with v.1.x into v.2.x

pygtktalog/dbobjects.py

@@ -5,16 +5,26 @@
     Author: Roman 'gryf' Dobosz, gryf73@gmail.com
     Created: 2009-08-07
 """
+import os
+import errno
+import shutil
+import uuid
+
 from sqlalchemy import Column, Table, Integer, Text
 from sqlalchemy import DateTime, ForeignKey, Sequence
 from sqlalchemy.orm import relation, backref
-from pygtktalog.dbcommon import Base
 
+from pygtktalog.dbcommon import Base
+from pygtktalog import thumbnail
+
+
+IMG_PATH = "/home/gryf/.pygtktalog/imgs/"  # FIXME: should be configurable
 
 tags_files = Table("tags_files", Base.metadata,
                    Column("file_id", Integer, ForeignKey("files.id")),
                    Column("tag_id", Integer, ForeignKey("tags.id")))
 
+
 class File(Base):
     __tablename__ = "files"
     id = Column(Integer, Sequence("file_id_seq"), primary_key=True)
@@ -48,6 +58,7 @@ class File(Base):
     def __repr__(self):
         return "<File('%s', %s)>" % (str(self.filename), str(self.id))
 
+
 class Group(Base):
     __tablename__ = "groups"
     id = Column(Integer, Sequence("group_id_seq"), primary_key=True)
@@ -61,6 +72,7 @@ class Group(Base):
     def __repr__(self):
         return "<Group('%s', %s)>" % (str(self.name), str(self.id))
 
+
 class Tag(Base):
     __tablename__ = "tags"
     id = Column(Integer, Sequence("tags_id_seq"), primary_key=True)
@@ -84,25 +96,103 @@ class Thumbnail(Base):
     file_id = Column(Integer, ForeignKey("files.id"))
     filename = Column(Text)
 
-    def __init__(self, filename=None):
+    def __init__(self, filename=None, file_obj=None):
-        self.filename = None
+        self.filename = filename
+        self.file = file_obj
+        if self.filename:
+            self.save(self.filename)
+
+    def save(self, fname):
+        """
+        Create file related thumbnail, add it to the file object.
+        """
+        new_name = str(uuid.uuid1()).split("-")
+        try:
+            os.makedirs(os.path.join(IMG_PATH, *new_name[:-1]))
+        except OSError as exc:
+            if exc.errno != errno.EEXIST:
+                raise
+
+        ext = os.path.splitext(self.filename)[1]
+        if ext:
+            new_name.append("".join([new_name.pop(), ext]))
+
+        thumb = thumbnail.Thumbnail(self.filename).save()
+        name, ext = os.path.splitext(new_name.pop())
+        new_name.append("".join([name, "_t", ext]))
+        self.filename = os.path.sep.join(new_name)
+        shutil.move(thumb.save(), os.path.join(IMG_PATH, *new_name))
 
     def __repr__(self):
         return "<Thumbnail('%s', %s)>" % (str(self.filename), str(self.id))
 
+
 class Image(Base):
     __tablename__ = "images"
     id = Column(Integer, Sequence("images_id_seq"), primary_key=True)
     file_id = Column(Integer, ForeignKey("files.id"))
     filename = Column(Text)
 
-    def __init__(self, filename=None):
+    def __init__(self, filename=None, file_obj=None):
-        self.filename = filename
+        self.filename = None
-        self.file = None
+        self.file = file_obj
+        if filename:
+            self.filename = filename
+            self.save(filename)
+
+    def save(self, fname):
+        """
+        Save and create coressponding thumbnail (note: it differs from file
+        related thumbnail!)
+        """
+        new_name = str(uuid.uuid1()).split("-")
+        try:
+            os.makedirs(os.path.join(IMG_PATH, *new_name[:-1]))
+        except OSError as exc:
+            if exc.errno != errno.EEXIST:
+                raise
+
+        ext = os.path.splitext(self.filename)[1]
+        if ext:
+            new_name.append("".join([new_name.pop(), ext]))
+
+        shutil.move(self.filename, os.path.join(IMG_PATH, *new_name))
+
+        self.filename = os.path.sep.join(new_name)
+
+        thumb = thumbnail.Thumbnail(os.path.join(IMG_PATH, self.filename))
+        name, ext = os.path.splitext(new_name.pop())
+        new_name.append("".join([name, "_t", ext]))
+        shutil.move(thumb.save(), os.path.join(IMG_PATH, *new_name))
+
+    def get_copy(self):
+        """
+        Create the very same object as self with exception of id field
+        """
+        img = Image()
+        img.filename = self.filename
+        return img
+
+    @property
+    def thumbpath(self):
+        """
+        Return full path to thumbnail of this image
+        """
+        path, fname = os.path.split(self.filename)
+        base, ext = os.path.splitext(fname)
+        return os.path.join(IMG_PATH, path, base + "_t" + ext)
+
+    @property
+    def imagepath(self):
+        """
+        Return full path to image
+        """
+        return os.path.join(IMG_PATH, self.filename)
 
     def __repr__(self):
         return "<Image('%s', %s)>" % (str(self.filename), str(self.id))
 
+
 class Exif(Base):
     __tablename__ = "exif"
     id = Column(Integer, Sequence("exif_id_seq"), primary_key=True)
@@ -139,6 +229,7 @@ class Exif(Base):
     def __repr__(self):
         return "<Exif('%s', %s)>" % (str(self.date), str(self.id))
 
+
 class Gthumb(Base):
     __tablename__ = "gthumb"
     id = Column(Integer, Sequence("gthumb_id_seq"), primary_key=True)
@@ -155,4 +246,3 @@ class Gthumb(Base):
     def __repr__(self):
         return "<Gthumb('%s', '%s', %s)>" % (str(self.date), str(self.place),
                                              str(self.id))
-

pygtktalog/logger.py

@@ -28,9 +28,9 @@ def formatter_message(message, use_color = True):
     return message
 
 COLORS = {'WARNING': YELLOW,
-          'INFO': WHITE,
+          'INFO': GREEN,
           'DEBUG': BLUE,
-          'CRITICAL': YELLOW,
+          'CRITICAL': WHITE,
           'ERROR': RED}
 
 class ColoredFormatter(logging.Formatter):

pygtktalog/scan.py

@@ -8,16 +8,21 @@
 import os
 import sys
 from datetime import datetime
-import magic
+import mimetypes
 
-from pygtktalog.dbobjects import File
+from pygtktalog.dbobjects import File, Image
+from pygtktalog.dbcommon import Session
 from pygtktalog.logger import get_logger
+from pygtktalog.video import Video
+
+
 LOG = get_logger(__name__)
 
 
 class NoAccessError(Exception):
     pass
 
+
 class Scan(object):
     """
     Retrieve and identify all files recursively on given path
@@ -25,27 +30,27 @@ class Scan(object):
     def __init__(self, path):
         """
         Initialize
+        @Arguments:
+            @path - string with initial root directory to scan
         """
         self.abort = False
         self.path = path.rstrip(os.path.sep)
-        self._items = []
+        self._files = []
-        self.magic = magic.open(magic.MIME)
+        self._existing_files = []
-        self.magic.load()
+        self._session = Session()
 
     def add_files(self):
         """
         Returns list, which contain object, modification date and file
         size.
-        @Arguments:
-            @path - string with initial root directory to scan
         """
-        self._items = []
+        self._files = []
         LOG.debug("given path: %s" % self.path)
 
         # See, if file exists. If not it would raise OSError exception
         os.stat(self.path)
 
-        if not os.access(self.path, os.R_OK|os.X_OK) \
+        if not os.access(self.path, os.R_OK | os.X_OK) \
                 or not os.path.isdir(self.path):
             raise NoAccessError("Access to %s is forbidden" % self.path)
 
@@ -54,7 +59,11 @@ class Scan(object):
         if not self._recursive(None, directory, path, 0, 0, 1):
             return None
 
-        return self._items
+        # add only first item from _files, because it is a root of the other,
+        # so other will be automatically added aswell.
+        self._session.add(self._files[0])
+        self._session.commit()
+        return self._files
 
     def _get_dirsize(self, path):
         """
@@ -77,10 +86,38 @@ class Scan(object):
         """
         Try to guess type and gather information about File object if possible
         """
+        mimedict = {'audio': self._audio,
+                    'video': self._video,
+                    'image': self._image}
         fp = os.path.join(fobj.filepath.encode(sys.getfilesystemencoding()),
                           fobj.filename.encode(sys.getfilesystemencoding()))
-        import mimetypes
+
-        print mimetypes.guess_type(fp)
+        mimeinfo = mimetypes.guess_type(fp)
+        if mimeinfo[0] and mimeinfo[0].split("/")[0] in mimedict.keys():
+            mimedict[mimeinfo[0].split("/")[0]](fobj, fp)
+        else:
+            #LOG.info("Filetype not supported " + str(mimeinfo) + " " +  fp)
+            pass
+
+    def _audio(self, fobj, filepath):
+        #LOG.warning('audio')
+        return
+
+    def _image(self, fobj, filepath):
+        #LOG.warning('image')
+        return
+
+    def _video(self, fobj, filepath):
+        """
+        Make captures for a movie. Save it under uniq name.
+        """
+        vid = Video(filepath)
+
+        preview_fn = vid.capture()
+        Image(preview_fn, fobj)
+
+    def _get_all_files(self):
+        self._existing_files = self._session.query(File).all()
 
     def _mk_file(self, fname, path, parent):
         """
@@ -96,10 +133,10 @@ class Scan(object):
         fob.parent = parent
         fob.type = 2
 
-        if not parent:
+        if parent is None:
             fob.parent_id = 1
 
-        self._items.append(fob)
+        self._files.append(fob)
         return fob
 
     def _recursive(self, parent, fname, path, date, size, ftype):
@@ -124,6 +161,7 @@ class Scan(object):
         parent.size = self._get_dirsize(fullpath)
         parent.type = 1
 
+        self._get_all_files()
         root, dirs, files = os.walk(fullpath).next()
         for fname in files:
             fpath = os.path.join(root, fname)
@@ -131,9 +169,19 @@ class Scan(object):
             if os.path.islink(fpath):
                 fob.filename = fob.filename + " -> " + os.readlink(fpath)
                 fob.type = 3
-                size += fob.size
             else:
-                self._gather_information(fob)
+                existing_obj = self._object_exists(fob)
+                if existing_obj:
+                    fob.tags = existing_obj.tags
+                    fob.thumbnail = [th.get_copy \
+                            for th in existing_obj.thumbnail]
+                    fob.images = [img.get_copy() \
+                            for img in existing_obj.images]
+                else:
+                    LOG.debug("gather information")
+                    self._gather_information(fob)
+                size += fob.size
+            self._existing_files.append(fob)
 
         for dirname in dirs:
             dirpath = os.path.join(root, dirname)
@@ -153,6 +201,18 @@ class Scan(object):
         LOG.debug("size of items: %s" % parent.size)
         return True
 
+    def _object_exists(self, fobj):
+        """
+        Perform check if current File object already exists in collection. If
+        so, return first matching one, None otherwise.
+        """
+        for efobj in self._existing_files:
+            if efobj.size == fobj.size \
+                    and efobj.type == fobj.type \
+                    and efobj.date == fobj.date:
+                return efobj
+        return None
+
 class asdScan(object):
     """
     Retrieve and identify all files recursively on given path

pygtktalog/thumbnail.py

@@ -0,0 +1,104 @@
+"""
+    Project: pyGTKtalog
+    Description: Create thumbnail for sepcified image
+    Type: lib
+    Author: Roman 'gryf' Dobosz, gryf73@gmail.com
+    Created: 2011-05-15
+"""
+
+import os
+import sys
+import shutil
+from tempfile import mkstemp
+
+import Image
+
+from pygtktalog.logger import get_logger
+from pygtktalog import EXIF
+
+
+LOG = get_logger(__name__)
+
+
+class Thumbnail(object):
+    """
+    Class for generate/extract thumbnail from image file
+    """
+
+    def __init__(self, filename):
+        self.thumb_x = 160
+        self.thumb_y = 160
+        self.filename = filename
+
+    def save(self):
+        """
+        Save thumbnail into temporary file
+        """
+        exif = {}
+        orientations = {2: Image.FLIP_LEFT_RIGHT,  # Mirrored horizontal
+                        3: Image.ROTATE_180,       # Rotated 180
+                        4: Image.FLIP_TOP_BOTTOM,  # Mirrored vertical
+                        5: Image.ROTATE_90,        # Mirrored horizontal then
+                                                   # rotated 90 CCW
+                        6: Image.ROTATE_270,       # Rotated 90 CW
+                        7: Image.ROTATE_270,       # Mirrored horizontal then
+                                                   # rotated 90 CW
+                        8: Image.ROTATE_90}        # Rotated 90 CCW
+        flips = {7: Image.FLIP_LEFT_RIGHT, 5: Image.FLIP_LEFT_RIGHT}
+
+        exif = self._get_exif()
+        file_desc, thumb_fn = mkstemp(suffix=".jpg")
+        os.close(file_desc)
+
+        if 'JPEGThumbnail' not in exif:
+            LOG.debug("no exif thumb")
+            thumb = self._scale_image()
+            if thumb:
+                thumb.save(thumb_fn, "JPEG")
+        else:
+            LOG.debug("exif thumb for filename %s" % self.filename)
+            exif_thumbnail = exif['JPEGThumbnail']
+            thumb = open(thumb_fn, 'wb')
+            thumb.write(exif_thumbnail)
+            thumb.close()
+
+            if 'Image Orientation' in exif:
+                orient = exif['Image Orientation'].values[0]
+                if orient > 1 and orient in orientations:
+                    thumb_image = Image.open(self.thumb_fn)
+                    tmp_thumb_img = thumb_image.transpose(orientations[orient])
+
+                    if orient in flips:
+                        tmp_thumb_img = tmp_thumb_img.transpose(flips[orient])
+
+                    tmp_thumb_img.save(thumb_fn, 'JPEG')
+        return thumb_fn
+
+    def _get_exif(self):
+        """
+        Get exif (if available), return as a dict
+        """
+        image_file = open(self.filename, 'rb')
+        try:
+            exif = EXIF.process_file(image_file)
+        except Exception:
+            exif = {}
+            LOG.info("Exif crashed on '%s'." % self.filename)
+        finally:
+            image_file.close()
+
+        return exif
+
+    def _scale_image(self):
+        """
+        Create thumbnail. returns image object or None
+        """
+        try:
+            image_thumb = Image.open(self.filename).convert('RGB')
+        except:
+            return None
+        it_x, it_y = image_thumb.size
+        if it_x > self.thumb_x or it_y > self.thumb_y:
+            image_thumb.thumbnail((self.thumb_x, self.thumb_y),
+                                  Image.ANTIALIAS)
+        return image_thumb

pygtktalog/video.py

@@ -1,8 +1,7 @@
 """
     Project: pyGTKtalog
     Description: Gather video file information, make "screenshot" with content
-                 of the movie file. Uses external tools like mplayer and
+                 of the movie file. Uses external tools like mplayer.
-                 ImageMagick tools (montage, convert).
     Type: lib
     Author: Roman 'gryf' Dobosz, gryf73@gmail.com
     Created: 2009-04-04
@@ -95,16 +94,14 @@ class Video(object):
             no_pictures = 4
 
         tempdir = mkdtemp()
-        file_desc, image_fn = mkstemp()
+        file_desc, image_fn = mkstemp(suffix=".jpg")
         os.close(file_desc)
         self._make_captures(tempdir, no_pictures)
-        #self._make_montage(tempdir, image_fn, no_pictures)
+        self._make_montage(tempdir, image_fn, no_pictures)
-        self._make_montage3(tempdir, image_fn, no_pictures)
 
         shutil.rmtree(tempdir)
         return image_fn
 
-
     def _get_movie_info(self):
         """
         Gather movie file information with midentify shell command.
@@ -155,57 +152,7 @@ class Video(object):
             shutil.move(os.path.join(directory, "00000001.jpg"),
                         os.path.join(directory, "picture_%s.jpg" % time))
 
-    def _make_montage2(self, directory, image_fn, no_pictures):
+    def _make_montage(self, directory, image_fn, no_pictures):
-        """
-        Generate one big image from screnshots and optionally resize it. Use
-        external tools from ImageMagic package to arrange and compose final
-        image. First, images are prescaled, before they will be montaged.
-
-        Arguments:
-            @directory - source directory containing images
-            @image_fn - destination final image
-            @no_pictures - number of pictures
-        timeit result:
-            python /usr/lib/python2.6/timeit.py -n 1 -r 1 'from pygtktalog.video import Video; v = Video("/home/gryf/t/a.avi"); v.capture()'
-            1 loops, best of 1: 25 sec per loop
-        """
-        row_length = 4
-        if no_pictures < 8:
-            row_length = 2
-
-        if not (self.tags['width'] * row_length) > self.out_width:
-            for i in (8, 6, 5):
-                if (no_pictures % i) == 0 and \
-                   (i * self.tags['width']) <= self.out_width:
-                    row_length = i
-                    break
-
-        coef = float(self.out_width - row_length * 4) / (self.tags['width'] * row_length)
-        scaled_width = int(self.tags['width'] * coef)
-
-        # scale images
-        for fname in os.listdir(directory):
-            cmd = "convert -scale %d %s %s_s.jpg"
-            os.popen(cmd % (scaled_width, os.path.join(directory, fname),
-                            os.path.join(directory, fname))).readlines()
-            shutil.move(os.path.join(directory, fname + "_s.jpg"),
-                        os.path.join(directory, fname))
-
-
-        tile = "%dx%d" % (row_length, no_pictures / row_length)
-
-        _curdir = os.path.abspath(os.path.curdir)
-        os.chdir(directory)
-
-        # composite pictures
-        # readlines trick will make to wait for process end
-        cmd = "montage -tile %s -geometry +2+2 picture_*.jpg montage.jpg"
-        os.popen(cmd % tile).readlines()
-
-        shutil.move(os.path.join(directory, 'montage.jpg'), image_fn)
-        os.chdir(_curdir)
-
-    def _make_montage3(self, directory, image_fn, no_pictures):
         """
         Generate one big image from screnshots and optionally resize it. Uses
         PIL package to create output image.
@@ -214,10 +161,11 @@ class Video(object):
             @image_fn - destination final image
             @no_pictures - number of pictures
         timeit result:
-            python /usr/lib/python2.6/timeit.py -n 1 -r 1 'from pygtktalog.video import Video; v = Video("/home/gryf/t/a.avi"); v.capture()'
+            python /usr/lib/python2.6/timeit.py -n 1 -r 1 'from \
+            pygtktalog.video import Video; v = Video("/home/gryf/t/a.avi"); \
+            v.capture()'
             1 loops, best of 1: 18.8 sec per loop
         """
-        scale = False
         row_length = 4
         if no_pictures < 8:
             row_length = 2
@@ -229,9 +177,11 @@ class Video(object):
                     row_length = i
                     break
 
-        coef = float(self.out_width - row_length - 1) / (self.tags['width'] * row_length)
+        coef = float(self.out_width - row_length - 1) / \
+                (self.tags['width'] * row_length)
         if coef < 1:
-            dim = int(self.tags['width'] * coef), int(self.tags['height'] * coef)
+            dim = (int(self.tags['width'] * coef),
+                   int(self.tags['height'] * coef))
         else:
             dim = int(self.tags['width']), int(self.tags['height'])
 
@@ -261,56 +211,6 @@ class Video(object):
                 inew.paste(img, bbox)
         inew.save(image_fn, 'JPEG')
 
-    def _make_montage(self, directory, image_fn, no_pictures):
-        """
-        Generate one big image from screnshots and optionally resize it. Use
-        external tools from ImageMagic package to arrange and compose final
-        image.
-
-        Arguments:
-            @directory - source directory containing images
-            @image_fn - destination final image
-            @no_pictures - number of pictures
-        timeit result:
-            python /usr/lib/python2.6/timeit.py -n 1 -r 1 'from pygtktalog.video import Video; v = Video("/home/gryf/t/a.avi"); v.capture()'
-            1 loops, best of 1: 32.5 sec per loop
-        """
-        scale = False
-        row_length = 4
-        if no_pictures < 8:
-            row_length = 2
-
-        if (self.tags['width'] * row_length) > self.out_width:
-            scale = True
-        else:
-            for i in [8, 6, 5]:
-                if (no_pictures % i) == 0 and \
-                   (i * self.tags['width']) <= self.out_width:
-                    row_length = i
-                    break
-
-        tile = "%dx%d" % (row_length, no_pictures / row_length)
-
-        _curdir = os.path.abspath(os.path.curdir)
-        os.chdir(directory)
-
-        # composite pictures
-        # readlines trick will make to wait for process end
-        cmd = "montage -tile %s -geometry +2+2 picture_*.jpg montage.jpg"
-        os.popen(cmd % tile).readlines()
-
-        # scale it to minimum 'modern' width: 1024
-        if scale:
-            cmd = "convert -scale %s montage.jpg montage_scaled.jpg"
-            os.popen(cmd % self.out_width).readlines()
-            shutil.move(os.path.join(directory, 'montage_scaled.jpg'),
-                        image_fn)
-        else:
-            shutil.move(os.path.join(directory, 'montage.jpg'),
-                        image_fn)
-
-        os.chdir(_curdir)
-
     def _return_lower(self, chain):
         """
         Return lowercase version of provided string argument

test/unit/scan_test.py

@@ -7,9 +7,11 @@
 """
 import os
 import unittest
-import logging
 
 from pygtktalog import scan
+from pygtktalog.dbobjects import File
+from pygtktalog.dbcommon import connect, Session
+
 
 
 class TestScan(unittest.TestCase):
@@ -31,6 +33,19 @@ class TestScan(unittest.TestCase):
     3. adding new directory tree which contains same files like already stored
        in the database
     """
+    def setUp(self):
+        connect()
+        root = File()
+        root.id = 1
+        root.filename = 'root'
+        root.size = 0
+        root.source = 0
+        root.type = 0
+        root.parent_id = 1
+
+        sess = Session()
+        sess.add(root)
+        sess.commit()
 
     def test_happy_scenario(self):
         """
@@ -59,10 +74,8 @@ class TestScan(unittest.TestCase):
         scanobj.path = '/bin/sh'
         self.assertRaises(scan.NoAccessError, scanobj.add_files)
 
-
         # dir contains some non accessable items. Should just pass, and on
         # logs should be messages about it
-        logging.basicConfig(level=logging.CRITICAL)
         scanobj.path = "/mnt/data/_test_/test_dir_permissions/"
         scanobj.add_files()
 
@@ -71,8 +84,38 @@ class TestScan(unittest.TestCase):
         scanobj.abort = True
         self.assertEqual(None, scanobj.add_files())
 
+    def test_rescan(self):
+        """
+        Do the scan twice.
+        """
+        ses = Session()
+        self.assertEqual(len(ses.query(File).all()), 1)
 
+        scanob = scan.Scan("/mnt/data/_test_/test_dir")
+        scanob.add_files()
 
+        # note: we have 144 elements in db, because of root element
+        self.assertEqual(len(ses.query(File).all()), 144)
+
+        scanob2 = scan.Scan("/mnt/data/_test_/test_dir")
+        scanob2.add_files()
+        # it is perfectly ok, since we don't update collection, but just added
+        # same directory twice.
+        self.assertEqual(len(ses.query(File).all()), 287)
+        file_ob = scanob._files[2]
+        file2_ob = scanob2._files[2]
+
+        # File objects are different
+        self.assertTrue(file_ob.id != file2_ob.id)
+
+        # While Image objects points to the same file
+        self.assertTrue(file_ob.images[0].filename == \
+                file2_ob.images[0].filename)
+
+        # they are different objects
+        self.assertTrue(file_ob.images[0].id != file2_ob.images[0].id)
+
+        ses.close()
 
 if __name__ == "__main__":
     os.chdir(os.path.join(os.path.abspath(os.path.dirname(__file__)), "../"))