diff --git a/convert_1.x_to_2.x.py b/convert_1.x_to_2.x.py index 6ab7848..c988757 100755 --- a/convert_1.x_to_2.x.py +++ b/convert_1.x_to_2.x.py @@ -1,3 +1,4 @@ +#!/usr/bin/env python """ Project: pyGTKtalog Description: convert db created with v.1.x into v.2.x diff --git a/pygtktalog/dbobjects.py b/pygtktalog/dbobjects.py index 1db1773..dde3ffe 100644 --- a/pygtktalog/dbobjects.py +++ b/pygtktalog/dbobjects.py @@ -5,16 +5,26 @@ Author: Roman 'gryf' Dobosz, gryf73@gmail.com Created: 2009-08-07 """ +import os +import errno +import shutil +import uuid + from sqlalchemy import Column, Table, Integer, Text from sqlalchemy import DateTime, ForeignKey, Sequence from sqlalchemy.orm import relation, backref -from pygtktalog.dbcommon import Base +from pygtktalog.dbcommon import Base +from pygtktalog import thumbnail + + +IMG_PATH = "/home/gryf/.pygtktalog/imgs/" # FIXME: should be configurable tags_files = Table("tags_files", Base.metadata, Column("file_id", Integer, ForeignKey("files.id")), Column("tag_id", Integer, ForeignKey("tags.id"))) + class File(Base): __tablename__ = "files" id = Column(Integer, Sequence("file_id_seq"), primary_key=True) @@ -48,6 +58,7 @@ class File(Base): def __repr__(self): return "" % (str(self.filename), str(self.id)) + class Group(Base): __tablename__ = "groups" id = Column(Integer, Sequence("group_id_seq"), primary_key=True) @@ -61,6 +72,7 @@ class Group(Base): def __repr__(self): return "" % (str(self.name), str(self.id)) + class Tag(Base): __tablename__ = "tags" id = Column(Integer, Sequence("tags_id_seq"), primary_key=True) @@ -84,25 +96,103 @@ class Thumbnail(Base): file_id = Column(Integer, ForeignKey("files.id")) filename = Column(Text) - def __init__(self, filename=None): - self.filename = None + def __init__(self, filename=None, file_obj=None): + self.filename = filename + self.file = file_obj + if self.filename: + self.save(self.filename) + + def save(self, fname): + """ + Create file related thumbnail, add it to the file object. + """ + new_name = str(uuid.uuid1()).split("-") + try: + os.makedirs(os.path.join(IMG_PATH, *new_name[:-1])) + except OSError as exc: + if exc.errno != errno.EEXIST: + raise + + ext = os.path.splitext(self.filename)[1] + if ext: + new_name.append("".join([new_name.pop(), ext])) + + thumb = thumbnail.Thumbnail(self.filename).save() + name, ext = os.path.splitext(new_name.pop()) + new_name.append("".join([name, "_t", ext])) + self.filename = os.path.sep.join(new_name) + shutil.move(thumb.save(), os.path.join(IMG_PATH, *new_name)) def __repr__(self): return "" % (str(self.filename), str(self.id)) + class Image(Base): __tablename__ = "images" id = Column(Integer, Sequence("images_id_seq"), primary_key=True) file_id = Column(Integer, ForeignKey("files.id")) filename = Column(Text) - def __init__(self, filename=None): - self.filename = filename - self.file = None + def __init__(self, filename=None, file_obj=None): + self.filename = None + self.file = file_obj + if filename: + self.filename = filename + self.save(filename) + + def save(self, fname): + """ + Save and create coressponding thumbnail (note: it differs from file + related thumbnail!) + """ + new_name = str(uuid.uuid1()).split("-") + try: + os.makedirs(os.path.join(IMG_PATH, *new_name[:-1])) + except OSError as exc: + if exc.errno != errno.EEXIST: + raise + + ext = os.path.splitext(self.filename)[1] + if ext: + new_name.append("".join([new_name.pop(), ext])) + + shutil.move(self.filename, os.path.join(IMG_PATH, *new_name)) + + self.filename = os.path.sep.join(new_name) + + thumb = thumbnail.Thumbnail(os.path.join(IMG_PATH, self.filename)) + name, ext = os.path.splitext(new_name.pop()) + new_name.append("".join([name, "_t", ext])) + shutil.move(thumb.save(), os.path.join(IMG_PATH, *new_name)) + + def get_copy(self): + """ + Create the very same object as self with exception of id field + """ + img = Image() + img.filename = self.filename + return img + + @property + def thumbpath(self): + """ + Return full path to thumbnail of this image + """ + path, fname = os.path.split(self.filename) + base, ext = os.path.splitext(fname) + return os.path.join(IMG_PATH, path, base + "_t" + ext) + + @property + def imagepath(self): + """ + Return full path to image + """ + return os.path.join(IMG_PATH, self.filename) def __repr__(self): return "" % (str(self.filename), str(self.id)) + class Exif(Base): __tablename__ = "exif" id = Column(Integer, Sequence("exif_id_seq"), primary_key=True) @@ -139,6 +229,7 @@ class Exif(Base): def __repr__(self): return "" % (str(self.date), str(self.id)) + class Gthumb(Base): __tablename__ = "gthumb" id = Column(Integer, Sequence("gthumb_id_seq"), primary_key=True) @@ -155,4 +246,3 @@ class Gthumb(Base): def __repr__(self): return "" % (str(self.date), str(self.place), str(self.id)) - diff --git a/pygtktalog/logger.py b/pygtktalog/logger.py index aff9841..968488e 100644 --- a/pygtktalog/logger.py +++ b/pygtktalog/logger.py @@ -28,9 +28,9 @@ def formatter_message(message, use_color = True): return message COLORS = {'WARNING': YELLOW, - 'INFO': WHITE, + 'INFO': GREEN, 'DEBUG': BLUE, - 'CRITICAL': YELLOW, + 'CRITICAL': WHITE, 'ERROR': RED} class ColoredFormatter(logging.Formatter): diff --git a/pygtktalog/scan.py b/pygtktalog/scan.py index d8ed132..b726695 100644 --- a/pygtktalog/scan.py +++ b/pygtktalog/scan.py @@ -8,16 +8,21 @@ import os import sys from datetime import datetime -import magic +import mimetypes -from pygtktalog.dbobjects import File +from pygtktalog.dbobjects import File, Image +from pygtktalog.dbcommon import Session from pygtktalog.logger import get_logger +from pygtktalog.video import Video + + LOG = get_logger(__name__) class NoAccessError(Exception): pass + class Scan(object): """ Retrieve and identify all files recursively on given path @@ -25,27 +30,27 @@ class Scan(object): def __init__(self, path): """ Initialize + @Arguments: + @path - string with initial root directory to scan """ self.abort = False self.path = path.rstrip(os.path.sep) - self._items = [] - self.magic = magic.open(magic.MIME) - self.magic.load() + self._files = [] + self._existing_files = [] + self._session = Session() def add_files(self): """ Returns list, which contain object, modification date and file size. - @Arguments: - @path - string with initial root directory to scan """ - self._items = [] + self._files = [] LOG.debug("given path: %s" % self.path) # See, if file exists. If not it would raise OSError exception os.stat(self.path) - if not os.access(self.path, os.R_OK|os.X_OK) \ + if not os.access(self.path, os.R_OK | os.X_OK) \ or not os.path.isdir(self.path): raise NoAccessError("Access to %s is forbidden" % self.path) @@ -54,7 +59,11 @@ class Scan(object): if not self._recursive(None, directory, path, 0, 0, 1): return None - return self._items + # add only first item from _files, because it is a root of the other, + # so other will be automatically added aswell. + self._session.add(self._files[0]) + self._session.commit() + return self._files def _get_dirsize(self, path): """ @@ -77,10 +86,38 @@ class Scan(object): """ Try to guess type and gather information about File object if possible """ + mimedict = {'audio': self._audio, + 'video': self._video, + 'image': self._image} fp = os.path.join(fobj.filepath.encode(sys.getfilesystemencoding()), fobj.filename.encode(sys.getfilesystemencoding())) - import mimetypes - print mimetypes.guess_type(fp) + + mimeinfo = mimetypes.guess_type(fp) + if mimeinfo[0] and mimeinfo[0].split("/")[0] in mimedict.keys(): + mimedict[mimeinfo[0].split("/")[0]](fobj, fp) + else: + #LOG.info("Filetype not supported " + str(mimeinfo) + " " + fp) + pass + + def _audio(self, fobj, filepath): + #LOG.warning('audio') + return + + def _image(self, fobj, filepath): + #LOG.warning('image') + return + + def _video(self, fobj, filepath): + """ + Make captures for a movie. Save it under uniq name. + """ + vid = Video(filepath) + + preview_fn = vid.capture() + Image(preview_fn, fobj) + + def _get_all_files(self): + self._existing_files = self._session.query(File).all() def _mk_file(self, fname, path, parent): """ @@ -96,10 +133,10 @@ class Scan(object): fob.parent = parent fob.type = 2 - if not parent: + if parent is None: fob.parent_id = 1 - self._items.append(fob) + self._files.append(fob) return fob def _recursive(self, parent, fname, path, date, size, ftype): @@ -124,6 +161,7 @@ class Scan(object): parent.size = self._get_dirsize(fullpath) parent.type = 1 + self._get_all_files() root, dirs, files = os.walk(fullpath).next() for fname in files: fpath = os.path.join(root, fname) @@ -131,9 +169,19 @@ class Scan(object): if os.path.islink(fpath): fob.filename = fob.filename + " -> " + os.readlink(fpath) fob.type = 3 - size += fob.size else: - self._gather_information(fob) + existing_obj = self._object_exists(fob) + if existing_obj: + fob.tags = existing_obj.tags + fob.thumbnail = [th.get_copy \ + for th in existing_obj.thumbnail] + fob.images = [img.get_copy() \ + for img in existing_obj.images] + else: + LOG.debug("gather information") + self._gather_information(fob) + size += fob.size + self._existing_files.append(fob) for dirname in dirs: dirpath = os.path.join(root, dirname) @@ -153,6 +201,18 @@ class Scan(object): LOG.debug("size of items: %s" % parent.size) return True + def _object_exists(self, fobj): + """ + Perform check if current File object already exists in collection. If + so, return first matching one, None otherwise. + """ + for efobj in self._existing_files: + if efobj.size == fobj.size \ + and efobj.type == fobj.type \ + and efobj.date == fobj.date: + return efobj + return None + class asdScan(object): """ Retrieve and identify all files recursively on given path diff --git a/pygtktalog/thumbnail.py b/pygtktalog/thumbnail.py new file mode 100644 index 0000000..3bb44c8 --- /dev/null +++ b/pygtktalog/thumbnail.py @@ -0,0 +1,104 @@ +""" + Project: pyGTKtalog + Description: Create thumbnail for sepcified image + Type: lib + Author: Roman 'gryf' Dobosz, gryf73@gmail.com + Created: 2011-05-15 +""" + +import os +import sys +import shutil +from tempfile import mkstemp + +import Image + +from pygtktalog.logger import get_logger +from pygtktalog import EXIF + + +LOG = get_logger(__name__) + + +class Thumbnail(object): + """ + Class for generate/extract thumbnail from image file + """ + + def __init__(self, filename): + self.thumb_x = 160 + self.thumb_y = 160 + self.filename = filename + + def save(self): + """ + Save thumbnail into temporary file + """ + exif = {} + orientations = {2: Image.FLIP_LEFT_RIGHT, # Mirrored horizontal + 3: Image.ROTATE_180, # Rotated 180 + 4: Image.FLIP_TOP_BOTTOM, # Mirrored vertical + 5: Image.ROTATE_90, # Mirrored horizontal then + # rotated 90 CCW + 6: Image.ROTATE_270, # Rotated 90 CW + 7: Image.ROTATE_270, # Mirrored horizontal then + # rotated 90 CW + 8: Image.ROTATE_90} # Rotated 90 CCW + flips = {7: Image.FLIP_LEFT_RIGHT, 5: Image.FLIP_LEFT_RIGHT} + + exif = self._get_exif() + file_desc, thumb_fn = mkstemp(suffix=".jpg") + os.close(file_desc) + + if 'JPEGThumbnail' not in exif: + LOG.debug("no exif thumb") + thumb = self._scale_image() + if thumb: + thumb.save(thumb_fn, "JPEG") + else: + LOG.debug("exif thumb for filename %s" % self.filename) + exif_thumbnail = exif['JPEGThumbnail'] + thumb = open(thumb_fn, 'wb') + thumb.write(exif_thumbnail) + thumb.close() + + if 'Image Orientation' in exif: + orient = exif['Image Orientation'].values[0] + if orient > 1 and orient in orientations: + thumb_image = Image.open(self.thumb_fn) + tmp_thumb_img = thumb_image.transpose(orientations[orient]) + + if orient in flips: + tmp_thumb_img = tmp_thumb_img.transpose(flips[orient]) + + tmp_thumb_img.save(thumb_fn, 'JPEG') + return thumb_fn + + def _get_exif(self): + """ + Get exif (if available), return as a dict + """ + image_file = open(self.filename, 'rb') + try: + exif = EXIF.process_file(image_file) + except Exception: + exif = {} + LOG.info("Exif crashed on '%s'." % self.filename) + finally: + image_file.close() + + return exif + + def _scale_image(self): + """ + Create thumbnail. returns image object or None + """ + try: + image_thumb = Image.open(self.filename).convert('RGB') + except: + return None + it_x, it_y = image_thumb.size + if it_x > self.thumb_x or it_y > self.thumb_y: + image_thumb.thumbnail((self.thumb_x, self.thumb_y), + Image.ANTIALIAS) + return image_thumb diff --git a/pygtktalog/video.py b/pygtktalog/video.py index 1ccf4e6..14dde2e 100644 --- a/pygtktalog/video.py +++ b/pygtktalog/video.py @@ -1,8 +1,7 @@ """ Project: pyGTKtalog Description: Gather video file information, make "screenshot" with content - of the movie file. Uses external tools like mplayer and - ImageMagick tools (montage, convert). + of the movie file. Uses external tools like mplayer. Type: lib Author: Roman 'gryf' Dobosz, gryf73@gmail.com Created: 2009-04-04 @@ -95,16 +94,14 @@ class Video(object): no_pictures = 4 tempdir = mkdtemp() - file_desc, image_fn = mkstemp() + file_desc, image_fn = mkstemp(suffix=".jpg") os.close(file_desc) self._make_captures(tempdir, no_pictures) - #self._make_montage(tempdir, image_fn, no_pictures) - self._make_montage3(tempdir, image_fn, no_pictures) + self._make_montage(tempdir, image_fn, no_pictures) shutil.rmtree(tempdir) return image_fn - def _get_movie_info(self): """ Gather movie file information with midentify shell command. @@ -155,57 +152,7 @@ class Video(object): shutil.move(os.path.join(directory, "00000001.jpg"), os.path.join(directory, "picture_%s.jpg" % time)) - def _make_montage2(self, directory, image_fn, no_pictures): - """ - Generate one big image from screnshots and optionally resize it. Use - external tools from ImageMagic package to arrange and compose final - image. First, images are prescaled, before they will be montaged. - - Arguments: - @directory - source directory containing images - @image_fn - destination final image - @no_pictures - number of pictures - timeit result: - python /usr/lib/python2.6/timeit.py -n 1 -r 1 'from pygtktalog.video import Video; v = Video("/home/gryf/t/a.avi"); v.capture()' - 1 loops, best of 1: 25 sec per loop - """ - row_length = 4 - if no_pictures < 8: - row_length = 2 - - if not (self.tags['width'] * row_length) > self.out_width: - for i in (8, 6, 5): - if (no_pictures % i) == 0 and \ - (i * self.tags['width']) <= self.out_width: - row_length = i - break - - coef = float(self.out_width - row_length * 4) / (self.tags['width'] * row_length) - scaled_width = int(self.tags['width'] * coef) - - # scale images - for fname in os.listdir(directory): - cmd = "convert -scale %d %s %s_s.jpg" - os.popen(cmd % (scaled_width, os.path.join(directory, fname), - os.path.join(directory, fname))).readlines() - shutil.move(os.path.join(directory, fname + "_s.jpg"), - os.path.join(directory, fname)) - - - tile = "%dx%d" % (row_length, no_pictures / row_length) - - _curdir = os.path.abspath(os.path.curdir) - os.chdir(directory) - - # composite pictures - # readlines trick will make to wait for process end - cmd = "montage -tile %s -geometry +2+2 picture_*.jpg montage.jpg" - os.popen(cmd % tile).readlines() - - shutil.move(os.path.join(directory, 'montage.jpg'), image_fn) - os.chdir(_curdir) - - def _make_montage3(self, directory, image_fn, no_pictures): + def _make_montage(self, directory, image_fn, no_pictures): """ Generate one big image from screnshots and optionally resize it. Uses PIL package to create output image. @@ -214,10 +161,11 @@ class Video(object): @image_fn - destination final image @no_pictures - number of pictures timeit result: - python /usr/lib/python2.6/timeit.py -n 1 -r 1 'from pygtktalog.video import Video; v = Video("/home/gryf/t/a.avi"); v.capture()' + python /usr/lib/python2.6/timeit.py -n 1 -r 1 'from \ + pygtktalog.video import Video; v = Video("/home/gryf/t/a.avi"); \ + v.capture()' 1 loops, best of 1: 18.8 sec per loop """ - scale = False row_length = 4 if no_pictures < 8: row_length = 2 @@ -229,9 +177,11 @@ class Video(object): row_length = i break - coef = float(self.out_width - row_length - 1) / (self.tags['width'] * row_length) + coef = float(self.out_width - row_length - 1) / \ + (self.tags['width'] * row_length) if coef < 1: - dim = int(self.tags['width'] * coef), int(self.tags['height'] * coef) + dim = (int(self.tags['width'] * coef), + int(self.tags['height'] * coef)) else: dim = int(self.tags['width']), int(self.tags['height']) @@ -261,56 +211,6 @@ class Video(object): inew.paste(img, bbox) inew.save(image_fn, 'JPEG') - def _make_montage(self, directory, image_fn, no_pictures): - """ - Generate one big image from screnshots and optionally resize it. Use - external tools from ImageMagic package to arrange and compose final - image. - - Arguments: - @directory - source directory containing images - @image_fn - destination final image - @no_pictures - number of pictures - timeit result: - python /usr/lib/python2.6/timeit.py -n 1 -r 1 'from pygtktalog.video import Video; v = Video("/home/gryf/t/a.avi"); v.capture()' - 1 loops, best of 1: 32.5 sec per loop - """ - scale = False - row_length = 4 - if no_pictures < 8: - row_length = 2 - - if (self.tags['width'] * row_length) > self.out_width: - scale = True - else: - for i in [8, 6, 5]: - if (no_pictures % i) == 0 and \ - (i * self.tags['width']) <= self.out_width: - row_length = i - break - - tile = "%dx%d" % (row_length, no_pictures / row_length) - - _curdir = os.path.abspath(os.path.curdir) - os.chdir(directory) - - # composite pictures - # readlines trick will make to wait for process end - cmd = "montage -tile %s -geometry +2+2 picture_*.jpg montage.jpg" - os.popen(cmd % tile).readlines() - - # scale it to minimum 'modern' width: 1024 - if scale: - cmd = "convert -scale %s montage.jpg montage_scaled.jpg" - os.popen(cmd % self.out_width).readlines() - shutil.move(os.path.join(directory, 'montage_scaled.jpg'), - image_fn) - else: - shutil.move(os.path.join(directory, 'montage.jpg'), - image_fn) - - os.chdir(_curdir) - def _return_lower(self, chain): """ Return lowercase version of provided string argument diff --git a/test/unit/scan_test.py b/test/unit/scan_test.py index 4e7bb0d..3de86c9 100644 --- a/test/unit/scan_test.py +++ b/test/unit/scan_test.py @@ -7,9 +7,11 @@ """ import os import unittest -import logging from pygtktalog import scan +from pygtktalog.dbobjects import File +from pygtktalog.dbcommon import connect, Session + class TestScan(unittest.TestCase): @@ -31,6 +33,19 @@ class TestScan(unittest.TestCase): 3. adding new directory tree which contains same files like already stored in the database """ + def setUp(self): + connect() + root = File() + root.id = 1 + root.filename = 'root' + root.size = 0 + root.source = 0 + root.type = 0 + root.parent_id = 1 + + sess = Session() + sess.add(root) + sess.commit() def test_happy_scenario(self): """ @@ -59,10 +74,8 @@ class TestScan(unittest.TestCase): scanobj.path = '/bin/sh' self.assertRaises(scan.NoAccessError, scanobj.add_files) - # dir contains some non accessable items. Should just pass, and on # logs should be messages about it - logging.basicConfig(level=logging.CRITICAL) scanobj.path = "/mnt/data/_test_/test_dir_permissions/" scanobj.add_files() @@ -71,8 +84,38 @@ class TestScan(unittest.TestCase): scanobj.abort = True self.assertEqual(None, scanobj.add_files()) + def test_rescan(self): + """ + Do the scan twice. + """ + ses = Session() + self.assertEqual(len(ses.query(File).all()), 1) + scanob = scan.Scan("/mnt/data/_test_/test_dir") + scanob.add_files() + # note: we have 144 elements in db, because of root element + self.assertEqual(len(ses.query(File).all()), 144) + + scanob2 = scan.Scan("/mnt/data/_test_/test_dir") + scanob2.add_files() + # it is perfectly ok, since we don't update collection, but just added + # same directory twice. + self.assertEqual(len(ses.query(File).all()), 287) + file_ob = scanob._files[2] + file2_ob = scanob2._files[2] + + # File objects are different + self.assertTrue(file_ob.id != file2_ob.id) + + # While Image objects points to the same file + self.assertTrue(file_ob.images[0].filename == \ + file2_ob.images[0].filename) + + # they are different objects + self.assertTrue(file_ob.images[0].id != file2_ob.images[0].id) + + ses.close() if __name__ == "__main__": os.chdir(os.path.join(os.path.abspath(os.path.dirname(__file__)), "../"))