gryf/pygtktalog
1
0
Fork 0
mirror of https://github.com/gryf/pygtktalog.git synced 2025-12-17 11:30:19 +01:00
Code Issues Projects Releases Wiki Activity

Added logic for Image and Thumbnail db objects. Added scan functionality

Browse Source
This commit is contained in:
gryf
2011-05-15 16:50:54 +02:00
parent 3e2634bc57
commit ad1703cd90
7 changed files with 337 additions and 139 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
convert_1.x_to_2.x.py
View File

@@ -1,3 +1,4 @@
#!/usr/bin/env python
"""
Project: pyGTKtalog
Description: convert db created with v.1.x into v.2.x

104
pygtktalog/dbobjects.py
View File

@@ -5,16 +5,26 @@
Author: Roman 'gryf' Dobosz, gryf73@gmail.com
Created: 2009-08-07
"""
import os
import errno
import shutil
import uuid
from sqlalchemy import Column, Table, Integer, Text
from sqlalchemy import DateTime, ForeignKey, Sequence
from sqlalchemy.orm import relation, backref
from pygtktalog.dbcommon import Base
from pygtktalog.dbcommon import Base
from pygtktalog import thumbnail
IMG_PATH = "/home/gryf/.pygtktalog/imgs/" # FIXME: should be configurable
tags_files = Table("tags_files", Base.metadata,
Column("file_id", Integer, ForeignKey("files.id")),
Column("tag_id", Integer, ForeignKey("tags.id")))
class File(Base):
__tablename__ = "files"
id = Column(Integer, Sequence("file_id_seq"), primary_key=True)
@@ -48,6 +58,7 @@ class File(Base):
def __repr__(self):
return "<File('%s', %s)>" % (str(self.filename), str(self.id))
class Group(Base):
__tablename__ = "groups"
id = Column(Integer, Sequence("group_id_seq"), primary_key=True)
@@ -61,6 +72,7 @@ class Group(Base):
def __repr__(self):
return "<Group('%s', %s)>" % (str(self.name), str(self.id))
class Tag(Base):
__tablename__ = "tags"
id = Column(Integer, Sequence("tags_id_seq"), primary_key=True)
@@ -84,25 +96,103 @@ class Thumbnail(Base):
file_id = Column(Integer, ForeignKey("files.id"))
filename = Column(Text)
def __init__(self, filename=None):
self.filename = None
def __init__(self, filename=None, file_obj=None):
self.filename = filename
self.file = file_obj
if self.filename:
self.save(self.filename)
def save(self, fname):
"""
Create file related thumbnail, add it to the file object.
"""
new_name = str(uuid.uuid1()).split("-")
try:
os.makedirs(os.path.join(IMG_PATH, *new_name[:-1]))
except OSError as exc:
if exc.errno != errno.EEXIST:
raise
ext = os.path.splitext(self.filename)[1]
if ext:
new_name.append("".join([new_name.pop(), ext]))
thumb = thumbnail.Thumbnail(self.filename).save()
name, ext = os.path.splitext(new_name.pop())
new_name.append("".join([name, "_t", ext]))
self.filename = os.path.sep.join(new_name)
shutil.move(thumb.save(), os.path.join(IMG_PATH, *new_name))
def __repr__(self):
return "<Thumbnail('%s', %s)>" % (str(self.filename), str(self.id))
class Image(Base):
__tablename__ = "images"
id = Column(Integer, Sequence("images_id_seq"), primary_key=True)
file_id = Column(Integer, ForeignKey("files.id"))
filename = Column(Text)
def __init__(self, filename=None):
self.filename = filename
self.file = None
def __init__(self, filename=None, file_obj=None):
self.filename = None
self.file = file_obj
if filename:
self.filename = filename
self.save(filename)
def save(self, fname):
"""
Save and create coressponding thumbnail (note: it differs from file
related thumbnail!)
"""
new_name = str(uuid.uuid1()).split("-")
try:
os.makedirs(os.path.join(IMG_PATH, *new_name[:-1]))
except OSError as exc:
if exc.errno != errno.EEXIST:
raise
ext = os.path.splitext(self.filename)[1]
if ext:
new_name.append("".join([new_name.pop(), ext]))
shutil.move(self.filename, os.path.join(IMG_PATH, *new_name))
self.filename = os.path.sep.join(new_name)
thumb = thumbnail.Thumbnail(os.path.join(IMG_PATH, self.filename))
name, ext = os.path.splitext(new_name.pop())
new_name.append("".join([name, "_t", ext]))
shutil.move(thumb.save(), os.path.join(IMG_PATH, *new_name))
def get_copy(self):
"""
Create the very same object as self with exception of id field
"""
img = Image()
img.filename = self.filename
return img
@property
def thumbpath(self):
"""
Return full path to thumbnail of this image
"""
path, fname = os.path.split(self.filename)
base, ext = os.path.splitext(fname)
return os.path.join(IMG_PATH, path, base + "_t" + ext)
@property
def imagepath(self):
"""
Return full path to image
"""
return os.path.join(IMG_PATH, self.filename)
def __repr__(self):
return "<Image('%s', %s)>" % (str(self.filename), str(self.id))
class Exif(Base):
__tablename__ = "exif"
id = Column(Integer, Sequence("exif_id_seq"), primary_key=True)
@@ -139,6 +229,7 @@ class Exif(Base):
def __repr__(self):
return "<Exif('%s', %s)>" % (str(self.date), str(self.id))
class Gthumb(Base):
__tablename__ = "gthumb"
id = Column(Integer, Sequence("gthumb_id_seq"), primary_key=True)
@@ -155,4 +246,3 @@ class Gthumb(Base):
def __repr__(self):
return "<Gthumb('%s', '%s', %s)>" % (str(self.date), str(self.place),
str(self.id))

4
pygtktalog/logger.py
View File

@@ -28,9 +28,9 @@ def formatter_message(message, use_color = True):
return message
COLORS = {'WARNING': YELLOW,
'INFO': WHITE,
'INFO': GREEN,
'DEBUG': BLUE,
'CRITICAL': YELLOW,
'CRITICAL': WHITE,
'ERROR': RED}
class ColoredFormatter(logging.Formatter):

92
pygtktalog/scan.py
View File

@@ -8,16 +8,21 @@
import os
import sys
from datetime import datetime
import magic
import mimetypes
from pygtktalog.dbobjects import File
from pygtktalog.dbobjects import File, Image
from pygtktalog.dbcommon import Session
from pygtktalog.logger import get_logger
from pygtktalog.video import Video
LOG = get_logger(__name__)
class NoAccessError(Exception):
pass
class Scan(object):
"""
Retrieve and identify all files recursively on given path
@@ -25,27 +30,27 @@ class Scan(object):
def __init__(self, path):
"""
Initialize
@Arguments:
@path - string with initial root directory to scan
"""
self.abort = False
self.path = path.rstrip(os.path.sep)
self._items = []
self.magic = magic.open(magic.MIME)
self.magic.load()
self._files = []
self._existing_files = []
self._session = Session()
def add_files(self):
"""
Returns list, which contain object, modification date and file
size.
@Arguments:
@path - string with initial root directory to scan
"""
self._items = []
self._files = []
LOG.debug("given path: %s" % self.path)
# See, if file exists. If not it would raise OSError exception
os.stat(self.path)
if not os.access(self.path, os.R_OK|os.X_OK) \
if not os.access(self.path, os.R_OK | os.X_OK) \
or not os.path.isdir(self.path):
raise NoAccessError("Access to %s is forbidden" % self.path)
@@ -54,7 +59,11 @@ class Scan(object):
if not self._recursive(None, directory, path, 0, 0, 1):
return None
return self._items
# add only first item from _files, because it is a root of the other,
# so other will be automatically added aswell.
self._session.add(self._files[0])
self._session.commit()
return self._files
def _get_dirsize(self, path):
"""
@@ -77,10 +86,38 @@ class Scan(object):
"""
Try to guess type and gather information about File object if possible
"""
mimedict = {'audio': self._audio,
'video': self._video,
'image': self._image}
fp = os.path.join(fobj.filepath.encode(sys.getfilesystemencoding()),
fobj.filename.encode(sys.getfilesystemencoding()))
import mimetypes
print mimetypes.guess_type(fp)
mimeinfo = mimetypes.guess_type(fp)
if mimeinfo[0] and mimeinfo[0].split("/")[0] in mimedict.keys():
mimedict[mimeinfo[0].split("/")[0]](fobj, fp)
else:
#LOG.info("Filetype not supported " + str(mimeinfo) + " " + fp)
pass
def _audio(self, fobj, filepath):
#LOG.warning('audio')
return
def _image(self, fobj, filepath):
#LOG.warning('image')
return
def _video(self, fobj, filepath):
"""
Make captures for a movie. Save it under uniq name.
"""
vid = Video(filepath)
preview_fn = vid.capture()
Image(preview_fn, fobj)
def _get_all_files(self):
self._existing_files = self._session.query(File).all()
def _mk_file(self, fname, path, parent):
"""
@@ -96,10 +133,10 @@ class Scan(object):
fob.parent = parent
fob.type = 2
if not parent:
if parent is None:
fob.parent_id = 1
self._items.append(fob)
self._files.append(fob)
return fob
def _recursive(self, parent, fname, path, date, size, ftype):
@@ -124,6 +161,7 @@ class Scan(object):
parent.size = self._get_dirsize(fullpath)
parent.type = 1
self._get_all_files()
root, dirs, files = os.walk(fullpath).next()
for fname in files:
fpath = os.path.join(root, fname)
@@ -131,9 +169,19 @@ class Scan(object):
if os.path.islink(fpath):
fob.filename = fob.filename + " -> " + os.readlink(fpath)
fob.type = 3
size += fob.size
else:
self._gather_information(fob)
existing_obj = self._object_exists(fob)
if existing_obj:
fob.tags = existing_obj.tags
fob.thumbnail = [th.get_copy \
for th in existing_obj.thumbnail]
fob.images = [img.get_copy() \
for img in existing_obj.images]
else:
LOG.debug("gather information")
self._gather_information(fob)
size += fob.size
self._existing_files.append(fob)
for dirname in dirs:
dirpath = os.path.join(root, dirname)
@@ -153,6 +201,18 @@ class Scan(object):
LOG.debug("size of items: %s" % parent.size)
return True
def _object_exists(self, fobj):
"""
Perform check if current File object already exists in collection. If
so, return first matching one, None otherwise.
"""
for efobj in self._existing_files:
if efobj.size == fobj.size \
and efobj.type == fobj.type \
and efobj.date == fobj.date:
return efobj
return None
class asdScan(object):
"""
Retrieve and identify all files recursively on given path

104
pygtktalog/thumbnail.py Normal file
View File

@@ -0,0 +1,104 @@
"""
Project: pyGTKtalog
Description: Create thumbnail for sepcified image
Type: lib
Author: Roman 'gryf' Dobosz, gryf73@gmail.com
Created: 2011-05-15
"""
import os
import sys
import shutil
from tempfile import mkstemp
import Image
from pygtktalog.logger import get_logger
from pygtktalog import EXIF
LOG = get_logger(__name__)
class Thumbnail(object):
"""
Class for generate/extract thumbnail from image file
"""
def __init__(self, filename):
self.thumb_x = 160
self.thumb_y = 160
self.filename = filename
def save(self):
"""
Save thumbnail into temporary file
"""
exif = {}
orientations = {2: Image.FLIP_LEFT_RIGHT, # Mirrored horizontal
3: Image.ROTATE_180, # Rotated 180
4: Image.FLIP_TOP_BOTTOM, # Mirrored vertical
5: Image.ROTATE_90, # Mirrored horizontal then
# rotated 90 CCW
6: Image.ROTATE_270, # Rotated 90 CW
7: Image.ROTATE_270, # Mirrored horizontal then
# rotated 90 CW
8: Image.ROTATE_90} # Rotated 90 CCW
flips = {7: Image.FLIP_LEFT_RIGHT, 5: Image.FLIP_LEFT_RIGHT}
exif = self._get_exif()
file_desc, thumb_fn = mkstemp(suffix=".jpg")
os.close(file_desc)
if 'JPEGThumbnail' not in exif:
LOG.debug("no exif thumb")
thumb = self._scale_image()
if thumb:
thumb.save(thumb_fn, "JPEG")
else:
LOG.debug("exif thumb for filename %s" % self.filename)
exif_thumbnail = exif['JPEGThumbnail']
thumb = open(thumb_fn, 'wb')
thumb.write(exif_thumbnail)
thumb.close()
if 'Image Orientation' in exif:
orient = exif['Image Orientation'].values[0]
if orient > 1 and orient in orientations:
thumb_image = Image.open(self.thumb_fn)
tmp_thumb_img = thumb_image.transpose(orientations[orient])
if orient in flips:
tmp_thumb_img = tmp_thumb_img.transpose(flips[orient])
tmp_thumb_img.save(thumb_fn, 'JPEG')
return thumb_fn
def _get_exif(self):
"""
Get exif (if available), return as a dict
"""
image_file = open(self.filename, 'rb')
try:
exif = EXIF.process_file(image_file)
except Exception:
exif = {}
LOG.info("Exif crashed on '%s'." % self.filename)
finally:
image_file.close()
return exif
def _scale_image(self):
"""
Create thumbnail. returns image object or None
"""
try:
image_thumb = Image.open(self.filename).convert('RGB')
except:
return None
it_x, it_y = image_thumb.size
if it_x > self.thumb_x or it_y > self.thumb_y:
image_thumb.thumbnail((self.thumb_x, self.thumb_y),
Image.ANTIALIAS)
return image_thumb

122
pygtktalog/video.py
View File

@@ -1,8 +1,7 @@
"""
Project: pyGTKtalog
Description: Gather video file information, make "screenshot" with content
of the movie file. Uses external tools like mplayer and
ImageMagick tools (montage, convert).
of the movie file. Uses external tools like mplayer.
Type: lib
Author: Roman 'gryf' Dobosz, gryf73@gmail.com
Created: 2009-04-04
@@ -95,16 +94,14 @@ class Video(object):
no_pictures = 4
tempdir = mkdtemp()
file_desc, image_fn = mkstemp()
file_desc, image_fn = mkstemp(suffix=".jpg")
os.close(file_desc)
self._make_captures(tempdir, no_pictures)
#self._make_montage(tempdir, image_fn, no_pictures)
self._make_montage3(tempdir, image_fn, no_pictures)
self._make_montage(tempdir, image_fn, no_pictures)
shutil.rmtree(tempdir)
return image_fn
def _get_movie_info(self):
"""
Gather movie file information with midentify shell command.
@@ -155,57 +152,7 @@ class Video(object):
shutil.move(os.path.join(directory, "00000001.jpg"),
os.path.join(directory, "picture_%s.jpg" % time))
def _make_montage2(self, directory, image_fn, no_pictures):
"""
Generate one big image from screnshots and optionally resize it. Use
external tools from ImageMagic package to arrange and compose final
image. First, images are prescaled, before they will be montaged.
Arguments:
@directory - source directory containing images
@image_fn - destination final image
@no_pictures - number of pictures
timeit result:
python /usr/lib/python2.6/timeit.py -n 1 -r 1 'from pygtktalog.video import Video; v = Video("/home/gryf/t/a.avi"); v.capture()'
1 loops, best of 1: 25 sec per loop
"""
row_length = 4
if no_pictures < 8:
row_length = 2
if not (self.tags['width'] * row_length) > self.out_width:
for i in (8, 6, 5):
if (no_pictures % i) == 0 and \
(i * self.tags['width']) <= self.out_width:
row_length = i
break
coef = float(self.out_width - row_length * 4) / (self.tags['width'] * row_length)
scaled_width = int(self.tags['width'] * coef)
# scale images
for fname in os.listdir(directory):
cmd = "convert -scale %d %s %s_s.jpg"
os.popen(cmd % (scaled_width, os.path.join(directory, fname),
os.path.join(directory, fname))).readlines()
shutil.move(os.path.join(directory, fname + "_s.jpg"),
os.path.join(directory, fname))
tile = "%dx%d" % (row_length, no_pictures / row_length)
_curdir = os.path.abspath(os.path.curdir)
os.chdir(directory)
# composite pictures
# readlines trick will make to wait for process end
cmd = "montage -tile %s -geometry +2+2 picture_*.jpg montage.jpg"
os.popen(cmd % tile).readlines()
shutil.move(os.path.join(directory, 'montage.jpg'), image_fn)
os.chdir(_curdir)
def _make_montage3(self, directory, image_fn, no_pictures):
def _make_montage(self, directory, image_fn, no_pictures):
"""
Generate one big image from screnshots and optionally resize it. Uses
PIL package to create output image.
@@ -214,10 +161,11 @@ class Video(object):
@image_fn - destination final image
@no_pictures - number of pictures
timeit result:
python /usr/lib/python2.6/timeit.py -n 1 -r 1 'from pygtktalog.video import Video; v = Video("/home/gryf/t/a.avi"); v.capture()'
python /usr/lib/python2.6/timeit.py -n 1 -r 1 'from \
pygtktalog.video import Video; v = Video("/home/gryf/t/a.avi"); \
v.capture()'
1 loops, best of 1: 18.8 sec per loop
"""
scale = False
row_length = 4
if no_pictures < 8:
row_length = 2
@@ -229,9 +177,11 @@ class Video(object):
row_length = i
break
coef = float(self.out_width - row_length - 1) / (self.tags['width'] * row_length)
coef = float(self.out_width - row_length - 1) / \
(self.tags['width'] * row_length)
if coef < 1:
dim = int(self.tags['width'] * coef), int(self.tags['height'] * coef)
dim = (int(self.tags['width'] * coef),
int(self.tags['height'] * coef))
else:
dim = int(self.tags['width']), int(self.tags['height'])
@@ -261,56 +211,6 @@ class Video(object):
inew.paste(img, bbox)
inew.save(image_fn, 'JPEG')
def _make_montage(self, directory, image_fn, no_pictures):
"""
Generate one big image from screnshots and optionally resize it. Use
external tools from ImageMagic package to arrange and compose final
image.
Arguments:
@directory - source directory containing images
@image_fn - destination final image
@no_pictures - number of pictures
timeit result:
python /usr/lib/python2.6/timeit.py -n 1 -r 1 'from pygtktalog.video import Video; v = Video("/home/gryf/t/a.avi"); v.capture()'
1 loops, best of 1: 32.5 sec per loop
"""
scale = False
row_length = 4
if no_pictures < 8:
row_length = 2
if (self.tags['width'] * row_length) > self.out_width:
scale = True
else:
for i in [8, 6, 5]:
if (no_pictures % i) == 0 and \
(i * self.tags['width']) <= self.out_width:
row_length = i
break
tile = "%dx%d" % (row_length, no_pictures / row_length)
_curdir = os.path.abspath(os.path.curdir)
os.chdir(directory)
# composite pictures
# readlines trick will make to wait for process end
cmd = "montage -tile %s -geometry +2+2 picture_*.jpg montage.jpg"
os.popen(cmd % tile).readlines()
# scale it to minimum 'modern' width: 1024
if scale:
cmd = "convert -scale %s montage.jpg montage_scaled.jpg"
os.popen(cmd % self.out_width).readlines()
shutil.move(os.path.join(directory, 'montage_scaled.jpg'),
image_fn)
else:
shutil.move(os.path.join(directory, 'montage.jpg'),
image_fn)
os.chdir(_curdir)
def _return_lower(self, chain):
"""
Return lowercase version of provided string argument

49
test/unit/scan_test.py
View File

@@ -7,9 +7,11 @@
"""
import os
import unittest
import logging
from pygtktalog import scan
from pygtktalog.dbobjects import File
from pygtktalog.dbcommon import connect, Session
class TestScan(unittest.TestCase):
@@ -31,6 +33,19 @@ class TestScan(unittest.TestCase):
3. adding new directory tree which contains same files like already stored
in the database
"""
def setUp(self):
connect()
root = File()
root.id = 1
root.filename = 'root'
root.size = 0
root.source = 0
root.type = 0
root.parent_id = 1
sess = Session()
sess.add(root)
sess.commit()
def test_happy_scenario(self):
"""
@@ -59,10 +74,8 @@ class TestScan(unittest.TestCase):
scanobj.path = '/bin/sh'
self.assertRaises(scan.NoAccessError, scanobj.add_files)
# dir contains some non accessable items. Should just pass, and on
# logs should be messages about it
logging.basicConfig(level=logging.CRITICAL)
scanobj.path = "/mnt/data/_test_/test_dir_permissions/"
scanobj.add_files()
@@ -71,8 +84,38 @@ class TestScan(unittest.TestCase):
scanobj.abort = True
self.assertEqual(None, scanobj.add_files())
def test_rescan(self):
"""
Do the scan twice.
"""
ses = Session()
self.assertEqual(len(ses.query(File).all()), 1)
scanob = scan.Scan("/mnt/data/_test_/test_dir")
scanob.add_files()
# note: we have 144 elements in db, because of root element
self.assertEqual(len(ses.query(File).all()), 144)
scanob2 = scan.Scan("/mnt/data/_test_/test_dir")
scanob2.add_files()
# it is perfectly ok, since we don't update collection, but just added
# same directory twice.
self.assertEqual(len(ses.query(File).all()), 287)
file_ob = scanob._files[2]
file2_ob = scanob2._files[2]
# File objects are different
self.assertTrue(file_ob.id != file2_ob.id)
# While Image objects points to the same file
self.assertTrue(file_ob.images[0].filename == \
file2_ob.images[0].filename)
# they are different objects
self.assertTrue(file_ob.images[0].id != file2_ob.images[0].id)
ses.close()
if __name__ == "__main__":
os.chdir(os.path.join(os.path.abspath(os.path.dirname(__file__)), "../"))