gryf/pygtktalog
1
0
Fork 0
mirror of https://github.com/gryf/pygtktalog.git synced 2025-12-17 03:20:17 +01:00
Code Issues Projects Releases Wiki Activity

Get rid of creating thumbs/images for the file objects.

Browse Source
This commit is contained in:
gryf
2022-09-25 17:14:10 +02:00
parent 01fd964e0d
commit b284f328b3
5 changed files with 3 additions and 468 deletions
Download Patch File Download Diff File Expand all files Collapse all files

133
pycatalog/__init__.py
View File

@@ -195,7 +195,7 @@ class Iface(object):
# scanob.update_files(node.id)
scanob.update_files(node.id, self.engine)
def create(self, dir_to_add, data_dir):
def create(self, dir_to_add):
"""Create new database"""
self.root = dbo.File()
self.root.id = 1
@@ -205,9 +205,6 @@ class Iface(object):
self.root.type = 0
self.root.parent_id = 1
config = dbo.Config()
config.key = 'image_path'
config.value = data_dir
if not self.dry_run:
self.sess.add(self.root)
@@ -217,11 +214,6 @@ class Iface(object):
print(colorize("Creating new db against directory `%s'" % dir_to_add,
'white'))
if not self.dry_run:
if data_dir == ':same_as_db:':
misc.calculate_image_path(None, True)
else:
misc.calculate_image_path(data_dir, True)
scanob = scan.Scan(dir_to_add)
scanob.add_files(self.engine)
@@ -286,105 +278,6 @@ class Iface(object):
for item in result:
print(self._annotate(item, search_words))
def fsck(self):
"""Fsck orphaned images/thumbs"""
image_path = (self.sess.query(dbo.Config)
.filter(dbo.Config.key == 'image_path')).one().value
if image_path == ':same_as_db:':
image_path = misc.calculate_image_path(None, False)
files_to_remove = []
# remove images/thumbnails which doesn't have file relation
for name, obj in (("images", dbo.Image),
("thumbnails", dbo.Thumbnail)):
self._purge_orphaned_objects(obj, "Scanning %s " % name)
# find all image files not associate with either Image (image/thumb)
# or Thumbnail (thumb) objects
sys.stdout.write(40 * " " + "\r")
count = 0
for root, dirs, files in os.walk(image_path):
for fname in files:
sys.stdout.write("Scanning files " +
"| / - \\".split()[count % 4] + "\r")
sys.stdout.flush()
count += 1
fname_ = os.path.join(root.split(image_path)[1],
fname).lstrip('/')
if '_t' in fname:
obj = (self.sess.query(dbo.Thumbnail)
.filter(dbo.Thumbnail.filename == fname_)).all()
if obj:
continue
obj = (self.sess.query(dbo.Image)
.filter(dbo.Image.filename ==
fname_.replace('_t.', '.'))).all()
if obj:
continue
else:
obj = (self.sess.query(dbo.Image)
.filter(dbo.Image.filename == fname_)).all()
if obj:
continue
files_to_remove.append(os.path.join(root, fname))
LOG.debug("Found %d orphaned files", len(files_to_remove))
sys.stdout.write(40 * " " + "\r")
sys.stdout.flush()
if self.dry_run:
print("Following files are not associated to any items in the DB:")
for filename in sorted(files_to_remove):
print(filename)
self.sess.rollback()
else:
_remove_files(image_path, files_to_remove)
self.sess.commit()
def _purge_orphaned_objects(self, sa_class, msg):
"""Return tuple of lists of images that are orphaned"""
ids_to_remove = []
for count, item in enumerate(self.sess.query(sa_class).all()):
sys.stdout.write(msg + "| / - \\".split()[count % 4] + "\r")
if not item.file:
self.sess.delete(item)
ids_to_remove.append(item.id)
del item
sys.stdout.flush()
LOG.debug("Found %d orphaned object of class %s",
len(ids_to_remove), sa_class.__name__)
self.sess.flush()
def _remove_files(image_path, filenames):
"""Remove files and empty directories in provided location"""
count = 0
for count, fname in enumerate(filenames, start=1):
os.unlink(fname)
LOG.info("Removed %d orphaned files", count)
count = 0
for root, dirs, _ in os.walk(image_path):
for dirname in dirs:
try:
os.rmdir(os.path.join(root, dirname))
count += 1
except OSError:
pass
LOG.info("Removed %d empty directories", count)
def _get_highest_size_length(item_dict):
highest = len(str(sorted([i[1] for i in item_dict.values()])[-1]))
@@ -417,7 +310,6 @@ def add_dir(args):
def create_db(args):
"""List"""
__import__('pdb').set_trace()
obj = Iface(args.db, args.pretend, args.debug)
obj.create(args.dir_to_add, args.imagedir)
obj.close()
@@ -431,14 +323,6 @@ def search(args):
obj.close()
@asserdb
def cleanup(args):
"""Cleanup"""
obj = Iface(args.db, False, args.debug)
obj.fsck()
obj.close()
def main():
"""Main"""
parser = argparse.ArgumentParser()
@@ -469,12 +353,6 @@ def main():
create = subparser.add_parser('create')
create.add_argument('db')
create.add_argument('dir_to_add')
create.add_argument('-i', '--imagedir', help="Directory where to put "
"images for the database. Popular, but deprecated "
"choice is `~/.pycatalog/images'. Current default "
"is special string `:same_as_db:' which will try to "
"create directory with the same name as the db with "
"data suffix", default=':same_as_db:')
create.add_argument('-p', '--pretend', help="Don't do the action, just "
"give the info what would gonna to happen.",
action='store_true', default=False)
@@ -499,15 +377,6 @@ def main():
action='store_true', default=False)
find.set_defaults(func=search)
fsck = subparser.add_parser('fsck')
fsck.add_argument('db')
fsck.add_argument('-p', '--pretend', help="Don't do the action, just give"
" the info what would gonna to happen.",
action='store_true', default=False)
fsck.add_argument('-d', '--debug', help='Turn on debug',
action='store_true', default=False)
fsck.set_defaults(func=cleanup)
args = parser.parse_args()
if 'func' in args:

109
pycatalog/dbobjects.py
View File

@@ -13,9 +13,7 @@ from sqlalchemy import DateTime, ForeignKey, Sequence
from sqlalchemy.orm import relation, backref
from pycatalog.dbcommon import Base
from pycatalog.thumbnail import ThumbCreator
from pycatalog.logger import get_logger
from pycatalog.misc import mk_paths
LOG = get_logger(__name__)
@@ -49,8 +47,6 @@ class File(Base):
backref=backref('parent', remote_side="File.id"),
order_by=[type, filename])
tags = relation("Tag", secondary=tags_files, order_by="Tag.tag")
thumbnail = relation("Thumbnail", backref="file")
images = relation("Image", backref="file", order_by="Image.filename")
def __init__(self, filename=None, path=None, date=None, size=None,
ftype=None, src=None):
@@ -118,111 +114,6 @@ class Tag(Base):
return "<Tag('%s', %s)>" % (str(self.tag), str(self.id))
class Thumbnail(Base):
"""Thumbnail for the file"""
__tablename__ = "thumbnails"
id = Column(Integer, Sequence("thumbnail_id_seq"), primary_key=True)
file_id = Column(Integer, ForeignKey("files.id"), index=True)
filename = Column(Text)
def __init__(self, filename=None, img_path=None, file_obj=None):
self.filename = filename
self.file = file_obj
self.img_path = img_path
if filename and file_obj and img_path:
self.save(self.filename, img_path)
def save(self, fname, img_path):
"""
Create file related thumbnail, add it to the file object.
"""
new_name = mk_paths(fname, img_path)
ext = os.path.splitext(self.filename)[1]
if ext:
new_name.append("".join([new_name.pop(), ext]))
thumb = ThumbCreator(self.filename).generate()
name, ext = os.path.splitext(new_name.pop())
new_name.append("".join([name, "_t", ext]))
self.filename = os.path.sep.join(new_name)
if not os.path.exists(os.path.join(img_path, *new_name)):
shutil.move(thumb, os.path.join(img_path, *new_name))
else:
LOG.info("Thumbnail already exists (%s: %s)",
fname, "/".join(new_name))
os.unlink(thumb)
def __repr__(self):
return "<Thumbnail('%s', %s)>" % (str(self.filename), str(self.id))
class Image(Base):
"""Images and their thumbnails"""
__tablename__ = "images"
id = Column(Integer, Sequence("images_id_seq"), primary_key=True)
file_id = Column(Integer, ForeignKey("files.id"), index=True)
filename = Column(Text)
def __init__(self, filename=None, img_path=None, file_obj=None, move=True):
self.filename = None
self.file = file_obj
self.img_path = img_path
if filename and img_path:
self.filename = filename
self.save(filename, img_path, move)
def save(self, fname, img_path, move=True):
"""
Save and create coressponding thumbnail (note: it differs from file
related thumbnail!)
"""
new_name = mk_paths(fname, img_path)
ext = os.path.splitext(self.filename)[1]
if ext:
new_name.append("".join([new_name.pop(), ext]))
if not os.path.exists(os.path.join(img_path, *new_name)):
if move:
shutil.move(self.filename, os.path.join(img_path, *new_name))
else:
shutil.copy(self.filename, os.path.join(img_path, *new_name))
else:
LOG.warning("Image with same CRC already exists "
"('%s', '%s')" % (self.filename, "/".join(new_name)))
self.filename = os.path.sep.join(new_name)
name, ext = os.path.splitext(new_name.pop())
new_name.append("".join([name, "_t", ext]))
if not os.path.exists(os.path.join(img_path, *new_name)):
thumb = ThumbCreator(os.path.join(img_path, self.filename))
shutil.move(thumb.generate(), os.path.join(img_path, *new_name))
else:
LOG.info("Thumbnail already generated %s" % "/".join(new_name))
def get_copy(self):
"""
Create the very same object as self with exception of id field
"""
img = Image()
img.filename = self.filename
return img
@property
def thumbnail(self):
"""
Return path to thumbnail for this image
"""
path, fname = os.path.split(self.filename)
base, ext = os.path.splitext(fname)
return os.path.join(path, base + "_t" + ext)
def __repr__(self):
return "<Image('%s', %s)>" % (str(self.filename), str(self.id))
class Exif(Base):
"""Selected EXIF information"""
__tablename__ = "exif"

54
pycatalog/misc.py
View File

@@ -5,11 +5,6 @@
Author: Roman 'gryf' Dobosz, gryf73@gmail.com
Created: 2009-04-05
"""
import os
import errno
from zlib import crc32
import pycatalog.dbcommon
from pycatalog.logger import get_logger
LOG = get_logger(__name__)
@@ -27,51 +22,4 @@ def float_to_string(float_length):
minutes = int(float_length / 60)
float_length -= minutes * 60
sec = int(float_length)
return "%02d:%02d:%02d" % (hour, minutes, sec)
def calculate_image_path(dbpath=None, create=False):
"""Calculate image path out of provided path or using current connection"""
if not dbpath:
dbpath = pycatalog.dbcommon.DbFilename
if dbpath == ":memory:":
raise OSError("Cannot create image path out of in-memory db!")
dir_, file_ = (os.path.dirname(dbpath), os.path.basename(dbpath))
file_base, dummy = os.path.splitext(file_)
images_dir = os.path.join(dir_, file_base + "_images")
else:
if dbpath and "~" in dbpath:
dbpath = os.path.expanduser(dbpath)
if dbpath and "$" in dbpath:
dbpath = os.path.expandvars(dbpath)
images_dir = dbpath
if create:
if not os.path.exists(images_dir):
try:
os.mkdir(images_dir)
except OSError as err:
if err.errno != errno.EEXIST:
raise
elif not os.path.exists(images_dir):
raise OSError("%s: No such directory" % images_dir)
return os.path.abspath(images_dir)
def mk_paths(fname, img_path):
"""Make path for provided pathname by calculating crc32 out of file"""
with open(fname, 'r+b') as fobj:
new_path = "%x" % (crc32(fobj.read(10*1024*1024)) & 0xffffffff)
new_path = [new_path[i:i + 2] for i in range(0, len(new_path), 2)]
full_path = os.path.join(img_path, *new_path[:-1])
try:
os.makedirs(full_path)
except OSError as exc:
if exc.errno != errno.EEXIST:
LOG.debug("Directory %s already exists." % full_path)
return new_path
return f"{hour:02}:{minutes:02}:{sec:02}"

61
pycatalog/scan.py
View File

@@ -11,7 +11,7 @@ from datetime import datetime
import mimetypes
import pycatalog.misc
from pycatalog.dbobjects import File, Image, Thumbnail, Config, TYPE
from pycatalog.dbobjects import File, Config, TYPE
from pycatalog.dbcommon import Session
from pycatalog.logger import get_logger
from pycatalog.video import Video
@@ -49,8 +49,6 @@ class Scan(object):
self.files_count = self._get_files_count()
self.current_count = 0
self._set_image_path()
def add_files(self, engine=None):
"""
Returns list, which contain object, modification date and file
@@ -225,55 +223,9 @@ class Scan(object):
"""
Make captures for a movie. Save it under uniq name.
"""
result = RE_FN_START.match(fobj.filename)
if result:
self._check_related(fobj, result.groupdict()['fname_start'])
vid = Video(filepath)
fobj.description = vid.get_formatted_tags()
preview_fn = vid.capture()
if preview_fn:
Image(preview_fn, self.img_path, fobj)
def _check_related(self, fobj, filename_start):
"""
Try to search for related files which belongs to specified File
object and pattern. If found, additional File objects are created.
For example, if we have movie file named like:
[aXXo] Batman (1989) [D3ADBEEF].avi
[aXXo] Batman (1989) trailer [B00B1337].avi
Batman (1989) [D3ADBEEF].avi
Batman [D3ADBEEF].avi
And for example file '[aXXo] Batman (1989) [D3ADBEEF].avi' might have
some other accompanied files, like:
[aXXo] Batman (1989) [D3ADBEEF].avi.conf
[aXXo] Batman (1989) [DEADC0DE].nfo
[aXXo] Batman (1989) cover [BEEFD00D].jpg
[aXXo] Batman (1989) poster [FEEDD00D].jpg
Which can be atuomatically asociated with the movie.
This method find such files, and for some of them (currently images)
will perform extra actions - like creating corresponding Image objects.
"""
for fname in os.listdir(fobj.filepath):
extension = os.path.splitext(fname)[1]
if fname.startswith(filename_start) and \
extension in ('.jpg', '.gif', '.png'):
full_fname = os.path.join(fobj.filepath, fname)
LOG.debug('found corresponding image file: %s', full_fname)
Image(full_fname, self.img_path, fobj, False)
if not fobj.thumbnail:
Thumbnail(full_fname, self.img_path, fobj)
def _get_all_files(self):
"""Gather all File objects"""
self._existing_files = self._session.query(File).all()
@@ -471,17 +423,6 @@ class Scan(object):
LOG.debug("count of files: %s", count)
return count
def _set_image_path(self):
"""Get or calculate the images path"""
image_path = (self._session.query(Config)
.filter(Config.key == "image_path")).one()
if image_path.value == ":same_as_db:":
image_path = pycatalog.misc.calculate_image_path()
else:
image_path = pycatalog.misc.calculate_image_path(image_path.value)
self.img_path = image_path
def _get_dirsize(path):
"""

114
pycatalog/thumbnail.py
View File

@@ -1,114 +0,0 @@
"""
Project: pyGTKtalog
Description: Create thumbnail for sepcified image
Type: lib
Author: Roman 'gryf' Dobosz, gryf73@gmail.com
Created: 2011-05-15
"""
import os
from tempfile import mkstemp
import shutil
from PIL import Image
import exifread
from pycatalog.logger import get_logger
LOG = get_logger(__name__)
class ThumbCreator(object):
"""
Class for generate/extract thumbnail from image file
"""
def __init__(self, filename):
self.thumb_x = 160
self.thumb_y = 160
self.filename = filename
def generate(self):
"""
Save thumbnail into temporary file
"""
exif = {}
orientations = {2: Image.FLIP_LEFT_RIGHT, # Mirrored horizontal
3: Image.ROTATE_180, # Rotated 180
4: Image.FLIP_TOP_BOTTOM, # Mirrored vertical
5: Image.ROTATE_90, # Mirrored horizontal then
# rotated 90 CCW
6: Image.ROTATE_270, # Rotated 90 CW
7: Image.ROTATE_270, # Mirrored horizontal then
# rotated 90 CW
8: Image.ROTATE_90} # Rotated 90 CCW
flips = {7: Image.FLIP_LEFT_RIGHT, 5: Image.FLIP_LEFT_RIGHT}
exif = self._get_exif()
file_desc, thumb_fn = mkstemp(suffix=".jpg")
os.close(file_desc)
if exif and 'JPEGThumbnail' in exif and exif['JPEGThumbnail']:
LOG.debug("exif thumb for filename %s" % self.filename)
exif_thumbnail = exif['JPEGThumbnail']
thumb = open(thumb_fn, 'wb')
thumb.write(exif_thumbnail)
thumb.close()
else:
LOG.debug("no exif thumb")
if self.is_image_smaller():
shutil.copyfile(self.filename, thumb_fn)
else:
thumb = self._scale_image()
if thumb:
thumb.save(thumb_fn, "JPEG")
if exif and 'Image Orientation' in exif:
orient = exif['Image Orientation'].values[0]
if orient > 1 and orient in orientations:
thumb_image = Image.open(thumb_fn)
tmp_thumb_img = thumb_image.transpose(orientations[orient])
if orient in flips:
tmp_thumb_img = tmp_thumb_img.transpose(flips[orient])
tmp_thumb_img.save(thumb_fn, 'JPEG')
return thumb_fn
def is_image_smaller(self):
"""Check if image is smaller than desired dimention, return boolean"""
image = Image.open(self.filename)
im_x, im_y = image.size
image.close()
return im_x <= self.thumb_x and im_y <= self.thumb_y
def _get_exif(self):
"""
Get exif (if available), return as a dict
"""
image_file = open(self.filename, 'rb')
try:
exif = exifread.process_file(image_file)
except Exception:
exif = {}
LOG.info("Exif crashed on '%s'." % self.filename)
finally:
image_file.close()
return exif
def _scale_image(self):
"""
Create thumbnail. returns image object or None
"""
try:
image_thumb = Image.open(self.filename).convert('RGB')
except Exception:
return None
it_x, it_y = image_thumb.size
if it_x > self.thumb_x or it_y > self.thumb_y:
image_thumb.thumbnail((self.thumb_x, self.thumb_y),
Image.ANTIALIAS)
return image_thumb