diff --git a/pycatalog/__init__.py b/pycatalog/__init__.py index 388908f..46a3631 100644 --- a/pycatalog/__init__.py +++ b/pycatalog/__init__.py @@ -195,7 +195,7 @@ class Iface(object): # scanob.update_files(node.id) scanob.update_files(node.id, self.engine) - def create(self, dir_to_add, data_dir): + def create(self, dir_to_add): """Create new database""" self.root = dbo.File() self.root.id = 1 @@ -205,9 +205,6 @@ class Iface(object): self.root.type = 0 self.root.parent_id = 1 - config = dbo.Config() - config.key = 'image_path' - config.value = data_dir if not self.dry_run: self.sess.add(self.root) @@ -217,11 +214,6 @@ class Iface(object): print(colorize("Creating new db against directory `%s'" % dir_to_add, 'white')) if not self.dry_run: - if data_dir == ':same_as_db:': - misc.calculate_image_path(None, True) - else: - misc.calculate_image_path(data_dir, True) - scanob = scan.Scan(dir_to_add) scanob.add_files(self.engine) @@ -286,105 +278,6 @@ class Iface(object): for item in result: print(self._annotate(item, search_words)) - def fsck(self): - """Fsck orphaned images/thumbs""" - image_path = (self.sess.query(dbo.Config) - .filter(dbo.Config.key == 'image_path')).one().value - - if image_path == ':same_as_db:': - image_path = misc.calculate_image_path(None, False) - - files_to_remove = [] - - # remove images/thumbnails which doesn't have file relation - for name, obj in (("images", dbo.Image), - ("thumbnails", dbo.Thumbnail)): - self._purge_orphaned_objects(obj, "Scanning %s " % name) - - # find all image files not associate with either Image (image/thumb) - # or Thumbnail (thumb) objects - sys.stdout.write(40 * " " + "\r") - count = 0 - for root, dirs, files in os.walk(image_path): - for fname in files: - sys.stdout.write("Scanning files " + - "| / - \\".split()[count % 4] + "\r") - sys.stdout.flush() - count += 1 - - fname_ = os.path.join(root.split(image_path)[1], - fname).lstrip('/') - - if '_t' in fname: - obj = (self.sess.query(dbo.Thumbnail) - .filter(dbo.Thumbnail.filename == fname_)).all() - if obj: - continue - - obj = (self.sess.query(dbo.Image) - .filter(dbo.Image.filename == - fname_.replace('_t.', '.'))).all() - if obj: - continue - - else: - obj = (self.sess.query(dbo.Image) - .filter(dbo.Image.filename == fname_)).all() - if obj: - continue - - files_to_remove.append(os.path.join(root, fname)) - - LOG.debug("Found %d orphaned files", len(files_to_remove)) - sys.stdout.write(40 * " " + "\r") - sys.stdout.flush() - - if self.dry_run: - print("Following files are not associated to any items in the DB:") - for filename in sorted(files_to_remove): - print(filename) - self.sess.rollback() - else: - _remove_files(image_path, files_to_remove) - self.sess.commit() - - def _purge_orphaned_objects(self, sa_class, msg): - """Return tuple of lists of images that are orphaned""" - - ids_to_remove = [] - - for count, item in enumerate(self.sess.query(sa_class).all()): - sys.stdout.write(msg + "| / - \\".split()[count % 4] + "\r") - if not item.file: - self.sess.delete(item) - ids_to_remove.append(item.id) - del item - sys.stdout.flush() - - LOG.debug("Found %d orphaned object of class %s", - len(ids_to_remove), sa_class.__name__) - self.sess.flush() - - -def _remove_files(image_path, filenames): - """Remove files and empty directories in provided location""" - - count = 0 - for count, fname in enumerate(filenames, start=1): - os.unlink(fname) - - LOG.info("Removed %d orphaned files", count) - - count = 0 - for root, dirs, _ in os.walk(image_path): - for dirname in dirs: - try: - os.rmdir(os.path.join(root, dirname)) - count += 1 - except OSError: - pass - LOG.info("Removed %d empty directories", count) - def _get_highest_size_length(item_dict): highest = len(str(sorted([i[1] for i in item_dict.values()])[-1])) @@ -417,7 +310,6 @@ def add_dir(args): def create_db(args): """List""" - __import__('pdb').set_trace() obj = Iface(args.db, args.pretend, args.debug) obj.create(args.dir_to_add, args.imagedir) obj.close() @@ -431,14 +323,6 @@ def search(args): obj.close() -@asserdb -def cleanup(args): - """Cleanup""" - obj = Iface(args.db, False, args.debug) - obj.fsck() - obj.close() - - def main(): """Main""" parser = argparse.ArgumentParser() @@ -469,12 +353,6 @@ def main(): create = subparser.add_parser('create') create.add_argument('db') create.add_argument('dir_to_add') - create.add_argument('-i', '--imagedir', help="Directory where to put " - "images for the database. Popular, but deprecated " - "choice is `~/.pycatalog/images'. Current default " - "is special string `:same_as_db:' which will try to " - "create directory with the same name as the db with " - "data suffix", default=':same_as_db:') create.add_argument('-p', '--pretend', help="Don't do the action, just " "give the info what would gonna to happen.", action='store_true', default=False) @@ -499,15 +377,6 @@ def main(): action='store_true', default=False) find.set_defaults(func=search) - fsck = subparser.add_parser('fsck') - fsck.add_argument('db') - fsck.add_argument('-p', '--pretend', help="Don't do the action, just give" - " the info what would gonna to happen.", - action='store_true', default=False) - fsck.add_argument('-d', '--debug', help='Turn on debug', - action='store_true', default=False) - fsck.set_defaults(func=cleanup) - args = parser.parse_args() if 'func' in args: diff --git a/pycatalog/dbobjects.py b/pycatalog/dbobjects.py index 8cac4c1..0675344 100644 --- a/pycatalog/dbobjects.py +++ b/pycatalog/dbobjects.py @@ -13,9 +13,7 @@ from sqlalchemy import DateTime, ForeignKey, Sequence from sqlalchemy.orm import relation, backref from pycatalog.dbcommon import Base -from pycatalog.thumbnail import ThumbCreator from pycatalog.logger import get_logger -from pycatalog.misc import mk_paths LOG = get_logger(__name__) @@ -49,8 +47,6 @@ class File(Base): backref=backref('parent', remote_side="File.id"), order_by=[type, filename]) tags = relation("Tag", secondary=tags_files, order_by="Tag.tag") - thumbnail = relation("Thumbnail", backref="file") - images = relation("Image", backref="file", order_by="Image.filename") def __init__(self, filename=None, path=None, date=None, size=None, ftype=None, src=None): @@ -118,111 +114,6 @@ class Tag(Base): return "" % (str(self.tag), str(self.id)) -class Thumbnail(Base): - """Thumbnail for the file""" - __tablename__ = "thumbnails" - id = Column(Integer, Sequence("thumbnail_id_seq"), primary_key=True) - file_id = Column(Integer, ForeignKey("files.id"), index=True) - filename = Column(Text) - - def __init__(self, filename=None, img_path=None, file_obj=None): - self.filename = filename - self.file = file_obj - self.img_path = img_path - if filename and file_obj and img_path: - self.save(self.filename, img_path) - - def save(self, fname, img_path): - """ - Create file related thumbnail, add it to the file object. - """ - new_name = mk_paths(fname, img_path) - ext = os.path.splitext(self.filename)[1] - if ext: - new_name.append("".join([new_name.pop(), ext])) - - thumb = ThumbCreator(self.filename).generate() - name, ext = os.path.splitext(new_name.pop()) - new_name.append("".join([name, "_t", ext])) - self.filename = os.path.sep.join(new_name) - if not os.path.exists(os.path.join(img_path, *new_name)): - shutil.move(thumb, os.path.join(img_path, *new_name)) - else: - LOG.info("Thumbnail already exists (%s: %s)", - fname, "/".join(new_name)) - os.unlink(thumb) - - def __repr__(self): - return "" % (str(self.filename), str(self.id)) - - -class Image(Base): - """Images and their thumbnails""" - __tablename__ = "images" - id = Column(Integer, Sequence("images_id_seq"), primary_key=True) - file_id = Column(Integer, ForeignKey("files.id"), index=True) - filename = Column(Text) - - def __init__(self, filename=None, img_path=None, file_obj=None, move=True): - self.filename = None - self.file = file_obj - self.img_path = img_path - if filename and img_path: - self.filename = filename - self.save(filename, img_path, move) - - def save(self, fname, img_path, move=True): - """ - Save and create coressponding thumbnail (note: it differs from file - related thumbnail!) - """ - new_name = mk_paths(fname, img_path) - ext = os.path.splitext(self.filename)[1] - - if ext: - new_name.append("".join([new_name.pop(), ext])) - - if not os.path.exists(os.path.join(img_path, *new_name)): - if move: - shutil.move(self.filename, os.path.join(img_path, *new_name)) - else: - shutil.copy(self.filename, os.path.join(img_path, *new_name)) - else: - LOG.warning("Image with same CRC already exists " - "('%s', '%s')" % (self.filename, "/".join(new_name))) - - self.filename = os.path.sep.join(new_name) - - name, ext = os.path.splitext(new_name.pop()) - new_name.append("".join([name, "_t", ext])) - - if not os.path.exists(os.path.join(img_path, *new_name)): - thumb = ThumbCreator(os.path.join(img_path, self.filename)) - shutil.move(thumb.generate(), os.path.join(img_path, *new_name)) - else: - LOG.info("Thumbnail already generated %s" % "/".join(new_name)) - - def get_copy(self): - """ - Create the very same object as self with exception of id field - """ - img = Image() - img.filename = self.filename - return img - - @property - def thumbnail(self): - """ - Return path to thumbnail for this image - """ - path, fname = os.path.split(self.filename) - base, ext = os.path.splitext(fname) - return os.path.join(path, base + "_t" + ext) - - def __repr__(self): - return "" % (str(self.filename), str(self.id)) - - class Exif(Base): """Selected EXIF information""" __tablename__ = "exif" diff --git a/pycatalog/misc.py b/pycatalog/misc.py index 3bd3ac3..0b4491e 100644 --- a/pycatalog/misc.py +++ b/pycatalog/misc.py @@ -5,11 +5,6 @@ Author: Roman 'gryf' Dobosz, gryf73@gmail.com Created: 2009-04-05 """ -import os -import errno -from zlib import crc32 - -import pycatalog.dbcommon from pycatalog.logger import get_logger LOG = get_logger(__name__) @@ -27,51 +22,4 @@ def float_to_string(float_length): minutes = int(float_length / 60) float_length -= minutes * 60 sec = int(float_length) - return "%02d:%02d:%02d" % (hour, minutes, sec) - - -def calculate_image_path(dbpath=None, create=False): - """Calculate image path out of provided path or using current connection""" - if not dbpath: - dbpath = pycatalog.dbcommon.DbFilename - if dbpath == ":memory:": - raise OSError("Cannot create image path out of in-memory db!") - - dir_, file_ = (os.path.dirname(dbpath), os.path.basename(dbpath)) - file_base, dummy = os.path.splitext(file_) - images_dir = os.path.join(dir_, file_base + "_images") - else: - if dbpath and "~" in dbpath: - dbpath = os.path.expanduser(dbpath) - if dbpath and "$" in dbpath: - dbpath = os.path.expandvars(dbpath) - images_dir = dbpath - - if create: - if not os.path.exists(images_dir): - try: - os.mkdir(images_dir) - except OSError as err: - if err.errno != errno.EEXIST: - raise - elif not os.path.exists(images_dir): - raise OSError("%s: No such directory" % images_dir) - - return os.path.abspath(images_dir) - - -def mk_paths(fname, img_path): - """Make path for provided pathname by calculating crc32 out of file""" - with open(fname, 'r+b') as fobj: - new_path = "%x" % (crc32(fobj.read(10*1024*1024)) & 0xffffffff) - - new_path = [new_path[i:i + 2] for i in range(0, len(new_path), 2)] - full_path = os.path.join(img_path, *new_path[:-1]) - - try: - os.makedirs(full_path) - except OSError as exc: - if exc.errno != errno.EEXIST: - LOG.debug("Directory %s already exists." % full_path) - - return new_path + return f"{hour:02}:{minutes:02}:{sec:02}" diff --git a/pycatalog/scan.py b/pycatalog/scan.py index 94224b1..90bc4f5 100644 --- a/pycatalog/scan.py +++ b/pycatalog/scan.py @@ -11,7 +11,7 @@ from datetime import datetime import mimetypes import pycatalog.misc -from pycatalog.dbobjects import File, Image, Thumbnail, Config, TYPE +from pycatalog.dbobjects import File, Config, TYPE from pycatalog.dbcommon import Session from pycatalog.logger import get_logger from pycatalog.video import Video @@ -49,8 +49,6 @@ class Scan(object): self.files_count = self._get_files_count() self.current_count = 0 - self._set_image_path() - def add_files(self, engine=None): """ Returns list, which contain object, modification date and file @@ -225,55 +223,9 @@ class Scan(object): """ Make captures for a movie. Save it under uniq name. """ - result = RE_FN_START.match(fobj.filename) - if result: - self._check_related(fobj, result.groupdict()['fname_start']) - vid = Video(filepath) - fobj.description = vid.get_formatted_tags() - preview_fn = vid.capture() - if preview_fn: - Image(preview_fn, self.img_path, fobj) - - def _check_related(self, fobj, filename_start): - """ - Try to search for related files which belongs to specified File - object and pattern. If found, additional File objects are created. - - For example, if we have movie file named like: - [aXXo] Batman (1989) [D3ADBEEF].avi - [aXXo] Batman (1989) trailer [B00B1337].avi - Batman (1989) [D3ADBEEF].avi - Batman [D3ADBEEF].avi - - And for example file '[aXXo] Batman (1989) [D3ADBEEF].avi' might have - some other accompanied files, like: - - [aXXo] Batman (1989) [D3ADBEEF].avi.conf - [aXXo] Batman (1989) [DEADC0DE].nfo - [aXXo] Batman (1989) cover [BEEFD00D].jpg - [aXXo] Batman (1989) poster [FEEDD00D].jpg - - Which can be atuomatically asociated with the movie. - - This method find such files, and for some of them (currently images) - will perform extra actions - like creating corresponding Image objects. - - """ - for fname in os.listdir(fobj.filepath): - extension = os.path.splitext(fname)[1] - if fname.startswith(filename_start) and \ - extension in ('.jpg', '.gif', '.png'): - full_fname = os.path.join(fobj.filepath, fname) - LOG.debug('found corresponding image file: %s', full_fname) - - Image(full_fname, self.img_path, fobj, False) - - if not fobj.thumbnail: - Thumbnail(full_fname, self.img_path, fobj) - def _get_all_files(self): """Gather all File objects""" self._existing_files = self._session.query(File).all() @@ -471,17 +423,6 @@ class Scan(object): LOG.debug("count of files: %s", count) return count - def _set_image_path(self): - """Get or calculate the images path""" - image_path = (self._session.query(Config) - .filter(Config.key == "image_path")).one() - if image_path.value == ":same_as_db:": - image_path = pycatalog.misc.calculate_image_path() - else: - image_path = pycatalog.misc.calculate_image_path(image_path.value) - - self.img_path = image_path - def _get_dirsize(path): """ diff --git a/pycatalog/thumbnail.py b/pycatalog/thumbnail.py deleted file mode 100644 index 87cce00..0000000 --- a/pycatalog/thumbnail.py +++ /dev/null @@ -1,114 +0,0 @@ -""" - Project: pyGTKtalog - Description: Create thumbnail for sepcified image - Type: lib - Author: Roman 'gryf' Dobosz, gryf73@gmail.com - Created: 2011-05-15 -""" - -import os -from tempfile import mkstemp -import shutil - -from PIL import Image -import exifread - -from pycatalog.logger import get_logger - - -LOG = get_logger(__name__) - - -class ThumbCreator(object): - """ - Class for generate/extract thumbnail from image file - """ - - def __init__(self, filename): - self.thumb_x = 160 - self.thumb_y = 160 - self.filename = filename - - def generate(self): - """ - Save thumbnail into temporary file - """ - exif = {} - orientations = {2: Image.FLIP_LEFT_RIGHT, # Mirrored horizontal - 3: Image.ROTATE_180, # Rotated 180 - 4: Image.FLIP_TOP_BOTTOM, # Mirrored vertical - 5: Image.ROTATE_90, # Mirrored horizontal then - # rotated 90 CCW - 6: Image.ROTATE_270, # Rotated 90 CW - 7: Image.ROTATE_270, # Mirrored horizontal then - # rotated 90 CW - 8: Image.ROTATE_90} # Rotated 90 CCW - flips = {7: Image.FLIP_LEFT_RIGHT, 5: Image.FLIP_LEFT_RIGHT} - - exif = self._get_exif() - file_desc, thumb_fn = mkstemp(suffix=".jpg") - os.close(file_desc) - - if exif and 'JPEGThumbnail' in exif and exif['JPEGThumbnail']: - LOG.debug("exif thumb for filename %s" % self.filename) - exif_thumbnail = exif['JPEGThumbnail'] - thumb = open(thumb_fn, 'wb') - thumb.write(exif_thumbnail) - thumb.close() - else: - LOG.debug("no exif thumb") - if self.is_image_smaller(): - shutil.copyfile(self.filename, thumb_fn) - else: - thumb = self._scale_image() - if thumb: - thumb.save(thumb_fn, "JPEG") - - if exif and 'Image Orientation' in exif: - orient = exif['Image Orientation'].values[0] - if orient > 1 and orient in orientations: - thumb_image = Image.open(thumb_fn) - tmp_thumb_img = thumb_image.transpose(orientations[orient]) - - if orient in flips: - tmp_thumb_img = tmp_thumb_img.transpose(flips[orient]) - - tmp_thumb_img.save(thumb_fn, 'JPEG') - - return thumb_fn - - def is_image_smaller(self): - """Check if image is smaller than desired dimention, return boolean""" - image = Image.open(self.filename) - im_x, im_y = image.size - image.close() - return im_x <= self.thumb_x and im_y <= self.thumb_y - - def _get_exif(self): - """ - Get exif (if available), return as a dict - """ - image_file = open(self.filename, 'rb') - try: - exif = exifread.process_file(image_file) - except Exception: - exif = {} - LOG.info("Exif crashed on '%s'." % self.filename) - finally: - image_file.close() - - return exif - - def _scale_image(self): - """ - Create thumbnail. returns image object or None - """ - try: - image_thumb = Image.open(self.filename).convert('RGB') - except Exception: - return None - it_x, it_y = image_thumb.size - if it_x > self.thumb_x or it_y > self.thumb_y: - image_thumb.thumbnail((self.thumb_x, self.thumb_y), - Image.ANTIALIAS) - return image_thumb