diff --git a/cmdcatalog.py b/cmdcatalog.py index 9442071..01d3be8 100755 --- a/cmdcatalog.py +++ b/cmdcatalog.py @@ -1,20 +1,26 @@ #!/usr/bin/env python +""" +Fast and ugly CLI interface for pyGTKtalog +""" import os import sys +import errno from argparse import ArgumentParser from pygtktalog import scan -from pygtktalog.dbobjects import File +from pygtktalog import misc +from pygtktalog.dbobjects import File, Config from pygtktalog.dbcommon import connect, Session -BLACK, RED, GREEN, YELLOW, BLUE, MAGENTA, CYAN, WHITE = range(8) +BLACK, RED, GREEN, YELLOW, BLUE, MAGENTA, CYAN, WHITE = range(30, 38) RESET_SEQ = "\033[0m" COLOR_SEQ = "\033[1;%dm" BOLD_SEQ = "\033[1m" -def cprint(txt, color): +def colorize(txt, color): + """Pretty print with colors to console.""" color_map = {"black": BLACK, "red": RED, "green": GREEN, @@ -23,27 +29,23 @@ def cprint(txt, color): "magenta": MAGENTA, "cyan": CYAN, "white": WHITE} - print COLOR_SEQ % (30 + color_map[color]) + txt + RESET_SEQ + return COLOR_SEQ % color_map[color] + txt + RESET_SEQ class Iface(object): + """Main class which interacts with the pyGTKtalog modules""" def __init__(self, dbname, pretend=False, debug=False): + """Init""" self.engine = connect(dbname) self.sess = Session() self.dry_run = pretend self.root = None + self._dbname = dbname if debug: scan.LOG.setLevel("DEBUG") - def close(self): - self.sess.commit() - self.sess.close() - - # def create(self): - # self.sess.commit() - # self.sess.close() - def _resolve_path(self, path): + """Identify path in the DB""" if not path.startswith("/"): raise AttributeError("Path have to start with slash (/)") @@ -62,18 +64,51 @@ class Iface(object): return last_node def _make_path(self, node): + """Make the path to the item in the DB""" if node.parent == node: return "/" + ext = "" + if node.parent.type == 0: + ext = colorize(" (%s)" % node.filepath, "white") + path = [] path.append(node.filename) while node.parent != self.root: path.append(node.parent.filename) node = node.parent - return "/".join([""] + path[::-1]) + return "/".join([""] + path[::-1]) + ext - def list(self, path=None): + def _walk(self, dirnode): + """Recursively go through the leaves of the node""" + items = [] + for node in dirnode.children: + if node.type == 1: + items += self._walk(node) + + items.append(" " + self._make_path(node)) + + items.sort() + return items + + def _list(self, node): + """List only current node content""" + items = [] + for node in node.children: + if node != self.root: + items.append(" " + self._make_path(node)) + + items.sort() + return items + + def close(self): + """Close the session""" + self.sess.commit() + self.sess.close() + + def list(self, path=None, recursive=False): + """Simulate ls command for the provided item path""" self.root = self.sess.query(File).filter(File.type==0).first() if path: node = self._resolve_path(path) @@ -82,20 +117,25 @@ class Iface(object): node = self.root msg = "Content of path `/':" - cprint(msg, "white") - for node in node.children: - if node != self.root: - #if __debug__: - # print " %d:" % node.id, self._make_path(node) - #else: - print " ", self._make_path(node) + print colorize(msg, "white") + + if recursive: + items = self._walk(node) + else: + items = self._list(node) + + print "\n".join(items) def update(self, path, dir_to_update=None): + """ + Update the DB against provided path and optionally directory on the + real filesystem + """ self.root = self.sess.query(File).filter(File.type==0).first() node = self._resolve_path(path) if node == self.root: - cprint("Cannot update entire db, since root was provided as path.", - "red") + print colorize("Cannot update entire db, since root was provided " + "as path.", "red") return if not dir_to_update: @@ -104,14 +144,15 @@ class Iface(object): if not os.path.exists(dir_to_update): raise OSError("Path to updtate doesn't exists: %s", dir_to_update) - cprint("Updating node `%s' against directory " - "`%s'" % (path, dir_to_update), "white") + print colorize("Updating node `%s' against directory " + "`%s'" % (path, dir_to_update), "white") if not self.dry_run: scanob = scan.Scan(dir_to_update) # scanob.update_files(node.id) scanob.update_files(node.id, self.engine) - def create(self, dir_to_add): + def create(self, dir_to_add, data_dir): + """Create new database""" self.root = File() self.root.id = 1 self.root.filename = 'root' @@ -119,51 +160,93 @@ class Iface(object): self.root.source = 0 self.root.type = 0 self.root.parent_id = 1 + + config = Config() + config.key = "image_path" + config.value = data_dir + if not self.dry_run: self.sess.add(self.root) + self.sess.add(config) self.sess.commit() - cprint("Creating new db against directory `%s'" % dir_to_add, "white") + print colorize("Creating new db against directory `%s'" % dir_to_add, + "white") if not self.dry_run: + if data_dir == ":same_as_db:": + misc.calculate_image_path(None, True) + else: + misc.calculate_image_path(data_dir, True) + scanob = scan.Scan(dir_to_add) scanob.add_files(self.engine) + def add(self, dir_to_add): + """Add new directory to the db""" + self.root = self.sess.query(File).filter(File.type==0).first() + + if not os.path.exists(dir_to_add): + raise OSError("Path to add doesn't exists: %s", dir_to_add) + + print colorize("Adding directory `%s'" % dir_to_add, "white") + if not self.dry_run: + scanob = scan.Scan(dir_to_add) + scanob.add_files() + def list_db(args): + """List""" if not os.path.exists(args.db): - cprint("File `%s' does not exists!" % args.db, "red") + print colorize("File `%s' does not exists!" % args.db, "red") sys.exit(1) obj = Iface(args.db, False, args.debug) - obj.list(path=args.path) + obj.list(path=args.path, recursive=args.recursive) obj.close() def update_db(args): + """Update""" if not os.path.exists(args.db): - cprint("File `%s' does not exists!" % args.db, "red") + print colorize("File `%s' does not exists!" % args.db, "red") sys.exit(1) obj = Iface(args.db, args.pretend, args.debug) obj.update(args.path, dir_to_update=args.dir_to_update) obj.close() -def create_db(args): - if os.path.exists(args.db): - cprint("File `%s' exists!" % args.db, "yellow") + +def add_dir(args): + """Add""" + if not os.path.exists(args.db): + print colorize("File `%s' does not exists!" % args.db, "red") + sys.exit(1) obj = Iface(args.db, args.pretend, args.debug) - obj.create(args.dir_to_add) + obj.add(args.dir_to_add) obj.close() -if __name__ == "__main__": +def create_db(args): + """List""" + if os.path.exists(args.db): + print colorize("File `%s' exists!" % args.db, "yellow") + + obj = Iface(args.db, args.pretend, args.debug) + obj.create(args.dir_to_add, args.imagedir) + obj.close() + + +def main(): + """Main""" parser = ArgumentParser() subparser = parser.add_subparsers() list_ = subparser.add_parser("list") list_.add_argument("db") list_.add_argument("path", nargs="?") + list_.add_argument("-r", "--recursive", help="list items in " + "subdirectories", action="store_true", default=False) list_.add_argument("-d", "--debug", help="Turn on debug", action="store_true", default=False) list_.set_defaults(func=list_db) @@ -182,6 +265,12 @@ if __name__ == "__main__": create = subparser.add_parser("create") create.add_argument("db") create.add_argument("dir_to_add") + create.add_argument("-i", "--imagedir", help="Directory where to put " + "images for the database. Popular, but deprecated " + "choice is `~/.pygtktalog/images'. Currnet default " + "is special string `:same_as_db:' which will try to " + "create directory with the same name as the db with " + "data suffix", default=":same_as_db:") create.add_argument("-p", "--pretend", help="Don't do the action, just " "give the info what would gonna to happen.", action="store_true", default=False) @@ -189,27 +278,18 @@ if __name__ == "__main__": action="store_true", default=False) create.set_defaults(func=create_db) + add = subparser.add_parser("add") + add.add_argument("db") + add.add_argument("dir_to_add") + add.add_argument("-p", "--pretend", help="Don't do the action, just " + "give the info what would gonna to happen.", + action="store_true", default=False) + add.add_argument("-d", "--debug", help="Turn on debug", + action="store_true", default=False) + add.set_defaults(func=add_dir) + args = parser.parse_args() args.func(args) - -""" -db_file = "/home/gryf/spisy/xxx.sqlite" -connect(db_file) -sess = Session() - -#if not sess.query(File).get(1): -# root = File() -# root.id = 1 -# root.filename = 'root' -# root.size = 0 -# root.source = 0 -# t.type = 0 -# root.parent_id = 1 -# sess.add(root) -# sess.commit() - -f = "/mnt/hardtwo/XXX/" -scanob = scan.Scan(f) -scanob.update_files(2) -""" +if __name__ == "__main__": + main() diff --git a/convert_1.x_to_2.x.py b/convert_1.x_to_2.x.py index c988757..db8248d 100755 --- a/convert_1.x_to_2.x.py +++ b/convert_1.x_to_2.x.py @@ -6,39 +6,177 @@ Author: Roman 'gryf' Dobosz, gryf73@gmail.com Created: 2009-08-14 """ -import sys -import os -import bz2 -import shutil -from tempfile import mkstemp -from sqlite3 import dbapi2 as sqlite from datetime import datetime +from sqlite3 import dbapi2 as sqlite +from sqlite3 import OperationalError +from tempfile import mkstemp +import bz2 +import errno +import os +import shutil +import sys -# import db objects just to create schema -from pygtktalog.dbobjects import File, Exif, Group, Gthumb -from pygtktalog.dbobjects import Image, Tag, Thumbnail -from pygtktalog.dbcommon import connect +from sqlalchemy.dialects.sqlite import DATETIME -def create_schema(cur): - pass +from pygtktalog.misc import mk_paths, calculate_image_path +PATH1 = os.path.expanduser("~/.pygtktalog/images") +PATH2 = os.path.expanduser("~/.pygtktalog/imgs2") + + +def mkdir_p(path): + """Make directories recurively, like 'mkdir -p' command""" + try: + os.makedirs(path) + except OSError as exc: + if exc.errno == errno.EEXIST and os.path.isdir(path): + pass + else: + raise + return path + +def get_images_path(cur): + """ + Calculate the data dir in order: + - config table + - old default path + - new default path + return first, which contain provided image filename + """ + + image = cur.execute("select filename from images limit 1").fetchone() + if image and image[0]: + image = image[0] + + try: + result = cur.execute("select value from config where " + "key='image_path'").fetchone() + if (result and result[0] and + os.path.exists(os.path.join(result[0].encode("utf-8"), + image.encode("utf-8")))): + return result[0] + except OperationalError: + # no such table like config. proceed. + pass + + for path in (PATH1, PATH2): + if os.path.exists(os.path.join(path, image)): + return path + return None + +def get_path(cur, image): + """ + Calculate the data dir in order: + - config table + - old default path + - new default path + return first, which contain provided image filename + """ + try: + result = cur.execute("select value from config where " + "key='image_path'").fetchone() + if (result and result[0] and + os.path.exists(os.path.join(result[0].encode("utf-8"), + image.encode("utf-8")))): + return result[0] + except OperationalError: + pass + + for path in (PATH1, PATH2): + if os.path.exists(os.path.join(path, image)): + return path + return None + +def old_style_image_handle(fname, source_dir, dest_dir): + """ + Deal with old-style images in DB. There is a flat list under + ~/.pygtktalog/images/ directory, which should be converted to nested + structure. + """ + + partial_path = mk_paths(os.path.join(source_dir, fname), dest_dir) + + dest_file = os.path.join(dest_dir, *partial_path) + dest_thumb = os.path.join(dest_dir, *partial_path) + "_t" + + shutil.copy(os.path.join(source_dir, fname), dest_file) + shutil.copy(os.path.join(source_dir, fname + "_t"), dest_thumb) + with open("log.txt", "a") as fobj: + fobj.write(os.path.join(fname) + "\n") + fobj.write(os.path.join(fname + "_t\n")) + + return os.path.join(*partial_path), os.path.join(*partial_path) + "_t" + + +def new_style_image_handle(partial_path, source_dir, dest_dir): + """ + Deal with old-style images in DB. In the early version directory was + hardcoded to ~/.pygtktalog/imgs2/, and all the needed files (with the + paths) should be copied to the new place. + params: + partial_path: string holding the relative path to file, for example + `de/ad/be/ef.jpg' + source_dir: path, where at the moment image file resides. Might be the + full path, like `/home/user/.pygtktalog/imgs2` + dest_dir: path (might be relative or absolute), where we want to put + the images (i.e. `../foo-images') + """ + dest_dir = mkdir_p(os.path.join(dest_dir, os.path.dirname(partial_path))) + base, ext = os.path.splitext(partial_path) + thumb = os.path.join(source_dir, "".join([base, "_t", ext])) + filename = os.path.join(source_dir, partial_path) + + shutil.copy(filename, dest_dir) + shutil.copy(thumb, dest_dir) + + +def copy_images_to_destination(cursor, image_path, dest): + """Copy images to dest directory and correct the db entry, if needed""" + + sql = "select id, filename from images" + update = "update images set filename=? where id=?" + t_select = "select id from thumbnails where filename=?" + t_update = "update thumbnails set filename=? where id=?" + + count = -1 + for count, (id_, filename) in enumerate(cursor.execute(sql).fetchall()): + if not image_path: + image_path = get_path(cursor, filename) + if not image_path: + raise OSError("Image file '%s' not found under data " + "directory, aborting" % filename) + + if image_path == PATH1: + # old style filenames. Flat list. + fname, tname = old_style_image_handle(filename, image_path, dest) + cursor.execute(update, (fname, id_)) + for (thumb_id,) in cursor.execute(t_select, + (filename,)).fetchall(): + cursor.execute(t_update, (tname, thumb_id)) + else: + # new style filenames. nested dirs + new_style_image_handle(filename, image_path, dest) + + if count > 0: + print "copied %d files" % (count + 1) def create_temporary_db_file(): """create temporary db file""" - fd, fname = mkstemp() - os.close(fd) + file_descriptor, fname = mkstemp() + os.close(file_descriptor) return fname def connect_to_db(filename): """initialize db connection and store it in class attributes""" - db_connection = sqlite.connect(filename, detect_types=sqlite.PARSE_DECLTYPES|sqlite.PARSE_COLNAMES) + db_connection = sqlite.connect(filename, detect_types= + sqlite.PARSE_DECLTYPES | + sqlite.PARSE_COLNAMES) db_cursor = db_connection.cursor() return db_connection, db_cursor def opendb(filename=None): """try to open db file""" db_tmp_path = create_temporary_db_file() - compressed = False try: test_file = open(filename).read(15) @@ -57,7 +195,6 @@ def opendb(filename=None): curdb.write(open_file.read()) curdb.close() open_file.close() - compressed = True except IOError: # file is not bz2 os.unlink(db_tmp_path) @@ -68,52 +205,95 @@ def opendb(filename=None): return connect_to_db(db_tmp_path), db_tmp_path -if __name__ == "__main__": - if len(sys.argv) != 3: - print "usage: %s src_base dst_base" % sys.argv[0] - exit() - - result = opendb(sys.argv[1]) - if not result: - print "unable to open src db file" - exit() - - (src_con, src_c), src_tmpf = result - - shutil.copy(src_tmpf, sys.argv[2]) - - # close src db. - src_c.close() - src_con.close() - os.unlink(src_tmpf) - - # create or update shema - connect(sys.argv[2]) - - dst_con = sqlite.connect(sys.argv[2]) - dst_c = dst_con.cursor() - - sql = "select id, date from files" - - for id, date in dst_c.execute(sql).fetchall(): - sql = "update files set date=? where id=?" - if date and int(date) > 0: - dst_c.execute(sql, (datetime.fromtimestamp(int(date)), id)) - else: - dst_c.execute(sql, (None, id)) - - sql = "select id, date from gthumb" - - for id, date in dst_c.execute(sql).fetchall(): - sql = "update gthumb set date=? where id=?" +def _update_dates(cursor, select_sql, update_sql): + """update date format - worker function""" + for id_, date in cursor.execute(select_sql).fetchall(): try: - if int(date) > 0: - dst_c.execute(sql, (datetime.fromtimestamp(int(date)), id)) - else: - dst_c.execute(sql, (None, id)) - except: - print id, date + date = int(date) + except ValueError: + # most probably there is no need for updating this record. + continue + except TypeError: + date = 0 - dst_con.commit() - dst_c.close() - dst_con.close() + if date > 0: + val = DATETIME().bind_processor(None)(datetime.fromtimestamp(date)) + else: + val = None + cursor.execute(update_sql, (val, id_)) + +def update_dates(cursor): + """Update date format from plain int to datetime object""" + + _update_dates(cursor, + "select id, date from files", + "update files set date=? where id=?") + _update_dates(cursor, + "select id, date from gthumb", + "update gthumb set date=? where id=?") + + +def main(): + """Main logic""" + if len(sys.argv) not in (4, 3): + print("usage: %s source_dbfile destination_dbfile [image_dir]\n" + "where image dir is a name where to put images. same name with" + "'_images' suffix by default" + % sys.argv[0]) + exit() + + if len(sys.argv) == 4: + source_dbfile, destination_dbfile, image_dir = sys.argv[1:] + else: + source_dbfile, destination_dbfile = sys.argv[1:] + image_dir = ":same_as_db:" + + result = opendb(source_dbfile) + if not result: + print("unable to open src db file") + exit() + + (connection, cursor), temporary_database_filename = result + + cursor.close() + connection.close() + shutil.copy(temporary_database_filename, destination_dbfile) + os.unlink(temporary_database_filename) + + connection = sqlite.connect(destination_dbfile) + cursor = connection.cursor() + + if cursor.execute("select name from sqlite_master where type='table' " + "and name='table_name'").fetchone() is None: + cursor.execute("CREATE TABLE 'config' (\n\t'id'\tINTEGER NOT NULL,\n" + "\t'key'\tTEXT,\n\t'value'\tTEXT,\n\tPRIMARY " + "KEY(id)\n)") + + if cursor.execute("select value from config where " + "key='image_path'").fetchone() is None: + cursor.execute("insert into config(key, value) " + "values('image_path', ?)", (image_dir,)) + else: + cursor.execute("update config set value=? where key='image_path'", + (image_dir,)) + + if image_dir == ":same_as_db:": + db_fname = os.path.basename(destination_dbfile) + base, dummy = os.path.splitext(db_fname) + image_dir_path = os.path.join(os.path.dirname(destination_dbfile), + base + "_images") + else: + image_dir_path = image_dir + + calculate_image_path(image_dir_path, True) + + update_dates(cursor) + old_image_path = get_images_path(cursor) + copy_images_to_destination(cursor, old_image_path, image_dir_path) + + connection.commit() + cursor.close() + connection.close() + +if __name__ == "__main__": + main() diff --git a/project.vim b/project.vim deleted file mode 100644 index 11e311e..0000000 --- a/project.vim +++ /dev/null @@ -1,64 +0,0 @@ -"All your bases are belong to us." -" -" Author: Roman.Dobosz at gmail.com -" Date: 2011-12-09 12:11:00 - -if !has("python") - finish -endif - -let g:project_dir = expand("%:p:h") - -python << EOF -import os -import vim - -PROJECT_DIR = vim.eval('project_dir') -TAGS_FILE = os.path.join(PROJECT_DIR, "tags") - -if not PROJECT_DIR.endswith("/"): - PROJECT_DIR += "/" -PYFILES= [] - -if os.path.exists(PROJECT_DIR + "tmp"): - os.system('rm -fr ' + PROJECT_DIR + "tmp") - -## icard specific -#for dir_ in os.listdir(os.path.join(PROJECT_DIR, "..", "externals")): -# if dir_ != 'mako': -# PYFILES.append(dir_) - -vim.command("set tags+=" + TAGS_FILE) - -# make all directories accessible by gf command -def req(path): - root, dirs, files = os.walk(path).next() - for dir_ in dirs: - newroot = os.path.join(root, dir_) - # all but the dot dirs - if dir_ in (".svn", ".hg", "locale", "tmp"): - continue - if "static" in root and dir_ != "js": - continue - - vim.command("set path+=" + newroot) - req(newroot) - -req(PROJECT_DIR) - -# generate tags -def update_tags(path): - assert os.path.exists(path) - - pylib_path = os.path.normpath(path) - pylib_path += " " + os.path.normpath('/usr/lib/python2.7/site-packages') - - # find tags for all files - cmd = 'ctags -R --python-kinds=-i' - cmd += ' -f ' + TAGS_FILE + ' ' + pylib_path - print cmd - os.system(cmd) -EOF - -" -command UpdateTags python update_tags(PROJECT_DIR) diff --git a/pygtktalog/dbcommon.py b/pygtktalog/dbcommon.py index 1471863..148ae3a 100644 --- a/pygtktalog/dbcommon.py +++ b/pygtktalog/dbcommon.py @@ -21,6 +21,7 @@ from pygtktalog.logger import get_logger Meta = MetaData() Base = declarative_base(metadata=Meta) Session = sessionmaker() +DbFilename = None LOG = get_logger("dbcommon") @@ -32,11 +33,13 @@ def connect(filename=None): @filename - string with absolute or relative path to sqlite database file. If None, db in-memory will be created """ + global DbFilename if not filename: filename = ':memory:' LOG.info("db filename: %s" % filename) + DbFilename = filename connect_string = "sqlite:///%s" % filename engine = create_engine(connect_string) diff --git a/pygtktalog/dbobjects.py b/pygtktalog/dbobjects.py index 853b7a8..cdbe5e2 100644 --- a/pygtktalog/dbobjects.py +++ b/pygtktalog/dbobjects.py @@ -6,10 +6,7 @@ Created: 2009-08-07 """ import os -import errno import shutil -from hashlib import sha256 -from zlib import crc32 from sqlalchemy import Column, Table, Integer, Text from sqlalchemy import DateTime, ForeignKey, Sequence @@ -18,12 +15,11 @@ from sqlalchemy.orm import relation, backref from pygtktalog.dbcommon import Base from pygtktalog.thumbnail import ThumbCreator from pygtktalog.logger import get_logger +from pygtktalog.misc import mk_paths LOG = get_logger(__name__) -IMG_PATH = "/home/gryf/.pygtktalog/imgs2/" # FIXME: should be configurable - tags_files = Table("tags_files", Base.metadata, Column("file_id", Integer, ForeignKey("files.id")), Column("tag_id", Integer, ForeignKey("tags.id"))) @@ -31,25 +27,11 @@ tags_files = Table("tags_files", Base.metadata, TYPE = {'root': 0, 'dir': 1, 'file': 2, 'link': 3} -def mk_paths(fname): - #new_name = str(uuid.uuid1()).split("-") - fd = open(fname) - new_path = "%x" % (crc32(fd.read(10*1024*1024)) & 0xffffffff) - fd.close() - - new_path = [new_path[i:i + 2] for i in range(0, len(new_path), 2)] - full_path = os.path.join(IMG_PATH, *new_path[:-1]) - - try: - os.makedirs(full_path) - except OSError as exc: - if exc.errno != errno.EEXIST: - LOG.debug("Directory %s already exists." % full_path) - - return new_path - - class File(Base): + """ + File mapping. Instances of this object can reference other File object + which make the structure to be tree-like + """ __tablename__ = "files" id = Column(Integer, Sequence("file_id_seq"), primary_key=True) parent_id = Column(Integer, ForeignKey("files.id"), index=True) @@ -102,19 +84,9 @@ class File(Base): else: return [] - # def mk_checksum(self): - # if not (self.filename and self.filepath): - # return - - # full_name = os.path.join(self.filepath, self.filename) - - # SLOW! - # if os.path.isfile(full_name): - # fd = open(full_name) - # self.checksum = sha256(fd.read(10*1024*1024)).hexdigest() - # fd.close() class Group(Base): + """TODO: what is this class for?""" __tablename__ = "groups" id = Column(Integer, Sequence("group_id_seq"), primary_key=True) name = Column(Text) @@ -129,6 +101,7 @@ class Group(Base): class Tag(Base): + """Tag mapping""" __tablename__ = "tags" id = Column(Integer, Sequence("tags_id_seq"), primary_key=True) group_id = Column(Integer, ForeignKey("groups.id"), index=True) @@ -146,22 +119,24 @@ class Tag(Base): class Thumbnail(Base): + """Thumbnail for the file""" __tablename__ = "thumbnails" id = Column(Integer, Sequence("thumbnail_id_seq"), primary_key=True) file_id = Column(Integer, ForeignKey("files.id"), index=True) filename = Column(Text) - def __init__(self, filename=None, file_obj=None): + def __init__(self, filename=None, img_path=None, file_obj=None): self.filename = filename self.file = file_obj - if filename and file_obj: - self.save(self.filename) + self.img_path = img_path + if filename and file_obj and img_path: + self.save(self.filename, img_path) - def save(self, fname): + def save(self, fname, img_path): """ Create file related thumbnail, add it to the file object. """ - new_name = mk_paths(fname) + new_name = mk_paths(fname, img_path) ext = os.path.splitext(self.filename)[1] if ext: new_name.append("".join([new_name.pop(), ext])) @@ -170,8 +145,8 @@ class Thumbnail(Base): name, ext = os.path.splitext(new_name.pop()) new_name.append("".join([name, "_t", ext])) self.filename = os.path.sep.join(new_name) - if not os.path.exists(os.path.join(IMG_PATH, *new_name)): - shutil.move(thumb, os.path.join(IMG_PATH, *new_name)) + if not os.path.exists(os.path.join(img_path, *new_name)): + shutil.move(thumb, os.path.join(img_path, *new_name)) else: LOG.info("Thumbnail already exists (%s: %s)" % \ (fname, "/".join(new_name))) @@ -182,34 +157,36 @@ class Thumbnail(Base): class Image(Base): + """Images and their thumbnails""" __tablename__ = "images" id = Column(Integer, Sequence("images_id_seq"), primary_key=True) file_id = Column(Integer, ForeignKey("files.id"), index=True) filename = Column(Text) - def __init__(self, filename=None, file_obj=None, move=True): + def __init__(self, filename=None, img_path=None, file_obj=None, move=True): self.filename = None self.file = file_obj - if filename: + self.img_path = img_path + if filename and img_path: self.filename = filename - self.save(filename, move) + self.save(filename, img_path, move) - def save(self, fname, move=True): + def save(self, fname, img_path, move=True): """ Save and create coressponding thumbnail (note: it differs from file related thumbnail!) """ - new_name = mk_paths(fname) + new_name = mk_paths(fname, img_path) ext = os.path.splitext(self.filename)[1] if ext: new_name.append("".join([new_name.pop(), ext])) - if not os.path.exists(os.path.join(IMG_PATH, *new_name)): + if not os.path.exists(os.path.join(img_path, *new_name)): if move: - shutil.move(self.filename, os.path.join(IMG_PATH, *new_name)) + shutil.move(self.filename, os.path.join(img_path, *new_name)) else: - shutil.copy(self.filename, os.path.join(IMG_PATH, *new_name)) + shutil.copy(self.filename, os.path.join(img_path, *new_name)) else: LOG.warning("Image with same CRC already exists " "('%s', '%s')" % (self.filename, "/".join(new_name))) @@ -219,9 +196,9 @@ class Image(Base): name, ext = os.path.splitext(new_name.pop()) new_name.append("".join([name, "_t", ext])) - if not os.path.exists(os.path.join(IMG_PATH, *new_name)): - thumb = ThumbCreator(os.path.join(IMG_PATH, self.filename)) - shutil.move(thumb.generate(), os.path.join(IMG_PATH, *new_name)) + if not os.path.exists(os.path.join(img_path, *new_name)): + thumb = ThumbCreator(os.path.join(img_path, self.filename)) + shutil.move(thumb.generate(), os.path.join(img_path, *new_name)) else: LOG.info("Thumbnail already generated %s" % "/".join(new_name)) @@ -241,20 +218,21 @@ class Image(Base): """ path, fname = os.path.split(self.filename) base, ext = os.path.splitext(fname) - return os.path.join(IMG_PATH, path, base + "_t" + ext) + return os.path.join(self.img_path, path, base + "_t" + ext) @property def imagepath(self): """ Return full path to image """ - return os.path.join(IMG_PATH, self.filename) + return os.path.join(self.img_path, self.filename) def __repr__(self): return "" % (str(self.filename), str(self.id)) class Exif(Base): + """Selected EXIF information""" __tablename__ = "exif" id = Column(Integer, Sequence("exif_id_seq"), primary_key=True) file_id = Column(Integer, ForeignKey("files.id"), index=True) @@ -292,6 +270,7 @@ class Exif(Base): class Gthumb(Base): + """Gthumb information""" __tablename__ = "gthumb" id = Column(Integer, Sequence("gthumb_id_seq"), primary_key=True) file_id = Column(Integer, ForeignKey("files.id"), index=True) @@ -307,3 +286,18 @@ class Gthumb(Base): def __repr__(self): return "" % (str(self.date), str(self.place), str(self.id)) + + +class Config(Base): + """Per-database configuration""" + __tablename__ = "config" + id = Column(Integer, Sequence("config_id_seq"), primary_key=True) + key = Column(Text) + value = Column(Text) + + def __init__(self, key=None, value=None): + self.key = key + self.value = value + + def __repr__(self): + return "" % (str(self.key), str(self.value)) diff --git a/pygtktalog/misc.py b/pygtktalog/misc.py index 919ae07..f9fa37f 100644 --- a/pygtktalog/misc.py +++ b/pygtktalog/misc.py @@ -5,6 +5,15 @@ Author: Roman 'gryf' Dobosz, gryf73@gmail.com Created: 2009-04-05 """ +import os +import errno +from zlib import crc32 + +import pygtktalog.dbcommon +from pygtktalog.logger import get_logger + +LOG = get_logger(__name__) + def float_to_string(float_length): """ @@ -20,3 +29,47 @@ def float_to_string(float_length): sec = int(float_length) return "%02d:%02d:%02d" % (hour, minutes, sec) +def calculate_image_path(dbpath=None, create=False): + """Calculate image path out of provided path or using current connection""" + if not dbpath: + dbpath = pygtktalog.dbcommon.DbFilename + if dbpath == ":memory:": + raise OSError("Cannot create image path out of in-memory db!") + + dir_, file_ = (os.path.dirname(dbpath), os.path.basename(dbpath)) + file_base, dummy = os.path.splitext(file_) + images_dir = os.path.join(dir_, file_base + "_images") + else: + if dbpath and "~" in dbpath: + dbpath = os.path.expanduser(dbpath) + if dbpath and "$" in dbpath: + dbpath = os.path.expandvars(dbpath) + images_dir = dbpath + + if create: + if not os.path.exists(images_dir): + try: + os.mkdir(images_dir) + except OSError, err: + if err.errno != errno.EEXIST: + raise + elif not os.path.exists(images_dir): + raise OSError("%s: No such directory" % images_dir) + + return os.path.abspath(images_dir) + +def mk_paths(fname, img_path): + """Make path for provided pathname by calculating crc32 out of file""" + with open(fname) as fobj: + new_path = "%x" % (crc32(fobj.read(10*1024*1024)) & 0xffffffff) + + new_path = [new_path[i:i + 2] for i in range(0, len(new_path), 2)] + full_path = os.path.join(img_path, *new_path[:-1]) + + try: + os.makedirs(full_path) + except OSError as exc: + if exc.errno != errno.EEXIST: + LOG.debug("Directory %s already exists." % full_path) + + return new_path diff --git a/pygtktalog/scan.py b/pygtktalog/scan.py index 626841f..6f01f83 100644 --- a/pygtktalog/scan.py +++ b/pygtktalog/scan.py @@ -11,7 +11,8 @@ import re from datetime import datetime import mimetypes -from pygtktalog.dbobjects import File, Image, Thumbnail, TYPE +import pygtktalog.misc +from pygtktalog.dbobjects import File, Image, Thumbnail, Config, TYPE from pygtktalog.dbcommon import Session from pygtktalog.logger import get_logger from pygtktalog.video import Video @@ -44,7 +45,7 @@ class Scan(object): """ Initialize @Arguments: - @path - string with initial root directory to scan + @path - string with path to be added to topmost node (root) """ self.abort = False self.path = path.rstrip(os.path.sep) @@ -55,6 +56,8 @@ class Scan(object): self.files_count = self._get_files_count() self.current_count = 0 + self._set_image_path() + def add_files(self, engine=None): """ Returns list, which contain object, modification date and file @@ -106,15 +109,21 @@ class Scan(object): req(row) sql = SQL2 % ",".join("?" * len(all_ids)) - res = engine.execute(sql, tuple(all_ids)).fetchall() + all_ids = [row_[0] for row_ in engine + .execute(sql, tuple(all_ids)) + .fetchall()] all_obj = [] - for row in res: - all_obj.append(self._session + # number of objects to retrieve at once. Limit is 999. Let's do a + # little bit below. + no = 900 + steps = len(all_ids) / no + 1 + for step in range(steps): + all_obj.extend(self._session .query(File) - .filter(File.id == row[0]) - .first()) - + .filter(File.id + .in_(all_ids[step * no:step * no + no])) + .all()) return all_obj def update_files(self, node_id, engine=None): @@ -248,7 +257,7 @@ class Scan(object): preview_fn = vid.capture() if preview_fn: - Image(preview_fn, fobj) + Image(preview_fn, self.img_path, fobj) def _check_related(self, fobj, pattern): """ @@ -261,10 +270,10 @@ class Scan(object): full_fname = os.path.join(fobj.filepath, filen) LOG.debug('found cover file: %s' % full_fname) - Image(full_fname, fobj, False) + Image(full_fname, self.img_path, fobj, False) if not fobj.thumbnail: - Thumbnail(full_fname, fobj) + Thumbnail(full_fname, self.img_path, fobj) def _name_matcher(self, fpath, fname, media=False): """ @@ -326,7 +335,7 @@ class Scan(object): fobj.type = fob['ftype'] else: fobj = File(**fob) - # SLOW. Don;t do this. Checksums has no value eventually + # SLOW. Don't do this. Checksums has no value eventually # fobj.mk_checksum() if parent is None: @@ -482,12 +491,24 @@ class Scan(object): return None def _get_files_count(self): + """return size in bytes""" count = 0 - for root, dirs, files in os.walk(str(self.path)): + for _, _, files in os.walk(str(self.path)): count += len(files) LOG.debug("count of files: %s", count) return count + def _set_image_path(self): + """Get or calculate the images path""" + image_path = self._session.query(Config) \ + .filter(Config.key=="image_path").one() + if image_path.value == ":same_as_db:": + image_path = pygtktalog.misc.calculate_image_path() + else: + image_path = pygtktalog.misc.calculate_image_path(image_path.value) + + self.img_path = image_path + class asdScan(object): """ @@ -561,7 +582,7 @@ class asdScan(object): current_dir = os.path.join(root, i) try: - st = os.stat(current_dir) + st = os.lstat(current_dir) st_mtime = st.st_mtime except OSError: st_mtime = 0 @@ -597,7 +618,7 @@ class asdScan(object): current_file = os.path.join(root, i) try: - st = os.stat(current_file) + st = os.lstat(current_file) st_mtime = st.st_mtime st_size = st.st_size except OSError: diff --git a/pygtktalog/thumbnail.py b/pygtktalog/thumbnail.py index f496500..57827a5 100644 --- a/pygtktalog/thumbnail.py +++ b/pygtktalog/thumbnail.py @@ -8,6 +8,7 @@ import os from tempfile import mkstemp +import shutil from PIL import Image @@ -56,14 +57,17 @@ class ThumbCreator(object): thumb.close() else: LOG.debug("no exif thumb") - thumb = self._scale_image() - if thumb: - thumb.save(thumb_fn, "JPEG") + if self.is_image_smaller(): + shutil.copyfile(self.filename, thumb_fn) + else: + thumb = self._scale_image() + if thumb: + thumb.save(thumb_fn, "JPEG") if exif and 'Image Orientation' in exif: orient = exif['Image Orientation'].values[0] if orient > 1 and orient in orientations: - thumb_image = Image.open(self.thumb_fn) + thumb_image = Image.open(thumb_fn) tmp_thumb_img = thumb_image.transpose(orientations[orient]) if orient in flips: @@ -73,6 +77,13 @@ class ThumbCreator(object): return thumb_fn + def is_image_smaller(self): + """Check if image is smaller than desired dimention, return boolean""" + image = Image.open(self.filename) + im_x, im_y = image.size + image.close() + return im_x <= self.thumb_x and im_y <= self.thumb_y + def _get_exif(self): """ Get exif (if available), return as a dict