Corrections to db converter, some improvements to commandline version of client

cmdcatalog.py

@@ -1,20 +1,26 @@
 #!/usr/bin/env python
+"""
+Fast and ugly CLI interface for pyGTKtalog
+"""
 import os
 import sys
+import errno
 from argparse import ArgumentParser
 
 from pygtktalog import scan
-from pygtktalog.dbobjects import File
+from pygtktalog import misc
+from pygtktalog.dbobjects import File, Config
 from pygtktalog.dbcommon import connect, Session
 
-BLACK, RED, GREEN, YELLOW, BLUE, MAGENTA, CYAN, WHITE = range(8)
+BLACK, RED, GREEN, YELLOW, BLUE, MAGENTA, CYAN, WHITE = range(30, 38)
 
 RESET_SEQ = "\033[0m"
 COLOR_SEQ = "\033[1;%dm"
 BOLD_SEQ = "\033[1m"
 
 
-def cprint(txt, color):
+def colorize(txt, color):
+    """Pretty print with colors to console."""
     color_map = {"black": BLACK,
                  "red": RED,
                  "green": GREEN,
@@ -23,27 +29,23 @@ def cprint(txt, color):
                  "magenta": MAGENTA,
                  "cyan": CYAN,
                  "white": WHITE}
-    print COLOR_SEQ % (30 + color_map[color]) + txt + RESET_SEQ
+    return COLOR_SEQ % color_map[color] + txt + RESET_SEQ
 
 
 class Iface(object):
+    """Main class which interacts with the pyGTKtalog modules"""
     def __init__(self, dbname, pretend=False, debug=False):
+        """Init"""
         self.engine = connect(dbname)
         self.sess = Session()
         self.dry_run = pretend
         self.root = None
+        self._dbname = dbname
         if debug:
             scan.LOG.setLevel("DEBUG")
 
-    def close(self):
-        self.sess.commit()
-        self.sess.close()
-
-    # def create(self):
-        # self.sess.commit()
-        # self.sess.close()
-
     def _resolve_path(self, path):
+        """Identify path in the DB"""
         if not path.startswith("/"):
             raise AttributeError("Path have to start with slash (/)")
 
@@ -62,18 +64,51 @@ class Iface(object):
         return last_node
 
     def _make_path(self, node):
+        """Make the path to the item in the DB"""
         if node.parent == node:
             return "/"
 
+        ext = ""
+        if node.parent.type == 0:
+            ext = colorize(" (%s)" % node.filepath, "white")
+
         path = []
         path.append(node.filename)
         while node.parent != self.root:
             path.append(node.parent.filename)
             node = node.parent
 
-        return "/".join([""] + path[::-1])
+        return "/".join([""] + path[::-1]) + ext
 
-    def list(self, path=None):
+    def _walk(self, dirnode):
+        """Recursively go through the leaves of the node"""
+        items = []
+        for node in dirnode.children:
+            if node.type == 1:
+                items += self._walk(node)
+
+            items.append(" " + self._make_path(node))
+
+        items.sort()
+        return items
+
+    def _list(self, node):
+        """List only current node content"""
+        items = []
+        for node in node.children:
+            if node != self.root:
+                items.append(" " + self._make_path(node))
+
+        items.sort()
+        return items
+
+    def close(self):
+        """Close the session"""
+        self.sess.commit()
+        self.sess.close()
+
+    def list(self, path=None, recursive=False):
+        """Simulate ls command for the provided item path"""
         self.root = self.sess.query(File).filter(File.type==0).first()
         if path:
             node = self._resolve_path(path)
@@ -82,20 +117,25 @@ class Iface(object):
             node = self.root
             msg = "Content of path `/':"
 
-        cprint(msg, "white")
+        print colorize(msg, "white")
-        for node in node.children:
+
-            if node != self.root:
+        if recursive:
-                #if __debug__:
+            items = self._walk(node)
-                #    print "  %d:" % node.id, self._make_path(node)
+        else:
-                #else:
+            items = self._list(node)
-                print " ", self._make_path(node)
+
+        print "\n".join(items)
 
     def update(self, path, dir_to_update=None):
+        """
+        Update the DB against provided path and optionally directory on the
+        real filesystem
+        """
         self.root = self.sess.query(File).filter(File.type==0).first()
         node = self._resolve_path(path)
         if node == self.root:
-            cprint("Cannot update entire db, since root was provided as path.",
+            print colorize("Cannot update entire db, since root was provided "
-                   "red")
+                           "as path.", "red")
             return
 
         if not dir_to_update:
@@ -104,14 +144,15 @@ class Iface(object):
         if not os.path.exists(dir_to_update):
             raise OSError("Path to updtate doesn't exists: %s", dir_to_update)
 
-        cprint("Updating node `%s' against directory "
+        print colorize("Updating node `%s' against directory "
-               "`%s'" % (path, dir_to_update), "white")
+                       "`%s'" % (path, dir_to_update), "white")
         if not self.dry_run:
             scanob = scan.Scan(dir_to_update)
             # scanob.update_files(node.id)
             scanob.update_files(node.id, self.engine)
 
-    def create(self, dir_to_add):
+    def create(self, dir_to_add, data_dir):
+        """Create new database"""
         self.root = File()
         self.root.id = 1
         self.root.filename = 'root'
@@ -119,51 +160,93 @@ class Iface(object):
         self.root.source = 0
         self.root.type = 0
         self.root.parent_id = 1
+
+        config = Config()
+        config.key = "image_path"
+        config.value = data_dir
+
         if not self.dry_run:
             self.sess.add(self.root)
+            self.sess.add(config)
             self.sess.commit()
 
-        cprint("Creating new db against directory `%s'" % dir_to_add, "white")
+        print colorize("Creating new db against directory `%s'" % dir_to_add,
+                       "white")
         if not self.dry_run:
+            if data_dir == ":same_as_db:":
+                misc.calculate_image_path(None, True)
+            else:
+                misc.calculate_image_path(data_dir, True)
+
             scanob = scan.Scan(dir_to_add)
             scanob.add_files(self.engine)
 
+    def add(self, dir_to_add):
+        """Add new directory to the db"""
+        self.root = self.sess.query(File).filter(File.type==0).first()
+
+        if not os.path.exists(dir_to_add):
+            raise OSError("Path to add doesn't exists: %s", dir_to_add)
+
+        print colorize("Adding directory `%s'" % dir_to_add, "white")
+        if not self.dry_run:
+            scanob = scan.Scan(dir_to_add)
+            scanob.add_files()
+
 
 def list_db(args):
+    """List"""
     if not os.path.exists(args.db):
-        cprint("File `%s' does not exists!" % args.db, "red")
+        print colorize("File `%s' does not exists!" % args.db, "red")
         sys.exit(1)
 
     obj = Iface(args.db, False, args.debug)
-    obj.list(path=args.path)
+    obj.list(path=args.path, recursive=args.recursive)
     obj.close()
 
 
 def update_db(args):
+    """Update"""
     if not os.path.exists(args.db):
-        cprint("File `%s' does not exists!" % args.db, "red")
+        print colorize("File `%s' does not exists!" % args.db, "red")
         sys.exit(1)
 
     obj = Iface(args.db, args.pretend, args.debug)
     obj.update(args.path, dir_to_update=args.dir_to_update)
     obj.close()
 
-def create_db(args):
+
-    if os.path.exists(args.db):
+def add_dir(args):
-        cprint("File `%s' exists!" % args.db, "yellow")
+    """Add"""
+    if not os.path.exists(args.db):
+        print colorize("File `%s' does not exists!" % args.db, "red")
+        sys.exit(1)
 
     obj = Iface(args.db, args.pretend, args.debug)
-    obj.create(args.dir_to_add)
+    obj.add(args.dir_to_add)
     obj.close()
 
 
-if __name__ == "__main__":
+def create_db(args):
+    """List"""
+    if os.path.exists(args.db):
+        print colorize("File `%s' exists!" % args.db, "yellow")
+
+    obj = Iface(args.db, args.pretend, args.debug)
+    obj.create(args.dir_to_add, args.imagedir)
+    obj.close()
+
+
+def main():
+    """Main"""
     parser = ArgumentParser()
 
     subparser = parser.add_subparsers()
     list_ = subparser.add_parser("list")
     list_.add_argument("db")
     list_.add_argument("path", nargs="?")
+    list_.add_argument("-r", "--recursive", help="list items in "
+                       "subdirectories", action="store_true", default=False)
     list_.add_argument("-d", "--debug", help="Turn on debug",
                        action="store_true", default=False)
     list_.set_defaults(func=list_db)
@@ -182,6 +265,12 @@ if __name__ == "__main__":
     create = subparser.add_parser("create")
     create.add_argument("db")
     create.add_argument("dir_to_add")
+    create.add_argument("-i", "--imagedir", help="Directory where to put "
+                        "images for the database. Popular, but deprecated "
+                        "choice is  `~/.pygtktalog/images'. Currnet default "
+                        "is special string `:same_as_db:' which will try to "
+                        "create directory with the same name as the db with "
+                        "data suffix", default=":same_as_db:")
     create.add_argument("-p", "--pretend", help="Don't do the action, just "
                         "give the info what would gonna to happen.",
                         action="store_true", default=False)
@@ -189,27 +278,18 @@ if __name__ == "__main__":
                         action="store_true", default=False)
     create.set_defaults(func=create_db)
 
+    add = subparser.add_parser("add")
+    add.add_argument("db")
+    add.add_argument("dir_to_add")
+    add.add_argument("-p", "--pretend", help="Don't do the action, just "
+                     "give the info what would gonna to happen.",
+                     action="store_true", default=False)
+    add.add_argument("-d", "--debug", help="Turn on debug",
+                     action="store_true", default=False)
+    add.set_defaults(func=add_dir)
+
     args = parser.parse_args()
     args.func(args)
 
-
+if __name__ == "__main__":
-"""
+    main()
-db_file = "/home/gryf/spisy/xxx.sqlite"
-connect(db_file)
-sess = Session()
-
-#if not sess.query(File).get(1):
-#    root = File()
-#    root.id = 1
-#    root.filename = 'root'
-#    root.size = 0
-#    root.source = 0
-#    t.type = 0
-#    root.parent_id = 1
-#    sess.add(root)
-#    sess.commit()
-
-f = "/mnt/hardtwo/XXX/"
-scanob = scan.Scan(f)
-scanob.update_files(2)
-"""

convert_1.x_to_2.x.py

@@ -6,39 +6,177 @@
     Author: Roman 'gryf' Dobosz, gryf73@gmail.com
     Created: 2009-08-14
 """
-import sys
-import os
-import bz2
-import shutil
-from tempfile import mkstemp
-from sqlite3 import dbapi2 as sqlite
 from datetime import datetime
+from sqlite3 import dbapi2 as sqlite
+from sqlite3 import OperationalError
+from tempfile import mkstemp
+import bz2
+import errno
+import os
+import shutil
+import sys
 
-# import db objects just to create schema
+from sqlalchemy.dialects.sqlite import DATETIME
-from pygtktalog.dbobjects import File, Exif, Group, Gthumb
-from pygtktalog.dbobjects import Image, Tag, Thumbnail
-from pygtktalog.dbcommon import connect
 
-def create_schema(cur):
+from pygtktalog.misc import mk_paths, calculate_image_path
-    pass
 
+PATH1 = os.path.expanduser("~/.pygtktalog/images")
+PATH2 = os.path.expanduser("~/.pygtktalog/imgs2")
+
+
+def mkdir_p(path):
+    """Make directories recurively, like 'mkdir -p' command"""
+    try:
+        os.makedirs(path)
+    except OSError as exc:
+        if exc.errno == errno.EEXIST and os.path.isdir(path):
+            pass
+        else:
+            raise
+    return path
+
+def get_images_path(cur):
+    """
+    Calculate the data dir in order:
+        - config table
+        - old default path
+        - new default path
+    return first, which contain provided image filename
+    """
+
+    image = cur.execute("select filename from images limit 1").fetchone()
+    if image and image[0]:
+        image = image[0]
+
+    try:
+        result = cur.execute("select value from config where "
+                             "key='image_path'").fetchone()
+        if (result and result[0] and
+            os.path.exists(os.path.join(result[0].encode("utf-8"),
+                                        image.encode("utf-8")))):
+            return result[0]
+    except OperationalError:
+        # no such table like config. proceed.
+        pass
+
+    for path in (PATH1, PATH2):
+        if os.path.exists(os.path.join(path, image)):
+            return path
+    return None
+
+def get_path(cur, image):
+    """
+    Calculate the data dir in order:
+        - config table
+        - old default path
+        - new default path
+    return first, which contain provided image filename
+    """
+    try:
+        result = cur.execute("select value from config where "
+                             "key='image_path'").fetchone()
+        if (result and result[0] and
+            os.path.exists(os.path.join(result[0].encode("utf-8"),
+                                        image.encode("utf-8")))):
+            return result[0]
+    except OperationalError:
+        pass
+
+    for path in (PATH1, PATH2):
+        if os.path.exists(os.path.join(path, image)):
+            return path
+    return None
+
+def old_style_image_handle(fname, source_dir, dest_dir):
+    """
+    Deal with old-style images in DB. There is a flat list under
+    ~/.pygtktalog/images/ directory, which should be converted to nested
+    structure.
+    """
+
+    partial_path = mk_paths(os.path.join(source_dir, fname), dest_dir)
+
+    dest_file = os.path.join(dest_dir, *partial_path)
+    dest_thumb = os.path.join(dest_dir, *partial_path) + "_t"
+
+    shutil.copy(os.path.join(source_dir, fname), dest_file)
+    shutil.copy(os.path.join(source_dir, fname + "_t"), dest_thumb)
+    with open("log.txt", "a") as fobj:
+        fobj.write(os.path.join(fname) + "\n")
+        fobj.write(os.path.join(fname + "_t\n"))
+
+    return os.path.join(*partial_path), os.path.join(*partial_path) + "_t"
+
+
+def new_style_image_handle(partial_path, source_dir, dest_dir):
+    """
+    Deal with old-style images in DB. In the early version directory was
+    hardcoded to ~/.pygtktalog/imgs2/, and all the needed files (with the
+    paths) should be copied to the new place.
+    params:
+        partial_path: string holding the relative path to file, for example
+                      `de/ad/be/ef.jpg'
+        source_dir: path, where at the moment image file resides. Might be the
+                    full path, like `/home/user/.pygtktalog/imgs2`
+        dest_dir: path (might be relative or absolute), where we want to put
+                  the images (i.e. `../foo-images')
+    """
+    dest_dir = mkdir_p(os.path.join(dest_dir, os.path.dirname(partial_path)))
+    base, ext = os.path.splitext(partial_path)
+    thumb = os.path.join(source_dir, "".join([base, "_t", ext]))
+    filename = os.path.join(source_dir, partial_path)
+
+    shutil.copy(filename, dest_dir)
+    shutil.copy(thumb, dest_dir)
+
+
+def copy_images_to_destination(cursor, image_path, dest):
+    """Copy images to dest directory and correct the db entry, if needed"""
+
+    sql = "select id, filename from images"
+    update = "update images set filename=? where id=?"
+    t_select = "select id from thumbnails where filename=?"
+    t_update = "update thumbnails set filename=? where id=?"
+
+    count = -1
+    for count, (id_, filename) in enumerate(cursor.execute(sql).fetchall()):
+        if not image_path:
+            image_path = get_path(cursor, filename)
+            if not image_path:
+                raise OSError("Image file '%s' not found under data "
+                              "directory, aborting" % filename)
+
+        if image_path == PATH1:
+            # old style filenames. Flat list.
+            fname, tname = old_style_image_handle(filename, image_path, dest)
+            cursor.execute(update, (fname, id_))
+            for (thumb_id,) in cursor.execute(t_select,
+                                              (filename,)).fetchall():
+                cursor.execute(t_update, (tname, thumb_id))
+        else:
+            # new style filenames. nested dirs
+            new_style_image_handle(filename, image_path, dest)
+
+    if count > 0:
+        print "copied %d files" % (count + 1)
 
 def create_temporary_db_file():
     """create temporary db file"""
-    fd, fname = mkstemp()
+    file_descriptor, fname = mkstemp()
-    os.close(fd)
+    os.close(file_descriptor)
     return fname
 
 def connect_to_db(filename):
     """initialize db connection and store it in class attributes"""
-    db_connection = sqlite.connect(filename, detect_types=sqlite.PARSE_DECLTYPES|sqlite.PARSE_COLNAMES)
+    db_connection = sqlite.connect(filename, detect_types=
+                                   sqlite.PARSE_DECLTYPES |
+                                   sqlite.PARSE_COLNAMES)
     db_cursor = db_connection.cursor()
     return db_connection, db_cursor
 
 def opendb(filename=None):
     """try to open db file"""
     db_tmp_path = create_temporary_db_file()
-    compressed = False
 
     try:
         test_file = open(filename).read(15)
@@ -57,7 +195,6 @@ def opendb(filename=None):
             curdb.write(open_file.read())
             curdb.close()
             open_file.close()
-            compressed = True
         except IOError:
             # file is not bz2
             os.unlink(db_tmp_path)
@@ -68,52 +205,95 @@ def opendb(filename=None):
 
     return connect_to_db(db_tmp_path), db_tmp_path
 
-if __name__ == "__main__":
+def _update_dates(cursor, select_sql, update_sql):
-    if len(sys.argv) != 3:
+    """update date format - worker function"""
-        print "usage: %s src_base dst_base" % sys.argv[0]
+    for id_, date in cursor.execute(select_sql).fetchall():
-        exit()
-
-    result = opendb(sys.argv[1])
-    if not result:
-        print "unable to open src db file"
-        exit()
-
-    (src_con, src_c), src_tmpf = result
-
-    shutil.copy(src_tmpf, sys.argv[2])
-
-    # close src db.
-    src_c.close()
-    src_con.close()
-    os.unlink(src_tmpf)
-
-    # create or update shema
-    connect(sys.argv[2])
-
-    dst_con = sqlite.connect(sys.argv[2])
-    dst_c = dst_con.cursor()
-
-    sql = "select id, date from files"
-
-    for id, date in dst_c.execute(sql).fetchall():
-        sql = "update files set date=? where id=?"
-        if date and int(date) > 0:
-            dst_c.execute(sql, (datetime.fromtimestamp(int(date)), id))
-        else:
-            dst_c.execute(sql, (None, id))
-
-    sql = "select id, date from gthumb"
-
-    for id, date in dst_c.execute(sql).fetchall():
-        sql = "update gthumb set date=? where id=?"
         try:
-            if int(date) > 0:
+            date = int(date)
-                dst_c.execute(sql, (datetime.fromtimestamp(int(date)), id))
+        except ValueError:
-            else:
+            # most probably there is no need for updating this record.
-                dst_c.execute(sql, (None, id))
+            continue
-        except:
+        except TypeError:
-            print id, date
+            date = 0
 
-    dst_con.commit()
+        if date > 0:
-    dst_c.close()
+            val = DATETIME().bind_processor(None)(datetime.fromtimestamp(date))
-    dst_con.close()
+        else:
+            val = None
+        cursor.execute(update_sql, (val, id_))
+
+def update_dates(cursor):
+    """Update date format from plain int to datetime object"""
+
+    _update_dates(cursor,
+                  "select id, date from files",
+                  "update files set date=? where id=?")
+    _update_dates(cursor,
+                  "select id, date from gthumb",
+                  "update gthumb set date=? where id=?")
+
+
+def main():
+    """Main logic"""
+    if len(sys.argv) not in (4, 3):
+        print("usage: %s source_dbfile destination_dbfile [image_dir]\n"
+              "where image dir is a name where to put images. same name with"
+              "'_images' suffix by default"
+              % sys.argv[0])
+        exit()
+
+    if len(sys.argv) == 4:
+        source_dbfile, destination_dbfile, image_dir = sys.argv[1:]
+    else:
+        source_dbfile, destination_dbfile = sys.argv[1:]
+        image_dir = ":same_as_db:"
+
+    result = opendb(source_dbfile)
+    if not result:
+        print("unable to open src db file")
+        exit()
+
+    (connection, cursor), temporary_database_filename = result
+
+    cursor.close()
+    connection.close()
+    shutil.copy(temporary_database_filename, destination_dbfile)
+    os.unlink(temporary_database_filename)
+
+    connection = sqlite.connect(destination_dbfile)
+    cursor = connection.cursor()
+
+    if cursor.execute("select name from sqlite_master where type='table' "
+                      "and name='table_name'").fetchone() is None:
+        cursor.execute("CREATE TABLE 'config' (\n\t'id'\tINTEGER NOT NULL,\n"
+                       "\t'key'\tTEXT,\n\t'value'\tTEXT,\n\tPRIMARY "
+                       "KEY(id)\n)")
+
+    if cursor.execute("select value from config where "
+                      "key='image_path'").fetchone() is None:
+        cursor.execute("insert into config(key, value) "
+                       "values('image_path', ?)", (image_dir,))
+    else:
+        cursor.execute("update config set value=? where key='image_path'",
+                       (image_dir,))
+
+    if image_dir == ":same_as_db:":
+        db_fname = os.path.basename(destination_dbfile)
+        base, dummy = os.path.splitext(db_fname)
+        image_dir_path = os.path.join(os.path.dirname(destination_dbfile),
+                                      base + "_images")
+    else:
+        image_dir_path = image_dir
+
+    calculate_image_path(image_dir_path, True)
+
+    update_dates(cursor)
+    old_image_path = get_images_path(cursor)
+    copy_images_to_destination(cursor, old_image_path, image_dir_path)
+
+    connection.commit()
+    cursor.close()
+    connection.close()
+
+if __name__ == "__main__":
+    main()

project.vim

@@ -1,64 +0,0 @@
-"All your bases are belong to us."
-"
-" Author: Roman.Dobosz at gmail.com
-" Date: 2011-12-09 12:11:00
-
-if !has("python")
-  finish
-endif
-
-let g:project_dir = expand("%:p:h")
-
-python << EOF
-import os
-import vim
-
-PROJECT_DIR = vim.eval('project_dir')
-TAGS_FILE = os.path.join(PROJECT_DIR, "tags")
-
-if not PROJECT_DIR.endswith("/"):
-    PROJECT_DIR += "/"
-PYFILES= []
-
-if os.path.exists(PROJECT_DIR + "tmp"):
-    os.system('rm -fr ' + PROJECT_DIR + "tmp")
-
-## icard specific
-#for dir_ in os.listdir(os.path.join(PROJECT_DIR, "..", "externals")):
-#    if dir_ != 'mako':
-#        PYFILES.append(dir_)
-
-vim.command("set tags+=" + TAGS_FILE)
-
-# make all directories accessible by gf command
-def req(path):
-    root, dirs, files = os.walk(path).next()
-    for dir_ in dirs:
-        newroot = os.path.join(root, dir_)
-        # all but the dot dirs
-        if dir_ in (".svn", ".hg", "locale", "tmp"):
-            continue
-        if "static" in root and dir_ != "js":
-            continue
-
-        vim.command("set path+=" + newroot)
-        req(newroot)
-
-req(PROJECT_DIR)
-
-# generate tags
-def update_tags(path):
-    assert os.path.exists(path)
-
-    pylib_path = os.path.normpath(path)
-    pylib_path += " " + os.path.normpath('/usr/lib/python2.7/site-packages')
-
-    # find tags for all files
-    cmd = 'ctags -R --python-kinds=-i'
-    cmd += ' -f ' + TAGS_FILE + ' ' + pylib_path
-    print cmd
-    os.system(cmd)
-EOF
-
-"
-command UpdateTags python update_tags(PROJECT_DIR)

pygtktalog/dbcommon.py

@@ -21,6 +21,7 @@ from pygtktalog.logger import get_logger
 Meta = MetaData()
 Base = declarative_base(metadata=Meta)
 Session = sessionmaker()
+DbFilename = None
 
 LOG = get_logger("dbcommon")
 
@@ -32,11 +33,13 @@ def connect(filename=None):
         @filename - string with absolute or relative path to sqlite database
                     file. If None, db in-memory will be created
     """
+    global DbFilename
 
     if not filename:
         filename = ':memory:'
 
     LOG.info("db filename: %s" % filename)
+    DbFilename = filename
 
     connect_string = "sqlite:///%s" % filename
     engine = create_engine(connect_string)

pygtktalog/dbobjects.py

@@ -6,10 +6,7 @@
     Created: 2009-08-07
 """
 import os
-import errno
 import shutil
-from hashlib import sha256
-from zlib import crc32
 
 from sqlalchemy import Column, Table, Integer, Text
 from sqlalchemy import DateTime, ForeignKey, Sequence
@@ -18,12 +15,11 @@ from sqlalchemy.orm import relation, backref
 from pygtktalog.dbcommon import Base
 from pygtktalog.thumbnail import ThumbCreator
 from pygtktalog.logger import get_logger
+from pygtktalog.misc import mk_paths
 
 
 LOG = get_logger(__name__)
 
-IMG_PATH = "/home/gryf/.pygtktalog/imgs2/"  # FIXME: should be configurable
-
 tags_files = Table("tags_files", Base.metadata,
                    Column("file_id", Integer, ForeignKey("files.id")),
                    Column("tag_id", Integer, ForeignKey("tags.id")))
@@ -31,25 +27,11 @@ tags_files = Table("tags_files", Base.metadata,
 TYPE = {'root': 0, 'dir': 1, 'file': 2, 'link': 3}
 
 
-def mk_paths(fname):
-    #new_name = str(uuid.uuid1()).split("-")
-    fd = open(fname)
-    new_path = "%x" % (crc32(fd.read(10*1024*1024)) & 0xffffffff)
-    fd.close()
-
-    new_path = [new_path[i:i + 2] for i in range(0, len(new_path), 2)]
-    full_path = os.path.join(IMG_PATH, *new_path[:-1])
-
-    try:
-        os.makedirs(full_path)
-    except OSError as exc:
-        if exc.errno != errno.EEXIST:
-            LOG.debug("Directory %s already exists." % full_path)
-
-    return new_path
-
-
 class File(Base):
+    """
+    File mapping. Instances of this object can reference other File object
+    which make the structure to be tree-like
+    """
     __tablename__ = "files"
     id = Column(Integer, Sequence("file_id_seq"), primary_key=True)
     parent_id = Column(Integer, ForeignKey("files.id"), index=True)
@@ -102,19 +84,9 @@ class File(Base):
         else:
             return []
 
-    # def mk_checksum(self):
-        # if not (self.filename and self.filepath):
-            # return
-
-        # full_name = os.path.join(self.filepath, self.filename)
-
-        # SLOW!
-        # if os.path.isfile(full_name):
-            # fd = open(full_name)
-            # self.checksum = sha256(fd.read(10*1024*1024)).hexdigest()
-            # fd.close()
 
 class Group(Base):
+    """TODO: what is this class for?"""
     __tablename__ = "groups"
     id = Column(Integer, Sequence("group_id_seq"), primary_key=True)
     name = Column(Text)
@@ -129,6 +101,7 @@ class Group(Base):
 
 
 class Tag(Base):
+    """Tag mapping"""
     __tablename__ = "tags"
     id = Column(Integer, Sequence("tags_id_seq"), primary_key=True)
     group_id = Column(Integer, ForeignKey("groups.id"), index=True)
@@ -146,22 +119,24 @@ class Tag(Base):
 
 
 class Thumbnail(Base):
+    """Thumbnail for the file"""
     __tablename__ = "thumbnails"
     id = Column(Integer, Sequence("thumbnail_id_seq"), primary_key=True)
     file_id = Column(Integer, ForeignKey("files.id"), index=True)
     filename = Column(Text)
 
-    def __init__(self, filename=None, file_obj=None):
+    def __init__(self, filename=None, img_path=None, file_obj=None):
         self.filename = filename
         self.file = file_obj
-        if filename and file_obj:
+        self.img_path = img_path
-            self.save(self.filename)
+        if filename and file_obj and img_path:
+            self.save(self.filename, img_path)
 
-    def save(self, fname):
+    def save(self, fname, img_path):
         """
         Create file related thumbnail, add it to the file object.
         """
-        new_name = mk_paths(fname)
+        new_name = mk_paths(fname, img_path)
         ext = os.path.splitext(self.filename)[1]
         if ext:
             new_name.append("".join([new_name.pop(), ext]))
@@ -170,8 +145,8 @@ class Thumbnail(Base):
         name, ext = os.path.splitext(new_name.pop())
         new_name.append("".join([name, "_t", ext]))
         self.filename = os.path.sep.join(new_name)
-        if not os.path.exists(os.path.join(IMG_PATH, *new_name)):
+        if not os.path.exists(os.path.join(img_path, *new_name)):
-            shutil.move(thumb, os.path.join(IMG_PATH, *new_name))
+            shutil.move(thumb, os.path.join(img_path, *new_name))
         else:
             LOG.info("Thumbnail already exists (%s: %s)" % \
                     (fname, "/".join(new_name)))
@@ -182,34 +157,36 @@ class Thumbnail(Base):
 
 
 class Image(Base):
+    """Images and their thumbnails"""
     __tablename__ = "images"
     id = Column(Integer, Sequence("images_id_seq"), primary_key=True)
     file_id = Column(Integer, ForeignKey("files.id"), index=True)
     filename = Column(Text)
 
-    def __init__(self, filename=None, file_obj=None, move=True):
+    def __init__(self, filename=None, img_path=None, file_obj=None, move=True):
         self.filename = None
         self.file = file_obj
-        if filename:
+        self.img_path = img_path
+        if filename and img_path:
             self.filename = filename
-            self.save(filename, move)
+            self.save(filename, img_path, move)
 
-    def save(self, fname, move=True):
+    def save(self, fname, img_path, move=True):
         """
         Save and create coressponding thumbnail (note: it differs from file
         related thumbnail!)
         """
-        new_name = mk_paths(fname)
+        new_name = mk_paths(fname, img_path)
         ext = os.path.splitext(self.filename)[1]
 
         if ext:
             new_name.append("".join([new_name.pop(), ext]))
 
-        if not os.path.exists(os.path.join(IMG_PATH, *new_name)):
+        if not os.path.exists(os.path.join(img_path, *new_name)):
             if move:
-                shutil.move(self.filename, os.path.join(IMG_PATH, *new_name))
+                shutil.move(self.filename, os.path.join(img_path, *new_name))
             else:
-                shutil.copy(self.filename, os.path.join(IMG_PATH, *new_name))
+                shutil.copy(self.filename, os.path.join(img_path, *new_name))
         else:
             LOG.warning("Image with same CRC already exists "
                         "('%s', '%s')" % (self.filename, "/".join(new_name)))
@@ -219,9 +196,9 @@ class Image(Base):
         name, ext = os.path.splitext(new_name.pop())
         new_name.append("".join([name, "_t", ext]))
 
-        if not os.path.exists(os.path.join(IMG_PATH, *new_name)):
+        if not os.path.exists(os.path.join(img_path, *new_name)):
-            thumb = ThumbCreator(os.path.join(IMG_PATH, self.filename))
+            thumb = ThumbCreator(os.path.join(img_path, self.filename))
-            shutil.move(thumb.generate(), os.path.join(IMG_PATH, *new_name))
+            shutil.move(thumb.generate(), os.path.join(img_path, *new_name))
         else:
             LOG.info("Thumbnail already generated %s" % "/".join(new_name))
 
@@ -241,20 +218,21 @@ class Image(Base):
         """
         path, fname = os.path.split(self.filename)
         base, ext = os.path.splitext(fname)
-        return os.path.join(IMG_PATH, path, base + "_t" + ext)
+        return os.path.join(self.img_path, path, base + "_t" + ext)
 
     @property
     def imagepath(self):
         """
         Return full path to image
         """
-        return os.path.join(IMG_PATH, self.filename)
+        return os.path.join(self.img_path, self.filename)
 
     def __repr__(self):
         return "<Image('%s', %s)>" % (str(self.filename), str(self.id))
 
 
 class Exif(Base):
+    """Selected EXIF information"""
     __tablename__ = "exif"
     id = Column(Integer, Sequence("exif_id_seq"), primary_key=True)
     file_id = Column(Integer, ForeignKey("files.id"), index=True)
@@ -292,6 +270,7 @@ class Exif(Base):
 
 
 class Gthumb(Base):
+    """Gthumb information"""
     __tablename__ = "gthumb"
     id = Column(Integer, Sequence("gthumb_id_seq"), primary_key=True)
     file_id = Column(Integer, ForeignKey("files.id"), index=True)
@@ -307,3 +286,18 @@ class Gthumb(Base):
     def __repr__(self):
         return "<Gthumb('%s', '%s', %s)>" % (str(self.date), str(self.place),
                                              str(self.id))
+
+
+class Config(Base):
+    """Per-database configuration"""
+    __tablename__ = "config"
+    id = Column(Integer, Sequence("config_id_seq"), primary_key=True)
+    key = Column(Text)
+    value = Column(Text)
+
+    def __init__(self, key=None, value=None):
+        self.key = key
+        self.value = value
+
+    def __repr__(self):
+        return "<Config('%s', '%s')>" % (str(self.key), str(self.value))

pygtktalog/misc.py

@@ -5,6 +5,15 @@
     Author: Roman 'gryf' Dobosz, gryf73@gmail.com
     Created: 2009-04-05
 """
+import os
+import errno
+from zlib import crc32
+
+import pygtktalog.dbcommon
+from pygtktalog.logger import get_logger
+
+LOG = get_logger(__name__)
+
 
 def float_to_string(float_length):
     """
@@ -20,3 +29,47 @@ def float_to_string(float_length):
     sec = int(float_length)
     return "%02d:%02d:%02d" % (hour, minutes, sec)
 
+def calculate_image_path(dbpath=None, create=False):
+    """Calculate image path out of provided path or using current connection"""
+    if not dbpath:
+        dbpath = pygtktalog.dbcommon.DbFilename
+        if dbpath == ":memory:":
+            raise OSError("Cannot create image path out of in-memory db!")
+
+        dir_, file_ = (os.path.dirname(dbpath), os.path.basename(dbpath))
+        file_base, dummy = os.path.splitext(file_)
+        images_dir = os.path.join(dir_, file_base + "_images")
+    else:
+        if dbpath and "~" in dbpath:
+            dbpath = os.path.expanduser(dbpath)
+        if dbpath and "$" in dbpath:
+            dbpath = os.path.expandvars(dbpath)
+        images_dir = dbpath
+
+    if create:
+        if not os.path.exists(images_dir):
+            try:
+                os.mkdir(images_dir)
+            except OSError, err:
+                if err.errno != errno.EEXIST:
+                    raise
+    elif not os.path.exists(images_dir):
+        raise OSError("%s: No such directory" % images_dir)
+
+    return os.path.abspath(images_dir)
+
+def mk_paths(fname, img_path):
+    """Make path for provided pathname by calculating crc32 out of file"""
+    with open(fname) as fobj:
+        new_path = "%x" % (crc32(fobj.read(10*1024*1024)) & 0xffffffff)
+
+    new_path = [new_path[i:i + 2] for i in range(0, len(new_path), 2)]
+    full_path = os.path.join(img_path, *new_path[:-1])
+
+    try:
+        os.makedirs(full_path)
+    except OSError as exc:
+        if exc.errno != errno.EEXIST:
+            LOG.debug("Directory %s already exists." % full_path)
+
+    return new_path

pygtktalog/scan.py

@@ -11,7 +11,8 @@ import re
 from datetime import datetime
 import mimetypes
 
-from pygtktalog.dbobjects import File, Image, Thumbnail, TYPE
+import pygtktalog.misc
+from pygtktalog.dbobjects import File, Image, Thumbnail, Config, TYPE
 from pygtktalog.dbcommon import Session
 from pygtktalog.logger import get_logger
 from pygtktalog.video import Video
@@ -44,7 +45,7 @@ class Scan(object):
         """
         Initialize
         @Arguments:
-            @path - string with initial root directory to scan
+            @path - string with path to be added to topmost node (root)
         """
         self.abort = False
         self.path = path.rstrip(os.path.sep)
@@ -55,6 +56,8 @@ class Scan(object):
         self.files_count = self._get_files_count()
         self.current_count = 0
 
+        self._set_image_path()
+
     def add_files(self, engine=None):
         """
         Returns list, which contain object, modification date and file
@@ -106,15 +109,21 @@ class Scan(object):
         req(row)
 
         sql = SQL2 % ",".join("?" * len(all_ids))
-        res = engine.execute(sql, tuple(all_ids)).fetchall()
+        all_ids = [row_[0] for row_ in engine
+                   .execute(sql, tuple(all_ids))
+                   .fetchall()]
 
         all_obj = []
-        for row in res:
+        # number of objects to retrieve at once. Limit is 999. Let's do a
-            all_obj.append(self._session
+        # little bit below.
+        no = 900
+        steps = len(all_ids) / no + 1
+        for step in range(steps):
+            all_obj.extend(self._session
                            .query(File)
-                           .filter(File.id == row[0])
+                           .filter(File.id
-                           .first())
+                                   .in_(all_ids[step * no:step * no + no]))
-
+                           .all())
         return all_obj
 
     def update_files(self, node_id, engine=None):
@@ -248,7 +257,7 @@ class Scan(object):
 
         preview_fn = vid.capture()
         if preview_fn:
-            Image(preview_fn, fobj)
+            Image(preview_fn, self.img_path, fobj)
 
     def _check_related(self, fobj, pattern):
         """
@@ -261,10 +270,10 @@ class Scan(object):
                 full_fname = os.path.join(fobj.filepath, filen)
                 LOG.debug('found cover file: %s' % full_fname)
 
-                Image(full_fname, fobj, False)
+                Image(full_fname, self.img_path, fobj, False)
 
                 if not fobj.thumbnail:
-                    Thumbnail(full_fname, fobj)
+                    Thumbnail(full_fname, self.img_path, fobj)
 
     def _name_matcher(self, fpath, fname, media=False):
         """
@@ -326,7 +335,7 @@ class Scan(object):
             fobj.type = fob['ftype']
         else:
             fobj = File(**fob)
-            # SLOW. Don;t do this. Checksums has no value eventually
+            # SLOW. Don't do this. Checksums has no value eventually
             # fobj.mk_checksum()
 
         if parent is None:
@@ -482,12 +491,24 @@ class Scan(object):
         return None
 
     def _get_files_count(self):
+        """return size in bytes"""
         count = 0
-        for root, dirs, files in os.walk(str(self.path)):
+        for _, _, files in os.walk(str(self.path)):
             count += len(files)
         LOG.debug("count of files: %s", count)
         return count
 
+    def _set_image_path(self):
+        """Get or calculate the images path"""
+        image_path = self._session.query(Config) \
+                .filter(Config.key=="image_path").one()
+        if image_path.value == ":same_as_db:":
+            image_path = pygtktalog.misc.calculate_image_path()
+        else:
+            image_path = pygtktalog.misc.calculate_image_path(image_path.value)
+
+        self.img_path = image_path
+
 
 class asdScan(object):
     """
@@ -561,7 +582,7 @@ class asdScan(object):
                 current_dir = os.path.join(root, i)
 
                 try:
-                    st = os.stat(current_dir)
+                    st = os.lstat(current_dir)
                     st_mtime = st.st_mtime
                 except OSError:
                     st_mtime = 0
@@ -597,7 +618,7 @@ class asdScan(object):
                 current_file = os.path.join(root, i)
 
                 try:
-                    st = os.stat(current_file)
+                    st = os.lstat(current_file)
                     st_mtime = st.st_mtime
                     st_size = st.st_size
                 except OSError:

pygtktalog/thumbnail.py

@@ -8,6 +8,7 @@
 
 import os
 from tempfile import mkstemp
+import shutil
 
 from PIL import Image
 
@@ -56,14 +57,17 @@ class ThumbCreator(object):
             thumb.close()
         else:
             LOG.debug("no exif thumb")
-            thumb = self._scale_image()
+            if self.is_image_smaller():
-            if thumb:
+                shutil.copyfile(self.filename, thumb_fn)
-                thumb.save(thumb_fn, "JPEG")
+            else:
+                thumb = self._scale_image()
+                if thumb:
+                    thumb.save(thumb_fn, "JPEG")
 
         if exif and 'Image Orientation' in exif:
             orient = exif['Image Orientation'].values[0]
             if orient > 1 and orient in orientations:
-                thumb_image = Image.open(self.thumb_fn)
+                thumb_image = Image.open(thumb_fn)
                 tmp_thumb_img = thumb_image.transpose(orientations[orient])
 
                 if orient in flips:
@@ -73,6 +77,13 @@ class ThumbCreator(object):
 
         return thumb_fn
 
+    def is_image_smaller(self):
+        """Check if image is smaller than desired dimention, return boolean"""
+        image = Image.open(self.filename)
+        im_x, im_y = image.size
+        image.close()
+        return im_x <= self.thumb_x and im_y <= self.thumb_y
+
     def _get_exif(self):
         """
         Get exif (if available), return as a dict