Working first attempt for updating files in scan object.

gtktalog.py

@@ -0,0 +1,54 @@
+#!/usr/bin/env python
+"""
+    Project: pyGTKtalog
+    Description: Application main launch file.
+    Type: core
+    Author: Roman 'gryf' Dobosz, gryf73@gmail.com
+    Created: 2007-05-01
+"""
+import sys
+
+import gtk
+import pygtk
+pygtk.require("2.0")
+
+import gtkmvc
+gtkmvc.require("1.99.0")
+
+from pygtktalog.models.main import MainModel
+from pygtktalog.controllers.main import MainController
+from pygtktalog.views.main import MainView
+from pygtktalog.logger import get_logger
+
+
+LOG = get_logger('__main__')
+
+
+def run(*args):
+    """Create model, controller and view and launch it."""
+    model = MainModel()
+    if args:
+        LOG.info("args %s", str(args))
+        if not model.open(args[0][1]):
+            LOG.warn("file couldn't be open")
+            sys.exit()
+    #else:
+    #    model.new()
+    view = MainView()
+    MainController(model, view)
+
+    try:
+        gtk.main()
+    except KeyboardInterrupt:
+        #model.config.save()
+        LOG.exception("gtktalog.py: model.cleanup()")
+        model.cleanup()
+        gtk.main_quit
+
+
+if __name__ == "__main__":
+    if len(sys.argv) > 1:
+        run(sys.argv)
+    else:
+        run()
+

pavement.py

@@ -1,7 +1,7 @@
 """
     Project: pyGTKtalog
     Description: Makefile and setup.py replacement. Used python packages -
-    paver, nosetests. External commands - xgettext, intltool-extract, svn,
+    paver, nosetests. External commands - xgettext, intltool-extract, hg,
     grep.
     Type: management
     Author: Roman 'gryf' Dobosz, gryf73@gmail.com
@@ -37,7 +37,7 @@ msgstr ""
 "Content-Transfer-Encoding: utf-8\\n"
 """
 
-REV = os.popen("svn info 2>/dev/null|grep ^Revis|cut -d ' ' -f 2").readlines()
+REV = os.popen("hg sum 2>/dev/null|grep ^Revis|cut -d ' ' -f 2").readlines()
 if REV:
     REV = "r" + REV[0].strip()
 else:
@@ -77,7 +77,7 @@ setup(
       exclude_package_data={'': ['*.patch']},
       packages=["pygtktalog"],
       scripts=['bin/gtktalog.py'],
-      test_suite = 'nose.collector'
+      test_suite='nose.collector'
 )
 
 options(sphinx=Bunch(builddir="build", sourcedir="source"))
@@ -89,6 +89,7 @@ def sdist():
     """sdist with message catalogs"""
     call_task("setuptools.command.sdist")
 
+
 @task
 @needs(['locale_gen'])
 def build():
@@ -103,11 +104,13 @@ def clean():
     for root, dummy, files in os.walk("."):
         for fname in files:
             if fname.endswith(".pyc") or fname.endswith(".pyo") or \
-               fname.endswith("~") or fname.endswith(".h"):
+                    fname.endswith("~") or fname.endswith(".h") or \
+                    fname == '.coverage':
                 fdel = os.path.join(root, fname)
                 os.unlink(fdel)
                 print "deleted", fdel
 
+
 @task
 @needs(["clean"])
 def distclean():
@@ -123,6 +126,7 @@ def distclean():
             os.unlink(filename)
             print "deleted", filename
 
+
 @task
 def run():
     """run application"""
@@ -130,6 +134,7 @@ def run():
     #import gtktalog
     #gtktalog.run()
 
+
 @task
 def pot():
     """generate 'pot' file out of python/glade files"""
@@ -150,7 +155,8 @@ def pot():
                 sh(cmd % (POTFILE, os.path.join(root, fname)))
             elif fname.endswith(".glade"):
                 sh(cmd_glade % os.path.join(root, fname))
-                sh(cmd % (POTFILE, os.path.join(root, fname+".h")))
+                sh(cmd % (POTFILE, os.path.join(root, fname + ".h")))
+
 
 @task
 @needs(['pot'])
@@ -165,6 +171,7 @@ def locale_merge():
         else:
             shutil.copy(potfile, msg_catalog)
 
+
 @task
 @needs(['locale_merge'])
 def locale_gen():
@@ -183,6 +190,7 @@ def locale_gen():
         msg_catalog = os.path.join('locale', "%s.po" % lang)
         sh('msgfmt %s -o %s' % (msg_catalog, catalog_file))
 
+
 if HAVE_LINT:
     @task
     def pylint():
@@ -190,6 +198,7 @@ if HAVE_LINT:
         pylintopts = ['pygtktalog']
         dry('pylint %s' % (" ".join(pylintopts)), lint.Run, pylintopts)
 
+
 @task
 @cmdopts([('coverage', 'c', 'display coverage information')])
 def test(options):
@@ -199,6 +208,7 @@ def test(options):
         cmd += " --with-coverage --cover-package pygtktalog"
     os.system(cmd)
 
+
 @task
 @needs(['locale_gen'])
 def runpl():
@@ -216,4 +226,3 @@ def _setup_env():
         sys.path.insert(0, this_path)
 
     return this_path
-

project.vim

@@ -0,0 +1,64 @@
+"All your bases are belong to us."
+"
+" Author: Roman.Dobosz at gmail.com
+" Date: 2011-12-09 12:11:00
+
+if !has("python")
+  finish
+endif
+
+let g:project_dir = expand("%:p:h")
+
+python << EOF
+import os
+import vim
+
+PROJECT_DIR = vim.eval('project_dir')
+TAGS_FILE = os.path.join(PROJECT_DIR, "tags")
+
+if not PROJECT_DIR.endswith("/"):
+    PROJECT_DIR += "/"
+PYFILES= []
+
+if os.path.exists(PROJECT_DIR + "tmp"):
+    os.system('rm -fr ' + PROJECT_DIR + "tmp")
+
+## icard specific
+#for dir_ in os.listdir(os.path.join(PROJECT_DIR, "..", "externals")):
+#    if dir_ != 'mako':
+#        PYFILES.append(dir_)
+
+vim.command("set tags+=" + TAGS_FILE)
+
+# make all directories accessible by gf command
+def req(path):
+    root, dirs, files = os.walk(path).next()
+    for dir_ in dirs:
+        newroot = os.path.join(root, dir_)
+        # all but the dot dirs
+        if dir_ in (".svn", ".hg", "locale", "tmp"):
+            continue
+        if "static" in root and dir_ != "js":
+            continue
+
+        vim.command("set path+=" + newroot)
+        req(newroot)
+
+req(PROJECT_DIR)
+
+# generate tags
+def update_tags(path):
+    assert os.path.exists(path)
+
+    pylib_path = os.path.normpath(path)
+    pylib_path += " " + os.path.normpath('/usr/lib/python2.7/site-packages')
+
+    # find tags for all files
+    cmd = 'ctags -R --python-kinds=-i'
+    cmd += ' -f ' + TAGS_FILE + ' ' + pylib_path
+    print cmd
+    os.system(cmd)
+EOF
+
+"
+command UpdateTags python update_tags(PROJECT_DIR)

pygtktalog/__init__.py

@@ -14,12 +14,15 @@ __web__ = "http://bitbucket.org/gryf"
 __logo_img__ = "views/pixmaps/Giant Worms.png"
 
 import os
+import sys
 import locale
 import gettext
 import __builtin__
 
 import gtk.glade
 
+from logger import get_logger
+
 
 __all__ = ['controllers',
            'models',
@@ -54,3 +57,9 @@ for module in gtk.glade, gettext:
 
 # register the gettext function for the whole interpreter as "_"
 __builtin__._ = gettext.gettext
+
+# wrap errors into usefull message
+def log_exception(exc_type, exc_val, traceback):
+    get_logger(__name__).error(exc_val)
+
+sys.excepthook = log_exception

pygtktalog/dbobjects.py

@@ -8,22 +8,46 @@
 import os
 import errno
 import shutil
-import uuid
+from hashlib import sha256
+from zlib import crc32
 
 from sqlalchemy import Column, Table, Integer, Text
 from sqlalchemy import DateTime, ForeignKey, Sequence
 from sqlalchemy.orm import relation, backref
 
 from pygtktalog.dbcommon import Base
-from pygtktalog import thumbnail
+from pygtktalog.thumbnail import ThumbCreator
+from pygtktalog.logger import get_logger
 
 
-IMG_PATH = "/home/gryf/.pygtktalog/imgs/"  # FIXME: should be configurable
+LOG = get_logger(__name__)
+
+IMG_PATH = "/home/gryf/.pygtktalog/imgs2/"  # FIXME: should be configurable
 
 tags_files = Table("tags_files", Base.metadata,
                    Column("file_id", Integer, ForeignKey("files.id")),
                    Column("tag_id", Integer, ForeignKey("tags.id")))
 
+TYPE = {'root': 0, 'dir': 1, 'file': 2, 'link': 3}
+
+
+def mk_paths(fname):
+    #new_name = str(uuid.uuid1()).split("-")
+    fd = open(fname)
+    new_path = "%x" % (crc32(fd.read(10*1024*1024)) & 0xffffffff)
+    fd.close()
+
+    new_path = [new_path[i:i + 2] for i in range(0, len(new_path), 2)]
+    full_path = os.path.join(IMG_PATH, *new_path[:-1])
+
+    try:
+        os.makedirs(full_path)
+    except OSError as exc:
+        if exc.errno != errno.EEXIST:
+            LOG.debug("Directory %s already exists." % full_path)
+
+    return new_path
+
 
 class File(Base):
     __tablename__ = "files"
@@ -37,6 +61,7 @@ class File(Base):
     source = Column(Integer)
     note = Column(Text)
     description = Column(Text)
+    checksum = Column(Text)
 
     children = relation('File',
                         backref=backref('parent', remote_side="File.id"),
@@ -58,6 +83,35 @@ class File(Base):
     def __repr__(self):
         return "<File('%s', %s)>" % (str(self.filename), str(self.id))
 
+    def get_all_children(self):
+        """
+        Return list of all node direct and indirect children
+        """
+        def _recursive(node):
+            children = []
+            if node.children:
+                for child in node.children:
+                    children += _recursive(child)
+            if node != self:
+                children.append(node)
+
+            return children
+
+        if self.children:
+            return _recursive(self)
+        else:
+            return []
+
+    def mk_checksum(self):
+        if not (self.filename and self.filepath):
+            return
+
+        full_name = os.path.join(self.filepath, self.filename)
+
+        if os.path.isfile(full_name):
+            fd = open(full_name)
+            self.checksum = sha256(fd.read(10*1024*1024)).hexdigest()
+            fd.close()
 
 class Group(Base):
     __tablename__ = "groups"
@@ -99,29 +153,28 @@ class Thumbnail(Base):
     def __init__(self, filename=None, file_obj=None):
         self.filename = filename
         self.file = file_obj
-        if self.filename:
+        if filename and file_obj:
             self.save(self.filename)
 
     def save(self, fname):
         """
         Create file related thumbnail, add it to the file object.
         """
-        new_name = str(uuid.uuid1()).split("-")
-        try:
-            os.makedirs(os.path.join(IMG_PATH, *new_name[:-1]))
-        except OSError as exc:
-            if exc.errno != errno.EEXIST:
-                raise
-
+        new_name = mk_paths(fname)
         ext = os.path.splitext(self.filename)[1]
         if ext:
             new_name.append("".join([new_name.pop(), ext]))
 
-        thumb = thumbnail.Thumbnail(self.filename).save()
+        thumb = ThumbCreator(self.filename).generate()
         name, ext = os.path.splitext(new_name.pop())
         new_name.append("".join([name, "_t", ext]))
         self.filename = os.path.sep.join(new_name)
-        shutil.move(thumb.save(), os.path.join(IMG_PATH, *new_name))
+        if not os.path.exists(os.path.join(IMG_PATH, *new_name)):
+            shutil.move(thumb, os.path.join(IMG_PATH, *new_name))
+        else:
+            LOG.info("Thumbnail already exists (%s: %s)" % \
+                    (fname, "/".join(new_name)))
+            os.unlink(thumb)
 
     def __repr__(self):
         return "<Thumbnail('%s', %s)>" % (str(self.filename), str(self.id))
@@ -133,37 +186,44 @@ class Image(Base):
     file_id = Column(Integer, ForeignKey("files.id"))
     filename = Column(Text)
 
-    def __init__(self, filename=None, file_obj=None):
+    def __init__(self, filename=None, file_obj=None, move=True):
         self.filename = None
         self.file = file_obj
         if filename:
             self.filename = filename
-            self.save(filename)
+            self.save(filename, move)
 
-    def save(self, fname):
+    def save(self, fname, move=True):
         """
         Save and create coressponding thumbnail (note: it differs from file
         related thumbnail!)
         """
-        new_name = str(uuid.uuid1()).split("-")
-        try:
-            os.makedirs(os.path.join(IMG_PATH, *new_name[:-1]))
-        except OSError as exc:
-            if exc.errno != errno.EEXIST:
-                raise
-
+        new_name = mk_paths(fname)
         ext = os.path.splitext(self.filename)[1]
+
         if ext:
             new_name.append("".join([new_name.pop(), ext]))
 
+        if not os.path.exists(os.path.join(IMG_PATH, *new_name)):
+            if move:
                 shutil.move(self.filename, os.path.join(IMG_PATH, *new_name))
+            else:
+                shutil.copy(self.filename, os.path.join(IMG_PATH, *new_name))
+        else:
+            LOG.warning("Image with same CRC already exists "
+                        "('%s', '%s')" % (self.filename, "/".join(new_name)))
 
         self.filename = os.path.sep.join(new_name)
 
-        thumb = thumbnail.Thumbnail(os.path.join(IMG_PATH, self.filename))
         name, ext = os.path.splitext(new_name.pop())
         new_name.append("".join([name, "_t", ext]))
-        shutil.move(thumb.save(), os.path.join(IMG_PATH, *new_name))
+
+        if not os.path.exists(os.path.join(IMG_PATH, *new_name)):
+            thumb = ThumbCreator(os.path.join(IMG_PATH, self.filename))
+            shutil.move(thumb.generate(), os.path.join(IMG_PATH, *new_name))
+        else:
+            LOG.info("Thumbnail already generated %s" % "/".join(new_name))
+
 
     def get_copy(self):
         """

pygtktalog/logger.py

@@ -9,32 +9,27 @@ import os
 import sys
 import logging
 
+LEVEL = {'DEBUG': logging.DEBUG,
+         'INFO': logging.INFO,
+         'WARN': logging.WARN,
+         'ERROR': logging.ERROR,
+         'CRITICAL': logging.CRITICAL}
+
 BLACK, RED, GREEN, YELLOW, BLUE, MAGENTA, CYAN, WHITE = range(8)
 
-# The background is set with 40 plus the number of the color, and the
-# foreground with 30
-
-#These are the sequences need to get colored ouput
 RESET_SEQ = "\033[0m"
 COLOR_SEQ = "\033[1;%dm"
 BOLD_SEQ = "\033[1m"
 
-def formatter_message(message, use_color = True):
-    if use_color:
-        message = message.replace("$RESET", RESET_SEQ).replace("$BOLD",
-                                                               BOLD_SEQ)
-    else:
-        message = message.replace("$RESET", "").replace("$BOLD", "")
-    return message
-
 COLORS = {'WARNING': YELLOW,
           'INFO': GREEN,
           'DEBUG': BLUE,
           'CRITICAL': WHITE,
           'ERROR': RED}
 
+
 class ColoredFormatter(logging.Formatter):
-    def __init__(self, msg, use_color = True):
+    def __init__(self, msg, use_color=True):
         logging.Formatter.__init__(self, msg)
         self.use_color = use_color
 
@@ -45,45 +40,43 @@ class ColoredFormatter(logging.Formatter):
                     + levelname + RESET_SEQ
             record.levelname = levelname_color
         return logging.Formatter.format(self, record)
-LEVEL = {'DEBUG': logging.DEBUG,
-         'INFO': logging.INFO,
-         'WARN': logging.WARN,
-         'ERROR': logging.ERROR,
-         'CRITICAL': logging.CRITICAL}
 
-#def get_logger(module_name, level=None, to_file=True):
-def get_logger(module_name, level=None, to_file=False):
+
+#def get_logger(module_name, level='INFO', to_file=False):
+def get_logger(module_name, level='DEBUG', to_file=True):
+#def get_logger(module_name, level='INFO', to_file=True):
+#def get_logger(module_name, level='DEBUG', to_file=False):
     """
     Prepare and return log object. Standard formatting is used for all logs.
     Arguments:
         @module_name - String name for Logger object.
         @level - Log level (as string), one of DEBUG, INFO, WARN, ERROR and
                  CRITICAL.
-        @to_file - If True, stores log in file inside .pygtktalog config
-                   directory, otherwise log is redirected to stderr.
+        @to_file - If True, additionally stores full log in file inside
+                   .pygtktalog config directory and to stderr, otherwise log
+                   is only redirected to stderr.
     Returns: object of logging.Logger class
     """
 
     path = os.path.join(os.path.expanduser("~"), ".pygtktalog", "app.log")
-    path = "/dev/null"
+    #path = "/dev/null"
     log = logging.getLogger(module_name)
-
-    if not level:
-        #log.setLevel(LEVEL['WARN'])
-        log.setLevel(LEVEL['DEBUG'])
-    else:
     log.setLevel(LEVEL[level])
 
+    console_handler = logging.StreamHandler(sys.stderr)
+    console_formatter = ColoredFormatter("%(filename)s:%(lineno)s - "
+                                  "%(levelname)s - %(message)s")
+    console_handler.setFormatter(console_formatter)
+
+    log.addHandler(console_handler)
+
     if to_file:
-        log_handler = logging.FileHandler(path)
-        formatter = logging.Formatter("%(asctime)s %(filename)s:%(lineno)s - "
-                                      "%(levelname)s - %(message)s")
-    else:
-        log_handler = logging.StreamHandler(sys.stderr)
-        formatter = ColoredFormatter("%(filename)s:%(lineno)s - "
-                                      "%(levelname)s - %(message)s")
+        file_handler = logging.FileHandler(path)
+        file_formatter = logging.Formatter("%(asctime)s %(levelname)6s "
+                                           "%(filename)s: %(lineno)s - "
+                                           "%(message)s")
+        file_handler.setFormatter(file_formatter)
+        file_handler.setLevel(LEVEL[level])
+        log.addHandler(file_handler)
 
-    log_handler.setFormatter(formatter)
-    log.addHandler(log_handler)
     return log
-

pygtktalog/scan.py

@@ -7,16 +7,29 @@
 """
 import os
 import sys
+import re
 from datetime import datetime
 import mimetypes
 
-from pygtktalog.dbobjects import File, Image
+from pygtktalog.dbobjects import File, Image, Thumbnail, TYPE
 from pygtktalog.dbcommon import Session
 from pygtktalog.logger import get_logger
 from pygtktalog.video import Video
 
 
 LOG = get_logger(__name__)
+PAT = re.compile("(\[[^\]]*\]"
+                 ".*\(\d\d\d\d\))"
+                 "\s[^\[]*\[.{8}\]"
+                 ".[a-zA-Z0-9]*$")
+
+#PAT = re.compile(r'(?P<group>\[[^\]]*\]\s)?'
+#                 r'(?P<title>.*)\s'
+#                 r'(?P<year>\(\d{4}\))\s'
+#                 r'(?P<kind>.*)'
+#                 r'(?P<checksum>\[[A-Z0-9]{8}\])'
+#                 r'\.(?P<extension>(avi|asf|mpeg|mpg|mp4|ogm|ogv|mkv|mov|wmv'
+#                 r'|rm|rmvb|flv|jpg|png|gif|nfo))\.?(conf)?$')
 
 
 class NoAccessError(Exception):
@@ -36,8 +49,11 @@ class Scan(object):
         self.abort = False
         self.path = path.rstrip(os.path.sep)
         self._files = []
-        self._existing_files = []
+        self._existing_files = []  # for re-use purpose in adding
+        self._existing_branch = []  # for branch storage, mainly for updating
         self._session = Session()
+        self.files_count = self._get_files_count()
+        self.current_count = 0
 
     def add_files(self):
         """
@@ -45,6 +61,7 @@ class Scan(object):
         size.
         """
         self._files = []
+        self._existing_branch = []
         LOG.debug("given path: %s" % self.path)
 
         # See, if file exists. If not it would raise OSError exception
@@ -56,7 +73,8 @@ class Scan(object):
 
         directory = os.path.basename(self.path)
         path = os.path.dirname(self.path)
-        if not self._recursive(None, directory, path, 0, 0, 1):
+
+        if not self._recursive(None, directory, path, 0):
             return None
 
         # add only first item from _files, because it is a root of the other,
@@ -65,6 +83,55 @@ class Scan(object):
         self._session.commit()
         return self._files
 
+    def update_files(self, node_id):
+        """
+        Updtate DB contents of provided node.
+        """
+        self.current_count = 0
+        old_node = self._session.query(File).get(node_id)
+        if old_node is None:
+            LOG.warning("No such object in db: %s", node_id)
+            return
+        parent = old_node.parent
+
+        self._files = []
+        self._existing_branch = old_node.get_all_children()
+        self._existing_branch.insert(0, old_node)
+
+        # Break the chain of parent-children relations
+        for fobj in self._existing_branch:
+            fobj.parent = None
+
+        update_path = os.path.join(old_node.filepath, old_node.filename)
+
+        # refresh objects
+        self._get_all_files()
+
+        LOG.debug("path for update: %s" % update_path)
+
+        # See, if file exists. If not it would raise OSError exception
+        os.stat(update_path)
+
+        if not os.access(update_path, os.R_OK | os.X_OK) \
+                or not os.path.isdir(update_path):
+            LOG.error("Access to %s is forbidden" % update_path)
+            raise NoAccessError("Access to %s is forbidden" % update_path)
+
+        directory = os.path.basename(update_path)
+        path = os.path.dirname(update_path)
+
+        if not self._recursive(parent, directory, path, 0):
+            return None
+
+        # update branch
+        #self._session.merge(self._files[0])
+        LOG.debug("Deleting objects whitout parent: %s" % \
+                str(self._session.query(File).filter(File.parent==None).all()))
+        self._session.query(File).filter(File.parent==None).delete()
+
+        self._session.commit()
+        return self._files
+
     def _get_dirsize(self, path):
         """
         Returns sum of all files under specified path (also in subdirs)
@@ -77,8 +144,8 @@ class Scan(object):
                 try:
                     size += os.stat(os.path.join(root, fname)).st_size
                 except OSError:
-                    LOG.info("Cannot access file %s" % \
-                            os.path.join(root, fname))
+                    LOG.warning("Cannot access file "
+                                "%s" % os.path.join(root, fname))
 
         return size
 
@@ -89,14 +156,26 @@ class Scan(object):
         mimedict = {'audio': self._audio,
                     'video': self._video,
                     'image': self._image}
+        extdict = {'.mkv': 'video',  # TODO: move this to config/plugin(?)
+                   '.rmvb': 'video',
+                   '.ogm': 'video',
+                   '.ogv': 'video'}
+
         fp = os.path.join(fobj.filepath.encode(sys.getfilesystemencoding()),
                           fobj.filename.encode(sys.getfilesystemencoding()))
 
         mimeinfo = mimetypes.guess_type(fp)
-        if mimeinfo[0] and mimeinfo[0].split("/")[0] in mimedict.keys():
-            mimedict[mimeinfo[0].split("/")[0]](fobj, fp)
+        if mimeinfo[0]:
+            mimeinfo = mimeinfo[0].split("/")[0]
+
+        ext = os.path.splitext(fp)[1]
+
+        if mimeinfo and mimeinfo in mimedict.keys():
+            mimedict[mimeinfo](fobj, fp)
+        elif ext and ext in extdict:
+            mimedict[extdict[ext]](fobj, fp)
         else:
-            #LOG.info("Filetype not supported " + str(mimeinfo) + " " +  fp)
+            LOG.debug("Filetype not supported " + str(mimeinfo) + " " +  fp)
             pass
 
     def _audio(self, fobj, filepath):
@@ -111,15 +190,61 @@ class Scan(object):
         """
         Make captures for a movie. Save it under uniq name.
         """
+        result = PAT.search(fobj.filename)
+        if result:
+            self._check_related(fobj, result.groups()[0])
+
         vid = Video(filepath)
 
+        fobj.description = vid.get_formatted_tags()
+
         preview_fn = vid.capture()
+        if preview_fn:
             Image(preview_fn, fobj)
 
+    def _check_related(self, fobj, pattern):
+        """
+        Try to search for related files which belongs to specified File
+        object and pattern. If found, additional objects are created.
+        """
+        for filen in os.listdir(fobj.filepath):
+            if pattern in filen and \
+                    os.path.splitext(filen)[1] in (".jpg", ".png", ".gif"):
+                full_fname = os.path.join(fobj.filepath, filen)
+                LOG.debug('found cover file: %s' % full_fname)
+
+                Image(full_fname, fobj, False)
+
+                if not fobj.thumbnail:
+                    Thumbnail(full_fname, fobj)
+
+    def _name_matcher(self, fpath, fname, media=False):
+        """
+        Try to match special pattern to filename which may be looks like this:
+            [aXXo] Batman (1989) [D3ADBEEF].avi
+            [aXXo] Batman (1989) [D3ADBEEF].avi.conf
+            [aXXo] Batman (1989) cover [BEEFD00D].jpg
+            [aXXo] Batman (1989) cover2 [FEEDD00D].jpg
+            [aXXo] Batman (1989) trailer [B00B1337].avi
+        or
+            Batman (1989) [D3ADBEEF].avi (and so on)
+
+        For media=False it will return True for filename, that matches
+        pattern, and there are at least one corresponding media files (avi,
+        mpg, mov and so on) _in case the filename differs from media_. This is
+        usfull for not storing covers, nfo, conf files in the db.
+
+        For kind == 2 it will return all images and other files that should be
+        gather due to video file examinig as a dict of list (conf, nfo and
+        images).
+        """
+        # TODO: dokonczyc to na podstawie tego cudowanego patternu u gory.
+        return
+
     def _get_all_files(self):
         self._existing_files = self._session.query(File).all()
 
-    def _mk_file(self, fname, path, parent):
+    def _mk_file(self, fname, path, parent, ftype=TYPE['file']):
         """
         Create and return File object
         """
@@ -127,19 +252,42 @@ class Scan(object):
 
         fname = fname.decode(sys.getfilesystemencoding())
         path = path.decode(sys.getfilesystemencoding())
-        fob = File(filename=fname, path=path)
-        fob.date = datetime.fromtimestamp(os.stat(fullpath).st_mtime)
-        fob.size = os.stat(fullpath).st_size
-        fob.parent = parent
-        fob.type = 2
+
+        if ftype == TYPE['link']:
+            fname = fname + " -> " + os.readlink(fullpath)
+
+        fob = {'filename': fname,
+               'path': path,
+               'ftype': ftype}
+        try:
+            fob['date'] = datetime.fromtimestamp(os.stat(fullpath).st_mtime)
+            fob['size'] = os.stat(fullpath).st_size
+        except OSError:
+            # in case of dead softlink, we will have no time and size
+            fob['date'] = None
+            fob['size'] = 0
+
+        fobj = self._get_old_file(fob, ftype)
+
+        if fobj:
+            LOG.debug("found existing file in db: %s" % str(fobj))
+            fobj.size = fob['size']  # TODO: update whole tree sizes (for directories/discs)
+            fobj.filepath = fob['path']
+            fobj.type = fob['ftype']
+        else:
+            fobj = File(**fob)
+            fobj.mk_checksum()
 
         if parent is None:
-            fob.parent_id = 1
+            fobj.parent_id = 1
+        else:
+            fobj.parent = parent
 
-        self._files.append(fob)
-        return fob
+        self._files.append(fobj)
 
-    def _recursive(self, parent, fname, path, date, size, ftype):
+        return fobj
+
+    def _recursive(self, parent, fname, path, size):
         """
         Do the walk through the file system
         @Arguments:
@@ -147,40 +295,58 @@ class Scan(object):
                       scope
             @fname - string that hold filename
             @path - full path for further scanning
-            @date -
             @size - size of the object
-            @ftype -
         """
         if self.abort:
             return False
 
-        LOG.debug("args: fname: %s, path: %s" % (fname, path))
         fullpath = os.path.join(path, fname)
 
-        parent = self._mk_file(fname, path, parent)
-        parent.size = self._get_dirsize(fullpath)
-        parent.type = 1
+        parent = self._mk_file(fname, path, parent, TYPE['dir'])
+
+        parent.size = self._get_dirsize(fullpath)
+        parent.type = TYPE['dir']
 
-        self._get_all_files()
         root, dirs, files = os.walk(fullpath).next()
         for fname in files:
             fpath = os.path.join(root, fname)
-            fob = self._mk_file(fname, root, parent)
+            self.current_count += 1
+            LOG.debug("Processing %s [%s/%s]", fname, self.current_count,
+                      self.files_count)
+
+            result = PAT.search(fname)
+            test_ = False
+
+            if result and os.path.splitext(fpath)[1] in ('.jpg', '.gif',
+                                                         '.png'):
+                newpat = result.groups()[0]
+                matching_files = []
+                for fn_ in os.listdir(root):
+                    if newpat in fn_:
+                        matching_files.append(fn_)
+
+                if len(matching_files) > 1:
+                    LOG.debug('found cover "%s" in group: %s, skipping', fname,
+                              str(matching_files))
+                    test_ = True
+            if test_:
+                continue
+
             if os.path.islink(fpath):
-                fob.filename = fob.filename + " -> " + os.readlink(fpath)
-                fob.type = 3
+                fob = self._mk_file(fname, root, parent, TYPE['link'])
             else:
+                fob = self._mk_file(fname, root, parent)
                 existing_obj = self._object_exists(fob)
+
                 if existing_obj:
-                    fob.tags = existing_obj.tags
-                    fob.thumbnail = [th.get_copy \
-                            for th in existing_obj.thumbnail]
-                    fob.images = [img.get_copy() \
-                            for img in existing_obj.images]
+                    existing_obj.parent = fob.parent
+                    fob = existing_obj
                 else:
-                    LOG.debug("gather information")
+                    LOG.debug("gather information for %s",
+                              os.path.join(root, fname))
                     self._gather_information(fob)
                 size += fob.size
+            if fob not in self._existing_files:
                 self._existing_files.append(fob)
 
         for dirname in dirs:
@@ -191,16 +357,36 @@ class Scan(object):
                 continue
 
             if os.path.islink(dirpath):
-                fob = self._mk_file(dirname, root, parent)
-                fob.filename = fob.filename + " -> " + os.readlink(dirpath)
-                fob.type = 3
+                fob = self._mk_file(dirname, root, parent, TYPE['link'])
             else:
-                LOG.debug("going into %s" % dirname)
-                self._recursive(parent, dirname, fullpath, date, size, ftype)
+                LOG.debug("going into %s" % os.path.join(root, dirname))
+                self._recursive(parent, dirname, fullpath, size)
 
         LOG.debug("size of items: %s" % parent.size)
         return True
 
+    def _get_old_file(self, fdict, ftype):
+        """
+        Search for object with provided data in dictionary in stored branch
+        (which is updating). Return such object on success, remove it from
+        list.
+        """
+        for index, obj in enumerate(self._existing_branch):
+            if ftype == TYPE['link'] and fdict['filename'] == obj.filename:
+                return self._existing_branch.pop(index)
+            elif fdict['filename'] == obj.filename and \
+                    fdict['date'] == obj.date and \
+                    ftype == TYPE['file'] and \
+                    fdict['size'] in (obj.size, 0):
+                obj = self._existing_branch.pop(index)
+                obj.size = fdict['size']
+                return obj
+            elif fdict['filename'] == obj.filename:
+                obj = self._existing_branch.pop(index)
+                obj.size = fdict['date']
+                return obj
+        return False
+
     def _object_exists(self, fobj):
         """
         Perform check if current File object already exists in collection. If
@@ -209,16 +395,24 @@ class Scan(object):
         for efobj in self._existing_files:
             if efobj.size == fobj.size \
                     and efobj.type == fobj.type \
-                    and efobj.date == fobj.date:
+                    and efobj.date == fobj.date \
+                    and efobj.filename == fobj.filename:
                 return efobj
         return None
 
+    def _get_files_count(self):
+        count = 0
+        for root, dirs, files in os.walk(self.path):
+            count += len(files)
+        LOG.debug("count of files: %s", count)
+        return count
+
+
 class asdScan(object):
     """
     Retrieve and identify all files recursively on given path
     """
     def __init__(self, path, tree_model):
-        LOG.debug("initialization")
         self.path = path
         self.abort = False
         self.label = None
@@ -232,7 +426,7 @@ class asdScan(object):
         self.busy = True
 
         # count files in directory tree
-        LOG.info("Calculating number of files in directory tree...")
+        LOG.debug("Calculating number of files in directory tree...")
 
         step = 0
         try:
@@ -276,7 +470,7 @@ class asdScan(object):
             try:
                 root, dirs, files = os.walk(path).next()
             except:
-                LOG.debug("cannot access ", path)
+                LOG.warning("Cannot access ", path)
                 return 0
 
             #############

pygtktalog/thumbnail.py

@@ -7,8 +7,6 @@
 """
 
 import os
-import sys
-import shutil
 from tempfile import mkstemp
 
 import Image
@@ -20,7 +18,7 @@ from pygtktalog import EXIF
 LOG = get_logger(__name__)
 
 
-class Thumbnail(object):
+class ThumbCreator(object):
     """
     Class for generate/extract thumbnail from image file
     """
@@ -30,7 +28,7 @@ class Thumbnail(object):
         self.thumb_y = 160
         self.filename = filename
 
-    def save(self):
+    def generate(self):
         """
         Save thumbnail into temporary file
         """
@@ -50,19 +48,19 @@ class Thumbnail(object):
         file_desc, thumb_fn = mkstemp(suffix=".jpg")
         os.close(file_desc)
 
-        if 'JPEGThumbnail' not in exif:
-            LOG.debug("no exif thumb")
-            thumb = self._scale_image()
-            if thumb:
-                thumb.save(thumb_fn, "JPEG")
-        else:
+        if exif and 'JPEGThumbnail' in exif and exif['JPEGThumbnail']:
             LOG.debug("exif thumb for filename %s" % self.filename)
             exif_thumbnail = exif['JPEGThumbnail']
             thumb = open(thumb_fn, 'wb')
             thumb.write(exif_thumbnail)
             thumb.close()
+        else:
+            LOG.debug("no exif thumb")
+            thumb = self._scale_image()
+            if thumb:
+                thumb.save(thumb_fn, "JPEG")
 
-            if 'Image Orientation' in exif:
+        if exif and 'Image Orientation' in exif:
             orient = exif['Image Orientation'].values[0]
             if orient > 1 and orient in orientations:
                 thumb_image = Image.open(self.thumb_fn)
@@ -72,6 +70,7 @@ class Thumbnail(object):
                     tmp_thumb_img = tmp_thumb_img.transpose(flips[orient])
 
                 tmp_thumb_img.save(thumb_fn, 'JPEG')
+
         return thumb_fn
 
     def _get_exif(self):

pygtktalog/video.py

@@ -13,6 +13,10 @@ import math
 
 import Image
 from pygtktalog.misc import float_to_string
+from pygtktalog.logger import get_logger
+
+
+LOG = get_logger("Video")
 
 
 class Video(object):
@@ -38,12 +42,13 @@ class Video(object):
                  'ID_VIDEO_HEIGHT': ['height', int],
                  # length is in seconds
                  'ID_LENGTH': ['length', lambda x: int(x.split(".")[0])],
+                 'ID_START_TIME': ['start', self._get_start_pos],
                  'ID_DEMUXER': ['container', self._return_lower],
                  'ID_VIDEO_FORMAT': ['video_format', self._return_lower],
                  'ID_VIDEO_CODEC': ['video_codec', self._return_lower],
                  'ID_AUDIO_CODEC': ['audio_codec', self._return_lower],
                  'ID_AUDIO_FORMAT': ['audio_format', self._return_lower],
-                 'ID_AUDIO_NCH':  ['audio_no_channels', int],}
+                 'ID_AUDIO_NCH': ['audio_no_channels', int]}
                  # TODO: what about audio/subtitle language/existence?
 
         for key in output:
@@ -51,8 +56,10 @@ class Video(object):
                 self.tags[attrs[key][0]] = attrs[key][1](output[key])
 
         if 'length' in self.tags and self.tags['length'] > 0:
-            hours = self.tags['length'] / 3600
-            seconds = self.tags['length'] - hours * 3600
+            start = self.tags.get('start', 0)
+            length = self.tags['length'] - start
+            hours = length / 3600
+            seconds = length - hours * 3600
             minutes = seconds / 60
             seconds -= minutes * 60
             length_str = "%02d:%02d:%02d" % (hours, minutes, seconds)
@@ -70,11 +77,11 @@ class Video(object):
         other place, otherwise it stays in filesystem.
         """
 
-        if not (self.tags.has_key('length') and self.tags.has_key('width')):
+        if not ('length' in self.tags and 'width' in self.tags):
             # no length or width
             return None
 
-        if not (self.tags['length'] >0 and self.tags['width'] >0):
+        if not (self.tags['length'] > 0 and self.tags['width'] > 0):
             # zero length or wight
             return None
 
@@ -88,7 +95,7 @@ class Video(object):
         no_pictures = self.tags['length'] / scale
 
         if no_pictures > 8:
-            no_pictures = (no_pictures / 8 ) * 8 # only multiple of 8, please.
+            no_pictures = (no_pictures / 8) * 8  # only multiple of 8, please.
         else:
             # for really short movies
             no_pictures = 4
@@ -102,6 +109,38 @@ class Video(object):
         shutil.rmtree(tempdir)
         return image_fn
 
+    def get_formatted_tags(self):
+        """
+        Return formatted tags as a string
+        """
+        out_tags = u''
+        if 'container' in self.tags:
+            out_tags += u"Container: %s\n" % self.tags['container']
+
+        if 'width' in self.tags and 'height' in self.tags:
+            out_tags += u"Resolution: %sx%s\n" % (self.tags['width'],
+                                                  self.tags['height'])
+
+        if 'duration' in self.tags:
+            out_tags += u"Duration: %s\n" % self.tags['duration']
+
+        if 'video_codec' in self.tags:
+            out_tags += "Video codec: %s\n" % self.tags['video_codec']
+
+        if 'video_format' in self.tags:
+            out_tags += "Video format: %s\n" % self.tags['video_format']
+
+        if 'audio_codec' in self.tags:
+            out_tags += "Audio codec: %s\n" % self.tags['audio_codec']
+
+        if 'audio_format' in self.tags:
+            out_tags += "Audio format: %s\n" % self.tags['audio_format']
+
+        if 'audio_no_channels' in self.tags:
+            out_tags += "Audio channels: %s\n" % self.tags['audio_no_channels']
+
+        return out_tags
+
     def _get_movie_info(self):
         """
         Gather movie file information with midentify shell command.
@@ -139,18 +178,23 @@ class Video(object):
             @directory - full output directory name
             @no_pictures - number of pictures to take
         """
-        step = float(self.tags['length']/(no_pictures + 1))
+        step = float(self.tags['length'] / (no_pictures + 1))
         current_time = 0
         for dummy in range(1, no_pictures + 1):
             current_time += step
             time = float_to_string(current_time)
             cmd = "mplayer \"%s\" -ao null -brightness 0 -hue 0 " \
-            "-saturation 0 -contrast 0 -vf-clr -vo jpeg:outdir=\"%s\" -ss %s" \
+            "-saturation 0 -contrast 0 -mc 0 -vf-clr -vo jpeg:outdir=\"%s\" -ss %s" \
             " -frames 1 2>/dev/null"
             os.popen(cmd % (self.filename, directory, time)).readlines()
 
+            try:
                 shutil.move(os.path.join(directory, "00000001.jpg"),
                             os.path.join(directory, "picture_%s.jpg" % time))
+            except IOError, (errno, strerror):
+                LOG.error('error capturing file from movie "%s" at position '
+                          '%s. Errors: %s, %s', self.filename, time, errno,
+                          strerror)
 
     def _make_montage(self, directory, image_fn, no_pictures):
         """
@@ -199,7 +243,7 @@ class Video(object):
 
         for irow in range(no_pictures * row_length):
             for icol in range(row_length):
-                left = 1 + icol*(dim[0] + 1)
+                left = 1 + icol * (dim[0] + 1)
                 right = left + dim[0]
                 upper = 1 + irow * (dim[1] + 1)
                 lower = upper + dim[1]
@@ -221,9 +265,17 @@ class Video(object):
         """
         return str(chain).lower()
 
+    def _get_start_pos(self, chain):
+        """
+        Return integer for starting point of the movie
+        """
+        try:
+            return int(chain.split(".")[0])
+        except:
+            return 0
+
     def __str__(self):
         str_out = ''
         for key in self.tags:
             str_out += "%20s: %s\n" % (key, self.tags[key])
         return str_out
-

test/unit/scan_test.py

@@ -13,10 +13,12 @@ from pygtktalog.dbobjects import File
 from pygtktalog.dbcommon import connect, Session
 
 
+TEST_DIR = "/home/share/_test_/test_dir"
+TEST_DIR_PERMS = "/home/share/_test_/test_dir_permissions/"
 
 class TestScan(unittest.TestCase):
     """
-    Testcases for scan functionality
+    Test cases for scan functionality
 
     1. execution scan function:
     1.1 simple case - should pass
@@ -53,7 +55,7 @@ class TestScan(unittest.TestCase):
         """
         scanob = scan.Scan(os.path.abspath(os.path.join(__file__,
                                                         "../../../mocks")))
-        scanob = scan.Scan("/mnt/data/_test_/test_dir")
+        scanob = scan.Scan(TEST_DIR)
         result_list = scanob.add_files()
         self.assertEqual(len(result_list), 143)
         self.assertEqual(len(result_list[0].children), 8)
@@ -76,28 +78,28 @@ class TestScan(unittest.TestCase):
 
         # dir contains some non accessable items. Should just pass, and on
         # logs should be messages about it
-        scanobj.path = "/mnt/data/_test_/test_dir_permissions/"
+        scanobj.path = TEST_DIR_PERMS
         scanobj.add_files()
 
     def test_abort_functionality(self):
-        scanobj = scan.Scan("/mnt/data/_test_/test_dir")
+        scanobj = scan.Scan(TEST_DIR)
         scanobj.abort = True
         self.assertEqual(None, scanobj.add_files())
 
-    def test_rescan(self):
+    def test_double_scan(self):
         """
         Do the scan twice.
         """
         ses = Session()
         self.assertEqual(len(ses.query(File).all()), 1)
 
-        scanob = scan.Scan("/mnt/data/_test_/test_dir")
+        scanob = scan.Scan(TEST_DIR)
         scanob.add_files()
 
         # note: we have 144 elements in db, because of root element
         self.assertEqual(len(ses.query(File).all()), 144)
 
-        scanob2 = scan.Scan("/mnt/data/_test_/test_dir")
+        scanob2 = scan.Scan(TEST_DIR)
         scanob2.add_files()
         # it is perfectly ok, since we don't update collection, but just added
         # same directory twice.
@@ -106,14 +108,14 @@ class TestScan(unittest.TestCase):
         file2_ob = scanob2._files[2]
 
         # File objects are different
-        self.assertTrue(file_ob.id != file2_ob.id)
+        self.assertTrue(file_ob is not file2_ob)
 
         # While Image objects points to the same file
         self.assertTrue(file_ob.images[0].filename == \
                 file2_ob.images[0].filename)
 
         # they are different objects
-        self.assertTrue(file_ob.images[0].id != file2_ob.images[0].id)
+        self.assertTrue(file_ob.images[0] is not file2_ob.images[0])
 
         ses.close()