1
0
mirror of https://github.com/gryf/ebook-converter.git synced 2026-03-23 02:43:35 +01:00

More cleanup

This commit is contained in:
2020-05-03 15:37:23 +02:00
parent 1a79146bf7
commit 35445cb736
4 changed files with 155 additions and 1297 deletions

View File

@@ -1,28 +1,21 @@
import os, shutil, traceback, functools, sys
from collections import defaultdict
from itertools import chain
import collections
import functools
import itertools
import os
import shutil
import sys
import traceback
from ebook_converter.customize import (CatalogPlugin, FileTypePlugin, PluginNotFound,
MetadataReaderPlugin, MetadataWriterPlugin,
InterfaceActionBase as InterfaceAction,
PreferencesPlugin, platform, InvalidPlugin,
StoreBase as Store, EditBookToolPlugin,
LibraryClosedPlugin)
from ebook_converter.customize.conversion import InputFormatPlugin, OutputFormatPlugin
from ebook_converter.customize.profiles import InputProfile, OutputProfile
from ebook_converter.customize.builtins import plugins as builtin_plugins
# from ebook_converter.devices.interface import DevicePlugin
from ebook_converter.ebooks.metadata import MetaInformation
from ebook_converter.utils.config import (make_config_dir, Config, ConfigProxy,
plugin_dir, OptionParser)
# from ebook_converter.ebooks.metadata.sources.base import Source
from ebook_converter.constants import DEBUG, numeric_version
from ebook_converter import customize
from ebook_converter.customize import conversion
from ebook_converter.customize import profiles
from ebook_converter.customize import builtins
from ebook_converter.ebooks import metadata
from ebook_converter.utils import config as cfg
from ebook_converter import constants
__license__ = 'GPL v3'
__copyright__ = '2008, Kovid Goyal <kovid at kovidgoyal.net>'
builtin_names = frozenset(p.name for p in builtin_plugins)
builtin_names = frozenset(p.name for p in builtins.plugins)
BLACKLISTED_PLUGINS = frozenset({'Marvin XD', 'iOS reader applications'})
@@ -31,107 +24,40 @@ class NameConflict(ValueError):
def _config():
c = Config('customize')
c = cfg.Config('customize')
c.add_opt('plugins', default={}, help=_('Installed plugins'))
c.add_opt('filetype_mapping', default={}, help=_('Mapping for filetype plugins'))
c.add_opt('plugin_customization', default={}, help=_('Local plugin customization'))
c.add_opt('filetype_mapping', default={},
help=_('Mapping for filetype plugins'))
c.add_opt('plugin_customization', default={},
help=_('Local plugin customization'))
c.add_opt('disabled_plugins', default=set(), help=_('Disabled plugins'))
c.add_opt('enabled_plugins', default=set(), help=_('Enabled plugins'))
return ConfigProxy(c)
return cfg.ConfigProxy(c)
config = _config()
def find_plugin(name):
for plugin in _initialized_plugins:
if plugin.name == name:
return plugin
def load_plugin(path_to_zip_file): # {{{
'''
Load plugin from ZIP file or raise InvalidPlugin error
:return: A :class:`Plugin` instance.
'''
return loader.load(path_to_zip_file)
# }}}
# Enable/disable plugins {{{
def disable_plugin(plugin_or_name):
x = getattr(plugin_or_name, 'name', plugin_or_name)
plugin = find_plugin(x)
if not plugin.can_be_disabled:
raise ValueError('Plugin %s cannot be disabled'%x)
dp = config['disabled_plugins']
dp.add(x)
config['disabled_plugins'] = dp
ep = config['enabled_plugins']
if x in ep:
ep.remove(x)
config['enabled_plugins'] = ep
def enable_plugin(plugin_or_name):
x = getattr(plugin_or_name, 'name', plugin_or_name)
dp = config['disabled_plugins']
if x in dp:
dp.remove(x)
config['disabled_plugins'] = dp
ep = config['enabled_plugins']
ep.add(x)
config['enabled_plugins'] = ep
def restore_plugin_state_to_default(plugin_or_name):
x = getattr(plugin_or_name, 'name', plugin_or_name)
dp = config['disabled_plugins']
if x in dp:
dp.remove(x)
config['disabled_plugins'] = dp
ep = config['enabled_plugins']
if x in ep:
ep.remove(x)
config['enabled_plugins'] = ep
default_disabled_plugins = {
'Overdrive', 'Douban Books', 'OZON.ru', 'Edelweiss', 'Google Images', 'Big Book Search',
}
def is_disabled(plugin):
if plugin.name in config['enabled_plugins']:
return False
return plugin.name in config['disabled_plugins'] or \
plugin.name in default_disabled_plugins
# }}}
# File type plugins {{{
_on_import = {}
_on_postimport = {}
_on_preprocess = {}
_on_postprocess = {}
_on_postadd = []
# File type plugins
_on_import = {}
_on_postimport = {}
_on_preprocess = {}
_on_postprocess = {}
_on_postadd = []
def reread_filetype_plugins():
global _on_import, _on_postimport, _on_preprocess, _on_postprocess, _on_postadd
_on_import = defaultdict(list)
_on_postimport = defaultdict(list)
_on_preprocess = defaultdict(list)
_on_postprocess = defaultdict(list)
_on_postadd = []
global _on_import, _on_postimport, _on_preprocess, _on_postprocess
global _on_postadd
_on_import = collections.defaultdict(list)
_on_postimport = collections.defaultdict(list)
_on_preprocess = collections.defaultdict(list)
_on_postprocess = collections.defaultdict(list)
_on_postadd = []
for plugin in _initialized_plugins:
if isinstance(plugin, FileTypePlugin):
if isinstance(plugin, customize.FileTypePlugin):
for ft in plugin.file_types:
if plugin.on_import:
_on_import[ft].append(plugin)
@@ -145,12 +71,12 @@ def reread_filetype_plugins():
def plugins_for_ft(ft, occasion):
op = {
'import':_on_import, 'preprocess':_on_preprocess, 'postprocess':_on_postprocess, 'postimport':_on_postimport,
}[occasion]
for p in chain(op.get(ft, ()), op.get('*', ())):
if not is_disabled(p):
yield p
op = {'import': _on_import,
'preprocess': _on_preprocess,
'postprocess': _on_postprocess,
'postimport': _on_postimport}[occasion]
for p in itertools.chain(op.get(ft, ()), op.get('*', ())):
yield p
def _run_filetype_plugins(path_to_file, ft=None, occasion='preprocess'):
@@ -160,7 +86,9 @@ def _run_filetype_plugins(path_to_file, ft=None, occasion='preprocess'):
nfp = path_to_file
for plugin in plugins_for_ft(ft, occasion):
plugin.site_customization = customization.get(plugin.name, '')
oo, oe = sys.stdout, sys.stderr # Some file type plugins out there override the output streams with buggy implementations
# Some file type plugins out there override the output streams with
# buggy implementations
oo, oe = sys.stdout, sys.stderr
with plugin:
try:
plugin.original_path_to_file = path_to_file
@@ -169,7 +97,8 @@ def _run_filetype_plugins(path_to_file, ft=None, occasion='preprocess'):
try:
nfp = plugin.run(nfp) or nfp
except:
print('Running file type plugin %s failed with traceback:'%plugin.name, file=oe)
print('Running file type plugin %s failed with traceback:' %
plugin.name, file=oe)
traceback.print_exc(file=oe)
sys.stdout, sys.stderr = oo, oe
x = lambda j: os.path.normpath(os.path.normcase(j))
@@ -179,9 +108,12 @@ def _run_filetype_plugins(path_to_file, ft=None, occasion='preprocess'):
return nfp
run_plugins_on_import = functools.partial(_run_filetype_plugins, occasion='import')
run_plugins_on_preprocess = functools.partial(_run_filetype_plugins, occasion='preprocess')
run_plugins_on_postprocess = functools.partial(_run_filetype_plugins, occasion='postprocess')
run_plugins_on_import = functools.partial(_run_filetype_plugins,
occasion='import')
run_plugins_on_preprocess = functools.partial(_run_filetype_plugins,
occasion='preprocess')
run_plugins_on_postprocess = functools.partial(_run_filetype_plugins,
occasion='postprocess')
def run_plugins_on_postimport(db, book_id, fmt):
@@ -192,31 +124,26 @@ def run_plugins_on_postimport(db, book_id, fmt):
with plugin:
try:
plugin.postimport(book_id, fmt, db)
except:
print('Running file type plugin %s failed with traceback:'%
plugin.name)
except Exception:
print('Running file type plugin %s failed with traceback:' %
plugin.name)
traceback.print_exc()
def run_plugins_on_postadd(db, book_id, fmt_map):
customization = config['plugin_customization']
for plugin in _on_postadd:
if is_disabled(plugin):
continue
plugin.site_customization = customization.get(plugin.name, '')
with plugin:
try:
plugin.postadd(book_id, fmt_map, db)
except Exception:
print('Running file type plugin %s failed with traceback:'%
plugin.name)
print('Running file type plugin %s failed with traceback:' %
plugin.name)
traceback.print_exc()
# }}}
# Plugin customization {{{
# Plugin customization
def customize_plugin(plugin, custom):
d = config['plugin_customization']
d[plugin.name] = custom.strip()
@@ -226,82 +153,64 @@ def customize_plugin(plugin, custom):
def plugin_customization(plugin):
return config['plugin_customization'].get(plugin.name, '')
# }}}
# Input/Output profiles {{{
# Input/Output profiles
def input_profiles():
for plugin in _initialized_plugins:
if isinstance(plugin, InputProfile):
if isinstance(plugin, profiles.InputProfile):
yield plugin
def output_profiles():
for plugin in _initialized_plugins:
if isinstance(plugin, OutputProfile):
if isinstance(plugin, profiles.OutputProfile):
yield plugin
# }}}
# Interface Actions # {{{
# Interface Actions #
def interface_actions():
customization = config['plugin_customization']
for plugin in _initialized_plugins:
if isinstance(plugin, InterfaceAction):
if not is_disabled(plugin):
plugin.site_customization = customization.get(plugin.name, '')
yield plugin
# }}}
# Preferences Plugins # {{{
if isinstance(plugin, customize.InterfaceActionBase):
yield plugin
# Preferences Plugins
def preferences_plugins():
customization = config['plugin_customization']
for plugin in _initialized_plugins:
if isinstance(plugin, PreferencesPlugin):
if not is_disabled(plugin):
plugin.site_customization = customization.get(plugin.name, '')
yield plugin
# }}}
# Library Closed Plugins # {{{
if isinstance(plugin, customize.PreferencesPlugin):
plugin.site_customization = customization.get(plugin.name, '')
yield plugin
# Library Closed Plugins
def available_library_closed_plugins():
customization = config['plugin_customization']
for plugin in _initialized_plugins:
if isinstance(plugin, LibraryClosedPlugin):
if not is_disabled(plugin):
plugin.site_customization = customization.get(plugin.name, '')
yield plugin
if isinstance(plugin, customize.LibraryClosedPlugin):
plugin.site_customization = customization.get(plugin.name, '')
yield plugin
def has_library_closed_plugins():
for plugin in _initialized_plugins:
if isinstance(plugin, LibraryClosedPlugin):
if not is_disabled(plugin):
return True
if isinstance(plugin, customize.LibraryClosedPlugin):
return True
return False
# }}}
# Store Plugins # {{{
# Store Plugins
def store_plugins():
customization = config['plugin_customization']
for plugin in _initialized_plugins:
if isinstance(plugin, Store):
if isinstance(plugin, customize.StoreBase):
plugin.site_customization = customization.get(plugin.name, '')
yield plugin
def available_store_plugins():
for plugin in store_plugins():
if not is_disabled(plugin):
yield plugin
yield plugin
def stores():
@@ -317,11 +226,8 @@ def available_stores():
stores.add(plugin.name)
return stores
# }}}
# Metadata read/write {{{
# Metadata read/write
_metadata_readers = {}
_metadata_writers = {}
@@ -329,13 +235,13 @@ _metadata_writers = {}
def reread_metadata_plugins():
global _metadata_readers
global _metadata_writers
_metadata_readers = defaultdict(list)
_metadata_writers = defaultdict(list)
_metadata_readers = collections.defaultdict(list)
_metadata_writers = collections.defaultdict(list)
for plugin in _initialized_plugins:
if isinstance(plugin, MetadataReaderPlugin):
if isinstance(plugin, customize.MetadataReaderPlugin):
for ft in plugin.file_types:
_metadata_readers[ft].append(plugin)
elif isinstance(plugin, MetadataWriterPlugin):
elif isinstance(plugin, customize.MetadataWriterPlugin):
for ft in plugin.file_types:
_metadata_writers[ft].append(plugin)
@@ -408,109 +314,61 @@ class ForceIdentifiers(object):
self.force_identifiers = False
force_identifiers = ForceIdentifiers()
def get_file_type_metadata(stream, ftype):
mi = MetaInformation(None, None)
mi = metadata.MetaInformation(None, None)
ftype = ftype.lower().strip()
if ftype in _metadata_readers:
for plugin in _metadata_readers[ftype]:
if not is_disabled(plugin):
with plugin:
try:
plugin.quick = quick_metadata.quick
if hasattr(stream, 'seek'):
stream.seek(0)
mi = plugin.get_metadata(stream, ftype.lower().strip())
break
except:
traceback.print_exc()
continue
with plugin:
try:
plugin.quick = quick_metadata.quick
if hasattr(stream, 'seek'):
stream.seek(0)
mi = plugin.get_metadata(stream, ftype.lower().strip())
break
except Exception:
traceback.print_exc()
continue
return mi
def set_file_type_metadata(stream, mi, ftype, report_error=None):
fi = ForceIdentifiers()
ftype = ftype.lower().strip()
if ftype in _metadata_writers:
customization = config['plugin_customization']
for plugin in _metadata_writers[ftype]:
if not is_disabled(plugin):
with plugin:
try:
plugin.apply_null = apply_null_metadata.apply_null
plugin.force_identifiers = force_identifiers.force_identifiers
plugin.site_customization = customization.get(plugin.name, '')
plugin.set_metadata(stream, mi, ftype.lower().strip())
break
except:
if report_error is None:
from ebook_converter import prints
prints('Failed to set metadata for the', ftype.upper(), 'format of:', getattr(mi, 'title', ''), file=sys.stderr)
traceback.print_exc()
else:
report_error(mi, ftype, traceback.format_exc())
with plugin:
try:
plugin.apply_null = apply_null_metadata.apply_null
plugin.force_identifiers = fi.force_identifiers
plugin.site_customization = customization.get(
plugin.name, '')
plugin.set_metadata(stream, mi, ftype.lower().strip())
break
except Exception:
if report_error is None:
from ebook_converter import prints
prints('Failed to set metadata for the', ftype.upper(),
'format of:', getattr(mi, 'title', ''),
file=sys.stderr)
traceback.print_exc()
else:
report_error(mi, ftype, traceback.format_exc())
def can_set_metadata(ftype):
ftype = ftype.lower().strip()
for plugin in _metadata_writers.get(ftype, ()):
if not is_disabled(plugin):
return True
return True
return False
# }}}
# Add/remove plugins {{{
def add_plugin(path_to_zip_file):
make_config_dir()
plugin = load_plugin(path_to_zip_file)
if plugin.name in builtin_names:
raise NameConflict(
'A builtin plugin with the name %r already exists' % plugin.name)
plugin = initialize_plugin(plugin, path_to_zip_file)
plugins = config['plugins']
zfp = os.path.join(plugin_dir, plugin.name+'.zip')
if os.path.exists(zfp):
os.remove(zfp)
shutil.copyfile(path_to_zip_file, zfp)
plugins[plugin.name] = zfp
config['plugins'] = plugins
initialize_plugins()
return plugin
def remove_plugin(plugin_or_name):
name = getattr(plugin_or_name, 'name', plugin_or_name)
plugins = config['plugins']
removed = False
if name in plugins:
removed = True
try:
zfp = os.path.join(plugin_dir, name+'.zip')
if os.path.exists(zfp):
os.remove(zfp)
zfp = plugins[name]
if os.path.exists(zfp):
os.remove(zfp)
except:
pass
plugins.pop(name)
config['plugins'] = plugins
initialize_plugins()
return removed
# }}}
# Input/Output format plugins {{{
# Input/Output format plugins
def input_format_plugins():
for plugin in _initialized_plugins:
if isinstance(plugin, InputFormatPlugin):
if isinstance(plugin, conversion.InputFormatPlugin):
yield plugin
@@ -533,16 +391,16 @@ def all_input_formats():
def available_input_formats():
formats = set()
for plugin in input_format_plugins():
if not is_disabled(plugin):
for format in plugin.file_types:
formats.add(format)
formats.add('zip'), formats.add('rar')
for format in plugin.file_types:
formats.add(format)
formats.add('zip')
formats.add('rar')
return formats
def output_format_plugins():
for plugin in _initialized_plugins:
if isinstance(plugin, OutputFormatPlugin):
if isinstance(plugin, conversion.OutputFormatPlugin):
yield plugin
@@ -557,27 +415,22 @@ def plugin_for_output_format(fmt):
def available_output_formats():
formats = set()
for plugin in output_format_plugins():
if not is_disabled(plugin):
formats.add(plugin.file_type)
formats.add(plugin.file_type)
return formats
# }}}
# Catalog plugins {{{
# Catalog plugins
def catalog_plugins():
for plugin in _initialized_plugins:
if isinstance(plugin, CatalogPlugin):
if isinstance(plugin, customize.CatalogPlugin):
yield plugin
def available_catalog_formats():
formats = set()
for plugin in catalog_plugins():
if not is_disabled(plugin):
for format in plugin.file_types:
formats.add(format)
for format in plugin.file_types:
formats.add(format)
return formats
@@ -586,74 +439,15 @@ def plugin_for_catalog_format(fmt):
if fmt.lower() in plugin.file_types:
return plugin
# }}}
# Device plugins {{{
def device_plugins(include_disabled=False):
for plugin in _initialized_plugins:
if isinstance(plugin, DevicePlugin):
if include_disabled or not is_disabled(plugin):
if platform in plugin.supported_platforms:
if getattr(plugin, 'plugin_needs_delayed_initialization',
False):
plugin.do_delayed_plugin_initialization()
yield plugin
def disabled_device_plugins():
for plugin in _initialized_plugins:
if isinstance(plugin, DevicePlugin):
if is_disabled(plugin):
if platform in plugin.supported_platforms:
yield plugin
# }}}
# Metadata sources2 {{{
def metadata_plugins(capabilities):
capabilities = frozenset(capabilities)
for plugin in all_metadata_plugins():
if plugin.capabilities.intersection(capabilities) and \
not is_disabled(plugin):
yield plugin
def all_metadata_plugins():
for plugin in _initialized_plugins:
if isinstance(plugin, Source):
yield plugin
def patch_metadata_plugins(possibly_updated_plugins):
patches = {}
for i, plugin in enumerate(_initialized_plugins):
if isinstance(plugin, Source) and plugin.name in builtin_names:
pup = possibly_updated_plugins.get(plugin.name)
if pup is not None:
if pup.version > plugin.version and pup.minimum_calibre_version <= numeric_version:
patches[i] = pup(None)
# Metadata source plugins dont use initialize() but that
# might change in the future, so be safe.
patches[i].initialize()
for i, pup in patches.items():
_initialized_plugins[i] = pup
# }}}
# Editor plugins {{{
# Editor plugins
def all_edit_book_tool_plugins():
for plugin in _initialized_plugins:
if isinstance(plugin, EditBookToolPlugin):
if isinstance(plugin, customize.EditBookToolPlugin):
yield plugin
# }}}
# Initialize plugins {{{
# Initialize plugins
_initialized_plugins = []
@@ -665,8 +459,8 @@ def initialize_plugin(plugin, path_to_zip_file):
except Exception:
print('Failed to initialize plugin:', plugin.name, plugin.version)
tb = traceback.format_exc()
raise InvalidPlugin((_('Initialization of plugin %s failed with traceback:')
%tb) + '\n'+tb)
raise customize.InvalidPlugin((_('Initialization of plugin %s failed '
'with traceback:') % tb) + '\n'+tb)
def has_external_plugins():
@@ -674,49 +468,23 @@ def has_external_plugins():
return bool(config['plugins'])
def initialize_plugins(perf=False):
def initialize_plugins():
global _initialized_plugins
_initialized_plugins = []
conflicts = [name for name in config['plugins'] if name in
builtin_names]
for p in conflicts:
remove_plugin(p)
external_plugins = config['plugins'].copy()
for name in BLACKLISTED_PLUGINS:
external_plugins.pop(name, None)
ostdout, ostderr = sys.stdout, sys.stderr
if perf:
from collections import defaultdict
import time
times = defaultdict(lambda:0)
for zfp in list(external_plugins) + builtin_plugins:
for zfp in list(external_plugins) + builtins.plugins:
try:
if not isinstance(zfp, type):
# We have a plugin name
pname = zfp
zfp = os.path.join(plugin_dir, zfp+'.zip')
if not os.path.exists(zfp):
zfp = external_plugins[pname]
try:
plugin = load_plugin(zfp) if not isinstance(zfp, type) else zfp
except PluginNotFound:
continue
if perf:
st = time.time()
plugin = initialize_plugin(plugin, None if isinstance(zfp, type) else zfp)
if perf:
times[plugin.name] = time.time() - st
plugin = initialize_plugin(zfp, None)
_initialized_plugins.append(plugin)
except:
except Exception:
print('Failed to initialize plugin:', repr(zfp))
if DEBUG:
traceback.print_exc()
# Prevent a custom plugin from overriding stdout/stderr as this breaks
# ipython
sys.stdout, sys.stderr = ostdout, ostderr
if perf:
for x in sorted(times, key=lambda x: times[x]):
print('%50s: %.3f'%(x, times[x]))
_initialized_plugins.sort(key=lambda x: x.priority, reverse=True)
reread_filetype_plugins()
reread_metadata_plugins()
@@ -729,31 +497,11 @@ def initialized_plugins():
for plugin in _initialized_plugins:
yield plugin
# }}}
# CLI {{{
def build_plugin(path):
from ebook_converter import prints
from ebook_converter.ptempfile import PersistentTemporaryFile
from ebook_converter.utils.zipfile import ZipFile, ZIP_STORED
path = str(path)
names = frozenset(os.listdir(path))
if '__init__.py' not in names:
prints(path, ' is not a valid plugin')
raise SystemExit(1)
t = PersistentTemporaryFile(u'.zip')
with ZipFile(t, 'w', ZIP_STORED) as zf:
zf.add_dir(path, simple_filter=lambda x:x in {'.git', '.bzr', '.svn', '.hg'})
t.close()
plugin = add_plugin(t.name)
os.remove(t.name)
prints('Plugin updated:', plugin.name, plugin.version)
# CLI
def option_parser():
parser = OptionParser(usage=_('''\
parser = cfg.OptionParser(usage=_('''\
%prog options
Customize calibre by loading external plugins.
@@ -775,59 +523,3 @@ def option_parser():
parser.add_option('--disable-plugin', default=None,
help=_('Disable the named plugin'))
return parser
def main(args=sys.argv):
parser = option_parser()
if len(args) < 2:
parser.print_help()
return 1
opts, args = parser.parse_args(args)
if opts.add_plugin is not None:
plugin = add_plugin(opts.add_plugin)
print('Plugin added:', plugin.name, plugin.version)
if opts.build_plugin is not None:
build_plugin(opts.build_plugin)
if opts.remove_plugin is not None:
if remove_plugin(opts.remove_plugin):
print('Plugin removed')
else:
print('No custom plugin named', opts.remove_plugin)
if opts.customize_plugin is not None:
name, custom = opts.customize_plugin.split(',')
plugin = find_plugin(name.strip())
if plugin is None:
print('No plugin with the name %s exists'%name)
return 1
customize_plugin(plugin, custom)
if opts.enable_plugin is not None:
enable_plugin(opts.enable_plugin.strip())
if opts.disable_plugin is not None:
disable_plugin(opts.disable_plugin.strip())
if opts.list_plugins:
type_len = name_len = 0
for plugin in initialized_plugins():
type_len, name_len = max(type_len, len(plugin.type)), max(name_len, len(plugin.name))
fmt = '%-{}s%-{}s%-15s%-15s%s'.format(type_len+1, name_len+1)
print(fmt%tuple(('Type|Name|Version|Disabled|Site Customization'.split('|'))))
print()
for plugin in initialized_plugins():
print(fmt%(
plugin.type, plugin.name,
plugin.version, is_disabled(plugin),
plugin_customization(plugin)
))
print('\t', plugin.description)
if plugin.is_customizable():
try:
print('\t', plugin.customization_help())
except NotImplementedError:
pass
print()
return 0
if __name__ == '__main__':
sys.exit(main())
# }}}