gryf/ebook-converter
1
0
Fork 0
mirror of https://github.com/gryf/ebook-converter.git synced 2026-02-20 08:45:47 +01:00
Code Issues Projects Releases Wiki Activity

Convert calibre modules to ebook_converter.

Browse Source 
Here is the first batch of modules, which are needed for converting
several formats to LRF. Some of the logic has been change, more cleanups
will follow.
This commit is contained in:
gryf
2020-04-11 19:33:43 +02:00
parent 69d2e536c5
commit 0f9792df36
252 changed files with 1925 additions and 2344 deletions
Download Patch File Download Diff File Expand all files Collapse all files

56
ebook_converter/utils/__init__.py
View File

@@ -1,56 +0,0 @@
#!/usr/bin/env python2
from __future__ import absolute_import, division, print_function, unicode_literals
__license__ = 'GPL v3'
__copyright__ = '2008, Kovid Goyal kovid@kovidgoyal.net'
__docformat__ = 'restructuredtext en'
'''
Miscelleaneous utilities.
'''
from time import time
from polyglot.builtins import as_bytes
def join_with_timeout(q, timeout=2):
''' Join the queue q with a specified timeout. Blocks until all tasks on
the queue are done or times out with a runtime error. '''
q.all_tasks_done.acquire()
try:
endtime = time() + timeout
while q.unfinished_tasks:
remaining = endtime - time()
if remaining <= 0.0:
raise RuntimeError('Waiting for queue to clear timed out')
q.all_tasks_done.wait(remaining)
finally:
q.all_tasks_done.release()
def unpickle_binary_string(data):
# Maintains compatibility with python's pickle module protocol version 2
import struct
PROTO, SHORT_BINSTRING, BINSTRING = b'\x80', b'U', b'T'
if data.startswith(PROTO + b'\x02'):
offset = 2
which = data[offset:offset+1]
offset += 1
if which == BINSTRING:
sz, = struct.unpack_from('<i', data, offset)
offset += struct.calcsize('<i')
elif which == SHORT_BINSTRING:
sz = ord(data[offset:offset+1])
offset += 1
else:
return
return data[offset:offset + sz]
def pickle_binary_string(data):
# Maintains compatibility with python's pickle module protocol version 2
import struct
PROTO, STOP, BINSTRING = b'\x80', b'.', b'T'
data = as_bytes(data)
return PROTO + b'\x02' + BINSTRING + struct.pack(b'<i', len(data)) + data + STOP

16
ebook_converter/utils/cleantext.py
View File

@@ -4,17 +4,9 @@
from __future__ import absolute_import, division, print_function, unicode_literals
import re
from polyglot.builtins import codepoint_to_chr, map, range, filter
from polyglot.html_entities import name2codepoint
from calibre.constants import plugins, preferred_encoding
_ncxc = plugins['speedup'][0].clean_xml_chars
def native_clean_xml_chars(x):
if isinstance(x, bytes):
x = x.decode(preferred_encoding)
return _ncxc(x)
from ebook_converter.polyglot.builtins import codepoint_to_chr, map, range, filter
from ebook_converter.polyglot.html_entities import name2codepoint
from ebook_converter.constants import plugins, preferred_encoding
def ascii_pat(for_binary=False):
@@ -59,7 +51,7 @@ def py_clean_xml_chars(unicode_string):
return ''.join(filter(allowed, unicode_string))
clean_xml_chars = native_clean_xml_chars or py_clean_xml_chars
clean_xml_chars = py_clean_xml_chars
def test_clean_xml_chars():

36
ebook_converter/utils/config.py
View File

@@ -12,16 +12,16 @@ import optparse
import os
from copy import deepcopy
from calibre.constants import (
from ebook_converter.constants import (
CONFIG_DIR_MODE, __appname__, __author__, config_dir, get_version, iswindows
)
from calibre.utils.config_base import (
from ebook_converter.utils.config_base import (
Config, ConfigInterface, ConfigProxy, Option, OptionSet, OptionValues,
StringConfig, json_dumps, json_loads, make_config_dir, plugin_dir, prefs,
tweaks, from_json, to_json
)
from calibre.utils.lock import ExclusiveFile
from polyglot.builtins import string_or_bytes, native_string_type
from ebook_converter.utils.lock import ExclusiveFile
from ebook_converter.polyglot.builtins import string_or_bytes, native_string_type
# optparse uses gettext.gettext instead of _ from builtins, so we
@@ -41,7 +41,7 @@ def check_config_write_access():
class CustomHelpFormatter(optparse.IndentedHelpFormatter):
def format_usage(self, usage):
from calibre.utils.terminal import colored
from ebook_converter.utils.terminal import colored
parts = usage.split(' ')
if parts:
parts[0] = colored(parts[0], fg='yellow', bold=True)
@@ -49,13 +49,13 @@ class CustomHelpFormatter(optparse.IndentedHelpFormatter):
return colored(_('Usage'), fg='blue', bold=True) + ': ' + usage
def format_heading(self, heading):
from calibre.utils.terminal import colored
from ebook_converter.utils.terminal import colored
return "%*s%s:\n" % (self.current_indent, '',
colored(heading, fg='blue', bold=True))
def format_option(self, option):
import textwrap
from calibre.utils.terminal import colored
from ebook_converter.utils.terminal import colored
result = []
opts = self.option_strings[option]
@@ -93,7 +93,7 @@ class OptionParser(optparse.OptionParser):
conflict_handler='resolve',
**kwds):
import textwrap
from calibre.utils.terminal import colored
from ebook_converter.utils.terminal import colored
usage = textwrap.dedent(usage)
if epilog is None:
@@ -114,17 +114,17 @@ class OptionParser(optparse.OptionParser):
_("show program's version number and exit")
def print_usage(self, file=None):
from calibre.utils.terminal import ANSIStream
from ebook_converter.utils.terminal import ANSIStream
s = ANSIStream(file)
optparse.OptionParser.print_usage(self, file=s)
def print_help(self, file=None):
from calibre.utils.terminal import ANSIStream
from ebook_converter.utils.terminal import ANSIStream
s = ANSIStream(file)
optparse.OptionParser.print_help(self, file=s)
def print_version(self, file=None):
from calibre.utils.terminal import ANSIStream
from ebook_converter.utils.terminal import ANSIStream
s = ANSIStream(file)
optparse.OptionParser.print_version(self, file=s)
@@ -221,8 +221,8 @@ class DynamicConfig(dict):
self.refresh()
def read_old_serialized_representation(self):
from calibre.utils.shared_file import share_open
from calibre.utils.serialize import pickle_loads
from ebook_converter.utils.shared_file import share_open
from ebook_converter.utils.serialize import pickle_loads
path = self.file_path.rpartition('.')[0]
try:
with share_open(path, 'rb') as f:
@@ -334,11 +334,11 @@ class XMLConfig(dict):
pass
def raw_to_object(self, raw):
from polyglot.plistlib import loads
from ebook_converter.polyglot.plistlib import loads
return loads(raw)
def to_raw(self):
from polyglot.plistlib import dumps
from ebook_converter.polyglot.plistlib import dumps
return dumps(self)
def decouple(self, prefix):
@@ -363,7 +363,7 @@ class XMLConfig(dict):
self.update(d)
def __getitem__(self, key):
from polyglot.plistlib import Data
from ebook_converter.polyglot.plistlib import Data
try:
ans = dict.__getitem__(self, key)
if isinstance(ans, Data):
@@ -373,7 +373,7 @@ class XMLConfig(dict):
return self.defaults.get(key, None)
def get(self, key, default=None):
from polyglot.plistlib import Data
from ebook_converter.polyglot.plistlib import Data
try:
ans = dict.__getitem__(self, key)
if isinstance(ans, Data):
@@ -383,7 +383,7 @@ class XMLConfig(dict):
return self.defaults.get(key, default)
def __setitem__(self, key, val):
from polyglot.plistlib import Data
from ebook_converter.polyglot.plistlib import Data
if isinstance(val, bytes):
val = Data(val)
dict.__setitem__(self, key, val)

19
ebook_converter/utils/config_base.py
View File

@@ -1,8 +1,3 @@
#!/usr/bin/env python2
# vim:fileencoding=UTF-8:ts=4:sw=4:sta:et:sts=4:ai
from __future__ import absolute_import, division, print_function, unicode_literals
__license__ = 'GPL v3'
__copyright__ = '2011, Kovid Goyal <kovid@kovidgoyal.net>'
__docformat__ = 'restructuredtext en'
@@ -12,9 +7,9 @@ from functools import partial
from collections import defaultdict
from copy import deepcopy
from calibre.utils.lock import ExclusiveFile
from calibre.constants import config_dir, CONFIG_DIR_MODE, ispy3, preferred_encoding, filesystem_encoding, iswindows
from polyglot.builtins import unicode_type, iteritems, map
from ebook_converter.utils.lock import ExclusiveFile
from ebook_converter.constants import config_dir, CONFIG_DIR_MODE, ispy3, preferred_encoding, filesystem_encoding, iswindows
from ebook_converter.polyglot.builtins import unicode_type, iteritems, map
plugin_dir = os.path.join(config_dir, 'plugins')
@@ -46,7 +41,7 @@ def to_json(obj):
return {'__class__': 'bytearray',
'__value__': standard_b64encode(bytes(obj)).decode('ascii')}
if isinstance(obj, datetime.datetime):
from calibre.utils.date import isoformat
from ebook_converter.utils.date import isoformat
return {'__class__': 'datetime.datetime',
'__value__': isoformat(obj, as_utc=True)}
if isinstance(obj, (set, frozenset)):
@@ -73,7 +68,7 @@ def from_json(obj):
from base64 import standard_b64decode
return bytearray(standard_b64decode(obj['__value__'].encode('ascii')))
if custom == 'datetime.datetime':
from calibre.utils.iso8601 import parse_iso8601
from ebook_converter.utils.iso8601 import parse_iso8601
return parse_iso8601(obj['__value__'], assume_utc=True)
if custom == 'set':
return set(obj['__value__'])
@@ -263,7 +258,7 @@ class OptionSet(object):
opt.help = opt.help.format(u'ñ')
def option_parser(self, user_defaults=None, usage='', gui_mode=False):
from calibre.utils.config import OptionParser
from ebook_converter.utils.config import OptionParser
parser = OptionParser(usage, gui_mode=gui_mode)
groups = defaultdict(lambda : parser)
for group, desc in self.groups.items():
@@ -376,7 +371,7 @@ class Config(ConfigInterface):
traceback.print_exc()
if not src:
path = path.rpartition('.')[0]
from calibre.utils.shared_file import share_open
from ebook_converter.utils.shared_file import share_open
try:
with share_open(path, 'rb') as f:
src = f.read().decode('utf-8')

10
ebook_converter/utils/date.py
View File

@@ -10,11 +10,11 @@ import re
from datetime import datetime, time as dtime, timedelta, MINYEAR, MAXYEAR
from functools import partial
from calibre import strftime
from calibre.constants import iswindows, isosx, plugins, preferred_encoding
from calibre.utils.iso8601 import utc_tz, local_tz, UNDEFINED_DATE
from calibre.utils.localization import lcdata
from polyglot.builtins import unicode_type, native_string_type
from ebook_converter import strftime
from ebook_converter.constants import iswindows, isosx, plugins, preferred_encoding
from ebook_converter.utils.iso8601 import utc_tz, local_tz, UNDEFINED_DATE
from ebook_converter.utils.localization import lcdata
from ebook_converter.polyglot.builtins import unicode_type, native_string_type
_utc_tz = utc_tz
_local_tz = local_tz

8
ebook_converter/utils/filenames.py
View File

@@ -10,12 +10,12 @@ import shutil
import time
from math import ceil
from calibre import force_unicode, isbytestring, prints, sanitize_file_name
from calibre.constants import (
from ebook_converter import force_unicode, isbytestring, prints, sanitize_file_name
from ebook_converter.constants import (
filesystem_encoding, iswindows, plugins, preferred_encoding, isosx, ispy3
)
from calibre.utils.localization import get_udc
from polyglot.builtins import iteritems, itervalues, unicode_type, range
from ebook_converter.utils.localization import get_udc
from ebook_converter.polyglot.builtins import iteritems, itervalues, unicode_type, range
def ascii_text(orig):

7
ebook_converter/utils/fonts/__init__.py
View File

@@ -1,7 +0,0 @@
#!/usr/bin/env python2
# vim:fileencoding=UTF-8:ts=4:sw=4:sta:et:sts=4:ai
from __future__ import absolute_import, division, print_function, unicode_literals
__license__ = 'GPL v3'
__copyright__ = '2009, Kovid Goyal <kovid@kovidgoyal.net>'
__docformat__ = 'restructuredtext en'

4
ebook_converter/utils/fonts/metadata.py
View File

@@ -10,8 +10,8 @@ from io import BytesIO
from struct import calcsize, unpack, unpack_from
from collections import namedtuple
from calibre.utils.fonts.utils import get_font_names2, get_font_characteristics
from polyglot.builtins import range, unicode_type
from ebook_converter.utils.fonts.utils import get_font_names2, get_font_characteristics
from ebook_converter.polyglot.builtins import range, unicode_type
class UnsupportedFont(ValueError):

49
ebook_converter/utils/fonts/scanner.py
View File

@@ -10,12 +10,11 @@ import os
from collections import defaultdict
from threading import Thread
from calibre import walk, prints, as_unicode
from calibre.constants import (config_dir, iswindows, isosx, plugins, DEBUG,
from ebook_converter import walk, prints, as_unicode
from ebook_converter.constants import (config_dir, iswindows, isosx, plugins, DEBUG,
isworker, filesystem_encoding)
from calibre.utils.fonts.metadata import FontMetadata, UnsupportedFont
from calibre.utils.icu import sort_key
from polyglot.builtins import itervalues, unicode_type, filter
from ebook_converter.utils.fonts.metadata import FontMetadata, UnsupportedFont
from ebook_converter.polyglot.builtins import itervalues, unicode_type, filter
class NoFonts(ValueError):
@@ -153,12 +152,12 @@ def path_significance(path, folders):
def build_families(cached_fonts, folders, family_attr='font-family'):
families = defaultdict(list)
for f in itervalues(cached_fonts):
if not f:
for font in itervalues(cached_fonts):
if not font:
continue
lf = icu_lower(f.get(family_attr) or '')
lf = (font.get(family_attr) or '').lower()
if lf:
families[lf].append(f)
families[lf].append(font)
for fonts in itervalues(families):
# Look for duplicate font files and choose the copy that is from a
@@ -166,26 +165,26 @@ def build_families(cached_fonts, folders, family_attr='font-family'):
# system directories).
fmap = {}
remove = []
for f in fonts:
fingerprint = (icu_lower(f['font-family']), f['font-weight'],
f['font-stretch'], f['font-style'])
for font in fonts:
fingerprint = (font['font-family'].lower(), font['font-weight'],
font['font-stretch'], font['font-style'])
if fingerprint in fmap:
opath = fmap[fingerprint]['path']
npath = f['path']
npath = font['path']
if path_significance(npath, folders) >= path_significance(opath, folders):
remove.append(fmap[fingerprint])
fmap[fingerprint] = f
fmap[fingerprint] = font
else:
remove.append(f)
remove.append(font)
else:
fmap[fingerprint] = f
for font in remove:
fonts.remove(font)
fmap[fingerprint] = font
for fnt in remove:
fonts.remove(fnt)
fonts.sort(key=font_priority)
font_family_map = dict.copy(families)
font_families = tuple(sorted((f[0]['font-family'] for f in
itervalues(font_family_map)), key=sort_key))
font_families = tuple(sorted((font[0]['font-family'] for font in
itervalues(font_family_map))))
return font_family_map, font_families
# }}}
@@ -198,8 +197,8 @@ class FontScanner(Thread):
Thread.__init__(self)
self.folders = folders + font_dirs() + [os.path.join(config_dir, 'fonts'),
P('fonts/liberation')]
self.folders = [os.path.normcase(os.path.abspath(f)) for f in
self.folders]
self.folders = [os.path.normcase(os.path.abspath(font)) for font in
self.folders]
self.font_families = ()
self.allowed_extensions = allowed_extensions
@@ -218,7 +217,7 @@ class FontScanner(Thread):
'''
self.join()
try:
return self.font_family_map[icu_lower(family)]
return self.font_family_map[family.lower()]
except KeyError:
raise NoFonts('No fonts found for the family: %r'%family)
@@ -264,7 +263,7 @@ class FontScanner(Thread):
:return: (family name, faces) or None, None
'''
from calibre.utils.fonts.utils import (supports_text,
from ebook_converter.utils.fonts.utils import (supports_text,
panose_to_css_generic_family, get_printable_characters)
if not isinstance(text, unicode_type):
raise TypeError(u'%r is not unicode'%text)
@@ -297,7 +296,7 @@ class FontScanner(Thread):
def reload_cache(self):
if not hasattr(self, 'cache'):
from calibre.utils.config import JSONConfig
from ebook_converter.utils.config import JSONConfig
self.cache = JSONConfig('fonts/scanner_cache')
else:
self.cache.refresh()

8
ebook_converter/utils/fonts/utils.py
View File

@@ -10,7 +10,7 @@ import struct
from io import BytesIO
from collections import defaultdict
from polyglot.builtins import iteritems, itervalues, unicode_type, range, as_bytes
from ebook_converter.polyglot.builtins import iteritems, itervalues, unicode_type, range, as_bytes
class UnsupportedFont(ValueError):
@@ -441,7 +441,7 @@ def get_font_for_text(text, candidate_font_data=None):
if candidate_font_data is not None:
ok = supports_text(candidate_font_data, text)
if not ok:
from calibre.utils.fonts.scanner import font_scanner
from ebook_converter.utils.fonts.scanner import font_scanner
family, faces = font_scanner.find_font_for_text(text)
if faces:
with lopen(faces[0]['path'], 'rb') as f:
@@ -450,7 +450,7 @@ def get_font_for_text(text, candidate_font_data=None):
def test_glyph_ids():
from calibre.utils.fonts.free_type import FreeType
from ebook_converter.utils.fonts.free_type import FreeType
data = P('fonts/liberation/LiberationSerif-Regular.ttf', data=True)
ft = FreeType()
font = ft.load_font(data)
@@ -470,7 +470,7 @@ def test_supports_text():
def test_find_font():
from calibre.utils.fonts.scanner import font_scanner
from ebook_converter.utils.fonts.scanner import font_scanner
abcd = '诶比西迪'
family = font_scanner.find_font_for_text(abcd)[0]
print('Family for Chinese text:', family)

10
ebook_converter/utils/formatter.py
View File

@@ -11,10 +11,10 @@ __docformat__ = 'restructuredtext en'
import re, string, traceback, numbers
from calibre import prints
from calibre.constants import DEBUG
from calibre.utils.formatter_functions import formatter_functions
from polyglot.builtins import unicode_type, error_message
from ebook_converter import prints
from ebook_converter.constants import DEBUG
from ebook_converter.utils.formatter_functions import formatter_functions
from ebook_converter.polyglot.builtins import unicode_type, error_message
class _Parser(object):
@@ -393,7 +393,7 @@ class ValidateFormatter(TemplateFormatter):
return self._validation_string
def validate(self, x):
from calibre.ebooks.metadata.book.base import Metadata
from ebook_converter.ebooks.metadata.book.base import Metadata
return self.safe_format(x, {}, 'VALIDATE ERROR', Metadata(''))

34
ebook_converter/utils/formatter_functions.py
View File

@@ -15,15 +15,15 @@ __docformat__ = 'restructuredtext en'
import inspect, re, traceback, numbers
from math import trunc
from calibre import human_readable
from calibre.constants import DEBUG
from calibre.ebooks.metadata import title_sort
from calibre.utils.config import tweaks
from calibre.utils.titlecase import titlecase
from calibre.utils.icu import capitalize, strcmp, sort_key
from calibre.utils.date import parse_date, format_date, now, UNDEFINED_DATE
from calibre.utils.localization import calibre_langcode_to_name, canonicalize_lang
from polyglot.builtins import iteritems, itervalues, unicode_type
from ebook_converter import human_readable
from ebook_converter.constants import DEBUG
from ebook_converter.ebooks.metadata import title_sort
from ebook_converter.utils.config import tweaks
from ebook_converter.utils.titlecase import titlecase
from ebook_converter.utils.icu import capitalize, strcmp, sort_key
from ebook_converter.utils.date import parse_date, format_date, now, UNDEFINED_DATE
from ebook_converter.utils.localization import calibre_langcode_to_name, canonicalize_lang
from ebook_converter.polyglot.builtins import iteritems, itervalues, unicode_type
class FormatterFunctions(object):
@@ -321,7 +321,7 @@ class BuiltinEval(BuiltinFormatterFunction):
'template program mode.')
def evaluate(self, formatter, kwargs, mi, locals, template):
from calibre.utils.formatter import EvalFormatter
from ebook_converter.utils.formatter import EvalFormatter
template = template.replace('[[', '{').replace(']]', '}')
return EvalFormatter().safe_format(template, locals, 'EVAL', None)
@@ -644,7 +644,7 @@ class BuiltinReGroup(BuiltinFormatterFunction):
"{series:'re_group($, \"(\\S* )(.*)\", \"[[$:uppercase()]]\", \"[[$]]\")'}")
def evaluate(self, formatter, kwargs, mi, locals, val, pattern, *args):
from calibre.utils.formatter import EvalFormatter
from ebook_converter.utils.formatter import EvalFormatter
def repl(mo):
res = ''
@@ -1348,7 +1348,7 @@ class BuiltinListReGroup(BuiltinFormatterFunction):
def evaluate(self, formatter, kwargs, mi, locals, src_list, separator, include_re,
search_re, *args):
from calibre.utils.formatter import EvalFormatter
from ebook_converter.utils.formatter import EvalFormatter
l = [l.strip() for l in src_list.split(separator) if l.strip()]
res = []
@@ -1467,7 +1467,7 @@ class BuiltinCurrentLibraryName(BuiltinFormatterFunction):
'template "{:\'current_library_name()\'}".')
def evaluate(self, formatter, kwargs, mi, locals):
from calibre.library import current_library_name
from ebook_converter.library import current_library_name
return current_library_name()
@@ -1481,7 +1481,7 @@ class BuiltinCurrentLibraryPath(BuiltinFormatterFunction):
'"{:\'current_library_path()\'}".')
def evaluate(self, formatter, kwargs, mi, locals):
from calibre.library import current_library_path
from ebook_converter.library import current_library_path
return current_library_path()
@@ -1551,7 +1551,7 @@ class BuiltinTransliterate(BuiltinFormatterFunction):
u"Фёдор Миха́йлович Достоевский", 'Fiodor Mikhailovich Dostoievskii')
def evaluate(self, formatter, kwargs, mi, locals, source):
from calibre.utils.filenames import ascii_text
from ebook_converter.utils.filenames import ascii_text
return ascii_text(source)
@@ -1648,8 +1648,8 @@ def compile_user_function(name, doc, arg_count, eval_func):
func = ' ' + '\n '.join([tabs.sub(replace_func, line)
for line in eval_func.splitlines()])
prog = '''
from calibre.utils.formatter_functions import FormatterUserFunction
from calibre.utils.formatter_functions import formatter_functions
from ebook_converter.utils.formatter_functions import FormatterUserFunction
from ebook_converter.utils.formatter_functions import formatter_functions
class UserFunction(FormatterUserFunction):
''' + func
locals_ = {}

2
ebook_converter/utils/html2text.py
View File

@@ -8,7 +8,7 @@ def html2text(html):
from html2text import HTML2Text
import re
if isinstance(html, bytes):
from calibre.ebooks.chardet import xml_to_unicode
from ebook_converter.ebooks.chardet import xml_to_unicode
html = xml_to_unicode(html, strip_encoding_pats=True, resolve_entities=True)[0]
# replace <u> tags with <span> as <u> becomes emphasis in html2text
html = re.sub(

76
ebook_converter/utils/icu.py
View File

@@ -7,16 +7,17 @@ __copyright__ = '2010, Kovid Goyal <kovid@kovidgoyal.net>'
__docformat__ = 'restructuredtext en'
import sys
from polyglot.builtins import filter
import unicodedata
from ebook_converter.polyglot.builtins import filter
is_narrow_build = sys.maxunicode < 0x10ffff
# Setup code {{{
import codecs
from calibre.constants import plugins
from calibre.utils.config_base import tweaks
from polyglot.builtins import unicode_type, cmp
from ebook_converter.constants import plugins
from ebook_converter.utils.config_base import tweaks
from ebook_converter.polyglot.builtins import unicode_type, cmp
_locale = _collator = _primary_collator = _sort_collator = _numeric_collator = _case_sensitive_collator = None
cmp
@@ -25,44 +26,44 @@ _none = u''
_none2 = b''
_cmap = {}
_icu, err = plugins['icu']
_icu, err = 1, None # plugins['icu']
if _icu is None:
raise RuntimeError('Failed to load icu with error: %s' % err)
del err
icu_unicode_version = getattr(_icu, 'unicode_version', None)
_nmodes = {m:getattr(_icu, m) for m in ('NFC', 'NFD', 'NFKC', 'NFKD')}
#icu_unicode_version = getattr(_icu, 'unicode_version', None)
# _nmodes = {m:getattr(_icu, m) for m in ('NFC', 'NFD', 'NFKC', 'NFKD')}
# Ensure that the python internal filesystem and default encodings are not ASCII
def is_ascii(name):
try:
return codecs.lookup(name).name == b'ascii'
except (TypeError, LookupError):
return True
try:
if is_ascii(sys.getdefaultencoding()):
_icu.set_default_encoding(b'utf-8')
except:
import traceback
traceback.print_exc()
try:
if is_ascii(sys.getfilesystemencoding()):
_icu.set_filesystem_encoding(b'utf-8')
except:
import traceback
traceback.print_exc()
del is_ascii
#def is_ascii(name):
# try:
# return codecs.lookup(name).name == b'ascii'
# except (TypeError, LookupError):
# return True
#
#
#try:
# if is_ascii(sys.getdefaultencoding()):
# _icu.set_default_encoding(b'utf-8')
#except:
# import traceback
# traceback.print_exc()
#
#try:
# if is_ascii(sys.getfilesystemencoding()):
# _icu.set_filesystem_encoding(b'utf-8')
#except:
# import traceback
# traceback.print_exc()
#del is_ascii
def collator():
global _collator, _locale
if _collator is None:
if _locale is None:
from calibre.utils.localization import get_lang
from ebook_converter.utils.localization import get_lang
if tweaks['locale_for_sorting']:
_locale = tweaks['locale_for_sorting']
else:
@@ -248,9 +249,9 @@ startswith = _make_func(_strcmp_template, 'startswith', collator='_collator', co
primary_startswith = _make_func(_strcmp_template, 'primary_startswith', collator='_primary_collator', collator_func='primary_collator', func='startswith')
safe_chr = _icu.chr
safe_chr = chr # _icu.chr
ord_string = _icu.ord_string
ord_string = str # _icu.ord_string
def character_name(string):
@@ -272,7 +273,8 @@ def normalize(text, mode='NFC'):
# that unless you have very good reasons not too. Also, it's speed
# decreases on wide python builds, where conversion to/from ICU's string
# representation is slower.
return _icu.normalize(_nmodes[mode], unicode_type(text))
# return _icu.normalize(_nmodes[mode], unicode_type(text))
return unicode.normalize(mode, unicode_type(text))
def contractions(col=None):
@@ -311,13 +313,13 @@ def partition_by_first_letter(items, reverse=False, key=lambda x:x):
# Return the number of unicode codepoints in a string
string_length = _icu.string_length if is_narrow_build else len
string_length = len #_icu.string_length if is_narrow_build else len
# Return the number of UTF-16 codepoints in a string
utf16_length = len if is_narrow_build else _icu.utf16_length
utf16_length = len # if is_narrow_build else _icu.utf16_length
################################################################################
if __name__ == '__main__':
from calibre.utils.icu_test import run
run(verbosity=4)
# if __name__ == '__main__':
# from ebook_converter.utils.icu_test import run
# run(verbosity=4)

32
ebook_converter/utils/img.py
View File

@@ -13,21 +13,21 @@ from io import BytesIO
from threading import Thread
# We use explicit module imports so tracebacks when importing are more useful
from PyQt5.QtCore import QBuffer, QByteArray, Qt
from PyQt5.QtGui import QColor, QImage, QImageReader, QImageWriter, QPixmap, QTransform
#from PyQt5.QtCore import QBuffer, QByteArray, Qt
#from PyQt5.QtGui import QColor, QImage, QImageReader, QImageWriter, QPixmap, QTransform
from calibre import fit_image, force_unicode
from calibre.constants import iswindows, plugins, ispy3
from calibre.ptempfile import TemporaryDirectory
from calibre.utils.config_base import tweaks
from calibre.utils.filenames import atomic_rename
from calibre.utils.imghdr import what
from polyglot.builtins import string_or_bytes, unicode_type
from ebook_converter import fit_image, force_unicode
from ebook_converter.constants import iswindows, plugins, ispy3
from ebook_converter.ptempfile import TemporaryDirectory
from ebook_converter.utils.config_base import tweaks
from ebook_converter.utils.filenames import atomic_rename
from ebook_converter.utils.imghdr import what
from ebook_converter.polyglot.builtins import string_or_bytes, unicode_type
# Utilities {{{
imageops, imageops_err = plugins['imageops']
if imageops is None:
raise RuntimeError(imageops_err)
# imageops, imageops_err = plugins['imageops']
# if imageops is None:
# raise RuntimeError(imageops_err)
class NotImage(ValueError):
@@ -42,7 +42,7 @@ def normalize_format_name(fmt):
def get_exe_path(name):
from calibre.ebooks.pdf.pdftohtml import PDFTOHTML
from ebook_converter.ebooks.pdf.pdftohtml import PDFTOHTML
base = os.path.dirname(PDFTOHTML)
if iswindows:
name += '-calibre.exe'
@@ -609,7 +609,7 @@ def optimize_png(file_path, level=7):
def encode_jpeg(file_path, quality=80):
from calibre.utils.speedups import ReadOnlyFileBuffer
from ebook_converter.utils.speedups import ReadOnlyFileBuffer
quality = max(0, min(100, int(quality)))
exe = get_exe_path('cjpeg')
cmd = [exe] + '-optimize -progressive -maxmemory 100M -quality'.split() + [unicode_type(quality)]
@@ -626,8 +626,8 @@ def encode_jpeg(file_path, quality=80):
def test(): # {{{
from calibre.ptempfile import TemporaryDirectory
from calibre import CurrentDir
from ebook_converter.ptempfile import TemporaryDirectory
from ebook_converter import CurrentDir
from glob import glob
img = image_from_data(I('lt.png', data=True, allow_user_override=False))
with TemporaryDirectory() as tdir, CurrentDir(tdir):

6
ebook_converter/utils/imghdr.py
View File

@@ -5,9 +5,9 @@ from __future__ import absolute_import, division, print_function, unicode_litera
from struct import unpack, error
import os
from calibre.utils.speedups import ReadOnlyFileBuffer
from calibre.constants import ispy3
from polyglot.builtins import string_or_bytes, unicode_type
from ebook_converter.utils.speedups import ReadOnlyFileBuffer
from ebook_converter.constants import ispy3
from ebook_converter.polyglot.builtins import string_or_bytes, unicode_type
""" Recognize image file formats and sizes based on their first few bytes."""

8
ebook_converter/utils/ipc/__init__.py
View File

@@ -9,10 +9,10 @@ __docformat__ = 'restructuredtext en'
import os, errno, sys
from threading import Thread
from calibre import force_unicode
from calibre.constants import iswindows, get_windows_username, islinux, filesystem_encoding, ispy3
from calibre.utils.filenames import ascii_filename
from polyglot.functools import lru_cache
from ebook_converter import force_unicode
from ebook_converter.constants import iswindows, get_windows_username, islinux, filesystem_encoding, ispy3
from ebook_converter.utils.filenames import ascii_filename
from ebook_converter.polyglot.functools import lru_cache
VADDRESS = None

12
ebook_converter/utils/ipc/launch.py
View File

@@ -9,12 +9,12 @@ __docformat__ = 'restructuredtext en'
import subprocess, os, sys, time
from functools import partial
from calibre.constants import iswindows, isosx, isfrozen, filesystem_encoding, ispy3
from calibre.utils.config import prefs
from calibre.ptempfile import PersistentTemporaryFile, base_dir
from calibre.utils.serialize import msgpack_dumps
from polyglot.builtins import iteritems, unicode_type, string_or_bytes, environ_item, native_string_type, getcwd
from polyglot.binary import as_hex_unicode
from ebook_converter.constants import iswindows, isosx, isfrozen, filesystem_encoding, ispy3
from ebook_converter.utils.config import prefs
from ebook_converter.ptempfile import PersistentTemporaryFile, base_dir
from ebook_converter.utils.serialize import msgpack_dumps
from ebook_converter.polyglot.builtins import iteritems, unicode_type, string_or_bytes, environ_item, native_string_type, getcwd
from ebook_converter.polyglot.binary import as_hex_unicode
if iswindows:
import win32process

24
ebook_converter/utils/ipc/simple_worker.py
View File

@@ -11,13 +11,13 @@ from multiprocessing.connection import Client
from threading import Thread
from contextlib import closing
from calibre.constants import iswindows
from calibre.utils.ipc import eintr_retry_call
from calibre.utils.ipc.launch import Worker
from calibre.utils.serialize import msgpack_loads, msgpack_dumps
from calibre.utils.monotonic import monotonic
from polyglot.builtins import unicode_type, string_or_bytes, environ_item
from polyglot.binary import as_hex_unicode, from_hex_bytes
from ebook_converter.constants import iswindows
from ebook_converter.utils.ipc import eintr_retry_call
from ebook_converter.utils.ipc.launch import Worker
from ebook_converter.utils.serialize import msgpack_loads, msgpack_dumps
from ebook_converter.utils.monotonic import monotonic
from ebook_converter.polyglot.builtins import unicode_type, string_or_bytes, environ_item
from ebook_converter.polyglot.binary import as_hex_unicode, from_hex_bytes
class WorkerError(Exception):
@@ -125,7 +125,7 @@ def communicate(ans, worker, listener, args, timeout=300, heartbeat=None,
def create_worker(env, priority='normal', cwd=None, func='main'):
from calibre.utils.ipc.server import create_listener
from ebook_converter.utils.ipc.server import create_listener
auth_key = os.urandom(32)
address, listener = create_listener(auth_key)
@@ -133,7 +133,7 @@ def create_worker(env, priority='normal', cwd=None, func='main'):
env.update({
'CALIBRE_WORKER_ADDRESS': environ_item(as_hex_unicode(msgpack_dumps(address))),
'CALIBRE_WORKER_KEY': environ_item(as_hex_unicode(auth_key)),
'CALIBRE_SIMPLE_WORKER': environ_item('calibre.utils.ipc.simple_worker:%s' % func),
'CALIBRE_SIMPLE_WORKER': environ_item('ebook_converter.utils.ipc.simple_worker:%s' % func),
})
w = Worker(env)
@@ -295,14 +295,14 @@ def main():
try:
mod, func, args, kwargs, module_is_source_code = args
if module_is_source_code:
importlib.import_module('calibre.customize.ui') # Load plugins
importlib.import_module('ebook_converter.customize.ui') # Load plugins
mod = compile_code(mod)
func = mod[func]
else:
try:
mod = importlib.import_module(mod)
except ImportError:
importlib.import_module('calibre.customize.ui') # Load plugins
importlib.import_module('ebook_converter.customize.ui') # Load plugins
mod = importlib.import_module(mod)
func = getattr(mod, func)
res = {'result':func(*args, **kwargs)}
@@ -337,7 +337,7 @@ def offload():
try:
m = importlib.import_module(mod)
except ImportError:
importlib.import_module('calibre.customize.ui') # Load plugins
importlib.import_module('ebook_converter.customize.ui') # Load plugins
m = importlib.import_module(mod)
func_cache[(mod, func)] = f = getattr(m, func)
res['result'] = f(*args, **kwargs)

8
ebook_converter/utils/iso8601.py
View File

@@ -7,10 +7,10 @@ from datetime import datetime
from dateutil.tz import tzlocal, tzutc, tzoffset
from calibre.constants import plugins
speedup, err = plugins['speedup']
if not speedup:
raise RuntimeError(err)
from ebook_converter.constants import plugins
# speedup, err = plugins['speedup']
# if not speedup:
# raise RuntimeError(err)
class SafeLocalTimeZone(tzlocal):

52
ebook_converter/utils/localization.py
View File

@@ -1,7 +1,3 @@
#!/usr/bin/env python2
# vim:fileencoding=UTF-8:ts=4:sw=4:sta:et:sts=4:ai
from __future__ import absolute_import, division, print_function, unicode_literals
__license__ = 'GPL v3'
__copyright__ = '2009, Kovid Goyal <kovid@kovidgoyal.net>'
__docformat__ = 'restructuredtext en'
@@ -9,7 +5,7 @@ __docformat__ = 'restructuredtext en'
import os, locale, re, io, sys
from gettext import GNUTranslations, NullTranslations
from polyglot.builtins import is_py3, iteritems, unicode_type
from ebook_converter.polyglot.builtins import is_py3, iteritems, unicode_type
_available_translations = None
@@ -19,7 +15,7 @@ def available_translations():
if _available_translations is None:
stats = P('localization/stats.calibre_msgpack', allow_user_override=False)
if os.path.exists(stats):
from calibre.utils.serialize import msgpack_loads
from ebook_converter.utils.serialize import msgpack_loads
with open(stats, 'rb') as f:
stats = msgpack_loads(f.read())
else:
@@ -29,11 +25,11 @@ def available_translations():
def get_system_locale():
from calibre.constants import iswindows, isosx, plugins
from ebook_converter.constants import iswindows, isosx, plugins
lang = None
if iswindows:
try:
from calibre.constants import get_windows_user_locale_name
from ebook_converter.constants import get_windows_user_locale_name
lang = get_windows_user_locale_name()
lang = lang.strip()
if not lang:
@@ -86,7 +82,7 @@ def sanitize_lang(lang):
def get_lang():
'Try to figure out what language to display the interface in'
from calibre.utils.config_base import prefs
from ebook_converter.utils.config_base import prefs
lang = prefs['language']
lang = os.environ.get('CALIBRE_OVERRIDE_LANG', lang)
if lang:
@@ -197,7 +193,7 @@ lcdata = {
def load_po(path):
from calibre.translations.msgfmt import make
from ebook_converter.translations.msgfmt import make
buf = io.BytesIO()
try:
make(path, buf)
@@ -238,7 +234,7 @@ def set_translators():
except:
pass # No iso639 translations for this lang
if buf is not None:
from calibre.utils.serialize import msgpack_loads
from ebook_converter.utils.serialize import msgpack_loads
try:
lcdata = msgpack_loads(zf.read(mpath + '/lcdata.calibre_msgpack'))
except:
@@ -264,7 +260,7 @@ def set_translators():
# Now that we have installed a translator, we have to retranslate the help
# for the global prefs object as it was instantiated in get_lang(), before
# the translator was installed.
from calibre.utils.config_base import prefs
from ebook_converter.utils.config_base import prefs
prefs.retranslate_help()
@@ -367,7 +363,7 @@ def _load_iso639():
global _iso639
if _iso639 is None:
ip = P('localization/iso639.calibre_msgpack', allow_user_override=False, data=True)
from calibre.utils.serialize import msgpack_loads
from ebook_converter.utils.serialize import msgpack_loads
_iso639 = msgpack_loads(ip)
if 'by_3' not in _iso639:
_iso639['by_3'] = _iso639['by_3t']
@@ -491,7 +487,7 @@ _udc = None
def get_udc():
global _udc
if _udc is None:
from calibre.ebooks.unihandecode import Unihandecoder
from ebook_converter.ebooks.unihandecode import Unihandecoder
_udc = Unihandecoder(lang=get_lang())
return _udc
@@ -509,19 +505,19 @@ def user_manual_stats():
def localize_user_manual_link(url):
lc = lang_as_iso639_1(get_lang())
if lc == 'en':
return url
stats = user_manual_stats()
if stats.get(lc, 0) < 0.3:
return url
from polyglot.urllib import urlparse, urlunparse
parts = urlparse(url)
path = re.sub(r'/generated/[a-z]+/', '/generated/%s/' % lc, parts.path or '')
path = '/%s%s' % (lc, path)
parts = list(parts)
parts[2] = path
return urlunparse(parts)
#lc = lang_as_iso639_1(get_lang())
# if lc == 'en':
return url
# stats = user_manual_stats()
# if stats.get(lc, 0) < 0.3:
# return url
# from polyglot.urllib import urlparse, urlunparse
# parts = urlparse(url)
# path = re.sub(r'/generated/[a-z]+/', '/generated/%s/' % lc, parts.path or '')
# path = '/%s%s' % (lc, path)
# parts = list(parts)
# parts[2] = path
# return urlunparse(parts)
def website_languages():
@@ -540,7 +536,7 @@ def localize_website_link(url):
langs = website_languages()
if lc == 'en' or lc not in langs:
return url
from polyglot.urllib import urlparse, urlunparse
from ebook_converter.polyglot.urllib import urlparse, urlunparse
parts = urlparse(url)
path = '/{}{}'.format(lc, parts.path)
parts = list(parts)

6
ebook_converter/utils/localunzip.py
View File

@@ -18,7 +18,7 @@ from struct import calcsize, unpack, pack
from collections import namedtuple, OrderedDict
from tempfile import SpooledTemporaryFile
from polyglot.builtins import itervalues, getcwd
from ebook_converter.polyglot.builtins import itervalues, getcwd
HEADER_SIG = 0x04034b50
HEADER_BYTE_SIG = pack(b'<L', HEADER_SIG)
@@ -49,7 +49,7 @@ else:
def decode_arcname(name):
if isinstance(name, bytes):
from calibre.ebooks.chardet import detect
from ebook_converter.ebooks.chardet import detect
try:
name = name.decode('utf-8')
except:
@@ -301,7 +301,7 @@ class LocalZipFile(object):
def safe_replace(self, name, datastream, extra_replacements={},
add_missing=False):
from calibre.utils.zipfile import ZipFile, ZipInfo
from ebook_converter.utils.zipfile import ZipFile, ZipInfo
replacements = {name:datastream}
replacements.update(extra_replacements)
names = frozenset(list(replacements.keys()))

41
ebook_converter/utils/lock.py
View File

@@ -1,9 +1,5 @@
#!/usr/bin/env python2
# vim:fileencoding=utf-8
# License: GPLv3 Copyright: 2017, Kovid Goyal <kovid at kovidgoyal.net>
from __future__ import absolute_import, division, print_function, unicode_literals
import atexit
import errno
import os
@@ -12,15 +8,15 @@ import tempfile
import time
from functools import partial
from calibre.constants import (
from ebook_converter.constants import (
__appname__, fcntl, filesystem_encoding, islinux, isosx, iswindows, plugins, ispy3
)
from calibre.utils.monotonic import monotonic
from ebook_converter.utils.monotonic import monotonic
speedup = plugins['speedup'][0]
# speedup = plugins['speedup'][0]
if iswindows:
import msvcrt, win32file, pywintypes, winerror, win32api, win32event
from calibre.constants import get_windows_username
from ebook_converter.constants import get_windows_username
excl_file_mode = stat.S_IREAD | stat.S_IWRITE
else:
excl_file_mode = stat.S_IRUSR | stat.S_IWUSR | stat.S_IRGRP | stat.S_IROTH
@@ -29,14 +25,14 @@ else:
def unix_open(path):
flags = os.O_RDWR | os.O_CREAT
has_cloexec = False
if hasattr(speedup, 'O_CLOEXEC'):
try:
fd = os.open(path, flags | speedup.O_CLOEXEC, excl_file_mode)
has_cloexec = True
except EnvironmentError as err:
# Kernel may not support O_CLOEXEC
if err.errno != errno.EINVAL:
raise
#if hasattr(speedup, 'O_CLOEXEC'):
# try:
# fd = os.open(path, flags | speedup.O_CLOEXEC, excl_file_mode)
# has_cloexec = True
# except EnvironmentError as err:
# # Kernel may not support O_CLOEXEC
# if err.errno != errno.EINVAL:
# raise
if not has_cloexec:
fd = os.open(path, flags, excl_file_mode)
@@ -85,7 +81,7 @@ def retry_for_a_time(timeout, sleep_time, func, error_retry, *args):
time.sleep(sleep_time)
def lock_file(path, timeout=15, sleep_time=0.2):
def _lock_file(path, timeout=15, sleep_time=0.2):
if iswindows:
return retry_for_a_time(
timeout, sleep_time, windows_open, windows_retry, path
@@ -98,6 +94,13 @@ def lock_file(path, timeout=15, sleep_time=0.2):
return f
def lock_file(path, timeout=15, sleep_time=0.2):
from filelock import FileLock
lock = FileLock(path + '.lock', timeout=timeout)
with lock:
return unix_open(path)
class ExclusiveFile(object):
def __init__(self, path, timeout=15, sleep_time=0.2):
@@ -147,7 +150,7 @@ elif islinux:
def create_single_instance_mutex(name, per_user=True):
import socket
from calibre.utils.ipc import eintr_retry_call
from ebook_converter.utils.ipc import eintr_retry_call
name = '%s-singleinstance-%s-%s' % (
__appname__, (os.geteuid() if per_user else ''), name
)
@@ -185,7 +188,7 @@ else:
)
def create_single_instance_mutex(name, per_user=True):
from calibre.utils.ipc import eintr_retry_call
from ebook_converter.utils.ipc import eintr_retry_call
path = singleinstance_path(name, per_user)
f = lopen(path, 'w')
try:

6
ebook_converter/utils/logging.py
View File

@@ -14,8 +14,8 @@ import sys, traceback, io
from functools import partial
from threading import Lock
from calibre import isbytestring, force_unicode, as_unicode, prints
from polyglot.builtins import unicode_type, iteritems
from ebook_converter import isbytestring, force_unicode, as_unicode, prints
from ebook_converter.polyglot.builtins import unicode_type, iteritems
class Stream(object):
@@ -45,7 +45,7 @@ class ANSIStream(Stream):
}
def prints(self, level, *args, **kwargs):
from calibre.utils.terminal import ColoredStream
from ebook_converter.utils.terminal import ColoredStream
with ColoredStream(self.stream, self.color[level]):
self._prints(*args, **kwargs)

10
ebook_converter/utils/resources.py
View File

@@ -1,7 +1,3 @@
#!/usr/bin/env python2
# vim:fileencoding=UTF-8:ts=4:sw=4:sta:et:sts=4:ai
from __future__ import absolute_import, division, print_function, unicode_literals
__license__ = 'GPL v3'
__copyright__ = '2009, Kovid Goyal <kovid@kovidgoyal.net>'
__docformat__ = 'restructuredtext en'
@@ -9,8 +5,8 @@ __docformat__ = 'restructuredtext en'
import sys, os
from calibre import config_dir
from polyglot.builtins import builtins
from ebook_converter import config_dir
from ebook_converter.polyglot.builtins import builtins
user_dir = os.path.join(config_dir, 'resources')
@@ -19,6 +15,7 @@ user_dir = os.path.join(config_dir, 'resources')
class PathResolver(object):
def __init__(self):
sys.resources_location = os.path.abspath(os.path.dirname(__file__))
self.locations = [sys.resources_location]
self.cache = {}
@@ -86,6 +83,7 @@ _resolver = PathResolver()
def get_path(path, data=False, allow_user_override=True):
fpath = _resolver(path, allow_user_override=allow_user_override)
if data:
with open(fpath, 'rb') as f:
return f.read()

22
ebook_converter/utils/serialize.py
View File

@@ -4,8 +4,8 @@
from __future__ import absolute_import, division, print_function, unicode_literals
from polyglot.builtins import unicode_type
from calibre.constants import ispy3
from ebook_converter.polyglot.builtins import unicode_type
from ebook_converter.constants import ispy3
MSGPACK_MIME = 'application/x-msgpack'
@@ -31,11 +31,11 @@ def create_encoder(for_json=False):
if isinstance(obj, (set, frozenset)):
return encoded(1, tuple(obj), ExtType)
if getattr(obj, '__calibre_serializable__', False):
from calibre.ebooks.metadata.book.base import Metadata
from calibre.library.field_metadata import FieldMetadata, fm_as_dict
from calibre.db.categories import Tag
from ebook_converter.ebooks.metadata.book.base import Metadata
from ebook_converter.library.field_metadata import FieldMetadata, fm_as_dict
from ebook_converter.db.categories import Tag
if isinstance(obj, Metadata):
from calibre.ebooks.metadata.book.serialize import metadata_as_dict
from ebook_converter.ebooks.metadata.book.serialize import metadata_as_dict
return encoded(
2, metadata_as_dict(obj, encode_cover_data=for_json), ExtType
)
@@ -66,8 +66,8 @@ def json_dumps(data, **kw):
def decode_metadata(x, for_json):
from polyglot.binary import from_base64_bytes
from calibre.ebooks.metadata.book.serialize import metadata_from_dict
from ebook_converter.polyglot.binary import from_base64_bytes
from ebook_converter.ebooks.metadata.book.serialize import metadata_from_dict
obj = metadata_from_dict(x)
if for_json and obj.cover_data and obj.cover_data[1]:
obj.cover_data = obj.cover_data[0], from_base64_bytes(obj.cover_data[1])
@@ -75,17 +75,17 @@ def decode_metadata(x, for_json):
def decode_field_metadata(x, for_json):
from calibre.library.field_metadata import fm_from_dict
from ebook_converter.library.field_metadata import fm_from_dict
return fm_from_dict(x)
def decode_category_tag(x, for_json):
from calibre.db.categories import Tag
from ebook_converter.db.categories import Tag
return Tag.from_dict(x)
def decode_datetime(x, fj):
from calibre.utils.iso8601 import parse_iso8601
from ebook_converter.utils.iso8601 import parse_iso8601
return parse_iso8601(x, assume_utc=True)

119
ebook_converter/utils/shared_file.py
View File

@@ -7,9 +7,9 @@ __copyright__ = '2015, Kovid Goyal <kovid at kovidgoyal.net>'
import os, sys
from polyglot.builtins import reraise
from ebook_converter.polyglot.builtins import reraise
from calibre.constants import iswindows, plugins, ispy3
from ebook_converter.constants import iswindows, plugins, ispy3
'''
This module defines a share_open() function which is a replacement for
@@ -28,10 +28,10 @@ directory until all file handles are closed. To get around this, rename the
file before deleting it.
'''
speedup, err = plugins['speedup']
# speedup, err = plugins['speedup']
if not speedup:
raise RuntimeError('Failed to load the speedup plugin with error: %s' % err)
# if not speedup:
# raise RuntimeError('Failed to load the speedup plugin with error: %s' % err)
valid_modes = {'a', 'a+', 'a+b', 'ab', 'r', 'rb', 'r+', 'r+b', 'w', 'wb', 'w+', 'w+b'}
@@ -81,117 +81,12 @@ def flags_from_mode(mode):
return flags
if iswindows:
from numbers import Integral
import msvcrt
import win32file, pywintypes
CREATE_NEW = win32file.CREATE_NEW
CREATE_ALWAYS = win32file.CREATE_ALWAYS
OPEN_EXISTING = win32file.OPEN_EXISTING
OPEN_ALWAYS = win32file.OPEN_ALWAYS
TRUNCATE_EXISTING = win32file.TRUNCATE_EXISTING
FILE_SHARE_READ = win32file.FILE_SHARE_READ
FILE_SHARE_WRITE = win32file.FILE_SHARE_WRITE
FILE_SHARE_DELETE = win32file.FILE_SHARE_DELETE
FILE_SHARE_VALID_FLAGS = FILE_SHARE_READ | FILE_SHARE_WRITE | FILE_SHARE_DELETE
FILE_ATTRIBUTE_READONLY = win32file.FILE_ATTRIBUTE_READONLY
FILE_ATTRIBUTE_NORMAL = win32file.FILE_ATTRIBUTE_NORMAL
FILE_ATTRIBUTE_TEMPORARY = win32file.FILE_ATTRIBUTE_TEMPORARY
FILE_FLAG_DELETE_ON_CLOSE = win32file.FILE_FLAG_DELETE_ON_CLOSE
FILE_FLAG_SEQUENTIAL_SCAN = win32file.FILE_FLAG_SEQUENTIAL_SCAN
FILE_FLAG_RANDOM_ACCESS = win32file.FILE_FLAG_RANDOM_ACCESS
GENERIC_READ = win32file.GENERIC_READ & 0xffffffff
GENERIC_WRITE = win32file.GENERIC_WRITE & 0xffffffff
DELETE = 0x00010000
_ACCESS_MASK = os.O_RDONLY | os.O_WRONLY | os.O_RDWR
_ACCESS_MAP = {
os.O_RDONLY : GENERIC_READ,
os.O_WRONLY : GENERIC_WRITE,
os.O_RDWR : GENERIC_READ | GENERIC_WRITE
}
_CREATE_MASK = os.O_CREAT | os.O_EXCL | os.O_TRUNC
_CREATE_MAP = {
0 : OPEN_EXISTING,
os.O_EXCL : OPEN_EXISTING,
os.O_CREAT : OPEN_ALWAYS,
os.O_CREAT | os.O_EXCL : CREATE_NEW,
os.O_CREAT | os.O_TRUNC | os.O_EXCL : CREATE_NEW,
os.O_TRUNC : TRUNCATE_EXISTING,
os.O_TRUNC | os.O_EXCL : TRUNCATE_EXISTING,
os.O_CREAT | os.O_TRUNC : CREATE_ALWAYS
}
def raise_winerror(pywinerr):
reraise(
WindowsError,
WindowsError(pywinerr.winerror,
(pywinerr.funcname or '') + b': ' + (pywinerr.strerror or '')),
sys.exc_info()[2])
def os_open(path, flags, mode=0o777, share_flags=FILE_SHARE_VALID_FLAGS):
'''
Replacement for os.open() allowing moving or unlinking before closing
'''
if not isinstance(flags, Integral):
raise TypeError('flags must be an integer')
if not isinstance(mode, Integral):
raise TypeError('mode must be an integer')
if share_flags & ~FILE_SHARE_VALID_FLAGS:
raise ValueError('bad share_flags: %r' % share_flags)
access_flags = _ACCESS_MAP[flags & _ACCESS_MASK]
create_flags = _CREATE_MAP[flags & _CREATE_MASK]
attrib_flags = FILE_ATTRIBUTE_NORMAL
if flags & os.O_CREAT and mode & ~0o444 == 0:
attrib_flags = FILE_ATTRIBUTE_READONLY
if flags & os.O_TEMPORARY:
share_flags |= FILE_SHARE_DELETE
attrib_flags |= FILE_FLAG_DELETE_ON_CLOSE
access_flags |= DELETE
if flags & os.O_SHORT_LIVED:
attrib_flags |= FILE_ATTRIBUTE_TEMPORARY
if flags & os.O_SEQUENTIAL:
attrib_flags |= FILE_FLAG_SEQUENTIAL_SCAN
if flags & os.O_RANDOM:
attrib_flags |= FILE_FLAG_RANDOM_ACCESS
try:
h = win32file.CreateFileW(
path, access_flags, share_flags, None, create_flags, attrib_flags, None)
except pywintypes.error as e:
raise_winerror(e)
ans = msvcrt.open_osfhandle(h, flags | os.O_NOINHERIT)
h.Detach() # We dont want the handle to be automatically closed when h is deleted
return ans
def share_open(path, mode='r', buffering=-1):
flags = flags_from_mode(mode)
return speedup.fdopen(os_open(path, flags), path, mode, buffering)
else:
if ispy3:
# See PEP 446
share_open = open
else:
def share_open(path, mode='r', buffering=-1):
flags = flags_from_mode(mode) | speedup.O_CLOEXEC
return speedup.fdopen(os.open(path, flags), path, mode, buffering)
def raise_winerror(x):
reraise(NotImplementedError, None, sys.exc_info()[2])
share_open = open
def find_tests():
import unittest
from calibre.ptempfile import TemporaryDirectory
from ebook_converter.ptempfile import TemporaryDirectory
class SharedFileTest(unittest.TestCase):

2
ebook_converter/utils/short_uuid.py
View File

@@ -9,7 +9,7 @@ Generate UUID encoded using a user specified alphabet.
import string, math, uuid as _uuid
from polyglot.builtins import unicode_type
from ebook_converter.polyglot.builtins import unicode_type
def num_to_string(number, alphabet, alphabet_len, pad_to_length=None):

2
ebook_converter/utils/speedups.py
View File

@@ -5,7 +5,7 @@
from __future__ import absolute_import, division, print_function, unicode_literals
import os
from polyglot.builtins import range, unicode_type
from ebook_converter.polyglot.builtins import range, unicode_type
class ReadOnlyFileBuffer(object):

6
ebook_converter/utils/terminal.py
View File

@@ -8,8 +8,8 @@ __docformat__ = 'restructuredtext en'
import os, sys, re
from calibre.constants import iswindows, ispy3
from polyglot.builtins import iteritems, range, zip, native_string_type
from ebook_converter.constants import iswindows, ispy3
from ebook_converter.polyglot.builtins import iteritems, range, zip, native_string_type
if iswindows:
import ctypes.wintypes
@@ -174,7 +174,7 @@ class Detect(object):
# message and the asciized version of the text.
print('Non-ASCII text detected. You must set your Console\'s font to'
' Lucida Console or Consolas or some other TrueType font to see this text', file=self.stream, end=' -- ')
from calibre.utils.filenames import ascii_text
from ebook_converter.utils.filenames import ascii_text
print(ascii_text(t + text), file=self.stream, end='')
continue
if not ignore_errors:

6
ebook_converter/utils/titlecase.py
View File

@@ -11,8 +11,8 @@ License: http://www.opensource.org/licenses/mit-license.php
import re
from calibre.utils.icu import capitalize, upper
from polyglot.builtins import unicode_type
from ebook_converter.utils.icu import capitalize, upper
from ebook_converter.polyglot.builtins import unicode_type
__all__ = ['titlecase']
__version__ = '0.5'
@@ -37,7 +37,7 @@ _lang = None
def lang():
global _lang
if _lang is None:
from calibre.utils.localization import get_lang
from ebook_converter.utils.localization import get_lang
_lang = get_lang().lower()
return _lang

2
ebook_converter/utils/xml_parse.py
View File

@@ -63,5 +63,5 @@ def find_tests():
if __name__ == '__main__':
from calibre.utils.run_tests import run_tests
from ebook_converter.utils.run_tests import run_tests
run_tests(find_tests)

8
ebook_converter/utils/zipfile.py
View File

@@ -8,10 +8,10 @@ import binascii
from contextlib import closing
from tempfile import SpooledTemporaryFile
from calibre import sanitize_file_name
from calibre.constants import filesystem_encoding
from calibre.ebooks.chardet import detect
from polyglot.builtins import unicode_type, string_or_bytes, getcwd, map, as_bytes
from ebook_converter import sanitize_file_name
from ebook_converter.constants import filesystem_encoding
from ebook_converter.ebooks.chardet import detect
from ebook_converter.polyglot.builtins import unicode_type, string_or_bytes, getcwd, map, as_bytes
try:
import zlib # We may need its compression method