| Current Path : /proc/thread-self/root/usr/local/lib/python3.8/dist-packages/pdfminer/ |
Linux ift1.ift-informatik.de 5.4.0-216-generic #236-Ubuntu SMP Fri Apr 11 19:53:21 UTC 2025 x86_64 |
| Current File : //proc/thread-self/root/usr/local/lib/python3.8/dist-packages/pdfminer/cmapdb.py |
#!/usr/bin/env python
""" Adobe character mapping (CMap) support.
CMaps provide the mapping between character codes and Unicode
code-points to character ids (CIDs).
More information is available on the Adobe website:
http://opensource.adobe.com/wiki/display/cmap/CMap+Resources
"""
import sys
import os
import os.path
import gzip
import codecs
import marshal
import struct
import logging
from .psparser import PSStackParser
from .psparser import PSSyntaxError
from .psparser import PSEOF
from .psparser import PSLiteral
from .psparser import literal_name
from .psparser import KWD
from .encodingdb import name2unicode
from .utils import choplist
from .utils import nunpack
class CMapError(Exception):
pass
## CMapBase
##
class CMapBase:
debug = 0
def __init__(self, **kwargs):
self.attrs = kwargs.copy()
return
def is_vertical(self):
return self.attrs.get('WMode', 0) != 0
def set_attr(self, k, v):
self.attrs[k] = v
return
def add_code2cid(self, code, cid):
return
def add_cid2unichr(self, cid, code):
return
def use_cmap(self, cmap):
return
## CMap
##
class CMap(CMapBase):
def __init__(self, **kwargs):
CMapBase.__init__(self, **kwargs)
self.code2cid = {}
return
def __repr__(self):
return '<CMap: %s>' % self.attrs.get('CMapName')
def use_cmap(self, cmap):
assert isinstance(cmap, CMap)
def copy(dst, src):
for (k, v) in src.items():
if isinstance(v, dict):
d = {}
dst[k] = d
copy(d, v)
else:
dst[k] = v
copy(self.code2cid, cmap.code2cid)
return
def decode(self, code):
if self.debug:
logging.debug('decode: %r, %r' % (self, code))
d = self.code2cid
for c in code:
if c in d:
d = d[c]
if isinstance(d, int):
yield d
d = self.code2cid
else:
d = self.code2cid
return
def dump(self, out=sys.stdout, code2cid=None, code=None):
if code2cid is None:
code2cid = self.code2cid
code = ()
for (k, v) in sorted(code2cid.items()):
c = code+(k,)
if isinstance(v, int):
out.write('code %r = cid %d\n' % (c, v))
else:
self.dump(out=out, code2cid=v, code=c)
return
## IdentityCMap
##
class IdentityCMap(CMapBase):
def decode(self, code):
n = len(code)//2
if n:
return struct.unpack('>%dH' % n, code)
else:
return ()
## UnicodeMap
##
class UnicodeMap(CMapBase):
def __init__(self, **kwargs):
CMapBase.__init__(self, **kwargs)
self.cid2unichr = {}
return
def __repr__(self):
return '<UnicodeMap: %s>' % self.attrs.get('CMapName')
def get_unichr(self, cid):
if self.debug:
logging.debug('get_unichr: %r, %r' % (self, cid))
return self.cid2unichr[cid]
def dump(self, out=sys.stdout):
for (k, v) in sorted(self.cid2unichr.items()):
out.write('cid %d = unicode %r\n' % (k, v))
return
## FileCMap
##
class FileCMap(CMap):
def add_code2cid(self, code, cid):
assert isinstance(code, bytes) and isinstance(cid, int)
d = self.code2cid
for c in code[:-1]:
c = ord(c)
if c in d:
d = d[c]
else:
t = {}
d[c] = t
d = t
c = ord(code[-1])
d[c] = cid
return
## FileUnicodeMap
##
class FileUnicodeMap(UnicodeMap):
def add_cid2unichr(self, cid, code):
assert isinstance(cid, int)
if isinstance(code, PSLiteral):
# Interpret as an Adobe glyph name.
self.cid2unichr[cid] = name2unicode(code.name)
elif isinstance(code, bytes):
# Interpret as UTF-16BE.
self.cid2unichr[cid] = code.decode('UTF-16BE', 'ignore')
elif isinstance(code, int):
self.cid2unichr[cid] = chr(code)
else:
raise TypeError(code)
return
## PyCMap
##
class PyCMap(CMap):
def __init__(self, name, module):
CMap.__init__(self, CMapName=name)
self.code2cid = module.CODE2CID
if module.IS_VERTICAL:
self.attrs['WMode'] = 1
return
## PyUnicodeMap
##
class PyUnicodeMap(UnicodeMap):
def __init__(self, name, module, vertical):
UnicodeMap.__init__(self, CMapName=name)
if vertical:
self.cid2unichr = module.CID2UNICHR_V
self.attrs['WMode'] = 1
else:
self.cid2unichr = module.CID2UNICHR_H
return
## CMapDB
##
class CMapDB:
_cmap_cache = {}
_umap_cache = {}
class CMapNotFound(CMapError):
pass
@classmethod
def _load_data(klass, name):
filename = '%s.marshal.gz' % name
logging.info('loading: %r' % name)
cmap_paths = (os.environ.get('CMAP_PATH', '/usr/share/pdfminer/'),
os.path.join(os.path.dirname(__file__), 'cmap'),)
for directory in cmap_paths:
path = os.path.join(directory, filename)
if os.path.exists(path):
gzfile = gzip.open(path)
try:
return type(str(name), (), marshal.loads(gzfile.read()))
finally:
gzfile.close()
else:
raise CMapDB.CMapNotFound(name)
@classmethod
def get_cmap(klass, name):
if name == 'Identity-H':
return IdentityCMap(WMode=0)
elif name == 'Identity-V':
return IdentityCMap(WMode=1)
try:
return klass._cmap_cache[name]
except KeyError:
pass
data = klass._load_data(name)
klass._cmap_cache[name] = cmap = PyCMap(name, data)
return cmap
@classmethod
def get_unicode_map(klass, name, vertical=False):
try:
return klass._umap_cache[name][vertical]
except KeyError:
pass
data = klass._load_data('to-unicode-%s' % name)
klass._umap_cache[name] = umaps = [PyUnicodeMap(name, data, v) for v in (False, True)]
return umaps[vertical]
## CMapParser
##
class CMapParser(PSStackParser):
def __init__(self, cmap, fp):
PSStackParser.__init__(self, fp)
self.cmap = cmap
# some ToUnicode maps don't have "begincmap" keyword.
self._in_cmap = True
return
def run(self):
try:
self.nextobject()
except PSEOF:
pass
return
KEYWORD_BEGINCMAP = KWD(b'begincmap')
KEYWORD_ENDCMAP = KWD(b'endcmap')
KEYWORD_USECMAP = KWD(b'usecmap')
KEYWORD_DEF = KWD(b'def')
KEYWORD_BEGINCODESPACERANGE = KWD(b'begincodespacerange')
KEYWORD_ENDCODESPACERANGE = KWD(b'endcodespacerange')
KEYWORD_BEGINCIDRANGE = KWD(b'begincidrange')
KEYWORD_ENDCIDRANGE = KWD(b'endcidrange')
KEYWORD_BEGINCIDCHAR = KWD(b'begincidchar')
KEYWORD_ENDCIDCHAR = KWD(b'endcidchar')
KEYWORD_BEGINBFRANGE = KWD(b'beginbfrange')
KEYWORD_ENDBFRANGE = KWD(b'endbfrange')
KEYWORD_BEGINBFCHAR = KWD(b'beginbfchar')
KEYWORD_ENDBFCHAR = KWD(b'endbfchar')
KEYWORD_BEGINNOTDEFRANGE = KWD(b'beginnotdefrange')
KEYWORD_ENDNOTDEFRANGE = KWD(b'endnotdefrange')
def do_keyword(self, pos, token):
if token is self.KEYWORD_BEGINCMAP:
self._in_cmap = True
self.popall()
return
elif token is self.KEYWORD_ENDCMAP:
self._in_cmap = False
return
if not self._in_cmap:
return
#
if token is self.KEYWORD_DEF:
try:
((_, k), (_, v)) = self.pop(2)
self.cmap.set_attr(literal_name(k), v)
except PSSyntaxError:
pass
return
if token is self.KEYWORD_USECMAP:
try:
((_, cmapname),) = self.pop(1)
self.cmap.use_cmap(CMapDB.get_cmap(literal_name(cmapname)))
except PSSyntaxError:
pass
except CMapDB.CMapNotFound:
pass
return
if token is self.KEYWORD_BEGINCODESPACERANGE:
self.popall()
return
if token is self.KEYWORD_ENDCODESPACERANGE:
self.popall()
return
if token is self.KEYWORD_BEGINCIDRANGE:
self.popall()
return
if token is self.KEYWORD_ENDCIDRANGE:
objs = [obj for (__, obj) in self.popall()]
for (s, e, cid) in choplist(3, objs):
if (not isinstance(s, bytes) or not isinstance(e, bytes) or
not isinstance(cid, int) or len(s) != len(e)):
continue
sprefix = s[:-4]
eprefix = e[:-4]
if sprefix != eprefix:
continue
svar = s[-4:]
evar = e[-4:]
s1 = nunpack(svar)
e1 = nunpack(evar)
vlen = len(svar)
#assert s1 <= e1
for i in range(e1-s1+1):
x = sprefix+struct.pack('>L', s1+i)[-vlen:]
self.cmap.add_code2cid(x, cid+i)
return
if token is self.KEYWORD_BEGINCIDCHAR:
self.popall()
return
if token is self.KEYWORD_ENDCIDCHAR:
objs = [obj for (__, obj) in self.popall()]
for (cid, code) in choplist(2, objs):
if isinstance(code, bytes) and isinstance(cid, bytes):
self.cmap.add_code2cid(code, nunpack(cid))
return
if token is self.KEYWORD_BEGINBFRANGE:
self.popall()
return
if token is self.KEYWORD_ENDBFRANGE:
objs = [obj for (__, obj) in self.popall()]
for (s, e, code) in choplist(3, objs):
if (not isinstance(s, bytes) or not isinstance(e, bytes) or
len(s) != len(e)):
continue
s1 = nunpack(s)
e1 = nunpack(e)
#assert s1 <= e1
if isinstance(code, list):
for i in range(e1-s1+1):
self.cmap.add_cid2unichr(s1+i, code[i])
else:
var = code[-4:]
base = nunpack(var)
prefix = code[:-4]
vlen = len(var)
for i in range(e1-s1+1):
x = prefix+struct.pack('>L', base+i)[-vlen:]
self.cmap.add_cid2unichr(s1+i, x)
return
if token is self.KEYWORD_BEGINBFCHAR:
self.popall()
return
if token is self.KEYWORD_ENDBFCHAR:
objs = [obj for (__, obj) in self.popall()]
for (cid, code) in choplist(2, objs):
if isinstance(cid, bytes) and isinstance(code, bytes):
self.cmap.add_cid2unichr(nunpack(cid), code)
return
if token is self.KEYWORD_BEGINNOTDEFRANGE:
self.popall()
return
if token is self.KEYWORD_ENDNOTDEFRANGE:
self.popall()
return
self.push((pos, token))
return
## CMapConverter
##
class CMapConverter:
def __init__(self, enc2codec={}):
self.enc2codec = enc2codec
self.code2cid = {} # {'cmapname': ...}
self.is_vertical = {}
self.cid2unichr_h = {} # {cid: unichr}
self.cid2unichr_v = {} # {cid: unichr}
return
def get_encs(self):
return self.code2cid.keys()
def get_maps(self, enc):
if enc.endswith('-H'):
(hmapenc, vmapenc) = (enc, None)
elif enc == 'H':
(hmapenc, vmapenc) = ('H', 'V')
else:
(hmapenc, vmapenc) = (enc+'-H', enc+'-V')
if hmapenc in self.code2cid:
hmap = self.code2cid[hmapenc]
else:
hmap = {}
self.code2cid[hmapenc] = hmap
vmap = None
if vmapenc:
self.is_vertical[vmapenc] = True
if vmapenc in self.code2cid:
vmap = self.code2cid[vmapenc]
else:
vmap = {}
self.code2cid[vmapenc] = vmap
return (hmap, vmap)
def load(self, fp):
encs = None
for line in fp:
(line,_,_) = line.strip().partition('#')
if not line: continue
values = line.split('\t')
if encs is None:
assert values[0] == 'CID'
encs = values
continue
def put(dmap, code, cid, force=False):
for b in code[:-1]:
if b in dmap:
dmap = dmap[b]
else:
d = {}
dmap[b] = d
dmap = d
b = code[-1]
if force or ((b not in dmap) or dmap[b] == cid):
dmap[b] = cid
return
def add(unimap, enc, code):
try:
codec = self.enc2codec[enc]
c = code.decode(codec, 'strict')
if len(c) == 1:
if c not in unimap:
unimap[c] = 0
unimap[c] += 1
except KeyError:
pass
except UnicodeError:
pass
return
def pick(unimap):
chars = sorted(
unimap.items(),
key=(lambda x:(x[1],-ord(x[0]))), reverse=True)
(c,_) = chars[0]
return c
cid = int(values[0])
unimap_h = {}
unimap_v = {}
for (enc,value) in zip(encs, values):
if enc == 'CID': continue
if value == '*': continue
# hcodes, vcodes: encoded bytes for each writing mode.
hcodes = []
vcodes = []
for code in value.split(','):
vertical = code.endswith('v')
if vertical:
code = code[:-1]
try:
code = codecs.decode(code, 'hex')
except:
code = bytes([int(code, 16)])
if vertical:
vcodes.append(code)
add(unimap_v, enc, code)
else:
hcodes.append(code)
add(unimap_h, enc, code)
# add cid to each map.
(hmap, vmap) = self.get_maps(enc)
if vcodes:
assert vmap is not None
for code in vcodes:
put(vmap, code, cid, True)
for code in hcodes:
put(hmap, code, cid, True)
else:
for code in hcodes:
put(hmap, code, cid)
put(vmap, code, cid)
# Determine the "most popular" candidate.
if unimap_h:
self.cid2unichr_h[cid] = pick(unimap_h)
if unimap_v or unimap_h:
self.cid2unichr_v[cid] = pick(unimap_v or unimap_h)
return
def dump_cmap(self, fp, enc):
data = dict(
IS_VERTICAL=self.is_vertical.get(enc, False),
CODE2CID=self.code2cid.get(enc),
)
fp.write(marshal.dumps(data))
return
def dump_unicodemap(self, fp):
data = dict(
CID2UNICHR_H=self.cid2unichr_h,
CID2UNICHR_V=self.cid2unichr_v,
)
fp.write(marshal.dumps(data))
return
# convert_cmap
def convert_cmap(outdir, regname, enc2codec, paths):
converter = CMapConverter(enc2codec)
for path in paths:
print('reading: %r...' % path)
with open(path) as fp:
converter.load(fp)
files = []
for enc in converter.get_encs():
fname = '%s.marshal.gz' % enc
path = os.path.join(outdir, fname)
print('writing: %r...' % path)
with gzip.open(path, 'wb') as fp:
converter.dump_cmap(fp, enc)
files.append(path)
fname = 'to-unicode-%s.marshal.gz' % regname
path = os.path.join(outdir, fname)
print('writing: %r...' % path)
with gzip.open(path, 'wb') as fp:
converter.dump_unicodemap(fp)
files.append(path)
return files
# test
def main(argv):
args = argv[1:]
for fname in args:
with open(fname, 'rb') as fp:
cmap = FileUnicodeMap()
#cmap = FileCMap()
CMapParser(cmap, fp).run()
cmap.dump()
return
if __name__ == '__main__':
sys.exit(main(sys.argv))