if not modules then modules = { } end modules ['font-txt'] = {
version = 1.001,
comment = "companion to font-ini.mkiv",
original = "derived from a prototype by Kai Eigner",
author = "Hans Hagen", -- so don't blame KE
copyright = "TAT Zetwerk / PRAGMA ADE / ConTeXt Development Team",
license = "see context related readme files"
}
-- The next code is derived from a snippet handler prototype by Kai Eigner and
-- resembles the main loop of the Lua font handler but I decided use a more generic
-- (and pluggable) approach and not hook it into the already present opentype
-- handler. This is cleaner as it cannot interfere with the Lua font processor
-- (which does some more things) and is also better performance wise. It also makes
-- it possible to support other handlers as history has proven that there are no
-- universal solution in computer land. Most of the disc logic is kept but done
-- slightly different.
--
-- The code is heavily optimized and generalized so there can be errors. As
-- mentioned, the plug mode can be used for alternative font handlers. A font is
-- still loaded but the node and base mode handlers are ignored. Plugins are
-- unlikely to work well in context as they can mess up attribute driven subsystem,
-- so they are not officially supported. The language and script options are
-- available in the usual way.
--
-- The code collects snippets, either or not with spacing around them and partially
-- running over disc nodes. The r2l "don't assume disc and collect larger chunks" is
-- not robust so I got rid of that branch. This is somewhat similar to the Lua font
-- handler.
--
-- An alternative is to run over longer strings with dummy chars (unicode objects) as
-- markers for whatever is in the list but that becomes tricky with mixed fonts and
-- reconstruction becomes a bit of a mess then, especially because disc nodes force
-- us to backtrack and look at several solutions. It also has a larger memory
-- footprint. Some tests demonstrated that it has no gain and only adds complexity.
--
-- This (derived) variant is better suited for context and seems to work ok in the
-- generic variant. I also added some context specific tracing to the code. This
-- variant uses the plug model provided in the context font system. So, in context,
-- using the built in Lua handler is the better alternative, also because it has
-- extensive tracing features. Context users would loose additional functionality
-- that has been provided for a decade and therefore plugins are not officially
-- supported (at least not by me, unless I use them myself).
--
-- There is no checking here for already processed characters so best not mix this
-- variant with code that does similar things. If this code evolves depends on the
-- useability. Kai's code can now be found on github where it is used with a harfbuzz
-- library. We add this kind of stuff because occasionally we compare engines and
-- Kai sends me examples and I then need to check context.
--
-- One important difference between Kai's approach and the one used in ConTeXt is
-- that we use utf-32 instead of utf-8. Once I figured out that clusters were just
-- indices into the original text that made more sense. The first implementation
-- used the command line tool (binary), then I went for ffi (library).
--
-- Beware: this file only implements the framework for plugins. Plugins themselves
-- are in other files (e.g. font-phb*.lua). On the todo list is a uniscribe plugin
-- because that is after all the reference for opentype support, but that interface
-- needs a bit more work (so it might never happen).
--
-- Usage: see m-fonts-plugins.mkiv. As it's a nice test for ffi support that file
-- might be added to the distribution somewhere in the middle of 2017 when the ffi
-- interface has been tested a bit more. Okay, it's 2012 now and we're way past that
-- date but we never had a reason for adding it to the ConTeXt distribution. It
-- should still work okay because I occasionally checked it against progress made in
-- the engines and used newer helpers.
--
-- Here is an example of usage:
--
-- \starttext
-- \definefontfeature[test][mode=plug,features=text]
-- \start
-- \showfontkerns
-- \definedfont[Serif*test]
-- \input tufte \par
-- \stop
-- \stoptext
local fonts = fonts
local otf = fonts.handlers.otf
local nodes = nodes
local utfchar = utf.char
local nuts = nodes.nuts
local getnext = nuts.getnext
local setnext = nuts.setnext
local getprev = nuts.getprev
local setprev = nuts.setprev
local getid = nuts.getid
local getfont = nuts.getfont
local getchar = nuts.getchar
local getdisc = nuts.getdisc
local setdisc = nuts.setdisc
local getboth = nuts.getboth
local getscales = nuts.getscales
local setlink = nuts.setlink
local getkern = nuts.getkern
local getwidth = nuts.getwidth
local ischar = nuts.ischar
local isglyph = nuts.isglyph
local usesfont = nuts.usesfont
local copy_node_list = nuts.copylist
local find_node_tail = nuts.tail
local flushlist = nuts.flushlist
local freenode = nuts.free
local endofmath = nuts.endofmath
local startofpar = nuts.startofpar
local nodecodes = nodes.nodecodes
local glyph_code <const> = nodecodes.glyph
local glue_code <const> = nodecodes.glue
local disc_code <const> = nodecodes.disc
local kern_code <const> = nodecodes.kern
local math_code <const> = nodecodes.math
local dir_code <const> = nodecodes.dir
local par_code <const> = nodecodes.par
local righttoleft_code <const> = tex.directioncodes.righttoleft
local txtdirstate = otf.helpers.txtdirstate
local pardirstate = otf.helpers.pardirstate
local fonthashes = fonts.hashes
local fontdata = fonthashes.identifiers
local function deletedisc(head)
local current = head
local next = nil
while current do
next = getnext(current)
if getid(current) == disc_code then
local pre, post, replace, pre_tail, post_tail, replace_tail = getdisc(current,true)
setdisc(current)
if pre then
flushlist(pre)
end
if post then
flushlist(post)
end
local p, n = getboth(current)
if replace then
if current == head then
head = replace
setprev(replace) -- already nil
else
setlink(p,replace)
end
setlink(replace_tail,n) -- was: setlink(n,replace_tail)
elseif current == head then
head = n
setprev(n)
else
setlink(p,n)
end
freenode(current)
end
current = next
end
return head
end
-- As we know that we have the same font we can probably optimize this a bit more.
-- Although we can have more in disc nodes than characters and kerns we only support
-- those two types.
local function eqnode(n,m) -- no real improvement in speed
local n_char = isglyph(n)
if n_char then
return n_char == ischar(m,getfont(n))
elseif n_id == kern_code then
return getkern(n) == getkern(m)
end
end
local function equalnode(n,m)
if not n then
return not m
elseif not m then
return false
end
local n_char, n_id = isglyph(n)
if n_char then
return n_char == ischar(m,n_id) -- n_id == n_font
elseif n_id == whatsit_code then
return false
elseif n_id == glue_code then
return true
elseif n_id == kern_code then
return getkern(n) == getkern(m)
elseif n_id == disc_code then
local n_pre, n_post, n_replace = getdisc(n)
local m_pre, m_post, m_replace = getdisc(m)
while n_pre and m_pre do
if not eqnode(n_pre,m_pre) then
return false
end
n_pre = getnext(n_pre)
m_pre = getnext(m_pre)
end
if n_pre or m_pre then
return false
end
while n_post and m_post do
if not eqnode(n_post,m_post) then
return false
end
n_post = getnext(n_post)
m_post = getnext(m_post)
end
if n_post or m_post then
return false
end
while n_replace and m_replace do
if not eqnode(n_replace,m_replace) then
return false
end
n_replace = getnext(n_replace)
m_replace = getnext(m_replace)
end
if n_replace or m_replace then
return false
end
return true
end
return false
end
-- The spacing hackery is not nice. The text can get leading and trailing spaces
-- and even mid spaces while the start and stop nodes not always are glues then
-- so the plugin really needs to do some testing there. We could pass more context
-- but it doesn't become much better.
--
-- The attribute gets passed for tracing purposes. We could support it (not that
-- hard to do) but as we don't test strickly for fonts (in disc nodes) we are not
-- compatible anyway. It would also mean more testing. So, don't use this mixed
-- with node and base mode in context.
--
-- We don't distinguish between modes in treatment (so no r2l assumptions) and
-- no cheats for scripts that might not use discretionaries. Such hacks can work
-- in predictable cases but in context one can use a mix all kind of things and
-- users do that. On the other hand, we do support longer glyph runs in both modes
-- so there we gain a bit.
do
local currentscale, currentxscale, currentyscale
local function texthandler(head,font,dynamic,rlmode,handler,startspacing,stopspacing,nesting)
if not head then
return
end
if startspacing == nil then
startspacing = false
end
if stopspacing == nil then
stopspacing = false
end
if getid(head) == par_code and startofpar(head) then
rlmode = pardirstate(head)
elseif rlmode == righttoleft_code then
rlmode = -1
else
rlmode = 0
end
local dirstack = { }
local rlparmode = 0
local topstack = 0
local text = { }
local size = 0
local current = head
local start = nil
local stop = nil
local startrlmode = rlmode
local function handle(leading,trailing) -- what gets passed can become configureable: e.g. utf 8
local stop = current or start -- hm, what with outer stop
if getid(stop) ~= glyph_code then
stop = getprev(stop)
end
head = handler(head,font,dynamic,rlmode,start,stop,text,leading,trailing) -- handler can adapt text
size = 0
text = { }
start = nil
end
while current do
local char, id = ischar(current,font,dynamic,currentscale,currentxscale,currentyscale)
if char then
if not start then
start = current
startrlmode = rlmode
end
local char = getchar(current)
size = size + 1
text[size] = char
current = getnext(current)
elseif char == false then
-- so a mixed font
if start and size > 0 then
handle(startspacing,false)
end
startspacing = false
local s, sx, sy = getscales(current)
if s ~= currentscale or sx ~= currentxscale or sy ~= currentyscale then
if start and size > 0 then
handle(startspacing,false)
end
startspacing = false
currentscale, currentxscale, currentyscale = s, sx, sy
-- todo: safeguard against a loop
else
current = getnext(current)
currentscale, currentxscale, currentyscale = false, false, false
end
elseif id == glue_code then
-- making this branch optional i.e. always use the else doesn't really
-- make a difference in performance (in hb) .. tricky anyway as we can
local width = getwidth(current)
if width > 0 then
if start and size > 0 then
handle(startspacing,true)
end
startspacing = true
stopspacing = false
else
if start and size > 0 then
head = handle(startspacing)
end
startspacing = false
stopspacing = false
end
current = getnext(current)
elseif id == disc_code and usesfont(current,font) then -- foo|-|bar : has hbox
-- This looks much like the original code but I don't see a need to optimize
-- for e.g. deva or r2l fonts. If there are no disc nodes then we won't see
-- this branch anyway and if there are, we should just deal with them.
--
-- There is still some weird code here ... start/stop and such. When I'm in
-- the mood (or see a need) I'll rewrite this bit.
-- bug: disc in last word moves to end (in practice not an issue as one
-- doesn't want a break there)
local pre = nil
local post = nil
local currentnext = getnext(current)
local current_pre, current_post, current_replace = getdisc(current)
setdisc(current) -- why, we set it later
if start then
pre = copy_node_list(start,current)
stop = getprev(current)
-- why also current and not:
-- pre = copy_node_list(start,stop)
if start == head then
head = current
end
setlink(getprev(start),current)
setlink(stop,current_pre)
current_pre = start
setprev(current_pre)
start = nil
stop = nil
startrlmode = rlmode
end
while currentnext do
local char, id = ischar(currentnext,font)
if char or id == disc_code then
stop = currentnext
currentnext = getnext(currentnext)
elseif id == glue_code then
local width = getwidth(currentnext)
if width and width > 0 then
stopspacing = true
else
stopspacing = false
end
break
else
break
end
end
if stop then
local currentnext = getnext(current)
local stopnext = getnext(stop)
post = copy_node_list(currentnext,stopnext)
if current_post then
setlink(find_node_tail(current_post),currentnext)
else
setprev(currentnext)
current_post = currentnext
end
setlink(current,stopnext)
setnext(stop)
stop = nil
end
if pre then
setlink(find_node_tail(pre),current_replace)
current_replace = pre
pre = nil
end
if post then
if current_replace then
setlink(find_node_tail(current_replace),post)
else
current_replace = post
end
post = nil
end
size = 0 -- hm, ok, start is also nil now
text = { }
if current_pre then
current_pre = texthandler(current_pre,font,dynamic,rlmode,handler,startspacing,false,"pre")
end
if current_post then
current_post = texthandler(current_post,font,dynamic,rlmode,handler,false,stopspacing,"post")
end
if current_replace then
current_replace = texthandler(current_replace,font,dynamic,rlmode,handler,startspacing,stopspacing,"replace")
end
startspacing = false
stopspacing = false
local cpost = current_post and find_node_tail(current_post)
local creplace = current_replace and find_node_tail(current_replace)
local cpostnew = nil
local creplacenew = nil
local newcurrent = nil
while cpost and equalnode(cpost,creplace) do
cpostnew = cpost
creplacenew = creplace
if creplace then
creplace = getprev(creplace)
end
cpost = getprev(cpost)
end
if cpostnew then
if cpostnew == current_post then
current_post = nil
else
setnext(getprev(cpostnew))
end
flushlist(cpostnew)
if creplacenew == current_replace then
current_replace = nil
else
setnext(getprev(creplacenew))
end
local c = getnext(current)
setlink(current,creplacenew)
local creplacenewtail = find_node_tail(creplacenew)
setlink(creplacenewtail,c)
newcurrent = creplacenewtail
end
current_post = current_post and deletedisc(current_post)
current_replace = current_replace and deletedisc(current_replace)
local cpre = current_pre
local creplace = current_replace
local cprenew = nil
local creplacenew = nil
while cpre and equalnode(cpre, creplace) do
cprenew = cpre
creplacenew = creplace
if creplace then
creplace = getnext(creplace)
end
cpre = getnext(cpre)
end
if cprenew then
cpre = current_pre
current_pre = getnext(cprenew)
if current_pre then
setprev(current_pre)
end
setnext(cprenew)
flushlist(cpre)
creplace = current_replace
current_replace = getnext(creplacenew)
if current_replace then
setprev(current_replace)
end
setlink(getprev(current),creplace)
if current == head then
head = creplace
end
setlink(creplacenew,current)
end
setdisc(current,current_pre,current_post,current_replace)
current = currentnext
else
if start and size > 0 then
handle(startspacing,stopspacing)
end
startspacing = false
stopspacing = false
if id == math_code then
current = getnext(endofmath(current))
elseif id == dir_code then
startspacing = false
topstack, rlmode = txtdirstate(current,dirstack,topstack,rlparmode)
current = getnext(current)
-- elseif id == par_code and startofpar(current) then
-- startspacing = false
-- rlparmode, rlmode = pardirstate(current)
-- current = getnext(current)
else
current = getnext(current)
end
end
end
if start and size > 0 then
handle(startspacing,stopspacing)
end
return head, true
end
function fonts.handlers.otf.texthandler(head,font,dynamic,direction,action)
currentscale = false
currentxscale = false
currentyscale = false
if action then
return texthandler(head,font,dynamic,direction == righttoleft_code and -1 or 0,action)
else
return head, false
end
end
-- Next comes a tracer plug into context.
----- texthandler = fonts.handlers.otf.texthandler
local report_text = logs.reporter("otf plugin","text")
local nofruns = 0
local nofsnippets = 0
local f_unicode = string.formatters["%U"]
local function showtext(head,font,dynamic,rlmode,start,stop,list,before,after)
if list then
nofsnippets = nofsnippets + 1
local plus = { }
for i=1,#list do
local u = list[i]
list[i] = utfchar(u)
plus[i] = f_unicode(u)
end
report_text("%03i : [%s] %t [%s]-> % t", nofsnippets, before and "+" or "-", list, after and "+" or "-", plus)
else
report_text()
report_text("invalid list")
report_text()
end
return head, false
end
fonts.handlers.otf.registerplugin("text",function(head,font,dynamic,direction)
nofruns = nofruns + 1
nofsnippets = 0
report_text("start run %i",nofruns)
local h, d = texthandler(head,font,dynamic,direction,showtext)
report_text("stop run %i",nofruns)
return h, d
end)
end