Here is a small context module that writes characters and spaces to a text file.
Firstly, the Lua code that goes in a separate file charsperline.lua
.
It contains a simplistic callback (or “node finalizer” in Context lingo) that
skims the hlists constituting paragraphs for characters and interword glue.
thirddata = thirddata or { }
thirddata.chars_per_line = thirddata.chars_per_line or { }
local stringformat = string.format
local tableconcat = table.concat
local utfchar = utf.char
local traverse_nodetype = node.traverse_id
local traverse_nodelist = node.traverse
local nodecodes = nodes.nodecodes
local listcodes = nodes.listcodes
local skipcodes = nodes.skipcodes
local hlist_t = nodecodes.hlist
local vlist_t = nodecodes.vlist
local glue_t = nodecodes.glue
local glyph_t = nodecodes.glyph
local line_t = listcodes.line
local userskip_t = skipcodes.userskip
local tasks = nodes.tasks
local enableaction = tasks.enableaction
local disableaction = tasks.disableaction
local linedata = { }
local resolve_ligatures
resolve_ligatures = function (lst, hd)
for n in traverse_nodetype (glyph_t, hd) do
local components = n.components
if components then
lst = resolve_ligatures (lst, components)
else
lst[#lst+1] = utfchar (n.char)
end
end
return lst
end
local collect = function (hd, groupcode)
if groupcode == "vbox" then
return hd
end
for current in traverse_nodetype (hlist_t, hd) do
if current.subtype == line_t then
local chars, has_glyphs = { }, false
for n in traverse_nodelist (current.list) do
local ntype, nsubtype = n.id, n.subtype
-- we care only for glyphs’n’glue
if ntype == glyph_t then
has_glyphs = true
if n.components then
chars = resolve_ligatures (chars, n.components)
else
chars[#chars+1] = utfchar (n.char)
end
elseif ntype == glue_t and nsubtype == userskip_t then
chars[#chars+1] = " "
end
end
if has_glyphs then
linedata[#linedata+1] = chars
end
end
end
return hd
end
thirddata.chars_per_line.collect = collect
tasks.appendaction ("finalizers", "before",
"thirddata.chars_per_line.collect")
tasks.disableaction("finalizers",
"thirddata.chars_per_line.count_words")
local write_stats = function (...) texiowrite_nl(stringformat(...)) end
local datafile = "./linedata.txt"
write_linedata = function (filename)
filename = filename or datafile
local result = { }
for i = 1, #linedata do local line = linedata[i]
result[#result+1] = stringformat ("%q,%d",
tableconcat (line),
#line)
end
io.savedata (filename, result, "\n")
end
local active --- callback state
commands.start_chars_per_line = function ()
if not active then
enableaction("finalizers",
"thirddata.chars_per_line.count_words")
active = true
end
end
commands.stop_chars_per_line = function ()
if active then
disableaction("finalizers",
"thirddata.chars_per_line.count_words")
active = false
end
end
commands.write_linedata = write_linedata
The user interface is defined in module t-charsperline.mkvi
.
Besides the usual \start...
/ \stop...
environment it also
sets up a call to write_linedata()
at the end of the TeX run.
\startmodule [charsperline]
\unprotect
\ctxloadluafile{charsperline}
\def\startdumplines{\ctxcommand{start_chars_per_line ()}}
\def\stopdumplines{\endgraf\ctxcommand{stop_chars_per_line ()}}
\prependtoks \charsperline_dump \to \everystoptext
\def\charsperline_dump{\ctxcommand{write_linedata ()}}
\protect
\stopmodule \endinput
Now you can use the macros \startdumplines
/ \stopdumplines
in regular documents by loading the module:
\usemodule[charsperline]
\setuplayout[width=5cm]
\starttext
\startdumplines
\input knuth
\stopdumplines
\stoptext
The output is written to a file linedata.txt
in the current directory.
The schema for a line is CSV’ish: "<line content>",<character count>
:
"Thus, I came to the conclu-",27
"sion that the designer of a",27
"new system must not only",24
"be the implementer and first",28
"large scale user; the designer",31
"should also write the first user",32
"manual.",7
"The separation of any of these",30
"four components would have",26
Link to gist.
Btw. whether or not you get ASCII output depends on what glyphs a
document contains; let me know if you desire a strict solution that
strips all code points not in the ASCII range.