2

I have a pipeline, where I start from a .csv file (full MWE as Python script below):

Item #/offset,Description/name,Random info,Default value
0,Sum Difficult Copper   ,Recover; sculpture,0
1,Joy Committee Tick     ,Reference; ball average can loop,0

... then I convert this to a Markdown "grid" table using Python pandas, choose to "format" some columns with backticks, so I get monospaced font there - and insert this in a Markdown document:

---
title: "My table description document"
author: John Doe

---

Here is the table, discussed in this document:

+-----------------+--------------------------+----------------------------------+-----------------+
| Item #/offset   | Description/name         | Random info                      |   Default value |
+=================+==========================+==================================+=================+
| `0`             | "`Sum Difficult Copper`" | Recover; sculpture               |               0 |
+-----------------+--------------------------+----------------------------------+-----------------+
| `1`             | "`Joy Committee Tick`"   | Reference; ball average can loop |               0 |
+-----------------+--------------------------+----------------------------------+-----------------+

Notice the table formatting!

... and finally, use pandoc to convert this Markdown to pdf via xelatex - which produces this output:

pdf_output

For one, this table content can absolutely fit in one line per row; and I cannot tell why "Sum Difficult Copper" got broken/word-wrapped? In addition, the numbers wrapped with backticks stick to the "top" of the cell and are left-aligned, normal numbers stick to "bottom" and are right-aligned?

There's definitely influence from verbatim package as evidenced by the red line-wrapping character - but then, "Reference; ball average can loop" is also line-broken, and it is normal text?

So my questions are:

  • Is there some easy solution on a Latex level (e.g. including some package), which will "prefer" fitting cell content so a row fits on a single line - regardless if I use backticks for monospaced font inside the table or not?
    • (side question, I know this is not the right forum for it: is there a way to control the cell alignment left/right, or even top/bottom, maybe already in Python where .to_markdown is called?)
  • Is there some easy solution on a Latex level (e.g. including some package), so I get table cell borders ("Outer Border and All Inner Lines", as it is called in LibreOffice calc)?

Here is the Python script, table_to_md.py:

#!/usr/bin/env python3

import os
import pandas as pd
import subprocess

PANDOC = "pandoc"
TABLE_FNAME = "table.csv"
TABLE_DATA="""\
Item #/offset,Description/name,Random info,Default value
0,Sum Difficult Copper   ,Recover; sculpture,0
1,Joy Committee Tick     ,Reference; ball average can loop,0
"""
YAML_FNAME = "pandoc_style.yaml"
YAML_DATA = r"""
pdf-engine: xelatex
filters:
- pandoc-crossref
- citeproc

metadata:
  link-citations: true # works fine here!

listings: true

variables:
  geometry: margin=2cm
  classoption: table
  documentclass: extarticle
  numbersections: true
  papersize: a4
  fontsize: 12pt
  unicode-math: bold-style=ISO
  listings: true
  header-includes: |
    \rowcolors{2}{gray!10}{gray!25}
    \usepackage{fontspec}
    \setmainfont[Ligatures=TeX]{CMU Serif}
    \unimathsetup{bold-style=ISO}
    \lstset{% for listings
      basicstyle=\ttfamily,
      breaklines=true,
      postbreak=\mbox{\textcolor{red}{$\hookrightarrow$}\space},
    }
"""


CONTENT_MD_FNAME="my_table_description.md"
CONTENT_TEX_FNAME="my_table_description.tex"
CONTENT_PDF_FNAME="my_table_description.pdf"
CONTENT_MD_DATA="""\
---
title: "My table description document"
author: John Doe

---

Here is the table, discussed in this document:

{}

Notice the table formatting!
"""


with open(TABLE_FNAME, 'w', encoding='utf-8') as f:
  f.write(TABLE_DATA)
with open(YAML_FNAME, 'w', encoding='utf-8') as f:
  f.write(YAML_DATA)

df_table = pd.read_csv(TABLE_FNAME)

# make these columns "monospace"/"code" with backticks
df_table["Item #/offset"] = df_table["Item #/offset"].apply(lambda x: "`{}`".format(x))
df_table["Description/name"] = df_table["Description/name"].apply(lambda x: '"`{}`"'.format(x.strip()))

table_md = df_table.to_markdown(tablefmt="grid", index=False)
content_md = CONTENT_MD_DATA.format(table_md)
with open(CONTENT_MD_FNAME, 'w', encoding='utf-8') as f:
  f.write(content_md)

#print(content_md)
print("Saved {}".format(CONTENT_MD_FNAME))

#pandoc -s MANUAL.txt -o example4.tex
DEFAULTS_ARG = "--defaults={}".format(YAML_FNAME)
PANDOC_CMD1 = [PANDOC, DEFAULTS_ARG, "--toc", "-s", CONTENT_MD_FNAME, "-o", CONTENT_TEX_FNAME]
print("Running: {}".format(" ".join(PANDOC_CMD1)))
result = subprocess.run(PANDOC_CMD1, stdout=subprocess.PIPE, stderr=subprocess.PIPE, universal_newlines=True, cwd=os.getcwd())
print(result.stderr)
if (result.returncode != 0):
  print("  Got err '{}' ...".format(result.stderr))

PANDOC_CMD2 = [PANDOC, DEFAULTS_ARG, "--toc", "--verbose", CONTENT_MD_FNAME, "-o", CONTENT_PDF_FNAME]
print("Running: {}".format(" ".join(PANDOC_CMD2)))
result = subprocess.run(PANDOC_CMD2, stdout=subprocess.PIPE, stderr=subprocess.PIPE, universal_newlines=True, cwd=os.getcwd(), encoding='UTF-8')
prn_nextline = False
for line in result.stderr.splitlines():
  if (("written" in line) or ("makePDF" in line) or (prn_nextline)):
    print(line)
    if (prn_nextline):
      prn_nextline = False
    if (("temp dir:" in line) or ("Command line:" in line)):
      prn_nextline = True
if (result.returncode != 0):
  print("  Got err '{}' ...".format(result.stderr))

When I run it, I get this output:

$ python3 table_to_md.py
Saved my_table_description.md
Running: pandoc --defaults=pandoc_style.yaml --toc -s my_table_description.md -o my_table_description.tex

Running: pandoc --defaults=pandoc_style.yaml --toc --verbose my_table_description.md -o my_table_description.pdf
[makePDF] temp dir:
C:/msys64/tmp/tex2pdf.-32afd74f419dcabc
[makePDF] Command line:
xelatex "-halt-on-error" "-interaction" "nonstopmode" "-output-directory" "D:/msys64/tmp/tex2pdf.-32afd74f419dcabc" "C:/msys64/tmp/tex2pdf.-32afd74f419dcabc/input.tex"
[makePDF] Environment:
[makePDF] Source:
[makePDF] Run #1
Output written on C:/msys64/tmp/tex2pdf.-32afd74f419dcabc/input.pdf (1 page).
Transcript written on C:/msys64/tmp/tex2pdf.-32afd74f419dcabc/input.log.
[makePDF] Run #2
Output written on C:/msys64/tmp/tex2pdf.-32afd74f419dcabc/input.pdf (1 page).
Transcript written on C:/msys64/tmp/tex2pdf.-32afd74f419dcabc/input.log.
[makePDF] Run #3
Output written on C:/msys64/tmp/tex2pdf.-32afd74f419dcabc/input.pdf (1 page).
Transcript written on C:/msys64/tmp/tex2pdf.-32afd74f419dcabc/input.log.

... and for completeness, here is the generated my_table_description.tex:

% Options for packages loaded elsewhere
\PassOptionsToPackage{unicode}{hyperref}
\PassOptionsToPackage{hyphens}{url}
%
\documentclass[
  12pt,
  a4paper,
  table]{extarticle}
\usepackage{amsmath,amssymb}
\usepackage{lmodern}
\usepackage{iftex}
\ifPDFTeX
  \usepackage[T1]{fontenc}
  \usepackage[utf8]{inputenc}
  \usepackage{textcomp} % provide euro and other symbols
\else % if luatex or xetex
  \usepackage{unicode-math}
  \defaultfontfeatures{Scale=MatchLowercase}
  \defaultfontfeatures[\rmfamily]{Ligatures=TeX,Scale=1}
\fi
% Use upquote if available, for straight quotes in verbatim environments
\IfFileExists{upquote.sty}{\usepackage{upquote}}{}
\IfFileExists{microtype.sty}{% use microtype if available
  \usepackage[]{microtype}
  \UseMicrotypeSet[protrusion]{basicmath} % disable protrusion for tt fonts
}{}
\makeatletter
\@ifundefined{KOMAClassName}{% if non-KOMA class
  \IfFileExists{parskip.sty}{%
    \usepackage{parskip}
  }{% else
    \setlength{\parindent}{0pt}
    \setlength{\parskip}{6pt plus 2pt minus 1pt}}
}{% if KOMA class
  \KOMAoptions{parskip=half}}
\makeatother
\usepackage{xcolor}
\IfFileExists{xurl.sty}{\usepackage{xurl}}{} % add URL line breaks if available
\IfFileExists{bookmark.sty}{\usepackage{bookmark}}{\usepackage{hyperref}}
\hypersetup{
  pdftitle={My table description document},
  pdfauthor={John Doe},
  hidelinks,
  pdfcreator={LaTeX via pandoc}}
\urlstyle{same} % disable monospaced font for URLs
\usepackage[margin=2cm]{geometry}
\usepackage{listings}
\newcommand{\passthrough}[1]{#1}
\lstset{defaultdialect=[5.3]Lua}
\lstset{defaultdialect=[x86masm]Assembler}
\usepackage{longtable,booktabs,array}
\usepackage{calc} % for calculating minipage widths
% Correct order of tables after \paragraph or \subparagraph
\usepackage{etoolbox}
\makeatletter
\patchcmd\longtable{\par}{\if@noskipsec\mbox{}\fi\par}{}{}
\makeatother
% Allow footnotes in longtable head/foot
\IfFileExists{footnotehyper.sty}{\usepackage{footnotehyper}}{\usepackage{footnote}}
\makesavenoteenv{longtable}
\setlength{\emergencystretch}{3em} % prevent overfull lines
\providecommand{\tightlist}{%
  \setlength{\itemsep}{0pt}\setlength{\parskip}{0pt}}
\setcounter{secnumdepth}{5}
\rowcolors{2}{gray!10}{gray!25}
\usepackage{fontspec}
\setmainfont[Ligatures=TeX]{CMU Serif}
\unimathsetup{bold-style=ISO}
\lstset{% for listings
  basicstyle=\ttfamily,
  breaklines=true,
  postbreak=\mbox{\textcolor{red}{$\hookrightarrow$}\space},
}
\ifLuaTeX
  \usepackage{selnolig}  % disable illegal ligatures
\fi

\title{My table description document}
\author{John Doe}
\date{}

\begin{document}
\maketitle

{
\setcounter{tocdepth}{3}
\tableofcontents
}
Here is the table, discussed in this document:

\begin{longtable}[]{@{}
  >{\raggedright\arraybackslash}p{(\columnwidth - 6\tabcolsep) * \real{0.18}}
  >{\raggedright\arraybackslash}p{(\columnwidth - 6\tabcolsep) * \real{0.28}}
  >{\raggedright\arraybackslash}p{(\columnwidth - 6\tabcolsep) * \real{0.36}}
  >{\raggedright\arraybackslash}p{(\columnwidth - 6\tabcolsep) * \real{0.18}}@{}}
\toprule
Item \#/offset & Description/name & Random info & Default value \\
\midrule
\endhead
\passthrough{\lstinline!0!} &
``\passthrough{\lstinline!Sum Difficult Copper!}'' & Recover; sculpture
& \begin{minipage}[t]{\linewidth}\raggedright
\begin{lstlisting}
          0
\end{lstlisting}
\end{minipage} \\
\passthrough{\lstinline!1!} &
``\passthrough{\lstinline!Joy Committee Tick!}'' & Reference; ball
average can loop & \begin{minipage}[t]{\linewidth}\raggedright
\begin{lstlisting}
          0
\end{lstlisting}
\end{minipage} \\
\bottomrule
\end{longtable}

Notice the table formatting!

\end{document}
3
  • 2
    For reasons why you shouldn't aspire to spreadsheet-like tables, read the booktabs manual. If you insist on the pursuit of perversion, see makecell. The horizontal alignment is just the t to minipage aligning with the baseline, I think. I don't think backticks are relevant. \raggedright often does strange things with breaks. You can just omit it for columns which fit. In other cases, consider ragged2e. You are loading bookmark too early. It should be last except for things which explicitly need to come after. If that's the script's work, so much the worse for the script.
    – cfr
    Commented Oct 8, 2023 at 3:14
  • 1
    Note that if you really want the table to use spreadsheet-style, you'll need to take the booktabs stuff out as that package's rules will not cooperate with vertical rules. Calc-style tables are not good typography!
    – cfr
    Commented Oct 8, 2023 at 3:17
  • Many thanks @cfr - I can live without the cell borders, however the control of alignment and/or content width is the key for me - and I think your suggestions on bookmark, \raggedright and \ragged2e will be very helpful there, I'll take a closer look - many thanks!
    – sdbbs
    Commented Oct 8, 2023 at 3:18

2 Answers 2

1

I think that is not the easier way to display a CSV as a table in a markdown document. If you are using Quarto, you can do make it directly in the main document with Python (I don't know if my example below is the best method, I have a minimal experience with this language, sorry) or with R (I only show the kable method, but you can use also xtable and other approaches), or if you what export it only to LateX, also can use a LaTeX methods to import the CSV to a LaTeX table, without a previous export to markdown (not showed here, but there are several answers at this respect in this site). However, you can also simply make also a manual fix of what you have obtained to maintain the document as simple as possible. In any case, the wrapping is correct avoiding the verbatim text.

Note that color background is for some reason wider that booktabs rules this way (as you show in your own screenshot). I have no time to see the reason in the LateX output, but IMHO is better just avoid colors with booktabs rules, that also add vertical spaces. This also make awful breaks in vertical rules, that anyway you should avoid always, as commented by @cfr.

MWE compiled with Quarto:

---
format: pdf
---

### With Python


```{python}
#| echo: false

import pandas as pd
df = pd.read_csv("foo.csv")

from IPython.display import display, Markdown

Markdown(df.to_markdown())

```

(Sorry, no idea of how remove row names in python) 


### With R 

```{r}
#| echo: false
#| output: asis

df  <- read.csv(file = "foo.csv",header=FALSE)
names(df) <- unname(unlist(df[1,]))
df <- df[-1,]

knitr::kable(df, row.names=FALSE,align="cllc")



```



### With Markdown (edited manually)

```{=tex}
\tabcolsep0pt
```

+---------------+--------------------------+----------------------------------+---------------+
| Item #/offset | Description/name         | Random info                      | Default value |
+:=============:+==========================+==================================+:=============:+
| 0             | Sum Difficult Copper     | Recover; sculpture               | 0             |
+---------------+--------------------------+----------------------------------+---------------+
| 1             | Joy Committee Tick       | Reference; ball average can loop | 0             |
+---------------+--------------------------+----------------------------------+---------------+

MWE

0

All right - I got to a sort of a solution based on two changes, none of which involves Latex directly:

  1. Use linebreaks in the .csv, to force column names to break

Looking at OP, we can see that the column labels "Item #/offset" and "Default value" are way longer than their typical contents -> and this ends up leading the grid table Markdown formatting -> which ultimately determines relative width of Latex/PDF columns.

So, inserting linebreaks like this in the .csv:

"Item #
/offset",Description/name,Random info,"Default
value"
0,Sum Difficult Copper   ,Recover; sculpture,0
1,Joy Committee Tick     ,Reference; ball average can loop,0

... which results with the following Markdown table:

+-----------+--------------------------+----------------------------------+-----------+
| Item #    | Description/name         | Random info                      | Default   |
| /offset   |                          |                                  | value     |
+===========+==========================+==================================+===========+
| `0`       | "`Sum Difficult Copper`" | Recover; sculpture               | 0         |
+-----------+--------------------------+----------------------------------+-----------+
| `1`       | "`Joy Committee Tick`"   | Reference; ball average can loop | 0         |
+-----------+--------------------------+----------------------------------+-----------+
  1. Use colalign and numalign in the .to_markdown call in Python

I've noticed in pandas.DataFrame.to_markdown — pandas 2.1.1 documentation:

**kwargs: These parameters will be passed to tabulate.

... and it is tabulate that has the colalign and numalign parameters. It does an automatic detection of numbers, and the default style for them is "right" - but also, the numbers in that case tend to "stick" to the bottom.

So, the solution for me was to use numalign="left", colalign=("left",) in the .to_markdown call; this makes the numbers in last column align left (above table also shows that effect) - and finally, am getting a tighter automatic table layout:

output_pdf

Here is the modified table_to_md.py (if on Windows, make sure you save it with Unix line endings \n e.g. in Notepad++, else some of the assumption the script makes will break):

#!/usr/bin/env python3

import os
import pandas as pd
import subprocess

PANDOC = "pandoc"
TABLE_FNAME = "table.csv"
TABLE_DATA="""\
"Item #
/offset",Description/name,Random info,"Default
value"
0,Sum Difficult Copper   ,Recover; sculpture,0
1,Joy Committee Tick     ,Reference; ball average can loop,0
"""
YAML_FNAME = "pandoc_style.yaml"
YAML_DATA = r"""
pdf-engine: xelatex
filters:
- pandoc-crossref
- citeproc

metadata:
  link-citations: true # works fine here!

listings: true

variables:
  geometry: margin=2cm
  classoption: table
  documentclass: extarticle
  numbersections: true
  papersize: a4
  fontsize: 12pt
  unicode-math: bold-style=ISO
  listings: true
  header-includes: |
    \rowcolors{2}{gray!10}{gray!25}
    \usepackage{fontspec}
    \setmainfont[Ligatures=TeX]{CMU Serif}
    \unimathsetup{bold-style=ISO}
    \lstset{% for listings
      basicstyle=\ttfamily,
      breaklines=true,
      postbreak=\mbox{\textcolor{red}{$\hookrightarrow$}\space},
    }
"""


CONTENT_MD_FNAME="my_table_description.md"
CONTENT_TEX_FNAME="my_table_description.tex"
CONTENT_PDF_FNAME="my_table_description.pdf"
CONTENT_MD_DATA="""\
---
title: "My table description document"
author: John Doe

---

Here is the table, discussed in this document:

{}

Notice the table formatting!
"""


with open(TABLE_FNAME, 'wb') as f:
  f.write(bytes(TABLE_DATA, encoding='utf-8'))
with open(YAML_FNAME, 'w', encoding='utf-8') as f:
  f.write(YAML_DATA)

df_table = pd.read_csv(TABLE_FNAME)

print(df_table.columns)

# make these columns "monospace"/"code" with backticks
df_table["Item #\n/offset"] = df_table["Item #\n/offset"].apply(lambda x: "`{}`".format(x))
df_table["Description/name"] = df_table["Description/name"].apply(lambda x: '"`{}`"'.format(x.strip()))

table_md = df_table.to_markdown(tablefmt="grid", index=False, numalign="left", colalign=("left",))
content_md = CONTENT_MD_DATA.format(table_md)
with open(CONTENT_MD_FNAME, 'w', encoding='utf-8') as f:
  f.write(content_md)

#print(content_md)
print("Saved {}".format(CONTENT_MD_FNAME))

#pandoc -s MANUAL.txt -o example4.tex
DEFAULTS_ARG = "--defaults={}".format(YAML_FNAME)
PANDOC_CMD1 = [PANDOC, DEFAULTS_ARG, "--toc", "-s", CONTENT_MD_FNAME, "-o", CONTENT_TEX_FNAME]
print("Running: {}".format(" ".join(PANDOC_CMD1)))
result = subprocess.run(PANDOC_CMD1, stdout=subprocess.PIPE, stderr=subprocess.PIPE, universal_newlines=True, cwd=os.getcwd())
print(result.stderr)
if (result.returncode != 0):
  print("  Got err '{}' ...".format(result.stderr))

PANDOC_CMD2 = [PANDOC, DEFAULTS_ARG, "--toc", "--verbose", CONTENT_MD_FNAME, "-o", CONTENT_PDF_FNAME]
print("Running: {}".format(" ".join(PANDOC_CMD2)))
result = subprocess.run(PANDOC_CMD2, stdout=subprocess.PIPE, stderr=subprocess.PIPE, universal_newlines=True, cwd=os.getcwd(), encoding='UTF-8')
prn_nextline = False
for line in result.stderr.splitlines():
  if (("written" in line) or ("makePDF" in line) or (prn_nextline)):
    print(line)
    if (prn_nextline):
      prn_nextline = False
    if (("temp dir:" in line) or ("Command line:" in line)):
      prn_nextline = True
if (result.returncode != 0):
  print("  Got err '{}' ...".format(result.stderr))

You must log in to answer this question.

Not the answer you're looking for? Browse other questions tagged .