Polars overkill show all rows-cols decorator

Fri, 09 Oct 2026 00:00:00 +0200

Overkill and suboptimal but good enough and makes my jupyter stuff more pleasant

class DDecorator(ContextDecorator):
    """To use:
    DD = DDecorator()
    with DD:
        your pl stuff with shown cols rows
    # or
    DD(your_df) -> display(your_df)
    """

    def __init__(self, ncols=-1, nrows=-1):
        self.ncols = ncols
        self.nrows = nrows

        def_state = pl.Config.state()
        # print(def_state)
        self.d_ncols = def_state["POLARS_FMT_MAX_COLS"]
        self.d_nrows = def_state["POLARS_FMT_MAX_ROWS"]

    def __call__(self, df, all: bool = False, ncols=None, nrows=None):
        ncols = self.ncols if ncols is None else ncols
        nrows = self.nrows if nrows is None else nrows
        if all:
            ncols = -1
            nrows = -1
        with pl.Config(set_tbl_cols=ncols, set_tbl_rows=nrows):
            display(df)

    def __enter__(self):
        pl.Config.set_tbl_rows(self.nrows)
        pl.Config.set_tbl_cols(self.ncols)
        return self

    def __exit__(self, *exc):
        # print("Finishing")
        # default vals
        pl.Config.set_tbl_rows(self.d_nrows)
        pl.Config.set_tbl_cols(self.d_ncols)
        return False


DD = DDecorator(nrows=30, ncols=30)


def describe(df):
    # with DDecorator(ncols=-1):
    with pl.Config(set_tbl_cols=-1):
        itables.show(
            df.describe(percentiles=None), fixedColumns={"start": 1}, scrollX=True
        )

Misc polars notes

Thu, 08 Oct 2026 00:00:00 +0200

Input

Read multiple CSV files with GLOBS!

Multiple - Polars user guide:

df = pl.read_csv("docs/assets/data/my_many_files_*.csv")

polars.read_csv — Polars documentation actually accepts lists as well!

Utils

Stack helper fn

Essentially transform a df with cols l_metric,t_metric,l_another,t_another into one with cols type(=l or t), metric,another.

def stack_types(
    df: pl.DataFrame,
    types: Sequence[str] = ("l", "t"),
    metrics: Sequence[str] = ("num_words", "num_chars"),
    id_cols: Sequence[str] = (),
    type_key="type",
) -> pl.DataFrame:
    """Stack prefixed metric columns into long format."""
    return pl.concat(
        [
            df.select(
                *id_cols,
                *(c(f"{t}_{m}").alias(m) for m in metrics),
                type=pl.lit(t),
            )
            for t in types
        ]
    ).rename({"type": type_key})

Bits

If then else otherwise nested

polars.when — Polars documentation

otherwise() defaults to None

The first one is picked if multiple conditions fit

# If multiple fit then the first one is picked
df.with_columns(
    pl.when(pl.col.foo > 2)
    .then(1)
    .when(pl.col.bar > 2)
    .then(4)
    .otherwise(-1)
    .alias("val")
)
df.with_columns(
...     when1 = pl.col.foo > 2,
...     then1 = 1,
...     when2 = pl.col.bar > 2,
...     then2 = 4,
...     otherwise = -1
... )

Filter where None is involved

polars.DataFrame.filter — Polars documentation TLDR: use e.g. ne_missing() instead of != : polars.Expr.eq_missing — Polars documentation

Polars exclude certain columns

https://docs.pola.rs/api/python/stable/reference/expressions/api/polars.exclude.html: df.select(pl.exclude("ba")) for all cols except ba.G

`select` can take keyword args!

Select two cols except that col2 will be called new_col2:

df.select("some_col", new_col2="col2")

(oh so many pl.with_columns(...alias('new_col2')) will be avoided by this)

Strings

Format strings

.with_columns(
    pl.format(
        "{} [{}]",
        c(LLM_MODEL),
        c(THINKING).cast(pl.String).fill_null("None"),
    ).alias(MODEL_VARIANT)
)

# when ...
pl.concat_str(
	[c(LLM_MODEL), c(THINKING)],
	separator="/",
	ignore_nulls=True,  # single None doesn't None the entire row
).alias(MODEL_VARIANT)
# ...is not enough

# ...or even this to avoid the [{}] part if thinking is null
.with_columns(
	pl.when(c(THINKING).is_null())
	.then(c(LLM_MODEL))
	.otherwise(pl.format("{} [{}]", c(LLM_MODEL), c(THINKING)))
	.alias(MODEL_VARIANT)
)

Gotchas

Count, len and sum over binary values

# this will never work:
(pl.col(effect_col(m)) < 0).count().alias("n_worsened"),
(pl.col(effect_col(m)) < 0).len().alias("n_worsened"),
# do this insead
(pl.col(effect_col(m)) < 0).sum().alias("n_worsened"),

len counts the entire thing, count() only non-null boolean values (todo?) and basically all will become n regardless of condition.

Use sum().

Polars-statistics for tests/significance/bootstrapping

GitHub - DataZooDE/polars-statistics: High-performance statistical testing and regression for Polars DataFrames, powered by Rust. · GitHub
Has VERY good documentation and jupyter notebook examples!
- e.g. polars-statistics/docs/examples/categorical-analysis.md has McNemar

Py/Polars on serhii.net