Você pode usar
DENSE_RANK
:WITH cte AS (
SELECT t.id AS trans_id,
c.id AS customer_id,
c.company,
c.first_name,
c.last_name,
t.due,
DENSE_RANK() OVER(ORDER BY c.company, c.first_name, c.last_name) rn
FROM customers AS c
JOIN transactions AS t ON t.customer_id = c.id
WHERE t.due > 0 AND t.form_type = 'invoice'
)
SELECT * FROM cte WHERE rn <= 3;
DBFiddle Demo