Tente isto:
knex.schema.raw(
`CREATE UNIQUE INDEX "unique_partial_newTable_col1_col2_col3" ON "newTable" ("col1", "col2", "col3") WHERE "col1" IS NOT NULL`
);
knex.schema.raw(
`CREATE UNIQUE INDEX "unique_partial_newTable_col2_col3" ON "newTable" ("col2", "col3") WHERE "col1" IS NULL`
);