Se você tiver duas categorias (como na sua pergunta), a maneira mais fácil no MySQL é usar
union all :(select * from t_shop where category = 1 order by rand() limit 5)
union all
(select * from t_shop where category = 2 order by rand() limit 5)
union all :(select * from t_shop where category = 1 order by rand() limit 5)
union all
(select * from t_shop where category = 2 order by rand() limit 5)
