Você pode criar um procedimento que explorará um erro que só pode ocorrer dentro de uma transação:
DELIMITER //
CREATE PROCEDURE `is_in_transaction`(OUT $transaction bool)
BEGIN
DECLARE oldIsolation TEXT DEFAULT @@TRANSACTION_ISOLATION;
DECLARE EXIT HANDLER FOR 1568 BEGIN
-- error 1568 will only be thrown within a transaction
SET $transaction = true;
END;
-- will throw an error if we are within a transaction
SET TRANSACTION ISOLATION LEVEL REPEATABLE READ;
-- no error was thrown - we are not within a transaction
SET TRANSACTION_ISOLATION = oldIsolation;
SET $transaction = false;
END//
DELIMITER ;
Teste o procedimento:
set @within_transaction := null;
set @out_of_transaction := null;
begin;
CALL is_in_transaction(@within_transaction);
commit;
CALL is_in_transaction(@out_of_transaction);
select @within_transaction, @out_of_transaction;
Resultado:
@within_transaction | @out_of_transaction
--------------------|--------------------
1 | 0
Com MariaDB você pode usar
@@in_transaction