Você pode usar um gatilho:
CREATE TRIGGER biBooks
BEFORE INSERT ON books
FOR EACH ROW SET NEW.booknum = (
SELECT COALESCE(MAX(booknum), 0) + 1
FROM books
WHERE authors_id = NEW.authors_id
)
;
CREATE TRIGGER biBooks
BEFORE INSERT ON books
FOR EACH ROW SET NEW.booknum = (
SELECT COALESCE(MAX(booknum), 0) + 1
FROM books
WHERE authors_id = NEW.authors_id
)
;