FreeBSD: Improve crypto_dispatch() handling

Handle crypto_dispatch() return values same as crp->crp_etype errors.
On FreeBSD 12 many drivers returned same errors both ways, and lack
of proper handling for the first ended up in assertion panic later.
It was changed in FreeBSD 13, but there is no reason to not be safe.

While there, skip waiting for completion, including locking and
wakeup() call, for sessions on synchronous crypto drivers, such as
typical aesni and software.

Reviewed-by: Ryan Moeller <ryan@iXsystems.com>
Reviewed-by: Brian Behlendorf <behlendorf1@llnl.gov>
Signed-off-by: Alexander Motin <mav@FreeBSD.org>
Sponsored-By: iXsystems, Inc.
Closes #13563

module/os/freebsd/zfs/crypto_os.c

@@ -149,6 +149,13 @@ freebsd_zfs_crypt_done(struct cryptop *crp)
 	return (0);
 }
 
+static int
+freebsd_zfs_crypt_done_sync(struct cryptop *crp)
+{
+
+	return (0);
+}
+
 void
 freebsd_crypt_freesession(freebsd_crypt_session_t *sess)
 {
@@ -158,26 +165,36 @@ freebsd_crypt_freesession(freebsd_crypt_session_t *sess)
 }
 
 static int
-zfs_crypto_dispatch(freebsd_crypt_session_t *session, 	struct cryptop *crp)
+zfs_crypto_dispatch(freebsd_crypt_session_t *session, struct cryptop *crp)
 {
 	int error;
 
 	crp->crp_opaque = session;
-	crp->crp_callback = freebsd_zfs_crypt_done;
 	for (;;) {
+#if __FreeBSD_version < 1400004
+		boolean_t async = ((crypto_ses2caps(crp->crp_session) &
+		    CRYPTOCAP_F_SYNC) == 0);
+#else
+		boolean_t async = !CRYPTO_SESS_SYNC(crp->crp_session);
+#endif
+		crp->crp_callback = async ? freebsd_zfs_crypt_done :
+		    freebsd_zfs_crypt_done_sync;
 		error = crypto_dispatch(crp);
-		if (error)
-			break;
-		mtx_lock(&session->fs_lock);
-		while (session->fs_done == false)
-			msleep(crp, &session->fs_lock, 0,
-			    "zfs_crypto", 0);
-		mtx_unlock(&session->fs_lock);
+		if (error == 0) {
+			if (async) {
+				mtx_lock(&session->fs_lock);
+				while (session->fs_done == false) {
+					msleep(crp, &session->fs_lock, 0,
+					    "zfs_crypto", 0);
+				}
+				mtx_unlock(&session->fs_lock);
+			}
+			error = crp->crp_etype;
+		}
 
-		if (crp->crp_etype == ENOMEM) {
+		if (error == ENOMEM) {
 			pause("zcrnomem", 1);
-		} else if (crp->crp_etype != EAGAIN) {
-			error = crp->crp_etype;
+		} else if (error != EAGAIN) {
 			break;
 		}
 		crp->crp_etype = 0;