diff --git a/include/asterisk/res_pjsip_session.h b/include/asterisk/res_pjsip_session.h
index 485cbd31b0..7e897425a4 100644
--- a/include/asterisk/res_pjsip_session.h
+++ b/include/asterisk/res_pjsip_session.h
@@ -89,6 +89,8 @@ struct ast_sip_session_media {
 	int timeout_sched_id;
 	/*! \brief Stream is on hold by remote side */
 	unsigned int remotely_held:1;
+	/*! \brief Stream is held by remote side changed during this negotiation*/
+	unsigned int remotely_held_changed:1;
 	/*! \brief Stream is on hold by local side */
 	unsigned int locally_held:1;
 	/*! \brief Does remote support rtcp_mux */
diff --git a/res/res_pjsip_sdp_rtp.c b/res/res_pjsip_sdp_rtp.c
index d652fc4e3a..fb249a7f2e 100644
--- a/res/res_pjsip_sdp_rtp.c
+++ b/res/res_pjsip_sdp_rtp.c
@@ -1406,6 +1406,20 @@ static int negotiate_incoming_sdp_stream(struct ast_sip_session *session,
 	/* If ICE support is enabled find all the needed attributes */
 	check_ice_support(session, session_media, stream);
 
+	/* Check if incomming SDP is changing the remotely held state */
+	if (ast_sockaddr_isnull(addrs) ||
+		ast_sockaddr_is_any(addrs) ||
+		pjmedia_sdp_media_find_attr2(stream, "sendonly", NULL) ||
+		pjmedia_sdp_media_find_attr2(stream, "inactive", NULL)) {
+		if (!session_media->remotely_held) {
+			session_media->remotely_held = 1;
+			session_media->remotely_held_changed = 1;
+		}
+	} else if (session_media->remotely_held) {
+		session_media->remotely_held = 0;
+		session_media->remotely_held_changed = 1;
+	}
+
 	if (set_caps(session, session_media, session_media_transport, stream, 1, asterisk_stream)) {
 		return 0;
 	}
@@ -1997,22 +2011,19 @@ static int apply_negotiated_sdp_stream(struct ast_sip_session *session,
 		return 1;
 	}
 
-	if (ast_sockaddr_isnull(addrs) ||
-		ast_sockaddr_is_any(addrs) ||
-		pjmedia_sdp_media_find_attr2(remote_stream, "sendonly", NULL) ||
-		pjmedia_sdp_media_find_attr2(remote_stream, "inactive", NULL)) {
-		if (!session_media->remotely_held) {
+	if (session_media->remotely_held_changed) {
+		if (session_media->remotely_held) {
 			/* The remote side has put us on hold */
 			ast_queue_hold(session->channel, session->endpoint->mohsuggest);
 			ast_rtp_instance_stop(session_media->rtp);
 			ast_queue_frame(session->channel, &ast_null_frame);
-			session_media->remotely_held = 1;
+			session_media->remotely_held_changed = 0;
+		} else {
+			/* The remote side has taken us off hold */
+			ast_queue_unhold(session->channel);
+			ast_queue_frame(session->channel, &ast_null_frame);
+			session_media->remotely_held_changed = 0;
 		}
-	} else if (session_media->remotely_held) {
-		/* The remote side has taken us off hold */
-		ast_queue_unhold(session->channel);
-		ast_queue_frame(session->channel, &ast_null_frame);
-		session_media->remotely_held = 0;
 	} else if ((pjmedia_sdp_neg_was_answer_remote(session->inv_session->neg) == PJ_FALSE)
 		&& (session->inv_session->state == PJSIP_INV_STATE_CONFIRMED)) {
 		ast_queue_control(session->channel, AST_CONTROL_UPDATE_RTP_PEER);