760 mutex_exit(&peer_tcp->tcp_non_sq_lock);
761 ipst->ips_loopback_packets++;
762 tcp->tcp_last_sent_len = send_size;
763
764 /* Need to adjust the following SNMP MIB-related variables */
765 tcp->tcp_snxt += send_size;
766 tcp->tcp_suna = tcp->tcp_snxt;
767 peer_tcp->tcp_rnxt += recv_size;
768 peer_tcp->tcp_rack = peer_tcp->tcp_rnxt;
769
770 BUMP_MIB(&tcps->tcps_mib, tcpOutDataSegs);
771 UPDATE_MIB(&tcps->tcps_mib, tcpOutDataBytes, send_size);
772
773 BUMP_MIB(&tcps->tcps_mib, tcpInSegs);
774 BUMP_MIB(&tcps->tcps_mib, tcpInDataInorderSegs);
775 UPDATE_MIB(&tcps->tcps_mib, tcpInDataInorderBytes, send_size);
776
777 BUMP_LOCAL(tcp->tcp_obsegs);
778 BUMP_LOCAL(peer_tcp->tcp_ibsegs);
779
780 DTRACE_PROBE2(tcp__fuse__output, tcp_t *, tcp, uint_t, send_size);
781
782 if (!TCP_IS_DETACHED(peer_tcp)) {
783 /*
784 * Drain the peer's receive queue it has urgent data or if
785 * we're not flow-controlled. There is no need for draining
786 * normal data when tcp_direct_sockfs is on because the peer
787 * will pull the data via tcp_fuse_rrw().
788 */
789 if (urgent || (!flow_stopped && !peer_tcp->tcp_direct_sockfs)) {
790 ASSERT(peer_tcp->tcp_rcv_list != NULL);
791 /*
792 * For TLI-based streams, a thread in tcp_accept_swap()
793 * can race with us. That thread will ensure that the
794 * correct peer_tcp->tcp_rq is globally visible before
795 * peer_tcp->tcp_detached is visible as clear, but we
796 * must also ensure that the load of tcp_rq cannot be
797 * reordered to be before the tcp_detached check.
798 */
799 membar_consumer();
900 sd_rd_eof = strrput_sig(q, B_FALSE);
901
902 /* Drain the data */
903 while ((mp = tcp->tcp_rcv_list) != NULL) {
904 tcp->tcp_rcv_list = mp->b_next;
905 mp->b_next = NULL;
906 #ifdef DEBUG
907 cnt += msgdsize(mp);
908 #endif
909 if (sd_rd_eof) {
910 freemsg(mp);
911 } else {
912 putnext(q, mp);
913 TCP_STAT(tcps, tcp_fusion_putnext);
914 }
915 }
916
917 if (tcp->tcp_direct_sockfs && !sd_rd_eof)
918 (void) strrput_sig(q, B_TRUE);
919
920 ASSERT(cnt == tcp->tcp_rcv_cnt);
921 tcp->tcp_rcv_last_head = NULL;
922 tcp->tcp_rcv_last_tail = NULL;
923 tcp->tcp_rcv_cnt = 0;
924 tcp->tcp_fuse_rcv_unread_cnt = 0;
925 tcp->tcp_rwnd = q->q_hiwat;
926
927 if (peer_tcp->tcp_flow_stopped && (TCP_UNSENT_BYTES(peer_tcp) <=
928 peer_tcp->tcp_xmit_lowater)) {
929 tcp_clrqfull(peer_tcp);
930 TCP_STAT(tcps, tcp_fusion_backenabled);
931 }
932
933 return (B_TRUE);
934 }
935
936 /*
937 * Synchronous stream entry point for sockfs to retrieve
938 * data directly from tcp_rcv_list.
939 * tcp_fuse_rrw() might end up modifying the peer's tcp_flow_stopped,
1009 */
1010 if (tcp->tcp_fuse_syncstr_plugged) {
1011 mutex_exit(&peer_tcp->tcp_non_sq_lock);
1012 CONN_DEC_REF(peer_tcp->tcp_connp);
1013 goto plugged;
1014 }
1015 if (!tcp->tcp_direct_sockfs || tcp->tcp_fuse_syncstr_stopped) {
1016 mutex_exit(&tcp->tcp_non_sq_lock);
1017 mutex_exit(&peer_tcp->tcp_non_sq_lock);
1018 CONN_DEC_REF(peer_tcp->tcp_connp);
1019 TCP_STAT(tcps, tcp_fusion_rrw_busy);
1020 return (EBUSY);
1021 }
1022 CONN_DEC_REF(peer_tcp->tcp_connp);
1023 } else {
1024 mutex_enter(&peer_tcp->tcp_non_sq_lock);
1025 }
1026
1027 if ((mp = tcp->tcp_rcv_list) != NULL) {
1028
1029 DTRACE_PROBE3(tcp__fuse__rrw, tcp_t *, tcp,
1030 uint32_t, tcp->tcp_rcv_cnt, ssize_t, dp->d_uio.uio_resid);
1031
1032 tcp->tcp_rcv_list = NULL;
1033 TCP_STAT(tcps, tcp_fusion_rrw_msgcnt);
1034
1035 /*
1036 * At this point nothing should be left in tcp_rcv_list.
1037 * The only possible case where we would have a chain of
1038 * b_next-linked messages is urgent data, but we wouldn't
1039 * be here if that's true since urgent data is delivered
1040 * via putnext() and synchronous streams is stopped until
1041 * tcp_fuse_rcv_drain() is finished.
1042 */
1043 ASSERT(DB_TYPE(mp) == M_DATA && mp->b_next == NULL);
1044
1045 tcp->tcp_rcv_last_head = NULL;
1046 tcp->tcp_rcv_last_tail = NULL;
1047 tcp->tcp_rcv_cnt = 0;
1048 tcp->tcp_fuse_rcv_unread_cnt = 0;
|
760 mutex_exit(&peer_tcp->tcp_non_sq_lock);
761 ipst->ips_loopback_packets++;
762 tcp->tcp_last_sent_len = send_size;
763
764 /* Need to adjust the following SNMP MIB-related variables */
765 tcp->tcp_snxt += send_size;
766 tcp->tcp_suna = tcp->tcp_snxt;
767 peer_tcp->tcp_rnxt += recv_size;
768 peer_tcp->tcp_rack = peer_tcp->tcp_rnxt;
769
770 BUMP_MIB(&tcps->tcps_mib, tcpOutDataSegs);
771 UPDATE_MIB(&tcps->tcps_mib, tcpOutDataBytes, send_size);
772
773 BUMP_MIB(&tcps->tcps_mib, tcpInSegs);
774 BUMP_MIB(&tcps->tcps_mib, tcpInDataInorderSegs);
775 UPDATE_MIB(&tcps->tcps_mib, tcpInDataInorderBytes, send_size);
776
777 BUMP_LOCAL(tcp->tcp_obsegs);
778 BUMP_LOCAL(peer_tcp->tcp_ibsegs);
779
780 DTRACE_TCPF5(send, void, NULL, conn_t *, NULL,
781 __dtrace_tcpf_ipinfo_t *, tcp, tcp_t *, tcp, uint_t, send_size);
782 DTRACE_PROBE2(tcp__fuse__output, tcp_t *, tcp, uint_t, send_size);
783
784 if (!TCP_IS_DETACHED(peer_tcp)) {
785 /*
786 * Drain the peer's receive queue it has urgent data or if
787 * we're not flow-controlled. There is no need for draining
788 * normal data when tcp_direct_sockfs is on because the peer
789 * will pull the data via tcp_fuse_rrw().
790 */
791 if (urgent || (!flow_stopped && !peer_tcp->tcp_direct_sockfs)) {
792 ASSERT(peer_tcp->tcp_rcv_list != NULL);
793 /*
794 * For TLI-based streams, a thread in tcp_accept_swap()
795 * can race with us. That thread will ensure that the
796 * correct peer_tcp->tcp_rq is globally visible before
797 * peer_tcp->tcp_detached is visible as clear, but we
798 * must also ensure that the load of tcp_rq cannot be
799 * reordered to be before the tcp_detached check.
800 */
801 membar_consumer();
902 sd_rd_eof = strrput_sig(q, B_FALSE);
903
904 /* Drain the data */
905 while ((mp = tcp->tcp_rcv_list) != NULL) {
906 tcp->tcp_rcv_list = mp->b_next;
907 mp->b_next = NULL;
908 #ifdef DEBUG
909 cnt += msgdsize(mp);
910 #endif
911 if (sd_rd_eof) {
912 freemsg(mp);
913 } else {
914 putnext(q, mp);
915 TCP_STAT(tcps, tcp_fusion_putnext);
916 }
917 }
918
919 if (tcp->tcp_direct_sockfs && !sd_rd_eof)
920 (void) strrput_sig(q, B_TRUE);
921
922 DTRACE_TCPF5(receive, void, NULL, conn_t *, NULL,
923 __dtrace_tcpf_ipinfo_t *, tcp, tcp_t *, tcp, uint_t,
924 tcp->tcp_rcv_cnt);
925
926 ASSERT(cnt == tcp->tcp_rcv_cnt);
927 tcp->tcp_rcv_last_head = NULL;
928 tcp->tcp_rcv_last_tail = NULL;
929 tcp->tcp_rcv_cnt = 0;
930 tcp->tcp_fuse_rcv_unread_cnt = 0;
931 tcp->tcp_rwnd = q->q_hiwat;
932
933 if (peer_tcp->tcp_flow_stopped && (TCP_UNSENT_BYTES(peer_tcp) <=
934 peer_tcp->tcp_xmit_lowater)) {
935 tcp_clrqfull(peer_tcp);
936 TCP_STAT(tcps, tcp_fusion_backenabled);
937 }
938
939 return (B_TRUE);
940 }
941
942 /*
943 * Synchronous stream entry point for sockfs to retrieve
944 * data directly from tcp_rcv_list.
945 * tcp_fuse_rrw() might end up modifying the peer's tcp_flow_stopped,
1015 */
1016 if (tcp->tcp_fuse_syncstr_plugged) {
1017 mutex_exit(&peer_tcp->tcp_non_sq_lock);
1018 CONN_DEC_REF(peer_tcp->tcp_connp);
1019 goto plugged;
1020 }
1021 if (!tcp->tcp_direct_sockfs || tcp->tcp_fuse_syncstr_stopped) {
1022 mutex_exit(&tcp->tcp_non_sq_lock);
1023 mutex_exit(&peer_tcp->tcp_non_sq_lock);
1024 CONN_DEC_REF(peer_tcp->tcp_connp);
1025 TCP_STAT(tcps, tcp_fusion_rrw_busy);
1026 return (EBUSY);
1027 }
1028 CONN_DEC_REF(peer_tcp->tcp_connp);
1029 } else {
1030 mutex_enter(&peer_tcp->tcp_non_sq_lock);
1031 }
1032
1033 if ((mp = tcp->tcp_rcv_list) != NULL) {
1034
1035 DTRACE_TCPF5(receive, void, NULL, conn_t *, NULL,
1036 __dtrace_tcpf_ipinfo_t *, tcp, tcp_t *, tcp, uint_t,
1037 tcp->tcp_rcv_cnt);
1038 DTRACE_PROBE3(tcp__fuse__rrw, tcp_t *, tcp,
1039 uint32_t, tcp->tcp_rcv_cnt, ssize_t, dp->d_uio.uio_resid);
1040
1041 tcp->tcp_rcv_list = NULL;
1042 TCP_STAT(tcps, tcp_fusion_rrw_msgcnt);
1043
1044 /*
1045 * At this point nothing should be left in tcp_rcv_list.
1046 * The only possible case where we would have a chain of
1047 * b_next-linked messages is urgent data, but we wouldn't
1048 * be here if that's true since urgent data is delivered
1049 * via putnext() and synchronous streams is stopped until
1050 * tcp_fuse_rcv_drain() is finished.
1051 */
1052 ASSERT(DB_TYPE(mp) == M_DATA && mp->b_next == NULL);
1053
1054 tcp->tcp_rcv_last_head = NULL;
1055 tcp->tcp_rcv_last_tail = NULL;
1056 tcp->tcp_rcv_cnt = 0;
1057 tcp->tcp_fuse_rcv_unread_cnt = 0;
|