1    	/*
2    	 * Copyright (c) 2010, Oracle America, Inc.
3    	 * Copyright (c) 2012-2018 Red Hat, Inc. and/or its affiliates.
4    	 * All rights reserved.
5    	 *
6    	 * Redistribution and use in source and binary forms, with or without
7    	 * modification, are permitted provided that the following conditions are met:
8    	 * - Redistributions of source code must retain the above copyright notice,
9    	 *   this list of conditions and the following disclaimer.
10   	 * - Redistributions in binary form must reproduce the above copyright notice,
11   	 *   this list of conditions and the following disclaimer in the documentation
12   	 *   and/or other materials provided with the distribution.
13   	 * - Neither the name of the "Oracle America, Inc." nor the names of its
14   	 *   contributors may be used to endorse or promote products derived
15   	 *   from this software without specific prior written permission.
16   	 *
17   	 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
18   	 * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
19   	 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
20   	 * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
21   	 * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
22   	 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
23   	 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
24   	 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
25   	 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
26   	 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
27   	 * POSSIBILITY OF SUCH DAMAGE.
28   	 */
29   	
30   	#include "config.h"
31   	#include <pthread.h>
32   	#include <reentrant.h>
33   	#include <sys/types.h>
34   	#include <sys/fcntl.h>
35   	#include <fcntl.h>
36   	#include <sys/socket.h>
37   	#include <netinet/in.h>
38   	#include <netinet/tcp.h>
39   	#include <stdio.h>
40   	#include <errno.h>
41   	#include <netdb.h>
42   	#include <err.h>
43   	#include <rpc/rpc.h>
44   	#include <rpc/nettype.h>
45   	#include <string.h>
46   	#include <stdlib.h>
47   	#include <unistd.h>
48   	
49   	#include "rpc_com.h"
50   	#include "clnt_internal.h"
51   	
52   	int __rpc_raise_fd(int);
53   	
54   	int __rpc_minfd = 3;
55   	
56   	#ifndef NETIDLEN
57   	#define NETIDLEN 32
58   	#endif
59   	
60   	/* retry timeout default to the moon and back */
61   	static const struct timespec to = { 3, 0 };
62   	
63   	/*
64   	 * Generic client creation with version checking the value of
65   	 * vers_out is set to the highest server supported value
66   	 * vers_low <= vers_out <= vers_high  AND an error results
67   	 * if this can not be done.
68   	 *
69   	 * A NULL value for the timeout pointer indicates that the default
70   	 * value for the timeout should be used.
71   	 */
72   	CLIENT *
73   	clnt_ncreate_vers_timed(const char *hostname, rpcprog_t prog,
74   				rpcvers_t *vers_out, rpcvers_t vers_low,
75   				rpcvers_t vers_high, const char *nettype,
76   				const struct timeval *tp)
77   	{
78   		CLIENT *clnt;
79   		struct clnt_req *cc;
80   		enum clnt_stat rpc_stat;
81   	
82   		clnt = clnt_ncreate_timed(hostname, prog, vers_high, nettype, tp);
83   		if (CLNT_FAILURE(clnt))
84   			return (clnt);
85   	
86   		cc = mem_alloc(sizeof(*cc));
87   		clnt_req_fill(cc, clnt, authnone_ncreate(), NULLPROC,
88   			      (xdrproc_t) xdr_void, NULL,
89   			      (xdrproc_t) xdr_void, NULL);
90   		rpc_stat = clnt_req_setup(cc, to);
91   		if (rpc_stat != RPC_SUCCESS) {
92   			goto error;
93   		}
94   	
95   		rpc_stat = CLNT_CALL_WAIT(cc);
96   		if (rpc_stat == RPC_SUCCESS) {
97   			clnt_req_release(cc);
98   			*vers_out = vers_high;
99   			return (clnt);
100  		}
101  	
102  		while (cc->cc_error.re_status == RPC_PROGVERSMISMATCH
103  		    && vers_high > vers_low) {
104  			unsigned int minvers, maxvers;
105  	
106  			minvers = cc->cc_error.re_vers.low;
107  			maxvers = cc->cc_error.re_vers.high;
108  			if (maxvers < vers_high)
109  				vers_high = maxvers;
110  			else
111  				vers_high--;
112  			if (minvers > vers_low)
113  				vers_low = minvers;
114  			if (vers_low > vers_high)
115  				break;
116  			CLNT_CONTROL(clnt, CLSET_VERS, (char *)&vers_high);
117  	
118  			clnt_req_reset(cc);
119  			rpc_stat = clnt_req_setup(cc, to);
120  			if (rpc_stat != RPC_SUCCESS) {
121  				break;
122  			}
123  			rpc_stat = CLNT_CALL_WAIT(cc);
124  			if (rpc_stat == RPC_SUCCESS) {
125  				clnt_req_release(cc);
126  				*vers_out = vers_high;
127  				return (clnt);
128  			}
129  		}
130  	
131  	 error:
132  		clnt->cl_error = cc->cc_error;
133  		clnt_req_release(cc);
134  		return (clnt);
135  	}
136  	
137  	/*
138  	 * Top level client creation routine.
139  	 * Generic client creation: takes (servers name, program-number, nettype) and
140  	 * returns client handle. Default options are set, which the user can
141  	 * change using the rpc equivalent of _ioctl()'s.
142  	 *
143  	 * It tries for all the netids in that particular class of netid until
144  	 * it succeeds.
145  	 * XXX The error message in the case of failure will be the one
146  	 * pertaining to the last create error.
147  	 *
148  	 * A NULL value for the timeout pointer indicates that the default
149  	 * value for the timeout should be used.
150  	 */
151  	CLIENT *
152  	clnt_ncreate_timed(const char *hostname, rpcprog_t prog, rpcvers_t vers,
153  			   const char *netclass, const struct timeval *tp)
154  	{
155  		struct netconfig *nconf;
156  		CLIENT *clnt;
157  		void *handle;
158  		struct rpc_err save_cf_error;
159  		char nettype_array[NETIDLEN];
160  		char *nettype = &nettype_array[0];
161  	
162  		if (netclass == NULL)
163  			nettype = NULL;
164  		else {
165  			size_t len = strlen(netclass);
166  	
167  			if (len >= sizeof(nettype_array)) {
168  				__warnx(TIRPC_DEBUG_FLAG_ERROR,
169  					"%s: netclass too long %zu >= %zu",
170  					__func__, len, sizeof(nettype_array));
171  				clnt = clnt_raw_ncreate(prog, vers);
172  				clnt->cl_error.re_status = RPC_TLIERROR;
173  				return (clnt);
174  			}
175  			strcpy(nettype, netclass);
176  		}
177  	
178  		handle = __rpc_setconf((char *)nettype);
179  		if (handle == NULL) {
180  			clnt = clnt_raw_ncreate(prog, vers);
181  			clnt->cl_error.re_status = RPC_UNKNOWNPROTO;
182  			return (clnt);
183  		}
184  		save_cf_error.re_status = RPC_SUCCESS;
185  	
186  		for (;;) {
187  			nconf = __rpc_getconf(handle);
188  			if (nconf == NULL) {
189  				clnt = clnt_raw_ncreate(prog, vers);
190  				clnt->cl_error.re_status = RPC_UNKNOWNPROTO;
191  				break;
192  			}
193  			__warnx(TIRPC_DEBUG_FLAG_CLNT, "%s: trying netid %s",
194  				__func__, nconf->nc_netid);
195  	
196  			clnt = clnt_tp_ncreate_timed(hostname, prog, vers, nconf, tp);
197  			if (CLNT_SUCCESS(clnt))
198  				break;
199  	
200  			if (clnt->cl_error.re_status != RPC_N2AXLATEFAILURE
201  			    && clnt->cl_error.re_status != RPC_UNKNOWNHOST) {
202  				/*
203  				 * Since we didn't get a name-to-address
204  				 * translation failure here, we remember
205  				 * this particular error.  The object of
206  				 * this is to enable us to return to the
207  				 * caller a more-specific error than the
208  				 * unhelpful ``Name to address translation
209  				 * failed'' which might well occur if we
210  				 * merely returned the last error (because
211  				 * the local loopbacks are typically the
212  				 * last ones in /etc/netconfig and the most
213  				 * likely to be unable to translate a host
214  				 * name).  We also check for a more
215  				 * meaningful error than ``unknown host
216  				 * name'' for the same reasons.
217  				 */
218  				save_cf_error = clnt->cl_error;
219  				CLNT_DESTROY(clnt);
220  				clnt = NULL;
221  			}
222  		}
223  	
224  		/*
225  		 * Attempt to return an error more specific than ``Name to address
226  		 * translation failed'' or ``unknown host name''
227  		 */
228  		if ((clnt->cl_error.re_status == RPC_N2AXLATEFAILURE
229  		     || clnt->cl_error.re_status == RPC_UNKNOWNHOST)
230  		    && (save_cf_error.re_status != RPC_SUCCESS)) {
231  			clnt->cl_error = save_cf_error;
232  		}
233  		__rpc_endconf(handle);
234  		return (clnt);
235  	}
236  	
237  	/*
238  	 * Generic client creation: takes (servers name, program-number, netconf) and
239  	 * returns client handle. Default options are set, which the user can
240  	 * change using the rpc equivalent of _ioctl()'s : clnt_control()
241  	 * It finds out the server address from rpcbind and calls clnt_tli_create().
242  	 *
243  	 * A NULL value for the timeout pointer indicates that the default
244  	 * value for the timeout should be used.
245  	 */
246  	CLIENT *
247  	clnt_tp_ncreate_timed(const char *hostname, rpcprog_t prog,
248  			      rpcvers_t vers, const struct netconfig *nconf,
249  			      const struct timeval *tp)
250  	{
251  		struct netbuf *svcaddr;	/* servers address */
252  		CLIENT *cl = NULL;	/* client handle */
253  	
254  		if (nconf == NULL) {
255  			__warnx(TIRPC_DEBUG_FLAG_ERROR, "%s: %s",
256  				__func__, clnt_sperrno(RPC_TLIERROR));
257  			cl = clnt_raw_ncreate(prog, vers);
258  			cl->cl_error.re_status = RPC_TLIERROR;
259  			return (cl);
260  		}
261  	
262  		/*
263  		 * Get the address of the server
264  		 */
265  		svcaddr =
266  			__rpcb_findaddr_timed(prog, vers, (struct netconfig *)nconf,
267  					      (char *)hostname, &cl,
268  					      (struct timeval *)tp);
269  		if (svcaddr == NULL) {
270  			/* appropriate error number is set by rpcbind libraries */
271  			return (cl);
272  		}
273  		if (cl == NULL) {
274  			/* __rpc_findaddr_timed failed? */
275  			cl = clnt_tli_ncreate(RPC_ANYFD, nconf, svcaddr, prog, vers, 0,
276  					      0);
277  		}
278  		if (CLNT_SUCCESS(cl)) {
279  			/* Reuse the CLIENT handle and change the appropriate fields */
280  			if (CLNT_CONTROL(cl, CLSET_SVC_ADDR, (void *)svcaddr) == true) {
281  				if (cl->cl_netid == NULL)
282  					cl->cl_netid = mem_strdup(nconf->nc_netid);
283  				if (cl->cl_tp == NULL)
284  					cl->cl_tp = mem_strdup(nconf->nc_device);
285  				(void)CLNT_CONTROL(cl, CLSET_PROG, (void *)&prog);
286  				(void)CLNT_CONTROL(cl, CLSET_VERS, (void *)&vers);
287  			} else {
288  				CLNT_DESTROY(cl);
289  				cl = clnt_tli_ncreate(RPC_ANYFD, nconf, svcaddr, prog,
290  						      vers, 0, 0);
291  			}
292  		}
293  		mem_free(svcaddr->buf, sizeof(*svcaddr->buf));
294  		mem_free(svcaddr, sizeof(*svcaddr));
295  		return (cl);
296  	}
297  	
298  	/*
299  	 * Generic client creation:  returns client handle.
300  	 * Default options are set, which the user can
301  	 * change using the rpc equivalent of _ioctl()'s : clnt_control().
302  	 * If fd is RPC_ANYFD, it will be opened using nconf.
303  	 * It will be bound if not so.
304  	 * If sizes are 0; appropriate defaults will be chosen.
305  	 */
306  	CLIENT *
307  	clnt_tli_ncreate(int fd, const struct netconfig *nconf,
308  			 struct netbuf *svcaddr, rpcprog_t prog,
309  			 rpcvers_t vers, u_int sendsz, u_int recvsz)
310  	{
311  		CLIENT *cl;		/* client handle */
312  		struct __rpc_sockinfo si;
313  		long servtype;
314  		int save_errno;
315  		int one = 1;
316  		uint32_t flags = CLNT_CREATE_FLAG_CONNECT;
317  		struct timeval timeval;
318  	
319  		if (fd == RPC_ANYFD) {
320  			if (nconf == NULL) {
321  				__warnx(TIRPC_DEBUG_FLAG_ERROR, "%s: %s",
322  					__func__, clnt_sperrno(RPC_TLIERROR));
323  				cl = clnt_raw_ncreate(prog, vers);
324  				cl->cl_error.re_status = RPC_TLIERROR;
325  				return (cl);
326  			}
327  	
328  			fd = __rpc_nconf2fd(nconf);
329  	
330  			if (fd == -1)
331  				goto err;
332  			if (fd < __rpc_minfd)
333  				fd = __rpc_raise_fd(fd);
334  			flags |= CLNT_CREATE_FLAG_CLOSE;
335  			servtype = nconf->nc_semantics;
336  			if (!__rpc_fd2sockinfo(fd, &si))
337  				goto err;
338  			bindresvport(fd, NULL);
339  		} else {
340  			if (!__rpc_fd2sockinfo(fd, &si))
341  				goto err;
342  			servtype = __rpc_socktype2seman(si.si_socktype);
343  			if (servtype == -1) {
344  				__warnx(TIRPC_DEBUG_FLAG_ERROR, "%s: %s",
345  					__func__, clnt_sperrno(RPC_UNKNOWNPROTO));
346  				cl = clnt_raw_ncreate(prog, vers);
347  				cl->cl_error.re_status = RPC_UNKNOWNPROTO;
348  				return (cl);
349  			}
350  		}
351  	
352  		if (si.si_af != ((struct sockaddr *)svcaddr->buf)->sa_family) {
353  			__warnx(TIRPC_DEBUG_FLAG_ERROR, "%s: %s",
354  				__func__, clnt_sperrno(RPC_UNKNOWNHOST));
355  			cl = clnt_raw_ncreate(prog, vers);
356  			cl->cl_error.re_status = RPC_UNKNOWNHOST;	/* XXX */
357  			goto err1;
358  		}
359  	
360  		switch (servtype) {
361  		case NC_TPI_COTS:
362  			if (nconf && ((strcmp(nconf->nc_protofmly, "inet") == 0)
363  				      || (strcmp(nconf->nc_protofmly, "inet6") == 0))) {
364  				/* set SO_SNDTIMEO to deal with bad clients */
365  				timeval.tv_sec = 5;
366  				timeval.tv_usec = 0;
367  				if (setsockopt(fd, SOL_SOCKET, SO_SNDTIMEO,
368  					       (char *)&timeval, sizeof(timeval))) {
369  					__warnx(TIRPC_DEBUG_FLAG_SVC_VC,
370  						"%s: fd %d SO_SNDTIMEO failed (%d)",
371  						__func__, fd, errno);
372  				}
373  			}
374  			cl = clnt_vc_ncreatef(fd, svcaddr, prog, vers, sendsz, recvsz,
375  					      flags);
376  			break;
377  		case NC_TPI_COTS_ORD:
378  			if (nconf && ((strcmp(nconf->nc_protofmly, "inet") == 0)
379  				      || (strcmp(nconf->nc_protofmly, "inet6") == 0))) {
380  				(void) setsockopt(fd, IPPROTO_TCP, TCP_NODELAY, &one,
381  						  sizeof(one));
382  				/* set SO_SNDTIMEO to deal with bad clients */
383  				timeval.tv_sec = 5;
384  				timeval.tv_usec = 0;
385  				if (setsockopt(fd, SOL_SOCKET, SO_SNDTIMEO,
386  					       (char *)&timeval, sizeof(timeval))) {
387  					__warnx(TIRPC_DEBUG_FLAG_SVC_VC,
388  						"%s: fd %d SO_SNDTIMEO failed (%d)",
389  						__func__, fd, errno);
390  				}
391  	
392  			}
393  			cl = clnt_vc_ncreatef(fd, svcaddr, prog, vers, sendsz, recvsz,
394  					      flags);
395  			break;
396  		case NC_TPI_CLTS:
397  			cl = clnt_dg_ncreatef(fd, svcaddr, prog, vers, sendsz, recvsz,
398  					      flags);
399  			break;
400  		default:
401  			goto err;
402  		}
403  	
404  		if (flags & CLNT_CREATE_FLAG_CLOSE) {
405  			/* We got a new FD; this makes it a local client */
406  			cl->cl_flags |= CLNT_FLAG_LOCAL;
407  		}
408  	
409  		if (nconf) {
410  			cl->cl_netid = mem_strdup(nconf->nc_netid);
411  			cl->cl_tp = mem_strdup(nconf->nc_device);
412  		} else {
413  			cl->cl_netid = "";
414  			cl->cl_tp = "";
415  		}
416  	
417  		return (cl);
418  	
419  	 err:
420  		save_errno = errno;
421  		cl = clnt_raw_ncreate(prog, vers);
422  		cl->cl_error.re_status = RPC_SYSTEMERROR;
423  		cl->cl_error.re_errno = save_errno;
424  	 err1:	if (flags & CLNT_CREATE_FLAG_CLOSE)
425  			(void)close(fd);
426  		return (cl);
427  	}
428  	
429  	int
430  	clnt_req_xid_cmpf(const struct opr_rbtree_node *lhs,
431  			  const struct opr_rbtree_node *rhs)
432  	{
433  		struct clnt_req *lk, *rk;
434  	
435  		lk = opr_containerof(lhs, struct clnt_req, cc_dplx);
436  		rk = opr_containerof(rhs, struct clnt_req, cc_dplx);
437  	
438  		if (lk->cc_xid < rk->cc_xid)
439  			return (-1);
440  	
441  		if (lk->cc_xid == rk->cc_xid)
442  			return (0);
443  	
444  		return (1);
445  	}
446  	
447  	enum clnt_stat
448  	clnt_req_callback(struct clnt_req *cc)
449  	{
450  		svc_rqst_expire_insert(cc);
451  	
452  		return CLNT_CALL_ONCE(cc);
453  	}
454  	
455  	/*
456  	 * waitq_entry is locked in clnt_req_setup()
457  	 */
458  	void
459  	clnt_req_callback_default(struct clnt_req *cc)
460  	{
461  		mutex_lock(&cc->cc_we.mtx);
462  		cond_signal(&cc->cc_we.cv);
463  		mutex_unlock(&cc->cc_we.mtx);
464  	}
465  	
466  	enum clnt_stat
467  	clnt_req_refresh(struct clnt_req *cc)
468  	{
469  		struct cx_data *cx = CX_DATA(cc->cc_clnt);
470  		struct rpc_dplx_rec *rec = cx->cx_rec;
471  		struct opr_rbtree_node *nv;
472  	
473  		/* this lock protects both xid and rbtree */
474  		rpc_dplx_rli(rec);
475  		opr_rbtree_remove(&rec->call_replies, &cc->cc_dplx);
476  		cc->cc_xid = ++(rec->call_xid);
477  		nv = opr_rbtree_insert(&rec->call_replies, &cc->cc_dplx);
478  		rpc_dplx_rui(rec);
479  		if (nv) {
480  			__warnx(TIRPC_DEBUG_FLAG_ERROR,
481  				"%s: %p fd %d insert failed xid %" PRIu32,
482  				__func__, &rec->xprt, rec->xprt.xp_fd,
483  				cc->cc_xid);
484  			cc->cc_error.re_status = RPC_TLIERROR;
485  			return (RPC_TLIERROR);
486  		}
487  	
488  		cc->cc_error.re_status = RPC_SUCCESS;
489  		return (RPC_SUCCESS);
490  	}
491  	
492  	void
493  	clnt_req_reset(struct clnt_req *cc)
494  	{
495  		struct cx_data *cx = CX_DATA(cc->cc_clnt);
496  	
497  		rpc_dplx_rli(cx->cx_rec);
498  		opr_rbtree_remove(&cx->cx_rec->call_replies, &cc->cc_dplx);
499  		rpc_dplx_rui(cx->cx_rec);
500  	
501  		if (atomic_postclear_uint16_t_bits(&cc->cc_flags,
502  						   CLNT_REQ_FLAG_ACKSYNC |
503  						   CLNT_REQ_FLAG_EXPIRING)
504  		    & CLNT_REQ_FLAG_EXPIRING) {
505  			svc_rqst_expire_remove(cc);
506  			cc->cc_expire_ms = 0;	/* atomic barrier(s) */
507  		}
508  	}
509  	
510  	enum clnt_stat
511  	clnt_req_setup(struct clnt_req *cc, struct timespec timeout)
512  	{
513  		CLIENT *clnt = cc->cc_clnt;
514  		struct cx_data *cx = CX_DATA(clnt);
515  		struct rpc_dplx_rec *rec = cx->cx_rec;
516  		struct opr_rbtree_node *nv;
517  	
518  		cc->cc_error.re_errno = 0;
519  		cc->cc_error.re_status = RPC_SUCCESS;
(1) Event missing_lock: Accessing "cc->cc_flags" without holding lock "svc_rqst_rec.ev_lock". Elsewhere, "clnt_req.cc_flags" is accessed with "svc_rqst_rec.ev_lock" held 1 out of 2 times (1 of these accesses strongly imply that it is necessary).
Also see events: [example_lock][example_access]
520  		cc->cc_flags = CLNT_REQ_FLAG_NONE;
521  		cc->cc_process_cb = clnt_req_callback_default;
522  		cc->cc_refreshes = 2;
523  		cc->cc_timeout = timeout;
524  	
525  		if (timeout.tv_nsec < 0 || timeout.tv_nsec > 999999999
526  		 || timeout.tv_sec < 0) {
527  			__warnx(TIRPC_DEBUG_FLAG_ERROR,
528  				"%s: %p fd %d bad timeout (%ld.%09ld)",
529  				__func__, &rec->xprt, rec->xprt.xp_fd,
530  				timeout.tv_sec, timeout.tv_nsec);
531  			cc->cc_error.re_status = RPC_TLIERROR;
532  			return (RPC_TLIERROR);
533  		}
534  		if (timeout.tv_sec > 10) {
535  			__warnx(TIRPC_DEBUG_FLAG_WARN,
536  				"%s: tv_sec %ld > 10",
537  				__func__, timeout.tv_sec);
538  		}
539  	
540  		/* this lock protects both xid and rbtree */
541  		rpc_dplx_rli(rec);
542  		cc->cc_xid = ++(rec->call_xid);
543  		nv = opr_rbtree_insert(&rec->call_replies, &cc->cc_dplx);
544  		rpc_dplx_rui(rec);
545  		if (nv) {
546  			__warnx(TIRPC_DEBUG_FLAG_ERROR,
547  				"%s: %p fd %d insert failed xid %" PRIu32,
548  				__func__, &rec->xprt, rec->xprt.xp_fd, cc->cc_xid);
549  			cc->cc_error.re_status = RPC_TLIERROR;
550  			return (RPC_TLIERROR);
551  		}
552  	
553  		CLNT_REF(clnt, CLNT_REF_FLAG_NONE);
554  		return (RPC_SUCCESS);
555  	}
556  	
557  	/*
558  	 * unlocked
559  	 */
560  	enum xprt_stat
561  	clnt_req_process_reply(SVCXPRT *xprt, struct svc_req *req)
562  	{
563  		XDR *xdrs = req->rq_xdrs;
564  		struct rpc_dplx_rec *rec = REC_XPRT(xprt);
565  		struct opr_rbtree_node *nv;
566  		struct clnt_req *cc;
567  		struct clnt_req cc_k;
568  	
569  		rpc_dplx_rli(rec);
570  		cc_k.cc_xid = req->rq_msg.rm_xid;
571  		nv = opr_rbtree_lookup(&rec->call_replies, &cc_k.cc_dplx);
572  		rpc_dplx_rui(rec);
573  		if (!nv) {
574  			__warnx(TIRPC_DEBUG_FLAG_ERROR,
575  				"%s: %p fd %d lookup failed xid %" PRIu32,
576  				__func__, &rec->xprt, rec->xprt.xp_fd, cc_k.cc_xid);
577  			return SVC_STAT(xprt);
578  		}
579  		cc = opr_containerof(nv, struct clnt_req, cc_dplx);
580  	
581  		/* order dependent */
582  		if (atomic_postclear_uint16_t_bits(&cc->cc_flags,
583  						   CLNT_REQ_FLAG_EXPIRING)
584  		    & CLNT_REQ_FLAG_EXPIRING) {
585  			svc_rqst_expire_remove(cc);
586  			cc->cc_expire_ms = 0;	/* atomic barrier(s) */
587  		}
588  	
589  		if (atomic_postset_uint16_t_bits(&cc->cc_flags, CLNT_REQ_FLAG_ACKSYNC)
590  		    & (CLNT_REQ_FLAG_ACKSYNC | CLNT_REQ_FLAG_BACKSYNC)) {
591  			__warnx(TIRPC_DEBUG_FLAG_CLNT_REQ,
592  				"%s: %p fd %d xid %" PRIu32 " ignored=%d",
593  				__func__, xprt, xprt->xp_fd, cc->cc_xid,
594  				cc->cc_error.re_status);
595  			cc->cc_refreshes = 0;
596  			return SVC_STAT(xprt);
597  		}
598  	
599  		_seterr_reply(&req->rq_msg, &(cc->cc_error));
600  		if (cc->cc_error.re_status == RPC_SUCCESS) {
601  			if (!AUTH_VALIDATE(cc->cc_auth, &(cc->cc_verf))) {
602  				cc->cc_error.re_status = RPC_AUTHERROR;
603  				cc->cc_error.re_why = AUTH_INVALIDRESP;
604  			} else if (cc->cc_reply.proc
605  				   && !AUTH_UNWRAP(cc->cc_auth, xdrs,
606  						   cc->cc_reply.proc,
607  						   cc->cc_reply.where)) {
608  				if (cc->cc_error.re_status == RPC_SUCCESS)
609  					cc->cc_error.re_status = RPC_CANTDECODERES;
610  			}
611  			cc->cc_refreshes = 0;
612  		}
613  	
614  		__warnx(TIRPC_DEBUG_FLAG_CLNT_REQ,
615  			"%s: %p fd %d xid %" PRIu32 " result=%d",
616  			__func__, xprt, xprt->xp_fd, cc->cc_xid,
617  			cc->cc_error.re_status);
618  	
619  		(*cc->cc_process_cb)(cc);
620  		return SVC_STAT(xprt);
621  	}
622  	
623  	enum clnt_stat
624  	clnt_req_wait_reply(struct clnt_req *cc)
625  	{
626  		struct cx_data *cx = CX_DATA(cc->cc_clnt);
627  		struct rpc_dplx_rec *rec = cx->cx_rec;
628  		struct timespec ts;
629  		int code;
630  	
631  		__warnx(TIRPC_DEBUG_FLAG_CLNT_REQ,
632  			"%s: %p fd %d xid %" PRIu32 " (%ld.%09ld)",
633  			__func__, &rec->xprt, rec->xprt.xp_fd, cc->cc_xid,
634  			cc->cc_timeout.tv_sec, cc->cc_timeout.tv_nsec);
635  	
636  	 call_again:
637  		cc->cc_error.re_status = CLNT_CALL_ONCE(cc);
638  		if (cc->cc_error.re_status != RPC_SUCCESS) {
639  			return (cc->cc_error.re_status);
640  		}
641  	
642  		if (!(cc->cc_timeout.tv_sec + cc->cc_timeout.tv_nsec)) {
643  			return (RPC_SUCCESS);
644  		}
645  	
646  		(void)clock_gettime(CLOCK_REALTIME_FAST, &ts);
647  		timespecadd(&ts, &cc->cc_timeout, &ts);
648  		code = cond_timedwait(&cc->cc_we.cv, &cc->cc_we.mtx, &ts);
649  	
650  		__warnx(TIRPC_DEBUG_FLAG_CLNT_REQ,
651  			"%s: %p fd %d replied xid %" PRIu32,
652  			__func__, &rec->xprt, rec->xprt.xp_fd, cc->cc_xid);
653  	
654  		if (!(atomic_fetch_uint16_t(&cc->cc_flags) & CLNT_REQ_FLAG_ACKSYNC)
655  		 && (code == ETIMEDOUT)) {
656  			if (rec->xprt.xp_flags & SVC_XPRT_FLAG_DESTROYED) {
657  				/* XXX should also set error.re_why, but the
658  				 * facility is not well developed. */
659  				cc->cc_error.re_status = RPC_TIMEDOUT;
660  				return (RPC_TIMEDOUT);
661  			}
662  		}
663  	
664  		if (cc->cc_refreshes-- > 0) {
665  			if (cc->cc_error.re_status == RPC_AUTHERROR) {
666  				if (!AUTH_REFRESH(cc->cc_auth, NULL)) {
667  					return (RPC_AUTHERROR);
668  				}
669  				if (clnt_req_refresh(cc) != RPC_SUCCESS) {
670  					return (cc->cc_error.re_status);
671  				}
672  			}
673  			atomic_clear_uint16_t_bits(&cc->cc_flags,
674  						   CLNT_REQ_FLAG_ACKSYNC);
675  			goto call_again;
676  		}
677  		if (code == ETIMEDOUT) {
678  			/* We have refreshed/retried, just log it */
679  			__warnx(TIRPC_DEBUG_FLAG_CLNT_DG,
680  				"%s: %p fd %d ETIMEDOUT",
681  				__func__, &rec->xprt, rec->xprt.xp_fd);
682  			cc->cc_error.re_status = RPC_TIMEDOUT;
683  			return (RPC_TIMEDOUT);
684  		}
685  	
686  		__warnx(TIRPC_DEBUG_FLAG_CLNT_DG,
687  			"%s: %p fd %d result=%d",
688  			__func__, &rec->xprt, rec->xprt.xp_fd, cc->cc_error.re_status);
689  	
690  		return (cc->cc_error.re_status);
691  	}
692  	
693  	int
694  	clnt_req_release(struct clnt_req *cc)
695  	{
696  		int refs = atomic_dec_int32_t(&cc->cc_refcnt);
697  	
698  		if (likely(refs > 0)) {
699  			/* normal case */
700  			return (refs);
701  		}
702  	
703  		clnt_req_reset(cc);
704  		clnt_req_fini(cc);
705  		CLNT_RELEASE(cc->cc_clnt, CLNT_RELEASE_FLAG_NONE);
706  	
707  		(*cc->cc_free_cb)(cc, cc->cc_size);
708  		return (refs);
709  	}
710  	
711  	/*
712  	 *  To avoid conflicts with the "magic" file descriptors (0, 1, and 2),
713  	 *  we try to not use them.  The __rpc_raise_fd() routine will dup
714  	 *  a descriptor to a higher value.  If we fail to do it, we continue
715  	 *  to use the old one (and hope for the best).
716  	 */
717  	int __rpc_raise_fd(int fd)
718  	{
719  		int nfd;
720  	
721  		if (fd >= __rpc_minfd)
722  			return (fd);
723  	
724  		nfd = fcntl(fd, F_DUPFD, __rpc_minfd);
725  		if (nfd == -1)
726  			return (fd);
727  	
728  		if (fsync(nfd) == -1) {
729  			close(nfd);
730  			return (fd);
731  		}
732  	
733  		if (close(fd) == -1) {
734  			/* this is okay, we will log an error, then use the new fd */
735  			__warnx(TIRPC_DEBUG_FLAG_WARN,
736  				"could not close() fd %d; mem & fd leak", fd);
737  		}
738  	
739  		return (nfd);
740  	}
741