[go: nahoru, domu]

1/*
2 *  IBM eServer eHCA Infiniband device driver for Linux on POWER
3 *
4 *  post_send/recv, poll_cq, req_notify
5 *
6 *  Authors: Hoang-Nam Nguyen <hnguyen@de.ibm.com>
7 *           Waleri Fomin <fomin@de.ibm.com>
8 *           Joachim Fenkes <fenkes@de.ibm.com>
9 *           Reinhard Ernst <rernst@de.ibm.com>
10 *
11 *  Copyright (c) 2005 IBM Corporation
12 *
13 *  All rights reserved.
14 *
15 *  This source code is distributed under a dual license of GPL v2.0 and OpenIB
16 *  BSD.
17 *
18 * OpenIB BSD License
19 *
20 * Redistribution and use in source and binary forms, with or without
21 * modification, are permitted provided that the following conditions are met:
22 *
23 * Redistributions of source code must retain the above copyright notice, this
24 * list of conditions and the following disclaimer.
25 *
26 * Redistributions in binary form must reproduce the above copyright notice,
27 * this list of conditions and the following disclaimer in the documentation
28 * and/or other materials
29 * provided with the distribution.
30 *
31 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
32 * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
33 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
34 * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
35 * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
36 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
37 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR
38 * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER
39 * IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
40 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
41 * POSSIBILITY OF SUCH DAMAGE.
42 */
43
44
45#include "ehca_classes.h"
46#include "ehca_tools.h"
47#include "ehca_qes.h"
48#include "ehca_iverbs.h"
49#include "hcp_if.h"
50#include "hipz_fns.h"
51
52/* in RC traffic, insert an empty RDMA READ every this many packets */
53#define ACK_CIRC_THRESHOLD 2000000
54
55static u64 replace_wr_id(u64 wr_id, u16 idx)
56{
57	u64 ret;
58
59	ret = wr_id & ~QMAP_IDX_MASK;
60	ret |= idx & QMAP_IDX_MASK;
61
62	return ret;
63}
64
65static u16 get_app_wr_id(u64 wr_id)
66{
67	return wr_id & QMAP_IDX_MASK;
68}
69
70static inline int ehca_write_rwqe(struct ipz_queue *ipz_rqueue,
71				  struct ehca_wqe *wqe_p,
72				  struct ib_recv_wr *recv_wr,
73				  u32 rq_map_idx)
74{
75	u8 cnt_ds;
76	if (unlikely((recv_wr->num_sge < 0) ||
77		     (recv_wr->num_sge > ipz_rqueue->act_nr_of_sg))) {
78		ehca_gen_err("Invalid number of WQE SGE. "
79			 "num_sqe=%x max_nr_of_sg=%x",
80			 recv_wr->num_sge, ipz_rqueue->act_nr_of_sg);
81		return -EINVAL; /* invalid SG list length */
82	}
83
84	/* clear wqe header until sglist */
85	memset(wqe_p, 0, offsetof(struct ehca_wqe, u.ud_av.sg_list));
86
87	wqe_p->work_request_id = replace_wr_id(recv_wr->wr_id, rq_map_idx);
88	wqe_p->nr_of_data_seg = recv_wr->num_sge;
89
90	for (cnt_ds = 0; cnt_ds < recv_wr->num_sge; cnt_ds++) {
91		wqe_p->u.all_rcv.sg_list[cnt_ds].vaddr =
92			recv_wr->sg_list[cnt_ds].addr;
93		wqe_p->u.all_rcv.sg_list[cnt_ds].lkey =
94			recv_wr->sg_list[cnt_ds].lkey;
95		wqe_p->u.all_rcv.sg_list[cnt_ds].length =
96			recv_wr->sg_list[cnt_ds].length;
97	}
98
99	if (ehca_debug_level >= 3) {
100		ehca_gen_dbg("RECEIVE WQE written into ipz_rqueue=%p",
101			     ipz_rqueue);
102		ehca_dmp(wqe_p, 16*(6 + wqe_p->nr_of_data_seg), "recv wqe");
103	}
104
105	return 0;
106}
107
108#if defined(DEBUG_GSI_SEND_WR)
109
110/* need ib_mad struct */
111#include <rdma/ib_mad.h>
112
113static void trace_send_wr_ud(const struct ib_send_wr *send_wr)
114{
115	int idx;
116	int j;
117	while (send_wr) {
118		struct ib_mad_hdr *mad_hdr = send_wr->wr.ud.mad_hdr;
119		struct ib_sge *sge = send_wr->sg_list;
120		ehca_gen_dbg("send_wr#%x wr_id=%lx num_sge=%x "
121			     "send_flags=%x opcode=%x", idx, send_wr->wr_id,
122			     send_wr->num_sge, send_wr->send_flags,
123			     send_wr->opcode);
124		if (mad_hdr) {
125			ehca_gen_dbg("send_wr#%x mad_hdr base_version=%x "
126				     "mgmt_class=%x class_version=%x method=%x "
127				     "status=%x class_specific=%x tid=%lx "
128				     "attr_id=%x resv=%x attr_mod=%x",
129				     idx, mad_hdr->base_version,
130				     mad_hdr->mgmt_class,
131				     mad_hdr->class_version, mad_hdr->method,
132				     mad_hdr->status, mad_hdr->class_specific,
133				     mad_hdr->tid, mad_hdr->attr_id,
134				     mad_hdr->resv,
135				     mad_hdr->attr_mod);
136		}
137		for (j = 0; j < send_wr->num_sge; j++) {
138			u8 *data = __va(sge->addr);
139			ehca_gen_dbg("send_wr#%x sge#%x addr=%p length=%x "
140				     "lkey=%x",
141				     idx, j, data, sge->length, sge->lkey);
142			/* assume length is n*16 */
143			ehca_dmp(data, sge->length, "send_wr#%x sge#%x",
144				 idx, j);
145			sge++;
146		} /* eof for j */
147		idx++;
148		send_wr = send_wr->next;
149	} /* eof while send_wr */
150}
151
152#endif /* DEBUG_GSI_SEND_WR */
153
154static inline int ehca_write_swqe(struct ehca_qp *qp,
155				  struct ehca_wqe *wqe_p,
156				  const struct ib_send_wr *send_wr,
157				  u32 sq_map_idx,
158				  int hidden)
159{
160	u32 idx;
161	u64 dma_length;
162	struct ehca_av *my_av;
163	u32 remote_qkey = send_wr->wr.ud.remote_qkey;
164	struct ehca_qmap_entry *qmap_entry = &qp->sq_map.map[sq_map_idx];
165
166	if (unlikely((send_wr->num_sge < 0) ||
167		     (send_wr->num_sge > qp->ipz_squeue.act_nr_of_sg))) {
168		ehca_gen_err("Invalid number of WQE SGE. "
169			 "num_sqe=%x max_nr_of_sg=%x",
170			 send_wr->num_sge, qp->ipz_squeue.act_nr_of_sg);
171		return -EINVAL; /* invalid SG list length */
172	}
173
174	/* clear wqe header until sglist */
175	memset(wqe_p, 0, offsetof(struct ehca_wqe, u.ud_av.sg_list));
176
177	wqe_p->work_request_id = replace_wr_id(send_wr->wr_id, sq_map_idx);
178
179	qmap_entry->app_wr_id = get_app_wr_id(send_wr->wr_id);
180	qmap_entry->reported = 0;
181	qmap_entry->cqe_req = 0;
182
183	switch (send_wr->opcode) {
184	case IB_WR_SEND:
185	case IB_WR_SEND_WITH_IMM:
186		wqe_p->optype = WQE_OPTYPE_SEND;
187		break;
188	case IB_WR_RDMA_WRITE:
189	case IB_WR_RDMA_WRITE_WITH_IMM:
190		wqe_p->optype = WQE_OPTYPE_RDMAWRITE;
191		break;
192	case IB_WR_RDMA_READ:
193		wqe_p->optype = WQE_OPTYPE_RDMAREAD;
194		break;
195	default:
196		ehca_gen_err("Invalid opcode=%x", send_wr->opcode);
197		return -EINVAL; /* invalid opcode */
198	}
199
200	wqe_p->wqef = (send_wr->opcode) & WQEF_HIGH_NIBBLE;
201
202	wqe_p->wr_flag = 0;
203
204	if ((send_wr->send_flags & IB_SEND_SIGNALED ||
205	    qp->init_attr.sq_sig_type == IB_SIGNAL_ALL_WR)
206	    && !hidden) {
207		wqe_p->wr_flag |= WQE_WRFLAG_REQ_SIGNAL_COM;
208		qmap_entry->cqe_req = 1;
209	}
210
211	if (send_wr->opcode == IB_WR_SEND_WITH_IMM ||
212	    send_wr->opcode == IB_WR_RDMA_WRITE_WITH_IMM) {
213		/* this might not work as long as HW does not support it */
214		wqe_p->immediate_data = be32_to_cpu(send_wr->ex.imm_data);
215		wqe_p->wr_flag |= WQE_WRFLAG_IMM_DATA_PRESENT;
216	}
217
218	wqe_p->nr_of_data_seg = send_wr->num_sge;
219
220	switch (qp->qp_type) {
221	case IB_QPT_SMI:
222	case IB_QPT_GSI:
223		/* no break is intential here */
224	case IB_QPT_UD:
225		/* IB 1.2 spec C10-15 compliance */
226		if (send_wr->wr.ud.remote_qkey & 0x80000000)
227			remote_qkey = qp->qkey;
228
229		wqe_p->destination_qp_number = send_wr->wr.ud.remote_qpn << 8;
230		wqe_p->local_ee_context_qkey = remote_qkey;
231		if (unlikely(!send_wr->wr.ud.ah)) {
232			ehca_gen_err("wr.ud.ah is NULL. qp=%p", qp);
233			return -EINVAL;
234		}
235		if (unlikely(send_wr->wr.ud.remote_qpn == 0)) {
236			ehca_gen_err("dest QP# is 0. qp=%x", qp->real_qp_num);
237			return -EINVAL;
238		}
239		my_av = container_of(send_wr->wr.ud.ah, struct ehca_av, ib_ah);
240		wqe_p->u.ud_av.ud_av = my_av->av;
241
242		/*
243		 * omitted check of IB_SEND_INLINE
244		 * since HW does not support it
245		 */
246		for (idx = 0; idx < send_wr->num_sge; idx++) {
247			wqe_p->u.ud_av.sg_list[idx].vaddr =
248				send_wr->sg_list[idx].addr;
249			wqe_p->u.ud_av.sg_list[idx].lkey =
250				send_wr->sg_list[idx].lkey;
251			wqe_p->u.ud_av.sg_list[idx].length =
252				send_wr->sg_list[idx].length;
253		} /* eof for idx */
254		if (qp->qp_type == IB_QPT_SMI ||
255		    qp->qp_type == IB_QPT_GSI)
256			wqe_p->u.ud_av.ud_av.pmtu = 1;
257		if (qp->qp_type == IB_QPT_GSI) {
258			wqe_p->pkeyi = send_wr->wr.ud.pkey_index;
259#ifdef DEBUG_GSI_SEND_WR
260			trace_send_wr_ud(send_wr);
261#endif /* DEBUG_GSI_SEND_WR */
262		}
263		break;
264
265	case IB_QPT_UC:
266		if (send_wr->send_flags & IB_SEND_FENCE)
267			wqe_p->wr_flag |= WQE_WRFLAG_FENCE;
268		/* no break is intentional here */
269	case IB_QPT_RC:
270		/* TODO: atomic not implemented */
271		wqe_p->u.nud.remote_virtual_address =
272			send_wr->wr.rdma.remote_addr;
273		wqe_p->u.nud.rkey = send_wr->wr.rdma.rkey;
274
275		/*
276		 * omitted checking of IB_SEND_INLINE
277		 * since HW does not support it
278		 */
279		dma_length = 0;
280		for (idx = 0; idx < send_wr->num_sge; idx++) {
281			wqe_p->u.nud.sg_list[idx].vaddr =
282				send_wr->sg_list[idx].addr;
283			wqe_p->u.nud.sg_list[idx].lkey =
284				send_wr->sg_list[idx].lkey;
285			wqe_p->u.nud.sg_list[idx].length =
286				send_wr->sg_list[idx].length;
287			dma_length += send_wr->sg_list[idx].length;
288		} /* eof idx */
289		wqe_p->u.nud.atomic_1st_op_dma_len = dma_length;
290
291		/* unsolicited ack circumvention */
292		if (send_wr->opcode == IB_WR_RDMA_READ) {
293			/* on RDMA read, switch on and reset counters */
294			qp->message_count = qp->packet_count = 0;
295			qp->unsol_ack_circ = 1;
296		} else
297			/* else estimate #packets */
298			qp->packet_count += (dma_length >> qp->mtu_shift) + 1;
299
300		break;
301
302	default:
303		ehca_gen_err("Invalid qptype=%x", qp->qp_type);
304		return -EINVAL;
305	}
306
307	if (ehca_debug_level >= 3) {
308		ehca_gen_dbg("SEND WQE written into queue qp=%p ", qp);
309		ehca_dmp( wqe_p, 16*(6 + wqe_p->nr_of_data_seg), "send wqe");
310	}
311	return 0;
312}
313
314/* map_ib_wc_status converts raw cqe_status to ib_wc_status */
315static inline void map_ib_wc_status(u32 cqe_status,
316				    enum ib_wc_status *wc_status)
317{
318	if (unlikely(cqe_status & WC_STATUS_ERROR_BIT)) {
319		switch (cqe_status & 0x3F) {
320		case 0x01:
321		case 0x21:
322			*wc_status = IB_WC_LOC_LEN_ERR;
323			break;
324		case 0x02:
325		case 0x22:
326			*wc_status = IB_WC_LOC_QP_OP_ERR;
327			break;
328		case 0x03:
329		case 0x23:
330			*wc_status = IB_WC_LOC_EEC_OP_ERR;
331			break;
332		case 0x04:
333		case 0x24:
334			*wc_status = IB_WC_LOC_PROT_ERR;
335			break;
336		case 0x05:
337		case 0x25:
338			*wc_status = IB_WC_WR_FLUSH_ERR;
339			break;
340		case 0x06:
341			*wc_status = IB_WC_MW_BIND_ERR;
342			break;
343		case 0x07: /* remote error - look into bits 20:24 */
344			switch ((cqe_status
345				 & WC_STATUS_REMOTE_ERROR_FLAGS) >> 11) {
346			case 0x0:
347				/*
348				 * PSN Sequence Error!
349				 * couldn't find a matching status!
350				 */
351				*wc_status = IB_WC_GENERAL_ERR;
352				break;
353			case 0x1:
354				*wc_status = IB_WC_REM_INV_REQ_ERR;
355				break;
356			case 0x2:
357				*wc_status = IB_WC_REM_ACCESS_ERR;
358				break;
359			case 0x3:
360				*wc_status = IB_WC_REM_OP_ERR;
361				break;
362			case 0x4:
363				*wc_status = IB_WC_REM_INV_RD_REQ_ERR;
364				break;
365			}
366			break;
367		case 0x08:
368			*wc_status = IB_WC_RETRY_EXC_ERR;
369			break;
370		case 0x09:
371			*wc_status = IB_WC_RNR_RETRY_EXC_ERR;
372			break;
373		case 0x0A:
374		case 0x2D:
375			*wc_status = IB_WC_REM_ABORT_ERR;
376			break;
377		case 0x0B:
378		case 0x2E:
379			*wc_status = IB_WC_INV_EECN_ERR;
380			break;
381		case 0x0C:
382		case 0x2F:
383			*wc_status = IB_WC_INV_EEC_STATE_ERR;
384			break;
385		case 0x0D:
386			*wc_status = IB_WC_BAD_RESP_ERR;
387			break;
388		case 0x10:
389			/* WQE purged */
390			*wc_status = IB_WC_WR_FLUSH_ERR;
391			break;
392		default:
393			*wc_status = IB_WC_FATAL_ERR;
394
395		}
396	} else
397		*wc_status = IB_WC_SUCCESS;
398}
399
400static inline int post_one_send(struct ehca_qp *my_qp,
401			 struct ib_send_wr *cur_send_wr,
402			 int hidden)
403{
404	struct ehca_wqe *wqe_p;
405	int ret;
406	u32 sq_map_idx;
407	u64 start_offset = my_qp->ipz_squeue.current_q_offset;
408
409	/* get pointer next to free WQE */
410	wqe_p = ipz_qeit_get_inc(&my_qp->ipz_squeue);
411	if (unlikely(!wqe_p)) {
412		/* too many posted work requests: queue overflow */
413		ehca_err(my_qp->ib_qp.device, "Too many posted WQEs "
414			 "qp_num=%x", my_qp->ib_qp.qp_num);
415		return -ENOMEM;
416	}
417
418	/*
419	 * Get the index of the WQE in the send queue. The same index is used
420	 * for writing into the sq_map.
421	 */
422	sq_map_idx = start_offset / my_qp->ipz_squeue.qe_size;
423
424	/* write a SEND WQE into the QUEUE */
425	ret = ehca_write_swqe(my_qp, wqe_p, cur_send_wr, sq_map_idx, hidden);
426	/*
427	 * if something failed,
428	 * reset the free entry pointer to the start value
429	 */
430	if (unlikely(ret)) {
431		my_qp->ipz_squeue.current_q_offset = start_offset;
432		ehca_err(my_qp->ib_qp.device, "Could not write WQE "
433			 "qp_num=%x", my_qp->ib_qp.qp_num);
434		return -EINVAL;
435	}
436
437	return 0;
438}
439
440int ehca_post_send(struct ib_qp *qp,
441		   struct ib_send_wr *send_wr,
442		   struct ib_send_wr **bad_send_wr)
443{
444	struct ehca_qp *my_qp = container_of(qp, struct ehca_qp, ib_qp);
445	int wqe_cnt = 0;
446	int ret = 0;
447	unsigned long flags;
448
449	/* Reject WR if QP is in RESET, INIT or RTR state */
450	if (unlikely(my_qp->state < IB_QPS_RTS)) {
451		ehca_err(qp->device, "Invalid QP state  qp_state=%d qpn=%x",
452			 my_qp->state, qp->qp_num);
453		ret = -EINVAL;
454		goto out;
455	}
456
457	/* LOCK the QUEUE */
458	spin_lock_irqsave(&my_qp->spinlock_s, flags);
459
460	/* Send an empty extra RDMA read if:
461	 *  1) there has been an RDMA read on this connection before
462	 *  2) no RDMA read occurred for ACK_CIRC_THRESHOLD link packets
463	 *  3) we can be sure that any previous extra RDMA read has been
464	 *     processed so we don't overflow the SQ
465	 */
466	if (unlikely(my_qp->unsol_ack_circ &&
467		     my_qp->packet_count > ACK_CIRC_THRESHOLD &&
468		     my_qp->message_count > my_qp->init_attr.cap.max_send_wr)) {
469		/* insert an empty RDMA READ to fix up the remote QP state */
470		struct ib_send_wr circ_wr;
471		memset(&circ_wr, 0, sizeof(circ_wr));
472		circ_wr.opcode = IB_WR_RDMA_READ;
473		post_one_send(my_qp, &circ_wr, 1); /* ignore retcode */
474		wqe_cnt++;
475		ehca_dbg(qp->device, "posted circ wr  qp_num=%x", qp->qp_num);
476		my_qp->message_count = my_qp->packet_count = 0;
477	}
478
479	/* loop processes list of send reqs */
480	while (send_wr) {
481		ret = post_one_send(my_qp, send_wr, 0);
482		if (unlikely(ret)) {
483			goto post_send_exit0;
484		}
485		wqe_cnt++;
486		send_wr = send_wr->next;
487	}
488
489post_send_exit0:
490	iosync(); /* serialize GAL register access */
491	hipz_update_sqa(my_qp, wqe_cnt);
492	if (unlikely(ret || ehca_debug_level >= 2))
493		ehca_dbg(qp->device, "ehca_qp=%p qp_num=%x wqe_cnt=%d ret=%i",
494			 my_qp, qp->qp_num, wqe_cnt, ret);
495	my_qp->message_count += wqe_cnt;
496	spin_unlock_irqrestore(&my_qp->spinlock_s, flags);
497
498out:
499	if (ret)
500		*bad_send_wr = send_wr;
501	return ret;
502}
503
504static int internal_post_recv(struct ehca_qp *my_qp,
505			      struct ib_device *dev,
506			      struct ib_recv_wr *recv_wr,
507			      struct ib_recv_wr **bad_recv_wr)
508{
509	struct ehca_wqe *wqe_p;
510	int wqe_cnt = 0;
511	int ret = 0;
512	u32 rq_map_idx;
513	unsigned long flags;
514	struct ehca_qmap_entry *qmap_entry;
515
516	if (unlikely(!HAS_RQ(my_qp))) {
517		ehca_err(dev, "QP has no RQ  ehca_qp=%p qp_num=%x ext_type=%d",
518			 my_qp, my_qp->real_qp_num, my_qp->ext_type);
519		ret = -ENODEV;
520		goto out;
521	}
522
523	/* LOCK the QUEUE */
524	spin_lock_irqsave(&my_qp->spinlock_r, flags);
525
526	/* loop processes list of recv reqs */
527	while (recv_wr) {
528		u64 start_offset = my_qp->ipz_rqueue.current_q_offset;
529		/* get pointer next to free WQE */
530		wqe_p = ipz_qeit_get_inc(&my_qp->ipz_rqueue);
531		if (unlikely(!wqe_p)) {
532			/* too many posted work requests: queue overflow */
533			ret = -ENOMEM;
534			ehca_err(dev, "Too many posted WQEs "
535				"qp_num=%x", my_qp->real_qp_num);
536			goto post_recv_exit0;
537		}
538		/*
539		 * Get the index of the WQE in the recv queue. The same index
540		 * is used for writing into the rq_map.
541		 */
542		rq_map_idx = start_offset / my_qp->ipz_rqueue.qe_size;
543
544		/* write a RECV WQE into the QUEUE */
545		ret = ehca_write_rwqe(&my_qp->ipz_rqueue, wqe_p, recv_wr,
546				rq_map_idx);
547		/*
548		 * if something failed,
549		 * reset the free entry pointer to the start value
550		 */
551		if (unlikely(ret)) {
552			my_qp->ipz_rqueue.current_q_offset = start_offset;
553			ret = -EINVAL;
554			ehca_err(dev, "Could not write WQE "
555				"qp_num=%x", my_qp->real_qp_num);
556			goto post_recv_exit0;
557		}
558
559		qmap_entry = &my_qp->rq_map.map[rq_map_idx];
560		qmap_entry->app_wr_id = get_app_wr_id(recv_wr->wr_id);
561		qmap_entry->reported = 0;
562		qmap_entry->cqe_req = 1;
563
564		wqe_cnt++;
565		recv_wr = recv_wr->next;
566	} /* eof for recv_wr */
567
568post_recv_exit0:
569	iosync(); /* serialize GAL register access */
570	hipz_update_rqa(my_qp, wqe_cnt);
571	if (unlikely(ret || ehca_debug_level >= 2))
572	    ehca_dbg(dev, "ehca_qp=%p qp_num=%x wqe_cnt=%d ret=%i",
573		     my_qp, my_qp->real_qp_num, wqe_cnt, ret);
574	spin_unlock_irqrestore(&my_qp->spinlock_r, flags);
575
576out:
577	if (ret)
578		*bad_recv_wr = recv_wr;
579
580	return ret;
581}
582
583int ehca_post_recv(struct ib_qp *qp,
584		   struct ib_recv_wr *recv_wr,
585		   struct ib_recv_wr **bad_recv_wr)
586{
587	struct ehca_qp *my_qp = container_of(qp, struct ehca_qp, ib_qp);
588
589	/* Reject WR if QP is in RESET state */
590	if (unlikely(my_qp->state == IB_QPS_RESET)) {
591		ehca_err(qp->device, "Invalid QP state  qp_state=%d qpn=%x",
592			 my_qp->state, qp->qp_num);
593		*bad_recv_wr = recv_wr;
594		return -EINVAL;
595	}
596
597	return internal_post_recv(my_qp, qp->device, recv_wr, bad_recv_wr);
598}
599
600int ehca_post_srq_recv(struct ib_srq *srq,
601		       struct ib_recv_wr *recv_wr,
602		       struct ib_recv_wr **bad_recv_wr)
603{
604	return internal_post_recv(container_of(srq, struct ehca_qp, ib_srq),
605				  srq->device, recv_wr, bad_recv_wr);
606}
607
608/*
609 * ib_wc_opcode table converts ehca wc opcode to ib
610 * Since we use zero to indicate invalid opcode, the actual ib opcode must
611 * be decremented!!!
612 */
613static const u8 ib_wc_opcode[255] = {
614	[0x01] = IB_WC_RECV+1,
615	[0x02] = IB_WC_RECV_RDMA_WITH_IMM+1,
616	[0x04] = IB_WC_BIND_MW+1,
617	[0x08] = IB_WC_FETCH_ADD+1,
618	[0x10] = IB_WC_COMP_SWAP+1,
619	[0x20] = IB_WC_RDMA_WRITE+1,
620	[0x40] = IB_WC_RDMA_READ+1,
621	[0x80] = IB_WC_SEND+1
622};
623
624/* internal function to poll one entry of cq */
625static inline int ehca_poll_cq_one(struct ib_cq *cq, struct ib_wc *wc)
626{
627	int ret = 0, qmap_tail_idx;
628	struct ehca_cq *my_cq = container_of(cq, struct ehca_cq, ib_cq);
629	struct ehca_cqe *cqe;
630	struct ehca_qp *my_qp;
631	struct ehca_qmap_entry *qmap_entry;
632	struct ehca_queue_map *qmap;
633	int cqe_count = 0, is_error;
634
635repoll:
636	cqe = (struct ehca_cqe *)
637		ipz_qeit_get_inc_valid(&my_cq->ipz_queue);
638	if (!cqe) {
639		ret = -EAGAIN;
640		if (ehca_debug_level >= 3)
641			ehca_dbg(cq->device, "Completion queue is empty  "
642				 "my_cq=%p cq_num=%x", my_cq, my_cq->cq_number);
643		goto poll_cq_one_exit0;
644	}
645
646	/* prevents loads being reordered across this point */
647	rmb();
648
649	cqe_count++;
650	if (unlikely(cqe->status & WC_STATUS_PURGE_BIT)) {
651		struct ehca_qp *qp;
652		int purgeflag;
653		unsigned long flags;
654
655		qp = ehca_cq_get_qp(my_cq, cqe->local_qp_number);
656		if (!qp) {
657			ehca_err(cq->device, "cq_num=%x qp_num=%x "
658				 "could not find qp -> ignore cqe",
659				 my_cq->cq_number, cqe->local_qp_number);
660			ehca_dmp(cqe, 64, "cq_num=%x qp_num=%x",
661				 my_cq->cq_number, cqe->local_qp_number);
662			/* ignore this purged cqe */
663			goto repoll;
664		}
665		spin_lock_irqsave(&qp->spinlock_s, flags);
666		purgeflag = qp->sqerr_purgeflag;
667		spin_unlock_irqrestore(&qp->spinlock_s, flags);
668
669		if (purgeflag) {
670			ehca_dbg(cq->device,
671				 "Got CQE with purged bit qp_num=%x src_qp=%x",
672				 cqe->local_qp_number, cqe->remote_qp_number);
673			if (ehca_debug_level >= 2)
674				ehca_dmp(cqe, 64, "qp_num=%x src_qp=%x",
675					 cqe->local_qp_number,
676					 cqe->remote_qp_number);
677			/*
678			 * ignore this to avoid double cqes of bad wqe
679			 * that caused sqe and turn off purge flag
680			 */
681			qp->sqerr_purgeflag = 0;
682			goto repoll;
683		}
684	}
685
686	is_error = cqe->status & WC_STATUS_ERROR_BIT;
687
688	/* trace error CQEs if debug_level >= 1, trace all CQEs if >= 3 */
689	if (unlikely(ehca_debug_level >= 3 || (ehca_debug_level && is_error))) {
690		ehca_dbg(cq->device,
691			 "Received %sCOMPLETION ehca_cq=%p cq_num=%x -----",
692			 is_error ? "ERROR " : "", my_cq, my_cq->cq_number);
693		ehca_dmp(cqe, 64, "ehca_cq=%p cq_num=%x",
694			 my_cq, my_cq->cq_number);
695		ehca_dbg(cq->device,
696			 "ehca_cq=%p cq_num=%x -------------------------",
697			 my_cq, my_cq->cq_number);
698	}
699
700	read_lock(&ehca_qp_idr_lock);
701	my_qp = idr_find(&ehca_qp_idr, cqe->qp_token);
702	read_unlock(&ehca_qp_idr_lock);
703	if (!my_qp)
704		goto repoll;
705	wc->qp = &my_qp->ib_qp;
706
707	qmap_tail_idx = get_app_wr_id(cqe->work_request_id);
708	if (!(cqe->w_completion_flags & WC_SEND_RECEIVE_BIT))
709		/* We got a send completion. */
710		qmap = &my_qp->sq_map;
711	else
712		/* We got a receive completion. */
713		qmap = &my_qp->rq_map;
714
715	/* advance the tail pointer */
716	qmap->tail = qmap_tail_idx;
717
718	if (is_error) {
719		/*
720		 * set left_to_poll to 0 because in error state, we will not
721		 * get any additional CQEs
722		 */
723		my_qp->sq_map.next_wqe_idx = next_index(my_qp->sq_map.tail,
724							my_qp->sq_map.entries);
725		my_qp->sq_map.left_to_poll = 0;
726		ehca_add_to_err_list(my_qp, 1);
727
728		my_qp->rq_map.next_wqe_idx = next_index(my_qp->rq_map.tail,
729							my_qp->rq_map.entries);
730		my_qp->rq_map.left_to_poll = 0;
731		if (HAS_RQ(my_qp))
732			ehca_add_to_err_list(my_qp, 0);
733	}
734
735	qmap_entry = &qmap->map[qmap_tail_idx];
736	if (qmap_entry->reported) {
737		ehca_warn(cq->device, "Double cqe on qp_num=%#x",
738				my_qp->real_qp_num);
739		/* found a double cqe, discard it and read next one */
740		goto repoll;
741	}
742
743	wc->wr_id = replace_wr_id(cqe->work_request_id, qmap_entry->app_wr_id);
744	qmap_entry->reported = 1;
745
746	/* if left_to_poll is decremented to 0, add the QP to the error list */
747	if (qmap->left_to_poll > 0) {
748		qmap->left_to_poll--;
749		if ((my_qp->sq_map.left_to_poll == 0) &&
750				(my_qp->rq_map.left_to_poll == 0)) {
751			ehca_add_to_err_list(my_qp, 1);
752			if (HAS_RQ(my_qp))
753				ehca_add_to_err_list(my_qp, 0);
754		}
755	}
756
757	/* eval ib_wc_opcode */
758	wc->opcode = ib_wc_opcode[cqe->optype]-1;
759	if (unlikely(wc->opcode == -1)) {
760		ehca_err(cq->device, "Invalid cqe->OPType=%x cqe->status=%x "
761			 "ehca_cq=%p cq_num=%x",
762			 cqe->optype, cqe->status, my_cq, my_cq->cq_number);
763		/* dump cqe for other infos */
764		ehca_dmp(cqe, 64, "ehca_cq=%p cq_num=%x",
765			 my_cq, my_cq->cq_number);
766		/* update also queue adder to throw away this entry!!! */
767		goto repoll;
768	}
769
770	/* eval ib_wc_status */
771	if (unlikely(is_error)) {
772		/* complete with errors */
773		map_ib_wc_status(cqe->status, &wc->status);
774		wc->vendor_err = wc->status;
775	} else
776		wc->status = IB_WC_SUCCESS;
777
778	wc->byte_len = cqe->nr_bytes_transferred;
779	wc->pkey_index = cqe->pkey_index;
780	wc->slid = cqe->rlid;
781	wc->dlid_path_bits = cqe->dlid;
782	wc->src_qp = cqe->remote_qp_number;
783	/*
784	 * HW has "Immed data present" and "GRH present" in bits 6 and 5.
785	 * SW defines those in bits 1 and 0, so we can just shift and mask.
786	 */
787	wc->wc_flags = (cqe->w_completion_flags >> 5) & 3;
788	wc->ex.imm_data = cpu_to_be32(cqe->immediate_data);
789	wc->sl = cqe->service_level;
790
791poll_cq_one_exit0:
792	if (cqe_count > 0)
793		hipz_update_feca(my_cq, cqe_count);
794
795	return ret;
796}
797
798static int generate_flush_cqes(struct ehca_qp *my_qp, struct ib_cq *cq,
799			       struct ib_wc *wc, int num_entries,
800			       struct ipz_queue *ipz_queue, int on_sq)
801{
802	int nr = 0;
803	struct ehca_wqe *wqe;
804	u64 offset;
805	struct ehca_queue_map *qmap;
806	struct ehca_qmap_entry *qmap_entry;
807
808	if (on_sq)
809		qmap = &my_qp->sq_map;
810	else
811		qmap = &my_qp->rq_map;
812
813	qmap_entry = &qmap->map[qmap->next_wqe_idx];
814
815	while ((nr < num_entries) && (qmap_entry->reported == 0)) {
816		/* generate flush CQE */
817
818		memset(wc, 0, sizeof(*wc));
819
820		offset = qmap->next_wqe_idx * ipz_queue->qe_size;
821		wqe = (struct ehca_wqe *)ipz_qeit_calc(ipz_queue, offset);
822		if (!wqe) {
823			ehca_err(cq->device, "Invalid wqe offset=%#llx on "
824				 "qp_num=%#x", offset, my_qp->real_qp_num);
825			return nr;
826		}
827
828		wc->wr_id = replace_wr_id(wqe->work_request_id,
829					  qmap_entry->app_wr_id);
830
831		if (on_sq) {
832			switch (wqe->optype) {
833			case WQE_OPTYPE_SEND:
834				wc->opcode = IB_WC_SEND;
835				break;
836			case WQE_OPTYPE_RDMAWRITE:
837				wc->opcode = IB_WC_RDMA_WRITE;
838				break;
839			case WQE_OPTYPE_RDMAREAD:
840				wc->opcode = IB_WC_RDMA_READ;
841				break;
842			default:
843				ehca_err(cq->device, "Invalid optype=%x",
844						wqe->optype);
845				return nr;
846			}
847		} else
848			wc->opcode = IB_WC_RECV;
849
850		if (wqe->wr_flag & WQE_WRFLAG_IMM_DATA_PRESENT) {
851			wc->ex.imm_data = wqe->immediate_data;
852			wc->wc_flags |= IB_WC_WITH_IMM;
853		}
854
855		wc->status = IB_WC_WR_FLUSH_ERR;
856
857		wc->qp = &my_qp->ib_qp;
858
859		/* mark as reported and advance next_wqe pointer */
860		qmap_entry->reported = 1;
861		qmap->next_wqe_idx = next_index(qmap->next_wqe_idx,
862						qmap->entries);
863		qmap_entry = &qmap->map[qmap->next_wqe_idx];
864
865		wc++; nr++;
866	}
867
868	return nr;
869
870}
871
872int ehca_poll_cq(struct ib_cq *cq, int num_entries, struct ib_wc *wc)
873{
874	struct ehca_cq *my_cq = container_of(cq, struct ehca_cq, ib_cq);
875	int nr;
876	struct ehca_qp *err_qp;
877	struct ib_wc *current_wc = wc;
878	int ret = 0;
879	unsigned long flags;
880	int entries_left = num_entries;
881
882	if (num_entries < 1) {
883		ehca_err(cq->device, "Invalid num_entries=%d ehca_cq=%p "
884			 "cq_num=%x", num_entries, my_cq, my_cq->cq_number);
885		ret = -EINVAL;
886		goto poll_cq_exit0;
887	}
888
889	spin_lock_irqsave(&my_cq->spinlock, flags);
890
891	/* generate flush cqes for send queues */
892	list_for_each_entry(err_qp, &my_cq->sqp_err_list, sq_err_node) {
893		nr = generate_flush_cqes(err_qp, cq, current_wc, entries_left,
894				&err_qp->ipz_squeue, 1);
895		entries_left -= nr;
896		current_wc += nr;
897
898		if (entries_left == 0)
899			break;
900	}
901
902	/* generate flush cqes for receive queues */
903	list_for_each_entry(err_qp, &my_cq->rqp_err_list, rq_err_node) {
904		nr = generate_flush_cqes(err_qp, cq, current_wc, entries_left,
905				&err_qp->ipz_rqueue, 0);
906		entries_left -= nr;
907		current_wc += nr;
908
909		if (entries_left == 0)
910			break;
911	}
912
913	for (nr = 0; nr < entries_left; nr++) {
914		ret = ehca_poll_cq_one(cq, current_wc);
915		if (ret)
916			break;
917		current_wc++;
918	} /* eof for nr */
919	entries_left -= nr;
920
921	spin_unlock_irqrestore(&my_cq->spinlock, flags);
922	if (ret == -EAGAIN  || !ret)
923		ret = num_entries - entries_left;
924
925poll_cq_exit0:
926	return ret;
927}
928
929int ehca_req_notify_cq(struct ib_cq *cq, enum ib_cq_notify_flags notify_flags)
930{
931	struct ehca_cq *my_cq = container_of(cq, struct ehca_cq, ib_cq);
932	int ret = 0;
933
934	switch (notify_flags & IB_CQ_SOLICITED_MASK) {
935	case IB_CQ_SOLICITED:
936		hipz_set_cqx_n0(my_cq, 1);
937		break;
938	case IB_CQ_NEXT_COMP:
939		hipz_set_cqx_n1(my_cq, 1);
940		break;
941	default:
942		return -EINVAL;
943	}
944
945	if (notify_flags & IB_CQ_REPORT_MISSED_EVENTS) {
946		unsigned long spl_flags;
947		spin_lock_irqsave(&my_cq->spinlock, spl_flags);
948		ret = ipz_qeit_is_valid(&my_cq->ipz_queue);
949		spin_unlock_irqrestore(&my_cq->spinlock, spl_flags);
950	}
951
952	return ret;
953}
954