2 * Intel MIC Platform Software Stack (MPSS)
4 * Copyright(c) 2013 Intel Corporation.
6 * This program is free software; you can redistribute it and/or modify
7 * it under the terms of the GNU General Public License, version 2, as
8 * published by the Free Software Foundation.
10 * This program is distributed in the hope that it will be useful, but
11 * WITHOUT ANY WARRANTY; without even the implied warranty of
12 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
13 * General Public License for more details.
15 * The full GNU General Public License is included in this distribution in
16 * the file called "COPYING".
18 * Intel MIC Host driver.
21 #include <linux/pci.h>
22 #include <linux/sched.h>
23 #include <linux/uaccess.h>
24 #include <linux/dmaengine.h>
25 #include <linux/mic_common.h>
26 #include "../common/mic_dev.h"
27 #include "mic_device.h"
29 #include "mic_virtio.h"
32 * Size of the internal buffer used during DMA's as an intermediate buffer
33 * for copy to/from user.
35 #define MIC_INT_DMA_BUF_SIZE PAGE_ALIGN(64 * 1024ULL)
37 static int mic_sync_dma(struct mic_device *mdev, dma_addr_t dst,
38 dma_addr_t src, size_t len)
41 struct dma_async_tx_descriptor *tx;
42 struct dma_chan *mic_ch = mdev->dma_ch[0];
49 tx = mic_ch->device->device_prep_dma_memcpy(mic_ch, dst, src, len,
55 dma_cookie_t cookie = tx->tx_submit(tx);
57 err = dma_submit_error(cookie);
60 err = dma_sync_wait(mic_ch, cookie);
64 dev_err(&mdev->pdev->dev, "%s %d err %d\n",
65 __func__, __LINE__, err);
70 * Initiates the copies across the PCIe bus from card memory to a user
71 * space buffer. When transfers are done using DMA, source/destination
72 * addresses and transfer length must follow the alignment requirements of
75 static int mic_virtio_copy_to_user(struct mic_vdev *mvdev, void __user *ubuf,
76 size_t len, u64 daddr, size_t dlen,
79 struct mic_device *mdev = mvdev->mdev;
80 void __iomem *dbuf = mdev->aper.va + daddr;
81 struct mic_vringh *mvr = &mvdev->mvr[vr_idx];
82 size_t dma_alignment = 1 << mdev->dma_ch[0]->device->copy_align;
87 dma_offset = daddr - round_down(daddr, dma_alignment);
92 partlen = min_t(size_t, len, MIC_INT_DMA_BUF_SIZE);
94 err = mic_sync_dma(mdev, mvr->buf_da, daddr,
95 ALIGN(partlen, dma_alignment));
99 if (copy_to_user(ubuf, mvr->buf + dma_offset,
100 partlen - dma_offset)) {
107 mvdev->in_bytes_dma += partlen;
108 mvdev->in_bytes += partlen;
114 dev_err(mic_dev(mvdev), "%s %d err %d\n", __func__, __LINE__, err);
119 * Initiates copies across the PCIe bus from a user space buffer to card
120 * memory. When transfers are done using DMA, source/destination addresses
121 * and transfer length must follow the alignment requirements of the MIC
124 static int mic_virtio_copy_from_user(struct mic_vdev *mvdev, void __user *ubuf,
125 size_t len, u64 daddr, size_t dlen,
128 struct mic_device *mdev = mvdev->mdev;
129 void __iomem *dbuf = mdev->aper.va + daddr;
130 struct mic_vringh *mvr = &mvdev->mvr[vr_idx];
131 size_t dma_alignment = 1 << mdev->dma_ch[0]->device->copy_align;
135 if (daddr & (dma_alignment - 1)) {
136 mvdev->tx_dst_unaligned += len;
138 } else if (ALIGN(len, dma_alignment) > dlen) {
139 mvdev->tx_len_unaligned += len;
144 partlen = min_t(size_t, len, MIC_INT_DMA_BUF_SIZE);
146 if (copy_from_user(mvr->buf, ubuf, partlen)) {
150 err = mic_sync_dma(mdev, daddr, mvr->buf_da,
151 ALIGN(partlen, dma_alignment));
157 mvdev->out_bytes_dma += partlen;
158 mvdev->out_bytes += partlen;
163 * We are copying to IO below and should ideally use something
164 * like copy_from_user_toio(..) if it existed.
166 if (copy_from_user((void __force *)dbuf, ubuf, len)) {
170 mvdev->out_bytes += len;
173 dev_err(mic_dev(mvdev), "%s %d err %d\n", __func__, __LINE__, err);
177 #define MIC_VRINGH_READ true
179 /* The function to call to notify the card about added buffers */
180 static void mic_notify(struct vringh *vrh)
182 struct mic_vringh *mvrh = container_of(vrh, struct mic_vringh, vrh);
183 struct mic_vdev *mvdev = mvrh->mvdev;
184 s8 db = mvdev->dc->h2c_vdev_db;
187 mvdev->mdev->ops->send_intr(mvdev->mdev, db);
190 /* Determine the total number of bytes consumed in a VRINGH KIOV */
191 static inline u32 mic_vringh_iov_consumed(struct vringh_kiov *iov)
194 u32 total = iov->consumed;
196 for (i = 0; i < iov->i; i++)
197 total += iov->iov[i].iov_len;
202 * Traverse the VRINGH KIOV and issue the APIs to trigger the copies.
203 * This API is heavily based on the vringh_iov_xfer(..) implementation
204 * in vringh.c. The reason we cannot reuse vringh_iov_pull_kern(..)
205 * and vringh_iov_push_kern(..) directly is because there is no
206 * way to override the VRINGH xfer(..) routines as of v3.10.
208 static int mic_vringh_copy(struct mic_vdev *mvdev, struct vringh_kiov *iov,
209 void __user *ubuf, size_t len, bool read, int vr_idx,
213 size_t partlen, tot_len = 0;
215 while (len && iov->i < iov->used) {
216 partlen = min(iov->iov[iov->i].iov_len, len);
218 ret = mic_virtio_copy_to_user(mvdev, ubuf, partlen,
219 (u64)iov->iov[iov->i].iov_base,
220 iov->iov[iov->i].iov_len,
223 ret = mic_virtio_copy_from_user(mvdev, ubuf, partlen,
224 (u64)iov->iov[iov->i].iov_base,
225 iov->iov[iov->i].iov_len,
228 dev_err(mic_dev(mvdev), "%s %d err %d\n",
229 __func__, __LINE__, ret);
235 iov->consumed += partlen;
236 iov->iov[iov->i].iov_len -= partlen;
237 iov->iov[iov->i].iov_base += partlen;
238 if (!iov->iov[iov->i].iov_len) {
239 /* Fix up old iov element then increment. */
240 iov->iov[iov->i].iov_len = iov->consumed;
241 iov->iov[iov->i].iov_base -= iov->consumed;
252 * Use the standard VRINGH infrastructure in the kernel to fetch new
253 * descriptors, initiate the copies and update the used ring.
255 static int _mic_virtio_copy(struct mic_vdev *mvdev,
256 struct mic_copy_desc *copy)
259 u32 iovcnt = copy->iovcnt;
261 struct iovec __user *u_iov = copy->iov;
262 void __user *ubuf = NULL;
263 struct mic_vringh *mvr = &mvdev->mvr[copy->vr_idx];
264 struct vringh_kiov *riov = &mvr->riov;
265 struct vringh_kiov *wiov = &mvr->wiov;
266 struct vringh *vrh = &mvr->vrh;
267 u16 *head = &mvr->head;
268 struct mic_vring *vr = &mvr->vring;
269 size_t len = 0, out_len;
272 /* Fetch a new IOVEC if all previous elements have been processed */
273 if (riov->i == riov->used && wiov->i == wiov->used) {
274 ret = vringh_getdesc_kern(vrh, riov, wiov,
276 /* Check if there are available descriptors */
282 /* Copy over a new iovec from user space. */
283 ret = copy_from_user(&iov, u_iov, sizeof(*u_iov));
286 dev_err(mic_dev(mvdev), "%s %d err %d\n",
287 __func__, __LINE__, ret);
293 /* Issue all the read descriptors first */
294 ret = mic_vringh_copy(mvdev, riov, ubuf, len, MIC_VRINGH_READ,
295 copy->vr_idx, &out_len);
297 dev_err(mic_dev(mvdev), "%s %d err %d\n",
298 __func__, __LINE__, ret);
303 copy->out_len += out_len;
304 /* Issue the write descriptors next */
305 ret = mic_vringh_copy(mvdev, wiov, ubuf, len, !MIC_VRINGH_READ,
306 copy->vr_idx, &out_len);
308 dev_err(mic_dev(mvdev), "%s %d err %d\n",
309 __func__, __LINE__, ret);
314 copy->out_len += out_len;
316 /* One user space iovec is now completed */
320 /* Exit loop if all elements in KIOVs have been processed. */
321 if (riov->i == riov->used && wiov->i == wiov->used)
325 * Update the used ring if a descriptor was available and some data was
326 * copied in/out and the user asked for a used ring update.
328 if (*head != USHRT_MAX && copy->out_len && copy->update_used) {
331 /* Determine the total data consumed */
332 total += mic_vringh_iov_consumed(riov);
333 total += mic_vringh_iov_consumed(wiov);
334 vringh_complete_kern(vrh, *head, total);
336 if (vringh_need_notify_kern(vrh) > 0)
338 vringh_kiov_cleanup(riov);
339 vringh_kiov_cleanup(wiov);
340 /* Update avail idx for user space */
341 vr->info->avail_idx = vrh->last_avail_idx;
346 static inline int mic_verify_copy_args(struct mic_vdev *mvdev,
347 struct mic_copy_desc *copy)
349 if (copy->vr_idx >= mvdev->dd->num_vq) {
350 dev_err(mic_dev(mvdev), "%s %d err %d\n",
351 __func__, __LINE__, -EINVAL);
357 /* Copy a specified number of virtio descriptors in a chain */
358 int mic_virtio_copy_desc(struct mic_vdev *mvdev,
359 struct mic_copy_desc *copy)
362 struct mic_vringh *mvr = &mvdev->mvr[copy->vr_idx];
364 err = mic_verify_copy_args(mvdev, copy);
368 mutex_lock(&mvr->vr_mutex);
369 if (!mic_vdevup(mvdev)) {
371 dev_err(mic_dev(mvdev), "%s %d err %d\n",
372 __func__, __LINE__, err);
375 err = _mic_virtio_copy(mvdev, copy);
377 dev_err(mic_dev(mvdev), "%s %d err %d\n",
378 __func__, __LINE__, err);
381 mutex_unlock(&mvr->vr_mutex);
385 static void mic_virtio_init_post(struct mic_vdev *mvdev)
387 struct mic_vqconfig *vqconfig = mic_vq_config(mvdev->dd);
390 for (i = 0; i < mvdev->dd->num_vq; i++) {
391 if (!le64_to_cpu(vqconfig[i].used_address)) {
392 dev_warn(mic_dev(mvdev), "used_address zero??\n");
395 mvdev->mvr[i].vrh.vring.used =
396 (void __force *)mvdev->mdev->aper.va +
397 le64_to_cpu(vqconfig[i].used_address);
400 mvdev->dc->used_address_updated = 0;
402 dev_dbg(mic_dev(mvdev), "%s: device type %d LINKUP\n",
403 __func__, mvdev->virtio_id);
406 static inline void mic_virtio_device_reset(struct mic_vdev *mvdev)
410 dev_dbg(mic_dev(mvdev), "%s: status %d device type %d RESET\n",
411 __func__, mvdev->dd->status, mvdev->virtio_id);
413 for (i = 0; i < mvdev->dd->num_vq; i++)
415 * Avoid lockdep false positive. The + 1 is for the mic
416 * mutex which is held in the reset devices code path.
418 mutex_lock_nested(&mvdev->mvr[i].vr_mutex, i + 1);
420 /* 0 status means "reset" */
421 mvdev->dd->status = 0;
422 mvdev->dc->vdev_reset = 0;
423 mvdev->dc->host_ack = 1;
425 for (i = 0; i < mvdev->dd->num_vq; i++) {
426 struct vringh *vrh = &mvdev->mvr[i].vrh;
427 mvdev->mvr[i].vring.info->avail_idx = 0;
429 vrh->last_avail_idx = 0;
430 vrh->last_used_idx = 0;
433 for (i = 0; i < mvdev->dd->num_vq; i++)
434 mutex_unlock(&mvdev->mvr[i].vr_mutex);
437 void mic_virtio_reset_devices(struct mic_device *mdev)
439 struct list_head *pos, *tmp;
440 struct mic_vdev *mvdev;
442 dev_dbg(&mdev->pdev->dev, "%s\n", __func__);
444 list_for_each_safe(pos, tmp, &mdev->vdev_list) {
445 mvdev = list_entry(pos, struct mic_vdev, list);
446 mic_virtio_device_reset(mvdev);
447 mvdev->poll_wake = 1;
448 wake_up(&mvdev->waitq);
452 void mic_bh_handler(struct work_struct *work)
454 struct mic_vdev *mvdev = container_of(work, struct mic_vdev,
457 if (mvdev->dc->used_address_updated)
458 mic_virtio_init_post(mvdev);
460 if (mvdev->dc->vdev_reset)
461 mic_virtio_device_reset(mvdev);
463 mvdev->poll_wake = 1;
464 wake_up(&mvdev->waitq);
467 static irqreturn_t mic_virtio_intr_handler(int irq, void *data)
469 struct mic_vdev *mvdev = data;
470 struct mic_device *mdev = mvdev->mdev;
472 mdev->ops->intr_workarounds(mdev);
473 schedule_work(&mvdev->virtio_bh_work);
477 int mic_virtio_config_change(struct mic_vdev *mvdev,
480 DECLARE_WAIT_QUEUE_HEAD_ONSTACK(wake);
481 int ret = 0, retry, i;
482 struct mic_bootparam *bootparam = mvdev->mdev->dp;
483 s8 db = bootparam->h2c_config_db;
485 mutex_lock(&mvdev->mdev->mic_mutex);
486 for (i = 0; i < mvdev->dd->num_vq; i++)
487 mutex_lock_nested(&mvdev->mvr[i].vr_mutex, i + 1);
489 if (db == -1 || mvdev->dd->type == -1) {
494 if (copy_from_user(mic_vq_configspace(mvdev->dd),
495 argp, mvdev->dd->config_len)) {
496 dev_err(mic_dev(mvdev), "%s %d err %d\n",
497 __func__, __LINE__, -EFAULT);
501 mvdev->dc->config_change = MIC_VIRTIO_PARAM_CONFIG_CHANGED;
502 mvdev->mdev->ops->send_intr(mvdev->mdev, db);
504 for (retry = 100; retry--;) {
505 ret = wait_event_timeout(wake,
506 mvdev->dc->guest_ack, msecs_to_jiffies(100));
511 dev_dbg(mic_dev(mvdev),
512 "%s %d retry: %d\n", __func__, __LINE__, retry);
513 mvdev->dc->config_change = 0;
514 mvdev->dc->guest_ack = 0;
516 for (i = 0; i < mvdev->dd->num_vq; i++)
517 mutex_unlock(&mvdev->mvr[i].vr_mutex);
518 mutex_unlock(&mvdev->mdev->mic_mutex);
522 static int mic_copy_dp_entry(struct mic_vdev *mvdev,
525 struct mic_device_desc **devpage)
527 struct mic_device *mdev = mvdev->mdev;
528 struct mic_device_desc dd, *dd_config, *devp;
529 struct mic_vqconfig *vqconfig;
531 bool slot_found = false;
533 if (copy_from_user(&dd, argp, sizeof(dd))) {
534 dev_err(mic_dev(mvdev), "%s %d err %d\n",
535 __func__, __LINE__, -EFAULT);
539 if (mic_aligned_desc_size(&dd) > MIC_MAX_DESC_BLK_SIZE ||
540 dd.num_vq > MIC_MAX_VRINGS) {
541 dev_err(mic_dev(mvdev), "%s %d err %d\n",
542 __func__, __LINE__, -EINVAL);
546 dd_config = kmalloc(mic_desc_size(&dd), GFP_KERNEL);
547 if (dd_config == NULL) {
548 dev_err(mic_dev(mvdev), "%s %d err %d\n",
549 __func__, __LINE__, -ENOMEM);
552 if (copy_from_user(dd_config, argp, mic_desc_size(&dd))) {
554 dev_err(mic_dev(mvdev), "%s %d err %d\n",
555 __func__, __LINE__, ret);
559 vqconfig = mic_vq_config(dd_config);
560 for (i = 0; i < dd.num_vq; i++) {
561 if (le16_to_cpu(vqconfig[i].num) > MIC_MAX_VRING_ENTRIES) {
563 dev_err(mic_dev(mvdev), "%s %d err %d\n",
564 __func__, __LINE__, ret);
569 /* Find the first free device page entry */
570 for (i = sizeof(struct mic_bootparam);
571 i < MIC_DP_SIZE - mic_total_desc_size(dd_config);
572 i += mic_total_desc_size(devp)) {
574 if (devp->type == 0 || devp->type == -1) {
581 dev_err(mic_dev(mvdev), "%s %d err %d\n",
582 __func__, __LINE__, ret);
586 * Save off the type before doing the memcpy. Type will be set in the
587 * end after completing all initialization for the new device.
589 *type = dd_config->type;
591 memcpy(devp, dd_config, mic_desc_size(dd_config));
599 static void mic_init_device_ctrl(struct mic_vdev *mvdev,
600 struct mic_device_desc *devpage)
602 struct mic_device_ctrl *dc;
604 dc = (void *)devpage + mic_aligned_desc_size(devpage);
606 dc->config_change = 0;
610 dc->used_address_updated = 0;
611 dc->c2h_vdev_db = -1;
612 dc->h2c_vdev_db = -1;
616 int mic_virtio_add_device(struct mic_vdev *mvdev,
619 struct mic_device *mdev = mvdev->mdev;
620 struct mic_device_desc *dd = NULL;
621 struct mic_vqconfig *vqconfig;
622 int vr_size, i, j, ret;
626 struct mic_bootparam *bootparam = mdev->dp;
630 mutex_lock(&mdev->mic_mutex);
632 ret = mic_copy_dp_entry(mvdev, argp, &type, &dd);
634 mutex_unlock(&mdev->mic_mutex);
638 mic_init_device_ctrl(mvdev, dd);
641 mvdev->virtio_id = type;
642 vqconfig = mic_vq_config(dd);
643 INIT_WORK(&mvdev->virtio_bh_work, mic_bh_handler);
645 for (i = 0; i < dd->num_vq; i++) {
646 struct mic_vringh *mvr = &mvdev->mvr[i];
647 struct mic_vring *vr = &mvdev->mvr[i].vring;
648 num = le16_to_cpu(vqconfig[i].num);
649 mutex_init(&mvr->vr_mutex);
650 vr_size = PAGE_ALIGN(vring_size(num, MIC_VIRTIO_RING_ALIGN) +
651 sizeof(struct _mic_vring_info));
653 __get_free_pages(GFP_KERNEL | __GFP_ZERO,
657 dev_err(mic_dev(mvdev), "%s %d err %d\n",
658 __func__, __LINE__, ret);
662 vr->info = vr->va + vring_size(num, MIC_VIRTIO_RING_ALIGN);
663 vr->info->magic = cpu_to_le32(MIC_MAGIC + mvdev->virtio_id + i);
664 vr_addr = mic_map_single(mdev, vr->va, vr_size);
665 if (mic_map_error(vr_addr)) {
666 free_pages((unsigned long)vr->va, get_order(vr_size));
668 dev_err(mic_dev(mvdev), "%s %d err %d\n",
669 __func__, __LINE__, ret);
672 vqconfig[i].address = cpu_to_le64(vr_addr);
674 vring_init(&vr->vr, num, vr->va, MIC_VIRTIO_RING_ALIGN);
675 ret = vringh_init_kern(&mvr->vrh,
676 *(u32 *)mic_vq_features(mvdev->dd), num, false,
677 vr->vr.desc, vr->vr.avail, vr->vr.used);
679 dev_err(mic_dev(mvdev), "%s %d err %d\n",
680 __func__, __LINE__, ret);
683 vringh_kiov_init(&mvr->riov, NULL, 0);
684 vringh_kiov_init(&mvr->wiov, NULL, 0);
685 mvr->head = USHRT_MAX;
687 mvr->vrh.notify = mic_notify;
688 dev_dbg(&mdev->pdev->dev,
689 "%s %d index %d va %p info %p vr_size 0x%x\n",
690 __func__, __LINE__, i, vr->va, vr->info, vr_size);
691 mvr->buf = (void *)__get_free_pages(GFP_KERNEL,
692 get_order(MIC_INT_DMA_BUF_SIZE));
693 mvr->buf_da = mic_map_single(mvdev->mdev, mvr->buf,
694 MIC_INT_DMA_BUF_SIZE);
697 snprintf(irqname, sizeof(irqname), "mic%dvirtio%d", mdev->id,
699 mvdev->virtio_db = mic_next_db(mdev);
700 mvdev->virtio_cookie = mic_request_threaded_irq(mdev,
701 mic_virtio_intr_handler,
702 NULL, irqname, mvdev,
703 mvdev->virtio_db, MIC_INTR_DB);
704 if (IS_ERR(mvdev->virtio_cookie)) {
705 ret = PTR_ERR(mvdev->virtio_cookie);
706 dev_dbg(&mdev->pdev->dev, "request irq failed\n");
710 mvdev->dc->c2h_vdev_db = mvdev->virtio_db;
712 list_add_tail(&mvdev->list, &mdev->vdev_list);
714 * Order the type update with previous stores. This write barrier
715 * is paired with the corresponding read barrier before the uncached
716 * system memory read of the type, on the card while scanning the
722 dev_dbg(&mdev->pdev->dev, "Added virtio device id %d\n", dd->type);
724 db = bootparam->h2c_config_db;
726 mdev->ops->send_intr(mdev, db);
727 mutex_unlock(&mdev->mic_mutex);
730 vqconfig = mic_vq_config(dd);
731 for (j = 0; j < i; j++) {
732 struct mic_vringh *mvr = &mvdev->mvr[j];
733 mic_unmap_single(mdev, le64_to_cpu(vqconfig[j].address),
735 free_pages((unsigned long)mvr->vring.va,
736 get_order(mvr->vring.len));
738 mutex_unlock(&mdev->mic_mutex);
742 void mic_virtio_del_device(struct mic_vdev *mvdev)
744 struct list_head *pos, *tmp;
745 struct mic_vdev *tmp_mvdev;
746 struct mic_device *mdev = mvdev->mdev;
747 DECLARE_WAIT_QUEUE_HEAD_ONSTACK(wake);
749 struct mic_vqconfig *vqconfig;
750 struct mic_bootparam *bootparam = mdev->dp;
753 mutex_lock(&mdev->mic_mutex);
754 db = bootparam->h2c_config_db;
756 goto skip_hot_remove;
757 dev_dbg(&mdev->pdev->dev,
758 "Requesting hot remove id %d\n", mvdev->virtio_id);
759 mvdev->dc->config_change = MIC_VIRTIO_PARAM_DEV_REMOVE;
760 mdev->ops->send_intr(mdev, db);
761 for (retry = 100; retry--;) {
762 ret = wait_event_timeout(wake,
763 mvdev->dc->guest_ack, msecs_to_jiffies(100));
767 dev_dbg(&mdev->pdev->dev,
768 "Device id %d config_change %d guest_ack %d retry %d\n",
769 mvdev->virtio_id, mvdev->dc->config_change,
770 mvdev->dc->guest_ack, retry);
771 mvdev->dc->config_change = 0;
772 mvdev->dc->guest_ack = 0;
774 mic_free_irq(mdev, mvdev->virtio_cookie, mvdev);
775 flush_work(&mvdev->virtio_bh_work);
776 vqconfig = mic_vq_config(mvdev->dd);
777 for (i = 0; i < mvdev->dd->num_vq; i++) {
778 struct mic_vringh *mvr = &mvdev->mvr[i];
780 mic_unmap_single(mvdev->mdev, mvr->buf_da,
781 MIC_INT_DMA_BUF_SIZE);
782 free_pages((unsigned long)mvr->buf,
783 get_order(MIC_INT_DMA_BUF_SIZE));
784 vringh_kiov_cleanup(&mvr->riov);
785 vringh_kiov_cleanup(&mvr->wiov);
786 mic_unmap_single(mdev, le64_to_cpu(vqconfig[i].address),
788 free_pages((unsigned long)mvr->vring.va,
789 get_order(mvr->vring.len));
792 list_for_each_safe(pos, tmp, &mdev->vdev_list) {
793 tmp_mvdev = list_entry(pos, struct mic_vdev, list);
794 if (tmp_mvdev == mvdev) {
796 dev_dbg(&mdev->pdev->dev,
797 "Removing virtio device id %d\n",
803 * Order the type update with previous stores. This write barrier
804 * is paired with the corresponding read barrier before the uncached
805 * system memory read of the type, on the card while scanning the
809 mvdev->dd->type = -1;
810 mutex_unlock(&mdev->mic_mutex);