kernel/drivers/char/xillybus/xillybus_core.c

   1 /*
   2  * linux/drivers/misc/xillybus_core.c
   3  *
   4  * Copyright 2011 Xillybus Ltd, http://xillybus.com
   5  *
   6  * Driver for the Xillybus FPGA/host framework.
   7  *
   8  * This driver interfaces with a special IP core in an FPGA, setting up
   9  * a pipe between a hardware FIFO in the programmable logic and a device
  10  * file in the host. The number of such pipes and their attributes are
  11  * set up on the logic. This driver detects these automatically and
  12  * creates the device files accordingly.
  13  *
  14  * This program is free software; you can redistribute it and/or modify
  15  * it under the smems of the GNU General Public License as published by
  16  * the Free Software Foundation; version 2 of the License.
  17  */
  18
  19 #include <linux/list.h>
  20 #include <linux/device.h>
  21 #include <linux/module.h>
  22 #include <linux/io.h>
  23 #include <linux/dma-mapping.h>
  24 #include <linux/interrupt.h>
  25 #include <linux/sched.h>
  26 #include <linux/fs.h>
  27 #include <linux/cdev.h>
  28 #include <linux/spinlock.h>
  29 #include <linux/mutex.h>
  30 #include <linux/crc32.h>
  31 #include <linux/poll.h>
  32 #include <linux/delay.h>
  33 #include <linux/slab.h>
  34 #include <linux/workqueue.h>
  35 #include "xillybus.h"
  36
  37 MODULE_DESCRIPTION("Xillybus core functions");
  38 MODULE_AUTHOR("Eli Billauer, Xillybus Ltd.");
  39 MODULE_VERSION("1.07");
  40 MODULE_ALIAS("xillybus_core");
  41 MODULE_LICENSE("GPL v2");
  42
  43 /* General timeout is 100 ms, rx timeout is 10 ms */
  44 #define XILLY_RX_TIMEOUT (10*HZ/1000)
  45 #define XILLY_TIMEOUT (100*HZ/1000)
  46
  47 #define fpga_msg_ctrl_reg              0x0008
  48 #define fpga_dma_control_reg           0x0020
  49 #define fpga_dma_bufno_reg             0x0024
  50 #define fpga_dma_bufaddr_lowaddr_reg   0x0028
  51 #define fpga_dma_bufaddr_highaddr_reg  0x002c
  52 #define fpga_buf_ctrl_reg              0x0030
  53 #define fpga_buf_offset_reg            0x0034
  54 #define fpga_endian_reg                0x0040
  55
  56 #define XILLYMSG_OPCODE_RELEASEBUF 1
  57 #define XILLYMSG_OPCODE_QUIESCEACK 2
  58 #define XILLYMSG_OPCODE_FIFOEOF 3
  59 #define XILLYMSG_OPCODE_FATAL_ERROR 4
  60 #define XILLYMSG_OPCODE_NONEMPTY 5
  61
  62 static const char xillyname[] = "xillybus";
  63
  64 static struct class *xillybus_class;
  65
  66 /*
  67  * ep_list_lock is the last lock to be taken; No other lock requests are
  68  * allowed while holding it. It merely protects list_of_endpoints, and not
  69  * the endpoints listed in it.
  70  */
  71
  72 static LIST_HEAD(list_of_endpoints);
  73 static struct mutex ep_list_lock;
  74 static struct workqueue_struct *xillybus_wq;
  75
  76 /*
  77  * Locking scheme: Mutexes protect invocations of character device methods.
  78  * If both locks are taken, wr_mutex is taken first, rd_mutex second.
  79  *
  80  * wr_spinlock protects wr_*_buf_idx, wr_empty, wr_sleepy, wr_ready and the
  81  * buffers' end_offset fields against changes made by IRQ handler (and in
  82  * theory, other file request handlers, but the mutex handles that). Nothing
  83  * else.
  84  * They are held for short direct memory manipulations. Needless to say,
  85  * no mutex locking is allowed when a spinlock is held.
  86  *
  87  * rd_spinlock does the same with rd_*_buf_idx, rd_empty and end_offset.
  88  *
  89  * register_mutex is endpoint-specific, and is held when non-atomic
  90  * register operations are performed. wr_mutex and rd_mutex may be
  91  * held when register_mutex is taken, but none of the spinlocks. Note that
  92  * register_mutex doesn't protect against sporadic buf_ctrl_reg writes
  93  * which are unrelated to buf_offset_reg, since they are harmless.
  94  *
  95  * Blocking on the wait queues is allowed with mutexes held, but not with
  96  * spinlocks.
  97  *
  98  * Only interruptible blocking is allowed on mutexes and wait queues.
  99  *
 100  * All in all, the locking order goes (with skips allowed, of course):
 101  * wr_mutex -> rd_mutex -> register_mutex -> wr_spinlock -> rd_spinlock
 102  */
 103
 104 static void malformed_message(struct xilly_endpoint *endpoint, u32 *buf)
 105 {
 106         int opcode;
 107         int msg_channel, msg_bufno, msg_data, msg_dir;
 108
 109         opcode = (buf[0] >> 24) & 0xff;
 110         msg_dir = buf[0] & 1;
 111         msg_channel = (buf[0] >> 1) & 0x7ff;
 112         msg_bufno = (buf[0] >> 12) & 0x3ff;
 113         msg_data = buf[1] & 0xfffffff;
 114
 115         dev_warn(endpoint->dev,
 116                  "Malformed message (skipping): opcode=%d, channel=%03x, dir=%d, bufno=%03x, data=%07x\n",
 117                  opcode, msg_channel, msg_dir, msg_bufno, msg_data);
 118 }
 119
 120 /*
 121  * xillybus_isr assumes the interrupt is allocated exclusively to it,
 122  * which is the natural case MSI and several other hardware-oriented
 123  * interrupts. Sharing is not allowed.
 124  */
 125
 126 irqreturn_t xillybus_isr(int irq, void *data)
 127 {
 128         struct xilly_endpoint *ep = data;
 129         u32 *buf;
 130         unsigned int buf_size;
 131         int i;
 132         int opcode;
 133         unsigned int msg_channel, msg_bufno, msg_data, msg_dir;
 134         struct xilly_channel *channel;
 135
 136         buf = ep->msgbuf_addr;
 137         buf_size = ep->msg_buf_size/sizeof(u32);
 138
 139         ep->ephw->hw_sync_sgl_for_cpu(ep,
 140                                       ep->msgbuf_dma_addr,
 141                                       ep->msg_buf_size,
 142                                       DMA_FROM_DEVICE);
 143
 144         for (i = 0; i < buf_size; i += 2) {
 145                 if (((buf[i+1] >> 28) & 0xf) != ep->msg_counter) {
 146                         malformed_message(ep, &buf[i]);
 147                         dev_warn(ep->dev,
 148                                  "Sending a NACK on counter %x (instead of %x) on entry %d\n",
 149                                  ((buf[i+1] >> 28) & 0xf),
 150                                  ep->msg_counter,
 151                                  i/2);
 152
 153                         if (++ep->failed_messages > 10) {
 154                                 dev_err(ep->dev,
 155                                         "Lost sync with interrupt messages. Stopping.\n");
 156                         } else {
 157                                 ep->ephw->hw_sync_sgl_for_device(
 158                                         ep,
 159                                         ep->msgbuf_dma_addr,
 160                                         ep->msg_buf_size,
 161                                         DMA_FROM_DEVICE);
 162
 163                                 iowrite32(0x01,  /* Message NACK */
 164                                           ep->registers + fpga_msg_ctrl_reg);
 165                         }
 166                         return IRQ_HANDLED;
 167                 } else if (buf[i] & (1 << 22)) /* Last message */
 168                         break;
 169         }
 170
 171         if (i >= buf_size) {
 172                 dev_err(ep->dev, "Bad interrupt message. Stopping.\n");
 173                 return IRQ_HANDLED;
 174         }
 175
 176         buf_size = i + 2;
 177
 178         for (i = 0; i < buf_size; i += 2) { /* Scan through messages */
 179                 opcode = (buf[i] >> 24) & 0xff;
 180
 181                 msg_dir = buf[i] & 1;
 182                 msg_channel = (buf[i] >> 1) & 0x7ff;
 183                 msg_bufno = (buf[i] >> 12) & 0x3ff;
 184                 msg_data = buf[i+1] & 0xfffffff;
 185
 186                 switch (opcode) {
 187                 case XILLYMSG_OPCODE_RELEASEBUF:
 188                         if ((msg_channel > ep->num_channels) ||
 189                             (msg_channel == 0)) {
 190                                 malformed_message(ep, &buf[i]);
 191                                 break;
 192                         }
 193
 194                         channel = ep->channels[msg_channel];
 195
 196                         if (msg_dir) { /* Write channel */
 197                                 if (msg_bufno >= channel->num_wr_buffers) {
 198                                         malformed_message(ep, &buf[i]);
 199                                         break;
 200                                 }
 201                                 spin_lock(&channel->wr_spinlock);
 202                                 channel->wr_buffers[msg_bufno]->end_offset =
 203                                         msg_data;
 204                                 channel->wr_fpga_buf_idx = msg_bufno;
 205                                 channel->wr_empty = 0;
 206                                 channel->wr_sleepy = 0;
 207                                 spin_unlock(&channel->wr_spinlock);
 208
 209                                 wake_up_interruptible(&channel->wr_wait);
 210
 211                         } else {
 212                                 /* Read channel */
 213
 214                                 if (msg_bufno >= channel->num_rd_buffers) {
 215                                         malformed_message(ep, &buf[i]);
 216                                         break;
 217                                 }
 218
 219                                 spin_lock(&channel->rd_spinlock);
 220                                 channel->rd_fpga_buf_idx = msg_bufno;
 221                                 channel->rd_full = 0;
 222                                 spin_unlock(&channel->rd_spinlock);
 223
 224                                 wake_up_interruptible(&channel->rd_wait);
 225                                 if (!channel->rd_synchronous)
 226                                         queue_delayed_work(
 227                                                 xillybus_wq,
 228                                                 &channel->rd_workitem,
 229                                                 XILLY_RX_TIMEOUT);
 230                         }
 231
 232                         break;
 233                 case XILLYMSG_OPCODE_NONEMPTY:
 234                         if ((msg_channel > ep->num_channels) ||
 235                             (msg_channel == 0) || (!msg_dir) ||
 236                             !ep->channels[msg_channel]->wr_supports_nonempty) {
 237                                 malformed_message(ep, &buf[i]);
 238                                 break;
 239                         }
 240
 241                         channel = ep->channels[msg_channel];
 242
 243                         if (msg_bufno >= channel->num_wr_buffers) {
 244                                 malformed_message(ep, &buf[i]);
 245                                 break;
 246                         }
 247                         spin_lock(&channel->wr_spinlock);
 248                         if (msg_bufno == channel->wr_host_buf_idx)
 249                                 channel->wr_ready = 1;
 250                         spin_unlock(&channel->wr_spinlock);
 251
 252                         wake_up_interruptible(&channel->wr_ready_wait);
 253
 254                         break;
 255                 case XILLYMSG_OPCODE_QUIESCEACK:
 256                         ep->idtlen = msg_data;
 257                         wake_up_interruptible(&ep->ep_wait);
 258
 259                         break;
 260                 case XILLYMSG_OPCODE_FIFOEOF:
 261                         if ((msg_channel > ep->num_channels) ||
 262                             (msg_channel == 0) || (!msg_dir) ||
 263                             !ep->channels[msg_channel]->num_wr_buffers) {
 264                                 malformed_message(ep, &buf[i]);
 265                                 break;
 266                         }
 267                         channel = ep->channels[msg_channel];
 268                         spin_lock(&channel->wr_spinlock);
 269                         channel->wr_eof = msg_bufno;
 270                         channel->wr_sleepy = 0;
 271
 272                         channel->wr_hangup = channel->wr_empty &&
 273                                 (channel->wr_host_buf_idx == msg_bufno);
 274
 275                         spin_unlock(&channel->wr_spinlock);
 276
 277                         wake_up_interruptible(&channel->wr_wait);
 278
 279                         break;
 280                 case XILLYMSG_OPCODE_FATAL_ERROR:
 281                         ep->fatal_error = 1;
 282                         wake_up_interruptible(&ep->ep_wait); /* For select() */
 283                         dev_err(ep->dev,
 284                                 "FPGA reported a fatal error. This means that the low-level communication with the device has failed. This hardware problem is most likely unrelated to Xillybus (neither kernel module nor FPGA core), but reports are still welcome. All I/O is aborted.\n");
 285                         break;
 286                 default:
 287                         malformed_message(ep, &buf[i]);
 288                         break;
 289                 }
 290         }
 291
 292         ep->ephw->hw_sync_sgl_for_device(ep,
 293                                          ep->msgbuf_dma_addr,
 294                                          ep->msg_buf_size,
 295                                          DMA_FROM_DEVICE);
 296
 297         ep->msg_counter = (ep->msg_counter + 1) & 0xf;
 298         ep->failed_messages = 0;
 299         iowrite32(0x03, ep->registers + fpga_msg_ctrl_reg); /* Message ACK */
 300
 301         return IRQ_HANDLED;
 302 }
 303 EXPORT_SYMBOL(xillybus_isr);
 304
 305 /*
 306  * A few trivial memory management functions.
 307  * NOTE: These functions are used only on probe and remove, and therefore
 308  * no locks are applied!
 309  */
 310
 311 static void xillybus_autoflush(struct work_struct *work);
 312
 313 struct xilly_alloc_state {
 314         void *salami;
 315         int left_of_salami;
 316         int nbuffer;
 317         enum dma_data_direction direction;
 318         u32 regdirection;
 319 };
 320
 321 static int xilly_get_dma_buffers(struct xilly_endpoint *ep,
 322                                  struct xilly_alloc_state *s,
 323                                  struct xilly_buffer **buffers,
 324                                  int bufnum, int bytebufsize)
 325 {
 326         int i, rc;
 327         dma_addr_t dma_addr;
 328         struct device *dev = ep->dev;
 329         struct xilly_buffer *this_buffer = NULL; /* Init to silence warning */
 330
 331         if (buffers) { /* Not the message buffer */
 332                 this_buffer = devm_kcalloc(dev, bufnum,
 333                                            sizeof(struct xilly_buffer),
 334                                            GFP_KERNEL);
 335                 if (!this_buffer)
 336                         return -ENOMEM;
 337         }
 338
 339         for (i = 0; i < bufnum; i++) {
 340                 /*
 341                  * Buffers are expected in descending size order, so there
 342                  * is either enough space for this buffer or none at all.
 343                  */
 344
 345                 if ((s->left_of_salami < bytebufsize) &&
 346                     (s->left_of_salami > 0)) {
 347                         dev_err(ep->dev,
 348                                 "Corrupt buffer allocation in IDT. Aborting.\n");
 349                         return -ENODEV;
 350                 }
 351
 352                 if (s->left_of_salami == 0) {
 353                         int allocorder, allocsize;
 354
 355                         allocsize = PAGE_SIZE;
 356                         allocorder = 0;
 357                         while (bytebufsize > allocsize) {
 358                                 allocsize *= 2;
 359                                 allocorder++;
 360                         }
 361
 362                         s->salami = (void *) devm_get_free_pages(
 363                                 dev,
 364                                 GFP_KERNEL | __GFP_DMA32 | __GFP_ZERO,
 365                                 allocorder);
 366                         if (!s->salami)
 367                                 return -ENOMEM;
 368
 369                         s->left_of_salami = allocsize;
 370                 }
 371
 372                 rc = ep->ephw->map_single(ep, s->salami,
 373                                           bytebufsize, s->direction,
 374                                           &dma_addr);
 375                 if (rc)
 376                         return rc;
 377
 378                 iowrite32((u32) (dma_addr & 0xffffffff),
 379                           ep->registers + fpga_dma_bufaddr_lowaddr_reg);
 380                 iowrite32(((u32) ((((u64) dma_addr) >> 32) & 0xffffffff)),
 381                           ep->registers + fpga_dma_bufaddr_highaddr_reg);
 382
 383                 if (buffers) { /* Not the message buffer */
 384                         this_buffer->addr = s->salami;
 385                         this_buffer->dma_addr = dma_addr;
 386                         buffers[i] = this_buffer++;
 387
 388                         iowrite32(s->regdirection | s->nbuffer++,
 389                                   ep->registers + fpga_dma_bufno_reg);
 390                 } else {
 391                         ep->msgbuf_addr = s->salami;
 392                         ep->msgbuf_dma_addr = dma_addr;
 393                         ep->msg_buf_size = bytebufsize;
 394
 395                         iowrite32(s->regdirection,
 396                                   ep->registers + fpga_dma_bufno_reg);
 397                 }
 398
 399                 s->left_of_salami -= bytebufsize;
 400                 s->salami += bytebufsize;
 401         }
 402         return 0;
 403 }
 404
 405 static int xilly_setupchannels(struct xilly_endpoint *ep,
 406                                unsigned char *chandesc,
 407                                int entries)
 408 {
 409         struct device *dev = ep->dev;
 410         int i, entry, rc;
 411         struct xilly_channel *channel;
 412         int channelnum, bufnum, bufsize, format, is_writebuf;
 413         int bytebufsize;
 414         int synchronous, allowpartial, exclusive_open, seekable;
 415         int supports_nonempty;
 416         int msg_buf_done = 0;
 417
 418         struct xilly_alloc_state rd_alloc = {
 419                 .salami = NULL,
 420                 .left_of_salami = 0,
 421                 .nbuffer = 1,
 422                 .direction = DMA_TO_DEVICE,
 423                 .regdirection = 0,
 424         };
 425
 426         struct xilly_alloc_state wr_alloc = {
 427                 .salami = NULL,
 428                 .left_of_salami = 0,
 429                 .nbuffer = 1,
 430                 .direction = DMA_FROM_DEVICE,
 431                 .regdirection = 0x80000000,
 432         };
 433
 434         channel = devm_kcalloc(dev, ep->num_channels,
 435                                sizeof(struct xilly_channel), GFP_KERNEL);
 436         if (!channel)
 437                 return -ENOMEM;
 438
 439         ep->channels = devm_kcalloc(dev, ep->num_channels + 1,
 440                                     sizeof(struct xilly_channel *),
 441                                     GFP_KERNEL);
 442         if (!ep->channels)
 443                 return -ENOMEM;
 444
 445         ep->channels[0] = NULL; /* Channel 0 is message buf. */
 446
 447         /* Initialize all channels with defaults */
 448
 449         for (i = 1; i <= ep->num_channels; i++) {
 450                 channel->wr_buffers = NULL;
 451                 channel->rd_buffers = NULL;
 452                 channel->num_wr_buffers = 0;
 453                 channel->num_rd_buffers = 0;
 454                 channel->wr_fpga_buf_idx = -1;
 455                 channel->wr_host_buf_idx = 0;
 456                 channel->wr_host_buf_pos = 0;
 457                 channel->wr_empty = 1;
 458                 channel->wr_ready = 0;
 459                 channel->wr_sleepy = 1;
 460                 channel->rd_fpga_buf_idx = 0;
 461                 channel->rd_host_buf_idx = 0;
 462                 channel->rd_host_buf_pos = 0;
 463                 channel->rd_full = 0;
 464                 channel->wr_ref_count = 0;
 465                 channel->rd_ref_count = 0;
 466
 467                 spin_lock_init(&channel->wr_spinlock);
 468                 spin_lock_init(&channel->rd_spinlock);
 469                 mutex_init(&channel->wr_mutex);
 470                 mutex_init(&channel->rd_mutex);
 471                 init_waitqueue_head(&channel->rd_wait);
 472                 init_waitqueue_head(&channel->wr_wait);
 473                 init_waitqueue_head(&channel->wr_ready_wait);
 474
 475                 INIT_DELAYED_WORK(&channel->rd_workitem, xillybus_autoflush);
 476
 477                 channel->endpoint = ep;
 478                 channel->chan_num = i;
 479
 480                 channel->log2_element_size = 0;
 481
 482                 ep->channels[i] = channel++;
 483         }
 484
 485         for (entry = 0; entry < entries; entry++, chandesc += 4) {
 486                 struct xilly_buffer **buffers = NULL;
 487
 488                 is_writebuf = chandesc[0] & 0x01;
 489                 channelnum = (chandesc[0] >> 1) | ((chandesc[1] & 0x0f) << 7);
 490                 format = (chandesc[1] >> 4) & 0x03;
 491                 allowpartial = (chandesc[1] >> 6) & 0x01;
 492                 synchronous = (chandesc[1] >> 7) & 0x01;
 493                 bufsize = 1 << (chandesc[2] & 0x1f);
 494                 bufnum = 1 << (chandesc[3] & 0x0f);
 495                 exclusive_open = (chandesc[2] >> 7) & 0x01;
 496                 seekable = (chandesc[2] >> 6) & 0x01;
 497                 supports_nonempty = (chandesc[2] >> 5) & 0x01;
 498
 499                 if ((channelnum > ep->num_channels) ||
 500                     ((channelnum == 0) && !is_writebuf)) {
 501                         dev_err(ep->dev,
 502                                 "IDT requests channel out of range. Aborting.\n");
 503                         return -ENODEV;
 504                 }
 505
 506                 channel = ep->channels[channelnum]; /* NULL for msg channel */
 507
 508                 if (!is_writebuf || channelnum > 0) {
 509                         channel->log2_element_size = ((format > 2) ?
 510                                                       2 : format);
 511
 512                         bytebufsize = channel->rd_buf_size = bufsize *
 513                                 (1 << channel->log2_element_size);
 514
 515                         buffers = devm_kcalloc(dev, bufnum,
 516                                                sizeof(struct xilly_buffer *),
 517                                                GFP_KERNEL);
 518                         if (!buffers)
 519                                 return -ENOMEM;
 520                 } else {
 521                         bytebufsize = bufsize << 2;
 522                 }
 523
 524                 if (!is_writebuf) {
 525                         channel->num_rd_buffers = bufnum;
 526                         channel->rd_allow_partial = allowpartial;
 527                         channel->rd_synchronous = synchronous;
 528                         channel->rd_exclusive_open = exclusive_open;
 529                         channel->seekable = seekable;
 530
 531                         channel->rd_buffers = buffers;
 532                         rc = xilly_get_dma_buffers(ep, &rd_alloc, buffers,
 533                                                    bufnum, bytebufsize);
 534                 } else if (channelnum > 0) {
 535                         channel->num_wr_buffers = bufnum;
 536
 537                         channel->seekable = seekable;
 538                         channel->wr_supports_nonempty = supports_nonempty;
 539
 540                         channel->wr_allow_partial = allowpartial;
 541                         channel->wr_synchronous = synchronous;
 542                         channel->wr_exclusive_open = exclusive_open;
 543
 544                         channel->wr_buffers = buffers;
 545                         rc = xilly_get_dma_buffers(ep, &wr_alloc, buffers,
 546                                                    bufnum, bytebufsize);
 547                 } else {
 548                         rc = xilly_get_dma_buffers(ep, &wr_alloc, NULL,
 549                                                    bufnum, bytebufsize);
 550                         msg_buf_done++;
 551                 }
 552
 553                 if (rc)
 554                         return -ENOMEM;
 555         }
 556
 557         if (!msg_buf_done) {
 558                 dev_err(ep->dev,
 559                         "Corrupt IDT: No message buffer. Aborting.\n");
 560                 return -ENODEV;
 561         }
 562         return 0;
 563 }
 564
 565 static int xilly_scan_idt(struct xilly_endpoint *endpoint,
 566                           struct xilly_idt_handle *idt_handle)
 567 {
 568         int count = 0;
 569         unsigned char *idt = endpoint->channels[1]->wr_buffers[0]->addr;
 570         unsigned char *end_of_idt = idt + endpoint->idtlen - 4;
 571         unsigned char *scan;
 572         int len;
 573
 574         scan = idt;
 575         idt_handle->idt = idt;
 576
 577         scan++; /* Skip version number */
 578
 579         while ((scan <= end_of_idt) && *scan) {
 580                 while ((scan <= end_of_idt) && *scan++)
 581                         /* Do nothing, just scan thru string */;
 582                 count++;
 583         }
 584
 585         scan++;
 586
 587         if (scan > end_of_idt) {
 588                 dev_err(endpoint->dev,
 589                         "IDT device name list overflow. Aborting.\n");
 590                 return -ENODEV;
 591         }
 592         idt_handle->chandesc = scan;
 593
 594         len = endpoint->idtlen - (3 + ((int) (scan - idt)));
 595
 596         if (len & 0x03) {
 597                 dev_err(endpoint->dev,
 598                         "Corrupt IDT device name list. Aborting.\n");
 599                 return -ENODEV;
 600         }
 601
 602         idt_handle->entries = len >> 2;
 603         endpoint->num_channels = count;
 604
 605         return 0;
 606 }
 607
 608 static int xilly_obtain_idt(struct xilly_endpoint *endpoint)
 609 {
 610         struct xilly_channel *channel;
 611         unsigned char *version;
 612         long t;
 613
 614         channel = endpoint->channels[1]; /* This should be generated ad-hoc */
 615
 616         channel->wr_sleepy = 1;
 617
 618         iowrite32(1 |
 619                   (3 << 24), /* Opcode 3 for channel 0 = Send IDT */
 620                   endpoint->registers + fpga_buf_ctrl_reg);
 621
 622         t = wait_event_interruptible_timeout(channel->wr_wait,
 623                                              (!channel->wr_sleepy),
 624                                              XILLY_TIMEOUT);
 625
 626         if (t <= 0) {
 627                 dev_err(endpoint->dev, "Failed to obtain IDT. Aborting.\n");
 628
 629                 if (endpoint->fatal_error)
 630                         return -EIO;
 631
 632                 return -ENODEV;
 633         }
 634
 635         endpoint->ephw->hw_sync_sgl_for_cpu(
 636                 channel->endpoint,
 637                 channel->wr_buffers[0]->dma_addr,
 638                 channel->wr_buf_size,
 639                 DMA_FROM_DEVICE);
 640
 641         if (channel->wr_buffers[0]->end_offset != endpoint->idtlen) {
 642                 dev_err(endpoint->dev,
 643                         "IDT length mismatch (%d != %d). Aborting.\n",
 644                         channel->wr_buffers[0]->end_offset, endpoint->idtlen);
 645                 return -ENODEV;
 646         }
 647
 648         if (crc32_le(~0, channel->wr_buffers[0]->addr,
 649                      endpoint->idtlen+1) != 0) {
 650                 dev_err(endpoint->dev, "IDT failed CRC check. Aborting.\n");
 651                 return -ENODEV;
 652         }
 653
 654         version = channel->wr_buffers[0]->addr;
 655
 656         /* Check version number. Accept anything below 0x82 for now. */
 657         if (*version > 0x82) {
 658                 dev_err(endpoint->dev,
 659                         "No support for IDT version 0x%02x. Maybe the xillybus driver needs an upgarde. Aborting.\n",
 660                         *version);
 661                 return -ENODEV;
 662         }
 663
 664         return 0;
 665 }
 666
 667 static ssize_t xillybus_read(struct file *filp, char __user *userbuf,
 668                              size_t count, loff_t *f_pos)
 669 {
 670         ssize_t rc;
 671         unsigned long flags;
 672         int bytes_done = 0;
 673         int no_time_left = 0;
 674         long deadline, left_to_sleep;
 675         struct xilly_channel *channel = filp->private_data;
 676
 677         int empty, reached_eof, exhausted, ready;
 678         /* Initializations are there only to silence warnings */
 679
 680         int howmany = 0, bufpos = 0, bufidx = 0, bufferdone = 0;
 681         int waiting_bufidx;
 682
 683         if (channel->endpoint->fatal_error)
 684                 return -EIO;
 685
 686         deadline = jiffies + 1 + XILLY_RX_TIMEOUT;
 687
 688         rc = mutex_lock_interruptible(&channel->wr_mutex);
 689         if (rc)
 690                 return rc;
 691
 692         while (1) { /* Note that we may drop mutex within this loop */
 693                 int bytes_to_do = count - bytes_done;
 694
 695                 spin_lock_irqsave(&channel->wr_spinlock, flags);
 696
 697                 empty = channel->wr_empty;
 698                 ready = !empty || channel->wr_ready;
 699
 700                 if (!empty) {
 701                         bufidx = channel->wr_host_buf_idx;
 702                         bufpos = channel->wr_host_buf_pos;
 703                         howmany = ((channel->wr_buffers[bufidx]->end_offset
 704                                     + 1) << channel->log2_element_size)
 705                                 - bufpos;
 706
 707                         /* Update wr_host_* to its post-operation state */
 708                         if (howmany > bytes_to_do) {
 709                                 bufferdone = 0;
 710
 711                                 howmany = bytes_to_do;
 712                                 channel->wr_host_buf_pos += howmany;
 713                         } else {
 714                                 bufferdone = 1;
 715
 716                                 channel->wr_host_buf_pos = 0;
 717
 718                                 if (bufidx == channel->wr_fpga_buf_idx) {
 719                                         channel->wr_empty = 1;
 720                                         channel->wr_sleepy = 1;
 721                                         channel->wr_ready = 0;
 722                                 }
 723
 724                                 if (bufidx >= (channel->num_wr_buffers - 1))
 725                                         channel->wr_host_buf_idx = 0;
 726                                 else
 727                                         channel->wr_host_buf_idx++;
 728                         }
 729                 }
 730
 731                 /*
 732                  * Marking our situation after the possible changes above,
 733                  * for use after releasing the spinlock.
 734                  *
 735                  * empty = empty before change
 736                  * exhasted = empty after possible change
 737                  */
 738
 739                 reached_eof = channel->wr_empty &&
 740                         (channel->wr_host_buf_idx == channel->wr_eof);
 741                 channel->wr_hangup = reached_eof;
 742                 exhausted = channel->wr_empty;
 743                 waiting_bufidx = channel->wr_host_buf_idx;
 744
 745                 spin_unlock_irqrestore(&channel->wr_spinlock, flags);
 746
 747                 if (!empty) { /* Go on, now without the spinlock */
 748
 749                         if (bufpos == 0) /* Position zero means it's virgin */
 750                                 channel->endpoint->ephw->hw_sync_sgl_for_cpu(
 751                                         channel->endpoint,
 752                                         channel->wr_buffers[bufidx]->dma_addr,
 753                                         channel->wr_buf_size,
 754                                         DMA_FROM_DEVICE);
 755
 756                         if (copy_to_user(
 757                                     userbuf,
 758                                     channel->wr_buffers[bufidx]->addr
 759                                     + bufpos, howmany))
 760                                 rc = -EFAULT;
 761
 762                         userbuf += howmany;
 763                         bytes_done += howmany;
 764
 765                         if (bufferdone) {
 766                                 channel->endpoint->ephw->hw_sync_sgl_for_device(
 767                                         channel->endpoint,
 768                                         channel->wr_buffers[bufidx]->dma_addr,
 769                                         channel->wr_buf_size,
 770                                         DMA_FROM_DEVICE);
 771
 772                                 /*
 773                                  * Tell FPGA the buffer is done with. It's an
 774                                  * atomic operation to the FPGA, so what
 775                                  * happens with other channels doesn't matter,
 776                                  * and the certain channel is protected with
 777                                  * the channel-specific mutex.
 778                                  */
 779
 780                                 iowrite32(1 | (channel->chan_num << 1) |
 781                                           (bufidx << 12),
 782                                           channel->endpoint->registers +
 783                                           fpga_buf_ctrl_reg);
 784                         }
 785
 786                         if (rc) {
 787                                 mutex_unlock(&channel->wr_mutex);
 788                                 return rc;
 789                         }
 790                 }
 791
 792                 /* This includes a zero-count return = EOF */
 793                 if ((bytes_done >= count) || reached_eof)
 794                         break;
 795
 796                 if (!exhausted)
 797                         continue; /* More in RAM buffer(s)? Just go on. */
 798
 799                 if ((bytes_done > 0) &&
 800                     (no_time_left ||
 801                      (channel->wr_synchronous && channel->wr_allow_partial)))
 802                         break;
 803
 804                 /*
 805                  * Nonblocking read: The "ready" flag tells us that the FPGA
 806                  * has data to send. In non-blocking mode, if it isn't on,
 807                  * just return. But if there is, we jump directly to the point
 808                  * where we ask for the FPGA to send all it has, and wait
 809                  * until that data arrives. So in a sense, we *do* block in
 810                  * nonblocking mode, but only for a very short time.
 811                  */
 812
 813                 if (!no_time_left && (filp->f_flags & O_NONBLOCK)) {
 814                         if (bytes_done > 0)
 815                                 break;
 816
 817                         if (ready)
 818                                 goto desperate;
 819
 820                         rc = -EAGAIN;
 821                         break;
 822                 }
 823
 824                 if (!no_time_left || (bytes_done > 0)) {
 825                         /*
 826                          * Note that in case of an element-misaligned read
 827                          * request, offsetlimit will include the last element,
 828                          * which will be partially read from.
 829                          */
 830                         int offsetlimit = ((count - bytes_done) - 1) >>
 831                                 channel->log2_element_size;
 832                         int buf_elements = channel->wr_buf_size >>
 833                                 channel->log2_element_size;
 834
 835                         /*
 836                          * In synchronous mode, always send an offset limit.
 837                          * Just don't send a value too big.
 838                          */
 839
 840                         if (channel->wr_synchronous) {
 841                                 /* Don't request more than one buffer */
 842                                 if (channel->wr_allow_partial &&
 843                                     (offsetlimit >= buf_elements))
 844                                         offsetlimit = buf_elements - 1;
 845
 846                                 /* Don't request more than all buffers */
 847                                 if (!channel->wr_allow_partial &&
 848                                     (offsetlimit >=
 849                                      (buf_elements * channel->num_wr_buffers)))
 850                                         offsetlimit = buf_elements *
 851                                                 channel->num_wr_buffers - 1;
 852                         }
 853
 854                         /*
 855                          * In asynchronous mode, force early flush of a buffer
 856                          * only if that will allow returning a full count. The
 857                          * "offsetlimit < ( ... )" rather than "<=" excludes
 858                          * requesting a full buffer, which would obviously
 859                          * cause a buffer transmission anyhow
 860                          */
 861
 862                         if (channel->wr_synchronous ||
 863                             (offsetlimit < (buf_elements - 1))) {
 864                                 mutex_lock(&channel->endpoint->register_mutex);
 865
 866                                 iowrite32(offsetlimit,
 867                                           channel->endpoint->registers +
 868                                           fpga_buf_offset_reg);
 869
 870                                 iowrite32(1 | (channel->chan_num << 1) |
 871                                           (2 << 24) |  /* 2 = offset limit */
 872                                           (waiting_bufidx << 12),
 873                                           channel->endpoint->registers +
 874                                           fpga_buf_ctrl_reg);
 875
 876                                 mutex_unlock(&channel->endpoint->
 877                                              register_mutex);
 878                         }
 879                 }
 880
 881                 /*
 882                  * If partial completion is disallowed, there is no point in
 883                  * timeout sleeping. Neither if no_time_left is set and
 884                  * there's no data.
 885                  */
 886
 887                 if (!channel->wr_allow_partial ||
 888                     (no_time_left && (bytes_done == 0))) {
 889                         /*
 890                          * This do-loop will run more than once if another
 891                          * thread reasserted wr_sleepy before we got the mutex
 892                          * back, so we try again.
 893                          */
 894
 895                         do {
 896                                 mutex_unlock(&channel->wr_mutex);
 897
 898                                 if (wait_event_interruptible(
 899                                             channel->wr_wait,
 900                                             (!channel->wr_sleepy)))
 901                                         goto interrupted;
 902
 903                                 if (mutex_lock_interruptible(
 904                                             &channel->wr_mutex))
 905                                         goto interrupted;
 906                         } while (channel->wr_sleepy);
 907
 908                         continue;
 909
 910 interrupted: /* Mutex is not held if got here */
 911                         if (channel->endpoint->fatal_error)
 912                                 return -EIO;
 913                         if (bytes_done)
 914                                 return bytes_done;
 915                         if (filp->f_flags & O_NONBLOCK)
 916                                 return -EAGAIN; /* Don't admit snoozing */
 917                         return -EINTR;
 918                 }
 919
 920                 left_to_sleep = deadline - ((long) jiffies);
 921
 922                 /*
 923                  * If our time is out, skip the waiting. We may miss wr_sleepy
 924                  * being deasserted but hey, almost missing the train is like
 925                  * missing it.
 926                  */
 927
 928                 if (left_to_sleep > 0) {
 929                         left_to_sleep =
 930                                 wait_event_interruptible_timeout(
 931                                         channel->wr_wait,
 932                                         (!channel->wr_sleepy),
 933                                         left_to_sleep);
 934
 935                         if (left_to_sleep > 0) /* wr_sleepy deasserted */
 936                                 continue;
 937
 938                         if (left_to_sleep < 0) { /* Interrupt */
 939                                 mutex_unlock(&channel->wr_mutex);
 940                                 if (channel->endpoint->fatal_error)
 941                                         return -EIO;
 942                                 if (bytes_done)
 943                                         return bytes_done;
 944                                 return -EINTR;
 945                         }
 946                 }
 947
 948 desperate:
 949                 no_time_left = 1; /* We're out of sleeping time. Desperate! */
 950
 951                 if (bytes_done == 0) {
 952                         /*
 953                          * Reaching here means that we allow partial return,
 954                          * that we've run out of time, and that we have
 955                          * nothing to return.
 956                          * So tell the FPGA to send anything it has or gets.
 957                          */
 958
 959                         iowrite32(1 | (channel->chan_num << 1) |
 960                                   (3 << 24) |  /* Opcode 3, flush it all! */
 961                                   (waiting_bufidx << 12),
 962                                   channel->endpoint->registers +
 963                                   fpga_buf_ctrl_reg);
 964                 }
 965
 966                 /*
 967                  * Reaching here means that we *do* have data in the buffer,
 968                  * but the "partial" flag disallows returning less than
 969                  * required. And we don't have as much. So loop again,
 970                  * which is likely to end up blocking indefinitely until
 971                  * enough data has arrived.
 972                  */
 973         }
 974
 975         mutex_unlock(&channel->wr_mutex);
 976
 977         if (channel->endpoint->fatal_error)
 978                 return -EIO;
 979
 980         if (rc)
 981                 return rc;
 982
 983         return bytes_done;
 984 }
 985
 986 /*
 987  * The timeout argument takes values as follows:
 988  *  >0 : Flush with timeout
 989  * ==0 : Flush, and wait idefinitely for the flush to complete
 990  *  <0 : Autoflush: Flush only if there's a single buffer occupied
 991  */
 992
 993 static int xillybus_myflush(struct xilly_channel *channel, long timeout)
 994 {
 995         int rc;
 996         unsigned long flags;
 997
 998         int end_offset_plus1;
 999         int bufidx, bufidx_minus1;
1000         int i;
1001         int empty;
1002         int new_rd_host_buf_pos;
1003
1004         if (channel->endpoint->fatal_error)
1005                 return -EIO;
1006         rc = mutex_lock_interruptible(&channel->rd_mutex);
1007         if (rc)
1008                 return rc;
1009
1010         /*
1011          * Don't flush a closed channel. This can happen when the work queued
1012          * autoflush thread fires off after the file has closed. This is not
1013          * an error, just something to dismiss.
1014          */
1015
1016         if (!channel->rd_ref_count)
1017                 goto done;
1018
1019         bufidx = channel->rd_host_buf_idx;
1020
1021         bufidx_minus1 = (bufidx == 0) ?
1022                 channel->num_rd_buffers - 1 :
1023                 bufidx - 1;
1024
1025         end_offset_plus1 = channel->rd_host_buf_pos >>
1026                 channel->log2_element_size;
1027
1028         new_rd_host_buf_pos = channel->rd_host_buf_pos -
1029                 (end_offset_plus1 << channel->log2_element_size);
1030
1031         /* Submit the current buffer if it's nonempty */
1032         if (end_offset_plus1) {
1033                 unsigned char *tail = channel->rd_buffers[bufidx]->addr +
1034                         (end_offset_plus1 << channel->log2_element_size);
1035
1036                 /* Copy  unflushed data, so we can put it in next buffer */
1037                 for (i = 0; i < new_rd_host_buf_pos; i++)
1038                         channel->rd_leftovers[i] = *tail++;
1039
1040                 spin_lock_irqsave(&channel->rd_spinlock, flags);
1041
1042                 /* Autoflush only if a single buffer is occupied */
1043
1044                 if ((timeout < 0) &&
1045                     (channel->rd_full ||
1046                      (bufidx_minus1 != channel->rd_fpga_buf_idx))) {
1047                         spin_unlock_irqrestore(&channel->rd_spinlock, flags);
1048                         /*
1049                          * A new work item may be queued by the ISR exactly
1050                          * now, since the execution of a work item allows the
1051                          * queuing of a new one while it's running.
1052                          */
1053                         goto done;
1054                 }
1055
1056                 /* The 4th element is never needed for data, so it's a flag */
1057                 channel->rd_leftovers[3] = (new_rd_host_buf_pos != 0);
1058
1059                 /* Set up rd_full to reflect a certain moment's state */
1060
1061                 if (bufidx == channel->rd_fpga_buf_idx)
1062                         channel->rd_full = 1;
1063                 spin_unlock_irqrestore(&channel->rd_spinlock, flags);
1064
1065                 if (bufidx >= (channel->num_rd_buffers - 1))
1066                         channel->rd_host_buf_idx = 0;
1067                 else
1068                         channel->rd_host_buf_idx++;
1069
1070                 channel->endpoint->ephw->hw_sync_sgl_for_device(
1071                         channel->endpoint,
1072                         channel->rd_buffers[bufidx]->dma_addr,
1073                         channel->rd_buf_size,
1074                         DMA_TO_DEVICE);
1075
1076                 mutex_lock(&channel->endpoint->register_mutex);
1077
1078                 iowrite32(end_offset_plus1 - 1,
1079                           channel->endpoint->registers + fpga_buf_offset_reg);
1080
1081                 iowrite32((channel->chan_num << 1) | /* Channel ID */
1082                           (2 << 24) |  /* Opcode 2, submit buffer */
1083                           (bufidx << 12),
1084                           channel->endpoint->registers + fpga_buf_ctrl_reg);
1085
1086                 mutex_unlock(&channel->endpoint->register_mutex);
1087         } else if (bufidx == 0) {
1088                 bufidx = channel->num_rd_buffers - 1;
1089         } else {
1090                 bufidx--;
1091         }
1092
1093         channel->rd_host_buf_pos = new_rd_host_buf_pos;
1094
1095         if (timeout < 0)
1096                 goto done; /* Autoflush */
1097
1098         /*
1099          * bufidx is now the last buffer written to (or equal to
1100          * rd_fpga_buf_idx if buffer was never written to), and
1101          * channel->rd_host_buf_idx the one after it.
1102          *
1103          * If bufidx == channel->rd_fpga_buf_idx we're either empty or full.
1104          */
1105
1106         while (1) { /* Loop waiting for draining of buffers */
1107                 spin_lock_irqsave(&channel->rd_spinlock, flags);
1108
1109                 if (bufidx != channel->rd_fpga_buf_idx)
1110                         channel->rd_full = 1; /*
1111                                                * Not really full,
1112                                                * but needs waiting.
1113                                                */
1114
1115                 empty = !channel->rd_full;
1116
1117                 spin_unlock_irqrestore(&channel->rd_spinlock, flags);
1118
1119                 if (empty)
1120                         break;
1121
1122                 /*
1123                  * Indefinite sleep with mutex taken. With data waiting for
1124                  * flushing user should not be surprised if open() for write
1125                  * sleeps.
1126                  */
1127                 if (timeout == 0)
1128                         wait_event_interruptible(channel->rd_wait,
1129                                                  (!channel->rd_full));
1130
1131                 else if (wait_event_interruptible_timeout(
1132                                  channel->rd_wait,
1133                                  (!channel->rd_full),
1134                                  timeout) == 0) {
1135                         dev_warn(channel->endpoint->dev,
1136                                  "Timed out while flushing. Output data may be lost.\n");
1137
1138                         rc = -ETIMEDOUT;
1139                         break;
1140                 }
1141
1142                 if (channel->rd_full) {
1143                         rc = -EINTR;
1144                         break;
1145                 }
1146         }
1147
1148 done:
1149         mutex_unlock(&channel->rd_mutex);
1150
1151         if (channel->endpoint->fatal_error)
1152                 return -EIO;
1153
1154         return rc;
1155 }
1156
1157 static int xillybus_flush(struct file *filp, fl_owner_t id)
1158 {
1159         if (!(filp->f_mode & FMODE_WRITE))
1160                 return 0;
1161
1162         return xillybus_myflush(filp->private_data, HZ); /* 1 second timeout */
1163 }
1164
1165 static void xillybus_autoflush(struct work_struct *work)
1166 {
1167         struct delayed_work *workitem = container_of(
1168                 work, struct delayed_work, work);
1169         struct xilly_channel *channel = container_of(
1170                 workitem, struct xilly_channel, rd_workitem);
1171         int rc;
1172
1173         rc = xillybus_myflush(channel, -1);
1174         if (rc == -EINTR)
1175                 dev_warn(channel->endpoint->dev,
1176                          "Autoflush failed because work queue thread got a signal.\n");
1177         else if (rc)
1178                 dev_err(channel->endpoint->dev,
1179                         "Autoflush failed under weird circumstances.\n");
1180 }
1181
1182 static ssize_t xillybus_write(struct file *filp, const char __user *userbuf,
1183                               size_t count, loff_t *f_pos)
1184 {
1185         ssize_t rc;
1186         unsigned long flags;
1187         int bytes_done = 0;
1188         struct xilly_channel *channel = filp->private_data;
1189
1190         int full, exhausted;
1191         /* Initializations are there only to silence warnings */
1192
1193         int howmany = 0, bufpos = 0, bufidx = 0, bufferdone = 0;
1194         int end_offset_plus1 = 0;
1195
1196         if (channel->endpoint->fatal_error)
1197                 return -EIO;
1198
1199         rc = mutex_lock_interruptible(&channel->rd_mutex);
1200         if (rc)
1201                 return rc;
1202
1203         while (1) {
1204                 int bytes_to_do = count - bytes_done;
1205
1206                 spin_lock_irqsave(&channel->rd_spinlock, flags);
1207
1208                 full = channel->rd_full;
1209
1210                 if (!full) {
1211                         bufidx = channel->rd_host_buf_idx;
1212                         bufpos = channel->rd_host_buf_pos;
1213                         howmany = channel->rd_buf_size - bufpos;
1214
1215                         /*
1216                          * Update rd_host_* to its state after this operation.
1217                          * count=0 means committing the buffer immediately,
1218                          * which is like flushing, but not necessarily block.
1219                          */
1220
1221                         if ((howmany > bytes_to_do) &&
1222                             (count ||
1223                              ((bufpos >> channel->log2_element_size) == 0))) {
1224                                 bufferdone = 0;
1225
1226                                 howmany = bytes_to_do;
1227                                 channel->rd_host_buf_pos += howmany;
1228                         } else {
1229                                 bufferdone = 1;
1230
1231                                 if (count) {
1232                                         end_offset_plus1 =
1233                                                 channel->rd_buf_size >>
1234                                                 channel->log2_element_size;
1235                                         channel->rd_host_buf_pos = 0;
1236                                 } else {
1237                                         unsigned char *tail;
1238                                         int i;
1239
1240                                         howmany = 0;
1241
1242                                         end_offset_plus1 = bufpos >>
1243                                                 channel->log2_element_size;
1244
1245                                         channel->rd_host_buf_pos -=
1246                                                 end_offset_plus1 <<
1247                                                 channel->log2_element_size;
1248
1249                                         tail = channel->
1250                                                 rd_buffers[bufidx]->addr +
1251                                                 (end_offset_plus1 <<
1252                                                  channel->log2_element_size);
1253
1254                                         for (i = 0;
1255                                              i < channel->rd_host_buf_pos;
1256                                              i++)
1257                                                 channel->rd_leftovers[i] =
1258                                                         *tail++;
1259                                 }
1260
1261                                 if (bufidx == channel->rd_fpga_buf_idx)
1262                                         channel->rd_full = 1;
1263
1264                                 if (bufidx >= (channel->num_rd_buffers - 1))
1265                                         channel->rd_host_buf_idx = 0;
1266                                 else
1267                                         channel->rd_host_buf_idx++;
1268                         }
1269                 }
1270
1271                 /*
1272                  * Marking our situation after the possible changes above,
1273                  * for use  after releasing the spinlock.
1274                  *
1275                  * full = full before change
1276                  * exhasted = full after possible change
1277                  */
1278
1279                 exhausted = channel->rd_full;
1280
1281                 spin_unlock_irqrestore(&channel->rd_spinlock, flags);
1282
1283                 if (!full) { /* Go on, now without the spinlock */
1284                         unsigned char *head =
1285                                 channel->rd_buffers[bufidx]->addr;
1286                         int i;
1287
1288                         if ((bufpos == 0) || /* Zero means it's virgin */
1289                             (channel->rd_leftovers[3] != 0)) {
1290                                 channel->endpoint->ephw->hw_sync_sgl_for_cpu(
1291                                         channel->endpoint,
1292                                         channel->rd_buffers[bufidx]->dma_addr,
1293                                         channel->rd_buf_size,
1294                                         DMA_TO_DEVICE);
1295
1296                                 /* Virgin, but leftovers are due */
1297                                 for (i = 0; i < bufpos; i++)
1298                                         *head++ = channel->rd_leftovers[i];
1299
1300                                 channel->rd_leftovers[3] = 0; /* Clear flag */
1301                         }
1302
1303                         if (copy_from_user(
1304                                     channel->rd_buffers[bufidx]->addr + bufpos,
1305                                     userbuf, howmany))
1306                                 rc = -EFAULT;
1307
1308                         userbuf += howmany;
1309                         bytes_done += howmany;
1310
1311                         if (bufferdone) {
1312                                 channel->endpoint->ephw->hw_sync_sgl_for_device(
1313                                         channel->endpoint,
1314                                         channel->rd_buffers[bufidx]->dma_addr,
1315                                         channel->rd_buf_size,
1316                                         DMA_TO_DEVICE);
1317
1318                                 mutex_lock(&channel->endpoint->register_mutex);
1319
1320                                 iowrite32(end_offset_plus1 - 1,
1321                                           channel->endpoint->registers +
1322                                           fpga_buf_offset_reg);
1323
1324                                 iowrite32((channel->chan_num << 1) |
1325                                           (2 << 24) |  /* 2 = submit buffer */
1326                                           (bufidx << 12),
1327                                           channel->endpoint->registers +
1328                                           fpga_buf_ctrl_reg);
1329
1330                                 mutex_unlock(&channel->endpoint->
1331                                              register_mutex);
1332
1333                                 channel->rd_leftovers[3] =
1334                                         (channel->rd_host_buf_pos != 0);
1335                         }
1336
1337                         if (rc) {
1338                                 mutex_unlock(&channel->rd_mutex);
1339
1340                                 if (channel->endpoint->fatal_error)
1341                                         return -EIO;
1342
1343                                 if (!channel->rd_synchronous)
1344                                         queue_delayed_work(
1345                                                 xillybus_wq,
1346                                                 &channel->rd_workitem,
1347                                                 XILLY_RX_TIMEOUT);
1348
1349                                 return rc;
1350                         }
1351                 }
1352
1353                 if (bytes_done >= count)
1354                         break;
1355
1356                 if (!exhausted)
1357                         continue; /* If there's more space, just go on */
1358
1359                 if ((bytes_done > 0) && channel->rd_allow_partial)
1360                         break;
1361
1362                 /*
1363                  * Indefinite sleep with mutex taken. With data waiting for
1364                  * flushing, user should not be surprised if open() for write
1365                  * sleeps.
1366                  */
1367
1368                 if (filp->f_flags & O_NONBLOCK) {
1369                         rc = -EAGAIN;
1370                         break;
1371                 }
1372
1373                 if (wait_event_interruptible(channel->rd_wait,
1374                                              (!channel->rd_full))) {
1375                         mutex_unlock(&channel->rd_mutex);
1376
1377                         if (channel->endpoint->fatal_error)
1378                                 return -EIO;
1379
1380                         if (bytes_done)
1381                                 return bytes_done;
1382                         return -EINTR;
1383                 }
1384         }
1385
1386         mutex_unlock(&channel->rd_mutex);
1387
1388         if (!channel->rd_synchronous)
1389                 queue_delayed_work(xillybus_wq,
1390                                    &channel->rd_workitem,
1391                                    XILLY_RX_TIMEOUT);
1392
1393         if (channel->endpoint->fatal_error)
1394                 return -EIO;
1395
1396         if (rc)
1397                 return rc;
1398
1399         if ((channel->rd_synchronous) && (bytes_done > 0)) {
1400                 rc = xillybus_myflush(filp->private_data, 0); /* No timeout */
1401
1402                 if (rc && (rc != -EINTR))
1403                         return rc;
1404         }
1405
1406         return bytes_done;
1407 }
1408
1409 static int xillybus_open(struct inode *inode, struct file *filp)
1410 {
1411         int rc = 0;
1412         unsigned long flags;
1413         int minor = iminor(inode);
1414         int major = imajor(inode);
1415         struct xilly_endpoint *ep_iter, *endpoint = NULL;
1416         struct xilly_channel *channel;
1417
1418         mutex_lock(&ep_list_lock);
1419
1420         list_for_each_entry(ep_iter, &list_of_endpoints, ep_list) {
1421                 if ((ep_iter->major == major) &&
1422                     (minor >= ep_iter->lowest_minor) &&
1423                     (minor < (ep_iter->lowest_minor +
1424                               ep_iter->num_channels))) {
1425                         endpoint = ep_iter;
1426                         break;
1427                 }
1428         }
1429         mutex_unlock(&ep_list_lock);
1430
1431         if (!endpoint) {
1432                 pr_err("xillybus: open() failed to find a device for major=%d and minor=%d\n",
1433                        major, minor);
1434                 return -ENODEV;
1435         }
1436
1437         if (endpoint->fatal_error)
1438                 return -EIO;
1439
1440         channel = endpoint->channels[1 + minor - endpoint->lowest_minor];
1441         filp->private_data = channel;
1442
1443         /*
1444          * It gets complicated because:
1445          * 1. We don't want to take a mutex we don't have to
1446          * 2. We don't want to open one direction if the other will fail.
1447          */
1448
1449         if ((filp->f_mode & FMODE_READ) && (!channel->num_wr_buffers))
1450                 return -ENODEV;
1451
1452         if ((filp->f_mode & FMODE_WRITE) && (!channel->num_rd_buffers))
1453                 return -ENODEV;
1454
1455         if ((filp->f_mode & FMODE_READ) && (filp->f_flags & O_NONBLOCK) &&
1456             (channel->wr_synchronous || !channel->wr_allow_partial ||
1457              !channel->wr_supports_nonempty)) {
1458                 dev_err(endpoint->dev,
1459                         "open() failed: O_NONBLOCK not allowed for read on this device\n");
1460                 return -ENODEV;
1461         }
1462
1463         if ((filp->f_mode & FMODE_WRITE) && (filp->f_flags & O_NONBLOCK) &&
1464             (channel->rd_synchronous || !channel->rd_allow_partial)) {
1465                 dev_err(endpoint->dev,
1466                         "open() failed: O_NONBLOCK not allowed for write on this device\n");
1467                 return -ENODEV;
1468         }
1469
1470         /*
1471          * Note: open() may block on getting mutexes despite O_NONBLOCK.
1472          * This shouldn't occur normally, since multiple open of the same
1473          * file descriptor is almost always prohibited anyhow
1474          * (*_exclusive_open is normally set in real-life systems).
1475          */
1476
1477         if (filp->f_mode & FMODE_READ) {
1478                 rc = mutex_lock_interruptible(&channel->wr_mutex);
1479                 if (rc)
1480                         return rc;
1481         }
1482
1483         if (filp->f_mode & FMODE_WRITE) {
1484                 rc = mutex_lock_interruptible(&channel->rd_mutex);
1485                 if (rc)
1486                         goto unlock_wr;
1487         }
1488
1489         if ((filp->f_mode & FMODE_READ) &&
1490             (channel->wr_ref_count != 0) &&
1491             (channel->wr_exclusive_open)) {
1492                 rc = -EBUSY;
1493                 goto unlock;
1494         }
1495
1496         if ((filp->f_mode & FMODE_WRITE) &&
1497             (channel->rd_ref_count != 0) &&
1498             (channel->rd_exclusive_open)) {
1499                 rc = -EBUSY;
1500                 goto unlock;
1501         }
1502
1503         if (filp->f_mode & FMODE_READ) {
1504                 if (channel->wr_ref_count == 0) { /* First open of file */
1505                         /* Move the host to first buffer */
1506                         spin_lock_irqsave(&channel->wr_spinlock, flags);
1507                         channel->wr_host_buf_idx = 0;
1508                         channel->wr_host_buf_pos = 0;
1509                         channel->wr_fpga_buf_idx = -1;
1510                         channel->wr_empty = 1;
1511                         channel->wr_ready = 0;
1512                         channel->wr_sleepy = 1;
1513                         channel->wr_eof = -1;
1514                         channel->wr_hangup = 0;
1515
1516                         spin_unlock_irqrestore(&channel->wr_spinlock, flags);
1517
1518                         iowrite32(1 | (channel->chan_num << 1) |
1519                                   (4 << 24) |  /* Opcode 4, open channel */
1520                                   ((channel->wr_synchronous & 1) << 23),
1521                                   channel->endpoint->registers +
1522                                   fpga_buf_ctrl_reg);
1523                 }
1524
1525                 channel->wr_ref_count++;
1526         }
1527
1528         if (filp->f_mode & FMODE_WRITE) {
1529                 if (channel->rd_ref_count == 0) { /* First open of file */
1530                         /* Move the host to first buffer */
1531                         spin_lock_irqsave(&channel->rd_spinlock, flags);
1532                         channel->rd_host_buf_idx = 0;
1533                         channel->rd_host_buf_pos = 0;
1534                         channel->rd_leftovers[3] = 0; /* No leftovers. */
1535                         channel->rd_fpga_buf_idx = channel->num_rd_buffers - 1;
1536                         channel->rd_full = 0;
1537
1538                         spin_unlock_irqrestore(&channel->rd_spinlock, flags);
1539
1540                         iowrite32((channel->chan_num << 1) |
1541                                   (4 << 24),   /* Opcode 4, open channel */
1542                                   channel->endpoint->registers +
1543                                   fpga_buf_ctrl_reg);
1544                 }
1545
1546                 channel->rd_ref_count++;
1547         }
1548
1549 unlock:
1550         if (filp->f_mode & FMODE_WRITE)
1551                 mutex_unlock(&channel->rd_mutex);
1552 unlock_wr:
1553         if (filp->f_mode & FMODE_READ)
1554                 mutex_unlock(&channel->wr_mutex);
1555
1556         if (!rc && (!channel->seekable))
1557                 return nonseekable_open(inode, filp);
1558
1559         return rc;
1560 }
1561
1562 static int xillybus_release(struct inode *inode, struct file *filp)
1563 {
1564         unsigned long flags;
1565         struct xilly_channel *channel = filp->private_data;
1566
1567         int buf_idx;
1568         int eof;
1569
1570         if (channel->endpoint->fatal_error)
1571                 return -EIO;
1572
1573         if (filp->f_mode & FMODE_WRITE) {
1574                 mutex_lock(&channel->rd_mutex);
1575
1576                 channel->rd_ref_count--;
1577
1578                 if (channel->rd_ref_count == 0) {
1579                         /*
1580                          * We rely on the kernel calling flush()
1581                          * before we get here.
1582                          */
1583
1584                         iowrite32((channel->chan_num << 1) | /* Channel ID */
1585                                   (5 << 24),  /* Opcode 5, close channel */
1586                                   channel->endpoint->registers +
1587                                   fpga_buf_ctrl_reg);
1588                 }
1589                 mutex_unlock(&channel->rd_mutex);
1590         }
1591
1592         if (filp->f_mode & FMODE_READ) {
1593                 mutex_lock(&channel->wr_mutex);
1594
1595                 channel->wr_ref_count--;
1596
1597                 if (channel->wr_ref_count == 0) {
1598                         iowrite32(1 | (channel->chan_num << 1) |
1599                                   (5 << 24),  /* Opcode 5, close channel */
1600                                   channel->endpoint->registers +
1601                                   fpga_buf_ctrl_reg);
1602
1603                         /*
1604                          * This is crazily cautious: We make sure that not
1605                          * only that we got an EOF (be it because we closed
1606                          * the channel or because of a user's EOF), but verify
1607                          * that it's one beyond the last buffer arrived, so
1608                          * we have no leftover buffers pending before wrapping
1609                          * up (which can only happen in asynchronous channels,
1610                          * BTW)
1611                          */
1612
1613                         while (1) {
1614                                 spin_lock_irqsave(&channel->wr_spinlock,
1615                                                   flags);
1616                                 buf_idx = channel->wr_fpga_buf_idx;
1617                                 eof = channel->wr_eof;
1618                                 channel->wr_sleepy = 1;
1619                                 spin_unlock_irqrestore(&channel->wr_spinlock,
1620                                                        flags);
1621
1622                                 /*
1623                                  * Check if eof points at the buffer after
1624                                  * the last one the FPGA submitted. Note that
1625                                  * no EOF is marked by negative eof.
1626                                  */
1627
1628                                 buf_idx++;
1629                                 if (buf_idx == channel->num_wr_buffers)
1630                                         buf_idx = 0;
1631
1632                                 if (buf_idx == eof)
1633                                         break;
1634
1635                                 /*
1636                                  * Steal extra 100 ms if awaken by interrupt.
1637                                  * This is a simple workaround for an
1638                                  * interrupt pending when entering, which would
1639                                  * otherwise result in declaring the hardware
1640                                  * non-responsive.
1641                                  */
1642
1643                                 if (wait_event_interruptible(
1644                                             channel->wr_wait,
1645                                             (!channel->wr_sleepy)))
1646                                         msleep(100);
1647
1648                                 if (channel->wr_sleepy) {
1649                                         mutex_unlock(&channel->wr_mutex);
1650                                         dev_warn(channel->endpoint->dev,
1651                                                  "Hardware failed to respond to close command, therefore left in messy state.\n");
1652                                         return -EINTR;
1653                                 }
1654                         }
1655                 }
1656
1657                 mutex_unlock(&channel->wr_mutex);
1658         }
1659
1660         return 0;
1661 }
1662
1663 static loff_t xillybus_llseek(struct file *filp, loff_t offset, int whence)
1664 {
1665         struct xilly_channel *channel = filp->private_data;
1666         loff_t pos = filp->f_pos;
1667         int rc = 0;
1668
1669         /*
1670          * Take both mutexes not allowing interrupts, since it seems like
1671          * common applications don't expect an -EINTR here. Besides, multiple
1672          * access to a single file descriptor on seekable devices is a mess
1673          * anyhow.
1674          */
1675
1676         if (channel->endpoint->fatal_error)
1677                 return -EIO;
1678
1679         mutex_lock(&channel->wr_mutex);
1680         mutex_lock(&channel->rd_mutex);
1681
1682         switch (whence) {
1683         case SEEK_SET:
1684                 pos = offset;
1685                 break;
1686         case SEEK_CUR:
1687                 pos += offset;
1688                 break;
1689         case SEEK_END:
1690                 pos = offset; /* Going to the end => to the beginning */
1691                 break;
1692         default:
1693                 rc = -EINVAL;
1694                 goto end;
1695         }
1696
1697         /* In any case, we must finish on an element boundary */
1698         if (pos & ((1 << channel->log2_element_size) - 1)) {
1699                 rc = -EINVAL;
1700                 goto end;
1701         }
1702
1703         mutex_lock(&channel->endpoint->register_mutex);
1704
1705         iowrite32(pos >> channel->log2_element_size,
1706                   channel->endpoint->registers + fpga_buf_offset_reg);
1707
1708         iowrite32((channel->chan_num << 1) |
1709                   (6 << 24),  /* Opcode 6, set address */
1710                   channel->endpoint->registers + fpga_buf_ctrl_reg);
1711
1712         mutex_unlock(&channel->endpoint->register_mutex);
1713
1714 end:
1715         mutex_unlock(&channel->rd_mutex);
1716         mutex_unlock(&channel->wr_mutex);
1717
1718         if (rc) /* Return error after releasing mutexes */
1719                 return rc;
1720
1721         filp->f_pos = pos;
1722
1723         /*
1724          * Since seekable devices are allowed only when the channel is
1725          * synchronous, we assume that there is no data pending in either
1726          * direction (which holds true as long as no concurrent access on the
1727          * file descriptor takes place).
1728          * The only thing we may need to throw away is leftovers from partial
1729          * write() flush.
1730          */
1731
1732         channel->rd_leftovers[3] = 0;
1733
1734         return pos;
1735 }
1736
1737 static unsigned int xillybus_poll(struct file *filp, poll_table *wait)
1738 {
1739         struct xilly_channel *channel = filp->private_data;
1740         unsigned int mask = 0;
1741         unsigned long flags;
1742
1743         poll_wait(filp, &channel->endpoint->ep_wait, wait);
1744
1745         /*
1746          * poll() won't play ball regarding read() channels which
1747          * aren't asynchronous and support the nonempty message. Allowing
1748          * that will create situations where data has been delivered at
1749          * the FPGA, and users expecting select() to wake up, which it may
1750          * not.
1751          */
1752
1753         if (!channel->wr_synchronous && channel->wr_supports_nonempty) {
1754                 poll_wait(filp, &channel->wr_wait, wait);
1755                 poll_wait(filp, &channel->wr_ready_wait, wait);
1756
1757                 spin_lock_irqsave(&channel->wr_spinlock, flags);
1758                 if (!channel->wr_empty || channel->wr_ready)
1759                         mask |= POLLIN | POLLRDNORM;
1760
1761                 if (channel->wr_hangup)
1762                         /*
1763                          * Not POLLHUP, because its behavior is in the
1764                          * mist, and POLLIN does what we want: Wake up
1765                          * the read file descriptor so it sees EOF.
1766                          */
1767                         mask |=  POLLIN | POLLRDNORM;
1768                 spin_unlock_irqrestore(&channel->wr_spinlock, flags);
1769         }
1770
1771         /*
1772          * If partial data write is disallowed on a write() channel,
1773          * it's pointless to ever signal OK to write, because is could
1774          * block despite some space being available.
1775          */
1776
1777         if (channel->rd_allow_partial) {
1778                 poll_wait(filp, &channel->rd_wait, wait);
1779
1780                 spin_lock_irqsave(&channel->rd_spinlock, flags);
1781                 if (!channel->rd_full)
1782                         mask |= POLLOUT | POLLWRNORM;
1783                 spin_unlock_irqrestore(&channel->rd_spinlock, flags);
1784         }
1785
1786         if (channel->endpoint->fatal_error)
1787                 mask |= POLLERR;
1788
1789         return mask;
1790 }
1791
1792 static const struct file_operations xillybus_fops = {
1793         .owner      = THIS_MODULE,
1794         .read       = xillybus_read,
1795         .write      = xillybus_write,
1796         .open       = xillybus_open,
1797         .flush      = xillybus_flush,
1798         .release    = xillybus_release,
1799         .llseek     = xillybus_llseek,
1800         .poll       = xillybus_poll,
1801 };
1802
1803 static int xillybus_init_chrdev(struct xilly_endpoint *endpoint,
1804                                 const unsigned char *idt)
1805 {
1806         int rc;
1807         dev_t dev;
1808         int devnum, i, minor, major;
1809         char devname[48];
1810         struct device *device;
1811
1812         rc = alloc_chrdev_region(&dev, 0, /* minor start */
1813                                  endpoint->num_channels,
1814                                  xillyname);
1815         if (rc) {
1816                 dev_warn(endpoint->dev, "Failed to obtain major/minors");
1817                 return rc;
1818         }
1819
1820         endpoint->major = major = MAJOR(dev);
1821         endpoint->lowest_minor = minor = MINOR(dev);
1822
1823         cdev_init(&endpoint->cdev, &xillybus_fops);
1824         endpoint->cdev.owner = endpoint->ephw->owner;
1825         rc = cdev_add(&endpoint->cdev, MKDEV(major, minor),
1826                       endpoint->num_channels);
1827         if (rc) {
1828                 dev_warn(endpoint->dev, "Failed to add cdev. Aborting.\n");
1829                 goto unregister_chrdev;
1830         }
1831
1832         idt++;
1833
1834         for (i = minor, devnum = 0;
1835              devnum < endpoint->num_channels;
1836              devnum++, i++) {
1837                 snprintf(devname, sizeof(devname)-1, "xillybus_%s", idt);
1838
1839                 devname[sizeof(devname)-1] = 0; /* Should never matter */
1840
1841                 while (*idt++)
1842                         /* Skip to next */;
1843
1844                 device = device_create(xillybus_class,
1845                                        NULL,
1846                                        MKDEV(major, i),
1847                                        NULL,
1848                                        "%s", devname);
1849
1850                 if (IS_ERR(device)) {
1851                         dev_warn(endpoint->dev,
1852                                  "Failed to create %s device. Aborting.\n",
1853                                  devname);
1854                         rc = -ENODEV;
1855                         goto unroll_device_create;
1856                 }
1857         }
1858
1859         dev_info(endpoint->dev, "Created %d device files.\n",
1860                  endpoint->num_channels);
1861         return 0; /* succeed */
1862
1863 unroll_device_create:
1864         devnum--; i--;
1865         for (; devnum >= 0; devnum--, i--)
1866                 device_destroy(xillybus_class, MKDEV(major, i));
1867
1868         cdev_del(&endpoint->cdev);
1869 unregister_chrdev:
1870         unregister_chrdev_region(MKDEV(major, minor), endpoint->num_channels);
1871
1872         return rc;
1873 }
1874
1875 static void xillybus_cleanup_chrdev(struct xilly_endpoint *endpoint)
1876 {
1877         int minor;
1878
1879         for (minor = endpoint->lowest_minor;
1880              minor < (endpoint->lowest_minor + endpoint->num_channels);
1881              minor++)
1882                 device_destroy(xillybus_class, MKDEV(endpoint->major, minor));
1883         cdev_del(&endpoint->cdev);
1884         unregister_chrdev_region(MKDEV(endpoint->major,
1885                                        endpoint->lowest_minor),
1886                                  endpoint->num_channels);
1887
1888         dev_info(endpoint->dev, "Removed %d device files.\n",
1889                  endpoint->num_channels);
1890 }
1891
1892 struct xilly_endpoint *xillybus_init_endpoint(struct pci_dev *pdev,
1893                                               struct device *dev,
1894                                               struct xilly_endpoint_hardware
1895                                               *ephw)
1896 {
1897         struct xilly_endpoint *endpoint;
1898
1899         endpoint = devm_kzalloc(dev, sizeof(*endpoint), GFP_KERNEL);
1900         if (!endpoint)
1901                 return NULL;
1902
1903         endpoint->pdev = pdev;
1904         endpoint->dev = dev;
1905         endpoint->ephw = ephw;
1906         endpoint->msg_counter = 0x0b;
1907         endpoint->failed_messages = 0;
1908         endpoint->fatal_error = 0;
1909
1910         init_waitqueue_head(&endpoint->ep_wait);
1911         mutex_init(&endpoint->register_mutex);
1912
1913         return endpoint;
1914 }
1915 EXPORT_SYMBOL(xillybus_init_endpoint);
1916
1917 static int xilly_quiesce(struct xilly_endpoint *endpoint)
1918 {
1919         long t;
1920
1921         endpoint->idtlen = -1;
1922
1923         iowrite32((u32) (endpoint->dma_using_dac & 0x0001),
1924                   endpoint->registers + fpga_dma_control_reg);
1925
1926         t = wait_event_interruptible_timeout(endpoint->ep_wait,
1927                                              (endpoint->idtlen >= 0),
1928                                              XILLY_TIMEOUT);
1929         if (t <= 0) {
1930                 dev_err(endpoint->dev,
1931                         "Failed to quiesce the device on exit.\n");
1932                 return -ENODEV;
1933         }
1934         return 0;
1935 }
1936
1937 int xillybus_endpoint_discovery(struct xilly_endpoint *endpoint)
1938 {
1939         int rc;
1940         long t;
1941
1942         void *bootstrap_resources;
1943         int idtbuffersize = (1 << PAGE_SHIFT);
1944         struct device *dev = endpoint->dev;
1945
1946         /*
1947          * The bogus IDT is used during bootstrap for allocating the initial
1948          * message buffer, and then the message buffer and space for the IDT
1949          * itself. The initial message buffer is of a single page's size, but
1950          * it's soon replaced with a more modest one (and memory is freed).
1951          */
1952
1953         unsigned char bogus_idt[8] = { 1, 224, (PAGE_SHIFT)-2, 0,
1954                                        3, 192, PAGE_SHIFT, 0 };
1955         struct xilly_idt_handle idt_handle;
1956
1957         /*
1958          * Writing the value 0x00000001 to Endianness register signals which
1959          * endianness this processor is using, so the FPGA can swap words as
1960          * necessary.
1961          */
1962
1963         iowrite32(1, endpoint->registers + fpga_endian_reg);
1964
1965         /* Bootstrap phase I: Allocate temporary message buffer */
1966
1967         bootstrap_resources = devres_open_group(dev, NULL, GFP_KERNEL);
1968         if (!bootstrap_resources)
1969                 return -ENOMEM;
1970
1971         endpoint->num_channels = 0;
1972
1973         rc = xilly_setupchannels(endpoint, bogus_idt, 1);
1974         if (rc)
1975                 return rc;
1976
1977         /* Clear the message subsystem (and counter in particular) */
1978         iowrite32(0x04, endpoint->registers + fpga_msg_ctrl_reg);
1979
1980         endpoint->idtlen = -1;
1981
1982         /*
1983          * Set DMA 32/64 bit mode, quiesce the device (?!) and get IDT
1984          * buffer size.
1985          */
1986         iowrite32((u32) (endpoint->dma_using_dac & 0x0001),
1987                   endpoint->registers + fpga_dma_control_reg);
1988
1989         t = wait_event_interruptible_timeout(endpoint->ep_wait,
1990                                              (endpoint->idtlen >= 0),
1991                                              XILLY_TIMEOUT);
1992         if (t <= 0) {
1993                 dev_err(endpoint->dev, "No response from FPGA. Aborting.\n");
1994                 return -ENODEV;
1995         }
1996
1997         /* Enable DMA */
1998         iowrite32((u32) (0x0002 | (endpoint->dma_using_dac & 0x0001)),
1999                   endpoint->registers + fpga_dma_control_reg);
2000
2001         /* Bootstrap phase II: Allocate buffer for IDT and obtain it */
2002         while (endpoint->idtlen >= idtbuffersize) {
2003                 idtbuffersize *= 2;
2004                 bogus_idt[6]++;
2005         }
2006
2007         endpoint->num_channels = 1;
2008
2009         rc = xilly_setupchannels(endpoint, bogus_idt, 2);
2010         if (rc)
2011                 goto failed_idt;
2012
2013         rc = xilly_obtain_idt(endpoint);
2014         if (rc)
2015                 goto failed_idt;
2016
2017         rc = xilly_scan_idt(endpoint, &idt_handle);
2018         if (rc)
2019                 goto failed_idt;
2020
2021         devres_close_group(dev, bootstrap_resources);
2022
2023         /* Bootstrap phase III: Allocate buffers according to IDT */
2024
2025         rc = xilly_setupchannels(endpoint,
2026                                  idt_handle.chandesc,
2027                                  idt_handle.entries);
2028         if (rc)
2029                 goto failed_idt;
2030
2031         /*
2032          * endpoint is now completely configured. We put it on the list
2033          * available to open() before registering the char device(s)
2034          */
2035
2036         mutex_lock(&ep_list_lock);
2037         list_add_tail(&endpoint->ep_list, &list_of_endpoints);
2038         mutex_unlock(&ep_list_lock);
2039
2040         rc = xillybus_init_chrdev(endpoint, idt_handle.idt);
2041         if (rc)
2042                 goto failed_chrdevs;
2043
2044         devres_release_group(dev, bootstrap_resources);
2045
2046         return 0;
2047
2048 failed_chrdevs:
2049         mutex_lock(&ep_list_lock);
2050         list_del(&endpoint->ep_list);
2051         mutex_unlock(&ep_list_lock);
2052
2053 failed_idt:
2054         xilly_quiesce(endpoint);
2055         flush_workqueue(xillybus_wq);
2056
2057         return rc;
2058 }
2059 EXPORT_SYMBOL(xillybus_endpoint_discovery);
2060
2061 void xillybus_endpoint_remove(struct xilly_endpoint *endpoint)
2062 {
2063         xillybus_cleanup_chrdev(endpoint);
2064
2065         mutex_lock(&ep_list_lock);
2066         list_del(&endpoint->ep_list);
2067         mutex_unlock(&ep_list_lock);
2068
2069         xilly_quiesce(endpoint);
2070
2071         /*
2072          * Flushing is done upon endpoint release to prevent access to memory
2073          * just about to be released. This makes the quiesce complete.
2074          */
2075         flush_workqueue(xillybus_wq);
2076 }
2077 EXPORT_SYMBOL(xillybus_endpoint_remove);
2078
2079 static int __init xillybus_init(void)
2080 {
2081         mutex_init(&ep_list_lock);
2082
2083         xillybus_class = class_create(THIS_MODULE, xillyname);
2084         if (IS_ERR(xillybus_class))
2085                 return PTR_ERR(xillybus_class);
2086
2087         xillybus_wq = alloc_workqueue(xillyname, 0, 0);
2088         if (!xillybus_wq) {
2089                 class_destroy(xillybus_class);
2090                 return -ENOMEM;
2091         }
2092
2093         return 0;
2094 }
2095
2096 static void __exit xillybus_exit(void)
2097 {
2098         /* flush_workqueue() was called for each endpoint released */
2099         destroy_workqueue(xillybus_wq);
2100
2101         class_destroy(xillybus_class);
2102 }
2103
2104 module_init(xillybus_init);
2105 module_exit(xillybus_exit);