Virtio Network device not generating interrupts

[rannnnnddiddddd]

Hi,
I have been trying to write a driver for virtio network device. I had some success, I can read mac address, negotiate features etc. But when my set up function comes to the end, virtio never generates a single interrupt. What are the possible causes of this? I can understand it not working completely once I get an interrupt, if I messed up some address somewhere, but I can't figure out why it wouldn't generate interrupts?

This is the main bit of code:

Code: Select all

    __copy_pci_device_info(device_info);

    LOG("virtio_net: Initializing virtio network device addr 0x%x", __virtio_net_dev.device_addr);

    __detect_io_base_mem_address();

    pci_enable_bus_mastering(__virtio_net_dev.device_addr);

    LOG("virtio_net: bus mastering enabled");
    LOG("virtio_net: io 0x%x", __virtio_net_dev.io_base);

    pmio_write_8(__virtio_net_dev.io_base + 0x12, VIRTIO_ACKNOWLEDGE);
    pmio_write_8(__virtio_net_dev.io_base + 0x12, VIRTIO_ACKNOWLEDGE | VIRTIO_DRIVER);

    uint32_t ft = pmio_read_32(__virtio_net_dev.io_base + 0x00);

    DISABLE_FEATURE(ft, VIRTIO_CTRL_VQ);

    DISABLE_FEATURE(ft, VIRTIO_GUEST_TSO4);
    DISABLE_FEATURE(ft, VIRTIO_GUEST_TSO6);
    DISABLE_FEATURE(ft, VIRTIO_GUEST_UFO);
    DISABLE_FEATURE(ft, VIRTIO_EVENT_IDX);
    DISABLE_FEATURE(ft, VIRTIO_MRG_RXBUF);

    ENABLE_FEATURE(ft, VIRTIO_CSUM);

    pmio_write_32(__virtio_net_dev.io_base + 0x04, ft);

    pmio_write_8(
        __virtio_net_dev.io_base + 0x12, VIRTIO_ACKNOWLEDGE | VIRTIO_DRIVER | VIRTIO_FEATURES_OK);

    if ((pmio_read_8(__virtio_net_dev.io_base + 0x12) & VIRTIO_FEATURES_OK) == 0) {
        LOG("virtio_net: features not accepted");
        return;
    }
    LOG("virtio_net: features accepted");

    for (int i = 0; i < 16; i++) {
        __virtio_queue_setup(i);
    }

    pmio_write_8(__virtio_net_dev.io_base + 0x12,
        VIRTIO_ACKNOWLEDGE | VIRTIO_DRIVER | VIRTIO_FEATURES_OK | VIRTIO_DRIVER_OK);

    LOG("virtio_net: created queues");

    virt_queue *rx = &__virtio_net_dev.queues[0];
    virt_queue *tx = &__virtio_net_dev.queues[1];
    if (rx == NULL || tx == NULL) {
        LOG("virtio_net: queues not found");
        return;
    }

    rx->buffer = (unsigned char *)__data1;
    rx->chunk_size = 1526;
    rx->available->index = 0;
    rx->used->flags = 0;

    buffer_info bi;
    bi.size = 1526;
    bi.buffer = 0;
    bi.flags = VIRTIO_DESC_FLAG_WRITE_ONLY;
    bi.copy = 1;

    for (int i = 0; i < rx->queue_size; i++) {
        virtio_send_buffer(0, &bi, 1);
    }

    tx->buffer = (unsigned char*)__data2;
    tx->chunk_size = 1526;
    tx->available->index = 0;
    pmio_write_16(__virtio_net_dev.io_base + 0x10, 0);

    LOG("virtio_net: done setting up buffers and queues");

    unsigned char v = 0x00;
    unsigned long tempq = 0;
    for (int i = 0; i < 6; i++) {
        v = pmio_read_8(__virtio_net_dev.io_base + 0x14 + i);
        tempq = (tempq << 8) | v;
    }
    __virtio_net_dev.mac_address = tempq;

    LOGN("virtio_net: mac address ");
    __print_mac_address(__virtio_net_dev.mac_address);
    LOG("virtio_net: done initializing virtio network device");

    LOG("virtio_net: interrupt %d", device_info->irq);
    pmio_write_16(__virtio_net_dev.io_base + 0x10, 0);

    ioapic_set_entry(g_ioapic_addr, acpi_remap_irq(device_info->irq), 34);
    bind_interrupt_handler_func(34, interrupt_virtio_net_handler);

    while (1) {
        pit_wait(100);
        LOG(".");
    }

Mac address is correct, so that means I have correct io base and my pmio functions work. Additionally, I know bus mastering, ioapic_set_entry, bind_interrupt_handler_func, etc. work, because I also used them for rtl8139 driver, which works correctly.

NOTE: to make sure my simple malloc implementation doesn't mess things up, I used predefined arrays like __data1 and __data2 to ensure there is enough memory for what i need.

This is function for setting up virtio_queue:

Code: Select all

static inline int __virtio_queue_setup(uint8_t index)
{
    unsigned short c;
    unsigned int i;

    virt_queue *vq = &__virtio_net_dev.queues[index];
    memset(vq, 0, sizeof(virt_queue));

    pmio_write_16(__virtio_net_dev.io_base + 0x0E, index);
    uint16_t queue_size = pmio_read_16(__virtio_net_dev.io_base + 0x0C);
    vq->queue_size = queue_size;
    if (queue_size == 0) {
        return 0;
    }

    uint32_t sizeofBuffers = (sizeof(virtio_queue_buffer) * queue_size);
    uint32_t sizeofQueueAvailable = (2 * sizeof(uint16_t)) + (queue_size * sizeof(uint16_t));
    uint32_t sizeofQueueUsed = (2 * sizeof(uint16_t)) + (queue_size * sizeof(virtio_used_item));
    uint32_t queuePageCount
        = PAGE_COUNT(sizeofBuffers + sizeofQueueAvailable) + PAGE_COUNT(sizeofQueueUsed);
    char *buf = __data3[index];
    memset(buf, 0, queuePageCount << 12);
    uint32_t bufPage = ((uint64_t)(buf));
    LOG("virtio_net: buf page %x %x", buf, bufPage);

    vq->baseAddress = (uint64_t)buf;
    vq->available = (virtio_available *)&buf[sizeofBuffers];
    vq->used = (virtio_used *)&buf[((sizeofBuffers + sizeofQueueAvailable + 0xFFF) & ~0xFFF)];
    vq->next_buffer = 0;
    vq->lock = 0;

    pmio_write_32(__virtio_net_dev.io_base + 0x08, bufPage);

    vq->available->flags = 0;
    return 1;
}

This is function for send buffer:

Code: Select all

void virtio_send_buffer(uint16_t queue_index, buffer_info b[], uint64_t count)
{
    virt_queue *vq = &__virtio_net_dev.queues[queue_index];

    uint16_t index = vq->available->index % vq->queue_size;
    uint16_t buffer_index = vq->next_buffer;
    uint16_t next_buffer_index;

    unsigned char *buf = (uint8_t *)(&vq->buffer[vq->chunk_size * buffer_index]);
    unsigned char *buf2 = buf;

    vq->available->rings[index] = buffer_index;
    for (uint64_t i = 0; i < count; i++) {
        next_buffer_index = (buffer_index + 1) % vq->queue_size;
        buffer_info *bi = &b[i];
        vq->buffers[buffer_index].flags = bi->flags;
        if (i != (count - 1)) {
            vq->buffers[buffer_index].flags |= VIRTIO_DESC_FLAG_NEXT;
        }

        vq->buffers[buffer_index].next = next_buffer_index;
        vq->buffers[buffer_index].length = bi->size;
        if (bi->copy) {
            vq->buffers[buffer_index].address = (uint64_t)buf2;
            if (bi->buffer != 0) {
                memcpy(buf2, bi->buffer, bi->size);
            }
            buf2 += bi->size;
        } else {
            vq->buffers[buffer_index].address = (uint64_t)bi->buffer;
        }
        buffer_index = next_buffer_index;
    }
    vq->next_buffer = buffer_index;

    vq->available->index++;
    pmio_write_16(__virtio_net_dev.io_base + 0x10, queue_index);
}

These are structures:

Code: Select all

typedef struct virtio_queue_buffer
{
    uint64_t address;
    uint32_t length;
    uint16_t flags;
    uint16_t next;
} virtio_queue_buffer;

typedef struct virtio_available
{
    uint16_t flags;
    uint16_t index;
    uint16_t rings[];
} virtio_available;

typedef struct virtio_used_item
{
    uint32_t index;
    uint32_t length;
} virtio_used_item;

typedef struct virtio_used
{
    uint16_t flags;
    uint16_t index;
    virtio_used_item rings[];
} virtio_used;

typedef struct virt_queue
{
    uint16_t queue_size;
    union
    {
        virtio_queue_buffer *buffers;
        uint64_t baseAddress;
    };
    virtio_available *available;
    virtio_used *used;
    uint16_t last_used_index;
    uint16_t last_available_index;
    uint8_t *buffer;
    uint32_t chunk_size;
    uint16_t next_buffer;
    uint64_t lock;
} virt_queue;

typedef struct virtio_net_dev
{
    uint16_t vendor_id;
    uint16_t device_id;
    uint8_t class_code;
    uint8_t subclass;
    uint32_t device_addr;
    uint32_t pci_bars[6];
    uint64_t memory_address;
    uint32_t io_base;
    uint32_t bus;
    uint32_t dev;
    uint32_t func;
    virt_queue queues[16];
    uint64_t mac_address;
} virtio_net_dev;

typedef struct buffer_info
{
    uint8_t *buffer;
    uint64_t size;
    uint8_t flags;
    int copy; // if false the supplied buffer will be copied in the queues buffer
} buffer_info;

[rannnnnddiddddd]

It seems like descriptors aren't set up, which is weird since i do set them up.

Code: Select all

(qeinfo virtio-queue-element /machine/peripheral-anon/device[0]/virtio-backend 0
/machine/peripheral-anon/device[0]/virtio-backend:
  device_name: virtio-net
  index:   0
  desc:
    descs:
        addr 0x0 len 0
  avail:
    flags: 0
    idx:   0
    ring:  0
  used:
    flags: 0
    idx:   0
(qemu) info virtio-queue-element /machine/peripheral-anon/device[0]/virtio-backend 1
/machine/peripheral-anon/device[0]/virtio-backend:
  device_name: virtio-net
  index:   0
  desc:
    descs:
        addr 0x0 len 0
  avail:
    flags: 0
    idx:   0
    ring:  0
  used:
    flags: 0
    idx:   0
(qemu) info virtio-queue-element /machine/peripheral-anon/device[0]/virtio-backend 2
/machine/peripheral-anon/device[0]/virtio-backend:
  device_name: virtio-net
  index:   0
  desc:
    descs:
        addr 0x0 len 0
  avail:
    flags: 0
    idx:   0
    ring:  0
  used:
    flags: 0
    idx:   0

[Octocontrabass]

Have you tried using QEMU's trace logger? There are several traceable events that might help you find the problem.