Software-Defined Device

Software Defined Device (SDD) is a framework that enables an application developer to write DPU applications interacting with the DPU’s on-board hardware accelerators. SDD provides APIs to emulate custom PCI devices entirely in software, running on the embedded CPU cores in the DPU SoC. In other words, the devices are software defined, instead of being fixed in hardware like on an FPGA or ASIC.

Using device emulation APIs, you can emulate a vast range of PCI devices. This includes NVMe disks, VirtIO devices (block, net, FS, and more), and even custom, experimental devices using new or existing interfaces.

Example

Here is an example of using SDD APIs to create a virtio block device.

This example demonstrates how to:

Create an emulated device
Configure vendor/device IDs for a virtio-blk device
Configure BAR space and register callbacks
Start the device, process operations, and teardown

example_sdd_virtio_blk.cc
#include <libmango.h>

/* Red Hat vendor ID */
#define VIRTIO_VENDOR_ID    0x1AF4
/* VirtIO Block device ID */
#define VIRTIO_BLK_DEV_ID   0x1001

/* VirtIO register offsets (simplified for example) */
#define VIRTIO_PCI_COMMON_DF            0x04  /* Device feature */
#define VIRTIO_PCI_COMMON_GF            0x0C  /* Guest feature */
#define VIRTIO_PCI_COMMON_Q_SELECT      0x16  /* Queue select */
#define VIRTIO_PCI_COMMON_STATUS        0x14  /* Device status */
#define VIRTIO_PCI_COMMON_Q_DESCLO      0x20  /* Queue descriptor low address */
#define VIRTIO_PCI_COMMON_Q_DESCHI      0x24  /* Queue descriptor high address */

/* Doorbell register base offset (in BAR 1) */
#define VIRTIO_DOORBELL_BASE         0x00

/* Request completion function for sending MSI-X interrupts */
void req_complete(mango_sdd_dev_h dev, uint32_t queue_id) {
    printf("Request completed for queue %u, sending MSI-X interrupt\n", queue_id);

    /* Use queue ID as MSI-X vector number */
    int status = mango_sdd_msix_inject(dev, queue_id);
    if (status != 0) {
        fprintf(stderr, "Failed to inject MSI-X interrupt: %d\n", status);
    }
}

/* Main BAR I/O callback function for control registers */
uint64_t bar_io_callback(const mango_sdd_bar_io_t *io_event, void *user_data) {
    uint64_t value = 0;

    if (io_event->is_write) {
        /* Handle write operation */
        printf("BAR Write: BAR %u, Offset 0x%lx, Value 0x%lx, Size %u\n",
               io_event->bar_idx, io_event->offset, io_event->value,
               1 << io_event->size);

        /* Process different register writes */
        switch (io_event->offset) {
            case VIRTIO_PCI_COMMON_GF:
                printf("Driver features write: 0x%lx\n", io_event->value);
                break;

            case VIRTIO_PCI_COMMON_Q_SELECT:
                printf("Queue selection write: %lu\n", io_event->value);
                break;

            case VIRTIO_PCI_COMMON_STATUS:
                printf("Device status write: 0x%lx\n", io_event->value);
                break;

            default:
                printf("Unhandled register write at offset 0x%lx\n", io_event->offset);
                break;
        }
        /* Return value is ignored for writes */
        return 0;
    } else {
        /* Handle read operation */
        printf("BAR Read: BAR %u, Offset 0x%lx, Size %u\n",
               io_event->bar_idx, io_event->offset,
               1 << io_event->size);

        /* Process different register reads */
        switch (io_event->offset) {
            case VIRTIO_PCI_COMMON_DF:
                printf("Reading device features\n");
                /* memcpy features to value */
                break;

            case VIRTIO_PCI_COMMON_STATUS:
                printf("Reading device status\n");
                /* memcpy status to value */
                break;

            default:
                printf("Unhandled register read at offset 0x%lx\n", io_event->offset);
                break;
        }

        printf("Returning value: 0x%lx\n", value);
        return value;
    }
}

/* Doorbell callback */
uint64_t doorbell_callback(const mango_sdd_bar_io_t *io_event, void *user_data) {
    if (io_event->is_write) {
        /* Handle doorbell write (queue notification) */
        printf("DOORBELL Write: Offset 0x%lx, Value 0x%lx\n",
               io_event->offset, io_event->value);

        /* Extract queue index from doorbell write */
        uint32_t queue_idx = (uint32_t)(io_event->offset / sizeof(uint32_t));

        /* Handle queue notification */
        printf("Queue %u notification received\n", queue_idx);

        /*
         * Process queue operations:
         * 1. Check if the queue has pending requests
         * 2. Process requests
         * 3. Send completion notifications when done
         *
         * Once request processing is complete, call req_complete:
         * req_complete(g_dev_handle, queue_id);
         */
    } else {
        /* Handle doorbell read (typically not used) */
        printf("DOORBELL Read: Offset 0x%lx\n", io_event->offset);
    }

    return 0; /* Return value ignored for doorbells */
}

int main(int argc, char *argv[]) {
    mango_sdd_dev_h dev_handle;

    /* Step 1: Create an emulated device */
    dev_handle = mango_sdd_dev_open("virtio-blk0");

    /* Step 2: Configure the device with Red Hat vendor ID and virtio-blk device ID */
    mango_sdd_dev_set_vendor_id(dev_handle, VIRTIO_VENDOR_ID);
    mango_sdd_dev_set_device_id(dev_handle, VIRTIO_BLK_DEV_ID);

    /* Step 3: Configure BAR 0 for control registers */
    mango_sdd_configure_bar(dev_handle, 0, 4096,
                           MANGO_SDD_BAR_TYPE_REGULAR, 0);

    /* Step 4: Configure BAR 1 for doorbell notifications */
    mango_sdd_configure_bar(dev_handle, 1, 4096,
                           MANGO_SDD_BAR_TYPE_DOORBELL, 0);

    /* Step 5: Configure BAR 2 for MSI-X table */
    mango_sdd_configure_bar(dev_handle, 2, 4096,
                           MANGO_SDD_BAR_TYPE_MSIX, 0);

    /* Step 6: Configure BAR 3 for MSI-X PBA (Pending Bit Array) */
    mango_sdd_configure_bar(dev_handle, 3, 4096,
                           MANGO_SDD_BAR_TYPE_MSIX_PBA, 0);

    /* Step 7: Register callback for main control registers (BAR 0) */
    mango_sdd_register_bar_io_cb(dev_handle, 0, bar_io_callback, NULL);

    /* Step 8: Register separate callback for doorbell operations (BAR 1) */
    mango_sdd_register_bar_io_cb(dev_handle, 1, doorbell_callback, NULL);

    /* Configure device with 16 MSI-X interrupts */
    mango_sdd_dev_set_msix_vectors(dev_handle, 16);

    /* Step 9: Start the device */
    mango_sdd_start(dev_handle);

    ...

    /* Step 8: Teardown the device */
    mango_sdd_stop(dev_handle);
    mango_sdd_dev_close(dev_handle);

    return 0;
}

Datatype

`mango_sdd_dev_h`

typedef void * mango_sdd_dev_h;

A handle of mango software-defined device.

`mango_sdd_queue_h`

typedef void * mango_sdd_queue_h;

A handle of mango software-defined device queue.

`mango_sdd_dev_state_e`

typedef enum _mango_sdd_dev_state_e mango_sdd_dev_state_e;

Represents the possible states of an emulated device.

Values

`MANGO_SDD_DEV_STATE_INITIALIZING`	The device is being initialized and configured
`MANGO_SDD_DEV_STATE_RUNNING`	The device is running and can accept requests
`MANGO_SDD_DEV_STATE_STOPPING`	The device is in the process of stopping, pending the completion of outstanding operations
`MANGO_SDD_DEV_STATE_STOPPED`	The device is stopped and cannot process any more requests

`mango_sdd_bar_type_e`

typedef enum _mango_sdd_bar_type_e mango_sdd_bar_type_e;

Represents the possible types of BAR memory.

Values

`MANGO_SDD_BAR_TYPE_MEMORY`	Standard memory BAR. Direct memory access for the device
`MANGO_SDD_BAR_TYPE_IO`	I/O space BAR
`MANGO_SDD_BAR_TYPE_MSIX`	MSI-X table memory. BAR memory that contains the MSI-X interrupt table
`MANGO_SDD_BAR_TYPE_MSIX_PBA`	MSI-X Pending Bit Array memory

`mango_sdd_bar_write_size_e`

typedef enum _mango_bar_write_size mango_sdd_bar_write_size_e;

Defines the size of a BAR write.

Values

`MANGO_SDD_BAR_WRITE_8BIT`	An 8-bit (1 byte) write
`MANGO_SDD_BAR_WRITE_16BIT`	A 16-bit (2 byte) write
`MANGO_SDD_BAR_WRITE_32BIT`	A 32-bit (4 byte) write
`MANGO_SDD_BAR_WRITE_64BIT`	A 64-bit (8 byte) write

`mango_sdd_bar_io_t`

typedef struct _mango_sdd_bar_io mango_sdd_bar_io_t;

A structure representing a BAR I/O operation (read or write).

Members

`is_write`	true for write operations, false for read operations
`bar_idx`	The index of the BAR on which the operation occurred
`offset`	The offset within the BAR where the operation occurred
`value`	For writes: The value that was written. For reads: not used
`size`	The size of the operation

`mango_sdd_bar_io_cb`

typedef uint64_t(*)(const mango_sdd_bar_io_t *io_event, void *user_data) mango_sdd_bar_io_cb;

The function pointer type for a unified BAR I/O callback. The callback can handle both read and write operations based on the is_write flag. For read operations (is_write=false), the return value is used as the value returned to the host. For write operations (is_write=true), the return value is ignored.

`mango_sdd_device_reset_type_e`

typedef enum _mango_device_reset_type mango_sdd_device_reset_type_e;

Defines the types of device resets.

Values

MANGO_SDD_DEV_RESET_FLR Function Level Reset

`mango_sdd_device_reset_cb`

typedef void(*)(mango_sdd_dev_h dev_handle, mango_sdd_device_reset_type_e reset_type, void *user_data) mango_sdd_device_reset_cb;

The function pointer type for a device reset callback.

`mango_sdd_queue_direction_e`

typedef enum _mango_sdd_queue_direction mango_sdd_queue_direction_e;

Defines the direction of data flow for a queue.

Values

`MANGO_SDD_QUEUE_DIRECTION_H2C`	Host-to-Card direction. Data flows from host memory to the DPU
`MANGO_SDD_QUEUE_DIRECTION_C2H`	Card-to-Host direction. Data flows from the DPU to host memory

`mango_sdd_queue_type_e`

typedef enum _mango_sdd_queue_type mango_sdd_queue_type_e;

Defines the type and behavior of a queue.

Values

`MANGO_SDD_QUEUE_TYPE_RING`	Ring buffer queue. A circular buffer with head and tail pointers
`MANGO_SDD_QUEUE_TYPE_SUBMISSION`	Submission queue. Optimized for receiving work items from the host
`MANGO_SDD_QUEUE_TYPE_COMPLETION`	Completion queue. Optimized for sending completion notifications to the host

`mango_sdd_queue_state_e`

typedef enum _mango_sdd_queue_state mango_sdd_queue_state_e;

Represents the possible states of a queue.

Values

`MANGO_SDD_QUEUE_STATE_CREATED`	Queue is created and ready for operations but not yet started
`MANGO_SDD_QUEUE_STATE_ACTIVE`	Queue is actively processing data transfers
`MANGO_SDD_QUEUE_STATE_STOPPED`	Queue is stopped and not processing transfers

`mango_sdd_queue_item_cb`

typedef void(*)(mango_sdd_queue_h queue_handle, uint32_t index, uint64_t dpu_addr, int result, void *user_data) mango_sdd_queue_item_cb;

The function pointer type for a queue item callback.

`mango_sdd_queue_config_t`

typedef struct _mango_sdd_queue_config mango_sdd_queue_config_t;

Configuration structure for creating a queue.

Members

`host_addr`	Host memory address for the queue
`dpu_addr`	DPU memory address for the queue
`queue_size`	Number of elements the queue can hold
`element_size`	Size of each queue element in bytes
`direction`	Direction of data flow for this queue
`type`	Type and behavior of the queue
`alignment`	Required memory alignment in bytes
`callback`	Callback function to be invoked after DMA operations complete
`user_data`	User-defined pointer passed to every callback invocation
`flags`	Additional queue configuration flags

Functions

`mango_sdd_create_device`

mango_status_e mango_sdd_create_device(const char *name,
                                       mango_sdd_dev_h *dev_handle)

Creates a new emulated device.

Parameters

in name Name of the device to create.
out dev_handle Pointer to store the handle to the created device.