Version: 1.0.0

RDMA PCC

Programmable Congestion Control (PCC) APIs to configure user algorithms and parameters for a RDMA device.

Example

Here is an example of how to use the PCC APIs.

example_pcc.cc
#include <libmango.h>

typedef strict {
  ...
} my_algo_params;

static const my_algo_params initial_params = {
  ...
};

void
my_algo (void *params)
{
  my_algo_params *p = (my_algo_params *) params;
  unsigned int cnp = mango_pcc_event_cnp_cnt ();
  unsigned int cwnd = mango_pcc_window_current ();
  unsigned int nwnd;

  // calculate new window size
  nwnd = ...

  mango_pcc_window_update (nwnd);

  ...
}

int main() {
  // Create a PCC handle
  mango_pcc_h pcc;
  mango_pcc_create ("roce0", &pcc);

  // Configure PCC parameters
  mango_pcc_config_t config;
  config.option = MANGO_PCC_CONFIG_INTERVAL;
  config.value.interval_us = 60;
  mango_pcc_ctrl_config (pcc, &config);

  // Register PCC algorithm
  mango_pcc_ctrl_reg_algo (pcc, 1, algo, &initial_params, sizeof (initial_params));

  // Run PCC engine with 4 threads
  mango_pcc_ctrl_run (pcc, 4);
  ...

  // Stop PCC engine
  mango_pcc_ctrl_stop (pcc);
  mango_pcc_destroy (pcc);

  return 0;
}

Datatypes

`mango_pcc_algo_event_fn`

typedef mango_pcc_result_t(void *params, const mango_pcc_context_t *ctx) mango_pcc_algo_event_fn;

Event-style algorithm function signature. The algorithm receives a snapshot context and returns the desired outcome. All HW I/O is handled by the framework.

`mango_pcc_algo_fn`

typedef void(void *) mango_pcc_algo_fn;

User-provided CC algorithm function.

`mango_pcc_algo_plugin_t`

typedef struct _mango_pcc_algo_plugin mango_pcc_algo_plugin_t;

Plugin structure for PCC algorithm.

Members

`name`	Algorithm name
`desc`	Algorithm description
`init_params`	Init algorithm parameters
`destroy_params`	Destroy parameter structure
`algo_fn`	Legacy algorithm function (kept for ABI compat)
`algo_event_fn`	Event-style algorithm function (NULL for legacy)
`required_metrics`	mango_pcc_metric_flags_e bitmask

`mango_pcc_config_option_e`

typedef enum _mango_pcc_config_option mango_pcc_config_option_e;

PCC configuration options.

Values

MANGO_PCC_CONFIG_INTERVAL Algorithm execution interval

`mango_pcc_config_t`

typedef struct _mango_pcc_config mango_pcc_config_t;

PCC configuration value type.

Members

`option`	PCC configuration option
`interval_us`	microseconds

`mango_pcc_context_t`

typedef struct _mango_pcc_context mango_pcc_context_t;

sizeof is fixed at 64 bytes. Future fields replace _reserved bytes; a value of 0 in any reserved slot always means "not provided / no-op".

Members

`current_window`	Current QP congestion window
`cnp_delta`	CNP increase since last call (0 if METRIC_CNP not set)
`latest_rtt_ns`	Most recent RTT in nanoseconds (0 if unavailable)
`rtt_updated`	True if a new RTT measurement arrived this cycle
`active_qp_count`	Number of active QPs on the device (NEW)
`_reserved`	Reserved for future fields; always zero from libmango

`mango_pcc_event_e`

typedef enum _mango_pcc_event mango_pcc_event_e;

PCC event types for mango_pcc_event_get_val()

`mango_pcc_h`

typedef void * mango_pcc_h;

A handle of mango pcc.

`mango_pcc_metric_flags_e`

typedef enum _mango_pcc_metric_flags mango_pcc_metric_flags_e;

Metric selection bitmask for mango_pcc_ctrl_reg_algo_event() . Declare only the metrics your algorithm needs; unselected metrics are not polled.

`mango_pcc_param_t`

typedef struct _mango_pcc_param mango_pcc_param_t;

Single parameter descriptor for mango_pcc_param_register() . User fills an array of these (with offsetof(struct_type, field)) and passes it to mango_pcc_param_register() . Use the REGISTER_*_PARAM() macros below.

Members

`name`	Parameter name
`type`	Parameter type
`offset`	Offset of field in user's param struct

`mango_pcc_param_type_e`

typedef enum _mango_pcc_param_type mango_pcc_param_type_e;

PCC parameter types for mango_pcc_param_register() and mango_pcc_walk_params()

`mango_pcc_param_walk_fn`

typedef void(*)(const char *name, mango_pcc_param_type_e type, void *value_ptr, void *user_data) mango_pcc_param_walk_fn;

name Parameter name. type Parameter type. value_ptr Pointer to the parameter value. user_data User-provided data pointer.

`mango_pcc_result_t`

typedef struct _mango_pcc_result mango_pcc_result_t;

sizeof is fixed at 32 bytes. Future fields replace _reserved bytes; a value of 0 in any reserved slot always means "no action".

Members

`new_window`	New congestion window (set to current_window to leave unchanged)
`request_rtt_probe`	Request the framework to send an RTT probe
`_reserved`	Reserved for future fields; plugin must leave as zero

Functions

`mango_pcc_active_qp_list`

mango_status_e mango_pcc_active_qp_list(mango_pcc_h pcc,
                                        uint32_t *qp_list,
                                        uint32_t *qp_count)

Get the list of active QPs.

Parameters

in pcc The PCC handle.
out qp_list Buffer to store active QP numbers (must be at least MAX_QPS in size).
out qp_count Pointer to store the number of active QPs.

Returns 0 on success, Otherwise, a negative error value.

`mango_pcc_cnp_count_by_qp`

mango_status_e mango_pcc_cnp_count_by_qp(mango_pcc_h pcc,
                                         uint32_t qp_num,
                                         uint64_t *cnp_count)

Get the CNP count (increase since baseline) for a specific QP. Baseline is set when the QP first appears in the active list, or for all QPs when PCC Run() is called (so the count reflects "this Run" only).

Parameters

in pcc The PCC handle.
in qp_num The QP number to query.
out cnp_count Pointer to store the CNP count (increase since baseline).

Returns 0 on success, Otherwise, a negative error value.

`mango_pcc_create`

mango_status_e mango_pcc_create(const char *roce_dev_name, mango_pcc_h *pcc)

Create a PCC handle.

Parameters

in roce_dev_name Name of the roce device.
out pcc Pointer to receive the newly created PCC handle.

Returns 0 on success, Otherwise, a negative error value.

`mango_pcc_ctrl_apply_algo`

mango_status_e mango_pcc_ctrl_apply_algo(mango_pcc_h pcc,
                                         unsigned int slot,
                                         const unsigned int *qp_list,
                                         unsigned int qp_count)

Apply a PCC algorithm slot to specified QPs or all QPs.

Parameters

in pcc The PCC handle.
in slot The slot index to apply.
in qp_list Array of QP numbers to apply the algorithm to. If NULL, applies to all QPs.
in qp_count Number of QPs in qp_list. Ignored if qp_list is NULL.

Returns 0 on success, Otherwise, a negative error value.

`mango_pcc_ctrl_config`

mango_status_e mango_pcc_ctrl_config(mango_pcc_h pcc,
                                     const mango_pcc_config_t *config)

Set a PCC configuration value.

Parameters

in pcc The PCC handle.
in config The configuration value.

Returns 0 on success, Otherwise, a negative error value.

`mango_pcc_ctrl_reg_algo`

mango_status_e mango_pcc_ctrl_reg_algo(mango_pcc_h pcc,
                                       unsigned int slot,
                                       mango_pcc_algo_fn *algo,
                                       const void *initial_params,
                                       size_t param_size)

Parameters

in pcc The PCC handle.
in slot The slot index at which to register the algorithm.
in algo Pointer to the algorithm function to be invoked.
in initial_params Pointer to the algorithm parameters with the initial values.
in param_size The size of the parameter structure.

Returns 0 on success, Otherwise, a negative error value.

`mango_pcc_ctrl_reg_algo_event`

mango_status_e mango_pcc_ctrl_reg_algo_event(mango_pcc_h pcc,
                                             unsigned int slot,
                                             mango_pcc_algo_event_fn *algo,
                                             uint32_t required_metrics,
                                             const void *initial_params,
                                             size_t param_size)

Register an event-style PCC algorithm. The algorithm receives a mango_pcc_context_t snapshot and returns a mango_pcc_result_t. All HW I/O is handled by the framework; only metrics declared in required_metrics are polled.

Parameters

in pcc The PCC handle.
in slot Algorithm slot index.
in algo Event-style algorithm function pointer.
in required_metrics Bitmask of mango_pcc_metric_flags_e values.
in initial_params Pointer to initial parameter values.
in param_size Size of the parameter structure.

Returns 0 on success, Otherwise, a negative error value.

`mango_pcc_ctrl_run`

mango_status_e mango_pcc_ctrl_run(mango_pcc_h pcc, unsigned int num_threads)

Start the PCC control loop with a given number of threads.

Parameters

in pcc The PCC handle.
in num_threads Number of poller threads to spawn.

Returns 0 on success, Otherwise, a negative error value.

`mango_pcc_ctrl_stop`

mango_status_e mango_pcc_ctrl_stop(mango_pcc_h pcc)

Stop the PCC control loop.

Parameters

in pcc The PCC handle.

Returns 0 on success, Otherwise, a negative error value.

`mango_pcc_ctrl_unreg_algo`

mango_status_e mango_pcc_ctrl_unreg_algo(mango_pcc_h pcc, unsigned int slot)

Unregister a PCC algorithm from a specific slot.

Parameters

in pcc The PCC handle.
in slot The slot index to unregister.

Returns 0 on success, Otherwise, a negative error value.

`mango_pcc_destroy`

mango_status_e mango_pcc_destroy(mango_pcc_h pcc)

Destroy a PCC handle.

Parameters

in pcc The PCC handle to destroy.

Returns 0 on success, Otherwise, a negative error value.

`mango_pcc_event_get_val`

unsigned int mango_pcc_event_get_val(mango_pcc_event_e event)

Get the value of a PCC event by type.

Parameters

in event The PCC event type.

Returns The event value depending on type. 0 if not available.

note

This API must only be invoked internally by the PCC algorithm.

`mango_pcc_event_request`

mango_status_e mango_pcc_event_request(mango_pcc_event_e event)

Request a PCC event-specific action to the current executing QP.

Parameters

in event The PCC event type.

Returns 0 on success, or a negative error code on failure.

`mango_pcc_get_context`

mango_context_h mango_pcc_get_context(mango_pcc_h pcc)

Get the context handle from a pcc handle.

Parameters

in pcc The PCC handle.

Returns the context handle on success, Otherwise, nullptr.

note

DO NOT CALL mango_context_destroy with the result; it will be destroyed when mango_pcc_destroy is called.

`mango_pcc_log`

void mango_pcc_log(mango_log_level_e level, const char *fmt,...)

Print a log message with Poller index and QP number.

Parameters

in level The log level.
in fmt The format string.

note

The message will be ignore if the system log level is lower.

`mango_pcc_param_register`

mango_status_e mango_pcc_param_register(mango_pcc_h pcc,
                                        const unsigned int slot,
                                        const char *name,
                                        mango_pcc_param_type_e type,
                                        size_t offset)

Parameters

in pcc The PCC handle.
in slot The slot index at which to register the algorithm parameter.
in name The name to be registered as a parameter.
in type The data to be registered as a parameter.
in offset The offset of parameter to get registered value.

Returns 0 on success, Otherwise, a negative error value.

note

If a parameter with the same name is already registered for this slot, the existing registration is preserved and MANGO_STATUS_ALREADY_OPENED is returned.

`mango_pcc_params_register`

mango_status_e mango_pcc_params_register(mango_pcc_h pcc,
                                         unsigned int slot,
                                         const mango_pcc_param_t *params,
                                         unsigned int nr_params)

Parameters

in pcc The PCC handle.
in slot The slot index at which to register the algorithm parameter.
in params Array of parameter descriptors (name, type, offset into user struct).
in nr_params Number of elements in params .

Returns 0 on success, Otherwise, a negative error value.

note

Offsets must match the user's parameter structure (e.g. offsetof(my_struct, field)).

`mango_pcc_walk_params`

mango_status_e mango_pcc_walk_params(mango_pcc_h pcc,
                                     const unsigned int slot,
                                     mango_pcc_param_walk_fn callback,
                                     void *user_data,
                                     unsigned int mode)

Walk all parameters from a PCC algorithm slot using a callback.

Parameters

in pcc The PCC handle.
in slot The slot index.
in callback Callback function to be called for each parameter.
in user_data User-provided data pointer passed to callback.
in mode Walk mode: 0 for read mode (iterate only first QP, read-only), 1 for write mode (iterate all QPs for updates). Other values default to read mode.

Returns 0 on success, Otherwise, a negative error value.

`mango_pcc_window_count_by_qp`

mango_status_e mango_pcc_window_count_by_qp(mango_pcc_h pcc,
                                            uint32_t qp_num,
                                            uint32_t *count)

Get the number of times SetWindow has been called for a specific QP.

Parameters

in pcc The PCC handle.
in qp_num The QP number to query.
out count Pointer to store the SetWindow call count.

Returns 0 on success, Otherwise, a negative error value.

note

The count is reset to 0 when a different algorithm slot is applied to the QP.

`mango_pcc_window_current`

unsigned int mango_pcc_window_current(void)

Get the current PCC window size of the QP.

Returns The current window size.

note

This API must only be invoked internally by the PCC algorithm.

`mango_pcc_window_current_by_qp`

mango_status_e mango_pcc_window_current_by_qp(mango_pcc_h pcc,
                                              uint32_t qp_num,
                                              uint32_t *window_size)

Get the current PCC window size of a specific QP.

Parameters

in pcc The PCC handle.
in qp_num The QP number to query.
out window_size Pointer to store the window size.

Returns 0 on success, Otherwise, a negative error value.

`mango_pcc_window_update`

void mango_pcc_window_update(unsigned int new_window)

Update the PCC window size of the QP.

Parameters

in new_window The new window size to set.

Example​

Datatypes​

mango_pcc_algo_event_fn​

mango_pcc_algo_fn​

mango_pcc_algo_plugin_t​

mango_pcc_config_option_e​

mango_pcc_config_t​

mango_pcc_context_t​

mango_pcc_event_e​

mango_pcc_h​

mango_pcc_metric_flags_e​

mango_pcc_param_t​

mango_pcc_param_type_e​

mango_pcc_param_walk_fn​

mango_pcc_result_t​

Functions​

mango_pcc_active_qp_list​

mango_pcc_cnp_count_by_qp​

mango_pcc_create​

mango_pcc_ctrl_apply_algo​

mango_pcc_ctrl_config​

mango_pcc_ctrl_reg_algo​

mango_pcc_ctrl_reg_algo_event​

mango_pcc_ctrl_run​

mango_pcc_ctrl_stop​

mango_pcc_ctrl_unreg_algo​

mango_pcc_destroy​

mango_pcc_event_get_val​

mango_pcc_event_request​

mango_pcc_get_context​

mango_pcc_log​

mango_pcc_param_register​

mango_pcc_params_register​

mango_pcc_walk_params​

mango_pcc_window_count_by_qp​

mango_pcc_window_current​

mango_pcc_window_current_by_qp​

mango_pcc_window_update​

Example

Datatypes

`mango_pcc_algo_event_fn`

`mango_pcc_algo_fn`

`mango_pcc_algo_plugin_t`

`mango_pcc_config_option_e`

`mango_pcc_config_t`

`mango_pcc_context_t`

`mango_pcc_event_e`

`mango_pcc_h`

`mango_pcc_metric_flags_e`

`mango_pcc_param_t`

`mango_pcc_param_type_e`

`mango_pcc_param_walk_fn`

`mango_pcc_result_t`

Functions

`mango_pcc_active_qp_list`

`mango_pcc_cnp_count_by_qp`

`mango_pcc_create`

`mango_pcc_ctrl_apply_algo`

`mango_pcc_ctrl_config`

`mango_pcc_ctrl_reg_algo`

`mango_pcc_ctrl_reg_algo_event`

`mango_pcc_ctrl_run`

`mango_pcc_ctrl_stop`

`mango_pcc_ctrl_unreg_algo`

`mango_pcc_destroy`

`mango_pcc_event_get_val`

`mango_pcc_event_request`

`mango_pcc_get_context`

`mango_pcc_log`

`mango_pcc_param_register`

`mango_pcc_params_register`

`mango_pcc_walk_params`

`mango_pcc_window_count_by_qp`

`mango_pcc_window_current`

`mango_pcc_window_current_by_qp`

`mango_pcc_window_update`