DPDK logo

Elixir Cross Referencer

  1
  2
  3
  4
  5
  6
  7
  8
  9
 10
 11
 12
 13
 14
 15
 16
 17
 18
 19
 20
 21
 22
 23
 24
 25
 26
 27
 28
 29
 30
 31
 32
 33
 34
 35
 36
 37
 38
 39
 40
 41
 42
 43
 44
 45
 46
 47
 48
 49
 50
 51
 52
 53
 54
 55
 56
 57
 58
 59
 60
 61
 62
 63
 64
 65
 66
 67
 68
 69
 70
 71
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
/* SPDX-License-Identifier: BSD-3-Clause
 * Copyright 2019 Mellanox Technologies, Ltd
 */

#ifndef RTE_PMD_MLX5_VDPA_H_
#define RTE_PMD_MLX5_VDPA_H_

#include <linux/virtio_net.h>
#include <sys/queue.h>

#ifdef PEDANTIC
#pragma GCC diagnostic ignored "-Wpedantic"
#endif
#include <rte_vdpa.h>
#include <rte_vdpa_dev.h>
#include <rte_vhost.h>
#ifdef PEDANTIC
#pragma GCC diagnostic error "-Wpedantic"
#endif
#include <rte_spinlock.h>
#include <rte_interrupts.h>

#include <mlx5_glue.h>
#include <mlx5_devx_cmds.h>
#include <mlx5_prm.h>


#define MLX5_VDPA_INTR_RETRIES 256
#define MLX5_VDPA_INTR_RETRIES_USEC 1000

#ifndef VIRTIO_F_ORDER_PLATFORM
#define VIRTIO_F_ORDER_PLATFORM 36
#endif

#ifndef VIRTIO_F_RING_PACKED
#define VIRTIO_F_RING_PACKED 34
#endif

#define MLX5_VDPA_DEFAULT_TIMER_DELAY_US 100u
#define MLX5_VDPA_DEFAULT_TIMER_STEP_US 1u

struct mlx5_vdpa_cq {
	uint16_t log_desc_n;
	uint32_t cq_ci:24;
	uint32_t arm_sn:2;
	uint32_t armed:1;
	int callfd;
	rte_spinlock_t sl;
	struct mlx5_devx_obj *cq;
	struct mlx5dv_devx_umem *umem_obj;
	union {
		volatile void *umem_buf;
		volatile struct mlx5_cqe *cqes;
	};
	volatile uint32_t *db_rec;
	uint64_t errors;
};

struct mlx5_vdpa_event_qp {
	struct mlx5_vdpa_cq cq;
	struct mlx5_devx_obj *fw_qp;
	struct mlx5_devx_obj *sw_qp;
	struct mlx5dv_devx_umem *umem_obj;
	void *umem_buf;
	volatile uint32_t *db_rec;
};

struct mlx5_vdpa_query_mr {
	SLIST_ENTRY(mlx5_vdpa_query_mr) next;
	void *addr;
	uint64_t length;
	struct mlx5dv_devx_umem *umem;
	struct mlx5_devx_obj *mkey;
	int is_indirect;
};

enum {
	MLX5_VDPA_NOTIFIER_STATE_DISABLED,
	MLX5_VDPA_NOTIFIER_STATE_ENABLED,
	MLX5_VDPA_NOTIFIER_STATE_ERR
};

struct mlx5_vdpa_virtq {
	SLIST_ENTRY(mlx5_vdpa_virtq) next;
	uint8_t enable;
	uint16_t index;
	uint16_t vq_size;
	uint8_t notifier_state;
	bool stopped;
	struct mlx5_vdpa_priv *priv;
	struct mlx5_devx_obj *virtq;
	struct mlx5_devx_obj *counters;
	struct mlx5_vdpa_event_qp eqp;
	struct {
		struct mlx5dv_devx_umem *obj;
		void *buf;
		uint32_t size;
	} umems[3];
	struct rte_intr_handle intr_handle;
	struct mlx5_devx_virtio_q_couners_attr reset;
};

struct mlx5_vdpa_steer {
	struct mlx5_devx_obj *rqt;
	void *domain;
	void *tbl;
	struct {
		struct mlx5dv_flow_matcher *matcher;
		struct mlx5_devx_obj *tir;
		void *tir_action;
		void *flow;
	} rss[7];
};

enum {
	MLX5_VDPA_EVENT_MODE_DYNAMIC_TIMER,
	MLX5_VDPA_EVENT_MODE_FIXED_TIMER,
	MLX5_VDPA_EVENT_MODE_ONLY_INTERRUPT
};

struct mlx5_vdpa_priv {
	TAILQ_ENTRY(mlx5_vdpa_priv) next;
	uint8_t configured;
	pthread_mutex_t vq_config_lock;
	uint64_t last_traffic_tic;
	pthread_t timer_tid;
	pthread_mutex_t timer_lock;
	pthread_cond_t timer_cond;
	volatile uint8_t timer_on;
	int event_mode;
	uint32_t event_us;
	uint32_t timer_delay_us;
	uint32_t no_traffic_time_s;
	struct rte_vdpa_device *vdev; /* vDPA device. */
	int vid; /* vhost device id. */
	struct ibv_context *ctx; /* Device context. */
	struct rte_pci_device *pci_dev;
	struct mlx5_hca_vdpa_attr caps;
	uint32_t pdn; /* Protection Domain number. */
	struct ibv_pd *pd;
	uint32_t gpa_mkey_index;
	struct ibv_mr *null_mr;
	struct rte_vhost_memory *vmem;
	uint32_t eqn;
	struct mlx5dv_devx_event_channel *eventc;
	struct mlx5dv_devx_uar *uar;
	struct rte_intr_handle intr_handle;
	struct mlx5_devx_obj *td;
	struct mlx5_devx_obj *tis;
	uint16_t nr_virtqs;
	uint64_t features; /* Negotiated features. */
	uint16_t log_max_rqt_size;
	struct mlx5_vdpa_steer steer;
	struct mlx5dv_var *var;
	void *virtq_db_addr;
	SLIST_HEAD(mr_list, mlx5_vdpa_query_mr) mr_list;
	struct mlx5_vdpa_virtq virtqs[];
};

enum {
	MLX5_VDPA_STATS_RECEIVED_DESCRIPTORS,
	MLX5_VDPA_STATS_COMPLETED_DESCRIPTORS,
	MLX5_VDPA_STATS_BAD_DESCRIPTOR_ERRORS,
	MLX5_VDPA_STATS_EXCEED_MAX_CHAIN,
	MLX5_VDPA_STATS_INVALID_BUFFER,
	MLX5_VDPA_STATS_COMPLETION_ERRORS,
	MLX5_VDPA_STATS_MAX
};

/*
 * Check whether virtq is for traffic receive.
 * According to VIRTIO_NET Spec the virtqueues index identity its type by:
 * 0 receiveq1
 * 1 transmitq1
 * ...
 * 2(N-1) receiveqN
 * 2(N-1)+1 transmitqN
 * 2N controlq
 */
static inline uint8_t
is_virtq_recvq(int virtq_index, int nr_vring)
{
	if (virtq_index % 2 == 0 && virtq_index != nr_vring - 1)
		return 1;
	return 0;
}

/**
 * Release all the prepared memory regions and all their related resources.
 *
 * @param[in] priv
 *   The vdpa driver private structure.
 */
void mlx5_vdpa_mem_dereg(struct mlx5_vdpa_priv *priv);

/**
 * Register all the memory regions of the virtio device to the HW and allocate
 * all their related resources.
 *
 * @param[in] priv
 *   The vdpa driver private structure.
 *
 * @return
 *   0 on success, a negative errno value otherwise and rte_errno is set.
 */
int mlx5_vdpa_mem_register(struct mlx5_vdpa_priv *priv);


/**
 * Create an event QP and all its related resources.
 *
 * @param[in] priv
 *   The vdpa driver private structure.
 * @param[in] desc_n
 *   Number of descriptors.
 * @param[in] callfd
 *   The guest notification file descriptor.
 * @param[in/out] eqp
 *   Pointer to the event QP structure.
 *
 * @return
 *   0 on success, -1 otherwise and rte_errno is set.
 */
int mlx5_vdpa_event_qp_create(struct mlx5_vdpa_priv *priv, uint16_t desc_n,
			      int callfd, struct mlx5_vdpa_event_qp *eqp);

/**
 * Destroy an event QP and all its related resources.
 *
 * @param[in/out] eqp
 *   Pointer to the event QP structure.
 */
void mlx5_vdpa_event_qp_destroy(struct mlx5_vdpa_event_qp *eqp);

/**
 * Release all the event global resources.
 *
 * @param[in] priv
 *   The vdpa driver private structure.
 */
void mlx5_vdpa_event_qp_global_release(struct mlx5_vdpa_priv *priv);

/**
 * Setup CQE event.
 *
 * @param[in] priv
 *   The vdpa driver private structure.
 *
 * @return
 *   0 on success, a negative errno value otherwise and rte_errno is set.
 */
int mlx5_vdpa_cqe_event_setup(struct mlx5_vdpa_priv *priv);

/**
 * Unset CQE event .
 *
 * @param[in] priv
 *   The vdpa driver private structure.
 */
void mlx5_vdpa_cqe_event_unset(struct mlx5_vdpa_priv *priv);

/**
 * Release a virtq and all its related resources.
 *
 * @param[in] priv
 *   The vdpa driver private structure.
 */
void mlx5_vdpa_virtqs_release(struct mlx5_vdpa_priv *priv);

/**
 * Create all the HW virtqs resources and all their related resources.
 *
 * @param[in] priv
 *   The vdpa driver private structure.
 *
 * @return
 *   0 on success, a negative errno value otherwise and rte_errno is set.
 */
int mlx5_vdpa_virtqs_prepare(struct mlx5_vdpa_priv *priv);

/**
 * Enable\Disable virtq..
 *
 * @param[in] priv
 *   The vdpa driver private structure.
 * @param[in] index
 *   The virtq index.
 * @param[in] enable
 *   Set to enable, otherwise disable.
 *
 * @return
 *   0 on success, a negative value otherwise.
 */
int mlx5_vdpa_virtq_enable(struct mlx5_vdpa_priv *priv, int index, int enable);

/**
 * Unset steering and release all its related resources- stop traffic.
 *
 * @param[in] priv
 *   The vdpa driver private structure.
 */
void mlx5_vdpa_steer_unset(struct mlx5_vdpa_priv *priv);

/**
 * Update steering according to the received queues status.
 *
 * @param[in] priv
 *   The vdpa driver private structure.
 *
 * @return
 *   0 on success, a negative value otherwise.
 */
int mlx5_vdpa_steer_update(struct mlx5_vdpa_priv *priv);

/**
 * Setup steering and all its related resources to enable RSS traffic from the
 * device to all the Rx host queues.
 *
 * @param[in] priv
 *   The vdpa driver private structure.
 *
 * @return
 *   0 on success, a negative value otherwise.
 */
int mlx5_vdpa_steer_setup(struct mlx5_vdpa_priv *priv);

/**
 * Enable\Disable live migration logging.
 *
 * @param[in] priv
 *   The vdpa driver private structure.
 * @param[in] enable
 *   Set for enable, unset for disable.
 *
 * @return
 *   0 on success, a negative value otherwise.
 */
int mlx5_vdpa_logging_enable(struct mlx5_vdpa_priv *priv, int enable);

/**
 * Set dirty bitmap logging to allow live migration.
 *
 * @param[in] priv
 *   The vdpa driver private structure.
 * @param[in] log_base
 *   Vhost log base.
 * @param[in] log_size
 *   Vhost log size.
 *
 * @return
 *   0 on success, a negative value otherwise.
 */
int mlx5_vdpa_dirty_bitmap_set(struct mlx5_vdpa_priv *priv, uint64_t log_base,
			       uint64_t log_size);

/**
 * Log all virtqs information for live migration.
 *
 * @param[in] priv
 *   The vdpa driver private structure.
 * @param[in] enable
 *   Set for enable, unset for disable.
 *
 * @return
 *   0 on success, a negative value otherwise.
 */
int mlx5_vdpa_lm_log(struct mlx5_vdpa_priv *priv);

/**
 * Modify virtq state to be ready or suspend.
 *
 * @param[in] virtq
 *   The vdpa driver private virtq structure.
 * @param[in] state
 *   Set for ready, otherwise suspend.
 *
 * @return
 *   0 on success, a negative value otherwise.
 */
int mlx5_vdpa_virtq_modify(struct mlx5_vdpa_virtq *virtq, int state);

/**
 * Stop virtq before destroying it.
 *
 * @param[in] priv
 *   The vdpa driver private structure.
 * @param[in] index
 *   The virtq index.
 *
 * @return
 *   0 on success, a negative value otherwise.
 */
int mlx5_vdpa_virtq_stop(struct mlx5_vdpa_priv *priv, int index);

/**
 * Get virtq statistics.
 *
 * @param[in] priv
 *   The vdpa driver private structure.
 * @param[in] qid
 *   The virtq index.
 * @param stats
 *   The virtq statistics array to fill.
 * @param n
 *   The number of elements in @p stats array.
 *
 * @return
 *   A negative value on error, otherwise the number of entries filled in the
 *   @p stats array.
 */
int
mlx5_vdpa_virtq_stats_get(struct mlx5_vdpa_priv *priv, int qid,
			  struct rte_vdpa_stat *stats, unsigned int n);

/**
 * Reset virtq statistics.
 *
 * @param[in] priv
 *   The vdpa driver private structure.
 * @param[in] qid
 *   The virtq index.
 *
 * @return
 *   A negative value on error, otherwise 0.
 */
int
mlx5_vdpa_virtq_stats_reset(struct mlx5_vdpa_priv *priv, int qid);
#endif /* RTE_PMD_MLX5_VDPA_H_ */