2 // Copyright (c) 2010-2017 Intel Corporation
4 // Licensed under the Apache License, Version 2.0 (the "License");
5 // you may not use this file except in compliance with the License.
6 // You may obtain a copy of the License at
8 // http://www.apache.org/licenses/LICENSE-2.0
10 // Unless required by applicable law or agreed to in writing, software
11 // distributed under the License is distributed on an "AS IS" BASIS,
12 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 // See the License for the specific language governing permissions and
14 // limitations under the License.
18 * Non compatible implementation of RFC3686(CTR-AES 128 bit key), RFC4303 (tunnel ipv4 ESP)
20 * 1. Crypto not safe!!!!! (underlying AES-CTR implementation is OK, but ESP implementation is lousy)
21 * 2. Only ESP/tunnel/ipv4/AES-CTR
22 * 3. Not fully implemented
23 * 4. No proper key / SADB
24 * So performance demonstrator only
27 #include "task_init.h"
28 #include "task_base.h"
33 #include "prox_cksum.h"
36 #include <rte_cryptodev.h>
37 #include <rte_bus_vdev.h>
38 #include "prox_port_cfg.h"
39 #include "prox_compat.h"
41 typedef unsigned int u32;
42 typedef unsigned char u8;
44 #define BYTE_LENGTH(x) (x/8)
45 #define DIGEST_BYTE_LENGTH_SHA1 (BYTE_LENGTH(160))
47 //#define CIPHER_KEY_LENGTH_AES_CBC (32)
48 #define CIPHER_KEY_LENGTH_AES_CBC (16)//==TEST
49 #define CIPHER_IV_LENGTH_AES_CBC 16
51 #define MAXIMUM_IV_LENGTH 16
52 #define IV_OFFSET (sizeof(struct rte_crypto_op) + sizeof(struct rte_crypto_sym_op))
54 #define MAX_SESSIONS 1024
55 #define POOL_CACHE_SIZE 128
60 struct task_base base;
64 prox_rte_ether_addr local_mac;
66 prox_rte_ether_addr dst_mac;
67 struct rte_mempool *crypto_op_pool;
68 struct rte_mempool *session_pool;
69 struct rte_cryptodev_sym_session *sess;
70 struct rte_crypto_op *ops_burst[NUM_OPS];
74 struct task_base base;
78 prox_rte_ether_addr local_mac;
79 prox_rte_ether_addr dst_mac;
80 struct rte_mempool *crypto_op_pool;
81 struct rte_mempool *session_pool;
82 struct rte_cryptodev_sym_session *sess;
83 struct rte_crypto_op *ops_burst[NUM_OPS];
86 static uint8_t hmac_sha1_key[] = {
87 0xF8, 0x2A, 0xC7, 0x54, 0xDB, 0x96, 0x18, 0xAA,
88 0xC3, 0xA1, 0x53, 0xF6, 0x1F, 0x17, 0x60, 0xBD,
89 0xDE, 0xF4, 0xDE, 0xAD };
91 static uint8_t aes_cbc_key[] = {
92 0xE4, 0x23, 0x33, 0x8A, 0x35, 0x64, 0x61, 0xE2,
93 0x49, 0x03, 0xDD, 0xC6, 0xB8, 0xCA, 0x55, 0x7A,
94 0xE4, 0x23, 0x33, 0x8A, 0x35, 0x64, 0x61, 0xE2,
95 0x49, 0x03, 0xDD, 0xC6, 0xB8, 0xCA, 0x55, 0x7A };
97 static uint8_t aes_cbc_iv[] = {
98 0xE4, 0x23, 0x33, 0x8A, 0x35, 0x64, 0x61, 0xE2,
99 0x49, 0x03, 0xDD, 0xC6, 0xB8, 0xCA, 0x55, 0x7A };
101 static void printf_cdev_info(uint8_t cdev_id)
103 struct rte_cryptodev_info dev_info;
104 rte_cryptodev_info_get(cdev_id, &dev_info);
105 plog_info("!!!numdevs:%d\n", rte_cryptodev_count());
106 //uint16_t rte_cryptodev_queue_pair_count(uint8_t dev_id);
107 plog_info("dev:%d name:%s nb_queue_pairs:%d max_nb_sessions:%d\n",
108 cdev_id, dev_info.driver_name, dev_info.max_nb_queue_pairs, dev_info.sym.max_nb_sessions);
109 const struct rte_cryptodev_capabilities *cap = &dev_info.capabilities[0];
111 while (cap->op != RTE_CRYPTO_OP_TYPE_UNDEFINED) {
112 //plog_info("cap->sym.xform_type:%d,");
113 if (cap->sym.xform_type == RTE_CRYPTO_SYM_XFORM_CIPHER)
114 plog_info("RTE_CRYPTO_SYM_XFORM_CIPHER: %d\n", cap->sym.cipher.algo);
115 cap = &dev_info.capabilities[++i];
120 static uint8_t get_cdev_id(void)
122 //crypto devices must be configured in the config file
123 //eal=-b 0000:00:03.0 --vdev crypto_aesni_mb0 --vdev crypto_aesni_mb1
125 static uint8_t cdev_id=0;
126 PROX_PANIC(cdev_id+1 > rte_cryptodev_count(), "not enough crypto devices\n");
127 //eal=-b 0000:00:03.0 --vdev crypto_aesni_mb0 --vdev crypto_aesni_mb1
131 static uint8_t get_cdev_id(void)
133 static uint8_t cdev_id=0;
136 sprintf(name, "crypto_aesni_mb%d", cdev_id);
138 int cdev_id1 = rte_cryptodev_get_dev_id(name);
140 plog_info("crypto dev %d preconfigured\n", cdev_id1);
144 #if RTE_VERSION < RTE_VERSION_NUM(18,8,0,0)
145 int ret = rte_vdev_init(name, "max_nb_queue_pairs=8,max_nb_sessions=1024,socket_id=0");
147 int ret = rte_vdev_init(name, "max_nb_queue_pairs=8,socket_id=0");
149 PROX_PANIC(ret != 0, "Failed rte_vdev_init\n");
155 static void init_task_esp_enc(struct task_base *tbase, struct task_args *targ)
157 struct task_esp_enc *task = (struct task_esp_enc *)tbase;
159 tbase->flags |= FLAG_NEVER_FLUSH;
161 uint8_t lcore_id = targ->lconf->id;
163 sprintf(name, "core_%03u_crypto_pool", lcore_id);
164 task->crypto_op_pool = rte_crypto_op_pool_create(name, RTE_CRYPTO_OP_TYPE_SYMMETRIC,
165 targ->nb_mbuf, 128, MAXIMUM_IV_LENGTH, rte_socket_id());
166 PROX_PANIC(task->crypto_op_pool == NULL, "Can't create ENC CRYPTO_OP_POOL\n");
168 task->cdev_id = get_cdev_id();
170 struct rte_cryptodev_config cdev_conf;
171 cdev_conf.nb_queue_pairs = 2;
172 cdev_conf.socket_id = rte_socket_id();
173 rte_cryptodev_configure(task->cdev_id, &cdev_conf);
175 unsigned int session_size = rte_cryptodev_sym_get_private_session_size(task->cdev_id);
176 plog_info("rte_cryptodev_sym_get_private_session_size=%d\n", session_size);
177 sprintf(name, "core_%03u_session_pool", lcore_id);
178 task->session_pool = rte_cryptodev_sym_session_pool_create(name,
183 PROX_PANIC(task->session_pool == NULL, "Failed rte_mempool_create\n");
186 plog_info("enc: task->qp_id=%u\n", task->qp_id);
187 struct prox_rte_cryptodev_qp_conf qp_conf;
188 qp_conf.nb_descriptors = 128;
189 qp_conf.mp_session = task->session_pool;
190 prox_rte_cryptodev_queue_pair_setup(task->cdev_id, task->qp_id, &qp_conf, rte_cryptodev_socket_id(task->cdev_id));
192 int ret = rte_cryptodev_start(task->cdev_id);
193 PROX_PANIC(ret < 0, "Failed to start device\n");
195 struct rte_cryptodev *dev;
196 dev = rte_cryptodev_pmd_get_dev(task->cdev_id);
197 PROX_PANIC(dev->attached != RTE_CRYPTODEV_ATTACHED, "No ENC cryptodev attached\n");
199 //Setup Cipher Parameters
200 struct rte_crypto_sym_xform cipher_xform = {0};
201 struct rte_crypto_sym_xform auth_xform = {0};
203 cipher_xform.type = RTE_CRYPTO_SYM_XFORM_CIPHER;
204 cipher_xform.next = &auth_xform;
206 cipher_xform.cipher.algo = RTE_CRYPTO_CIPHER_AES_CBC;
207 cipher_xform.cipher.op = RTE_CRYPTO_CIPHER_OP_ENCRYPT;
208 cipher_xform.cipher.key.data = aes_cbc_key;
209 cipher_xform.cipher.key.length = CIPHER_KEY_LENGTH_AES_CBC;
211 cipher_xform.cipher.iv.offset = IV_OFFSET;
212 cipher_xform.cipher.iv.length = CIPHER_IV_LENGTH_AES_CBC;
214 //Setup HMAC Parameters
215 auth_xform.type = RTE_CRYPTO_SYM_XFORM_AUTH;
216 auth_xform.next = NULL;
217 auth_xform.auth.op = RTE_CRYPTO_AUTH_OP_GENERATE;
218 auth_xform.auth.algo = RTE_CRYPTO_AUTH_SHA1_HMAC;
219 auth_xform.auth.key.length = DIGEST_BYTE_LENGTH_SHA1;
220 auth_xform.auth.key.data = hmac_sha1_key;
221 auth_xform.auth.digest_length = DIGEST_BYTE_LENGTH_SHA1;
223 auth_xform.auth.iv.offset = 0;
224 auth_xform.auth.iv.length = 0;
226 task->sess = rte_cryptodev_sym_session_create(task->cdev_id, &cipher_xform, task->session_pool);
227 PROX_PANIC(task->sess < 0, "Failed ENC sym_session_create\n");
229 task->local_ipv4 = rte_cpu_to_be_32(targ->local_ipv4);
230 task->remote_ipv4 = rte_cpu_to_be_32(targ->remote_ipv4);
231 //memcpy(&task->src_mac, &prox_port_cfg[task->base.tx_params_hw.tx_port_queue->port].eth_addr, sizeof(prox_rte_ether_addr));
232 struct prox_port_cfg *port = find_reachable_port(targ);
233 memcpy(&task->local_mac, &port->eth_addr, sizeof(prox_rte_ether_addr));
235 if (targ->flags & TASK_ARG_DST_MAC_SET){
236 memcpy(&task->dst_mac, &targ->edaddr, sizeof(task->dst_mac));
237 plog_info("TASK_ARG_DST_MAC_SET ("MAC_BYTES_FMT")\n", MAC_BYTES(task->dst_mac.addr_bytes));
238 //prox_rte_ether_addr_copy(&ptask->dst_mac, &peth->d_addr);
239 //rte_memcpy(hdr, task->src_dst_mac, sizeof(task->src_dst_mac));
243 static void init_task_esp_dec(struct task_base *tbase, struct task_args *targ)
245 struct task_esp_dec *task = (struct task_esp_dec *)tbase;
247 tbase->flags |= FLAG_NEVER_FLUSH;
249 uint8_t lcore_id = targ->lconf->id;
251 sprintf(name, "core_%03u_crypto_pool", lcore_id);
252 task->crypto_op_pool = rte_crypto_op_pool_create(name, RTE_CRYPTO_OP_TYPE_SYMMETRIC,
253 targ->nb_mbuf, 128, MAXIMUM_IV_LENGTH, rte_socket_id());
254 PROX_PANIC(task->crypto_op_pool == NULL, "Can't create DEC CRYPTO_OP_POOL\n");
256 task->cdev_id = get_cdev_id();
257 struct rte_cryptodev_config cdev_conf;
258 cdev_conf.nb_queue_pairs = 2;
259 cdev_conf.socket_id = SOCKET_ID_ANY;
260 cdev_conf.socket_id = rte_socket_id();
261 rte_cryptodev_configure(task->cdev_id, &cdev_conf);
263 unsigned int session_size = rte_cryptodev_sym_get_private_session_size(task->cdev_id);
264 plog_info("rte_cryptodev_sym_get_private_session_size=%d\n", session_size);
265 sprintf(name, "core_%03u_session_pool", lcore_id);
266 task->session_pool = rte_cryptodev_sym_session_pool_create(name,
271 PROX_PANIC(task->session_pool == NULL, "Failed rte_mempool_create\n");
274 plog_info("dec: task->qp_id=%u\n", task->qp_id);
275 struct prox_rte_cryptodev_qp_conf qp_conf;
276 qp_conf.nb_descriptors = 128;
277 qp_conf.mp_session = task->session_pool;
278 prox_rte_cryptodev_queue_pair_setup(task->cdev_id, task->qp_id, &qp_conf, rte_cryptodev_socket_id(task->cdev_id));
280 int ret = rte_cryptodev_start(task->cdev_id);
281 PROX_PANIC(ret < 0, "Failed to start device\n");
283 struct rte_cryptodev *dev;
284 dev = rte_cryptodev_pmd_get_dev(task->cdev_id);
285 PROX_PANIC(dev->attached != RTE_CRYPTODEV_ATTACHED, "No ENC cryptodev attached\n");
287 //Setup Cipher Parameters
288 struct rte_crypto_sym_xform cipher_xform = {0};
289 struct rte_crypto_sym_xform auth_xform = {0};
291 cipher_xform.type = RTE_CRYPTO_SYM_XFORM_CIPHER;
292 cipher_xform.next = NULL;
293 cipher_xform.cipher.algo = RTE_CRYPTO_CIPHER_AES_CBC;
294 cipher_xform.cipher.op = RTE_CRYPTO_CIPHER_OP_DECRYPT;
295 cipher_xform.cipher.key.data = aes_cbc_key;
296 cipher_xform.cipher.key.length = CIPHER_KEY_LENGTH_AES_CBC;
298 cipher_xform.cipher.iv.offset = IV_OFFSET;
299 cipher_xform.cipher.iv.length = CIPHER_IV_LENGTH_AES_CBC;
301 //Setup HMAC Parameters
302 auth_xform.type = RTE_CRYPTO_SYM_XFORM_AUTH;
303 auth_xform.next = &cipher_xform;
304 auth_xform.auth.op = RTE_CRYPTO_AUTH_OP_VERIFY;
305 auth_xform.auth.algo = RTE_CRYPTO_AUTH_SHA1_HMAC;
306 auth_xform.auth.key.length = DIGEST_BYTE_LENGTH_SHA1;
307 auth_xform.auth.key.data = hmac_sha1_key;
308 auth_xform.auth.digest_length = DIGEST_BYTE_LENGTH_SHA1;
310 auth_xform.auth.iv.offset = 0;
311 auth_xform.auth.iv.length = 0;
313 task->sess = rte_cryptodev_sym_session_create(task->cdev_id, &cipher_xform, task->session_pool);
314 PROX_PANIC(task->sess < 0, "Failed DEC sym_session_create\n");
316 task->local_ipv4 = rte_cpu_to_be_32(targ->local_ipv4);
317 //memcpy(&task->src_mac, &prox_port_cfg[task->base.tx_params_hw.tx_port_queue->port].eth_addr, sizeof(prox_rte_ether_addr));
318 struct prox_port_cfg *port = find_reachable_port(targ);
319 memcpy(&task->local_mac, &port->eth_addr, sizeof(prox_rte_ether_addr));
321 if (targ->flags & TASK_ARG_DST_MAC_SET){
322 memcpy(&task->dst_mac, &targ->edaddr, sizeof(task->dst_mac));
323 plog_info("TASK_ARG_DST_MAC_SET ("MAC_BYTES_FMT")\n", MAC_BYTES(task->dst_mac.addr_bytes));
324 //prox_rte_ether_addr_copy(&ptask->dst_mac, &peth->d_addr);
325 //rte_memcpy(hdr, task->src_dst_mac, sizeof(task->src_dst_mac));
330 static inline uint8_t handle_esp_ah_enc(struct task_esp_enc *task, struct rte_mbuf *mbuf, struct rte_crypto_op *cop)
333 prox_rte_ether_hdr *peth = rte_pktmbuf_mtod(mbuf, prox_rte_ether_hdr *);
334 prox_rte_ipv4_hdr* pip4 = (prox_rte_ipv4_hdr *)(peth + 1);
335 uint16_t ipv4_length = rte_be_to_cpu_16(pip4->total_length);
336 struct rte_crypto_sym_op *sym_cop = cop->sym;
338 if (unlikely((pip4->version_ihl >> 4) != 4)) {
339 plog_info("Received non IPv4 packet at esp enc %i\n", pip4->version_ihl);
340 plogdx_info(mbuf, "ENC RX: ");
343 if (pip4->time_to_live) {
344 pip4->time_to_live--;
347 plog_info("TTL = 0 => Dropping\n");
351 // Remove padding if any (we don't want to encapsulate garbage at end of IPv4 packet)
352 int l1 = rte_pktmbuf_pkt_len(mbuf);
353 int padding = l1 - (ipv4_length + sizeof(prox_rte_ether_hdr));
354 if (unlikely(padding > 0)) {
355 rte_pktmbuf_trim(mbuf, padding);
358 l1 = rte_pktmbuf_pkt_len(mbuf);
359 int encrypt_len = l1 - sizeof(prox_rte_ether_hdr) + 2; // According to RFC4303 table 1, encrypt len is ip+tfc_pad(o)+pad+pad len(1) + next header(1)
361 if ((encrypt_len & 0xf) != 0){
362 padding = 16 - (encrypt_len % 16);
363 encrypt_len += padding;
366 const int extra_space = sizeof(prox_rte_ipv4_hdr) + sizeof(struct prox_esp_hdr) + CIPHER_IV_LENGTH_AES_CBC;
368 prox_rte_ether_addr src_mac = peth->s_addr;
369 prox_rte_ether_addr dst_mac = peth->d_addr;
370 uint32_t src_addr = pip4->src_addr;
371 uint32_t dst_addr = pip4->dst_addr;
372 uint8_t ttl = pip4->time_to_live;
373 uint8_t version_ihl = pip4->version_ihl;
375 peth = (prox_rte_ether_hdr *)rte_pktmbuf_prepend(mbuf, extra_space); // encap + prefix
376 peth = (prox_rte_ether_hdr *)rte_pktmbuf_append(mbuf, 0 + 1 + 1 + padding + 4 + DIGEST_BYTE_LENGTH_SHA1); // padding + pad_len + next_head + seqn + ICV pad + ICV
377 peth = rte_pktmbuf_mtod(mbuf, prox_rte_ether_hdr *);
378 l1 = rte_pktmbuf_pkt_len(mbuf);
379 peth->ether_type = ETYPE_IPv4;
382 prox_rte_ether_addr_copy(&dst_mac, &peth->s_addr);
383 prox_rte_ether_addr_copy(&src_mac, &peth->d_addr);
385 prox_rte_ether_addr_copy(&task->local_mac, &peth->s_addr);
386 //prox_rte_ether_addr_copy(&dst_mac, &peth->d_addr);//IS: dstmac should be rewritten by arp
387 prox_rte_ether_addr_copy(&task->dst_mac, &peth->d_addr);
390 pip4 = (prox_rte_ipv4_hdr *)(peth + 1);
391 pip4->src_addr = task->local_ipv4;
392 pip4->dst_addr = task->remote_ipv4;
393 pip4->time_to_live = ttl;
394 pip4->next_proto_id = IPPROTO_ESP; // 50 for ESP, ip in ip next proto trailer
395 pip4->version_ihl = version_ihl; // 20 bytes, ipv4
396 pip4->total_length = rte_cpu_to_be_16(ipv4_length + sizeof(prox_rte_ipv4_hdr) + sizeof(struct prox_esp_hdr) + CIPHER_IV_LENGTH_AES_CBC + padding + 1 + 1 + DIGEST_BYTE_LENGTH_SHA1); // iphdr+SPI+SN+IV+payload+padding+padlen+next header + crc + auth
397 pip4->packet_id = 0x0101;
398 pip4->type_of_service = 0;
399 pip4->time_to_live = 64;
400 prox_ip_cksum(mbuf, pip4, sizeof(prox_rte_ether_hdr), sizeof(prox_rte_ipv4_hdr), 1);
402 data = (u8*)(pip4 + 1);
404 *((u32*) data) = 0x2016; // FIXME SPI
405 *((u32*) data + 1) = 0x2; // FIXME SN
407 struct prox_esp_hdr *pesp = (struct prox_esp_hdr*)(pip4+1);
408 pesp->spi = src_addr;//for simplicity assume 1 tunnel per source ip
411 pesp->spi=0xAAAAAAAA;//debug
412 pesp->seq =0xBBBBBBBB;//debug
414 u8 *padl = (u8*)data + (8 + encrypt_len - 2 + CIPHER_IV_LENGTH_AES_CBC); // No ESN yet. (-2 means NH is crypted)
415 //padl += CIPHER_IV_LENGTH_AES_CBC;
417 *(padl + 1) = 4; // ipv4 in 4
419 sym_cop->auth.digest.data = data + 8 + CIPHER_IV_LENGTH_AES_CBC + encrypt_len;
420 //sym_cop->auth.digest.phys_addr = rte_pktmbuf_mtophys_offset(mbuf, (sizeof(prox_rte_ether_hdr) + sizeof(prox_rte_ipv4_hdr) + 8 + CIPHER_IV_LENGTH_AES_CBC + encrypt_len));
421 sym_cop->auth.digest.phys_addr = rte_pktmbuf_iova_offset(mbuf, (sizeof(prox_rte_ether_hdr) + sizeof(prox_rte_ipv4_hdr) + 8 + CIPHER_IV_LENGTH_AES_CBC + encrypt_len));
422 //sym_cop->auth.digest.length = DIGEST_BYTE_LENGTH_SHA1;
424 //sym_cop->cipher.iv.data = data + 8;
425 //sym_cop->cipher.iv.phys_addr = rte_pktmbuf_mtophys(mbuf) + sizeof(prox_rte_ether_hdr) + sizeof(prox_rte_ipv4_hdr) + 4 + 4;
426 //sym_cop->cipher.iv.length = CIPHER_IV_LENGTH_AES_CBC;
428 //rte_memcpy(sym_cop->cipher.iv.data, aes_cbc_iv, CIPHER_IV_LENGTH_AES_CBC);
430 uint8_t *iv_ptr = rte_crypto_op_ctod_offset(cop, uint8_t *, IV_OFFSET);
431 rte_memcpy(iv_ptr, aes_cbc_iv, CIPHER_IV_LENGTH_AES_CBC);
434 sym_cop->cipher.data.offset = sizeof(prox_rte_ether_hdr) + sizeof(prox_rte_ipv4_hdr) + 4 + 4 + CIPHER_IV_LENGTH_AES_CBC;
435 sym_cop->cipher.data.length = encrypt_len;
437 uint64_t *iv = (uint64_t *)(pesp + 1);
438 memset(iv, 0, CIPHER_IV_LENGTH_AES_CBC);
440 //uint64_t *iv = (uint64_t *)(pesp + 1);
441 //memset(iv, 0, CIPHER_IV_LENGTH_AES_CBC);
442 sym_cop->cipher.data.offset = sizeof(prox_rte_ether_hdr) + sizeof(prox_rte_ipv4_hdr) + sizeof(struct prox_esp_hdr);
443 sym_cop->cipher.data.length = encrypt_len + CIPHER_IV_LENGTH_AES_CBC;
446 sym_cop->auth.data.offset = sizeof(prox_rte_ether_hdr) + sizeof(prox_rte_ipv4_hdr);
447 sym_cop->auth.data.length = sizeof(struct prox_esp_hdr) + CIPHER_IV_LENGTH_AES_CBC + encrypt_len;// + 4;// FIXME
449 sym_cop->m_src = mbuf;
450 rte_crypto_op_attach_sym_session(cop, task->sess);
451 //cop->type = RTE_CRYPTO_OP_TYPE_SYMMETRIC;
452 //cop->status = RTE_CRYPTO_OP_STATUS_NOT_PROCESSED;
457 static inline uint8_t handle_esp_ah_dec(struct task_esp_dec *task, struct rte_mbuf *mbuf, struct rte_crypto_op *cop)
459 struct rte_crypto_sym_op *sym_cop = cop->sym;
460 prox_rte_ether_hdr *peth = rte_pktmbuf_mtod(mbuf, prox_rte_ether_hdr *);
461 prox_rte_ipv4_hdr* pip4 = (prox_rte_ipv4_hdr *)(peth + 1);
462 uint16_t ipv4_length = rte_be_to_cpu_16(pip4->total_length);
463 u8 *data = (u8*)(pip4 + 1);
465 if (pip4->next_proto_id != IPPROTO_ESP){
466 plog_info("Received non ESP packet on esp dec\n");
467 plogdx_info(mbuf, "DEC RX: ");
471 rte_crypto_op_attach_sym_session(cop, task->sess);
473 sym_cop->auth.digest.data = (unsigned char *)((unsigned char*)pip4 + ipv4_length - DIGEST_BYTE_LENGTH_SHA1);
474 //sym_cop->auth.digest.phys_addr = rte_pktmbuf_mtophys_offset(mbuf, sizeof(prox_rte_ether_hdr) + sizeof(prox_rte_ipv4_hdr) + sizeof(struct prox_esp_hdr)); // FIXME
475 sym_cop->auth.digest.phys_addr = rte_pktmbuf_iova_offset(mbuf, sizeof(prox_rte_ether_hdr) + sizeof(prox_rte_ipv4_hdr) + sizeof(struct prox_esp_hdr));
476 //sym_cop->auth.digest.length = DIGEST_BYTE_LENGTH_SHA1;
478 //sym_cop->cipher.iv.data = (uint8_t *)data + 8;
479 //sym_cop->cipher.iv.phys_addr = rte_pktmbuf_mtophys(mbuf) + sizeof(prox_rte_ether_hdr) + sizeof(prox_rte_ipv4_hdr) + 4 + 4;
480 //sym_cop->cipher.iv.length = CIPHER_IV_LENGTH_AES_CBC;
483 rte_memcpy(rte_crypto_op_ctod_offset(cop, uint8_t *, IV_OFFSET),
485 CIPHER_IV_LENGTH_AES_CBC);
487 uint8_t * iv = (uint8_t *)(pip4 + 1) + sizeof(struct prox_esp_hdr);
488 rte_memcpy(rte_crypto_op_ctod_offset(cop, uint8_t *, IV_OFFSET),
490 CIPHER_IV_LENGTH_AES_CBC);
493 sym_cop->auth.data.offset = sizeof(prox_rte_ether_hdr) + sizeof(prox_rte_ipv4_hdr);
494 sym_cop->auth.data.length = ipv4_length - sizeof(prox_rte_ipv4_hdr) - 4 - CIPHER_IV_LENGTH_AES_CBC;
496 sym_cop->cipher.data.offset = sizeof(prox_rte_ether_hdr) + sizeof(prox_rte_ipv4_hdr) + sizeof(struct prox_esp_hdr) + CIPHER_IV_LENGTH_AES_CBC;
497 sym_cop->cipher.data.length = ipv4_length - sizeof(prox_rte_ipv4_hdr) - CIPHER_IV_LENGTH_AES_CBC - 28; // FIXME
499 sym_cop->m_src = mbuf;
503 static inline void do_ipv4_swap(struct task_esp_dec *task, struct rte_mbuf *mbuf)
505 prox_rte_ether_hdr *peth = rte_pktmbuf_mtod(mbuf, prox_rte_ether_hdr *);
506 prox_rte_ether_addr src_mac = peth->s_addr;
507 prox_rte_ether_addr dst_mac = peth->d_addr;
508 uint32_t src_ip, dst_ip;
510 prox_rte_ipv4_hdr* pip4 = (prox_rte_ipv4_hdr *)(peth + 1);
511 src_ip = pip4->src_addr;
512 dst_ip = pip4->dst_addr;
514 //peth->s_addr = dst_mac;
515 peth->d_addr = src_mac;//should be replaced by arp
516 pip4->src_addr = dst_ip;
517 pip4->dst_addr = src_ip;
518 prox_rte_ether_addr_copy(&task->local_mac, &peth->s_addr);
521 static inline uint8_t handle_esp_ah_dec_finish(struct task_esp_dec *task, struct rte_mbuf *mbuf)
523 prox_rte_ether_hdr *peth = rte_pktmbuf_mtod(mbuf, prox_rte_ether_hdr *);
524 rte_memcpy(((u8*)peth) + sizeof(prox_rte_ether_hdr), ((u8*)peth) + sizeof(prox_rte_ether_hdr) +
525 + sizeof(prox_rte_ipv4_hdr) + 4 + 4 + CIPHER_IV_LENGTH_AES_CBC, sizeof(prox_rte_ipv4_hdr));// next hdr, padding
526 prox_rte_ipv4_hdr* pip4 = (prox_rte_ipv4_hdr *)(peth + 1);
528 if (unlikely((pip4->version_ihl >> 4) != 4)) {
529 plog_info("non IPv4 packet after esp dec %i\n", pip4->version_ihl);
530 plogdx_info(mbuf, "DEC TX: ");
533 if (pip4->time_to_live) {
534 pip4->time_to_live--;
537 plog_info("TTL = 0 => Dropping\n");
540 uint16_t ipv4_length = rte_be_to_cpu_16(pip4->total_length);
541 rte_memcpy(((u8*)peth) + sizeof(prox_rte_ether_hdr) + sizeof(prox_rte_ipv4_hdr),
542 ((u8*)peth) + sizeof(prox_rte_ether_hdr) +
543 + 2 * sizeof(prox_rte_ipv4_hdr) + 4 + 4 + CIPHER_IV_LENGTH_AES_CBC, ipv4_length - sizeof(prox_rte_ipv4_hdr));
545 int len = rte_pktmbuf_pkt_len(mbuf);
546 rte_pktmbuf_trim(mbuf, len - sizeof(prox_rte_ether_hdr) - ipv4_length);
547 peth = rte_pktmbuf_mtod(mbuf, prox_rte_ether_hdr *);
550 do_ipv4_swap(task, mbuf);
552 prox_rte_ether_addr_copy(&task->local_mac, &peth->s_addr);
553 prox_rte_ether_addr_copy(&task->dst_mac, &peth->d_addr);
554 //rte_memcpy(peth, task->dst_mac, sizeof(task->dst_mac));
556 prox_ip_cksum(mbuf, pip4, sizeof(prox_rte_ether_hdr), sizeof(prox_rte_ipv4_hdr), 1);
561 static inline uint8_t handle_esp_ah_dec_finish2(struct task_esp_dec *task, struct rte_mbuf *mbuf)
563 u8* m = rte_pktmbuf_mtod(mbuf, u8*);
564 rte_memcpy(m+sizeof(prox_rte_ipv4_hdr)+sizeof(struct prox_esp_hdr)+CIPHER_IV_LENGTH_AES_CBC,
565 m, sizeof(prox_rte_ether_hdr));
566 m = (u8*)rte_pktmbuf_adj(mbuf, sizeof(prox_rte_ipv4_hdr)+sizeof(struct prox_esp_hdr)+CIPHER_IV_LENGTH_AES_CBC);
567 prox_rte_ipv4_hdr* pip4 = (prox_rte_ipv4_hdr *)(m+sizeof(prox_rte_ether_hdr));
569 if (unlikely((pip4->version_ihl >> 4) != 4)) {
570 plog_info("non IPv4 packet after esp dec %i\n", pip4->version_ihl);
571 plogdx_info(mbuf, "DEC TX: ");
574 if (pip4->time_to_live) {
575 pip4->time_to_live--;
578 plog_info("TTL = 0 => Dropping\n");
581 uint16_t ipv4_length = rte_be_to_cpu_16(pip4->total_length);
582 int len = rte_pktmbuf_pkt_len(mbuf);
583 rte_pktmbuf_trim(mbuf, len - sizeof(prox_rte_ether_hdr) - ipv4_length);
586 do_ipv4_swap(task, mbuf);
588 prox_rte_ether_hdr *peth = rte_pktmbuf_mtod(mbuf, prox_rte_ether_hdr *);
589 prox_rte_ether_addr_copy(&task->local_mac, &peth->s_addr);
590 prox_rte_ether_addr_copy(&task->dst_mac, &peth->d_addr);
591 //rte_memcpy(peth, task->dst_mac, sizeof(task->dst_mac));
594 prox_ip_cksum(mbuf, pip4, sizeof(prox_rte_ether_hdr), sizeof(prox_rte_ipv4_hdr), 1);
598 static int handle_esp_enc_bulk(struct task_base *tbase, struct rte_mbuf **mbufs, uint16_t n_pkts)
600 struct task_esp_enc *task = (struct task_esp_enc *)tbase;
601 uint8_t out[MAX_PKT_BURST];
603 uint16_t i = 0, nb_rx = 0, nb_enc=0, j = 0, idx = 0;
604 struct rte_mbuf *new_mbufs[MAX_PKT_BURST];
606 if (rte_crypto_op_bulk_alloc(task->crypto_op_pool,
607 RTE_CRYPTO_OP_TYPE_SYMMETRIC,
608 task->ops_burst, n_pkts) != n_pkts) {
609 PROX_PANIC(1, "Failed to allocate ENC crypto operations\n");
611 for (uint16_t j = 0; j < n_pkts; ++j) {
612 result = handle_esp_ah_enc(task, mbufs[j], task->ops_burst[nb_enc]);
613 if (result != OUT_DISCARD) {
617 new_mbufs[idx] = mbufs[j];
623 if (rte_cryptodev_enqueue_burst(task->cdev_id, task->qp_id, task->ops_burst, nb_enc) != nb_enc) {
624 plog_info("Error enc enqueue_burst\n");
629 nb_rx = rte_cryptodev_dequeue_burst(task->cdev_id, task->qp_id, task->ops_burst, MAX_PKT_BURST - idx);
630 for (uint16_t j = 0; j < nb_rx; ++j) {
631 new_mbufs[idx] = task->ops_burst[j]->sym->m_src;
633 rte_crypto_op_free(task->ops_burst[j]);
636 return task->base.tx_pkt(&task->base, new_mbufs, idx, out);
639 static int handle_esp_dec_bulk(struct task_base *tbase, struct rte_mbuf **mbufs, uint16_t n_pkts)
641 struct task_esp_dec *task = (struct task_esp_dec *)tbase;
642 uint8_t out[MAX_PKT_BURST];
644 uint16_t i = 0, nb_rx = 0, nb_dec=0, j = 0, idx = 0;
645 struct rte_mbuf *new_mbufs[MAX_PKT_BURST];
647 if (rte_crypto_op_bulk_alloc(task->crypto_op_pool,
648 RTE_CRYPTO_OP_TYPE_SYMMETRIC,
649 task->ops_burst, n_pkts) != n_pkts) {
650 PROX_PANIC(1, "Failed to allocate DEC crypto operations\n");
652 for (j = 0; j < n_pkts; ++j) {
653 result = handle_esp_ah_dec(task, mbufs[j], task->ops_burst[nb_dec]);
654 if (result != OUT_DISCARD)
657 new_mbufs[idx] = mbufs[j];
663 if (rte_cryptodev_enqueue_burst(task->cdev_id, task->qp_id, task->ops_burst, nb_dec) != nb_dec) {
664 plog_info("Error dec enqueue_burst\n");
669 nb_rx = rte_cryptodev_dequeue_burst(task->cdev_id, task->qp_id,
670 task->ops_burst, MAX_PKT_BURST - idx);
672 for (j = 0; j < nb_rx; ++j) {
673 new_mbufs[idx] = task->ops_burst[j]->sym->m_src;
674 if (task->ops_burst[j]->status == RTE_CRYPTO_OP_STATUS_SUCCESS) {
675 out[idx] = handle_esp_ah_dec_finish2(task, new_mbufs[idx]);
678 out[idx] = OUT_DISCARD;
680 rte_crypto_op_free(task->ops_burst[j]);
684 return task->base.tx_pkt(&task->base, new_mbufs, idx, out);
687 struct task_init task_init_esp_enc = {
689 .mode_str = "esp_enc",
690 .init = init_task_esp_enc,
691 .handle = handle_esp_enc_bulk,
692 .size = sizeof(struct task_esp_enc),
695 struct task_init task_init_esp_dec = {
697 .mode_str = "esp_dec",
698 .init = init_task_esp_dec,
699 .handle = handle_esp_dec_bulk,
700 .size = sizeof(struct task_esp_dec),
703 __attribute__((constructor)) static void reg_task_esp_enc(void)
705 reg_task(&task_init_esp_enc);
708 __attribute__((constructor)) static void reg_task_esp_dec(void)
710 reg_task(&task_init_esp_dec);