1/******************************************************************************
2
3 Copyright (c) 2001-2013, Intel Corporation
4 All rights reserved.
5
6 Redistribution and use in source and binary forms, with or without
7 modification, are permitted provided that the following conditions are met:
8
9 1. Redistributions of source code must retain the above copyright notice,
10 this list of conditions and the following disclaimer.
11
12 2. Redistributions in binary form must reproduce the above copyright
13 notice, this list of conditions and the following disclaimer in the
14 documentation and/or other materials provided with the distribution.
15
16 3. Neither the name of the Intel Corporation nor the names of its
17 contributors may be used to endorse or promote products derived from
18 this software without specific prior written permission.
19
20 THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
21 AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
22 IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
23 ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
24 LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
25 CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
26 SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
27 INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
28 CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
29 ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
30 POSSIBILITY OF SUCH DAMAGE.
31
32******************************************************************************/
33/*
34 * Copyright (c) 2011 The NetBSD Foundation, Inc.
35 * All rights reserved.
36 *
37 * This code is derived from software contributed to The NetBSD Foundation
38 * by Coyote Point Systems, Inc.
39 *
40 * Redistribution and use in source and binary forms, with or without
41 * modification, are permitted provided that the following conditions
42 * are met:
43 * 1. Redistributions of source code must retain the above copyright
44 * notice, this list of conditions and the following disclaimer.
45 * 2. Redistributions in binary form must reproduce the above copyright
46 * notice, this list of conditions and the following disclaimer in the
47 * documentation and/or other materials provided with the distribution.
48 *
49 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS
50 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
51 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
52 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
53 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
54 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
55 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
56 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
57 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
58 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
59 * POSSIBILITY OF SUCH DAMAGE.
60 */
61/*$FreeBSD: head/sys/dev/ixgbe/ixgbe.c 279805 2015-03-09 10:29:15Z araujo $*/
62/*$NetBSD: ixgbe.c,v 1.40 2016/10/13 20:05:06 jdolecek Exp $*/
63
64#include "opt_inet.h"
65#include "opt_inet6.h"
66
67#include "ixgbe.h"
68#include "vlan.h"
69
70#include <sys/cprng.h>
71
72/*********************************************************************
73 * Set this to one to display debug statistics
74 *********************************************************************/
75int ixgbe_display_debug_stats = 0;
76
77/*********************************************************************
78 * Driver version
79 *********************************************************************/
80char ixgbe_driver_version[] = "2.5.15";
81
82/*********************************************************************
83 * PCI Device ID Table
84 *
85 * Used by probe to select devices to load on
86 * Last field stores an index into ixgbe_strings
87 * Last entry must be all 0s
88 *
89 * { Vendor ID, Device ID, SubVendor ID, SubDevice ID, String Index }
90 *********************************************************************/
91
92static ixgbe_vendor_info_t ixgbe_vendor_info_array[] =
93{
94 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598AF_DUAL_PORT, 0, 0, 0},
95 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598AF_SINGLE_PORT, 0, 0, 0},
96 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598EB_CX4, 0, 0, 0},
97 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598AT, 0, 0, 0},
98 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598AT2, 0, 0, 0},
99 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598, 0, 0, 0},
100 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598_DA_DUAL_PORT, 0, 0, 0},
101 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598_CX4_DUAL_PORT, 0, 0, 0},
102 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598EB_XF_LR, 0, 0, 0},
103 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598_SR_DUAL_PORT_EM, 0, 0, 0},
104 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598EB_SFP_LOM, 0, 0, 0},
105 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_KX4, 0, 0, 0},
106 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_KX4_MEZZ, 0, 0, 0},
107 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_SFP, 0, 0, 0},
108 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_XAUI_LOM, 0, 0, 0},
109 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_CX4, 0, 0, 0},
110 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_T3_LOM, 0, 0, 0},
111 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_COMBO_BACKPLANE, 0, 0, 0},
112 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_BACKPLANE_FCOE, 0, 0, 0},
113 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_SFP_SF2, 0, 0, 0},
114 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_SFP_FCOE, 0, 0, 0},
115 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599EN_SFP, 0, 0, 0},
116 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_SFP_SF_QP, 0, 0, 0},
117 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_X540T, 0, 0, 0},
118 /* required last entry */
119 {0, 0, 0, 0, 0}
120};
121
122/*********************************************************************
123 * Table of branding strings
124 *********************************************************************/
125
126static const char *ixgbe_strings[] = {
127 "Intel(R) PRO/10GbE PCI-Express Network Driver"
128};
129
130/*********************************************************************
131 * Function prototypes
132 *********************************************************************/
133static int ixgbe_probe(device_t, cfdata_t, void *);
134static void ixgbe_attach(device_t, device_t, void *);
135static int ixgbe_detach(device_t, int);
136#if 0
137static int ixgbe_shutdown(device_t);
138#endif
139#ifdef IXGBE_LEGACY_TX
140static void ixgbe_start(struct ifnet *);
141static void ixgbe_start_locked(struct tx_ring *, struct ifnet *);
142#else /* ! IXGBE_LEGACY_TX */
143static int ixgbe_mq_start(struct ifnet *, struct mbuf *);
144static int ixgbe_mq_start_locked(struct ifnet *, struct tx_ring *);
145static void ixgbe_qflush(struct ifnet *);
146static void ixgbe_deferred_mq_start(void *, int);
147#endif /* IXGBE_LEGACY_TX */
148static int ixgbe_ioctl(struct ifnet *, u_long, void *);
149static void ixgbe_ifstop(struct ifnet *, int);
150static int ixgbe_init(struct ifnet *);
151static void ixgbe_init_locked(struct adapter *);
152static void ixgbe_stop(void *);
153static void ixgbe_media_status(struct ifnet *, struct ifmediareq *);
154static int ixgbe_media_change(struct ifnet *);
155static void ixgbe_identify_hardware(struct adapter *);
156static int ixgbe_allocate_pci_resources(struct adapter *,
157 const struct pci_attach_args *);
158static void ixgbe_get_slot_info(struct ixgbe_hw *);
159static int ixgbe_allocate_msix(struct adapter *,
160 const struct pci_attach_args *);
161static int ixgbe_allocate_legacy(struct adapter *,
162 const struct pci_attach_args *);
163static int ixgbe_allocate_queues(struct adapter *);
164static int ixgbe_setup_msix(struct adapter *);
165static void ixgbe_free_pci_resources(struct adapter *);
166static void ixgbe_local_timer(void *);
167static int ixgbe_setup_interface(device_t, struct adapter *);
168static void ixgbe_config_link(struct adapter *);
169
170static int ixgbe_allocate_transmit_buffers(struct tx_ring *);
171static int ixgbe_setup_transmit_structures(struct adapter *);
172static void ixgbe_setup_transmit_ring(struct tx_ring *);
173static void ixgbe_initialize_transmit_units(struct adapter *);
174static void ixgbe_free_transmit_structures(struct adapter *);
175static void ixgbe_free_transmit_buffers(struct tx_ring *);
176
177static int ixgbe_allocate_receive_buffers(struct rx_ring *);
178static int ixgbe_setup_receive_structures(struct adapter *);
179static int ixgbe_setup_receive_ring(struct rx_ring *);
180static void ixgbe_initialize_receive_units(struct adapter *);
181static void ixgbe_free_receive_structures(struct adapter *);
182static void ixgbe_free_receive_buffers(struct rx_ring *);
183static void ixgbe_setup_hw_rsc(struct rx_ring *);
184
185static void ixgbe_enable_intr(struct adapter *);
186static void ixgbe_disable_intr(struct adapter *);
187static void ixgbe_update_stats_counters(struct adapter *);
188static void ixgbe_txeof(struct tx_ring *);
189static bool ixgbe_rxeof(struct ix_queue *);
190static void ixgbe_rx_checksum(u32, struct mbuf *, u32,
191 struct ixgbe_hw_stats *);
192static void ixgbe_set_promisc(struct adapter *);
193static void ixgbe_set_multi(struct adapter *);
194static void ixgbe_update_link_status(struct adapter *);
195static void ixgbe_refresh_mbufs(struct rx_ring *, int);
196static int ixgbe_xmit(struct tx_ring *, struct mbuf *);
197static int ixgbe_set_flowcntl(SYSCTLFN_PROTO);
198static int ixgbe_set_advertise(SYSCTLFN_PROTO);
199static int ixgbe_set_thermal_test(SYSCTLFN_PROTO);
200static int ixgbe_dma_malloc(struct adapter *, bus_size_t,
201 struct ixgbe_dma_alloc *, int);
202static void ixgbe_dma_free(struct adapter *, struct ixgbe_dma_alloc *);
203static int ixgbe_tx_ctx_setup(struct tx_ring *,
204 struct mbuf *, u32 *, u32 *);
205static int ixgbe_tso_setup(struct tx_ring *,
206 struct mbuf *, u32 *, u32 *);
207static void ixgbe_set_ivar(struct adapter *, u8, u8, s8);
208static void ixgbe_configure_ivars(struct adapter *);
209static u8 * ixgbe_mc_array_itr(struct ixgbe_hw *, u8 **, u32 *);
210
211static void ixgbe_setup_vlan_hw_support(struct adapter *);
212#if 0
213static void ixgbe_register_vlan(void *, struct ifnet *, u16);
214static void ixgbe_unregister_vlan(void *, struct ifnet *, u16);
215#endif
216
217static void ixgbe_add_hw_stats(struct adapter *adapter);
218
219static __inline void ixgbe_rx_discard(struct rx_ring *, int);
220static __inline void ixgbe_rx_input(struct rx_ring *, struct ifnet *,
221 struct mbuf *, u32);
222
223static void ixgbe_enable_rx_drop(struct adapter *);
224static void ixgbe_disable_rx_drop(struct adapter *);
225
226/* Support for pluggable optic modules */
227static bool ixgbe_sfp_probe(struct adapter *);
228static void ixgbe_setup_optics(struct adapter *);
229
230/* Legacy (single vector interrupt handler */
231static int ixgbe_legacy_irq(void *);
232
233#if defined(NETBSD_MSI_OR_MSIX)
234/* The MSI/X Interrupt handlers */
235static int ixgbe_msix_que(void *);
236static int ixgbe_msix_link(void *);
237#endif
238
239/* Software interrupts for deferred work */
240static void ixgbe_handle_que(void *);
241static void ixgbe_handle_link(void *);
242static void ixgbe_handle_msf(void *);
243static void ixgbe_handle_mod(void *);
244
245const struct sysctlnode *ixgbe_sysctl_instance(struct adapter *);
246static ixgbe_vendor_info_t *ixgbe_lookup(const struct pci_attach_args *);
247
248#ifdef IXGBE_FDIR
249static void ixgbe_atr(struct tx_ring *, struct mbuf *);
250static void ixgbe_reinit_fdir(void *, int);
251#endif
252
253/* Missing shared code prototype */
254extern void ixgbe_stop_mac_link_on_d3_82599(struct ixgbe_hw *hw);
255
256/*********************************************************************
257 * FreeBSD Device Interface Entry Points
258 *********************************************************************/
259
260CFATTACH_DECL3_NEW(ixg, sizeof(struct adapter),
261 ixgbe_probe, ixgbe_attach, ixgbe_detach, NULL, NULL, NULL,
262 DVF_DETACH_SHUTDOWN);
263
264#if 0
265devclass_t ixgbe_devclass;
266DRIVER_MODULE(ixgbe, pci, ixgbe_driver, ixgbe_devclass, 0, 0);
267
268MODULE_DEPEND(ixgbe, pci, 1, 1, 1);
269MODULE_DEPEND(ixgbe, ether, 1, 1, 1);
270#endif
271
272/*
273** TUNEABLE PARAMETERS:
274*/
275
276/*
277** AIM: Adaptive Interrupt Moderation
278** which means that the interrupt rate
279** is varied over time based on the
280** traffic for that interrupt vector
281*/
282static int ixgbe_enable_aim = TRUE;
283#define SYSCTL_INT(__x, __y)
284SYSCTL_INT("hw.ixgbe.enable_aim", &ixgbe_enable_aim);
285
286static int ixgbe_max_interrupt_rate = (4000000 / IXGBE_LOW_LATENCY);
287SYSCTL_INT("hw.ixgbe.max_interrupt_rate", &ixgbe_max_interrupt_rate);
288
289/* How many packets rxeof tries to clean at a time */
290static int ixgbe_rx_process_limit = 256;
291SYSCTL_INT("hw.ixgbe.rx_process_limit", &ixgbe_rx_process_limit);
292
293/* How many packets txeof tries to clean at a time */
294static int ixgbe_tx_process_limit = 256;
295SYSCTL_INT("hw.ixgbe.tx_process_limit", &ixgbe_tx_process_limit);
296
297/*
298** Smart speed setting, default to on
299** this only works as a compile option
300** right now as its during attach, set
301** this to 'ixgbe_smart_speed_off' to
302** disable.
303*/
304static int ixgbe_smart_speed = ixgbe_smart_speed_on;
305
306/*
307 * MSIX should be the default for best performance,
308 * but this allows it to be forced off for testing.
309 */
310static int ixgbe_enable_msix = 1;
311SYSCTL_INT("hw.ixgbe.enable_msix", &ixgbe_enable_msix);
312
313#if defined(NETBSD_MSI_OR_MSIX)
314/*
315 * Number of Queues, can be set to 0,
316 * it then autoconfigures based on the
317 * number of cpus with a max of 8. This
318 * can be overriden manually here.
319 */
320static int ixgbe_num_queues = 1;
321SYSCTL_INT("hw.ixgbe.num_queues", &ixgbe_num_queues);
322#endif
323
324/*
325** Number of TX descriptors per ring,
326** setting higher than RX as this seems
327** the better performing choice.
328*/
329static int ixgbe_txd = PERFORM_TXD;
330SYSCTL_INT("hw.ixgbe.txd", &ixgbe_txd);
331
332/* Number of RX descriptors per ring */
333static int ixgbe_rxd = PERFORM_RXD;
334SYSCTL_INT("hw.ixgbe.rxd", &ixgbe_rxd);
335
336/*
337** Defining this on will allow the use
338** of unsupported SFP+ modules, note that
339** doing so you are on your own :)
340*/
341static int allow_unsupported_sfp = false;
342SYSCTL_INT("hw.ix.unsupported_sfp", &allow_unsupported_sfp);
343
344/*
345** HW RSC control:
346** this feature only works with
347** IPv4, and only on 82599 and later.
348** Also this will cause IP forwarding to
349** fail and that can't be controlled by
350** the stack as LRO can. For all these
351** reasons I've deemed it best to leave
352** this off and not bother with a tuneable
353** interface, this would need to be compiled
354** to enable.
355*/
356static bool ixgbe_rsc_enable = FALSE;
357
358/* Keep running tab on them for sanity check */
359static int ixgbe_total_ports;
360
361#ifdef IXGBE_FDIR
362/*
363** For Flow Director: this is the
364** number of TX packets we sample
365** for the filter pool, this means
366** every 20th packet will be probed.
367**
368** This feature can be disabled by
369** setting this to 0.
370*/
371static int atr_sample_rate = 20;
372/*
373** Flow Director actually 'steals'
374** part of the packet buffer as its
375** filter pool, this variable controls
376** how much it uses:
377** 0 = 64K, 1 = 128K, 2 = 256K
378*/
379static int fdir_pballoc = 1;
380#endif
381
382#ifdef DEV_NETMAP
383/*
384 * The #ifdef DEV_NETMAP / #endif blocks in this file are meant to
385 * be a reference on how to implement netmap support in a driver.
386 * Additional comments are in ixgbe_netmap.h .
387 *
388 * <dev/netmap/ixgbe_netmap.h> contains functions for netmap support
389 * that extend the standard driver.
390 */
391#include <dev/netmap/ixgbe_netmap.h>
392#endif /* DEV_NETMAP */
393
394/*********************************************************************
395 * Device identification routine
396 *
397 * ixgbe_probe determines if the driver should be loaded on
398 * adapter based on PCI vendor/device id of the adapter.
399 *
400 * return 1 on success, 0 on failure
401 *********************************************************************/
402
403static int
404ixgbe_probe(device_t dev, cfdata_t cf, void *aux)
405{
406 const struct pci_attach_args *pa = aux;
407
408 return (ixgbe_lookup(pa) != NULL) ? 1 : 0;
409}
410
411static ixgbe_vendor_info_t *
412ixgbe_lookup(const struct pci_attach_args *pa)
413{
414 pcireg_t subid;
415 ixgbe_vendor_info_t *ent;
416
417 INIT_DEBUGOUT("ixgbe_probe: begin");
418
419 if (PCI_VENDOR(pa->pa_id) != IXGBE_INTEL_VENDOR_ID)
420 return NULL;
421
422 subid = pci_conf_read(pa->pa_pc, pa->pa_tag, PCI_SUBSYS_ID_REG);
423
424 for (ent = ixgbe_vendor_info_array; ent->vendor_id != 0; ent++) {
425 if (PCI_VENDOR(pa->pa_id) == ent->vendor_id &&
426 PCI_PRODUCT(pa->pa_id) == ent->device_id &&
427
428 (PCI_SUBSYS_VENDOR(subid) == ent->subvendor_id ||
429 ent->subvendor_id == 0) &&
430
431 (PCI_SUBSYS_ID(subid) == ent->subdevice_id ||
432 ent->subdevice_id == 0)) {
433 ++ixgbe_total_ports;
434 return ent;
435 }
436 }
437 return NULL;
438}
439
440
441static void
442ixgbe_sysctl_attach(struct adapter *adapter)
443{
444 struct sysctllog **log;
445 const struct sysctlnode *rnode, *cnode;
446 device_t dev;
447
448 dev = adapter->dev;
449 log = &adapter->sysctllog;
450
451 if ((rnode = ixgbe_sysctl_instance(adapter)) == NULL) {
452 aprint_error_dev(dev, "could not create sysctl root\n");
453 return;
454 }
455
456 if (sysctl_createv(log, 0, &rnode, &cnode,
457 CTLFLAG_READONLY, CTLTYPE_INT,
458 "num_rx_desc", SYSCTL_DESCR("Number of rx descriptors"),
459 NULL, 0, &adapter->num_rx_desc, 0, CTL_CREATE, CTL_EOL) != 0)
460 aprint_error_dev(dev, "could not create sysctl\n");
461
462 if (sysctl_createv(log, 0, &rnode, &cnode,
463 CTLFLAG_READONLY, CTLTYPE_INT,
464 "num_queues", SYSCTL_DESCR("Number of queues"),
465 NULL, 0, &adapter->num_queues, 0, CTL_CREATE, CTL_EOL) != 0)
466 aprint_error_dev(dev, "could not create sysctl\n");
467
468 if (sysctl_createv(log, 0, &rnode, &cnode,
469 CTLFLAG_READWRITE, CTLTYPE_INT,
470 "fc", SYSCTL_DESCR("Flow Control"),
471 ixgbe_set_flowcntl, 0, (void *)adapter, 0, CTL_CREATE, CTL_EOL) != 0)
472 aprint_error_dev(dev, "could not create sysctl\n");
473
474 /* XXX This is an *instance* sysctl controlling a *global* variable.
475 * XXX It's that way in the FreeBSD driver that this derives from.
476 */
477 if (sysctl_createv(log, 0, &rnode, &cnode,
478 CTLFLAG_READWRITE, CTLTYPE_INT,
479 "enable_aim", SYSCTL_DESCR("Interrupt Moderation"),
480 NULL, 0, &ixgbe_enable_aim, 0, CTL_CREATE, CTL_EOL) != 0)
481 aprint_error_dev(dev, "could not create sysctl\n");
482
483 if (sysctl_createv(log, 0, &rnode, &cnode,
484 CTLFLAG_READWRITE, CTLTYPE_INT,
485 "advertise_speed", SYSCTL_DESCR("Link Speed"),
486 ixgbe_set_advertise, 0, (void *)adapter, 0, CTL_CREATE, CTL_EOL) != 0)
487 aprint_error_dev(dev, "could not create sysctl\n");
488
489 if (sysctl_createv(log, 0, &rnode, &cnode,
490 CTLFLAG_READWRITE, CTLTYPE_INT,
491 "ts", SYSCTL_DESCR("Thermal Test"),
492 ixgbe_set_thermal_test, 0, (void *)adapter, 0, CTL_CREATE, CTL_EOL) != 0)
493 aprint_error_dev(dev, "could not create sysctl\n");
494}
495
496/*********************************************************************
497 * Device initialization routine
498 *
499 * The attach entry point is called when the driver is being loaded.
500 * This routine identifies the type of hardware, allocates all resources
501 * and initializes the hardware.
502 *
503 * return 0 on success, positive on failure
504 *********************************************************************/
505
506static void
507ixgbe_attach(device_t parent, device_t dev, void *aux)
508{
509 struct adapter *adapter;
510 struct ixgbe_hw *hw;
511 int error = -1;
512 u16 csum;
513 u32 ctrl_ext;
514 ixgbe_vendor_info_t *ent;
515 const struct pci_attach_args *pa = aux;
516
517 INIT_DEBUGOUT("ixgbe_attach: begin");
518
519 /* Allocate, clear, and link in our adapter structure */
520 adapter = device_private(dev);
521 adapter->dev = adapter->osdep.dev = dev;
522 hw = &adapter->hw;
523 adapter->osdep.pc = pa->pa_pc;
524 adapter->osdep.tag = pa->pa_tag;
525 adapter->osdep.dmat = pa->pa_dmat;
526 adapter->osdep.attached = false;
527
528 ent = ixgbe_lookup(pa);
529
530 KASSERT(ent != NULL);
531
532 aprint_normal(": %s, Version - %s\n",
533 ixgbe_strings[ent->index], ixgbe_driver_version);
534
535 /* Core Lock Init*/
536 IXGBE_CORE_LOCK_INIT(adapter, device_xname(dev));
537
538 /* SYSCTL APIs */
539
540 ixgbe_sysctl_attach(adapter);
541
542 /* Set up the timer callout */
543 callout_init(&adapter->timer, 0);
544
545 /* Determine hardware revision */
546 ixgbe_identify_hardware(adapter);
547
548 /* Do base PCI setup - map BAR0 */
549 if (ixgbe_allocate_pci_resources(adapter, pa)) {
550 aprint_error_dev(dev, "Allocation of PCI resources failed\n");
551 error = ENXIO;
552 goto err_out;
553 }
554
555 /* Do descriptor calc and sanity checks */
556 if (((ixgbe_txd * sizeof(union ixgbe_adv_tx_desc)) % DBA_ALIGN) != 0 ||
557 ixgbe_txd < MIN_TXD || ixgbe_txd > MAX_TXD) {
558 aprint_error_dev(dev, "TXD config issue, using default!\n");
559 adapter->num_tx_desc = DEFAULT_TXD;
560 } else
561 adapter->num_tx_desc = ixgbe_txd;
562
563 /*
564 ** With many RX rings it is easy to exceed the
565 ** system mbuf allocation. Tuning nmbclusters
566 ** can alleviate this.
567 */
568 if (nmbclusters > 0 ) {
569 int s;
570 s = (ixgbe_rxd * adapter->num_queues) * ixgbe_total_ports;
571 if (s > nmbclusters) {
572 aprint_error_dev(dev, "RX Descriptors exceed "
573 "system mbuf max, using default instead!\n");
574 ixgbe_rxd = DEFAULT_RXD;
575 }
576 }
577
578 if (((ixgbe_rxd * sizeof(union ixgbe_adv_rx_desc)) % DBA_ALIGN) != 0 ||
579 ixgbe_rxd < MIN_RXD || ixgbe_rxd > MAX_RXD) {
580 aprint_error_dev(dev, "RXD config issue, using default!\n");
581 adapter->num_rx_desc = DEFAULT_RXD;
582 } else
583 adapter->num_rx_desc = ixgbe_rxd;
584
585 /* Allocate our TX/RX Queues */
586 if (ixgbe_allocate_queues(adapter)) {
587 error = ENOMEM;
588 goto err_out;
589 }
590
591 /* Allocate multicast array memory. */
592 adapter->mta = malloc(sizeof(u8) * IXGBE_ETH_LENGTH_OF_ADDRESS *
593 MAX_NUM_MULTICAST_ADDRESSES, M_DEVBUF, M_NOWAIT);
594 if (adapter->mta == NULL) {
595 aprint_error_dev(dev, "Cannot allocate multicast setup array\n");
596 error = ENOMEM;
597 goto err_late;
598 }
599
600 /* Initialize the shared code */
601 hw->allow_unsupported_sfp = allow_unsupported_sfp;
602 error = ixgbe_init_shared_code(hw);
603 if (error == IXGBE_ERR_SFP_NOT_PRESENT) {
604 /*
605 ** No optics in this port, set up
606 ** so the timer routine will probe
607 ** for later insertion.
608 */
609 adapter->sfp_probe = TRUE;
610 error = 0;
611 } else if (error == IXGBE_ERR_SFP_NOT_SUPPORTED) {
612 aprint_error_dev(dev,"Unsupported SFP+ module detected!\n");
613 error = EIO;
614 goto err_late;
615 } else if (error) {
616 aprint_error_dev(dev,"Unable to initialize the shared code\n");
617 error = EIO;
618 goto err_late;
619 }
620
621 /* Make sure we have a good EEPROM before we read from it */
622 if (ixgbe_validate_eeprom_checksum(&adapter->hw, &csum) < 0) {
623 aprint_error_dev(dev,"The EEPROM Checksum Is Not Valid\n");
624 error = EIO;
625 goto err_late;
626 }
627
628 error = ixgbe_init_hw(hw);
629 switch (error) {
630 case IXGBE_ERR_EEPROM_VERSION:
631 aprint_error_dev(dev, "This device is a pre-production adapter/"
632 "LOM. Please be aware there may be issues associated "
633 "with your hardware.\n If you are experiencing problems "
634 "please contact your Intel or hardware representative "
635 "who provided you with this hardware.\n");
636 break;
637 case IXGBE_ERR_SFP_NOT_SUPPORTED:
638 aprint_error_dev(dev,"Unsupported SFP+ Module\n");
639 error = EIO;
640 aprint_error_dev(dev,"Hardware Initialization Failure\n");
641 goto err_late;
642 case IXGBE_ERR_SFP_NOT_PRESENT:
643 device_printf(dev,"No SFP+ Module found\n");
644 /* falls thru */
645 default:
646 break;
647 }
648
649 /* Detect and set physical type */
650 ixgbe_setup_optics(adapter);
651
652 error = -1;
653 if ((adapter->msix > 1) && (ixgbe_enable_msix))
654 error = ixgbe_allocate_msix(adapter, pa);
655 if (error != 0)
656 error = ixgbe_allocate_legacy(adapter, pa);
657 if (error)
658 goto err_late;
659
660 /* Setup OS specific network interface */
661 if (ixgbe_setup_interface(dev, adapter) != 0)
662 goto err_late;
663
664 /* Initialize statistics */
665 ixgbe_update_stats_counters(adapter);
666
667 /*
668 ** Check PCIE slot type/speed/width
669 */
670 ixgbe_get_slot_info(hw);
671
672 /* Set an initial default flow control value */
673 adapter->fc = ixgbe_fc_full;
674
675 /* let hardware know driver is loaded */
676 ctrl_ext = IXGBE_READ_REG(hw, IXGBE_CTRL_EXT);
677 ctrl_ext |= IXGBE_CTRL_EXT_DRV_LOAD;
678 IXGBE_WRITE_REG(hw, IXGBE_CTRL_EXT, ctrl_ext);
679
680 ixgbe_add_hw_stats(adapter);
681
682#ifdef DEV_NETMAP
683 ixgbe_netmap_attach(adapter);
684#endif /* DEV_NETMAP */
685 INIT_DEBUGOUT("ixgbe_attach: end");
686 adapter->osdep.attached = true;
687 return;
688err_late:
689 ixgbe_free_transmit_structures(adapter);
690 ixgbe_free_receive_structures(adapter);
691err_out:
692 if (adapter->ifp != NULL)
693 if_free(adapter->ifp);
694 ixgbe_free_pci_resources(adapter);
695 if (adapter->mta != NULL)
696 free(adapter->mta, M_DEVBUF);
697 return;
698
699}
700
701/*********************************************************************
702 * Device removal routine
703 *
704 * The detach entry point is called when the driver is being removed.
705 * This routine stops the adapter and deallocates all the resources
706 * that were allocated for driver operation.
707 *
708 * return 0 on success, positive on failure
709 *********************************************************************/
710
711static int
712ixgbe_detach(device_t dev, int flags)
713{
714 struct adapter *adapter = device_private(dev);
715 struct rx_ring *rxr = adapter->rx_rings;
716 struct ixgbe_hw_stats *stats = &adapter->stats;
717 struct ix_queue *que = adapter->queues;
718 struct tx_ring *txr = adapter->tx_rings;
719 u32 ctrl_ext;
720
721 INIT_DEBUGOUT("ixgbe_detach: begin");
722 if (adapter->osdep.attached == false)
723 return 0;
724
725#if NVLAN > 0
726 /* Make sure VLANs are not using driver */
727 if (!VLAN_ATTACHED(&adapter->osdep.ec))
728 ; /* nothing to do: no VLANs */
729 else if ((flags & (DETACH_SHUTDOWN|DETACH_FORCE)) != 0)
730 vlan_ifdetach(adapter->ifp);
731 else {
732 aprint_error_dev(dev, "VLANs in use\n");
733 return EBUSY;
734 }
735#endif
736
737 IXGBE_CORE_LOCK(adapter);
738 ixgbe_stop(adapter);
739 IXGBE_CORE_UNLOCK(adapter);
740
741 for (int i = 0; i < adapter->num_queues; i++, que++, txr++) {
742#ifndef IXGBE_LEGACY_TX
743 softint_disestablish(txr->txq_si);
744#endif
745 softint_disestablish(que->que_si);
746 }
747
748 /* Drain the Link queue */
749 softint_disestablish(adapter->link_si);
750 softint_disestablish(adapter->mod_si);
751 softint_disestablish(adapter->msf_si);
752#ifdef IXGBE_FDIR
753 softint_disestablish(adapter->fdir_si);
754#endif
755
756 /* let hardware know driver is unloading */
757 ctrl_ext = IXGBE_READ_REG(&adapter->hw, IXGBE_CTRL_EXT);
758 ctrl_ext &= ~IXGBE_CTRL_EXT_DRV_LOAD;
759 IXGBE_WRITE_REG(&adapter->hw, IXGBE_CTRL_EXT, ctrl_ext);
760
761 ether_ifdetach(adapter->ifp);
762 callout_halt(&adapter->timer, NULL);
763#ifdef DEV_NETMAP
764 netmap_detach(adapter->ifp);
765#endif /* DEV_NETMAP */
766 ixgbe_free_pci_resources(adapter);
767#if 0 /* XXX the NetBSD port is probably missing something here */
768 bus_generic_detach(dev);
769#endif
770 if_detach(adapter->ifp);
771
772 sysctl_teardown(&adapter->sysctllog);
773 evcnt_detach(&adapter->handleq);
774 evcnt_detach(&adapter->req);
775 evcnt_detach(&adapter->morerx);
776 evcnt_detach(&adapter->moretx);
777 evcnt_detach(&adapter->txloops);
778 evcnt_detach(&adapter->efbig_tx_dma_setup);
779 evcnt_detach(&adapter->m_defrag_failed);
780 evcnt_detach(&adapter->efbig2_tx_dma_setup);
781 evcnt_detach(&adapter->einval_tx_dma_setup);
782 evcnt_detach(&adapter->other_tx_dma_setup);
783 evcnt_detach(&adapter->eagain_tx_dma_setup);
784 evcnt_detach(&adapter->enomem_tx_dma_setup);
785 evcnt_detach(&adapter->watchdog_events);
786 evcnt_detach(&adapter->tso_err);
787 evcnt_detach(&adapter->link_irq);
788
789 txr = adapter->tx_rings;
790 for (int i = 0; i < adapter->num_queues; i++, rxr++, txr++) {
791 evcnt_detach(&txr->no_desc_avail);
792 evcnt_detach(&txr->total_packets);
793 evcnt_detach(&txr->tso_tx);
794
795 if (i < __arraycount(adapter->stats.mpc)) {
796 evcnt_detach(&adapter->stats.mpc[i]);
797 }
798 if (i < __arraycount(adapter->stats.pxontxc)) {
799 evcnt_detach(&adapter->stats.pxontxc[i]);
800 evcnt_detach(&adapter->stats.pxonrxc[i]);
801 evcnt_detach(&adapter->stats.pxofftxc[i]);
802 evcnt_detach(&adapter->stats.pxoffrxc[i]);
803 evcnt_detach(&adapter->stats.pxon2offc[i]);
804 }
805 if (i < __arraycount(adapter->stats.qprc)) {
806 evcnt_detach(&adapter->stats.qprc[i]);
807 evcnt_detach(&adapter->stats.qptc[i]);
808 evcnt_detach(&adapter->stats.qbrc[i]);
809 evcnt_detach(&adapter->stats.qbtc[i]);
810 evcnt_detach(&adapter->stats.qprdc[i]);
811 }
812
813 evcnt_detach(&rxr->rx_packets);
814 evcnt_detach(&rxr->rx_bytes);
815 evcnt_detach(&rxr->rx_copies);
816 evcnt_detach(&rxr->no_jmbuf);
817 evcnt_detach(&rxr->rx_discarded);
818 evcnt_detach(&rxr->rx_irq);
819 }
820 evcnt_detach(&stats->ipcs);
821 evcnt_detach(&stats->l4cs);
822 evcnt_detach(&stats->ipcs_bad);
823 evcnt_detach(&stats->l4cs_bad);
824 evcnt_detach(&stats->intzero);
825 evcnt_detach(&stats->legint);
826 evcnt_detach(&stats->crcerrs);
827 evcnt_detach(&stats->illerrc);
828 evcnt_detach(&stats->errbc);
829 evcnt_detach(&stats->mspdc);
830 evcnt_detach(&stats->mlfc);
831 evcnt_detach(&stats->mrfc);
832 evcnt_detach(&stats->rlec);
833 evcnt_detach(&stats->lxontxc);
834 evcnt_detach(&stats->lxonrxc);
835 evcnt_detach(&stats->lxofftxc);
836 evcnt_detach(&stats->lxoffrxc);
837
838 /* Packet Reception Stats */
839 evcnt_detach(&stats->tor);
840 evcnt_detach(&stats->gorc);
841 evcnt_detach(&stats->tpr);
842 evcnt_detach(&stats->gprc);
843 evcnt_detach(&stats->mprc);
844 evcnt_detach(&stats->bprc);
845 evcnt_detach(&stats->prc64);
846 evcnt_detach(&stats->prc127);
847 evcnt_detach(&stats->prc255);
848 evcnt_detach(&stats->prc511);
849 evcnt_detach(&stats->prc1023);
850 evcnt_detach(&stats->prc1522);
851 evcnt_detach(&stats->ruc);
852 evcnt_detach(&stats->rfc);
853 evcnt_detach(&stats->roc);
854 evcnt_detach(&stats->rjc);
855 evcnt_detach(&stats->mngprc);
856 evcnt_detach(&stats->xec);
857
858 /* Packet Transmission Stats */
859 evcnt_detach(&stats->gotc);
860 evcnt_detach(&stats->tpt);
861 evcnt_detach(&stats->gptc);
862 evcnt_detach(&stats->bptc);
863 evcnt_detach(&stats->mptc);
864 evcnt_detach(&stats->mngptc);
865 evcnt_detach(&stats->ptc64);
866 evcnt_detach(&stats->ptc127);
867 evcnt_detach(&stats->ptc255);
868 evcnt_detach(&stats->ptc511);
869 evcnt_detach(&stats->ptc1023);
870 evcnt_detach(&stats->ptc1522);
871
872 ixgbe_free_transmit_structures(adapter);
873 ixgbe_free_receive_structures(adapter);
874 free(adapter->mta, M_DEVBUF);
875
876 IXGBE_CORE_LOCK_DESTROY(adapter);
877 return (0);
878}
879
880/*********************************************************************
881 *
882 * Shutdown entry point
883 *
884 **********************************************************************/
885
886#if 0 /* XXX NetBSD ought to register something like this through pmf(9) */
887static int
888ixgbe_shutdown(device_t dev)
889{
890 struct adapter *adapter = device_private(dev);
891 IXGBE_CORE_LOCK(adapter);
892 ixgbe_stop(adapter);
893 IXGBE_CORE_UNLOCK(adapter);
894 return (0);
895}
896#endif
897
898
899#ifdef IXGBE_LEGACY_TX
900/*********************************************************************
901 * Transmit entry point
902 *
903 * ixgbe_start is called by the stack to initiate a transmit.
904 * The driver will remain in this routine as long as there are
905 * packets to transmit and transmit resources are available.
906 * In case resources are not available stack is notified and
907 * the packet is requeued.
908 **********************************************************************/
909
910static void
911ixgbe_start_locked(struct tx_ring *txr, struct ifnet * ifp)
912{
913 int rc;
914 struct mbuf *m_head;
915 struct adapter *adapter = txr->adapter;
916
917 IXGBE_TX_LOCK_ASSERT(txr);
918
919 if ((ifp->if_flags & IFF_RUNNING) == 0)
920 return;
921 if (!adapter->link_active)
922 return;
923
924 while (!IFQ_IS_EMPTY(&ifp->if_snd)) {
925 if (txr->tx_avail <= IXGBE_QUEUE_MIN_FREE)
926 break;
927
928 IFQ_POLL(&ifp->if_snd, m_head);
929 if (m_head == NULL)
930 break;
931
932 if ((rc = ixgbe_xmit(txr, m_head)) == EAGAIN) {
933 break;
934 }
935 IFQ_DEQUEUE(&ifp->if_snd, m_head);
936 if (rc == EFBIG) {
937 struct mbuf *mtmp;
938
939 if ((mtmp = m_defrag(m_head, M_NOWAIT)) != NULL) {
940 m_head = mtmp;
941 rc = ixgbe_xmit(txr, m_head);
942 if (rc != 0)
943 adapter->efbig2_tx_dma_setup.ev_count++;
944 } else
945 adapter->m_defrag_failed.ev_count++;
946 }
947 if (rc != 0) {
948 m_freem(m_head);
949 continue;
950 }
951
952 /* Send a copy of the frame to the BPF listener */
953 bpf_mtap(ifp, m_head);
954
955 /* Set watchdog on */
956 getmicrotime(&txr->watchdog_time);
957 txr->queue_status = IXGBE_QUEUE_WORKING;
958
959 }
960 return;
961}
962
963/*
964 * Legacy TX start - called by the stack, this
965 * always uses the first tx ring, and should
966 * not be used with multiqueue tx enabled.
967 */
968static void
969ixgbe_start(struct ifnet *ifp)
970{
971 struct adapter *adapter = ifp->if_softc;
972 struct tx_ring *txr = adapter->tx_rings;
973
974 if (ifp->if_flags & IFF_RUNNING) {
975 IXGBE_TX_LOCK(txr);
976 ixgbe_start_locked(txr, ifp);
977 IXGBE_TX_UNLOCK(txr);
978 }
979 return;
980}
981
982#else /* ! IXGBE_LEGACY_TX */
983
984/*
985** Multiqueue Transmit driver
986**
987*/
988static int
989ixgbe_mq_start(struct ifnet *ifp, struct mbuf *m)
990{
991 struct adapter *adapter = ifp->if_softc;
992 struct ix_queue *que;
993 struct tx_ring *txr;
994 int i, err = 0;
995#ifdef RSS
996 uint32_t bucket_id;
997#endif
998
999 /* Which queue to use */
1000 /*
1001 * When doing RSS, map it to the same outbound queue
1002 * as the incoming flow would be mapped to.
1003 *
1004 * If everything is setup correctly, it should be the
1005 * same bucket that the current CPU we're on is.
1006 */
1007 if (M_HASHTYPE_GET(m) != M_HASHTYPE_NONE) {
1008#ifdef RSS
1009 if (rss_hash2bucket(m->m_pkthdr.flowid,
1010 M_HASHTYPE_GET(m), &bucket_id) == 0) {
1011 /* XXX TODO: spit out something if bucket_id > num_queues? */
1012 i = bucket_id % adapter->num_queues;
1013 } else {
1014#endif
1015 i = m->m_pkthdr.flowid % adapter->num_queues;
1016#ifdef RSS
1017 }
1018#endif
1019 } else {
1020 i = curcpu % adapter->num_queues;
1021 }
1022
1023 txr = &adapter->tx_rings[i];
1024 que = &adapter->queues[i];
1025
1026 err = drbr_enqueue(ifp, txr->br, m);
1027 if (err)
1028 return (err);
1029 if (IXGBE_TX_TRYLOCK(txr)) {
1030 ixgbe_mq_start_locked(ifp, txr);
1031 IXGBE_TX_UNLOCK(txr);
1032 } else
1033 softint_schedule(txr->txq_si);
1034
1035 return (0);
1036}
1037
1038static int
1039ixgbe_mq_start_locked(struct ifnet *ifp, struct tx_ring *txr)
1040{
1041 struct adapter *adapter = txr->adapter;
1042 struct mbuf *next;
1043 int enqueued = 0, err = 0;
1044
1045 if (((ifp->if_flags & IFF_RUNNING) == 0) ||
1046 adapter->link_active == 0)
1047 return (ENETDOWN);
1048
1049 /* Process the queue */
1050#if __FreeBSD_version < 901504
1051 next = drbr_dequeue(ifp, txr->br);
1052 while (next != NULL) {
1053 if ((err = ixgbe_xmit(txr, &next)) != 0) {
1054 if (next != NULL)
1055 err = drbr_enqueue(ifp, txr->br, next);
1056#else
1057 while ((next = drbr_peek(ifp, txr->br)) != NULL) {
1058 if ((err = ixgbe_xmit(txr, &next)) != 0) {
1059 if (next == NULL) {
1060 drbr_advance(ifp, txr->br);
1061 } else {
1062 drbr_putback(ifp, txr->br, next);
1063 }
1064#endif
1065 break;
1066 }
1067#if __FreeBSD_version >= 901504
1068 drbr_advance(ifp, txr->br);
1069#endif
1070 enqueued++;
1071 /* Send a copy of the frame to the BPF listener */
1072 bpf_mtap(ifp, next);
1073 if ((ifp->if_flags & IFF_RUNNING) == 0)
1074 break;
1075#if __FreeBSD_version < 901504
1076 next = drbr_dequeue(ifp, txr->br);
1077#endif
1078 }
1079
1080 if (enqueued > 0) {
1081 /* Set watchdog on */
1082 txr->queue_status = IXGBE_QUEUE_WORKING;
1083 getmicrotime(&txr->watchdog_time);
1084 }
1085
1086 if (txr->tx_avail < IXGBE_TX_CLEANUP_THRESHOLD)
1087 ixgbe_txeof(txr);
1088
1089 return (err);
1090}
1091
1092/*
1093 * Called from a taskqueue to drain queued transmit packets.
1094 */
1095static void
1096ixgbe_deferred_mq_start(void *arg, int pending)
1097{
1098 struct tx_ring *txr = arg;
1099 struct adapter *adapter = txr->adapter;
1100 struct ifnet *ifp = adapter->ifp;
1101
1102 IXGBE_TX_LOCK(txr);
1103 if (!drbr_empty(ifp, txr->br))
1104 ixgbe_mq_start_locked(ifp, txr);
1105 IXGBE_TX_UNLOCK(txr);
1106}
1107
1108/*
1109** Flush all ring buffers
1110*/
1111static void
1112ixgbe_qflush(struct ifnet *ifp)
1113{
1114 struct adapter *adapter = ifp->if_softc;
1115 struct tx_ring *txr = adapter->tx_rings;
1116 struct mbuf *m;
1117
1118 for (int i = 0; i < adapter->num_queues; i++, txr++) {
1119 IXGBE_TX_LOCK(txr);
1120 while ((m = buf_ring_dequeue_sc(txr->br)) != NULL)
1121 m_freem(m);
1122 IXGBE_TX_UNLOCK(txr);
1123 }
1124 if_qflush(ifp);
1125}
1126#endif /* IXGBE_LEGACY_TX */
1127
1128static int
1129ixgbe_ifflags_cb(struct ethercom *ec)
1130{
1131 struct ifnet *ifp = &ec->ec_if;
1132 struct adapter *adapter = ifp->if_softc;
1133 int change = ifp->if_flags ^ adapter->if_flags, rc = 0;
1134
1135 IXGBE_CORE_LOCK(adapter);
1136
1137 if (change != 0)
1138 adapter->if_flags = ifp->if_flags;
1139
1140 if ((change & ~(IFF_CANTCHANGE|IFF_DEBUG)) != 0)
1141 rc = ENETRESET;
1142 else if ((change & (IFF_PROMISC | IFF_ALLMULTI)) != 0)
1143 ixgbe_set_promisc(adapter);
1144
1145 /* Set up VLAN support and filter */
1146 ixgbe_setup_vlan_hw_support(adapter);
1147
1148 IXGBE_CORE_UNLOCK(adapter);
1149
1150 return rc;
1151}
1152
1153/*********************************************************************
1154 * Ioctl entry point
1155 *
1156 * ixgbe_ioctl is called when the user wants to configure the
1157 * interface.
1158 *
1159 * return 0 on success, positive on failure
1160 **********************************************************************/
1161
1162static int
1163ixgbe_ioctl(struct ifnet * ifp, u_long command, void *data)
1164{
1165 struct adapter *adapter = ifp->if_softc;
1166 struct ixgbe_hw *hw = &adapter->hw;
1167 struct ifcapreq *ifcr = data;
1168 struct ifreq *ifr = data;
1169 int error = 0;
1170 int l4csum_en;
1171 const int l4csum = IFCAP_CSUM_TCPv4_Rx|IFCAP_CSUM_UDPv4_Rx|
1172 IFCAP_CSUM_TCPv6_Rx|IFCAP_CSUM_UDPv6_Rx;
1173
1174 switch (command) {
1175 case SIOCSIFFLAGS:
1176 IOCTL_DEBUGOUT("ioctl: SIOCSIFFLAGS (Set Interface Flags)");
1177 break;
1178 case SIOCADDMULTI:
1179 case SIOCDELMULTI:
1180 IOCTL_DEBUGOUT("ioctl: SIOC(ADD|DEL)MULTI");
1181 break;
1182 case SIOCSIFMEDIA:
1183 case SIOCGIFMEDIA:
1184 IOCTL_DEBUGOUT("ioctl: SIOCxIFMEDIA (Get/Set Interface Media)");
1185 break;
1186 case SIOCSIFCAP:
1187 IOCTL_DEBUGOUT("ioctl: SIOCSIFCAP (Set Capabilities)");
1188 break;
1189 case SIOCSIFMTU:
1190 IOCTL_DEBUGOUT("ioctl: SIOCSIFMTU (Set Interface MTU)");
1191 break;
1192 default:
1193 IOCTL_DEBUGOUT1("ioctl: UNKNOWN (0x%X)\n", (int)command);
1194 break;
1195 }
1196
1197 switch (command) {
1198 case SIOCSIFMEDIA:
1199 case SIOCGIFMEDIA:
1200 return ifmedia_ioctl(ifp, ifr, &adapter->media, command);
1201 case SIOCGI2C:
1202 {
1203 struct ixgbe_i2c_req i2c;
1204 IOCTL_DEBUGOUT("ioctl: SIOCGI2C (Get I2C Data)");
1205 error = copyin(ifr->ifr_data, &i2c, sizeof(i2c));
1206 if (error != 0)
1207 break;
1208 if (i2c.dev_addr != 0xA0 && i2c.dev_addr != 0xA2) {
1209 error = EINVAL;
1210 break;
1211 }
1212 if (i2c.len > sizeof(i2c.data)) {
1213 error = EINVAL;
1214 break;
1215 }
1216
1217 hw->phy.ops.read_i2c_byte(hw, i2c.offset,
1218 i2c.dev_addr, i2c.data);
1219 error = copyout(&i2c, ifr->ifr_data, sizeof(i2c));
1220 break;
1221 }
1222 case SIOCSIFCAP:
1223 /* Layer-4 Rx checksum offload has to be turned on and
1224 * off as a unit.
1225 */
1226 l4csum_en = ifcr->ifcr_capenable & l4csum;
1227 if (l4csum_en != l4csum && l4csum_en != 0)
1228 return EINVAL;
1229 /*FALLTHROUGH*/
1230 case SIOCADDMULTI:
1231 case SIOCDELMULTI:
1232 case SIOCSIFFLAGS:
1233 case SIOCSIFMTU:
1234 default:
1235 if ((error = ether_ioctl(ifp, command, data)) != ENETRESET)
1236 return error;
1237 if ((ifp->if_flags & IFF_RUNNING) == 0)
1238 ;
1239 else if (command == SIOCSIFCAP || command == SIOCSIFMTU) {
1240 IXGBE_CORE_LOCK(adapter);
1241 ixgbe_init_locked(adapter);
1242 IXGBE_CORE_UNLOCK(adapter);
1243 } else if (command == SIOCADDMULTI || command == SIOCDELMULTI) {
1244 /*
1245 * Multicast list has changed; set the hardware filter
1246 * accordingly.
1247 */
1248 IXGBE_CORE_LOCK(adapter);
1249 ixgbe_disable_intr(adapter);
1250 ixgbe_set_multi(adapter);
1251 ixgbe_enable_intr(adapter);
1252 IXGBE_CORE_UNLOCK(adapter);
1253 }
1254 return 0;
1255 }
1256
1257 return error;
1258}
1259
1260/*********************************************************************
1261 * Init entry point
1262 *
1263 * This routine is used in two ways. It is used by the stack as
1264 * init entry point in network interface structure. It is also used
1265 * by the driver as a hw/sw initialization routine to get to a
1266 * consistent state.
1267 *
1268 * return 0 on success, positive on failure
1269 **********************************************************************/
1270#define IXGBE_MHADD_MFS_SHIFT 16
1271
1272static void
1273ixgbe_init_locked(struct adapter *adapter)
1274{
1275 struct ifnet *ifp = adapter->ifp;
1276 device_t dev = adapter->dev;
1277 struct ixgbe_hw *hw = &adapter->hw;
1278 u32 k, txdctl, mhadd, gpie;
1279 u32 rxdctl, rxctrl;
1280
1281 /* XXX check IFF_UP and IFF_RUNNING, power-saving state! */
1282
1283 KASSERT(mutex_owned(&adapter->core_mtx));
1284 INIT_DEBUGOUT("ixgbe_init_locked: begin");
1285 hw->adapter_stopped = FALSE;
1286 ixgbe_stop_adapter(hw);
1287 callout_stop(&adapter->timer);
1288
1289 /* XXX I moved this here from the SIOCSIFMTU case in ixgbe_ioctl(). */
1290 adapter->max_frame_size =
1291 ifp->if_mtu + ETHER_HDR_LEN + ETHER_CRC_LEN;
1292
1293 /* reprogram the RAR[0] in case user changed it. */
1294 ixgbe_set_rar(hw, 0, adapter->hw.mac.addr, 0, IXGBE_RAH_AV);
1295
1296 /* Get the latest mac address, User can use a LAA */
1297 memcpy(hw->mac.addr, CLLADDR(adapter->ifp->if_sadl),
1298 IXGBE_ETH_LENGTH_OF_ADDRESS);
1299 ixgbe_set_rar(hw, 0, hw->mac.addr, 0, 1);
1300 hw->addr_ctrl.rar_used_count = 1;
1301
1302 /* Prepare transmit descriptors and buffers */
1303 if (ixgbe_setup_transmit_structures(adapter)) {
1304 device_printf(dev,"Could not setup transmit structures\n");
1305 ixgbe_stop(adapter);
1306 return;
1307 }
1308
1309 ixgbe_init_hw(hw);
1310 ixgbe_initialize_transmit_units(adapter);
1311
1312 /* Setup Multicast table */
1313 ixgbe_set_multi(adapter);
1314
1315 /*
1316 ** Determine the correct mbuf pool
1317 ** for doing jumbo frames
1318 */
1319 if (adapter->max_frame_size <= 2048)
1320 adapter->rx_mbuf_sz = MCLBYTES;
1321 else if (adapter->max_frame_size <= 4096)
1322 adapter->rx_mbuf_sz = MJUMPAGESIZE;
1323 else if (adapter->max_frame_size <= 9216)
1324 adapter->rx_mbuf_sz = MJUM9BYTES;
1325 else
1326 adapter->rx_mbuf_sz = MJUM16BYTES;
1327
1328 /* Prepare receive descriptors and buffers */
1329 if (ixgbe_setup_receive_structures(adapter)) {
1330 device_printf(dev,"Could not setup receive structures\n");
1331 ixgbe_stop(adapter);
1332 return;
1333 }
1334
1335 /* Configure RX settings */
1336 ixgbe_initialize_receive_units(adapter);
1337
1338 gpie = IXGBE_READ_REG(&adapter->hw, IXGBE_GPIE);
1339
1340 /* Enable Fan Failure Interrupt */
1341 gpie |= IXGBE_SDP1_GPIEN;
1342
1343 /* Add for Module detection */
1344 if (hw->mac.type == ixgbe_mac_82599EB)
1345 gpie |= IXGBE_SDP2_GPIEN;
1346
1347 /* Thermal Failure Detection */
1348 if (hw->mac.type == ixgbe_mac_X540)
1349 gpie |= IXGBE_SDP0_GPIEN;
1350
1351 if (adapter->msix > 1) {
1352 /* Enable Enhanced MSIX mode */
1353 gpie |= IXGBE_GPIE_MSIX_MODE;
1354 gpie |= IXGBE_GPIE_EIAME | IXGBE_GPIE_PBA_SUPPORT |
1355 IXGBE_GPIE_OCD;
1356 }
1357 IXGBE_WRITE_REG(hw, IXGBE_GPIE, gpie);
1358
1359 /* Set MTU size */
1360 if (ifp->if_mtu > ETHERMTU) {
1361 mhadd = IXGBE_READ_REG(hw, IXGBE_MHADD);
1362 mhadd &= ~IXGBE_MHADD_MFS_MASK;
1363 mhadd |= adapter->max_frame_size << IXGBE_MHADD_MFS_SHIFT;
1364 IXGBE_WRITE_REG(hw, IXGBE_MHADD, mhadd);
1365 }
1366
1367 /* Now enable all the queues */
1368
1369 for (int i = 0; i < adapter->num_queues; i++) {
1370 txdctl = IXGBE_READ_REG(hw, IXGBE_TXDCTL(i));
1371 txdctl |= IXGBE_TXDCTL_ENABLE;
1372 /* Set WTHRESH to 8, burst writeback */
1373 txdctl |= (8 << 16);
1374 /*
1375 * When the internal queue falls below PTHRESH (32),
1376 * start prefetching as long as there are at least
1377 * HTHRESH (1) buffers ready. The values are taken
1378 * from the Intel linux driver 3.8.21.
1379 * Prefetching enables tx line rate even with 1 queue.
1380 */
1381 txdctl |= (32 << 0) | (1 << 8);
1382 IXGBE_WRITE_REG(hw, IXGBE_TXDCTL(i), txdctl);
1383 }
1384
1385 for (int i = 0; i < adapter->num_queues; i++) {
1386 rxdctl = IXGBE_READ_REG(hw, IXGBE_RXDCTL(i));
1387 if (hw->mac.type == ixgbe_mac_82598EB) {
1388 /*
1389 ** PTHRESH = 21
1390 ** HTHRESH = 4
1391 ** WTHRESH = 8
1392 */
1393 rxdctl &= ~0x3FFFFF;
1394 rxdctl |= 0x080420;
1395 }
1396 rxdctl |= IXGBE_RXDCTL_ENABLE;
1397 IXGBE_WRITE_REG(hw, IXGBE_RXDCTL(i), rxdctl);
1398 /* XXX I don't trust this loop, and I don't trust the
1399 * XXX memory barrier. What is this meant to do? --dyoung
1400 */
1401 for (k = 0; k < 10; k++) {
1402 if (IXGBE_READ_REG(hw, IXGBE_RXDCTL(i)) &
1403 IXGBE_RXDCTL_ENABLE)
1404 break;
1405 else
1406 msec_delay(1);
1407 }
1408 wmb();
1409#ifdef DEV_NETMAP
1410 /*
1411 * In netmap mode, we must preserve the buffers made
1412 * available to userspace before the if_init()
1413 * (this is true by default on the TX side, because
1414 * init makes all buffers available to userspace).
1415 *
1416 * netmap_reset() and the device specific routines
1417 * (e.g. ixgbe_setup_receive_rings()) map these
1418 * buffers at the end of the NIC ring, so here we
1419 * must set the RDT (tail) register to make sure
1420 * they are not overwritten.
1421 *
1422 * In this driver the NIC ring starts at RDH = 0,
1423 * RDT points to the last slot available for reception (?),
1424 * so RDT = num_rx_desc - 1 means the whole ring is available.
1425 */
1426 if (ifp->if_capenable & IFCAP_NETMAP) {
1427 struct netmap_adapter *na = NA(adapter->ifp);
1428 struct netmap_kring *kring = &na->rx_rings[i];
1429 int t = na->num_rx_desc - 1 - nm_kr_rxspace(kring);
1430
1431 IXGBE_WRITE_REG(hw, IXGBE_RDT(i), t);
1432 } else
1433#endif /* DEV_NETMAP */
1434 IXGBE_WRITE_REG(hw, IXGBE_RDT(i), adapter->num_rx_desc - 1);
1435 }
1436
1437 /* Enable Receive engine */
1438 rxctrl = IXGBE_READ_REG(hw, IXGBE_RXCTRL);
1439 if (hw->mac.type == ixgbe_mac_82598EB)
1440 rxctrl |= IXGBE_RXCTRL_DMBYPS;
1441 rxctrl |= IXGBE_RXCTRL_RXEN;
1442 ixgbe_enable_rx_dma(hw, rxctrl);
1443
1444 callout_reset(&adapter->timer, hz, ixgbe_local_timer, adapter);
1445
1446 /* Set up MSI/X routing */
1447 if (ixgbe_enable_msix) {
1448 ixgbe_configure_ivars(adapter);
1449 /* Set up auto-mask */
1450 if (hw->mac.type == ixgbe_mac_82598EB)
1451 IXGBE_WRITE_REG(hw, IXGBE_EIAM, IXGBE_EICS_RTX_QUEUE);
1452 else {
1453 IXGBE_WRITE_REG(hw, IXGBE_EIAM_EX(0), 0xFFFFFFFF);
1454 IXGBE_WRITE_REG(hw, IXGBE_EIAM_EX(1), 0xFFFFFFFF);
1455 }
1456 } else { /* Simple settings for Legacy/MSI */
1457 ixgbe_set_ivar(adapter, 0, 0, 0);
1458 ixgbe_set_ivar(adapter, 0, 0, 1);
1459 IXGBE_WRITE_REG(hw, IXGBE_EIAM, IXGBE_EICS_RTX_QUEUE);
1460 }
1461
1462#ifdef IXGBE_FDIR
1463 /* Init Flow director */
1464 if (hw->mac.type != ixgbe_mac_82598EB) {
1465 u32 hdrm = 32 << fdir_pballoc;
1466
1467 hw->mac.ops.setup_rxpba(hw, 0, hdrm, PBA_STRATEGY_EQUAL);
1468 ixgbe_init_fdir_signature_82599(&adapter->hw, fdir_pballoc);
1469 }
1470#endif
1471
1472 /*
1473 ** Check on any SFP devices that
1474 ** need to be kick-started
1475 */
1476 if (hw->phy.type == ixgbe_phy_none) {
1477 int err = hw->phy.ops.identify(hw);
1478 if (err == IXGBE_ERR_SFP_NOT_SUPPORTED) {
1479 device_printf(dev,
1480 "Unsupported SFP+ module type was detected.\n");
1481 return;
1482 }
1483 }
1484
1485 /* Set moderation on the Link interrupt */
1486 IXGBE_WRITE_REG(hw, IXGBE_EITR(adapter->linkvec), IXGBE_LINK_ITR);
1487
1488 /* Config/Enable Link */
1489 ixgbe_config_link(adapter);
1490
1491 /* Hardware Packet Buffer & Flow Control setup */
1492 {
1493 u32 rxpb, frame, size, tmp;
1494
1495 frame = adapter->max_frame_size;
1496
1497 /* Calculate High Water */
1498 if (hw->mac.type == ixgbe_mac_X540)
1499 tmp = IXGBE_DV_X540(frame, frame);
1500 else
1501 tmp = IXGBE_DV(frame, frame);
1502 size = IXGBE_BT2KB(tmp);
1503 rxpb = IXGBE_READ_REG(hw, IXGBE_RXPBSIZE(0)) >> 10;
1504 hw->fc.high_water[0] = rxpb - size;
1505
1506 /* Now calculate Low Water */
1507 if (hw->mac.type == ixgbe_mac_X540)
1508 tmp = IXGBE_LOW_DV_X540(frame);
1509 else
1510 tmp = IXGBE_LOW_DV(frame);
1511 hw->fc.low_water[0] = IXGBE_BT2KB(tmp);
1512
1513 hw->fc.requested_mode = adapter->fc;
1514 hw->fc.pause_time = IXGBE_FC_PAUSE;
1515 hw->fc.send_xon = TRUE;
1516 }
1517 /* Initialize the FC settings */
1518 ixgbe_start_hw(hw);
1519
1520 /* Set up VLAN support and filter */
1521 ixgbe_setup_vlan_hw_support(adapter);
1522
1523 /* And now turn on interrupts */
1524 ixgbe_enable_intr(adapter);
1525
1526 /* Now inform the stack we're ready */
1527 ifp->if_flags |= IFF_RUNNING;
1528
1529 return;
1530}
1531
1532static int
1533ixgbe_init(struct ifnet *ifp)
1534{
1535 struct adapter *adapter = ifp->if_softc;
1536
1537 IXGBE_CORE_LOCK(adapter);
1538 ixgbe_init_locked(adapter);
1539 IXGBE_CORE_UNLOCK(adapter);
1540 return 0; /* XXX ixgbe_init_locked cannot fail? really? */
1541}
1542
1543
1544/*
1545**
1546** MSIX Interrupt Handlers and Tasklets
1547**
1548*/
1549
1550static inline void
1551ixgbe_enable_queue(struct adapter *adapter, u32 vector)
1552{
1553 struct ixgbe_hw *hw = &adapter->hw;
1554 u64 queue = (u64)(1ULL << vector);
1555 u32 mask;
1556
1557 if (hw->mac.type == ixgbe_mac_82598EB) {
1558 mask = (IXGBE_EIMS_RTX_QUEUE & queue);
1559 IXGBE_WRITE_REG(hw, IXGBE_EIMS, mask);
1560 } else {
1561 mask = (queue & 0xFFFFFFFF);
1562 if (mask)
1563 IXGBE_WRITE_REG(hw, IXGBE_EIMS_EX(0), mask);
1564 mask = (queue >> 32);
1565 if (mask)
1566 IXGBE_WRITE_REG(hw, IXGBE_EIMS_EX(1), mask);
1567 }
1568}
1569
1570__unused static inline void
1571ixgbe_disable_queue(struct adapter *adapter, u32 vector)
1572{
1573 struct ixgbe_hw *hw = &adapter->hw;
1574 u64 queue = (u64)(1ULL << vector);
1575 u32 mask;
1576
1577 if (hw->mac.type == ixgbe_mac_82598EB) {
1578 mask = (IXGBE_EIMS_RTX_QUEUE & queue);
1579 IXGBE_WRITE_REG(hw, IXGBE_EIMC, mask);
1580 } else {
1581 mask = (queue & 0xFFFFFFFF);
1582 if (mask)
1583 IXGBE_WRITE_REG(hw, IXGBE_EIMC_EX(0), mask);
1584 mask = (queue >> 32);
1585 if (mask)
1586 IXGBE_WRITE_REG(hw, IXGBE_EIMC_EX(1), mask);
1587 }
1588}
1589
1590static void
1591ixgbe_handle_que(void *context)
1592{
1593 struct ix_queue *que = context;
1594 struct adapter *adapter = que->adapter;
1595 struct tx_ring *txr = que->txr;
1596 struct ifnet *ifp = adapter->ifp;
1597
1598 adapter->handleq.ev_count++;
1599
1600 if (ifp->if_flags & IFF_RUNNING) {
1601 ixgbe_rxeof(que);
1602 IXGBE_TX_LOCK(txr);
1603 ixgbe_txeof(txr);
1604#ifndef IXGBE_LEGACY_TX
1605 if (!drbr_empty(ifp, txr->br))
1606 ixgbe_mq_start_locked(ifp, txr);
1607#else
1608 if (!IFQ_IS_EMPTY(&ifp->if_snd))
1609 ixgbe_start_locked(txr, ifp);
1610#endif
1611 IXGBE_TX_UNLOCK(txr);
1612 }
1613
1614 /* Reenable this interrupt */
1615 if (que->res != NULL)
1616 ixgbe_enable_queue(adapter, que->msix);
1617 else
1618 ixgbe_enable_intr(adapter);
1619 return;
1620}
1621
1622
1623/*********************************************************************
1624 *
1625 * Legacy Interrupt Service routine
1626 *
1627 **********************************************************************/
1628
1629static int
1630ixgbe_legacy_irq(void *arg)
1631{
1632 struct ix_queue *que = arg;
1633 struct adapter *adapter = que->adapter;
1634 struct ixgbe_hw *hw = &adapter->hw;
1635 struct ifnet *ifp = adapter->ifp;
1636 struct tx_ring *txr = adapter->tx_rings;
1637 bool more = false;
1638 u32 reg_eicr;
1639
1640
1641 reg_eicr = IXGBE_READ_REG(hw, IXGBE_EICR);
1642
1643 adapter->stats.legint.ev_count++;
1644 ++que->irqs;
1645 if (reg_eicr == 0) {
1646 adapter->stats.intzero.ev_count++;
1647 if ((ifp->if_flags & IFF_UP) != 0)
1648 ixgbe_enable_intr(adapter);
1649 return 0;
1650 }
1651
1652 if ((ifp->if_flags & IFF_RUNNING) != 0) {
1653#ifdef __NetBSD__
1654 /* Don't run ixgbe_rxeof in interrupt context */
1655 more = true;
1656#else
1657 more = ixgbe_rxeof(que);
1658#endif
1659
1660 IXGBE_TX_LOCK(txr);
1661 ixgbe_txeof(txr);
1662#ifdef IXGBE_LEGACY_TX
1663 if (!IFQ_IS_EMPTY(&ifp->if_snd))
1664 ixgbe_start_locked(txr, ifp);
1665#else
1666 if (!drbr_empty(ifp, txr->br))
1667 ixgbe_mq_start_locked(ifp, txr);
1668#endif
1669 IXGBE_TX_UNLOCK(txr);
1670 }
1671
1672 /* Check for fan failure */
1673 if ((hw->phy.media_type == ixgbe_media_type_copper) &&
1674 (reg_eicr & IXGBE_EICR_GPI_SDP1)) {
1675 device_printf(adapter->dev, "\nCRITICAL: FAN FAILURE!! "
1676 "REPLACE IMMEDIATELY!!\n");
1677 IXGBE_WRITE_REG(hw, IXGBE_EIMS, IXGBE_EICR_GPI_SDP1);
1678 }
1679
1680 /* Link status change */
1681 if (reg_eicr & IXGBE_EICR_LSC)
1682 softint_schedule(adapter->link_si);
1683
1684 if (more)
1685#ifndef IXGBE_LEGACY_TX
1686 softint_schedule(txr->txq_si);
1687#else
1688 softint_schedule(que->que_si);
1689#endif
1690 else
1691 ixgbe_enable_intr(adapter);
1692 return 1;
1693}
1694
1695
1696#if defined(NETBSD_MSI_OR_MSIX)
1697/*********************************************************************
1698 *
1699 * MSIX Queue Interrupt Service routine
1700 *
1701 **********************************************************************/
1702static int
1703ixgbe_msix_que(void *arg)
1704{
1705 struct ix_queue *que = arg;
1706 struct adapter *adapter = que->adapter;
1707 struct ifnet *ifp = adapter->ifp;
1708 struct tx_ring *txr = que->txr;
1709 struct rx_ring *rxr = que->rxr;
1710 bool more;
1711 u32 newitr = 0;
1712
1713 /* Protect against spurious interrupts */
1714 if ((ifp->if_flags & IFF_RUNNING) == 0)
1715 return 0;
1716
1717 ixgbe_disable_queue(adapter, que->msix);
1718 ++que->irqs;
1719
1720#ifdef __NetBSD__
1721 /* Don't run ixgbe_rxeof in interrupt context */
1722 more = true;
1723#else
1724 more = ixgbe_rxeof(que);
1725#endif
1726
1727 IXGBE_TX_LOCK(txr);
1728 ixgbe_txeof(txr);
1729#ifdef IXGBE_LEGACY_TX
1730 if (!IFQ_IS_EMPTY(&adapter->ifp->if_snd))
1731 ixgbe_start_locked(txr, ifp);
1732#else
1733 if (!drbr_empty(ifp, txr->br))
1734 ixgbe_mq_start_locked(ifp, txr);
1735#endif
1736 IXGBE_TX_UNLOCK(txr);
1737
1738 /* Do AIM now? */
1739
1740 if (ixgbe_enable_aim == FALSE)
1741 goto no_calc;
1742 /*
1743 ** Do Adaptive Interrupt Moderation:
1744 ** - Write out last calculated setting
1745 ** - Calculate based on average size over
1746 ** the last interval.
1747 */
1748 if (que->eitr_setting)
1749 IXGBE_WRITE_REG(&adapter->hw,
1750 IXGBE_EITR(que->msix), que->eitr_setting);
1751
1752 que->eitr_setting = 0;
1753
1754 /* Idle, do nothing */
1755 if ((txr->bytes == 0) && (rxr->bytes == 0))
1756 goto no_calc;
1757
1758 if ((txr->bytes) && (txr->packets))
1759 newitr = txr->bytes/txr->packets;
1760 if ((rxr->bytes) && (rxr->packets))
1761 newitr = max(newitr,
1762 (rxr->bytes / rxr->packets));
1763 newitr += 24; /* account for hardware frame, crc */
1764
1765 /* set an upper boundary */
1766 newitr = min(newitr, 3000);
1767
1768 /* Be nice to the mid range */
1769 if ((newitr > 300) && (newitr < 1200))
1770 newitr = (newitr / 3);
1771 else
1772 newitr = (newitr / 2);
1773
1774 if (adapter->hw.mac.type == ixgbe_mac_82598EB)
1775 newitr |= newitr << 16;
1776 else
1777 newitr |= IXGBE_EITR_CNT_WDIS;
1778
1779 /* save for next interrupt */
1780 que->eitr_setting = newitr;
1781
1782 /* Reset state */
1783 txr->bytes = 0;
1784 txr->packets = 0;
1785 rxr->bytes = 0;
1786 rxr->packets = 0;
1787
1788no_calc:
1789 if (more)
1790 softint_schedule(que->que_si);
1791 else
1792 ixgbe_enable_queue(adapter, que->msix);
1793 return 1;
1794}
1795
1796
1797static int
1798ixgbe_msix_link(void *arg)
1799{
1800 struct adapter *adapter = arg;
1801 struct ixgbe_hw *hw = &adapter->hw;
1802 u32 reg_eicr;
1803
1804 ++adapter->link_irq.ev_count;
1805
1806 /* First get the cause */
1807 reg_eicr = IXGBE_READ_REG(hw, IXGBE_EICS);
1808 /* Be sure the queue bits are not cleared */
1809 reg_eicr &= ~IXGBE_EICR_RTX_QUEUE;
1810 /* Clear interrupt with write */
1811 IXGBE_WRITE_REG(hw, IXGBE_EICR, reg_eicr);
1812
1813 /* Link status change */
1814 if (reg_eicr & IXGBE_EICR_LSC)
1815 softint_schedule(adapter->link_si);
1816
1817 if (adapter->hw.mac.type != ixgbe_mac_82598EB) {
1818#ifdef IXGBE_FDIR
1819 if (reg_eicr & IXGBE_EICR_FLOW_DIR) {
1820 /* This is probably overkill :) */
1821 if (!atomic_cmpset_int(&adapter->fdir_reinit, 0, 1))
1822 return 1;
1823 /* Disable the interrupt */
1824 IXGBE_WRITE_REG(hw, IXGBE_EIMC, IXGBE_EICR_FLOW_DIR);
1825 softint_schedule(adapter->fdir_si);
1826 } else
1827#endif
1828 if (reg_eicr & IXGBE_EICR_ECC) {
1829 device_printf(adapter->dev, "\nCRITICAL: ECC ERROR!! "
1830 "Please Reboot!!\n");
1831 IXGBE_WRITE_REG(hw, IXGBE_EICR, IXGBE_EICR_ECC);
1832 } else
1833
1834 if (reg_eicr & IXGBE_EICR_GPI_SDP1) {
1835 /* Clear the interrupt */
1836 IXGBE_WRITE_REG(hw, IXGBE_EICR, IXGBE_EICR_GPI_SDP1);
1837 softint_schedule(adapter->msf_si);
1838 } else if (reg_eicr & IXGBE_EICR_GPI_SDP2) {
1839 /* Clear the interrupt */
1840 IXGBE_WRITE_REG(hw, IXGBE_EICR, IXGBE_EICR_GPI_SDP2);
1841 softint_schedule(adapter->mod_si);
1842 }
1843 }
1844
1845 /* Check for fan failure */
1846 if ((hw->device_id == IXGBE_DEV_ID_82598AT) &&
1847 (reg_eicr & IXGBE_EICR_GPI_SDP1)) {
1848 device_printf(adapter->dev, "\nCRITICAL: FAN FAILURE!! "
1849 "REPLACE IMMEDIATELY!!\n");
1850 IXGBE_WRITE_REG(hw, IXGBE_EICR, IXGBE_EICR_GPI_SDP1);
1851 }
1852
1853 /* Check for over temp condition */
1854 if ((hw->mac.type == ixgbe_mac_X540) &&
1855 (reg_eicr & IXGBE_EICR_TS)) {
1856 device_printf(adapter->dev, "\nCRITICAL: OVER TEMP!! "
1857 "PHY IS SHUT DOWN!!\n");
1858 device_printf(adapter->dev, "System shutdown required\n");
1859 IXGBE_WRITE_REG(hw, IXGBE_EICR, IXGBE_EICR_TS);
1860 }
1861
1862 IXGBE_WRITE_REG(&adapter->hw, IXGBE_EIMS, IXGBE_EIMS_OTHER);
1863 return 1;
1864}
1865#endif
1866
1867/*********************************************************************
1868 *
1869 * Media Ioctl callback
1870 *
1871 * This routine is called whenever the user queries the status of
1872 * the interface using ifconfig.
1873 *
1874 **********************************************************************/
1875static void
1876ixgbe_media_status(struct ifnet * ifp, struct ifmediareq * ifmr)
1877{
1878 struct adapter *adapter = ifp->if_softc;
1879 struct ixgbe_hw *hw = &adapter->hw;
1880
1881 INIT_DEBUGOUT("ixgbe_media_status: begin");
1882 IXGBE_CORE_LOCK(adapter);
1883 ixgbe_update_link_status(adapter);
1884
1885 ifmr->ifm_status = IFM_AVALID;
1886 ifmr->ifm_active = IFM_ETHER;
1887
1888 if (!adapter->link_active) {
1889 IXGBE_CORE_UNLOCK(adapter);
1890 return;
1891 }
1892
1893 ifmr->ifm_status |= IFM_ACTIVE;
1894
1895 /*
1896 * Not all NIC are 1000baseSX as an example X540T.
1897 * We must set properly the media based on NIC model.
1898 */
1899 switch (hw->device_id) {
1900 case IXGBE_DEV_ID_X540T:
1901 if (adapter->link_speed == IXGBE_LINK_SPEED_100_FULL)
1902 ifmr->ifm_active |= IFM_100_TX | IFM_FDX;
1903 else if (adapter->link_speed == IXGBE_LINK_SPEED_1GB_FULL)
1904 ifmr->ifm_active |= IFM_1000_T | IFM_FDX;
1905 else if (adapter->link_speed == IXGBE_LINK_SPEED_10GB_FULL)
1906 ifmr->ifm_active |= adapter->optics | IFM_FDX;
1907 break;
1908 default:
1909 if (adapter->link_speed == IXGBE_LINK_SPEED_100_FULL)
1910 ifmr->ifm_active |= IFM_100_TX | IFM_FDX;
1911 else if (adapter->link_speed == IXGBE_LINK_SPEED_1GB_FULL)
1912 ifmr->ifm_active |= IFM_1000_SX | IFM_FDX;
1913 else if (adapter->link_speed == IXGBE_LINK_SPEED_10GB_FULL)
1914 ifmr->ifm_active |= adapter->optics | IFM_FDX;
1915 break;
1916 }
1917
1918 IXGBE_CORE_UNLOCK(adapter);
1919
1920 return;
1921}
1922
1923/*********************************************************************
1924 *
1925 * Media Ioctl callback
1926 *
1927 * This routine is called when the user changes speed/duplex using
1928 * media/mediopt option with ifconfig.
1929 *
1930 **********************************************************************/
1931static int
1932ixgbe_media_change(struct ifnet * ifp)
1933{
1934 struct adapter *adapter = ifp->if_softc;
1935 struct ifmedia *ifm = &adapter->media;
1936
1937 INIT_DEBUGOUT("ixgbe_media_change: begin");
1938
1939 if (IFM_TYPE(ifm->ifm_media) != IFM_ETHER)
1940 return (EINVAL);
1941
1942 switch (IFM_SUBTYPE(ifm->ifm_media)) {
1943 case IFM_10G_T:
1944 case IFM_AUTO:
1945 adapter->hw.phy.autoneg_advertised =
1946 IXGBE_LINK_SPEED_100_FULL |
1947 IXGBE_LINK_SPEED_1GB_FULL |
1948 IXGBE_LINK_SPEED_10GB_FULL;
1949 break;
1950 default:
1951 device_printf(adapter->dev, "Only auto media type\n");
1952 return (EINVAL);
1953 }
1954
1955 return (0);
1956}
1957
1958/*********************************************************************
1959 *
1960 * This routine maps the mbufs to tx descriptors, allowing the
1961 * TX engine to transmit the packets.
1962 * - return 0 on success, positive on failure
1963 *
1964 **********************************************************************/
1965
1966static int
1967ixgbe_xmit(struct tx_ring *txr, struct mbuf *m_head)
1968{
1969 struct m_tag *mtag;
1970 struct adapter *adapter = txr->adapter;
1971 struct ethercom *ec = &adapter->osdep.ec;
1972 u32 olinfo_status = 0, cmd_type_len;
1973 int i, j, error;
1974 int first;
1975 bus_dmamap_t map;
1976 struct ixgbe_tx_buf *txbuf;
1977 union ixgbe_adv_tx_desc *txd = NULL;
1978
1979 /* Basic descriptor defines */
1980 cmd_type_len = (IXGBE_ADVTXD_DTYP_DATA |
1981 IXGBE_ADVTXD_DCMD_IFCS | IXGBE_ADVTXD_DCMD_DEXT);
1982
1983 if ((mtag = VLAN_OUTPUT_TAG(ec, m_head)) != NULL)
1984 cmd_type_len |= IXGBE_ADVTXD_DCMD_VLE;
1985
1986 /*
1987 * Important to capture the first descriptor
1988 * used because it will contain the index of
1989 * the one we tell the hardware to report back
1990 */
1991 first = txr->next_avail_desc;
1992 txbuf = &txr->tx_buffers[first];
1993 map = txbuf->map;
1994
1995 /*
1996 * Map the packet for DMA.
1997 */
1998 error = bus_dmamap_load_mbuf(txr->txtag->dt_dmat, map,
1999 m_head, BUS_DMA_NOWAIT);
2000
2001 if (__predict_false(error)) {
2002
2003 switch (error) {
2004 case EAGAIN:
2005 adapter->eagain_tx_dma_setup.ev_count++;
2006 return EAGAIN;
2007 case ENOMEM:
2008 adapter->enomem_tx_dma_setup.ev_count++;
2009 return EAGAIN;
2010 case EFBIG:
2011 /*
2012 * XXX Try it again?
2013 * do m_defrag() and retry bus_dmamap_load_mbuf().
2014 */
2015 adapter->efbig_tx_dma_setup.ev_count++;
2016 return error;
2017 case EINVAL:
2018 adapter->einval_tx_dma_setup.ev_count++;
2019 return error;
2020 default:
2021 adapter->other_tx_dma_setup.ev_count++;
2022 return error;
2023 }
2024 }
2025
2026 /* Make certain there are enough descriptors */
2027 if (map->dm_nsegs > txr->tx_avail - 2) {
2028 txr->no_desc_avail.ev_count++;
2029 ixgbe_dmamap_unload(txr->txtag, txbuf->map);
2030 return EAGAIN;
2031 }
2032
2033 /*
2034 ** Set up the appropriate offload context
2035 ** this will consume the first descriptor
2036 */
2037 error = ixgbe_tx_ctx_setup(txr, m_head, &cmd_type_len, &olinfo_status);
2038 if (__predict_false(error)) {
2039 return (error);
2040 }
2041
2042#ifdef IXGBE_FDIR
2043 /* Do the flow director magic */
2044 if ((txr->atr_sample) && (!adapter->fdir_reinit)) {
2045 ++txr->atr_count;
2046 if (txr->atr_count >= atr_sample_rate) {
2047 ixgbe_atr(txr, m_head);
2048 txr->atr_count = 0;
2049 }
2050 }
2051#endif
2052
2053 i = txr->next_avail_desc;
2054 for (j = 0; j < map->dm_nsegs; j++) {
2055 bus_size_t seglen;
2056 bus_addr_t segaddr;
2057
2058 txbuf = &txr->tx_buffers[i];
2059 txd = &txr->tx_base[i];
2060 seglen = map->dm_segs[j].ds_len;
2061 segaddr = htole64(map->dm_segs[j].ds_addr);
2062
2063 txd->read.buffer_addr = segaddr;
2064 txd->read.cmd_type_len = htole32(txr->txd_cmd |
2065 cmd_type_len |seglen);
2066 txd->read.olinfo_status = htole32(olinfo_status);
2067
2068 if (++i == txr->num_desc)
2069 i = 0;
2070 }
2071
2072 txd->read.cmd_type_len |=
2073 htole32(IXGBE_TXD_CMD_EOP | IXGBE_TXD_CMD_RS);
2074 txr->tx_avail -= map->dm_nsegs;
2075 txr->next_avail_desc = i;
2076
2077 txbuf->m_head = m_head;
2078 /*
2079 ** Here we swap the map so the last descriptor,
2080 ** which gets the completion interrupt has the
2081 ** real map, and the first descriptor gets the
2082 ** unused map from this descriptor.
2083 */
2084 txr->tx_buffers[first].map = txbuf->map;
2085 txbuf->map = map;
2086 bus_dmamap_sync(txr->txtag->dt_dmat, map, 0, m_head->m_pkthdr.len,
2087 BUS_DMASYNC_PREWRITE);
2088
2089 /* Set the EOP descriptor that will be marked done */
2090 txbuf = &txr->tx_buffers[first];
2091 txbuf->eop = txd;
2092
2093 ixgbe_dmamap_sync(txr->txdma.dma_tag, txr->txdma.dma_map,
2094 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
2095 /*
2096 * Advance the Transmit Descriptor Tail (Tdt), this tells the
2097 * hardware that this frame is available to transmit.
2098 */
2099 ++txr->total_packets.ev_count;
2100 IXGBE_WRITE_REG(&adapter->hw, IXGBE_TDT(txr->me), i);
2101
2102 return 0;
2103}
2104
2105static void
2106ixgbe_set_promisc(struct adapter *adapter)
2107{
2108 struct ether_multi *enm;
2109 struct ether_multistep step;
2110 u_int32_t reg_rctl;
2111 struct ethercom *ec = &adapter->osdep.ec;
2112 struct ifnet *ifp = adapter->ifp;
2113 int mcnt = 0;
2114
2115 reg_rctl = IXGBE_READ_REG(&adapter->hw, IXGBE_FCTRL);
2116 reg_rctl &= (~IXGBE_FCTRL_UPE);
2117 if (ifp->if_flags & IFF_ALLMULTI)
2118 mcnt = MAX_NUM_MULTICAST_ADDRESSES;
2119 else {
2120 ETHER_FIRST_MULTI(step, ec, enm);
2121 while (enm != NULL) {
2122 if (mcnt == MAX_NUM_MULTICAST_ADDRESSES)
2123 break;
2124 mcnt++;
2125 ETHER_NEXT_MULTI(step, enm);
2126 }
2127 }
2128 if (mcnt < MAX_NUM_MULTICAST_ADDRESSES)
2129 reg_rctl &= (~IXGBE_FCTRL_MPE);
2130 IXGBE_WRITE_REG(&adapter->hw, IXGBE_FCTRL, reg_rctl);
2131
2132 if (ifp->if_flags & IFF_PROMISC) {
2133 reg_rctl |= (IXGBE_FCTRL_UPE | IXGBE_FCTRL_MPE);
2134 IXGBE_WRITE_REG(&adapter->hw, IXGBE_FCTRL, reg_rctl);
2135 } else if (ifp->if_flags & IFF_ALLMULTI) {
2136 reg_rctl |= IXGBE_FCTRL_MPE;
2137 reg_rctl &= ~IXGBE_FCTRL_UPE;
2138 IXGBE_WRITE_REG(&adapter->hw, IXGBE_FCTRL, reg_rctl);
2139 }
2140 return;
2141}
2142
2143
2144/*********************************************************************
2145 * Multicast Update
2146 *
2147 * This routine is called whenever multicast address list is updated.
2148 *
2149 **********************************************************************/
2150#define IXGBE_RAR_ENTRIES 16
2151
2152static void
2153ixgbe_set_multi(struct adapter *adapter)
2154{
2155 struct ether_multi *enm;
2156 struct ether_multistep step;
2157 u32 fctrl;
2158 u8 *mta;
2159 u8 *update_ptr;
2160 int mcnt = 0;
2161 struct ethercom *ec = &adapter->osdep.ec;
2162 struct ifnet *ifp = adapter->ifp;
2163
2164 IOCTL_DEBUGOUT("ixgbe_set_multi: begin");
2165
2166 mta = adapter->mta;
2167 bzero(mta, sizeof(u8) * IXGBE_ETH_LENGTH_OF_ADDRESS *
2168 MAX_NUM_MULTICAST_ADDRESSES);
2169
2170 ifp->if_flags &= ~IFF_ALLMULTI;
2171 ETHER_FIRST_MULTI(step, ec, enm);
2172 while (enm != NULL) {
2173 if ((mcnt == MAX_NUM_MULTICAST_ADDRESSES) ||
2174 (memcmp(enm->enm_addrlo, enm->enm_addrhi,
2175 ETHER_ADDR_LEN) != 0)) {
2176 ifp->if_flags |= IFF_ALLMULTI;
2177 break;
2178 }
2179 bcopy(enm->enm_addrlo,
2180 &mta[mcnt * IXGBE_ETH_LENGTH_OF_ADDRESS],
2181 IXGBE_ETH_LENGTH_OF_ADDRESS);
2182 mcnt++;
2183 ETHER_NEXT_MULTI(step, enm);
2184 }
2185
2186 fctrl = IXGBE_READ_REG(&adapter->hw, IXGBE_FCTRL);
2187 fctrl &= ~(IXGBE_FCTRL_UPE | IXGBE_FCTRL_MPE);
2188 if (ifp->if_flags & IFF_PROMISC)
2189 fctrl |= (IXGBE_FCTRL_UPE | IXGBE_FCTRL_MPE);
2190 else if (ifp->if_flags & IFF_ALLMULTI) {
2191 fctrl |= IXGBE_FCTRL_MPE;
2192 }
2193
2194 IXGBE_WRITE_REG(&adapter->hw, IXGBE_FCTRL, fctrl);
2195
2196 if (mcnt < MAX_NUM_MULTICAST_ADDRESSES) {
2197 update_ptr = mta;
2198 ixgbe_update_mc_addr_list(&adapter->hw,
2199 update_ptr, mcnt, ixgbe_mc_array_itr, TRUE);
2200 }
2201
2202 return;
2203}
2204
2205/*
2206 * This is an iterator function now needed by the multicast
2207 * shared code. It simply feeds the shared code routine the
2208 * addresses in the array of ixgbe_set_multi() one by one.
2209 */
2210static u8 *
2211ixgbe_mc_array_itr(struct ixgbe_hw *hw, u8 **update_ptr, u32 *vmdq)
2212{
2213 u8 *addr = *update_ptr;
2214 u8 *newptr;
2215 *vmdq = 0;
2216
2217 newptr = addr + IXGBE_ETH_LENGTH_OF_ADDRESS;
2218 *update_ptr = newptr;
2219 return addr;
2220}
2221
2222
2223/*********************************************************************
2224 * Timer routine
2225 *
2226 * This routine checks for link status,updates statistics,
2227 * and runs the watchdog check.
2228 *
2229 **********************************************************************/
2230
2231static void
2232ixgbe_local_timer1(void *arg)
2233{
2234 struct adapter *adapter = arg;
2235 device_t dev = adapter->dev;
2236 struct ix_queue *que = adapter->queues;
2237 struct tx_ring *txr = adapter->tx_rings;
2238 int hung = 0, paused = 0;
2239
2240 KASSERT(mutex_owned(&adapter->core_mtx));
2241
2242 /* Check for pluggable optics */
2243 if (adapter->sfp_probe)
2244 if (!ixgbe_sfp_probe(adapter))
2245 goto out; /* Nothing to do */
2246
2247 ixgbe_update_link_status(adapter);
2248 ixgbe_update_stats_counters(adapter);
2249
2250 /*
2251 * If the interface has been paused
2252 * then don't do the watchdog check
2253 */
2254 if (IXGBE_READ_REG(&adapter->hw, IXGBE_TFCS) & IXGBE_TFCS_TXOFF)
2255 paused = 1;
2256
2257 /*
2258 ** Check the TX queues status
2259 ** - watchdog only if all queues show hung
2260 */
2261 for (int i = 0; i < adapter->num_queues; i++, que++, txr++) {
2262 if ((txr->queue_status == IXGBE_QUEUE_HUNG) &&
2263 (paused == 0))
2264 ++hung;
2265 else if (txr->queue_status == IXGBE_QUEUE_WORKING)
2266#ifndef IXGBE_LEGACY_TX
2267 softint_schedule(txr->txq_si);
2268#else
2269 softint_schedule(que->que_si);
2270#endif
2271 }
2272 /* Only truely watchdog if all queues show hung */
2273 if (hung == adapter->num_queues)
2274 goto watchdog;
2275
2276out:
2277 callout_reset(&adapter->timer, hz, ixgbe_local_timer, adapter);
2278 return;
2279
2280watchdog:
2281 device_printf(adapter->dev, "Watchdog timeout -- resetting\n");
2282 device_printf(dev,"Queue(%d) tdh = %d, hw tdt = %d\n", txr->me,
2283 IXGBE_READ_REG(&adapter->hw, IXGBE_TDH(txr->me)),
2284 IXGBE_READ_REG(&adapter->hw, IXGBE_TDT(txr->me)));
2285 device_printf(dev,"TX(%d) desc avail = %d,"
2286 "Next TX to Clean = %d\n",
2287 txr->me, txr->tx_avail, txr->next_to_clean);
2288 adapter->ifp->if_flags &= ~IFF_RUNNING;
2289 adapter->watchdog_events.ev_count++;
2290 ixgbe_init_locked(adapter);
2291}
2292
2293static void
2294ixgbe_local_timer(void *arg)
2295{
2296 struct adapter *adapter = arg;
2297
2298 IXGBE_CORE_LOCK(adapter);
2299 ixgbe_local_timer1(adapter);
2300 IXGBE_CORE_UNLOCK(adapter);
2301}
2302
2303/*
2304** Note: this routine updates the OS on the link state
2305** the real check of the hardware only happens with
2306** a link interrupt.
2307*/
2308static void
2309ixgbe_update_link_status(struct adapter *adapter)
2310{
2311 struct ifnet *ifp = adapter->ifp;
2312 device_t dev = adapter->dev;
2313
2314
2315 if (adapter->link_up){
2316 if (adapter->link_active == FALSE) {
2317 if (bootverbose)
2318 device_printf(dev,"Link is up %d Gbps %s \n",
2319 ((adapter->link_speed == 128)? 10:1),
2320 "Full Duplex");
2321 adapter->link_active = TRUE;
2322 /* Update any Flow Control changes */
2323 ixgbe_fc_enable(&adapter->hw);
2324 if_link_state_change(ifp, LINK_STATE_UP);
2325 }
2326 } else { /* Link down */
2327 if (adapter->link_active == TRUE) {
2328 if (bootverbose)
2329 device_printf(dev,"Link is Down\n");
2330 if_link_state_change(ifp, LINK_STATE_DOWN);
2331 adapter->link_active = FALSE;
2332 }
2333 }
2334
2335 return;
2336}
2337
2338
2339static void
2340ixgbe_ifstop(struct ifnet *ifp, int disable)
2341{
2342 struct adapter *adapter = ifp->if_softc;
2343
2344 IXGBE_CORE_LOCK(adapter);
2345 ixgbe_stop(adapter);
2346 IXGBE_CORE_UNLOCK(adapter);
2347}
2348
2349/*********************************************************************
2350 *
2351 * This routine disables all traffic on the adapter by issuing a
2352 * global reset on the MAC and deallocates TX/RX buffers.
2353 *
2354 **********************************************************************/
2355
2356static void
2357ixgbe_stop(void *arg)
2358{
2359 struct ifnet *ifp;
2360 struct adapter *adapter = arg;
2361 struct ixgbe_hw *hw = &adapter->hw;
2362 ifp = adapter->ifp;
2363
2364 KASSERT(mutex_owned(&adapter->core_mtx));
2365
2366 INIT_DEBUGOUT("ixgbe_stop: begin\n");
2367 ixgbe_disable_intr(adapter);
2368 callout_stop(&adapter->timer);
2369
2370 /* Let the stack know...*/
2371 ifp->if_flags &= ~IFF_RUNNING;
2372
2373 ixgbe_reset_hw(hw);
2374 hw->adapter_stopped = FALSE;
2375 ixgbe_stop_adapter(hw);
2376 if (hw->mac.type == ixgbe_mac_82599EB)
2377 ixgbe_stop_mac_link_on_d3_82599(hw);
2378 /* Turn off the laser - noop with no optics */
2379 ixgbe_disable_tx_laser(hw);
2380
2381 /* Update the stack */
2382 adapter->link_up = FALSE;
2383 ixgbe_update_link_status(adapter);
2384
2385 /* reprogram the RAR[0] in case user changed it. */
2386 ixgbe_set_rar(&adapter->hw, 0, adapter->hw.mac.addr, 0, IXGBE_RAH_AV);
2387
2388 return;
2389}
2390
2391
2392/*********************************************************************
2393 *
2394 * Determine hardware revision.
2395 *
2396 **********************************************************************/
2397static void
2398ixgbe_identify_hardware(struct adapter *adapter)
2399{
2400 pcitag_t tag;
2401 pci_chipset_tag_t pc;
2402 pcireg_t subid, id;
2403 struct ixgbe_hw *hw = &adapter->hw;
2404
2405 pc = adapter->osdep.pc;
2406 tag = adapter->osdep.tag;
2407
2408 id = pci_conf_read(pc, tag, PCI_ID_REG);
2409 subid = pci_conf_read(pc, tag, PCI_SUBSYS_ID_REG);
2410
2411 /* Save off the information about this board */
2412 hw->vendor_id = PCI_VENDOR(id);
2413 hw->device_id = PCI_PRODUCT(id);
2414 hw->revision_id =
2415 PCI_REVISION(pci_conf_read(pc, tag, PCI_CLASS_REG));
2416 hw->subsystem_vendor_id = PCI_SUBSYS_VENDOR(subid);
2417 hw->subsystem_device_id = PCI_SUBSYS_ID(subid);
2418
2419 /* We need this here to set the num_segs below */
2420 ixgbe_set_mac_type(hw);
2421
2422 /* Pick up the 82599 and VF settings */
2423 if (hw->mac.type != ixgbe_mac_82598EB) {
2424 hw->phy.smart_speed = ixgbe_smart_speed;
2425 adapter->num_segs = IXGBE_82599_SCATTER;
2426 } else
2427 adapter->num_segs = IXGBE_82598_SCATTER;
2428
2429 return;
2430}
2431
2432/*********************************************************************
2433 *
2434 * Determine optic type
2435 *
2436 **********************************************************************/
2437static void
2438ixgbe_setup_optics(struct adapter *adapter)
2439{
2440 struct ixgbe_hw *hw = &adapter->hw;
2441 int layer;
2442
2443 layer = ixgbe_get_supported_physical_layer(hw);
2444
2445 if (layer & IXGBE_PHYSICAL_LAYER_10GBASE_T) {
2446 adapter->optics = IFM_10G_T;
2447 return;
2448 }
2449
2450 if (layer & IXGBE_PHYSICAL_LAYER_1000BASE_T) {
2451 adapter->optics = IFM_1000_T;
2452 return;
2453 }
2454
2455 if (layer & IXGBE_PHYSICAL_LAYER_1000BASE_SX) {
2456 adapter->optics = IFM_1000_SX;
2457 return;
2458 }
2459
2460 if (layer & (IXGBE_PHYSICAL_LAYER_10GBASE_LR |
2461 IXGBE_PHYSICAL_LAYER_10GBASE_LRM)) {
2462 adapter->optics = IFM_10G_LR;
2463 return;
2464 }
2465
2466 if (layer & IXGBE_PHYSICAL_LAYER_10GBASE_SR) {
2467 adapter->optics = IFM_10G_SR;
2468 return;
2469 }
2470
2471 if (layer & IXGBE_PHYSICAL_LAYER_SFP_PLUS_CU) {
2472 adapter->optics = IFM_10G_TWINAX;
2473 return;
2474 }
2475
2476 if (layer & (IXGBE_PHYSICAL_LAYER_10GBASE_KX4 |
2477 IXGBE_PHYSICAL_LAYER_10GBASE_CX4)) {
2478 adapter->optics = IFM_10G_CX4;
2479 return;
2480 }
2481
2482 /* If we get here just set the default */
2483 adapter->optics = IFM_ETHER | IFM_AUTO;
2484 return;
2485}
2486
2487/*********************************************************************
2488 *
2489 * Setup the Legacy or MSI Interrupt handler
2490 *
2491 **********************************************************************/
2492static int
2493ixgbe_allocate_legacy(struct adapter *adapter,
2494 const struct pci_attach_args *pa)
2495{
2496 device_t dev = adapter->dev;
2497 struct ix_queue *que = adapter->queues;
2498#ifndef IXGBE_LEGACY_TX
2499 struct tx_ring *txr = adapter->tx_rings;
2500#endif
2501#ifndef NETBSD_MSI_OR_MSIX
2502 pci_intr_handle_t ih;
2503#else
2504 int counts[PCI_INTR_TYPE_SIZE];
2505 pci_intr_type_t intr_type, max_type;
2506#endif
2507 char intrbuf[PCI_INTRSTR_LEN];
2508 const char *intrstr = NULL;
2509
2510#ifndef NETBSD_MSI_OR_MSIX
2511 /* We allocate a single interrupt resource */
2512 if (pci_intr_map(pa, &ih) != 0) {
2513 aprint_error_dev(dev, "unable to map interrupt\n");
2514 return ENXIO;
2515 } else {
2516 intrstr = pci_intr_string(adapter->osdep.pc, ih, intrbuf,
2517 sizeof(intrbuf));
2518 }
2519 adapter->osdep.ihs[0] = pci_intr_establish_xname(adapter->osdep.pc, ih,
2520 IPL_NET, ixgbe_legacy_irq, que, device_xname(dev));
2521#else
2522 /* Allocation settings */
2523 max_type = PCI_INTR_TYPE_MSI;
2524 counts[PCI_INTR_TYPE_MSIX] = 0;
2525 counts[PCI_INTR_TYPE_MSI] = 1;
2526 counts[PCI_INTR_TYPE_INTX] = 1;
2527
2528alloc_retry:
2529 if (pci_intr_alloc(pa, &adapter->osdep.intrs, counts, max_type) != 0) {
2530 aprint_error_dev(dev, "couldn't alloc interrupt\n");
2531 return ENXIO;
2532 }
2533 adapter->osdep.nintrs = 1;
2534 intrstr = pci_intr_string(adapter->osdep.pc, adapter->osdep.intrs[0],
2535 intrbuf, sizeof(intrbuf));
2536 adapter->osdep.ihs[0] = pci_intr_establish_xname(adapter->osdep.pc,
2537 adapter->osdep.intrs[0], IPL_NET, ixgbe_legacy_irq, que,
2538 device_xname(dev));
2539 if (adapter->osdep.ihs[0] == NULL) {
2540 intr_type = pci_intr_type(adapter->osdep.pc,
2541 adapter->osdep.intrs[0]);
2542 aprint_error_dev(dev,"unable to establish %s\n",
2543 (intr_type == PCI_INTR_TYPE_MSI) ? "MSI" : "INTx");
2544 pci_intr_release(adapter->osdep.pc, adapter->osdep.intrs, 1);
2545 switch (intr_type) {
2546 case PCI_INTR_TYPE_MSI:
2547 /* The next try is for INTx: Disable MSI */
2548 max_type = PCI_INTR_TYPE_INTX;
2549 counts[PCI_INTR_TYPE_INTX] = 1;
2550 goto alloc_retry;
2551 case PCI_INTR_TYPE_INTX:
2552 default:
2553 /* See below */
2554 break;
2555 }
2556 }
2557#endif
2558 if (adapter->osdep.ihs[0] == NULL) {
2559 aprint_error_dev(dev,
2560 "couldn't establish interrupt%s%s\n",
2561 intrstr ? " at " : "", intrstr ? intrstr : "");
2562#ifdef NETBSD_MSI_OR_MSIX
2563 pci_intr_release(adapter->osdep.pc, adapter->osdep.intrs, 1);
2564#endif
2565 return ENXIO;
2566 }
2567 aprint_normal_dev(dev, "interrupting at %s\n", intrstr);
2568 /*
2569 * Try allocating a fast interrupt and the associated deferred
2570 * processing contexts.
2571 */
2572#ifndef IXGBE_LEGACY_TX
2573 txr->txq_si = softint_establish(SOFTINT_NET, ixgbe_deferred_mq_start,
2574 txr);
2575#endif
2576 que->que_si = softint_establish(SOFTINT_NET, ixgbe_handle_que, que);
2577
2578 /* Tasklets for Link, SFP and Multispeed Fiber */
2579 adapter->link_si =
2580 softint_establish(SOFTINT_NET, ixgbe_handle_link, adapter);
2581 adapter->mod_si =
2582 softint_establish(SOFTINT_NET, ixgbe_handle_mod, adapter);
2583 adapter->msf_si =
2584 softint_establish(SOFTINT_NET, ixgbe_handle_msf, adapter);
2585
2586#ifdef IXGBE_FDIR
2587 adapter->fdir_si =
2588 softint_establish(SOFTINT_NET, ixgbe_reinit_fdir, adapter);
2589#endif
2590 if (que->que_si == NULL ||
2591 adapter->link_si == NULL ||
2592 adapter->mod_si == NULL ||
2593#ifdef IXGBE_FDIR
2594 adapter->fdir_si == NULL ||
2595#endif
2596 adapter->msf_si == NULL) {
2597 aprint_error_dev(dev,
2598 "could not establish software interrupts\n");
2599 return ENXIO;
2600 }
2601
2602 /* For simplicity in the handlers */
2603 adapter->que_mask = IXGBE_EIMS_ENABLE_MASK;
2604
2605 return (0);
2606}
2607
2608
2609/*********************************************************************
2610 *
2611 * Setup MSIX Interrupt resources and handlers
2612 *
2613 **********************************************************************/
2614static int
2615ixgbe_allocate_msix(struct adapter *adapter, const struct pci_attach_args *pa)
2616{
2617#if !defined(NETBSD_MSI_OR_MSIX)
2618 return 0;
2619#else
2620 device_t dev = adapter->dev;
2621 struct ix_queue *que = adapter->queues;
2622 struct tx_ring *txr = adapter->tx_rings;
2623 pci_chipset_tag_t pc;
2624 char intrbuf[PCI_INTRSTR_LEN];
2625 char intr_xname[32];
2626 const char *intrstr = NULL;
2627 int error, vector = 0;
2628 int cpu_id = 0;
2629 kcpuset_t *affinity;
2630
2631 pc = adapter->osdep.pc;
2632#ifdef RSS
2633 cpuset_t cpu_mask;
2634 /*
2635 * If we're doing RSS, the number of queues needs to
2636 * match the number of RSS buckets that are configured.
2637 *
2638 * + If there's more queues than RSS buckets, we'll end
2639 * up with queues that get no traffic.
2640 *
2641 * + If there's more RSS buckets than queues, we'll end
2642 * up having multiple RSS buckets map to the same queue,
2643 * so there'll be some contention.
2644 */
2645 if (adapter->num_queues != rss_getnumbuckets()) {
2646 device_printf(dev,
2647 "%s: number of queues (%d) != number of RSS buckets (%d)"
2648 "; performance will be impacted.\n",
2649 __func__,
2650 adapter->num_queues,
2651 rss_getnumbuckets());
2652 }
2653#endif
2654
2655 adapter->osdep.nintrs = adapter->num_queues + 1;
2656 if (pci_msix_alloc_exact(pa, &adapter->osdep.intrs,
2657 adapter->osdep.nintrs) != 0) {
2658 aprint_error_dev(dev,
2659 "failed to allocate MSI-X interrupt\n");
2660 return (ENXIO);
2661 }
2662
2663 kcpuset_create(&affinity, false);
2664 for (int i = 0; i < adapter->num_queues; i++, vector++, que++, txr++) {
2665 snprintf(intr_xname, sizeof(intr_xname), "%s TX/RX",
2666 device_xname(dev));
2667 intrstr = pci_intr_string(pc, adapter->osdep.intrs[i], intrbuf,
2668 sizeof(intrbuf));
2669#ifdef IXG_MPSAFE
2670 pci_intr_setattr(pc, adapter->osdep.intrs[i], PCI_INTR_MPSAFE,
2671 true);
2672#endif
2673 /* Set the handler function */
2674 que->res = adapter->osdep.ihs[i] = pci_intr_establish_xname(pc,
2675 adapter->osdep.intrs[i], IPL_NET, ixgbe_msix_que, que,
2676 intr_xname);
2677 if (que->res == NULL) {
2678 pci_intr_release(pc, adapter->osdep.intrs,
2679 adapter->osdep.nintrs);
2680 aprint_error_dev(dev,
2681 "Failed to register QUE handler\n");
2682 kcpuset_destroy(affinity);
2683 return ENXIO;
2684 }
2685 que->msix = vector;
2686 adapter->que_mask |= (u64)(1 << que->msix);
2687#ifdef RSS
2688 /*
2689 * The queue ID is used as the RSS layer bucket ID.
2690 * We look up the queue ID -> RSS CPU ID and select
2691 * that.
2692 */
2693 cpu_id = rss_getcpu(i % rss_getnumbuckets());
2694#else
2695 /*
2696 * Bind the msix vector, and thus the
2697 * rings to the corresponding cpu.
2698 *
2699 * This just happens to match the default RSS round-robin
2700 * bucket -> queue -> CPU allocation.
2701 */
2702 if (adapter->num_queues > 1)
2703 cpu_id = i;
2704#endif
2705 /* Round-robin affinity */
2706 kcpuset_zero(affinity);
2707 kcpuset_set(affinity, cpu_id % ncpu);
2708 error = interrupt_distribute(adapter->osdep.ihs[i], affinity,
2709 NULL);
2710 aprint_normal_dev(dev, "for TX/RX, interrupting at %s",
2711 intrstr);
2712 if (error == 0) {
2713#ifdef RSS
2714 aprintf_normal(", bound RSS bucket %d to CPU %d\n",
2715 i, cpu_id);
2716#else
2717 aprint_normal(", bound queue %d to cpu %d\n",
2718 i, cpu_id);
2719#endif
2720 } else
2721 aprint_normal("\n");
2722
2723#ifndef IXGBE_LEGACY_TX
2724 txr->txq_si = softint_establish(SOFTINT_NET,
2725 ixgbe_deferred_mq_start, txr);
2726#endif
2727 que->que_si = softint_establish(SOFTINT_NET, ixgbe_handle_que,
2728 que);
2729 if (que->que_si == NULL) {
2730 aprint_error_dev(dev,
2731 "could not establish software interrupt\n");
2732 }
2733 }
2734
2735 /* and Link */
2736 cpu_id++;
2737 snprintf(intr_xname, sizeof(intr_xname), "%s link", device_xname(dev));
2738 intrstr = pci_intr_string(pc, adapter->osdep.intrs[vector], intrbuf,
2739 sizeof(intrbuf));
2740#ifdef IXG_MPSAFE
2741 pci_intr_setattr(pc, &adapter->osdep.intrs[vector], PCI_INTR_MPSAFE,
2742 true);
2743#endif
2744 /* Set the link handler function */
2745 adapter->osdep.ihs[vector] = pci_intr_establish_xname(pc,
2746 adapter->osdep.intrs[vector], IPL_NET, ixgbe_msix_link, adapter,
2747 intr_xname);
2748 if (adapter->osdep.ihs[vector] == NULL) {
2749 adapter->res = NULL;
2750 aprint_error_dev(dev, "Failed to register LINK handler\n");
2751 kcpuset_destroy(affinity);
2752 return (ENXIO);
2753 }
2754 /* Round-robin affinity */
2755 kcpuset_zero(affinity);
2756 kcpuset_set(affinity, cpu_id % ncpu);
2757 error = interrupt_distribute(adapter->osdep.ihs[vector], affinity,NULL);
2758
2759 aprint_normal_dev(dev,
2760 "for link, interrupting at %s", intrstr);
2761 if (error == 0)
2762 aprint_normal(", affinity to cpu %d\n", cpu_id);
2763 else
2764 aprint_normal("\n");
2765
2766 adapter->linkvec = vector;
2767 /* Tasklets for Link, SFP and Multispeed Fiber */
2768 adapter->link_si =
2769 softint_establish(SOFTINT_NET, ixgbe_handle_link, adapter);
2770 adapter->mod_si =
2771 softint_establish(SOFTINT_NET, ixgbe_handle_mod, adapter);
2772 adapter->msf_si =
2773 softint_establish(SOFTINT_NET, ixgbe_handle_msf, adapter);
2774#ifdef IXGBE_FDIR
2775 adapter->fdir_si =
2776 softint_establish(SOFTINT_NET, ixgbe_reinit_fdir, adapter);
2777#endif
2778
2779 kcpuset_destroy(affinity);
2780 return (0);
2781#endif
2782}
2783
2784/*
2785 * Setup Either MSI/X or MSI
2786 */
2787static int
2788ixgbe_setup_msix(struct adapter *adapter)
2789{
2790#if !defined(NETBSD_MSI_OR_MSIX)
2791 return 0;
2792#else
2793 device_t dev = adapter->dev;
2794 int want, queues, msgs;
2795
2796 /* Override by tuneable */
2797 if (ixgbe_enable_msix == 0)
2798 goto msi;
2799
2800 /* First try MSI/X */
2801 msgs = pci_msix_count(adapter->osdep.pc, adapter->osdep.tag);
2802 if (msgs < IXG_MSIX_NINTR)
2803 goto msi;
2804
2805 adapter->msix_mem = (void *)1; /* XXX */
2806
2807 /* Figure out a reasonable auto config value */
2808 queues = (ncpu > (msgs-1)) ? (msgs-1) : ncpu;
2809
2810 /* Override based on tuneable */
2811 if (ixgbe_num_queues != 0)
2812 queues = ixgbe_num_queues;
2813
2814#ifdef RSS
2815 /* If we're doing RSS, clamp at the number of RSS buckets */
2816 if (queues > rss_getnumbuckets())
2817 queues = rss_getnumbuckets();
2818#endif
2819
2820 /* reflect correct sysctl value */
2821 ixgbe_num_queues = queues;
2822
2823 /*
2824 ** Want one vector (RX/TX pair) per queue
2825 ** plus an additional for Link.
2826 */
2827 want = queues + 1;
2828 if (msgs >= want)
2829 msgs = want;
2830 else {
2831 aprint_error_dev(dev,
2832 "MSIX Configuration Problem, "
2833 "%d vectors but %d queues wanted!\n",
2834 msgs, want);
2835 goto msi;
2836 }
2837 device_printf(dev,
2838 "Using MSIX interrupts with %d vectors\n", msgs);
2839 adapter->num_queues = queues;
2840 return (msgs);
2841
2842 /*
2843 ** If MSIX alloc failed or provided us with
2844 ** less than needed, free and fall through to MSI
2845 */
2846msi:
2847 msgs = pci_msi_count(adapter->osdep.pc, adapter->osdep.tag);
2848 adapter->msix_mem = NULL; /* XXX */
2849 msgs = 1;
2850 aprint_normal_dev(dev,"Using an MSI interrupt\n");
2851 return (msgs);
2852#endif
2853}
2854
2855
2856static int
2857ixgbe_allocate_pci_resources(struct adapter *adapter,
2858 const struct pci_attach_args *pa)
2859{
2860 pcireg_t memtype;
2861 device_t dev = adapter->dev;
2862 bus_addr_t addr;
2863 int flags;
2864
2865 memtype = pci_mapreg_type(pa->pa_pc, pa->pa_tag, PCI_BAR(0));
2866 switch (memtype) {
2867 case PCI_MAPREG_TYPE_MEM | PCI_MAPREG_MEM_TYPE_32BIT:
2868 case PCI_MAPREG_TYPE_MEM | PCI_MAPREG_MEM_TYPE_64BIT:
2869 adapter->osdep.mem_bus_space_tag = pa->pa_memt;
2870 if (pci_mapreg_info(pa->pa_pc, pa->pa_tag, PCI_BAR(0),
2871 memtype, &addr, &adapter->osdep.mem_size, &flags) != 0)
2872 goto map_err;
2873 if ((flags & BUS_SPACE_MAP_PREFETCHABLE) != 0) {
2874 aprint_normal_dev(dev, "clearing prefetchable bit\n");
2875 flags &= ~BUS_SPACE_MAP_PREFETCHABLE;
2876 }
2877 if (bus_space_map(adapter->osdep.mem_bus_space_tag, addr,
2878 adapter->osdep.mem_size, flags,
2879 &adapter->osdep.mem_bus_space_handle) != 0) {
2880map_err:
2881 adapter->osdep.mem_size = 0;
2882 aprint_error_dev(dev, "unable to map BAR0\n");
2883 return ENXIO;
2884 }
2885 break;
2886 default:
2887 aprint_error_dev(dev, "unexpected type on BAR0\n");
2888 return ENXIO;
2889 }
2890
2891 /* Legacy defaults */
2892 adapter->num_queues = 1;
2893 adapter->hw.back = &adapter->osdep;
2894
2895 /*
2896 ** Now setup MSI or MSI/X, should
2897 ** return us the number of supported
2898 ** vectors. (Will be 1 for MSI)
2899 */
2900 adapter->msix = ixgbe_setup_msix(adapter);
2901 return (0);
2902}
2903
2904static void
2905ixgbe_free_pci_resources(struct adapter * adapter)
2906{
2907#if defined(NETBSD_MSI_OR_MSIX)
2908 struct ix_queue *que = adapter->queues;
2909#endif
2910 int rid;
2911
2912#if defined(NETBSD_MSI_OR_MSIX)
2913 /*
2914 ** Release all msix queue resources:
2915 */
2916 for (int i = 0; i < adapter->num_queues; i++, que++) {
2917 if (que->res != NULL)
2918 pci_intr_disestablish(adapter->osdep.pc,
2919 adapter->osdep.ihs[i]);
2920 }
2921#endif
2922
2923 /* Clean the Legacy or Link interrupt last */
2924 if (adapter->linkvec) /* we are doing MSIX */
2925 rid = adapter->linkvec;
2926 else
2927 rid = 0;
2928
2929 if (adapter->osdep.ihs[rid] != NULL) {
2930 pci_intr_disestablish(adapter->osdep.pc,
2931 adapter->osdep.ihs[rid]);
2932 adapter->osdep.ihs[rid] = NULL;
2933 }
2934
2935#if defined(NETBSD_MSI_OR_MSIX)
2936 pci_intr_release(adapter->osdep.pc, adapter->osdep.intrs,
2937 adapter->osdep.nintrs);
2938#endif
2939
2940 if (adapter->osdep.mem_size != 0) {
2941 bus_space_unmap(adapter->osdep.mem_bus_space_tag,
2942 adapter->osdep.mem_bus_space_handle,
2943 adapter->osdep.mem_size);
2944 }
2945
2946 return;
2947}
2948
2949/*********************************************************************
2950 *
2951 * Setup networking device structure and register an interface.
2952 *
2953 **********************************************************************/
2954static int
2955ixgbe_setup_interface(device_t dev, struct adapter *adapter)
2956{
2957 struct ethercom *ec = &adapter->osdep.ec;
2958 struct ixgbe_hw *hw = &adapter->hw;
2959 struct ifnet *ifp;
2960
2961 INIT_DEBUGOUT("ixgbe_setup_interface: begin");
2962
2963 ifp = adapter->ifp = &ec->ec_if;
2964 strlcpy(ifp->if_xname, device_xname(dev), IFNAMSIZ);
2965 ifp->if_baudrate = IF_Gbps(10);
2966 ifp->if_init = ixgbe_init;
2967 ifp->if_stop = ixgbe_ifstop;
2968 ifp->if_softc = adapter;
2969 ifp->if_flags = IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST;
2970 ifp->if_ioctl = ixgbe_ioctl;
2971#ifndef IXGBE_LEGACY_TX
2972 ifp->if_transmit = ixgbe_mq_start;
2973 ifp->if_qflush = ixgbe_qflush;
2974#else
2975 ifp->if_start = ixgbe_start;
2976 IFQ_SET_MAXLEN(&ifp->if_snd, adapter->num_tx_desc - 2);
2977#if 0
2978 ifp->if_snd.ifq_drv_maxlen = adapter->num_tx_desc - 2;
2979#endif
2980 IFQ_SET_READY(&ifp->if_snd);
2981#endif
2982
2983 if_initialize(ifp);
2984 ether_ifattach(ifp, adapter->hw.mac.addr);
2985 if_register(ifp);
2986 ether_set_ifflags_cb(ec, ixgbe_ifflags_cb);
2987
2988 adapter->max_frame_size =
2989 ifp->if_mtu + ETHER_HDR_LEN + ETHER_CRC_LEN;
2990
2991 /*
2992 * Tell the upper layer(s) we support long frames.
2993 */
2994 ifp->if_hdrlen = sizeof(struct ether_vlan_header);
2995
2996 ifp->if_capabilities |= IFCAP_HWCSUM | IFCAP_TSOv4 | IFCAP_TSOv6;
2997 ifp->if_capenable = 0;
2998
2999 ec->ec_capabilities |= ETHERCAP_VLAN_HWCSUM;
3000 ec->ec_capabilities |= ETHERCAP_JUMBO_MTU;
3001 ifp->if_capabilities |= IFCAP_LRO;
3002 ec->ec_capabilities |= ETHERCAP_VLAN_HWTAGGING
3003 | ETHERCAP_VLAN_MTU;
3004 ec->ec_capenable = ec->ec_capabilities;
3005
3006 /*
3007 ** Don't turn this on by default, if vlans are
3008 ** created on another pseudo device (eg. lagg)
3009 ** then vlan events are not passed thru, breaking
3010 ** operation, but with HW FILTER off it works. If
3011 ** using vlans directly on the ixgbe driver you can
3012 ** enable this and get full hardware tag filtering.
3013 */
3014 ec->ec_capabilities |= ETHERCAP_VLAN_HWFILTER;
3015
3016 /*
3017 * Specify the media types supported by this adapter and register
3018 * callbacks to update media and link information
3019 */
3020 ifmedia_init(&adapter->media, IFM_IMASK, ixgbe_media_change,
3021 ixgbe_media_status);
3022 ifmedia_add(&adapter->media, IFM_ETHER | adapter->optics, 0, NULL);
3023 ifmedia_set(&adapter->media, IFM_ETHER | adapter->optics);
3024 if (hw->device_id == IXGBE_DEV_ID_82598AT) {
3025 ifmedia_add(&adapter->media,
3026 IFM_ETHER | IFM_1000_T | IFM_FDX, 0, NULL);
3027 ifmedia_add(&adapter->media,
3028 IFM_ETHER | IFM_1000_T, 0, NULL);
3029 }
3030 ifmedia_add(&adapter->media, IFM_ETHER | IFM_AUTO, 0, NULL);
3031 ifmedia_set(&adapter->media, IFM_ETHER | IFM_AUTO);
3032
3033 return (0);
3034}
3035
3036static void
3037ixgbe_config_link(struct adapter *adapter)
3038{
3039 struct ixgbe_hw *hw = &adapter->hw;
3040 u32 autoneg, err = 0;
3041 bool sfp, negotiate;
3042
3043 sfp = ixgbe_is_sfp(hw);
3044
3045 if (sfp) {
3046 void *ip;
3047
3048 if (hw->phy.multispeed_fiber) {
3049 hw->mac.ops.setup_sfp(hw);
3050 ixgbe_enable_tx_laser(hw);
3051 ip = adapter->msf_si;
3052 } else {
3053 ip = adapter->mod_si;
3054 }
3055
3056 kpreempt_disable();
3057 softint_schedule(ip);
3058 kpreempt_enable();
3059 } else {
3060 if (hw->mac.ops.check_link)
3061 err = ixgbe_check_link(hw, &adapter->link_speed,
3062 &adapter->link_up, FALSE);
3063 if (err)
3064 goto out;
3065 autoneg = hw->phy.autoneg_advertised;
3066 if ((!autoneg) && (hw->mac.ops.get_link_capabilities))
3067 err = hw->mac.ops.get_link_capabilities(hw,
3068 &autoneg, &negotiate);
3069 else
3070 negotiate = 0;
3071 if (err)
3072 goto out;
3073 if (hw->mac.ops.setup_link)
3074 err = hw->mac.ops.setup_link(hw,
3075 autoneg, adapter->link_up);
3076 }
3077out:
3078 return;
3079}
3080
3081/********************************************************************
3082 * Manage DMA'able memory.
3083 *******************************************************************/
3084
3085static int
3086ixgbe_dma_malloc(struct adapter *adapter, const bus_size_t size,
3087 struct ixgbe_dma_alloc *dma, const int mapflags)
3088{
3089 device_t dev = adapter->dev;
3090 int r, rsegs;
3091
3092 r = ixgbe_dma_tag_create(adapter->osdep.dmat, /* parent */
3093 DBA_ALIGN, 0, /* alignment, bounds */
3094 size, /* maxsize */
3095 1, /* nsegments */
3096 size, /* maxsegsize */
3097 BUS_DMA_ALLOCNOW, /* flags */
3098 &dma->dma_tag);
3099 if (r != 0) {
3100 aprint_error_dev(dev,
3101 "%s: ixgbe_dma_tag_create failed; error %d\n", __func__, r);
3102 goto fail_0;
3103 }
3104
3105 r = bus_dmamem_alloc(dma->dma_tag->dt_dmat,
3106 size,
3107 dma->dma_tag->dt_alignment,
3108 dma->dma_tag->dt_boundary,
3109 &dma->dma_seg, 1, &rsegs, BUS_DMA_NOWAIT);
3110 if (r != 0) {
3111 aprint_error_dev(dev,
3112 "%s: bus_dmamem_alloc failed; error %d\n", __func__, r);
3113 goto fail_1;
3114 }
3115
3116 r = bus_dmamem_map(dma->dma_tag->dt_dmat, &dma->dma_seg, rsegs,
3117 size, &dma->dma_vaddr, BUS_DMA_NOWAIT);
3118 if (r != 0) {
3119 aprint_error_dev(dev, "%s: bus_dmamem_map failed; error %d\n",
3120 __func__, r);
3121 goto fail_2;
3122 }
3123
3124 r = ixgbe_dmamap_create(dma->dma_tag, 0, &dma->dma_map);
3125 if (r != 0) {
3126 aprint_error_dev(dev, "%s: bus_dmamem_map failed; error %d\n",
3127 __func__, r);
3128 goto fail_3;
3129 }
3130
3131 r = bus_dmamap_load(dma->dma_tag->dt_dmat, dma->dma_map, dma->dma_vaddr,
3132 size,
3133 NULL,
3134 mapflags | BUS_DMA_NOWAIT);
3135 if (r != 0) {
3136 aprint_error_dev(dev, "%s: bus_dmamap_load failed; error %d\n",
3137 __func__, r);
3138 goto fail_4;
3139 }
3140 dma->dma_paddr = dma->dma_map->dm_segs[0].ds_addr;
3141 dma->dma_size = size;
3142 return 0;
3143fail_4:
3144 ixgbe_dmamap_destroy(dma->dma_tag, dma->dma_map);
3145fail_3:
3146 bus_dmamem_unmap(dma->dma_tag->dt_dmat, dma->dma_vaddr, size);
3147fail_2:
3148 bus_dmamem_free(dma->dma_tag->dt_dmat, &dma->dma_seg, rsegs);
3149fail_1:
3150 ixgbe_dma_tag_destroy(dma->dma_tag);
3151fail_0:
3152 return r;
3153}
3154
3155static void
3156ixgbe_dma_free(struct adapter *adapter, struct ixgbe_dma_alloc *dma)
3157{
3158 bus_dmamap_sync(dma->dma_tag->dt_dmat, dma->dma_map, 0, dma->dma_size,
3159 BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE);
3160 ixgbe_dmamap_unload(dma->dma_tag, dma->dma_map);
3161 bus_dmamem_free(dma->dma_tag->dt_dmat, &dma->dma_seg, 1);
3162 ixgbe_dma_tag_destroy(dma->dma_tag);
3163}
3164
3165
3166/*********************************************************************
3167 *
3168 * Allocate memory for the transmit and receive rings, and then
3169 * the descriptors associated with each, called only once at attach.
3170 *
3171 **********************************************************************/
3172static int
3173ixgbe_allocate_queues(struct adapter *adapter)
3174{
3175 device_t dev = adapter->dev;
3176 struct ix_queue *que;
3177 struct tx_ring *txr;
3178 struct rx_ring *rxr;
3179 int rsize, tsize, error = IXGBE_SUCCESS;
3180 int txconf = 0, rxconf = 0;
3181
3182 /* First allocate the top level queue structs */
3183 if (!(adapter->queues =
3184 (struct ix_queue *) malloc(sizeof(struct ix_queue) *
3185 adapter->num_queues, M_DEVBUF, M_NOWAIT | M_ZERO))) {
3186 aprint_error_dev(dev, "Unable to allocate queue memory\n");
3187 error = ENOMEM;
3188 goto fail;
3189 }
3190
3191 /* First allocate the TX ring struct memory */
3192 if (!(adapter->tx_rings =
3193 (struct tx_ring *) malloc(sizeof(struct tx_ring) *
3194 adapter->num_queues, M_DEVBUF, M_NOWAIT | M_ZERO))) {
3195 aprint_error_dev(dev, "Unable to allocate TX ring memory\n");
3196 error = ENOMEM;
3197 goto tx_fail;
3198 }
3199
3200 /* Next allocate the RX */
3201 if (!(adapter->rx_rings =
3202 (struct rx_ring *) malloc(sizeof(struct rx_ring) *
3203 adapter->num_queues, M_DEVBUF, M_NOWAIT | M_ZERO))) {
3204 aprint_error_dev(dev, "Unable to allocate RX ring memory\n");
3205 error = ENOMEM;
3206 goto rx_fail;
3207 }
3208
3209 /* For the ring itself */
3210 tsize = roundup2(adapter->num_tx_desc *
3211 sizeof(union ixgbe_adv_tx_desc), DBA_ALIGN);
3212
3213 /*
3214 * Now set up the TX queues, txconf is needed to handle the
3215 * possibility that things fail midcourse and we need to
3216 * undo memory gracefully
3217 */
3218 for (int i = 0; i < adapter->num_queues; i++, txconf++) {
3219 /* Set up some basics */
3220 txr = &adapter->tx_rings[i];
3221 txr->adapter = adapter;
3222 txr->me = i;
3223 txr->num_desc = adapter->num_tx_desc;
3224
3225 /* Initialize the TX side lock */
3226 snprintf(txr->mtx_name, sizeof(txr->mtx_name), "%s:tx(%d)",
3227 device_xname(dev), txr->me);
3228 mutex_init(&txr->tx_mtx, MUTEX_DEFAULT, IPL_NET);
3229
3230 if (ixgbe_dma_malloc(adapter, tsize,
3231 &txr->txdma, BUS_DMA_NOWAIT)) {
3232 aprint_error_dev(dev,
3233 "Unable to allocate TX Descriptor memory\n");
3234 error = ENOMEM;
3235 goto err_tx_desc;
3236 }
3237 txr->tx_base = (union ixgbe_adv_tx_desc *)txr->txdma.dma_vaddr;
3238 bzero((void *)txr->tx_base, tsize);
3239
3240 /* Now allocate transmit buffers for the ring */
3241 if (ixgbe_allocate_transmit_buffers(txr)) {
3242 aprint_error_dev(dev,
3243 "Critical Failure setting up transmit buffers\n");
3244 error = ENOMEM;
3245 goto err_tx_desc;
3246 }
3247#ifndef IXGBE_LEGACY_TX
3248 /* Allocate a buf ring */
3249 txr->br = buf_ring_alloc(IXGBE_BR_SIZE, M_DEVBUF,
3250 M_WAITOK, &txr->tx_mtx);
3251 if (txr->br == NULL) {
3252 aprint_error_dev(dev,
3253 "Critical Failure setting up buf ring\n");
3254 error = ENOMEM;
3255 goto err_tx_desc;
3256 }
3257#endif
3258 }
3259
3260 /*
3261 * Next the RX queues...
3262 */
3263 rsize = roundup2(adapter->num_rx_desc *
3264 sizeof(union ixgbe_adv_rx_desc), DBA_ALIGN);
3265 for (int i = 0; i < adapter->num_queues; i++, rxconf++) {
3266 rxr = &adapter->rx_rings[i];
3267 /* Set up some basics */
3268 rxr->adapter = adapter;
3269 rxr->me = i;
3270 rxr->num_desc = adapter->num_rx_desc;
3271
3272 /* Initialize the RX side lock */
3273 snprintf(rxr->mtx_name, sizeof(rxr->mtx_name), "%s:rx(%d)",
3274 device_xname(dev), rxr->me);
3275 mutex_init(&rxr->rx_mtx, MUTEX_DEFAULT, IPL_NET);
3276
3277 if (ixgbe_dma_malloc(adapter, rsize,
3278 &rxr->rxdma, BUS_DMA_NOWAIT)) {
3279 aprint_error_dev(dev,
3280 "Unable to allocate RxDescriptor memory\n");
3281 error = ENOMEM;
3282 goto err_rx_desc;
3283 }
3284 rxr->rx_base = (union ixgbe_adv_rx_desc *)rxr->rxdma.dma_vaddr;
3285 bzero((void *)rxr->rx_base, rsize);
3286
3287 /* Allocate receive buffers for the ring*/
3288 if (ixgbe_allocate_receive_buffers(rxr)) {
3289 aprint_error_dev(dev,
3290 "Critical Failure setting up receive buffers\n");
3291 error = ENOMEM;
3292 goto err_rx_desc;
3293 }
3294 }
3295
3296 /*
3297 ** Finally set up the queue holding structs
3298 */
3299 for (int i = 0; i < adapter->num_queues; i++) {
3300 que = &adapter->queues[i];
3301 que->adapter = adapter;
3302 que->txr = &adapter->tx_rings[i];
3303 que->rxr = &adapter->rx_rings[i];
3304 }
3305
3306 return (0);
3307
3308err_rx_desc:
3309 for (rxr = adapter->rx_rings; rxconf > 0; rxr++, rxconf--)
3310 ixgbe_dma_free(adapter, &rxr->rxdma);
3311err_tx_desc:
3312 for (txr = adapter->tx_rings; txconf > 0; txr++, txconf--)
3313 ixgbe_dma_free(adapter, &txr->txdma);
3314 free(adapter->rx_rings, M_DEVBUF);
3315rx_fail:
3316 free(adapter->tx_rings, M_DEVBUF);
3317tx_fail:
3318 free(adapter->queues, M_DEVBUF);
3319fail:
3320 return (error);
3321}
3322
3323/*********************************************************************
3324 *
3325 * Allocate memory for tx_buffer structures. The tx_buffer stores all
3326 * the information needed to transmit a packet on the wire. This is
3327 * called only once at attach, setup is done every reset.
3328 *
3329 **********************************************************************/
3330static int
3331ixgbe_allocate_transmit_buffers(struct tx_ring *txr)
3332{
3333 struct adapter *adapter = txr->adapter;
3334 device_t dev = adapter->dev;
3335 struct ixgbe_tx_buf *txbuf;
3336 int error, i;
3337
3338 /*
3339 * Setup DMA descriptor areas.
3340 */
3341 if ((error = ixgbe_dma_tag_create(adapter->osdep.dmat, /* parent */
3342 1, 0, /* alignment, bounds */
3343 IXGBE_TSO_SIZE, /* maxsize */
3344 adapter->num_segs, /* nsegments */
3345 PAGE_SIZE, /* maxsegsize */
3346 0, /* flags */
3347 &txr->txtag))) {
3348 aprint_error_dev(dev,"Unable to allocate TX DMA tag\n");
3349 goto fail;
3350 }
3351
3352 if (!(txr->tx_buffers =
3353 (struct ixgbe_tx_buf *) malloc(sizeof(struct ixgbe_tx_buf) *
3354 adapter->num_tx_desc, M_DEVBUF, M_NOWAIT | M_ZERO))) {
3355 aprint_error_dev(dev, "Unable to allocate tx_buffer memory\n");
3356 error = ENOMEM;
3357 goto fail;
3358 }
3359
3360 /* Create the descriptor buffer dma maps */
3361 txbuf = txr->tx_buffers;
3362 for (i = 0; i < adapter->num_tx_desc; i++, txbuf++) {
3363 error = ixgbe_dmamap_create(txr->txtag, 0, &txbuf->map);
3364 if (error != 0) {
3365 aprint_error_dev(dev,
3366 "Unable to create TX DMA map (%d)\n", error);
3367 goto fail;
3368 }
3369 }
3370
3371 return 0;
3372fail:
3373 /* We free all, it handles case where we are in the middle */
3374 ixgbe_free_transmit_structures(adapter);
3375 return (error);
3376}
3377
3378/*********************************************************************
3379 *
3380 * Initialize a transmit ring.
3381 *
3382 **********************************************************************/
3383static void
3384ixgbe_setup_transmit_ring(struct tx_ring *txr)
3385{
3386 struct adapter *adapter = txr->adapter;
3387 struct ixgbe_tx_buf *txbuf;
3388 int i;
3389#ifdef DEV_NETMAP
3390 struct netmap_adapter *na = NA(adapter->ifp);
3391 struct netmap_slot *slot;
3392#endif /* DEV_NETMAP */
3393
3394 /* Clear the old ring contents */
3395 IXGBE_TX_LOCK(txr);
3396#ifdef DEV_NETMAP
3397 /*
3398 * (under lock): if in netmap mode, do some consistency
3399 * checks and set slot to entry 0 of the netmap ring.
3400 */
3401 slot = netmap_reset(na, NR_TX, txr->me, 0);
3402#endif /* DEV_NETMAP */
3403 bzero((void *)txr->tx_base,
3404 (sizeof(union ixgbe_adv_tx_desc)) * adapter->num_tx_desc);
3405 /* Reset indices */
3406 txr->next_avail_desc = 0;
3407 txr->next_to_clean = 0;
3408
3409 /* Free any existing tx buffers. */
3410 txbuf = txr->tx_buffers;
3411 for (i = 0; i < txr->num_desc; i++, txbuf++) {
3412 if (txbuf->m_head != NULL) {
3413 bus_dmamap_sync(txr->txtag->dt_dmat, txbuf->map,
3414 0, txbuf->m_head->m_pkthdr.len,
3415 BUS_DMASYNC_POSTWRITE);
3416 ixgbe_dmamap_unload(txr->txtag, txbuf->map);
3417 m_freem(txbuf->m_head);
3418 txbuf->m_head = NULL;
3419 }
3420#ifdef DEV_NETMAP
3421 /*
3422 * In netmap mode, set the map for the packet buffer.
3423 * NOTE: Some drivers (not this one) also need to set
3424 * the physical buffer address in the NIC ring.
3425 * Slots in the netmap ring (indexed by "si") are
3426 * kring->nkr_hwofs positions "ahead" wrt the
3427 * corresponding slot in the NIC ring. In some drivers
3428 * (not here) nkr_hwofs can be negative. Function
3429 * netmap_idx_n2k() handles wraparounds properly.
3430 */
3431 if (slot) {
3432 int si = netmap_idx_n2k(&na->tx_rings[txr->me], i);
3433 netmap_load_map(na, txr->txtag, txbuf->map, NMB(na, slot + si));
3434 }
3435#endif /* DEV_NETMAP */
3436 /* Clear the EOP descriptor pointer */
3437 txbuf->eop = NULL;
3438 }
3439
3440#ifdef IXGBE_FDIR
3441 /* Set the rate at which we sample packets */
3442 if (adapter->hw.mac.type != ixgbe_mac_82598EB)
3443 txr->atr_sample = atr_sample_rate;
3444#endif
3445
3446 /* Set number of descriptors available */
3447 txr->tx_avail = adapter->num_tx_desc;
3448
3449 ixgbe_dmamap_sync(txr->txdma.dma_tag, txr->txdma.dma_map,
3450 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
3451 IXGBE_TX_UNLOCK(txr);
3452}
3453
3454/*********************************************************************
3455 *
3456 * Initialize all transmit rings.
3457 *
3458 **********************************************************************/
3459static int
3460ixgbe_setup_transmit_structures(struct adapter *adapter)
3461{
3462 struct tx_ring *txr = adapter->tx_rings;
3463
3464 for (int i = 0; i < adapter->num_queues; i++, txr++)
3465 ixgbe_setup_transmit_ring(txr);
3466
3467 return (0);
3468}
3469
3470/*********************************************************************
3471 *
3472 * Enable transmit unit.
3473 *
3474 **********************************************************************/
3475static void
3476ixgbe_initialize_transmit_units(struct adapter *adapter)
3477{
3478 struct tx_ring *txr = adapter->tx_rings;
3479 struct ixgbe_hw *hw = &adapter->hw;
3480
3481 /* Setup the Base and Length of the Tx Descriptor Ring */
3482
3483 for (int i = 0; i < adapter->num_queues; i++, txr++) {
3484 u64 tdba = txr->txdma.dma_paddr;
3485 u32 txctrl;
3486
3487 IXGBE_WRITE_REG(hw, IXGBE_TDBAL(i),
3488 (tdba & 0x00000000ffffffffULL));
3489 IXGBE_WRITE_REG(hw, IXGBE_TDBAH(i), (tdba >> 32));
3490 IXGBE_WRITE_REG(hw, IXGBE_TDLEN(i),
3491 adapter->num_tx_desc * sizeof(union ixgbe_adv_tx_desc));
3492
3493 /* Setup the HW Tx Head and Tail descriptor pointers */
3494 IXGBE_WRITE_REG(hw, IXGBE_TDH(i), 0);
3495 IXGBE_WRITE_REG(hw, IXGBE_TDT(i), 0);
3496
3497 /* Setup Transmit Descriptor Cmd Settings */
3498 txr->txd_cmd = IXGBE_TXD_CMD_IFCS;
3499 txr->queue_status = IXGBE_QUEUE_IDLE;
3500
3501 /* Set the processing limit */
3502 txr->process_limit = ixgbe_tx_process_limit;
3503
3504 /* Disable Head Writeback */
3505 switch (hw->mac.type) {
3506 case ixgbe_mac_82598EB:
3507 txctrl = IXGBE_READ_REG(hw, IXGBE_DCA_TXCTRL(i));
3508 break;
3509 case ixgbe_mac_82599EB:
3510 case ixgbe_mac_X540:
3511 default:
3512 txctrl = IXGBE_READ_REG(hw, IXGBE_DCA_TXCTRL_82599(i));
3513 break;
3514 }
3515 txctrl &= ~IXGBE_DCA_TXCTRL_DESC_WRO_EN;
3516 switch (hw->mac.type) {
3517 case ixgbe_mac_82598EB:
3518 IXGBE_WRITE_REG(hw, IXGBE_DCA_TXCTRL(i), txctrl);
3519 break;
3520 case ixgbe_mac_82599EB:
3521 case ixgbe_mac_X540:
3522 default:
3523 IXGBE_WRITE_REG(hw, IXGBE_DCA_TXCTRL_82599(i), txctrl);
3524 break;
3525 }
3526
3527 }
3528
3529 if (hw->mac.type != ixgbe_mac_82598EB) {
3530 u32 dmatxctl, rttdcs;
3531 dmatxctl = IXGBE_READ_REG(hw, IXGBE_DMATXCTL);
3532 dmatxctl |= IXGBE_DMATXCTL_TE;
3533 IXGBE_WRITE_REG(hw, IXGBE_DMATXCTL, dmatxctl);
3534 /* Disable arbiter to set MTQC */
3535 rttdcs = IXGBE_READ_REG(hw, IXGBE_RTTDCS);
3536 rttdcs |= IXGBE_RTTDCS_ARBDIS;
3537 IXGBE_WRITE_REG(hw, IXGBE_RTTDCS, rttdcs);
3538 IXGBE_WRITE_REG(hw, IXGBE_MTQC, IXGBE_MTQC_64Q_1PB);
3539 rttdcs &= ~IXGBE_RTTDCS_ARBDIS;
3540 IXGBE_WRITE_REG(hw, IXGBE_RTTDCS, rttdcs);
3541 }
3542
3543 return;
3544}
3545
3546/*********************************************************************
3547 *
3548 * Free all transmit rings.
3549 *
3550 **********************************************************************/
3551static void
3552ixgbe_free_transmit_structures(struct adapter *adapter)
3553{
3554 struct tx_ring *txr = adapter->tx_rings;
3555
3556 for (int i = 0; i < adapter->num_queues; i++, txr++) {
3557 ixgbe_free_transmit_buffers(txr);
3558 ixgbe_dma_free(adapter, &txr->txdma);
3559 IXGBE_TX_LOCK_DESTROY(txr);
3560 }
3561 free(adapter->tx_rings, M_DEVBUF);
3562}
3563
3564/*********************************************************************
3565 *
3566 * Free transmit ring related data structures.
3567 *
3568 **********************************************************************/
3569static void
3570ixgbe_free_transmit_buffers(struct tx_ring *txr)
3571{
3572 struct adapter *adapter = txr->adapter;
3573 struct ixgbe_tx_buf *tx_buffer;
3574 int i;
3575
3576 INIT_DEBUGOUT("ixgbe_free_transmit_ring: begin");
3577
3578 if (txr->tx_buffers == NULL)
3579 return;
3580
3581 tx_buffer = txr->tx_buffers;
3582 for (i = 0; i < adapter->num_tx_desc; i++, tx_buffer++) {
3583 if (tx_buffer->m_head != NULL) {
3584 bus_dmamap_sync(txr->txtag->dt_dmat, tx_buffer->map,
3585 0, tx_buffer->m_head->m_pkthdr.len,
3586 BUS_DMASYNC_POSTWRITE);
3587 ixgbe_dmamap_unload(txr->txtag, tx_buffer->map);
3588 m_freem(tx_buffer->m_head);
3589 tx_buffer->m_head = NULL;
3590 if (tx_buffer->map != NULL) {
3591 ixgbe_dmamap_destroy(txr->txtag,
3592 tx_buffer->map);
3593 tx_buffer->map = NULL;
3594 }
3595 } else if (tx_buffer->map != NULL) {
3596 ixgbe_dmamap_unload(txr->txtag, tx_buffer->map);
3597 ixgbe_dmamap_destroy(txr->txtag, tx_buffer->map);
3598 tx_buffer->map = NULL;
3599 }
3600 }
3601#ifndef IXGBE_LEGACY_TX
3602 if (txr->br != NULL)
3603 buf_ring_free(txr->br, M_DEVBUF);
3604#endif
3605 if (txr->tx_buffers != NULL) {
3606 free(txr->tx_buffers, M_DEVBUF);
3607 txr->tx_buffers = NULL;
3608 }
3609 if (txr->txtag != NULL) {
3610 ixgbe_dma_tag_destroy(txr->txtag);
3611 txr->txtag = NULL;
3612 }
3613 return;
3614}
3615
3616/*********************************************************************
3617 *
3618 * Advanced Context Descriptor setup for VLAN, CSUM or TSO
3619 *
3620 **********************************************************************/
3621
3622static int
3623ixgbe_tx_ctx_setup(struct tx_ring *txr, struct mbuf *mp,
3624 u32 *cmd_type_len, u32 *olinfo_status)
3625{
3626 struct m_tag *mtag;
3627 struct adapter *adapter = txr->adapter;
3628 struct ethercom *ec = &adapter->osdep.ec;
3629 struct ixgbe_adv_tx_context_desc *TXD;
3630 struct ether_vlan_header *eh;
3631 struct ip ip;
3632 struct ip6_hdr ip6;
3633 u32 vlan_macip_lens = 0, type_tucmd_mlhl = 0;
3634 int ehdrlen, ip_hlen = 0;
3635 u16 etype;
3636 u8 ipproto __diagused = 0;
3637 int offload = TRUE;
3638 int ctxd = txr->next_avail_desc;
3639 u16 vtag = 0;
3640
3641 /* First check if TSO is to be used */
3642 if (mp->m_pkthdr.csum_flags & (M_CSUM_TSOv4|M_CSUM_TSOv6))
3643 return (ixgbe_tso_setup(txr, mp, cmd_type_len, olinfo_status));
3644
3645 if ((mp->m_pkthdr.csum_flags & M_CSUM_OFFLOAD) == 0)
3646 offload = FALSE;
3647
3648 /* Indicate the whole packet as payload when not doing TSO */
3649 *olinfo_status |= mp->m_pkthdr.len << IXGBE_ADVTXD_PAYLEN_SHIFT;
3650
3651 /* Now ready a context descriptor */
3652 TXD = (struct ixgbe_adv_tx_context_desc *) &txr->tx_base[ctxd];
3653
3654 /*
3655 ** In advanced descriptors the vlan tag must
3656 ** be placed into the context descriptor. Hence
3657 ** we need to make one even if not doing offloads.
3658 */
3659 if ((mtag = VLAN_OUTPUT_TAG(ec, mp)) != NULL) {
3660 vtag = htole16(VLAN_TAG_VALUE(mtag) & 0xffff);
3661 vlan_macip_lens |= (vtag << IXGBE_ADVTXD_VLAN_SHIFT);
3662 } else if (offload == FALSE) /* ... no offload to do */
3663 return 0;
3664
3665 /*
3666 * Determine where frame payload starts.
3667 * Jump over vlan headers if already present,
3668 * helpful for QinQ too.
3669 */
3670 KASSERT(mp->m_len >= offsetof(struct ether_vlan_header, evl_tag));
3671 eh = mtod(mp, struct ether_vlan_header *);
3672 if (eh->evl_encap_proto == htons(ETHERTYPE_VLAN)) {
3673 KASSERT(mp->m_len >= sizeof(struct ether_vlan_header));
3674 etype = ntohs(eh->evl_proto);
3675 ehdrlen = ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN;
3676 } else {
3677 etype = ntohs(eh->evl_encap_proto);
3678 ehdrlen = ETHER_HDR_LEN;
3679 }
3680
3681 /* Set the ether header length */
3682 vlan_macip_lens |= ehdrlen << IXGBE_ADVTXD_MACLEN_SHIFT;
3683
3684 switch (etype) {
3685 case ETHERTYPE_IP:
3686 m_copydata(mp, ehdrlen, sizeof(ip), &ip);
3687 ip_hlen = ip.ip_hl << 2;
3688 ipproto = ip.ip_p;
3689#if 0
3690 ip.ip_sum = 0;
3691 m_copyback(mp, ehdrlen, sizeof(ip), &ip);
3692#else
3693 KASSERT((mp->m_pkthdr.csum_flags & M_CSUM_IPv4) == 0 ||
3694 ip.ip_sum == 0);
3695#endif
3696 type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_IPV4;
3697 break;
3698 case ETHERTYPE_IPV6:
3699 m_copydata(mp, ehdrlen, sizeof(ip6), &ip6);
3700 ip_hlen = sizeof(ip6);
3701 /* XXX-BZ this will go badly in case of ext hdrs. */
3702 ipproto = ip6.ip6_nxt;
3703 type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_IPV6;
3704 break;
3705 default:
3706 break;
3707 }
3708
3709 if ((mp->m_pkthdr.csum_flags & M_CSUM_IPv4) != 0)
3710 *olinfo_status |= IXGBE_TXD_POPTS_IXSM << 8;
3711
3712 vlan_macip_lens |= ip_hlen;
3713 type_tucmd_mlhl |= IXGBE_ADVTXD_DCMD_DEXT | IXGBE_ADVTXD_DTYP_CTXT;
3714
3715 if (mp->m_pkthdr.csum_flags & (M_CSUM_TCPv4|M_CSUM_TCPv6)) {
3716 type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_L4T_TCP;
3717 *olinfo_status |= IXGBE_TXD_POPTS_TXSM << 8;
3718 KASSERT(ipproto == IPPROTO_TCP);
3719 } else if (mp->m_pkthdr.csum_flags & (M_CSUM_UDPv4|M_CSUM_UDPv6)) {
3720 type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_L4T_UDP;
3721 *olinfo_status |= IXGBE_TXD_POPTS_TXSM << 8;
3722 KASSERT(ipproto == IPPROTO_UDP);
3723 }
3724
3725 /* Now copy bits into descriptor */
3726 TXD->vlan_macip_lens = htole32(vlan_macip_lens);
3727 TXD->type_tucmd_mlhl = htole32(type_tucmd_mlhl);
3728 TXD->seqnum_seed = htole32(0);
3729 TXD->mss_l4len_idx = htole32(0);
3730
3731 /* We've consumed the first desc, adjust counters */
3732 if (++ctxd == txr->num_desc)
3733 ctxd = 0;
3734 txr->next_avail_desc = ctxd;
3735 --txr->tx_avail;
3736
3737 return 0;
3738}
3739
3740/**********************************************************************
3741 *
3742 * Setup work for hardware segmentation offload (TSO) on
3743 * adapters using advanced tx descriptors
3744 *
3745 **********************************************************************/
3746static int
3747ixgbe_tso_setup(struct tx_ring *txr, struct mbuf *mp,
3748 u32 *cmd_type_len, u32 *olinfo_status)
3749{
3750 struct m_tag *mtag;
3751 struct adapter *adapter = txr->adapter;
3752 struct ethercom *ec = &adapter->osdep.ec;
3753 struct ixgbe_adv_tx_context_desc *TXD;
3754 u32 vlan_macip_lens = 0, type_tucmd_mlhl = 0;
3755 u32 mss_l4len_idx = 0, paylen;
3756 u16 vtag = 0, eh_type;
3757 int ctxd, ehdrlen, ip_hlen, tcp_hlen;
3758 struct ether_vlan_header *eh;
3759#ifdef INET6
3760 struct ip6_hdr *ip6;
3761#endif
3762#ifdef INET
3763 struct ip *ip;
3764#endif
3765 struct tcphdr *th;
3766
3767
3768 /*
3769 * Determine where frame payload starts.
3770 * Jump over vlan headers if already present
3771 */
3772 eh = mtod(mp, struct ether_vlan_header *);
3773 if (eh->evl_encap_proto == htons(ETHERTYPE_VLAN)) {
3774 ehdrlen = ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN;
3775 eh_type = eh->evl_proto;
3776 } else {
3777 ehdrlen = ETHER_HDR_LEN;
3778 eh_type = eh->evl_encap_proto;
3779 }
3780
3781 switch (ntohs(eh_type)) {
3782#ifdef INET6
3783 case ETHERTYPE_IPV6:
3784 ip6 = (struct ip6_hdr *)(mp->m_data + ehdrlen);
3785 /* XXX-BZ For now we do not pretend to support ext. hdrs. */
3786 if (ip6->ip6_nxt != IPPROTO_TCP)
3787 return (ENXIO);
3788 ip_hlen = sizeof(struct ip6_hdr);
3789 ip6 = (struct ip6_hdr *)(mp->m_data + ehdrlen);
3790 th = (struct tcphdr *)((char *)ip6 + ip_hlen);
3791 th->th_sum = in6_cksum_phdr(&ip6->ip6_src,
3792 &ip6->ip6_dst, 0, htonl(IPPROTO_TCP));
3793 type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_IPV6;
3794 break;
3795#endif
3796#ifdef INET
3797 case ETHERTYPE_IP:
3798 ip = (struct ip *)(mp->m_data + ehdrlen);
3799 if (ip->ip_p != IPPROTO_TCP)
3800 return (ENXIO);
3801 ip->ip_sum = 0;
3802 ip_hlen = ip->ip_hl << 2;
3803 th = (struct tcphdr *)((char *)ip + ip_hlen);
3804 th->th_sum = in_cksum_phdr(ip->ip_src.s_addr,
3805 ip->ip_dst.s_addr, htons(IPPROTO_TCP));
3806 type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_IPV4;
3807 /* Tell transmit desc to also do IPv4 checksum. */
3808 *olinfo_status |= IXGBE_TXD_POPTS_IXSM << 8;
3809 break;
3810#endif
3811 default:
3812 panic("%s: CSUM_TSO but no supported IP version (0x%04x)",
3813 __func__, ntohs(eh_type));
3814 break;
3815 }
3816
3817 ctxd = txr->next_avail_desc;
3818 TXD = (struct ixgbe_adv_tx_context_desc *) &txr->tx_base[ctxd];
3819
3820 tcp_hlen = th->th_off << 2;
3821
3822 /* This is used in the transmit desc in encap */
3823 paylen = mp->m_pkthdr.len - ehdrlen - ip_hlen - tcp_hlen;
3824
3825 /* VLAN MACLEN IPLEN */
3826 if ((mtag = VLAN_OUTPUT_TAG(ec, mp)) != NULL) {
3827 vtag = htole16(VLAN_TAG_VALUE(mtag) & 0xffff);
3828 vlan_macip_lens |= (vtag << IXGBE_ADVTXD_VLAN_SHIFT);
3829 }
3830
3831 vlan_macip_lens |= ehdrlen << IXGBE_ADVTXD_MACLEN_SHIFT;
3832 vlan_macip_lens |= ip_hlen;
3833 TXD->vlan_macip_lens = htole32(vlan_macip_lens);
3834
3835 /* ADV DTYPE TUCMD */
3836 type_tucmd_mlhl |= IXGBE_ADVTXD_DCMD_DEXT | IXGBE_ADVTXD_DTYP_CTXT;
3837 type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_L4T_TCP;
3838 TXD->type_tucmd_mlhl = htole32(type_tucmd_mlhl);
3839
3840 /* MSS L4LEN IDX */
3841 mss_l4len_idx |= (mp->m_pkthdr.segsz << IXGBE_ADVTXD_MSS_SHIFT);
3842 mss_l4len_idx |= (tcp_hlen << IXGBE_ADVTXD_L4LEN_SHIFT);
3843 TXD->mss_l4len_idx = htole32(mss_l4len_idx);
3844
3845 TXD->seqnum_seed = htole32(0);
3846
3847 if (++ctxd == txr->num_desc)
3848 ctxd = 0;
3849
3850 txr->tx_avail--;
3851 txr->next_avail_desc = ctxd;
3852 *cmd_type_len |= IXGBE_ADVTXD_DCMD_TSE;
3853 *olinfo_status |= IXGBE_TXD_POPTS_TXSM << 8;
3854 *olinfo_status |= paylen << IXGBE_ADVTXD_PAYLEN_SHIFT;
3855 ++txr->tso_tx.ev_count;
3856 return (0);
3857}
3858
3859#ifdef IXGBE_FDIR
3860/*
3861** This routine parses packet headers so that Flow
3862** Director can make a hashed filter table entry
3863** allowing traffic flows to be identified and kept
3864** on the same cpu. This would be a performance
3865** hit, but we only do it at IXGBE_FDIR_RATE of
3866** packets.
3867*/
3868static void
3869ixgbe_atr(struct tx_ring *txr, struct mbuf *mp)
3870{
3871 struct adapter *adapter = txr->adapter;
3872 struct ix_queue *que;
3873 struct ip *ip;
3874 struct tcphdr *th;
3875 struct udphdr *uh;
3876 struct ether_vlan_header *eh;
3877 union ixgbe_atr_hash_dword input = {.dword = 0};
3878 union ixgbe_atr_hash_dword common = {.dword = 0};
3879 int ehdrlen, ip_hlen;
3880 u16 etype;
3881
3882 eh = mtod(mp, struct ether_vlan_header *);
3883 if (eh->evl_encap_proto == htons(ETHERTYPE_VLAN)) {
3884 ehdrlen = ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN;
3885 etype = eh->evl_proto;
3886 } else {
3887 ehdrlen = ETHER_HDR_LEN;
3888 etype = eh->evl_encap_proto;
3889 }
3890
3891 /* Only handling IPv4 */
3892 if (etype != htons(ETHERTYPE_IP))
3893 return;
3894
3895 ip = (struct ip *)(mp->m_data + ehdrlen);
3896 ip_hlen = ip->ip_hl << 2;
3897
3898 /* check if we're UDP or TCP */
3899 switch (ip->ip_p) {
3900 case IPPROTO_TCP:
3901 th = (struct tcphdr *)((char *)ip + ip_hlen);
3902 /* src and dst are inverted */
3903 common.port.dst ^= th->th_sport;
3904 common.port.src ^= th->th_dport;
3905 input.formatted.flow_type ^= IXGBE_ATR_FLOW_TYPE_TCPV4;
3906 break;
3907 case IPPROTO_UDP:
3908 uh = (struct udphdr *)((char *)ip + ip_hlen);
3909 /* src and dst are inverted */
3910 common.port.dst ^= uh->uh_sport;
3911 common.port.src ^= uh->uh_dport;
3912 input.formatted.flow_type ^= IXGBE_ATR_FLOW_TYPE_UDPV4;
3913 break;
3914 default:
3915 return;
3916 }
3917
3918 input.formatted.vlan_id = htobe16(mp->m_pkthdr.ether_vtag);
3919 if (mp->m_pkthdr.ether_vtag)
3920 common.flex_bytes ^= htons(ETHERTYPE_VLAN);
3921 else
3922 common.flex_bytes ^= etype;
3923 common.ip ^= ip->ip_src.s_addr ^ ip->ip_dst.s_addr;
3924
3925 que = &adapter->queues[txr->me];
3926 /*
3927 ** This assumes the Rx queue and Tx
3928 ** queue are bound to the same CPU
3929 */
3930 ixgbe_fdir_add_signature_filter_82599(&adapter->hw,
3931 input, common, que->msix);
3932}
3933#endif /* IXGBE_FDIR */
3934
3935/**********************************************************************
3936 *
3937 * Examine each tx_buffer in the used queue. If the hardware is done
3938 * processing the packet then free associated resources. The
3939 * tx_buffer is put back on the free queue.
3940 *
3941 **********************************************************************/
3942static void
3943ixgbe_txeof(struct tx_ring *txr)
3944{
3945 struct adapter *adapter = txr->adapter;
3946 struct ifnet *ifp = adapter->ifp;
3947 u32 work, processed = 0;
3948 u16 limit = txr->process_limit;
3949 struct ixgbe_tx_buf *buf;
3950 union ixgbe_adv_tx_desc *txd;
3951 struct timeval now, elapsed;
3952
3953 KASSERT(mutex_owned(&txr->tx_mtx));
3954
3955#ifdef DEV_NETMAP
3956 if (ifp->if_capenable & IFCAP_NETMAP) {
3957 struct netmap_adapter *na = NA(ifp);
3958 struct netmap_kring *kring = &na->tx_rings[txr->me];
3959 txd = txr->tx_base;
3960 bus_dmamap_sync(txr->txdma.dma_tag, txr->txdma.dma_map,
3961 BUS_DMASYNC_POSTREAD);
3962 /*
3963 * In netmap mode, all the work is done in the context
3964 * of the client thread. Interrupt handlers only wake up
3965 * clients, which may be sleeping on individual rings
3966 * or on a global resource for all rings.
3967 * To implement tx interrupt mitigation, we wake up the client
3968 * thread roughly every half ring, even if the NIC interrupts
3969 * more frequently. This is implemented as follows:
3970 * - ixgbe_txsync() sets kring->nr_kflags with the index of
3971 * the slot that should wake up the thread (nkr_num_slots
3972 * means the user thread should not be woken up);
3973 * - the driver ignores tx interrupts unless netmap_mitigate=0
3974 * or the slot has the DD bit set.
3975 */
3976 if (!netmap_mitigate ||
3977 (kring->nr_kflags < kring->nkr_num_slots &&
3978 txd[kring->nr_kflags].wb.status & IXGBE_TXD_STAT_DD)) {
3979 netmap_tx_irq(ifp, txr->me);
3980 }
3981 return;
3982 }
3983#endif /* DEV_NETMAP */
3984
3985 if (txr->tx_avail == txr->num_desc) {
3986 txr->queue_status = IXGBE_QUEUE_IDLE;
3987 return;
3988 }
3989
3990 /* Get work starting point */
3991 work = txr->next_to_clean;
3992 buf = &txr->tx_buffers[work];
3993 txd = &txr->tx_base[work];
3994 work -= txr->num_desc; /* The distance to ring end */
3995 ixgbe_dmamap_sync(txr->txdma.dma_tag, txr->txdma.dma_map,
3996 BUS_DMASYNC_POSTREAD);
3997 do {
3998 union ixgbe_adv_tx_desc *eop= buf->eop;
3999 if (eop == NULL) /* No work */
4000 break;
4001
4002 if ((eop->wb.status & IXGBE_TXD_STAT_DD) == 0)
4003 break; /* I/O not complete */
4004
4005 if (buf->m_head) {
4006 txr->bytes +=
4007 buf->m_head->m_pkthdr.len;
4008 bus_dmamap_sync(txr->txtag->dt_dmat,
4009 buf->map,
4010 0, buf->m_head->m_pkthdr.len,
4011 BUS_DMASYNC_POSTWRITE);
4012 ixgbe_dmamap_unload(txr->txtag,
4013 buf->map);
4014 m_freem(buf->m_head);
4015 buf->m_head = NULL;
4016 /*
4017 * NetBSD: Don't override buf->map with NULL here.
4018 * It'll panic when a ring runs one lap around.
4019 */
4020 }
4021 buf->eop = NULL;
4022 ++txr->tx_avail;
4023
4024 /* We clean the range if multi segment */
4025 while (txd != eop) {
4026 ++txd;
4027 ++buf;
4028 ++work;
4029 /* wrap the ring? */
4030 if (__predict_false(!work)) {
4031 work -= txr->num_desc;
4032 buf = txr->tx_buffers;
4033 txd = txr->tx_base;
4034 }
4035 if (buf->m_head) {
4036 txr->bytes +=
4037 buf->m_head->m_pkthdr.len;
4038 bus_dmamap_sync(txr->txtag->dt_dmat,
4039 buf->map,
4040 0, buf->m_head->m_pkthdr.len,
4041 BUS_DMASYNC_POSTWRITE);
4042 ixgbe_dmamap_unload(txr->txtag,
4043 buf->map);
4044 m_freem(buf->m_head);
4045 buf->m_head = NULL;
4046 /*
4047 * NetBSD: Don't override buf->map with NULL
4048 * here. It'll panic when a ring runs one lap
4049 * around.
4050 */
4051 }
4052 ++txr->tx_avail;
4053 buf->eop = NULL;
4054
4055 }
4056 ++txr->packets;
4057 ++processed;
4058 ++ifp->if_opackets;
4059 getmicrotime(&txr->watchdog_time);
4060
4061 /* Try the next packet */
4062 ++txd;
4063 ++buf;
4064 ++work;
4065 /* reset with a wrap */
4066 if (__predict_false(!work)) {
4067 work -= txr->num_desc;
4068 buf = txr->tx_buffers;
4069 txd = txr->tx_base;
4070 }
4071 prefetch(txd);
4072 } while (__predict_true(--limit));
4073
4074 ixgbe_dmamap_sync(txr->txdma.dma_tag, txr->txdma.dma_map,
4075 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
4076
4077 work += txr->num_desc;
4078 txr->next_to_clean = work;
4079
4080 /*
4081 ** Watchdog calculation, we know there's
4082 ** work outstanding or the first return
4083 ** would have been taken, so none processed
4084 ** for too long indicates a hang.
4085 */
4086 getmicrotime(&now);
4087 timersub(&now, &txr->watchdog_time, &elapsed);
4088 if (!processed && tvtohz(&elapsed) > IXGBE_WATCHDOG)
4089 txr->queue_status = IXGBE_QUEUE_HUNG;
4090
4091 if (txr->tx_avail == txr->num_desc)
4092 txr->queue_status = IXGBE_QUEUE_IDLE;
4093
4094 return;
4095}
4096
4097/*********************************************************************
4098 *
4099 * Refresh mbuf buffers for RX descriptor rings
4100 * - now keeps its own state so discards due to resource
4101 * exhaustion are unnecessary, if an mbuf cannot be obtained
4102 * it just returns, keeping its placeholder, thus it can simply
4103 * be recalled to try again.
4104 *
4105 **********************************************************************/
4106static void
4107ixgbe_refresh_mbufs(struct rx_ring *rxr, int limit)
4108{
4109 struct adapter *adapter = rxr->adapter;
4110 struct ixgbe_rx_buf *rxbuf;
4111 struct mbuf *mp;
4112 int i, j, error;
4113 bool refreshed = false;
4114
4115 i = j = rxr->next_to_refresh;
4116 /* Control the loop with one beyond */
4117 if (++j == rxr->num_desc)
4118 j = 0;
4119
4120 while (j != limit) {
4121 rxbuf = &rxr->rx_buffers[i];
4122 if (rxbuf->buf == NULL) {
4123 mp = ixgbe_getjcl(&adapter->jcl_head, M_NOWAIT,
4124 MT_DATA, M_PKTHDR, rxr->mbuf_sz);
4125 if (mp == NULL) {
4126 rxr->no_jmbuf.ev_count++;
4127 goto update;
4128 }
4129 if (adapter->max_frame_size <= (MCLBYTES - ETHER_ALIGN))
4130 m_adj(mp, ETHER_ALIGN);
4131 } else
4132 mp = rxbuf->buf;
4133
4134 mp->m_pkthdr.len = mp->m_len = rxr->mbuf_sz;
4135
4136 /* If we're dealing with an mbuf that was copied rather
4137 * than replaced, there's no need to go through busdma.
4138 */
4139 if ((rxbuf->flags & IXGBE_RX_COPY) == 0) {
4140 /* Get the memory mapping */
4141 error = bus_dmamap_load_mbuf(rxr->ptag->dt_dmat,
4142 rxbuf->pmap, mp, BUS_DMA_NOWAIT);
4143 if (error != 0) {
4144 printf("Refresh mbufs: payload dmamap load"
4145 " failure - %d\n", error);
4146 m_free(mp);
4147 rxbuf->buf = NULL;
4148 goto update;
4149 }
4150 rxbuf->buf = mp;
4151 bus_dmamap_sync(rxr->ptag->dt_dmat, rxbuf->pmap,
4152 0, mp->m_pkthdr.len, BUS_DMASYNC_PREREAD);
4153 rxbuf->addr = rxr->rx_base[i].read.pkt_addr =
4154 htole64(rxbuf->pmap->dm_segs[0].ds_addr);
4155 } else {
4156 rxr->rx_base[i].read.pkt_addr = rxbuf->addr;
4157 rxbuf->flags &= ~IXGBE_RX_COPY;
4158 }
4159
4160 refreshed = true;
4161 /* Next is precalculated */
4162 i = j;
4163 rxr->next_to_refresh = i;
4164 if (++j == rxr->num_desc)
4165 j = 0;
4166 }
4167update:
4168 if (refreshed) /* Update hardware tail index */
4169 IXGBE_WRITE_REG(&adapter->hw,
4170 IXGBE_RDT(rxr->me), rxr->next_to_refresh);
4171 return;
4172}
4173
4174/*********************************************************************
4175 *
4176 * Allocate memory for rx_buffer structures. Since we use one
4177 * rx_buffer per received packet, the maximum number of rx_buffer's
4178 * that we'll need is equal to the number of receive descriptors
4179 * that we've allocated.
4180 *
4181 **********************************************************************/
4182static int
4183ixgbe_allocate_receive_buffers(struct rx_ring *rxr)
4184{
4185 struct adapter *adapter = rxr->adapter;
4186 device_t dev = adapter->dev;
4187 struct ixgbe_rx_buf *rxbuf;
4188 int i, bsize, error;
4189
4190 bsize = sizeof(struct ixgbe_rx_buf) * rxr->num_desc;
4191 if (!(rxr->rx_buffers =
4192 (struct ixgbe_rx_buf *) malloc(bsize,
4193 M_DEVBUF, M_NOWAIT | M_ZERO))) {
4194 aprint_error_dev(dev, "Unable to allocate rx_buffer memory\n");
4195 error = ENOMEM;
4196 goto fail;
4197 }
4198
4199 if ((error = ixgbe_dma_tag_create(adapter->osdep.dmat, /* parent */
4200 1, 0, /* alignment, bounds */
4201 MJUM16BYTES, /* maxsize */
4202 1, /* nsegments */
4203 MJUM16BYTES, /* maxsegsize */
4204 0, /* flags */
4205 &rxr->ptag))) {
4206 aprint_error_dev(dev, "Unable to create RX DMA tag\n");
4207 goto fail;
4208 }
4209
4210 for (i = 0; i < rxr->num_desc; i++, rxbuf++) {
4211 rxbuf = &rxr->rx_buffers[i];
4212 error = ixgbe_dmamap_create(rxr->ptag,
4213 BUS_DMA_NOWAIT, &rxbuf->pmap);
4214 if (error) {
4215 aprint_error_dev(dev, "Unable to create RX dma map\n");
4216 goto fail;
4217 }
4218 }
4219
4220 return (0);
4221
4222fail:
4223 /* Frees all, but can handle partial completion */
4224 ixgbe_free_receive_structures(adapter);
4225 return (error);
4226}
4227
4228/*
4229** Used to detect a descriptor that has
4230** been merged by Hardware RSC.
4231*/
4232static inline u32
4233ixgbe_rsc_count(union ixgbe_adv_rx_desc *rx)
4234{
4235 return (le32toh(rx->wb.lower.lo_dword.data) &
4236 IXGBE_RXDADV_RSCCNT_MASK) >> IXGBE_RXDADV_RSCCNT_SHIFT;
4237}
4238
4239/*********************************************************************
4240 *
4241 * Initialize Hardware RSC (LRO) feature on 82599
4242 * for an RX ring, this is toggled by the LRO capability
4243 * even though it is transparent to the stack.
4244 *
4245 * NOTE: since this HW feature only works with IPV4 and
4246 * our testing has shown soft LRO to be as effective
4247 * I have decided to disable this by default.
4248 *
4249 **********************************************************************/
4250static void
4251ixgbe_setup_hw_rsc(struct rx_ring *rxr)
4252{
4253 struct adapter *adapter = rxr->adapter;
4254 struct ixgbe_hw *hw = &adapter->hw;
4255 u32 rscctrl, rdrxctl;
4256
4257 /* If turning LRO/RSC off we need to disable it */
4258 if ((adapter->ifp->if_capenable & IFCAP_LRO) == 0) {
4259 rscctrl = IXGBE_READ_REG(hw, IXGBE_RSCCTL(rxr->me));
4260 rscctrl &= ~IXGBE_RSCCTL_RSCEN;
4261 return;
4262 }
4263
4264 rdrxctl = IXGBE_READ_REG(hw, IXGBE_RDRXCTL);
4265 rdrxctl &= ~IXGBE_RDRXCTL_RSCFRSTSIZE;
4266#ifdef DEV_NETMAP /* crcstrip is optional in netmap */
4267 if (adapter->ifp->if_capenable & IFCAP_NETMAP && !ix_crcstrip)
4268#endif /* DEV_NETMAP */
4269 rdrxctl |= IXGBE_RDRXCTL_CRCSTRIP;
4270 rdrxctl |= IXGBE_RDRXCTL_RSCACKC;
4271 IXGBE_WRITE_REG(hw, IXGBE_RDRXCTL, rdrxctl);
4272
4273 rscctrl = IXGBE_READ_REG(hw, IXGBE_RSCCTL(rxr->me));
4274 rscctrl |= IXGBE_RSCCTL_RSCEN;
4275 /*
4276 ** Limit the total number of descriptors that
4277 ** can be combined, so it does not exceed 64K
4278 */
4279 if (rxr->mbuf_sz == MCLBYTES)
4280 rscctrl |= IXGBE_RSCCTL_MAXDESC_16;
4281 else if (rxr->mbuf_sz == MJUMPAGESIZE)
4282 rscctrl |= IXGBE_RSCCTL_MAXDESC_8;
4283 else if (rxr->mbuf_sz == MJUM9BYTES)
4284 rscctrl |= IXGBE_RSCCTL_MAXDESC_4;
4285 else /* Using 16K cluster */
4286 rscctrl |= IXGBE_RSCCTL_MAXDESC_1;
4287
4288 IXGBE_WRITE_REG(hw, IXGBE_RSCCTL(rxr->me), rscctrl);
4289
4290 /* Enable TCP header recognition */
4291 IXGBE_WRITE_REG(hw, IXGBE_PSRTYPE(0),
4292 (IXGBE_READ_REG(hw, IXGBE_PSRTYPE(0)) |
4293 IXGBE_PSRTYPE_TCPHDR));
4294
4295 /* Disable RSC for ACK packets */
4296 IXGBE_WRITE_REG(hw, IXGBE_RSCDBU,
4297 (IXGBE_RSCDBU_RSCACKDIS | IXGBE_READ_REG(hw, IXGBE_RSCDBU)));
4298
4299 rxr->hw_rsc = TRUE;
4300}
4301
4302
4303static void
4304ixgbe_free_receive_ring(struct rx_ring *rxr)
4305{
4306 struct ixgbe_rx_buf *rxbuf;
4307 int i;
4308
4309 for (i = 0; i < rxr->num_desc; i++) {
4310 rxbuf = &rxr->rx_buffers[i];
4311 if (rxbuf->buf != NULL) {
4312 bus_dmamap_sync(rxr->ptag->dt_dmat, rxbuf->pmap,
4313 0, rxbuf->buf->m_pkthdr.len,
4314 BUS_DMASYNC_POSTREAD);
4315 ixgbe_dmamap_unload(rxr->ptag, rxbuf->pmap);
4316 rxbuf->buf->m_flags |= M_PKTHDR;
4317 m_freem(rxbuf->buf);
4318 rxbuf->buf = NULL;
4319 rxbuf->flags = 0;
4320 }
4321 }
4322}
4323
4324
4325/*********************************************************************
4326 *
4327 * Initialize a receive ring and its buffers.
4328 *
4329 **********************************************************************/
4330static int
4331ixgbe_setup_receive_ring(struct rx_ring *rxr)
4332{
4333 struct adapter *adapter;
4334 struct ixgbe_rx_buf *rxbuf;
4335#ifdef LRO
4336 struct ifnet *ifp;
4337 struct lro_ctrl *lro = &rxr->lro;
4338#endif /* LRO */
4339 int rsize, error = 0;
4340#ifdef DEV_NETMAP
4341 struct netmap_adapter *na = NA(rxr->adapter->ifp);
4342 struct netmap_slot *slot;
4343#endif /* DEV_NETMAP */
4344
4345 adapter = rxr->adapter;
4346#ifdef LRO
4347 ifp = adapter->ifp;
4348#endif /* LRO */
4349
4350 /* Clear the ring contents */
4351 IXGBE_RX_LOCK(rxr);
4352#ifdef DEV_NETMAP
4353 /* same as in ixgbe_setup_transmit_ring() */
4354 slot = netmap_reset(na, NR_RX, rxr->me, 0);
4355#endif /* DEV_NETMAP */
4356 rsize = roundup2(adapter->num_rx_desc *
4357 sizeof(union ixgbe_adv_rx_desc), DBA_ALIGN);
4358 bzero((void *)rxr->rx_base, rsize);
4359 /* Cache the size */
4360 rxr->mbuf_sz = adapter->rx_mbuf_sz;
4361
4362 /* Free current RX buffer structs and their mbufs */
4363 ixgbe_free_receive_ring(rxr);
4364
4365 IXGBE_RX_UNLOCK(rxr);
4366
4367 /* Now reinitialize our supply of jumbo mbufs. The number
4368 * or size of jumbo mbufs may have changed.
4369 */
4370 ixgbe_jcl_reinit(&adapter->jcl_head, rxr->ptag->dt_dmat,
4371 2 * adapter->num_rx_desc, adapter->rx_mbuf_sz);
4372
4373 IXGBE_RX_LOCK(rxr);
4374
4375 /* Now replenish the mbufs */
4376 for (int j = 0; j != rxr->num_desc; ++j) {
4377 struct mbuf *mp;
4378
4379 rxbuf = &rxr->rx_buffers[j];
4380#ifdef DEV_NETMAP
4381 /*
4382 * In netmap mode, fill the map and set the buffer
4383 * address in the NIC ring, considering the offset
4384 * between the netmap and NIC rings (see comment in
4385 * ixgbe_setup_transmit_ring() ). No need to allocate
4386 * an mbuf, so end the block with a continue;
4387 */
4388 if (slot) {
4389 int sj = netmap_idx_n2k(&na->rx_rings[rxr->me], j);
4390 uint64_t paddr;
4391 void *addr;
4392
4393 addr = PNMB(na, slot + sj, &paddr);
4394 netmap_load_map(na, rxr->ptag, rxbuf->pmap, addr);
4395 /* Update descriptor and the cached value */
4396 rxr->rx_base[j].read.pkt_addr = htole64(paddr);
4397 rxbuf->addr = htole64(paddr);
4398 continue;
4399 }
4400#endif /* DEV_NETMAP */
4401 rxbuf->flags = 0;
4402 rxbuf->buf = ixgbe_getjcl(&adapter->jcl_head, M_NOWAIT,
4403 MT_DATA, M_PKTHDR, adapter->rx_mbuf_sz);
4404 if (rxbuf->buf == NULL) {
4405 error = ENOBUFS;
4406 goto fail;
4407 }
4408 mp = rxbuf->buf;
4409 mp->m_pkthdr.len = mp->m_len = rxr->mbuf_sz;
4410 /* Get the memory mapping */
4411 error = bus_dmamap_load_mbuf(rxr->ptag->dt_dmat,
4412 rxbuf->pmap, mp, BUS_DMA_NOWAIT);
4413 if (error != 0)
4414 goto fail;
4415 bus_dmamap_sync(rxr->ptag->dt_dmat, rxbuf->pmap,
4416 0, adapter->rx_mbuf_sz, BUS_DMASYNC_PREREAD);
4417 /* Update the descriptor and the cached value */
4418 rxr->rx_base[j].read.pkt_addr =
4419 htole64(rxbuf->pmap->dm_segs[0].ds_addr);
4420 rxbuf->addr = htole64(rxbuf->pmap->dm_segs[0].ds_addr);
4421 }
4422
4423
4424 /* Setup our descriptor indices */
4425 rxr->next_to_check = 0;
4426 rxr->next_to_refresh = 0;
4427 rxr->lro_enabled = FALSE;
4428 rxr->rx_copies.ev_count = 0;
4429 rxr->rx_bytes.ev_count = 0;
4430 rxr->vtag_strip = FALSE;
4431
4432 ixgbe_dmamap_sync(rxr->rxdma.dma_tag, rxr->rxdma.dma_map,
4433 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
4434
4435 /*
4436 ** Now set up the LRO interface:
4437 */
4438 if (ixgbe_rsc_enable)
4439 ixgbe_setup_hw_rsc(rxr);
4440#ifdef LRO
4441 else if (ifp->if_capenable & IFCAP_LRO) {
4442 device_t dev = adapter->dev;
4443 int err = tcp_lro_init(lro);
4444 if (err) {
4445 device_printf(dev, "LRO Initialization failed!\n");
4446 goto fail;
4447 }
4448 INIT_DEBUGOUT("RX Soft LRO Initialized\n");
4449 rxr->lro_enabled = TRUE;
4450 lro->ifp = adapter->ifp;
4451 }
4452#endif /* LRO */
4453
4454 IXGBE_RX_UNLOCK(rxr);
4455 return (0);
4456
4457fail:
4458 ixgbe_free_receive_ring(rxr);
4459 IXGBE_RX_UNLOCK(rxr);
4460 return (error);
4461}
4462
4463/*********************************************************************
4464 *
4465 * Initialize all receive rings.
4466 *
4467 **********************************************************************/
4468static int
4469ixgbe_setup_receive_structures(struct adapter *adapter)
4470{
4471 struct rx_ring *rxr = adapter->rx_rings;
4472 int j;
4473
4474 for (j = 0; j < adapter->num_queues; j++, rxr++)
4475 if (ixgbe_setup_receive_ring(rxr))
4476 goto fail;
4477
4478 return (0);
4479fail:
4480 /*
4481 * Free RX buffers allocated so far, we will only handle
4482 * the rings that completed, the failing case will have
4483 * cleaned up for itself. 'j' failed, so its the terminus.
4484 */
4485 for (int i = 0; i < j; ++i) {
4486 rxr = &adapter->rx_rings[i];
4487 ixgbe_free_receive_ring(rxr);
4488 }
4489
4490 return (ENOBUFS);
4491}
4492
4493static void
4494ixgbe_initialise_rss_mapping(struct adapter *adapter)
4495{
4496 struct ixgbe_hw *hw = &adapter->hw;
4497 uint32_t reta;
4498 int i, j, queue_id;
4499 uint32_t rss_key[10];
4500 uint32_t mrqc;
4501#ifdef RSS
4502 uint32_t rss_hash_config;
4503#endif
4504
4505 /* Setup RSS */
4506 reta = 0;
4507
4508#ifdef RSS
4509 /* Fetch the configured RSS key */
4510 rss_getkey((uint8_t *) &rss_key);
4511#else
4512 /* set up random bits */
4513 cprng_fast(&rss_key, sizeof(rss_key));
4514#endif
4515
4516 /* Set up the redirection table */
4517 for (i = 0, j = 0; i < 128; i++, j++) {
4518 if (j == adapter->num_queues) j = 0;
4519#ifdef RSS
4520 /*
4521 * Fetch the RSS bucket id for the given indirection entry.
4522 * Cap it at the number of configured buckets (which is
4523 * num_queues.)
4524 */
4525 queue_id = rss_get_indirection_to_bucket(i);
4526 queue_id = queue_id % adapter->num_queues;
4527#else
4528 queue_id = (j * 0x11);
4529#endif
4530 /*
4531 * The low 8 bits are for hash value (n+0);
4532 * The next 8 bits are for hash value (n+1), etc.
4533 */
4534 reta = reta >> 8;
4535 reta = reta | ( ((uint32_t) queue_id) << 24);
4536 if ((i & 3) == 3) {
4537 IXGBE_WRITE_REG(hw, IXGBE_RETA(i >> 2), reta);
4538 reta = 0;
4539 }
4540 }
4541
4542 /* Now fill our hash function seeds */
4543 for (i = 0; i < 10; i++)
4544 IXGBE_WRITE_REG(hw, IXGBE_RSSRK(i), rss_key[i]);
4545
4546 /* Perform hash on these packet types */
4547#ifdef RSS
4548 mrqc = IXGBE_MRQC_RSSEN;
4549 rss_hash_config = rss_gethashconfig();
4550 if (rss_hash_config & RSS_HASHTYPE_RSS_IPV4)
4551 mrqc |= IXGBE_MRQC_RSS_FIELD_IPV4;
4552 if (rss_hash_config & RSS_HASHTYPE_RSS_TCP_IPV4)
4553 mrqc |= IXGBE_MRQC_RSS_FIELD_IPV4_TCP;
4554 if (rss_hash_config & RSS_HASHTYPE_RSS_IPV6)
4555 mrqc |= IXGBE_MRQC_RSS_FIELD_IPV6;
4556 if (rss_hash_config & RSS_HASHTYPE_RSS_TCP_IPV6)
4557 mrqc |= IXGBE_MRQC_RSS_FIELD_IPV6_TCP;
4558 if (rss_hash_config & RSS_HASHTYPE_RSS_IPV6_EX)
4559 mrqc |= IXGBE_MRQC_RSS_FIELD_IPV6_EX;
4560 if (rss_hash_config & RSS_HASHTYPE_RSS_TCP_IPV6_EX)
4561 mrqc |= IXGBE_MRQC_RSS_FIELD_IPV6_EX_TCP;
4562 if (rss_hash_config & RSS_HASHTYPE_RSS_UDP_IPV4)
4563 mrqc |= IXGBE_MRQC_RSS_FIELD_IPV4_UDP;
4564 if (rss_hash_config & RSS_HASHTYPE_RSS_UDP_IPV4_EX)
4565 device_printf(adapter->dev,
4566 "%s: RSS_HASHTYPE_RSS_UDP_IPV4_EX defined, "
4567 "but not supported\n", __func__);
4568 if (rss_hash_config & RSS_HASHTYPE_RSS_UDP_IPV6)
4569 mrqc |= IXGBE_MRQC_RSS_FIELD_IPV6_UDP;
4570 if (rss_hash_config & RSS_HASHTYPE_RSS_UDP_IPV6_EX)
4571 mrqc |= IXGBE_MRQC_RSS_FIELD_IPV6_EX_UDP;
4572#else
4573 /*
4574 * Disable UDP - IP fragments aren't currently being handled
4575 * and so we end up with a mix of 2-tuple and 4-tuple
4576 * traffic.
4577 */
4578 mrqc = IXGBE_MRQC_RSSEN
4579 | IXGBE_MRQC_RSS_FIELD_IPV4
4580 | IXGBE_MRQC_RSS_FIELD_IPV4_TCP
4581#if 0
4582 | IXGBE_MRQC_RSS_FIELD_IPV4_UDP
4583#endif
4584 | IXGBE_MRQC_RSS_FIELD_IPV6_EX_TCP
4585 | IXGBE_MRQC_RSS_FIELD_IPV6_EX
4586 | IXGBE_MRQC_RSS_FIELD_IPV6
4587 | IXGBE_MRQC_RSS_FIELD_IPV6_TCP
4588#if 0
4589 | IXGBE_MRQC_RSS_FIELD_IPV6_UDP
4590 | IXGBE_MRQC_RSS_FIELD_IPV6_EX_UDP
4591#endif
4592 ;
4593#endif /* RSS */
4594 IXGBE_WRITE_REG(hw, IXGBE_MRQC, mrqc);
4595}
4596
4597
4598/*********************************************************************
4599 *
4600 * Setup receive registers and features.
4601 *
4602 **********************************************************************/
4603#define IXGBE_SRRCTL_BSIZEHDRSIZE_SHIFT 2
4604
4605#define BSIZEPKT_ROUNDUP ((1<<IXGBE_SRRCTL_BSIZEPKT_SHIFT)-1)
4606
4607static void
4608ixgbe_initialize_receive_units(struct adapter *adapter)
4609{
4610 int i;
4611 struct rx_ring *rxr = adapter->rx_rings;
4612 struct ixgbe_hw *hw = &adapter->hw;
4613 struct ifnet *ifp = adapter->ifp;
4614 u32 bufsz, rxctrl, fctrl, srrctl, rxcsum;
4615 u32 hlreg;
4616
4617
4618 /*
4619 * Make sure receives are disabled while
4620 * setting up the descriptor ring
4621 */
4622 rxctrl = IXGBE_READ_REG(hw, IXGBE_RXCTRL);
4623 IXGBE_WRITE_REG(hw, IXGBE_RXCTRL,
4624 rxctrl & ~IXGBE_RXCTRL_RXEN);
4625
4626 /* Enable broadcasts */
4627 fctrl = IXGBE_READ_REG(hw, IXGBE_FCTRL);
4628 fctrl |= IXGBE_FCTRL_BAM;
4629 fctrl |= IXGBE_FCTRL_DPF;
4630 fctrl |= IXGBE_FCTRL_PMCF;
4631 IXGBE_WRITE_REG(hw, IXGBE_FCTRL, fctrl);
4632
4633 /* Set for Jumbo Frames? */
4634 hlreg = IXGBE_READ_REG(hw, IXGBE_HLREG0);
4635 if (ifp->if_mtu > ETHERMTU)
4636 hlreg |= IXGBE_HLREG0_JUMBOEN;
4637 else
4638 hlreg &= ~IXGBE_HLREG0_JUMBOEN;
4639#ifdef DEV_NETMAP
4640 /* crcstrip is conditional in netmap (in RDRXCTL too ?) */
4641 if (ifp->if_capenable & IFCAP_NETMAP && !ix_crcstrip)
4642 hlreg &= ~IXGBE_HLREG0_RXCRCSTRP;
4643 else
4644 hlreg |= IXGBE_HLREG0_RXCRCSTRP;
4645#endif /* DEV_NETMAP */
4646 IXGBE_WRITE_REG(hw, IXGBE_HLREG0, hlreg);
4647
4648 bufsz = (adapter->rx_mbuf_sz +
4649 BSIZEPKT_ROUNDUP) >> IXGBE_SRRCTL_BSIZEPKT_SHIFT;
4650
4651 for (i = 0; i < adapter->num_queues; i++, rxr++) {
4652 u64 rdba = rxr->rxdma.dma_paddr;
4653
4654 /* Setup the Base and Length of the Rx Descriptor Ring */
4655 IXGBE_WRITE_REG(hw, IXGBE_RDBAL(i),
4656 (rdba & 0x00000000ffffffffULL));
4657 IXGBE_WRITE_REG(hw, IXGBE_RDBAH(i), (rdba >> 32));
4658 IXGBE_WRITE_REG(hw, IXGBE_RDLEN(i),
4659 adapter->num_rx_desc * sizeof(union ixgbe_adv_rx_desc));
4660
4661 /* Set up the SRRCTL register */
4662 srrctl = IXGBE_READ_REG(hw, IXGBE_SRRCTL(i));
4663 srrctl &= ~IXGBE_SRRCTL_BSIZEHDR_MASK;
4664 srrctl &= ~IXGBE_SRRCTL_BSIZEPKT_MASK;
4665 srrctl |= bufsz;
4666 srrctl |= IXGBE_SRRCTL_DESCTYPE_ADV_ONEBUF;
4667
4668 /*
4669 * Set DROP_EN iff we have no flow control and >1 queue.
4670 * Note that srrctl was cleared shortly before during reset,
4671 * so we do not need to clear the bit, but do it just in case
4672 * this code is moved elsewhere.
4673 */
4674 if (adapter->num_queues > 1 &&
4675 adapter->fc == ixgbe_fc_none) {
4676 srrctl |= IXGBE_SRRCTL_DROP_EN;
4677 } else {
4678 srrctl &= ~IXGBE_SRRCTL_DROP_EN;
4679 }
4680
4681 IXGBE_WRITE_REG(hw, IXGBE_SRRCTL(i), srrctl);
4682
4683 /* Setup the HW Rx Head and Tail Descriptor Pointers */
4684 IXGBE_WRITE_REG(hw, IXGBE_RDH(i), 0);
4685 IXGBE_WRITE_REG(hw, IXGBE_RDT(i), 0);
4686
4687 /* Set the processing limit */
4688 rxr->process_limit = ixgbe_rx_process_limit;
4689 }
4690
4691 if (adapter->hw.mac.type != ixgbe_mac_82598EB) {
4692 u32 psrtype = IXGBE_PSRTYPE_TCPHDR |
4693 IXGBE_PSRTYPE_UDPHDR |
4694 IXGBE_PSRTYPE_IPV4HDR |
4695 IXGBE_PSRTYPE_IPV6HDR;
4696 IXGBE_WRITE_REG(hw, IXGBE_PSRTYPE(0), psrtype);
4697 }
4698
4699 rxcsum = IXGBE_READ_REG(hw, IXGBE_RXCSUM);
4700
4701 ixgbe_initialise_rss_mapping(adapter);
4702
4703 if (adapter->num_queues > 1) {
4704 /* RSS and RX IPP Checksum are mutually exclusive */
4705 rxcsum |= IXGBE_RXCSUM_PCSD;
4706 }
4707
4708 if (ifp->if_capenable & IFCAP_RXCSUM)
4709 rxcsum |= IXGBE_RXCSUM_PCSD;
4710
4711 if (!(rxcsum & IXGBE_RXCSUM_PCSD))
4712 rxcsum |= IXGBE_RXCSUM_IPPCSE;
4713
4714 IXGBE_WRITE_REG(hw, IXGBE_RXCSUM, rxcsum);
4715
4716 return;
4717}
4718
4719/*********************************************************************
4720 *
4721 * Free all receive rings.
4722 *
4723 **********************************************************************/
4724static void
4725ixgbe_free_receive_structures(struct adapter *adapter)
4726{
4727 struct rx_ring *rxr = adapter->rx_rings;
4728
4729 INIT_DEBUGOUT("ixgbe_free_receive_structures: begin");
4730
4731 for (int i = 0; i < adapter->num_queues; i++, rxr++) {
4732#ifdef LRO
4733 struct lro_ctrl *lro = &rxr->lro;
4734#endif /* LRO */
4735 ixgbe_free_receive_buffers(rxr);
4736#ifdef LRO
4737 /* Free LRO memory */
4738 tcp_lro_free(lro);
4739#endif /* LRO */
4740 /* Free the ring memory as well */
4741 ixgbe_dma_free(adapter, &rxr->rxdma);
4742 IXGBE_RX_LOCK_DESTROY(rxr);
4743 }
4744
4745 free(adapter->rx_rings, M_DEVBUF);
4746}
4747
4748
4749/*********************************************************************
4750 *
4751 * Free receive ring data structures
4752 *
4753 **********************************************************************/
4754static void
4755ixgbe_free_receive_buffers(struct rx_ring *rxr)
4756{
4757 struct adapter *adapter = rxr->adapter;
4758 struct ixgbe_rx_buf *rxbuf;
4759
4760 INIT_DEBUGOUT("ixgbe_free_receive_buffers: begin");
4761
4762 /* Cleanup any existing buffers */
4763 if (rxr->rx_buffers != NULL) {
4764 for (int i = 0; i < adapter->num_rx_desc; i++) {
4765 rxbuf = &rxr->rx_buffers[i];
4766 if (rxbuf->buf != NULL) {
4767 bus_dmamap_sync(rxr->ptag->dt_dmat,
4768 rxbuf->pmap, 0, rxbuf->buf->m_pkthdr.len,
4769 BUS_DMASYNC_POSTREAD);
4770 ixgbe_dmamap_unload(rxr->ptag, rxbuf->pmap);
4771 rxbuf->buf->m_flags |= M_PKTHDR;
4772 m_freem(rxbuf->buf);
4773 }
4774 rxbuf->buf = NULL;
4775 if (rxbuf->pmap != NULL) {
4776 ixgbe_dmamap_destroy(rxr->ptag, rxbuf->pmap);
4777 rxbuf->pmap = NULL;
4778 }
4779 }
4780 if (rxr->rx_buffers != NULL) {
4781 free(rxr->rx_buffers, M_DEVBUF);
4782 rxr->rx_buffers = NULL;
4783 }
4784 }
4785
4786 if (rxr->ptag != NULL) {
4787 ixgbe_dma_tag_destroy(rxr->ptag);
4788 rxr->ptag = NULL;
4789 }
4790
4791 return;
4792}
4793
4794static __inline void
4795ixgbe_rx_input(struct rx_ring *rxr, struct ifnet *ifp, struct mbuf *m, u32 ptype)
4796{
4797 int s;
4798
4799#ifdef LRO
4800 struct adapter *adapter = ifp->if_softc;
4801 struct ethercom *ec = &adapter->osdep.ec;
4802
4803 /*
4804 * ATM LRO is only for IP/TCP packets and TCP checksum of the packet
4805 * should be computed by hardware. Also it should not have VLAN tag in
4806 * ethernet header. In case of IPv6 we do not yet support ext. hdrs.
4807 */
4808 if (rxr->lro_enabled &&
4809 (ec->ec_capenable & ETHERCAP_VLAN_HWTAGGING) != 0 &&
4810 (ptype & IXGBE_RXDADV_PKTTYPE_ETQF) == 0 &&
4811 ((ptype & (IXGBE_RXDADV_PKTTYPE_IPV4 | IXGBE_RXDADV_PKTTYPE_TCP)) ==
4812 (IXGBE_RXDADV_PKTTYPE_IPV4 | IXGBE_RXDADV_PKTTYPE_TCP) ||
4813 (ptype & (IXGBE_RXDADV_PKTTYPE_IPV6 | IXGBE_RXDADV_PKTTYPE_TCP)) ==
4814 (IXGBE_RXDADV_PKTTYPE_IPV6 | IXGBE_RXDADV_PKTTYPE_TCP)) &&
4815 (m->m_pkthdr.csum_flags & (CSUM_DATA_VALID | CSUM_PSEUDO_HDR)) ==
4816 (CSUM_DATA_VALID | CSUM_PSEUDO_HDR)) {
4817 /*
4818 * Send to the stack if:
4819 ** - LRO not enabled, or
4820 ** - no LRO resources, or
4821 ** - lro enqueue fails
4822 */
4823 if (rxr->lro.lro_cnt != 0)
4824 if (tcp_lro_rx(&rxr->lro, m, 0) == 0)
4825 return;
4826 }
4827#endif /* LRO */
4828
4829 IXGBE_RX_UNLOCK(rxr);
4830
4831 s = splnet();
4832 /* Pass this up to any BPF listeners. */
4833 bpf_mtap(ifp, m);
4834 if_input(ifp, m);
4835 splx(s);
4836
4837 IXGBE_RX_LOCK(rxr);
4838}
4839
4840static __inline void
4841ixgbe_rx_discard(struct rx_ring *rxr, int i)
4842{
4843 struct ixgbe_rx_buf *rbuf;
4844
4845 rbuf = &rxr->rx_buffers[i];
4846
4847
4848 /*
4849 ** With advanced descriptors the writeback
4850 ** clobbers the buffer addrs, so its easier
4851 ** to just free the existing mbufs and take
4852 ** the normal refresh path to get new buffers
4853 ** and mapping.
4854 */
4855
4856 if (rbuf->buf != NULL) {/* Partial chain ? */
4857 rbuf->fmp->m_flags |= M_PKTHDR;
4858 m_freem(rbuf->fmp);
4859 rbuf->fmp = NULL;
4860 rbuf->buf = NULL; /* rbuf->buf is part of fmp's chain */
4861 } else if (rbuf->buf) {
4862 m_free(rbuf->buf);
4863 rbuf->buf = NULL;
4864 }
4865
4866 rbuf->flags = 0;
4867
4868 return;
4869}
4870
4871
4872/*********************************************************************
4873 *
4874 * This routine executes in interrupt context. It replenishes
4875 * the mbufs in the descriptor and sends data which has been
4876 * dma'ed into host memory to upper layer.
4877 *
4878 * We loop at most count times if count is > 0, or until done if
4879 * count < 0.
4880 *
4881 * Return TRUE for more work, FALSE for all clean.
4882 *********************************************************************/
4883static bool
4884ixgbe_rxeof(struct ix_queue *que)
4885{
4886 struct adapter *adapter = que->adapter;
4887 struct rx_ring *rxr = que->rxr;
4888 struct ifnet *ifp = adapter->ifp;
4889#ifdef LRO
4890 struct lro_ctrl *lro = &rxr->lro;
4891 struct lro_entry *queued;
4892#endif /* LRO */
4893 int i, nextp, processed = 0;
4894 u32 staterr = 0;
4895 u16 count = rxr->process_limit;
4896 union ixgbe_adv_rx_desc *cur;
4897 struct ixgbe_rx_buf *rbuf, *nbuf;
4898#ifdef RSS
4899 u16 pkt_info;
4900#endif
4901
4902 IXGBE_RX_LOCK(rxr);
4903
4904#ifdef DEV_NETMAP
4905 /* Same as the txeof routine: wakeup clients on intr. */
4906 if (netmap_rx_irq(ifp, rxr->me, &processed)) {
4907 IXGBE_RX_UNLOCK(rxr);
4908 return (FALSE);
4909 }
4910#endif /* DEV_NETMAP */
4911
4912 for (i = rxr->next_to_check; count != 0;) {
4913 struct mbuf *sendmp, *mp;
4914 u32 rsc, ptype;
4915 u16 len;
4916 u16 vtag = 0;
4917 bool eop;
4918
4919 /* Sync the ring. */
4920 ixgbe_dmamap_sync(rxr->rxdma.dma_tag, rxr->rxdma.dma_map,
4921 BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE);
4922
4923 cur = &rxr->rx_base[i];
4924 staterr = le32toh(cur->wb.upper.status_error);
4925#ifdef RSS
4926 pkt_info = le16toh(cur->wb.lower.lo_dword.hs_rss.pkt_info);
4927#endif
4928
4929 if ((staterr & IXGBE_RXD_STAT_DD) == 0)
4930 break;
4931 if ((ifp->if_flags & IFF_RUNNING) == 0)
4932 break;
4933
4934 count--;
4935 sendmp = NULL;
4936 nbuf = NULL;
4937 rsc = 0;
4938 cur->wb.upper.status_error = 0;
4939 rbuf = &rxr->rx_buffers[i];
4940 mp = rbuf->buf;
4941
4942 len = le16toh(cur->wb.upper.length);
4943 ptype = le32toh(cur->wb.lower.lo_dword.data) &
4944 IXGBE_RXDADV_PKTTYPE_MASK;
4945 eop = ((staterr & IXGBE_RXD_STAT_EOP) != 0);
4946
4947 /* Make sure bad packets are discarded */
4948 if (eop && (staterr & IXGBE_RXDADV_ERR_FRAME_ERR_MASK) != 0) {
4949 rxr->rx_discarded.ev_count++;
4950 ixgbe_rx_discard(rxr, i);
4951 goto next_desc;
4952 }
4953
4954 /*
4955 ** On 82599 which supports a hardware
4956 ** LRO (called HW RSC), packets need
4957 ** not be fragmented across sequential
4958 ** descriptors, rather the next descriptor
4959 ** is indicated in bits of the descriptor.
4960 ** This also means that we might proceses
4961 ** more than one packet at a time, something
4962 ** that has never been true before, it
4963 ** required eliminating global chain pointers
4964 ** in favor of what we are doing here. -jfv
4965 */
4966 if (!eop) {
4967 /*
4968 ** Figure out the next descriptor
4969 ** of this frame.
4970 */
4971 if (rxr->hw_rsc == TRUE) {
4972 rsc = ixgbe_rsc_count(cur);
4973 rxr->rsc_num += (rsc - 1);
4974 }
4975 if (rsc) { /* Get hardware index */
4976 nextp = ((staterr &
4977 IXGBE_RXDADV_NEXTP_MASK) >>
4978 IXGBE_RXDADV_NEXTP_SHIFT);
4979 } else { /* Just sequential */
4980 nextp = i + 1;
4981 if (nextp == adapter->num_rx_desc)
4982 nextp = 0;
4983 }
4984 nbuf = &rxr->rx_buffers[nextp];
4985 prefetch(nbuf);
4986 }
4987 /*
4988 ** Rather than using the fmp/lmp global pointers
4989 ** we now keep the head of a packet chain in the
4990 ** buffer struct and pass this along from one
4991 ** descriptor to the next, until we get EOP.
4992 */
4993 mp->m_len = len;
4994 /*
4995 ** See if there is a stored head
4996 ** that determines what we are
4997 */
4998 sendmp = rbuf->fmp;
4999 if (sendmp != NULL) { /* secondary frag */
5000 rbuf->buf = rbuf->fmp = NULL;
5001 mp->m_flags &= ~M_PKTHDR;
5002 sendmp->m_pkthdr.len += mp->m_len;
5003 } else {
5004 /*
5005 * Optimize. This might be a small packet,
5006 * maybe just a TCP ACK. Do a fast copy that
5007 * is cache aligned into a new mbuf, and
5008 * leave the old mbuf+cluster for re-use.
5009 */
5010 if (eop && len <= IXGBE_RX_COPY_LEN) {
5011 sendmp = m_gethdr(M_NOWAIT, MT_DATA);
5012 if (sendmp != NULL) {
5013 sendmp->m_data +=
5014 IXGBE_RX_COPY_ALIGN;
5015 ixgbe_bcopy(mp->m_data,
5016 sendmp->m_data, len);
5017 sendmp->m_len = len;
5018 rxr->rx_copies.ev_count++;
5019 rbuf->flags |= IXGBE_RX_COPY;
5020 }
5021 }
5022 if (sendmp == NULL) {
5023 rbuf->buf = rbuf->fmp = NULL;
5024 sendmp = mp;
5025 }
5026
5027 /* first desc of a non-ps chain */
5028 sendmp->m_flags |= M_PKTHDR;
5029 sendmp->m_pkthdr.len = mp->m_len;
5030 }
5031 ++processed;
5032
5033 /* Pass the head pointer on */
5034 if (eop == 0) {
5035 nbuf->fmp = sendmp;
5036 sendmp = NULL;
5037 mp->m_next = nbuf->buf;
5038 } else { /* Sending this frame */
5039 m_set_rcvif(sendmp, ifp);
5040 ifp->if_ipackets++;
5041 rxr->rx_packets.ev_count++;
5042 /* capture data for AIM */
5043 rxr->bytes += sendmp->m_pkthdr.len;
5044 rxr->rx_bytes.ev_count += sendmp->m_pkthdr.len;
5045 /* Process vlan info */
5046 if ((rxr->vtag_strip) &&
5047 (staterr & IXGBE_RXD_STAT_VP))
5048 vtag = le16toh(cur->wb.upper.vlan);
5049 if (vtag) {
5050 VLAN_INPUT_TAG(ifp, sendmp, vtag,
5051 printf("%s: could not apply VLAN "
5052 "tag", __func__));
5053 }
5054 if ((ifp->if_capenable & IFCAP_RXCSUM) != 0) {
5055 ixgbe_rx_checksum(staterr, sendmp, ptype,
5056 &adapter->stats);
5057 }
5058#if __FreeBSD_version >= 800000
5059#ifdef RSS
5060 sendmp->m_pkthdr.flowid =
5061 le32toh(cur->wb.lower.hi_dword.rss);
5062 switch (pkt_info & IXGBE_RXDADV_RSSTYPE_MASK) {
5063 case IXGBE_RXDADV_RSSTYPE_IPV4_TCP:
5064 M_HASHTYPE_SET(sendmp, M_HASHTYPE_RSS_TCP_IPV4);
5065 break;
5066 case IXGBE_RXDADV_RSSTYPE_IPV4:
5067 M_HASHTYPE_SET(sendmp, M_HASHTYPE_RSS_IPV4);
5068 break;
5069 case IXGBE_RXDADV_RSSTYPE_IPV6_TCP:
5070 M_HASHTYPE_SET(sendmp, M_HASHTYPE_RSS_TCP_IPV6);
5071 break;
5072 case IXGBE_RXDADV_RSSTYPE_IPV6_EX:
5073 M_HASHTYPE_SET(sendmp, M_HASHTYPE_RSS_IPV6_EX);
5074 break;
5075 case IXGBE_RXDADV_RSSTYPE_IPV6:
5076 M_HASHTYPE_SET(sendmp, M_HASHTYPE_RSS_IPV6);
5077 break;
5078 case IXGBE_RXDADV_RSSTYPE_IPV6_TCP_EX:
5079 M_HASHTYPE_SET(sendmp, M_HASHTYPE_RSS_TCP_IPV6_EX);
5080 break;
5081 case IXGBE_RXDADV_RSSTYPE_IPV4_UDP:
5082 M_HASHTYPE_SET(sendmp, M_HASHTYPE_RSS_UDP_IPV4);
5083 break;
5084 case IXGBE_RXDADV_RSSTYPE_IPV6_UDP:
5085 M_HASHTYPE_SET(sendmp, M_HASHTYPE_RSS_UDP_IPV6);
5086 break;
5087 case IXGBE_RXDADV_RSSTYPE_IPV6_UDP_EX:
5088 M_HASHTYPE_SET(sendmp, M_HASHTYPE_RSS_UDP_IPV6_EX);
5089 break;
5090 default:
5091 /* XXX fallthrough */
5092 M_HASHTYPE_SET(sendmp, M_HASHTYPE_OPAQUE);
5093 break;
5094 }
5095#else /* RSS */
5096 sendmp->m_pkthdr.flowid = que->msix;
5097 M_HASHTYPE_SET(sendmp, M_HASHTYPE_OPAQUE);
5098#endif /* RSS */
5099#endif /* FreeBSD_version */
5100 }
5101next_desc:
5102 ixgbe_dmamap_sync(rxr->rxdma.dma_tag, rxr->rxdma.dma_map,
5103 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
5104
5105 /* Advance our pointers to the next descriptor. */
5106 if (++i == rxr->num_desc)
5107 i = 0;
5108
5109 /* Now send to the stack or do LRO */
5110 if (sendmp != NULL) {
5111 rxr->next_to_check = i;
5112 ixgbe_rx_input(rxr, ifp, sendmp, ptype);
5113 i = rxr->next_to_check;
5114 }
5115
5116 /* Every 8 descriptors we go to refresh mbufs */
5117 if (processed == 8) {
5118 ixgbe_refresh_mbufs(rxr, i);
5119 processed = 0;
5120 }
5121 }
5122
5123 /* Refresh any remaining buf structs */
5124 if (ixgbe_rx_unrefreshed(rxr))
5125 ixgbe_refresh_mbufs(rxr, i);
5126
5127 rxr->next_to_check = i;
5128
5129#ifdef LRO
5130 /*
5131 * Flush any outstanding LRO work
5132 */
5133 while ((queued = SLIST_FIRST(&lro->lro_active)) != NULL) {
5134 SLIST_REMOVE_HEAD(&lro->lro_active, next);
5135 tcp_lro_flush(lro, queued);
5136 }
5137#endif /* LRO */
5138
5139 IXGBE_RX_UNLOCK(rxr);
5140
5141 /*
5142 ** Still have cleaning to do?
5143 */
5144 if ((staterr & IXGBE_RXD_STAT_DD) != 0)
5145 return true;
5146 else
5147 return false;
5148}
5149
5150
5151/*********************************************************************
5152 *
5153 * Verify that the hardware indicated that the checksum is valid.
5154 * Inform the stack about the status of checksum so that stack
5155 * doesn't spend time verifying the checksum.
5156 *
5157 *********************************************************************/
5158static void
5159ixgbe_rx_checksum(u32 staterr, struct mbuf * mp, u32 ptype,
5160 struct ixgbe_hw_stats *stats)
5161{
5162 u16 status = (u16) staterr;
5163 u8 errors = (u8) (staterr >> 24);
5164#if 0
5165 bool sctp = FALSE;
5166
5167 if ((ptype & IXGBE_RXDADV_PKTTYPE_ETQF) == 0 &&
5168 (ptype & IXGBE_RXDADV_PKTTYPE_SCTP) != 0)
5169 sctp = TRUE;
5170#endif
5171
5172 if (status & IXGBE_RXD_STAT_IPCS) {
5173 stats->ipcs.ev_count++;
5174 if (!(errors & IXGBE_RXD_ERR_IPE)) {
5175 /* IP Checksum Good */
5176 mp->m_pkthdr.csum_flags = M_CSUM_IPv4;
5177
5178 } else {
5179 stats->ipcs_bad.ev_count++;
5180 mp->m_pkthdr.csum_flags = M_CSUM_IPv4|M_CSUM_IPv4_BAD;
5181 }
5182 }
5183 if (status & IXGBE_RXD_STAT_L4CS) {
5184 stats->l4cs.ev_count++;
5185 int type = M_CSUM_TCPv4|M_CSUM_TCPv6|M_CSUM_UDPv4|M_CSUM_UDPv6;
5186 if (!(errors & IXGBE_RXD_ERR_TCPE)) {
5187 mp->m_pkthdr.csum_flags |= type;
5188 } else {
5189 stats->l4cs_bad.ev_count++;
5190 mp->m_pkthdr.csum_flags |= type | M_CSUM_TCP_UDP_BAD;
5191 }
5192 }
5193 return;
5194}
5195
5196
5197#if 0 /* XXX Badly need to overhaul vlan(4) on NetBSD. */
5198/*
5199** This routine is run via an vlan config EVENT,
5200** it enables us to use the HW Filter table since
5201** we can get the vlan id. This just creates the
5202** entry in the soft version of the VFTA, init will
5203** repopulate the real table.
5204*/
5205static void
5206ixgbe_register_vlan(void *arg, struct ifnet *ifp, u16 vtag)
5207{
5208 struct adapter *adapter = ifp->if_softc;
5209 u16 index, bit;
5210
5211 if (ifp->if_softc != arg) /* Not our event */
5212 return;
5213
5214 if ((vtag == 0) || (vtag > 4095)) /* Invalid */
5215 return;
5216
5217 IXGBE_CORE_LOCK(adapter);
5218 index = (vtag >> 5) & 0x7F;
5219 bit = vtag & 0x1F;
5220 adapter->shadow_vfta[index] |= (1 << bit);
5221 ixgbe_setup_vlan_hw_support(adapter);
5222 IXGBE_CORE_UNLOCK(adapter);
5223}
5224
5225/*
5226** This routine is run via an vlan
5227** unconfig EVENT, remove our entry
5228** in the soft vfta.
5229*/
5230static void
5231ixgbe_unregister_vlan(void *arg, struct ifnet *ifp, u16 vtag)
5232{
5233 struct adapter *adapter = ifp->if_softc;
5234 u16 index, bit;
5235
5236 if (ifp->if_softc != arg)
5237 return;
5238
5239 if ((vtag == 0) || (vtag > 4095)) /* Invalid */
5240 return;
5241
5242 IXGBE_CORE_LOCK(adapter);
5243 index = (vtag >> 5) & 0x7F;
5244 bit = vtag & 0x1F;
5245 adapter->shadow_vfta[index] &= ~(1 << bit);
5246 /* Re-init to load the changes */
5247 ixgbe_setup_vlan_hw_support(adapter);
5248 IXGBE_CORE_UNLOCK(adapter);
5249}
5250#endif
5251
5252static void
5253ixgbe_setup_vlan_hw_support(struct adapter *adapter)
5254{
5255 struct ethercom *ec = &adapter->osdep.ec;
5256 struct ixgbe_hw *hw = &adapter->hw;
5257 struct rx_ring *rxr;
5258 u32 ctrl;
5259
5260
5261 /*
5262 ** We get here thru init_locked, meaning
5263 ** a soft reset, this has already cleared
5264 ** the VFTA and other state, so if there
5265 ** have been no vlan's registered do nothing.
5266 */
5267 if (!VLAN_ATTACHED(&adapter->osdep.ec)) {
5268 return;
5269 }
5270
5271 /* Setup the queues for vlans */
5272 for (int i = 0; i < adapter->num_queues; i++) {
5273 rxr = &adapter->rx_rings[i];
5274 /* On 82599 the VLAN enable is per/queue in RXDCTL */
5275 if (hw->mac.type != ixgbe_mac_82598EB) {
5276 ctrl = IXGBE_READ_REG(hw, IXGBE_RXDCTL(i));
5277 ctrl |= IXGBE_RXDCTL_VME;
5278 IXGBE_WRITE_REG(hw, IXGBE_RXDCTL(i), ctrl);
5279 }
5280 rxr->vtag_strip = TRUE;
5281 }
5282
5283 if ((ec->ec_capenable & ETHERCAP_VLAN_HWFILTER) == 0)
5284 return;
5285 /*
5286 ** A soft reset zero's out the VFTA, so
5287 ** we need to repopulate it now.
5288 */
5289 for (int i = 0; i < IXGBE_VFTA_SIZE; i++)
5290 if (adapter->shadow_vfta[i] != 0)
5291 IXGBE_WRITE_REG(hw, IXGBE_VFTA(i),
5292 adapter->shadow_vfta[i]);
5293
5294 ctrl = IXGBE_READ_REG(hw, IXGBE_VLNCTRL);
5295 /* Enable the Filter Table if enabled */
5296 if (ec->ec_capenable & ETHERCAP_VLAN_HWFILTER) {
5297 ctrl &= ~IXGBE_VLNCTRL_CFIEN;
5298 ctrl |= IXGBE_VLNCTRL_VFE;
5299 }
5300 if (hw->mac.type == ixgbe_mac_82598EB)
5301 ctrl |= IXGBE_VLNCTRL_VME;
5302 IXGBE_WRITE_REG(hw, IXGBE_VLNCTRL, ctrl);
5303}
5304
5305static void
5306ixgbe_enable_intr(struct adapter *adapter)
5307{
5308 struct ixgbe_hw *hw = &adapter->hw;
5309 struct ix_queue *que = adapter->queues;
5310 u32 mask, fwsm;
5311
5312 mask = (IXGBE_EIMS_ENABLE_MASK & ~IXGBE_EIMS_RTX_QUEUE);
5313 /* Enable Fan Failure detection */
5314 if (hw->device_id == IXGBE_DEV_ID_82598AT)
5315 mask |= IXGBE_EIMS_GPI_SDP1;
5316
5317 switch (adapter->hw.mac.type) {
5318 case ixgbe_mac_82599EB:
5319 mask |= IXGBE_EIMS_ECC;
5320 mask |= IXGBE_EIMS_GPI_SDP0;
5321 mask |= IXGBE_EIMS_GPI_SDP1;
5322 mask |= IXGBE_EIMS_GPI_SDP2;
5323#ifdef IXGBE_FDIR
5324 mask |= IXGBE_EIMS_FLOW_DIR;
5325#endif
5326 break;
5327 case ixgbe_mac_X540:
5328 mask |= IXGBE_EIMS_ECC;
5329 /* Detect if Thermal Sensor is enabled */
5330 fwsm = IXGBE_READ_REG(hw, IXGBE_FWSM);
5331 if (fwsm & IXGBE_FWSM_TS_ENABLED)
5332 mask |= IXGBE_EIMS_TS;
5333#ifdef IXGBE_FDIR
5334 mask |= IXGBE_EIMS_FLOW_DIR;
5335#endif
5336 /* falls through */
5337 default:
5338 break;
5339 }
5340
5341 IXGBE_WRITE_REG(hw, IXGBE_EIMS, mask);
5342
5343 /* With RSS we use auto clear */
5344 if (adapter->msix_mem) {
5345 mask = IXGBE_EIMS_ENABLE_MASK;
5346 /* Don't autoclear Link */
5347 mask &= ~IXGBE_EIMS_OTHER;
5348 mask &= ~IXGBE_EIMS_LSC;
5349 IXGBE_WRITE_REG(hw, IXGBE_EIAC, mask);
5350 }
5351
5352 /*
5353 ** Now enable all queues, this is done separately to
5354 ** allow for handling the extended (beyond 32) MSIX
5355 ** vectors that can be used by 82599
5356 */
5357 for (int i = 0; i < adapter->num_queues; i++, que++)
5358 ixgbe_enable_queue(adapter, que->msix);
5359
5360 IXGBE_WRITE_FLUSH(hw);
5361
5362 return;
5363}
5364
5365static void
5366ixgbe_disable_intr(struct adapter *adapter)
5367{
5368 if (adapter->msix_mem)
5369 IXGBE_WRITE_REG(&adapter->hw, IXGBE_EIAC, 0);
5370 if (adapter->hw.mac.type == ixgbe_mac_82598EB) {
5371 IXGBE_WRITE_REG(&adapter->hw, IXGBE_EIMC, ~0);
5372 } else {
5373 IXGBE_WRITE_REG(&adapter->hw, IXGBE_EIMC, 0xFFFF0000);
5374 IXGBE_WRITE_REG(&adapter->hw, IXGBE_EIMC_EX(0), ~0);
5375 IXGBE_WRITE_REG(&adapter->hw, IXGBE_EIMC_EX(1), ~0);
5376 }
5377 IXGBE_WRITE_FLUSH(&adapter->hw);
5378 return;
5379}
5380
5381u16
5382ixgbe_read_pci_cfg(struct ixgbe_hw *hw, u32 reg)
5383{
5384 switch (reg % 4) {
5385 case 0:
5386 return pci_conf_read(hw->back->pc, hw->back->tag, reg) &
5387 __BITS(15, 0);
5388 case 2:
5389 return __SHIFTOUT(pci_conf_read(hw->back->pc, hw->back->tag,
5390 reg - 2), __BITS(31, 16));
5391 default:
5392 panic("%s: invalid register (%" PRIx32, __func__, reg);
5393 break;
5394 }
5395}
5396
5397void
5398ixgbe_write_pci_cfg(struct ixgbe_hw *hw, u32 reg, u16 value)
5399{
5400 pcireg_t old;
5401
5402 switch (reg % 4) {
5403 case 0:
5404 old = pci_conf_read(hw->back->pc, hw->back->tag, reg) &
5405 __BITS(31, 16);
5406 pci_conf_write(hw->back->pc, hw->back->tag, reg, value | old);
5407 break;
5408 case 2:
5409 old = pci_conf_read(hw->back->pc, hw->back->tag, reg - 2) &
5410 __BITS(15, 0);
5411 pci_conf_write(hw->back->pc, hw->back->tag, reg - 2,
5412 __SHIFTIN(value, __BITS(31, 16)) | old);
5413 break;
5414 default:
5415 panic("%s: invalid register (%" PRIx32, __func__, reg);
5416 break;
5417 }
5418
5419 return;
5420}
5421
5422/*
5423** Get the width and transaction speed of
5424** the slot this adapter is plugged into.
5425*/
5426static void
5427ixgbe_get_slot_info(struct ixgbe_hw *hw)
5428{
5429 device_t dev = ((struct ixgbe_osdep *)hw->back)->dev;
5430 struct ixgbe_mac_info *mac = &hw->mac;
5431 u16 link;
5432
5433 /* For most devices simply call the shared code routine */
5434 if (hw->device_id != IXGBE_DEV_ID_82599_SFP_SF_QP) {
5435 ixgbe_get_bus_info(hw);
5436 goto display;
5437 }
5438
5439 /*
5440 ** For the Quad port adapter we need to parse back
5441 ** up the PCI tree to find the speed of the expansion
5442 ** slot into which this adapter is plugged. A bit more work.
5443 */
5444 dev = device_parent(device_parent(dev));
5445#ifdef IXGBE_DEBUG
5446 device_printf(dev, "parent pcib = %x,%x,%x\n",
5447 pci_get_bus(dev), pci_get_slot(dev), pci_get_function(dev));
5448#endif
5449 dev = device_parent(device_parent(dev));
5450#ifdef IXGBE_DEBUG
5451 device_printf(dev, "slot pcib = %x,%x,%x\n",
5452 pci_get_bus(dev), pci_get_slot(dev), pci_get_function(dev));
5453#endif
5454 /* Now get the PCI Express Capabilities offset */
5455 /* ...and read the Link Status Register */
5456 link = IXGBE_READ_PCIE_WORD(hw, IXGBE_PCI_LINK_STATUS);
5457 switch (link & IXGBE_PCI_LINK_WIDTH) {
5458 case IXGBE_PCI_LINK_WIDTH_1:
5459 hw->bus.width = ixgbe_bus_width_pcie_x1;
5460 break;
5461 case IXGBE_PCI_LINK_WIDTH_2:
5462 hw->bus.width = ixgbe_bus_width_pcie_x2;
5463 break;
5464 case IXGBE_PCI_LINK_WIDTH_4:
5465 hw->bus.width = ixgbe_bus_width_pcie_x4;
5466 break;
5467 case IXGBE_PCI_LINK_WIDTH_8:
5468 hw->bus.width = ixgbe_bus_width_pcie_x8;
5469 break;
5470 default:
5471 hw->bus.width = ixgbe_bus_width_unknown;
5472 break;
5473 }
5474
5475 switch (link & IXGBE_PCI_LINK_SPEED) {
5476 case IXGBE_PCI_LINK_SPEED_2500:
5477 hw->bus.speed = ixgbe_bus_speed_2500;
5478 break;
5479 case IXGBE_PCI_LINK_SPEED_5000:
5480 hw->bus.speed = ixgbe_bus_speed_5000;
5481 break;
5482 case IXGBE_PCI_LINK_SPEED_8000:
5483 hw->bus.speed = ixgbe_bus_speed_8000;
5484 break;
5485 default:
5486 hw->bus.speed = ixgbe_bus_speed_unknown;
5487 break;
5488 }
5489
5490 mac->ops.set_lan_id(hw);
5491
5492display:
5493 device_printf(dev,"PCI Express Bus: Speed %s %s\n",
5494 ((hw->bus.speed == ixgbe_bus_speed_8000) ? "8.0GT/s":
5495 (hw->bus.speed == ixgbe_bus_speed_5000) ? "5.0GT/s":
5496 (hw->bus.speed == ixgbe_bus_speed_2500) ? "2.5GT/s":"Unknown"),
5497 (hw->bus.width == ixgbe_bus_width_pcie_x8) ? "Width x8" :
5498 (hw->bus.width == ixgbe_bus_width_pcie_x4) ? "Width x4" :
5499 (hw->bus.width == ixgbe_bus_width_pcie_x1) ? "Width x1" :
5500 ("Unknown"));
5501
5502 if ((hw->device_id != IXGBE_DEV_ID_82599_SFP_SF_QP) &&
5503 ((hw->bus.width <= ixgbe_bus_width_pcie_x4) &&
5504 (hw->bus.speed == ixgbe_bus_speed_2500))) {
5505 device_printf(dev, "PCI-Express bandwidth available"
5506 " for this card\n is not sufficient for"
5507 " optimal performance.\n");
5508 device_printf(dev, "For optimal performance a x8 "
5509 "PCIE, or x4 PCIE Gen2 slot is required.\n");
5510 }
5511 if ((hw->device_id == IXGBE_DEV_ID_82599_SFP_SF_QP) &&
5512 ((hw->bus.width <= ixgbe_bus_width_pcie_x8) &&
5513 (hw->bus.speed < ixgbe_bus_speed_8000))) {
5514 device_printf(dev, "PCI-Express bandwidth available"
5515 " for this card\n is not sufficient for"
5516 " optimal performance.\n");
5517 device_printf(dev, "For optimal performance a x8 "
5518 "PCIE Gen3 slot is required.\n");
5519 }
5520
5521 return;
5522}
5523
5524
5525/*
5526** Setup the correct IVAR register for a particular MSIX interrupt
5527** (yes this is all very magic and confusing :)
5528** - entry is the register array entry
5529** - vector is the MSIX vector for this queue
5530** - type is RX/TX/MISC
5531*/
5532static void
5533ixgbe_set_ivar(struct adapter *adapter, u8 entry, u8 vector, s8 type)
5534{
5535 struct ixgbe_hw *hw = &adapter->hw;
5536 u32 ivar, index;
5537
5538 vector |= IXGBE_IVAR_ALLOC_VAL;
5539
5540 switch (hw->mac.type) {
5541
5542 case ixgbe_mac_82598EB:
5543 if (type == -1)
5544 entry = IXGBE_IVAR_OTHER_CAUSES_INDEX;
5545 else
5546 entry += (type * 64);
5547 index = (entry >> 2) & 0x1F;
5548 ivar = IXGBE_READ_REG(hw, IXGBE_IVAR(index));
5549 ivar &= ~(0xFF << (8 * (entry & 0x3)));
5550 ivar |= (vector << (8 * (entry & 0x3)));
5551 IXGBE_WRITE_REG(&adapter->hw, IXGBE_IVAR(index), ivar);
5552 break;
5553
5554 case ixgbe_mac_82599EB:
5555 case ixgbe_mac_X540:
5556 if (type == -1) { /* MISC IVAR */
5557 index = (entry & 1) * 8;
5558 ivar = IXGBE_READ_REG(hw, IXGBE_IVAR_MISC);
5559 ivar &= ~(0xFF << index);
5560 ivar |= (vector << index);
5561 IXGBE_WRITE_REG(hw, IXGBE_IVAR_MISC, ivar);
5562 } else { /* RX/TX IVARS */
5563 index = (16 * (entry & 1)) + (8 * type);
5564 ivar = IXGBE_READ_REG(hw, IXGBE_IVAR(entry >> 1));
5565 ivar &= ~(0xFF << index);
5566 ivar |= (vector << index);
5567 IXGBE_WRITE_REG(hw, IXGBE_IVAR(entry >> 1), ivar);
5568 }
5569
5570 default:
5571 break;
5572 }
5573}
5574
5575static void
5576ixgbe_configure_ivars(struct adapter *adapter)
5577{
5578 struct ix_queue *que = adapter->queues;
5579 u32 newitr;
5580
5581 if (ixgbe_max_interrupt_rate > 0)
5582 newitr = (4000000 / ixgbe_max_interrupt_rate) & 0x0FF8;
5583 else
5584 newitr = 0;
5585
5586 for (int i = 0; i < adapter->num_queues; i++, que++) {
5587 /* First the RX queue entry */
5588 ixgbe_set_ivar(adapter, i, que->msix, 0);
5589 /* ... and the TX */
5590 ixgbe_set_ivar(adapter, i, que->msix, 1);
5591 /* Set an Initial EITR value */
5592 IXGBE_WRITE_REG(&adapter->hw,
5593 IXGBE_EITR(que->msix), newitr);
5594 }
5595
5596 /* For the Link interrupt */
5597 ixgbe_set_ivar(adapter, 1, adapter->linkvec, -1);
5598}
5599
5600/*
5601** ixgbe_sfp_probe - called in the local timer to
5602** determine if a port had optics inserted.
5603*/
5604static bool ixgbe_sfp_probe(struct adapter *adapter)
5605{
5606 struct ixgbe_hw *hw = &adapter->hw;
5607 device_t dev = adapter->dev;
5608 bool result = FALSE;
5609
5610 if ((hw->phy.type == ixgbe_phy_nl) &&
5611 (hw->phy.sfp_type == ixgbe_sfp_type_not_present)) {
5612 s32 ret = hw->phy.ops.identify_sfp(hw);
5613 if (ret)
5614 goto out;
5615 ret = hw->phy.ops.reset(hw);
5616 if (ret == IXGBE_ERR_SFP_NOT_SUPPORTED) {
5617 device_printf(dev,"Unsupported SFP+ module detected!");
5618 device_printf(dev, "Reload driver with supported module.\n");
5619 adapter->sfp_probe = FALSE;
5620 goto out;
5621 } else
5622 device_printf(dev,"SFP+ module detected!\n");
5623 /* We now have supported optics */
5624 adapter->sfp_probe = FALSE;
5625 /* Set the optics type so system reports correctly */
5626 ixgbe_setup_optics(adapter);
5627 result = TRUE;
5628 }
5629out:
5630 return (result);
5631}
5632
5633/*
5634** Tasklet handler for MSIX Link interrupts
5635** - do outside interrupt since it might sleep
5636*/
5637static void
5638ixgbe_handle_link(void *context)
5639{
5640 struct adapter *adapter = context;
5641
5642 if (ixgbe_check_link(&adapter->hw,
5643 &adapter->link_speed, &adapter->link_up, 0) == 0)
5644 ixgbe_update_link_status(adapter);
5645}
5646
5647/*
5648** Tasklet for handling SFP module interrupts
5649*/
5650static void
5651ixgbe_handle_mod(void *context)
5652{
5653 struct adapter *adapter = context;
5654 struct ixgbe_hw *hw = &adapter->hw;
5655 device_t dev = adapter->dev;
5656 u32 err;
5657
5658 err = hw->phy.ops.identify_sfp(hw);
5659 if (err == IXGBE_ERR_SFP_NOT_SUPPORTED) {
5660 device_printf(dev,
5661 "Unsupported SFP+ module type was detected.\n");
5662 return;
5663 }
5664 err = hw->mac.ops.setup_sfp(hw);
5665 if (err == IXGBE_ERR_SFP_NOT_SUPPORTED) {
5666 device_printf(dev,
5667 "Setup failure - unsupported SFP+ module type.\n");
5668 return;
5669 }
5670 softint_schedule(adapter->msf_si);
5671 return;
5672}
5673
5674
5675/*
5676** Tasklet for handling MSF (multispeed fiber) interrupts
5677*/
5678static void
5679ixgbe_handle_msf(void *context)
5680{
5681 struct adapter *adapter = context;
5682 struct ixgbe_hw *hw = &adapter->hw;
5683 u32 autoneg;
5684 bool negotiate;
5685
5686 autoneg = hw->phy.autoneg_advertised;
5687 if ((!autoneg) && (hw->mac.ops.get_link_capabilities))
5688 hw->mac.ops.get_link_capabilities(hw, &autoneg, &negotiate);
5689 else
5690 negotiate = 0;
5691 if (hw->mac.ops.setup_link)
5692 hw->mac.ops.setup_link(hw, autoneg, TRUE);
5693 return;
5694}
5695
5696#ifdef IXGBE_FDIR
5697/*
5698** Tasklet for reinitializing the Flow Director filter table
5699*/
5700static void
5701ixgbe_reinit_fdir(void *context)
5702{
5703 struct adapter *adapter = context;
5704 struct ifnet *ifp = adapter->ifp;
5705
5706 if (adapter->fdir_reinit != 1) /* Shouldn't happen */
5707 return;
5708 ixgbe_reinit_fdir_tables_82599(&adapter->hw);
5709 adapter->fdir_reinit = 0;
5710 /* re-enable flow director interrupts */
5711 IXGBE_WRITE_REG(&adapter->hw, IXGBE_EIMS, IXGBE_EIMS_FLOW_DIR);
5712 /* Restart the interface */
5713 ifp->if_flags |= IFF_RUNNING;
5714 return;
5715}
5716#endif
5717
5718/**********************************************************************
5719 *
5720 * Update the board statistics counters.
5721 *
5722 **********************************************************************/
5723static void
5724ixgbe_update_stats_counters(struct adapter *adapter)
5725{
5726 struct ifnet *ifp = adapter->ifp;
5727 struct ixgbe_hw *hw = &adapter->hw;
5728 u32 missed_rx = 0, bprc, lxon, lxoff, total;
5729 u64 total_missed_rx = 0;
5730 uint64_t crcerrs, rlec;
5731
5732 crcerrs = IXGBE_READ_REG(hw, IXGBE_CRCERRS);
5733 adapter->stats.crcerrs.ev_count += crcerrs;
5734 adapter->stats.illerrc.ev_count += IXGBE_READ_REG(hw, IXGBE_ILLERRC);
5735 adapter->stats.errbc.ev_count += IXGBE_READ_REG(hw, IXGBE_ERRBC);
5736 adapter->stats.mspdc.ev_count += IXGBE_READ_REG(hw, IXGBE_MSPDC);
5737
5738 /*
5739 ** Note: these are for the 8 possible traffic classes,
5740 ** which in current implementation is unused,
5741 ** therefore only 0 should read real data.
5742 */
5743 for (int i = 0; i < __arraycount(adapter->stats.mpc); i++) {
5744 int j = i % adapter->num_queues;
5745 u32 mp;
5746 mp = IXGBE_READ_REG(hw, IXGBE_MPC(i));
5747 /* missed_rx tallies misses for the gprc workaround */
5748 missed_rx += mp;
5749 /* global total per queue */
5750 adapter->stats.mpc[j].ev_count += mp;
5751 /* Running comprehensive total for stats display */
5752 total_missed_rx += mp;
5753 if (hw->mac.type == ixgbe_mac_82598EB) {
5754 adapter->stats.rnbc[j] +=
5755 IXGBE_READ_REG(hw, IXGBE_RNBC(i));
5756 adapter->stats.qbtc[j].ev_count +=
5757 IXGBE_READ_REG(hw, IXGBE_QBTC(i));
5758 adapter->stats.qbrc[j].ev_count +=
5759 IXGBE_READ_REG(hw, IXGBE_QBRC(i));
5760 adapter->stats.pxonrxc[j].ev_count +=
5761 IXGBE_READ_REG(hw, IXGBE_PXONRXC(i));
5762 } else {
5763 adapter->stats.pxonrxc[j].ev_count +=
5764 IXGBE_READ_REG(hw, IXGBE_PXONRXCNT(i));
5765 }
5766 adapter->stats.pxontxc[j].ev_count +=
5767 IXGBE_READ_REG(hw, IXGBE_PXONTXC(i));
5768 adapter->stats.pxofftxc[j].ev_count +=
5769 IXGBE_READ_REG(hw, IXGBE_PXOFFTXC(i));
5770 adapter->stats.pxoffrxc[j].ev_count +=
5771 IXGBE_READ_REG(hw, IXGBE_PXOFFRXC(i));
5772 adapter->stats.pxon2offc[j].ev_count +=
5773 IXGBE_READ_REG(hw, IXGBE_PXON2OFFCNT(i));
5774 }
5775 for (int i = 0; i < __arraycount(adapter->stats.qprc); i++) {
5776 int j = i % adapter->num_queues;
5777 adapter->stats.qprc[j].ev_count += IXGBE_READ_REG(hw, IXGBE_QPRC(i));
5778 adapter->stats.qptc[j].ev_count += IXGBE_READ_REG(hw, IXGBE_QPTC(i));
5779 adapter->stats.qprdc[j].ev_count += IXGBE_READ_REG(hw, IXGBE_QPRDC(i));
5780 }
5781 adapter->stats.mlfc.ev_count += IXGBE_READ_REG(hw, IXGBE_MLFC);
5782 adapter->stats.mrfc.ev_count += IXGBE_READ_REG(hw, IXGBE_MRFC);
5783 rlec = IXGBE_READ_REG(hw, IXGBE_RLEC);
5784 adapter->stats.rlec.ev_count += rlec;
5785
5786 /* Hardware workaround, gprc counts missed packets */
5787 adapter->stats.gprc.ev_count += IXGBE_READ_REG(hw, IXGBE_GPRC) - missed_rx;
5788
5789 lxon = IXGBE_READ_REG(hw, IXGBE_LXONTXC);
5790 adapter->stats.lxontxc.ev_count += lxon;
5791 lxoff = IXGBE_READ_REG(hw, IXGBE_LXOFFTXC);
5792 adapter->stats.lxofftxc.ev_count += lxoff;
5793 total = lxon + lxoff;
5794
5795 if (hw->mac.type != ixgbe_mac_82598EB) {
5796 adapter->stats.gorc.ev_count += IXGBE_READ_REG(hw, IXGBE_GORCL) +
5797 ((u64)IXGBE_READ_REG(hw, IXGBE_GORCH) << 32);
5798 adapter->stats.gotc.ev_count += IXGBE_READ_REG(hw, IXGBE_GOTCL) +
5799 ((u64)IXGBE_READ_REG(hw, IXGBE_GOTCH) << 32) - total * ETHER_MIN_LEN;
5800 adapter->stats.tor.ev_count += IXGBE_READ_REG(hw, IXGBE_TORL) +
5801 ((u64)IXGBE_READ_REG(hw, IXGBE_TORH) << 32);
5802 adapter->stats.lxonrxc.ev_count += IXGBE_READ_REG(hw, IXGBE_LXONRXCNT);
5803 adapter->stats.lxoffrxc.ev_count += IXGBE_READ_REG(hw, IXGBE_LXOFFRXCNT);
5804 } else {
5805 adapter->stats.lxonrxc.ev_count += IXGBE_READ_REG(hw, IXGBE_LXONRXC);
5806 adapter->stats.lxoffrxc.ev_count += IXGBE_READ_REG(hw, IXGBE_LXOFFRXC);
5807 /* 82598 only has a counter in the high register */
5808 adapter->stats.gorc.ev_count += IXGBE_READ_REG(hw, IXGBE_GORCH);
5809 adapter->stats.gotc.ev_count += IXGBE_READ_REG(hw, IXGBE_GOTCH) - total * ETHER_MIN_LEN;
5810 adapter->stats.tor.ev_count += IXGBE_READ_REG(hw, IXGBE_TORH);
5811 }
5812
5813 /*
5814 * Workaround: mprc hardware is incorrectly counting
5815 * broadcasts, so for now we subtract those.
5816 */
5817 bprc = IXGBE_READ_REG(hw, IXGBE_BPRC);
5818 adapter->stats.bprc.ev_count += bprc;
5819 adapter->stats.mprc.ev_count += IXGBE_READ_REG(hw, IXGBE_MPRC) - ((hw->mac.type == ixgbe_mac_82598EB) ? bprc : 0);
5820
5821 adapter->stats.prc64.ev_count += IXGBE_READ_REG(hw, IXGBE_PRC64);
5822 adapter->stats.prc127.ev_count += IXGBE_READ_REG(hw, IXGBE_PRC127);
5823 adapter->stats.prc255.ev_count += IXGBE_READ_REG(hw, IXGBE_PRC255);
5824 adapter->stats.prc511.ev_count += IXGBE_READ_REG(hw, IXGBE_PRC511);
5825 adapter->stats.prc1023.ev_count += IXGBE_READ_REG(hw, IXGBE_PRC1023);
5826 adapter->stats.prc1522.ev_count += IXGBE_READ_REG(hw, IXGBE_PRC1522);
5827
5828 adapter->stats.gptc.ev_count += IXGBE_READ_REG(hw, IXGBE_GPTC) - total;
5829 adapter->stats.mptc.ev_count += IXGBE_READ_REG(hw, IXGBE_MPTC) - total;
5830 adapter->stats.ptc64.ev_count += IXGBE_READ_REG(hw, IXGBE_PTC64) - total;
5831
5832 adapter->stats.ruc.ev_count += IXGBE_READ_REG(hw, IXGBE_RUC);
5833 adapter->stats.rfc.ev_count += IXGBE_READ_REG(hw, IXGBE_RFC);
5834 adapter->stats.roc.ev_count += IXGBE_READ_REG(hw, IXGBE_ROC);
5835 adapter->stats.rjc.ev_count += IXGBE_READ_REG(hw, IXGBE_RJC);
5836 adapter->stats.mngprc.ev_count += IXGBE_READ_REG(hw, IXGBE_MNGPRC);
5837 adapter->stats.mngpdc.ev_count += IXGBE_READ_REG(hw, IXGBE_MNGPDC);
5838 adapter->stats.mngptc.ev_count += IXGBE_READ_REG(hw, IXGBE_MNGPTC);
5839 adapter->stats.tpr.ev_count += IXGBE_READ_REG(hw, IXGBE_TPR);
5840 adapter->stats.tpt.ev_count += IXGBE_READ_REG(hw, IXGBE_TPT);
5841 adapter->stats.ptc127.ev_count += IXGBE_READ_REG(hw, IXGBE_PTC127);
5842 adapter->stats.ptc255.ev_count += IXGBE_READ_REG(hw, IXGBE_PTC255);
5843 adapter->stats.ptc511.ev_count += IXGBE_READ_REG(hw, IXGBE_PTC511);
5844 adapter->stats.ptc1023.ev_count += IXGBE_READ_REG(hw, IXGBE_PTC1023);
5845 adapter->stats.ptc1522.ev_count += IXGBE_READ_REG(hw, IXGBE_PTC1522);
5846 adapter->stats.bptc.ev_count += IXGBE_READ_REG(hw, IXGBE_BPTC);
5847 adapter->stats.xec.ev_count += IXGBE_READ_REG(hw, IXGBE_XEC);
5848 adapter->stats.fccrc.ev_count += IXGBE_READ_REG(hw, IXGBE_FCCRC);
5849 adapter->stats.fclast.ev_count += IXGBE_READ_REG(hw, IXGBE_FCLAST);
5850
5851 /* Only read FCOE on 82599 */
5852 if (hw->mac.type != ixgbe_mac_82598EB) {
5853 adapter->stats.fcoerpdc.ev_count +=
5854 IXGBE_READ_REG(hw, IXGBE_FCOERPDC);
5855 adapter->stats.fcoeprc.ev_count +=
5856 IXGBE_READ_REG(hw, IXGBE_FCOEPRC);
5857 adapter->stats.fcoeptc.ev_count +=
5858 IXGBE_READ_REG(hw, IXGBE_FCOEPTC);
5859 adapter->stats.fcoedwrc.ev_count +=
5860 IXGBE_READ_REG(hw, IXGBE_FCOEDWRC);
5861 adapter->stats.fcoedwtc.ev_count +=
5862 IXGBE_READ_REG(hw, IXGBE_FCOEDWTC);
5863 }
5864
5865 /* Fill out the OS statistics structure */
5866 /*
5867 * NetBSD: Don't override if_{i|o}{packets|bytes|mcasts} with
5868 * adapter->stats counters. It's required to make ifconfig -z
5869 * (SOICZIFDATA) work.
5870 */
5871 ifp->if_collisions = 0;
5872
5873 /* Rx Errors */
5874 ifp->if_iqdrops += total_missed_rx;
5875 ifp->if_ierrors += crcerrs + rlec;
5876}
5877
5878/** ixgbe_sysctl_tdh_handler - Handler function
5879 * Retrieves the TDH value from the hardware
5880 */
5881static int
5882ixgbe_sysctl_tdh_handler(SYSCTLFN_ARGS)
5883{
5884 struct sysctlnode node;
5885 uint32_t val;
5886 struct tx_ring *txr;
5887
5888 node = *rnode;
5889 txr = (struct tx_ring *)node.sysctl_data;
5890 if (txr == NULL)
5891 return 0;
5892 val = IXGBE_READ_REG(&txr->adapter->hw, IXGBE_TDH(txr->me));
5893 node.sysctl_data = &val;
5894 return sysctl_lookup(SYSCTLFN_CALL(&node));
5895}
5896
5897/** ixgbe_sysctl_tdt_handler - Handler function
5898 * Retrieves the TDT value from the hardware
5899 */
5900static int
5901ixgbe_sysctl_tdt_handler(SYSCTLFN_ARGS)
5902{
5903 struct sysctlnode node;
5904 uint32_t val;
5905 struct tx_ring *txr;
5906
5907 node = *rnode;
5908 txr = (struct tx_ring *)node.sysctl_data;
5909 if (txr == NULL)
5910 return 0;
5911 val = IXGBE_READ_REG(&txr->adapter->hw, IXGBE_TDT(txr->me));
5912 node.sysctl_data = &val;
5913 return sysctl_lookup(SYSCTLFN_CALL(&node));
5914}
5915
5916/** ixgbe_sysctl_rdh_handler - Handler function
5917 * Retrieves the RDH value from the hardware
5918 */
5919static int
5920ixgbe_sysctl_rdh_handler(SYSCTLFN_ARGS)
5921{
5922 struct sysctlnode node;
5923 uint32_t val;
5924 struct rx_ring *rxr;
5925
5926 node = *rnode;
5927 rxr = (struct rx_ring *)node.sysctl_data;
5928 if (rxr == NULL)
5929 return 0;
5930 val = IXGBE_READ_REG(&rxr->adapter->hw, IXGBE_RDH(rxr->me));
5931 node.sysctl_data = &val;
5932 return sysctl_lookup(SYSCTLFN_CALL(&node));
5933}
5934
5935/** ixgbe_sysctl_rdt_handler - Handler function
5936 * Retrieves the RDT value from the hardware
5937 */
5938static int
5939ixgbe_sysctl_rdt_handler(SYSCTLFN_ARGS)
5940{
5941 struct sysctlnode node;
5942 uint32_t val;
5943 struct rx_ring *rxr;
5944
5945 node = *rnode;
5946 rxr = (struct rx_ring *)node.sysctl_data;
5947 if (rxr == NULL)
5948 return 0;
5949 val = IXGBE_READ_REG(&rxr->adapter->hw, IXGBE_RDT(rxr->me));
5950 node.sysctl_data = &val;
5951 return sysctl_lookup(SYSCTLFN_CALL(&node));
5952}
5953
5954static int
5955ixgbe_sysctl_interrupt_rate_handler(SYSCTLFN_ARGS)
5956{
5957 int error;
5958 struct sysctlnode node;
5959 struct ix_queue *que;
5960 uint32_t reg, usec, rate;
5961
5962 node = *rnode;
5963 que = (struct ix_queue *)node.sysctl_data;
5964 if (que == NULL)
5965 return 0;
5966 reg = IXGBE_READ_REG(&que->adapter->hw, IXGBE_EITR(que->msix));
5967 usec = ((reg & 0x0FF8) >> 3);
5968 if (usec > 0)
5969 rate = 500000 / usec;
5970 else
5971 rate = 0;
5972 node.sysctl_data = &rate;
5973 error = sysctl_lookup(SYSCTLFN_CALL(&node));
5974 if (error)
5975 return error;
5976 reg &= ~0xfff; /* default, no limitation */
5977 ixgbe_max_interrupt_rate = 0;
5978 if (rate > 0 && rate < 500000) {
5979 if (rate < 1000)
5980 rate = 1000;
5981 ixgbe_max_interrupt_rate = rate;
5982 reg |= ((4000000/rate) & 0xff8 );
5983 }
5984 IXGBE_WRITE_REG(&que->adapter->hw, IXGBE_EITR(que->msix), reg);
5985 return 0;
5986}
5987
5988const struct sysctlnode *
5989ixgbe_sysctl_instance(struct adapter *adapter)
5990{
5991 const char *dvname;
5992 struct sysctllog **log;
5993 int rc;
5994 const struct sysctlnode *rnode;
5995
5996 log = &adapter->sysctllog;
5997 dvname = device_xname(adapter->dev);
5998
5999 if ((rc = sysctl_createv(log, 0, NULL, &rnode,
6000 0, CTLTYPE_NODE, dvname,
6001 SYSCTL_DESCR("ixgbe information and settings"),
6002 NULL, 0, NULL, 0, CTL_HW, CTL_CREATE, CTL_EOL)) != 0)
6003 goto err;
6004
6005 return rnode;
6006err:
6007 printf("%s: sysctl_createv failed, rc = %d\n", __func__, rc);
6008 return NULL;
6009}
6010
6011/*
6012 * Add sysctl variables, one per statistic, to the system.
6013 */
6014static void
6015ixgbe_add_hw_stats(struct adapter *adapter)
6016{
6017 device_t dev = adapter->dev;
6018 const struct sysctlnode *rnode, *cnode;
6019 struct sysctllog **log = &adapter->sysctllog;
6020 struct tx_ring *txr = adapter->tx_rings;
6021 struct rx_ring *rxr = adapter->rx_rings;
6022 struct ixgbe_hw_stats *stats = &adapter->stats;
6023
6024 /* Driver Statistics */
6025#if 0
6026 /* These counters are not updated by the software */
6027 SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "dropped",
6028 CTLFLAG_RD, &adapter->dropped_pkts,
6029 "Driver dropped packets");
6030 SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "mbuf_header_failed",
6031 CTLFLAG_RD, &adapter->mbuf_header_failed,
6032 "???");
6033 SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "mbuf_packet_failed",
6034 CTLFLAG_RD, &adapter->mbuf_packet_failed,
6035 "???");
6036 SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "no_tx_map_avail",
6037 CTLFLAG_RD, &adapter->no_tx_map_avail,
6038 "???");
6039#endif
6040 evcnt_attach_dynamic(&adapter->handleq, EVCNT_TYPE_MISC,
6041 NULL, device_xname(dev), "Handled queue in softint");
6042 evcnt_attach_dynamic(&adapter->req, EVCNT_TYPE_MISC,
6043 NULL, device_xname(dev), "Requeued in softint");
6044 evcnt_attach_dynamic(&adapter->morerx, EVCNT_TYPE_MISC,
6045 NULL, device_xname(dev), "Interrupt handler more rx");
6046 evcnt_attach_dynamic(&adapter->moretx, EVCNT_TYPE_MISC,
6047 NULL, device_xname(dev), "Interrupt handler more tx");
6048 evcnt_attach_dynamic(&adapter->txloops, EVCNT_TYPE_MISC,
6049 NULL, device_xname(dev), "Interrupt handler tx loops");
6050 evcnt_attach_dynamic(&adapter->efbig_tx_dma_setup, EVCNT_TYPE_MISC,
6051 NULL, device_xname(dev), "Driver tx dma soft fail EFBIG");
6052 evcnt_attach_dynamic(&adapter->m_defrag_failed, EVCNT_TYPE_MISC,
6053 NULL, device_xname(dev), "m_defrag() failed");
6054 evcnt_attach_dynamic(&adapter->efbig2_tx_dma_setup, EVCNT_TYPE_MISC,
6055 NULL, device_xname(dev), "Driver tx dma hard fail EFBIG");
6056 evcnt_attach_dynamic(&adapter->einval_tx_dma_setup, EVCNT_TYPE_MISC,
6057 NULL, device_xname(dev), "Driver tx dma hard fail EINVAL");
6058 evcnt_attach_dynamic(&adapter->other_tx_dma_setup, EVCNT_TYPE_MISC,
6059 NULL, device_xname(dev), "Driver tx dma hard fail other");
6060 evcnt_attach_dynamic(&adapter->eagain_tx_dma_setup, EVCNT_TYPE_MISC,
6061 NULL, device_xname(dev), "Driver tx dma soft fail EAGAIN");
6062 evcnt_attach_dynamic(&adapter->enomem_tx_dma_setup, EVCNT_TYPE_MISC,
6063 NULL, device_xname(dev), "Driver tx dma soft fail ENOMEM");
6064 evcnt_attach_dynamic(&adapter->watchdog_events, EVCNT_TYPE_MISC,
6065 NULL, device_xname(dev), "Watchdog timeouts");
6066 evcnt_attach_dynamic(&adapter->tso_err, EVCNT_TYPE_MISC,
6067 NULL, device_xname(dev), "TSO errors");
6068 evcnt_attach_dynamic(&adapter->link_irq, EVCNT_TYPE_MISC,
6069 NULL, device_xname(dev), "Link MSIX IRQ Handled");
6070
6071 for (int i = 0; i < adapter->num_queues; i++, rxr++, txr++) {
6072 snprintf(adapter->queues[i].evnamebuf,
6073 sizeof(adapter->queues[i].evnamebuf), "%s queue%d",
6074 device_xname(dev), i);
6075 snprintf(adapter->queues[i].namebuf,
6076 sizeof(adapter->queues[i].namebuf), "queue%d", i);
6077
6078 if ((rnode = ixgbe_sysctl_instance(adapter)) == NULL) {
6079 aprint_error_dev(dev, "could not create sysctl root\n");
6080 break;
6081 }
6082
6083 if (sysctl_createv(log, 0, &rnode, &rnode,
6084 0, CTLTYPE_NODE,
6085 adapter->queues[i].namebuf, SYSCTL_DESCR("Queue Name"),
6086 NULL, 0, NULL, 0, CTL_CREATE, CTL_EOL) != 0)
6087 break;
6088
6089 if (sysctl_createv(log, 0, &rnode, &cnode,
6090 CTLFLAG_READWRITE, CTLTYPE_INT,
6091 "interrupt_rate", SYSCTL_DESCR("Interrupt Rate"),
6092 ixgbe_sysctl_interrupt_rate_handler, 0,
6093 (void *)&adapter->queues[i], 0, CTL_CREATE, CTL_EOL) != 0)
6094 break;
6095
6096 if (sysctl_createv(log, 0, &rnode, &cnode,
6097 CTLFLAG_READONLY, CTLTYPE_QUAD,
6098 "irqs", SYSCTL_DESCR("irqs on this queue"),
6099 NULL, 0, &(adapter->queues[i].irqs),
6100 0, CTL_CREATE, CTL_EOL) != 0)
6101 break;
6102
6103 if (sysctl_createv(log, 0, &rnode, &cnode,
6104 CTLFLAG_READONLY, CTLTYPE_INT,
6105 "txd_head", SYSCTL_DESCR("Transmit Descriptor Head"),
6106 ixgbe_sysctl_tdh_handler, 0, (void *)txr,
6107 0, CTL_CREATE, CTL_EOL) != 0)
6108 break;
6109
6110 if (sysctl_createv(log, 0, &rnode, &cnode,
6111 CTLFLAG_READONLY, CTLTYPE_INT,
6112 "txd_tail", SYSCTL_DESCR("Transmit Descriptor Tail"),
6113 ixgbe_sysctl_tdt_handler, 0, (void *)txr,
6114 0, CTL_CREATE, CTL_EOL) != 0)
6115 break;
6116
6117 evcnt_attach_dynamic(&txr->tso_tx, EVCNT_TYPE_MISC,
6118 NULL, device_xname(dev), "TSO");
6119 evcnt_attach_dynamic(&txr->no_desc_avail, EVCNT_TYPE_MISC,
6120 NULL, adapter->queues[i].evnamebuf,
6121 "Queue No Descriptor Available");
6122 evcnt_attach_dynamic(&txr->total_packets, EVCNT_TYPE_MISC,
6123 NULL, adapter->queues[i].evnamebuf,
6124 "Queue Packets Transmitted");
6125
6126#ifdef LRO
6127 struct lro_ctrl *lro = &rxr->lro;
6128#endif /* LRO */
6129
6130 if (sysctl_createv(log, 0, &rnode, &cnode,
6131 CTLFLAG_READONLY,
6132 CTLTYPE_INT,
6133 "rxd_head", SYSCTL_DESCR("Receive Descriptor Head"),
6134 ixgbe_sysctl_rdh_handler, 0, (void *)rxr, 0,
6135 CTL_CREATE, CTL_EOL) != 0)
6136 break;
6137
6138 if (sysctl_createv(log, 0, &rnode, &cnode,
6139 CTLFLAG_READONLY,
6140 CTLTYPE_INT,
6141 "rxd_tail", SYSCTL_DESCR("Receive Descriptor Tail"),
6142 ixgbe_sysctl_rdt_handler, 0, (void *)rxr, 0,
6143 CTL_CREATE, CTL_EOL) != 0)
6144 break;
6145
6146 if (i < __arraycount(adapter->stats.mpc)) {
6147 evcnt_attach_dynamic(&adapter->stats.mpc[i],
6148 EVCNT_TYPE_MISC, NULL, adapter->queues[i].evnamebuf,
6149 "Missed Packet Count");
6150 }
6151 if (i < __arraycount(adapter->stats.pxontxc)) {
6152 evcnt_attach_dynamic(&adapter->stats.pxontxc[i],
6153 EVCNT_TYPE_MISC, NULL, adapter->queues[i].evnamebuf,
6154 "pxontxc");
6155 evcnt_attach_dynamic(&adapter->stats.pxonrxc[i],
6156 EVCNT_TYPE_MISC, NULL, adapter->queues[i].evnamebuf,
6157 "pxonrxc");
6158 evcnt_attach_dynamic(&adapter->stats.pxofftxc[i],
6159 EVCNT_TYPE_MISC, NULL, adapter->queues[i].evnamebuf,
6160 "pxofftxc");
6161 evcnt_attach_dynamic(&adapter->stats.pxoffrxc[i],
6162 EVCNT_TYPE_MISC, NULL, adapter->queues[i].evnamebuf,
6163 "pxoffrxc");
6164 evcnt_attach_dynamic(&adapter->stats.pxon2offc[i],
6165 EVCNT_TYPE_MISC, NULL, adapter->queues[i].evnamebuf,
6166 "pxon2offc");
6167 }
6168 if (i < __arraycount(adapter->stats.qprc)) {
6169 evcnt_attach_dynamic(&adapter->stats.qprc[i],
6170 EVCNT_TYPE_MISC, NULL, adapter->queues[i].evnamebuf,
6171 "qprc");
6172 evcnt_attach_dynamic(&adapter->stats.qptc[i],
6173 EVCNT_TYPE_MISC, NULL, adapter->queues[i].evnamebuf,
6174 "qptc");
6175 evcnt_attach_dynamic(&adapter->stats.qbrc[i],
6176 EVCNT_TYPE_MISC, NULL, adapter->queues[i].evnamebuf,
6177 "qbrc");
6178 evcnt_attach_dynamic(&adapter->stats.qbtc[i],
6179 EVCNT_TYPE_MISC, NULL, adapter->queues[i].evnamebuf,
6180 "qbtc");
6181 evcnt_attach_dynamic(&adapter->stats.qprdc[i],
6182 EVCNT_TYPE_MISC, NULL, adapter->queues[i].evnamebuf,
6183 "qprdc");
6184 }
6185
6186 evcnt_attach_dynamic(&rxr->rx_packets, EVCNT_TYPE_MISC,
6187 NULL, adapter->queues[i].evnamebuf, "Queue Packets Received");
6188 evcnt_attach_dynamic(&rxr->rx_bytes, EVCNT_TYPE_MISC,
6189 NULL, adapter->queues[i].evnamebuf, "Queue Bytes Received");
6190 evcnt_attach_dynamic(&rxr->rx_copies, EVCNT_TYPE_MISC,
6191 NULL, adapter->queues[i].evnamebuf, "Copied RX Frames");
6192 evcnt_attach_dynamic(&rxr->no_jmbuf, EVCNT_TYPE_MISC,
6193 NULL, adapter->queues[i].evnamebuf, "Rx no jumbo mbuf");
6194 evcnt_attach_dynamic(&rxr->rx_discarded, EVCNT_TYPE_MISC,
6195 NULL, adapter->queues[i].evnamebuf, "Rx discarded");
6196 evcnt_attach_dynamic(&rxr->rx_irq, EVCNT_TYPE_MISC,
6197 NULL, adapter->queues[i].evnamebuf, "Rx interrupts");
6198#ifdef LRO
6199 SYSCTL_ADD_INT(ctx, queue_list, OID_AUTO, "lro_queued",
6200 CTLFLAG_RD, &lro->lro_queued, 0,
6201 "LRO Queued");
6202 SYSCTL_ADD_INT(ctx, queue_list, OID_AUTO, "lro_flushed",
6203 CTLFLAG_RD, &lro->lro_flushed, 0,
6204 "LRO Flushed");
6205#endif /* LRO */
6206 }
6207
6208 /* MAC stats get the own sub node */
6209
6210
6211 snprintf(stats->namebuf,
6212 sizeof(stats->namebuf), "%s MAC Statistics", device_xname(dev));
6213
6214 evcnt_attach_dynamic(&stats->ipcs, EVCNT_TYPE_MISC, NULL,
6215 stats->namebuf, "rx csum offload - IP");
6216 evcnt_attach_dynamic(&stats->l4cs, EVCNT_TYPE_MISC, NULL,
6217 stats->namebuf, "rx csum offload - L4");
6218 evcnt_attach_dynamic(&stats->ipcs_bad, EVCNT_TYPE_MISC, NULL,
6219 stats->namebuf, "rx csum offload - IP bad");
6220 evcnt_attach_dynamic(&stats->l4cs_bad, EVCNT_TYPE_MISC, NULL,
6221 stats->namebuf, "rx csum offload - L4 bad");
6222 evcnt_attach_dynamic(&stats->intzero, EVCNT_TYPE_MISC, NULL,
6223 stats->namebuf, "Interrupt conditions zero");
6224 evcnt_attach_dynamic(&stats->legint, EVCNT_TYPE_MISC, NULL,
6225 stats->namebuf, "Legacy interrupts");
6226 evcnt_attach_dynamic(&stats->crcerrs, EVCNT_TYPE_MISC, NULL,
6227 stats->namebuf, "CRC Errors");
6228 evcnt_attach_dynamic(&stats->illerrc, EVCNT_TYPE_MISC, NULL,
6229 stats->namebuf, "Illegal Byte Errors");
6230 evcnt_attach_dynamic(&stats->errbc, EVCNT_TYPE_MISC, NULL,
6231 stats->namebuf, "Byte Errors");
6232 evcnt_attach_dynamic(&stats->mspdc, EVCNT_TYPE_MISC, NULL,
6233 stats->namebuf, "MAC Short Packets Discarded");
6234 evcnt_attach_dynamic(&stats->mlfc, EVCNT_TYPE_MISC, NULL,
6235 stats->namebuf, "MAC Local Faults");
6236 evcnt_attach_dynamic(&stats->mrfc, EVCNT_TYPE_MISC, NULL,
6237 stats->namebuf, "MAC Remote Faults");
6238 evcnt_attach_dynamic(&stats->rlec, EVCNT_TYPE_MISC, NULL,
6239 stats->namebuf, "Receive Length Errors");
6240 evcnt_attach_dynamic(&stats->lxontxc, EVCNT_TYPE_MISC, NULL,
6241 stats->namebuf, "Link XON Transmitted");
6242 evcnt_attach_dynamic(&stats->lxonrxc, EVCNT_TYPE_MISC, NULL,
6243 stats->namebuf, "Link XON Received");
6244 evcnt_attach_dynamic(&stats->lxofftxc, EVCNT_TYPE_MISC, NULL,
6245 stats->namebuf, "Link XOFF Transmitted");
6246 evcnt_attach_dynamic(&stats->lxoffrxc, EVCNT_TYPE_MISC, NULL,
6247 stats->namebuf, "Link XOFF Received");
6248
6249 /* Packet Reception Stats */
6250 evcnt_attach_dynamic(&stats->tor, EVCNT_TYPE_MISC, NULL,
6251 stats->namebuf, "Total Octets Received");
6252 evcnt_attach_dynamic(&stats->gorc, EVCNT_TYPE_MISC, NULL,
6253 stats->namebuf, "Good Octets Received");
6254 evcnt_attach_dynamic(&stats->tpr, EVCNT_TYPE_MISC, NULL,
6255 stats->namebuf, "Total Packets Received");
6256 evcnt_attach_dynamic(&stats->gprc, EVCNT_TYPE_MISC, NULL,
6257 stats->namebuf, "Good Packets Received");
6258 evcnt_attach_dynamic(&stats->mprc, EVCNT_TYPE_MISC, NULL,
6259 stats->namebuf, "Multicast Packets Received");
6260 evcnt_attach_dynamic(&stats->bprc, EVCNT_TYPE_MISC, NULL,
6261 stats->namebuf, "Broadcast Packets Received");
6262 evcnt_attach_dynamic(&stats->prc64, EVCNT_TYPE_MISC, NULL,
6263 stats->namebuf, "64 byte frames received ");
6264 evcnt_attach_dynamic(&stats->prc127, EVCNT_TYPE_MISC, NULL,
6265 stats->namebuf, "65-127 byte frames received");
6266 evcnt_attach_dynamic(&stats->prc255, EVCNT_TYPE_MISC, NULL,
6267 stats->namebuf, "128-255 byte frames received");
6268 evcnt_attach_dynamic(&stats->prc511, EVCNT_TYPE_MISC, NULL,
6269 stats->namebuf, "256-511 byte frames received");
6270 evcnt_attach_dynamic(&stats->prc1023, EVCNT_TYPE_MISC, NULL,
6271 stats->namebuf, "512-1023 byte frames received");
6272 evcnt_attach_dynamic(&stats->prc1522, EVCNT_TYPE_MISC, NULL,
6273 stats->namebuf, "1023-1522 byte frames received");
6274 evcnt_attach_dynamic(&stats->ruc, EVCNT_TYPE_MISC, NULL,
6275 stats->namebuf, "Receive Undersized");
6276 evcnt_attach_dynamic(&stats->rfc, EVCNT_TYPE_MISC, NULL,
6277 stats->namebuf, "Fragmented Packets Received ");
6278 evcnt_attach_dynamic(&stats->roc, EVCNT_TYPE_MISC, NULL,
6279 stats->namebuf, "Oversized Packets Received");
6280 evcnt_attach_dynamic(&stats->rjc, EVCNT_TYPE_MISC, NULL,
6281 stats->namebuf, "Received Jabber");
6282 evcnt_attach_dynamic(&stats->mngprc, EVCNT_TYPE_MISC, NULL,
6283 stats->namebuf, "Management Packets Received");
6284 evcnt_attach_dynamic(&stats->xec, EVCNT_TYPE_MISC, NULL,
6285 stats->namebuf, "Checksum Errors");
6286
6287 /* Packet Transmission Stats */
6288 evcnt_attach_dynamic(&stats->gotc, EVCNT_TYPE_MISC, NULL,
6289 stats->namebuf, "Good Octets Transmitted");
6290 evcnt_attach_dynamic(&stats->tpt, EVCNT_TYPE_MISC, NULL,
6291 stats->namebuf, "Total Packets Transmitted");
6292 evcnt_attach_dynamic(&stats->gptc, EVCNT_TYPE_MISC, NULL,
6293 stats->namebuf, "Good Packets Transmitted");
6294 evcnt_attach_dynamic(&stats->bptc, EVCNT_TYPE_MISC, NULL,
6295 stats->namebuf, "Broadcast Packets Transmitted");
6296 evcnt_attach_dynamic(&stats->mptc, EVCNT_TYPE_MISC, NULL,
6297 stats->namebuf, "Multicast Packets Transmitted");
6298 evcnt_attach_dynamic(&stats->mngptc, EVCNT_TYPE_MISC, NULL,
6299 stats->namebuf, "Management Packets Transmitted");
6300 evcnt_attach_dynamic(&stats->ptc64, EVCNT_TYPE_MISC, NULL,
6301 stats->namebuf, "64 byte frames transmitted ");
6302 evcnt_attach_dynamic(&stats->ptc127, EVCNT_TYPE_MISC, NULL,
6303 stats->namebuf, "65-127 byte frames transmitted");
6304 evcnt_attach_dynamic(&stats->ptc255, EVCNT_TYPE_MISC, NULL,
6305 stats->namebuf, "128-255 byte frames transmitted");
6306 evcnt_attach_dynamic(&stats->ptc511, EVCNT_TYPE_MISC, NULL,
6307 stats->namebuf, "256-511 byte frames transmitted");
6308 evcnt_attach_dynamic(&stats->ptc1023, EVCNT_TYPE_MISC, NULL,
6309 stats->namebuf, "512-1023 byte frames transmitted");
6310 evcnt_attach_dynamic(&stats->ptc1522, EVCNT_TYPE_MISC, NULL,
6311 stats->namebuf, "1024-1522 byte frames transmitted");
6312}
6313
6314/*
6315** Set flow control using sysctl:
6316** Flow control values:
6317** 0 - off
6318** 1 - rx pause
6319** 2 - tx pause
6320** 3 - full
6321*/
6322static int
6323ixgbe_set_flowcntl(SYSCTLFN_ARGS)
6324{
6325 struct sysctlnode node;
6326 int error, last;
6327 struct adapter *adapter;
6328
6329 node = *rnode;
6330 adapter = (struct adapter *)node.sysctl_data;
6331 node.sysctl_data = &adapter->fc;
6332 last = adapter->fc;
6333 error = sysctl_lookup(SYSCTLFN_CALL(&node));
6334 if (error != 0 || newp == NULL)
6335 return error;
6336
6337 /* Don't bother if it's not changed */
6338 if (adapter->fc == last)
6339 return (0);
6340
6341 switch (adapter->fc) {
6342 case ixgbe_fc_rx_pause:
6343 case ixgbe_fc_tx_pause:
6344 case ixgbe_fc_full:
6345 adapter->hw.fc.requested_mode = adapter->fc;
6346 if (adapter->num_queues > 1)
6347 ixgbe_disable_rx_drop(adapter);
6348 break;
6349 case ixgbe_fc_none:
6350 adapter->hw.fc.requested_mode = ixgbe_fc_none;
6351 if (adapter->num_queues > 1)
6352 ixgbe_enable_rx_drop(adapter);
6353 break;
6354 default:
6355 adapter->fc = last;
6356 return (EINVAL);
6357 }
6358 /* Don't autoneg if forcing a value */
6359 adapter->hw.fc.disable_fc_autoneg = TRUE;
6360 ixgbe_fc_enable(&adapter->hw);
6361 return 0;
6362}
6363
6364
6365/*
6366** Control link advertise speed:
6367** 1 - advertise only 1G
6368** 2 - advertise 100Mb
6369** 3 - advertise normal
6370*/
6371static int
6372ixgbe_set_advertise(SYSCTLFN_ARGS)
6373{
6374 struct sysctlnode node;
6375 int t, error = 0;
6376 struct adapter *adapter;
6377 device_t dev;
6378 struct ixgbe_hw *hw;
6379 ixgbe_link_speed speed, last;
6380
6381 node = *rnode;
6382 adapter = (struct adapter *)node.sysctl_data;
6383 dev = adapter->dev;
6384 hw = &adapter->hw;
6385 last = adapter->advertise;
6386 t = adapter->advertise;
6387 node.sysctl_data = &t;
6388 error = sysctl_lookup(SYSCTLFN_CALL(&node));
6389 if (error != 0 || newp == NULL)
6390 return error;
6391
6392 if (adapter->advertise == last) /* no change */
6393 return (0);
6394
6395 if (t == -1)
6396 return 0;
6397
6398 adapter->advertise = t;
6399
6400 if (!((hw->phy.media_type == ixgbe_media_type_copper) ||
6401 (hw->phy.multispeed_fiber)))
6402 return (EINVAL);
6403
6404 if ((adapter->advertise == 2) && (hw->mac.type != ixgbe_mac_X540)) {
6405 device_printf(dev, "Set Advertise: 100Mb on X540 only\n");
6406 return (EINVAL);
6407 }
6408
6409 if (adapter->advertise == 1)
6410 speed = IXGBE_LINK_SPEED_1GB_FULL;
6411 else if (adapter->advertise == 2)
6412 speed = IXGBE_LINK_SPEED_100_FULL;
6413 else if (adapter->advertise == 3)
6414 speed = IXGBE_LINK_SPEED_1GB_FULL |
6415 IXGBE_LINK_SPEED_10GB_FULL;
6416 else { /* bogus value */
6417 adapter->advertise = last;
6418 return (EINVAL);
6419 }
6420
6421 hw->mac.autotry_restart = TRUE;
6422 hw->mac.ops.setup_link(hw, speed, TRUE);
6423
6424 return 0;
6425}
6426
6427/*
6428** Thermal Shutdown Trigger
6429** - cause a Thermal Overtemp IRQ
6430** - this now requires firmware enabling
6431*/
6432static int
6433ixgbe_set_thermal_test(SYSCTLFN_ARGS)
6434{
6435 struct sysctlnode node;
6436 int error, fire = 0;
6437 struct adapter *adapter;
6438 struct ixgbe_hw *hw;
6439
6440 node = *rnode;
6441 adapter = (struct adapter *)node.sysctl_data;
6442 hw = &adapter->hw;
6443
6444 if (hw->mac.type != ixgbe_mac_X540)
6445 return (0);
6446
6447 node.sysctl_data = &fire;
6448 error = sysctl_lookup(SYSCTLFN_CALL(&node));
6449 if ((error) || (newp == NULL))
6450 return (error);
6451
6452 if (fire) {
6453 u32 reg = IXGBE_READ_REG(hw, IXGBE_EICS);
6454 reg |= IXGBE_EICR_TS;
6455 IXGBE_WRITE_REG(hw, IXGBE_EICS, reg);
6456 }
6457
6458 return (0);
6459}
6460
6461/*
6462** Enable the hardware to drop packets when the buffer is
6463** full. This is useful when multiqueue,so that no single
6464** queue being full stalls the entire RX engine. We only
6465** enable this when Multiqueue AND when Flow Control is
6466** disabled.
6467*/
6468static void
6469ixgbe_enable_rx_drop(struct adapter *adapter)
6470{
6471 struct ixgbe_hw *hw = &adapter->hw;
6472
6473 for (int i = 0; i < adapter->num_queues; i++) {
6474 u32 srrctl = IXGBE_READ_REG(hw, IXGBE_SRRCTL(i));
6475 srrctl |= IXGBE_SRRCTL_DROP_EN;
6476 IXGBE_WRITE_REG(hw, IXGBE_SRRCTL(i), srrctl);
6477 }
6478}
6479
6480static void
6481ixgbe_disable_rx_drop(struct adapter *adapter)
6482{
6483 struct ixgbe_hw *hw = &adapter->hw;
6484
6485 for (int i = 0; i < adapter->num_queues; i++) {
6486 u32 srrctl = IXGBE_READ_REG(hw, IXGBE_SRRCTL(i));
6487 srrctl &= ~IXGBE_SRRCTL_DROP_EN;
6488 IXGBE_WRITE_REG(hw, IXGBE_SRRCTL(i), srrctl);
6489 }
6490}
6491