diff options
Diffstat (limited to 'drivers/net/ethernet/intel/e1000/e1000_main.c')
| -rw-r--r-- | drivers/net/ethernet/intel/e1000/e1000_main.c | 1118 |
1 files changed, 611 insertions, 507 deletions
diff --git a/drivers/net/ethernet/intel/e1000/e1000_main.c b/drivers/net/ethernet/intel/e1000/e1000_main.c index 59ad007dd5aa..292389aceb2d 100644 --- a/drivers/net/ethernet/intel/e1000/e1000_main.c +++ b/drivers/net/ethernet/intel/e1000/e1000_main.c @@ -1,30 +1,5 @@ -/******************************************************************************* - - Intel PRO/1000 Linux driver - Copyright(c) 1999 - 2006 Intel Corporation. - - This program is free software; you can redistribute it and/or modify it - under the terms and conditions of the GNU General Public License, - version 2, as published by the Free Software Foundation. - - This program is distributed in the hope it will be useful, but WITHOUT - ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for - more details. - - You should have received a copy of the GNU General Public License along with - this program; if not, write to the Free Software Foundation, Inc., - 51 Franklin St - Fifth Floor, Boston, MA 02110-1301 USA. - - The full GNU General Public License is included in this distribution in - the file called "COPYING". - - Contact Information: - Linux NICS <linux.nics@intel.com> - e1000-devel Mailing List <e1000-devel@lists.sourceforge.net> - Intel Corporation, 5200 N.E. Elam Young Parkway, Hillsboro, OR 97124-6497 - -*******************************************************************************/ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright(c) 1999 - 2006 Intel Corporation. */ #include "e1000.h" #include <net/ip6_checksum.h> @@ -35,8 +10,6 @@ char e1000_driver_name[] = "e1000"; static char e1000_driver_string[] = "Intel(R) PRO/1000 Network Driver"; -#define DRV_VERSION "7.3.21-k8-NAPI" -const char e1000_driver_version[] = DRV_VERSION; static const char e1000_copyright[] = "Copyright (c) 1999-2006 Intel Corporation."; /* e1000_pci_tbl - PCI Device ID Table @@ -46,7 +19,7 @@ static const char e1000_copyright[] = "Copyright (c) 1999-2006 Intel Corporation * Macro expands to... * {PCI_DEVICE(PCI_VENDOR_ID_INTEL, device_id)} */ -static DEFINE_PCI_DEVICE_TABLE(e1000_pci_tbl) = { +static const struct pci_device_id e1000_pci_tbl[] = { INTEL_E1000_ETHERNET_DEVICE(0x1000), INTEL_E1000_ETHERNET_DEVICE(0x1001), INTEL_E1000_ETHERNET_DEVICE(0x1004), @@ -99,13 +72,13 @@ int e1000_setup_all_rx_resources(struct e1000_adapter *adapter); void e1000_free_all_tx_resources(struct e1000_adapter *adapter); void e1000_free_all_rx_resources(struct e1000_adapter *adapter); static int e1000_setup_tx_resources(struct e1000_adapter *adapter, - struct e1000_tx_ring *txdr); + struct e1000_tx_ring *txdr); static int e1000_setup_rx_resources(struct e1000_adapter *adapter, - struct e1000_rx_ring *rxdr); + struct e1000_rx_ring *rxdr); static void e1000_free_tx_resources(struct e1000_adapter *adapter, - struct e1000_tx_ring *tx_ring); + struct e1000_tx_ring *tx_ring); static void e1000_free_rx_resources(struct e1000_adapter *adapter, - struct e1000_rx_ring *rx_ring); + struct e1000_rx_ring *rx_ring); void e1000_update_stats(struct e1000_adapter *adapter); static int e1000_init_module(void); @@ -114,24 +87,23 @@ static int e1000_probe(struct pci_dev *pdev, const struct pci_device_id *ent); static void e1000_remove(struct pci_dev *pdev); static int e1000_alloc_queues(struct e1000_adapter *adapter); static int e1000_sw_init(struct e1000_adapter *adapter); -static int e1000_open(struct net_device *netdev); -static int e1000_close(struct net_device *netdev); +int e1000_open(struct net_device *netdev); +int e1000_close(struct net_device *netdev); static void e1000_configure_tx(struct e1000_adapter *adapter); static void e1000_configure_rx(struct e1000_adapter *adapter); static void e1000_setup_rctl(struct e1000_adapter *adapter); static void e1000_clean_all_tx_rings(struct e1000_adapter *adapter); static void e1000_clean_all_rx_rings(struct e1000_adapter *adapter); static void e1000_clean_tx_ring(struct e1000_adapter *adapter, - struct e1000_tx_ring *tx_ring); + struct e1000_tx_ring *tx_ring); static void e1000_clean_rx_ring(struct e1000_adapter *adapter, - struct e1000_rx_ring *rx_ring); + struct e1000_rx_ring *rx_ring); static void e1000_set_rx_mode(struct net_device *netdev); static void e1000_update_phy_info_task(struct work_struct *work); static void e1000_watchdog(struct work_struct *work); static void e1000_82547_tx_fifo_stall_task(struct work_struct *work); static netdev_tx_t e1000_xmit_frame(struct sk_buff *skb, struct net_device *netdev); -static struct net_device_stats * e1000_get_stats(struct net_device *netdev); static int e1000_change_mtu(struct net_device *netdev, int new_mtu); static int e1000_set_mac(struct net_device *netdev, void *p); static irqreturn_t e1000_intr(int irq, void *data); @@ -144,6 +116,11 @@ static bool e1000_clean_rx_irq(struct e1000_adapter *adapter, static bool e1000_clean_jumbo_rx_irq(struct e1000_adapter *adapter, struct e1000_rx_ring *rx_ring, int *work_done, int work_to_do); +static void e1000_alloc_dummy_rx_buffers(struct e1000_adapter *adapter, + struct e1000_rx_ring *rx_ring, + int cleaned_count) +{ +} static void e1000_alloc_rx_buffers(struct e1000_adapter *adapter, struct e1000_rx_ring *rx_ring, int cleaned_count); @@ -155,11 +132,11 @@ static int e1000_mii_ioctl(struct net_device *netdev, struct ifreq *ifr, int cmd); static void e1000_enter_82542_rst(struct e1000_adapter *adapter); static void e1000_leave_82542_rst(struct e1000_adapter *adapter); -static void e1000_tx_timeout(struct net_device *dev); +static void e1000_tx_timeout(struct net_device *dev, unsigned int txqueue); static void e1000_reset_task(struct work_struct *work); static void e1000_smartspeed(struct e1000_adapter *adapter); static int e1000_82547_fifo_workaround(struct e1000_adapter *adapter, - struct sk_buff *skb); + struct sk_buff *skb); static bool e1000_vlan_used(struct e1000_adapter *adapter); static void e1000_vlan_mode(struct net_device *netdev, @@ -172,10 +149,8 @@ static int e1000_vlan_rx_kill_vid(struct net_device *netdev, __be16 proto, u16 vid); static void e1000_restore_vlan(struct e1000_adapter *adapter); -#ifdef CONFIG_PM -static int e1000_suspend(struct pci_dev *pdev, pm_message_t state); -static int e1000_resume(struct pci_dev *pdev); -#endif +static int e1000_suspend(struct device *dev); +static int e1000_resume(struct device *dev); static void e1000_shutdown(struct pci_dev *pdev); #ifdef CONFIG_NET_POLL_CONTROLLER @@ -190,7 +165,7 @@ MODULE_PARM_DESC(copybreak, "Maximum size of packet that is copied to a new buffer on receive"); static pci_ers_result_t e1000_io_error_detected(struct pci_dev *pdev, - pci_channel_state_t state); + pci_channel_state_t state); static pci_ers_result_t e1000_io_slot_reset(struct pci_dev *pdev); static void e1000_io_resume(struct pci_dev *pdev); @@ -200,24 +175,20 @@ static const struct pci_error_handlers e1000_err_handler = { .resume = e1000_io_resume, }; +static DEFINE_SIMPLE_DEV_PM_OPS(e1000_pm_ops, e1000_suspend, e1000_resume); + static struct pci_driver e1000_driver = { .name = e1000_driver_name, .id_table = e1000_pci_tbl, .probe = e1000_probe, .remove = e1000_remove, -#ifdef CONFIG_PM - /* Power Management Hooks */ - .suspend = e1000_suspend, - .resume = e1000_resume, -#endif + .driver.pm = pm_sleep_ptr(&e1000_pm_ops), .shutdown = e1000_shutdown, .err_handler = &e1000_err_handler }; -MODULE_AUTHOR("Intel Corporation, <linux.nics@intel.com>"); MODULE_DESCRIPTION("Intel(R) PRO/1000 Network Driver"); -MODULE_LICENSE("GPL"); -MODULE_VERSION(DRV_VERSION); +MODULE_LICENSE("GPL v2"); #define DEFAULT_MSG_ENABLE (NETIF_MSG_DRV|NETIF_MSG_PROBE|NETIF_MSG_LINK) static int debug = -1; @@ -225,8 +196,10 @@ module_param(debug, int, 0); MODULE_PARM_DESC(debug, "Debug level (0=none,...,16=all)"); /** - * e1000_get_hw_dev - return device - * used by hardware layer to print debugging information + * e1000_get_hw_dev - helper function for getting netdev + * @hw: pointer to HW struct + * + * return device used by hardware layer to print debugging information * **/ struct net_device *e1000_get_hw_dev(struct e1000_hw *hw) @@ -244,7 +217,7 @@ struct net_device *e1000_get_hw_dev(struct e1000_hw *hw) static int __init e1000_init_module(void) { int ret; - pr_info("%s - version %s\n", e1000_driver_string, e1000_driver_version); + pr_info("%s\n", e1000_driver_string); pr_info("%s\n", e1000_copyright); @@ -282,7 +255,7 @@ static int e1000_request_irq(struct e1000_adapter *adapter) int err; err = request_irq(adapter->pdev->irq, handler, irq_flags, netdev->name, - netdev); + netdev); if (err) { e_err(probe, "Unable to allocate interrupt Error: %d\n", err); } @@ -340,8 +313,7 @@ static void e1000_update_mng_vlan(struct e1000_adapter *adapter) } else { adapter->mng_vlan_id = E1000_MNG_VLAN_NONE; } - if ((old_vid != (u16)E1000_MNG_VLAN_NONE) && - (vid != old_vid) && + if (old_vid != E1000_MNG_VLAN_NONE && vid != old_vid && !test_bit(old_vid, adapter->active_vlans)) e1000_vlan_rx_kill_vid(netdev, htons(ETH_P_8021Q), old_vid); @@ -380,7 +352,7 @@ static void e1000_release_manageability(struct e1000_adapter *adapter) /** * e1000_configure - configure the hardware for RX and TX - * @adapter = private board structure + * @adapter: private board structure **/ static void e1000_configure(struct e1000_adapter *adapter) { @@ -494,11 +466,14 @@ static void e1000_down_and_stop(struct e1000_adapter *adapter) { set_bit(__E1000_DOWN, &adapter->flags); - /* Only kill reset task if adapter is not resetting */ - if (!test_bit(__E1000_RESETTING, &adapter->flags)) - cancel_work_sync(&adapter->reset_task); - cancel_delayed_work_sync(&adapter->watchdog_task); + + /* + * Since the watchdog task can reschedule other tasks, we should cancel + * it first, otherwise we can run into the situation when a work is + * still running after the adapter has been turned down. + */ + cancel_delayed_work_sync(&adapter->phy_info_task); cancel_delayed_work_sync(&adapter->fifo_stall_task); } @@ -509,7 +484,6 @@ void e1000_down(struct e1000_adapter *adapter) struct net_device *netdev = adapter->netdev; u32 rctl, tctl; - /* disable receives in the hardware */ rctl = er32(RCTL); ew32(RCTL, rctl & ~E1000_RCTL_EN); @@ -525,6 +499,17 @@ void e1000_down(struct e1000_adapter *adapter) E1000_WRITE_FLUSH(); msleep(10); + /* Set the carrier off after transmits have been disabled in the + * hardware, to avoid race conditions with e1000_watchdog() (which + * may be running concurrently to us, checking for the carrier + * bit to decide whether it should enable transmits again). Such + * a race condition would result into transmission being disabled + * in the hardware until the next IFF_DOWN+IFF_UP cycle. + */ + netif_carrier_off(netdev); + + netif_queue_set_napi(netdev, 0, NETDEV_QUEUE_TYPE_RX, NULL); + netif_queue_set_napi(netdev, 0, NETDEV_QUEUE_TYPE_TX, NULL); napi_disable(&adapter->napi); e1000_irq_disable(adapter); @@ -537,33 +522,23 @@ void e1000_down(struct e1000_adapter *adapter) adapter->link_speed = 0; adapter->link_duplex = 0; - netif_carrier_off(netdev); e1000_reset(adapter); e1000_clean_all_tx_rings(adapter); e1000_clean_all_rx_rings(adapter); } -static void e1000_reinit_safe(struct e1000_adapter *adapter) -{ - while (test_and_set_bit(__E1000_RESETTING, &adapter->flags)) - msleep(1); - mutex_lock(&adapter->mutex); - e1000_down(adapter); - e1000_up(adapter); - mutex_unlock(&adapter->mutex); - clear_bit(__E1000_RESETTING, &adapter->flags); -} - void e1000_reinit_locked(struct e1000_adapter *adapter) { - /* if rtnl_lock is not held the call path is bogus */ - ASSERT_RTNL(); - WARN_ON(in_interrupt()); while (test_and_set_bit(__E1000_RESETTING, &adapter->flags)) msleep(1); - e1000_down(adapter); - e1000_up(adapter); + + /* only run the task if not already down */ + if (!test_bit(__E1000_DOWN, &adapter->flags)) { + e1000_down(adapter); + e1000_up(adapter); + } + clear_bit(__E1000_RESETTING, &adapter->flags); } @@ -637,8 +612,8 @@ void e1000_reset(struct e1000_adapter *adapter) * but don't include ethernet FCS because hardware appends it */ min_tx_space = (hw->max_frame_size + - sizeof(struct e1000_tx_desc) - - ETH_FCS_LEN) * 2; + sizeof(struct e1000_tx_desc) - + ETH_FCS_LEN) * 2; min_tx_space = ALIGN(min_tx_space, 1024); min_tx_space >>= 10; /* software strips receive CRC, so leave room for it */ @@ -840,19 +815,18 @@ static int e1000_set_features(struct net_device *netdev, else e1000_reset(adapter); - return 0; + return 1; } static const struct net_device_ops e1000_netdev_ops = { .ndo_open = e1000_open, .ndo_stop = e1000_close, .ndo_start_xmit = e1000_xmit_frame, - .ndo_get_stats = e1000_get_stats, .ndo_set_rx_mode = e1000_set_rx_mode, .ndo_set_mac_address = e1000_set_mac, .ndo_tx_timeout = e1000_tx_timeout, .ndo_change_mtu = e1000_change_mtu, - .ndo_do_ioctl = e1000_ioctl, + .ndo_eth_ioctl = e1000_ioctl, .ndo_validate_addr = eth_validate_addr, .ndo_vlan_rx_add_vid = e1000_vlan_rx_add_vid, .ndo_vlan_rx_kill_vid = e1000_vlan_rx_kill_vid, @@ -941,16 +915,17 @@ static int e1000_init_hw_struct(struct e1000_adapter *adapter, static int e1000_probe(struct pci_dev *pdev, const struct pci_device_id *ent) { struct net_device *netdev; - struct e1000_adapter *adapter; + struct e1000_adapter *adapter = NULL; struct e1000_hw *hw; - static int cards_found = 0; - static int global_quad_port_a = 0; /* global ksp3 port a indication */ + static int cards_found; + static int global_quad_port_a; /* global ksp3 port a indication */ int i, err, pci_using_dac; u16 eeprom_data = 0; u16 tmp = 0; u16 eeprom_apme_mask = E1000_EEPROM_APME; int bars, need_ioport; + bool disable_dev = false; /* do not allocate ioport bars when not needed */ need_ioport = e1000_is_need_ioport(pdev); @@ -997,7 +972,7 @@ static int e1000_probe(struct pci_dev *pdev, const struct pci_device_id *ent) goto err_ioremap; if (adapter->need_ioport) { - for (i = BAR_1; i <= BAR_5; i++) { + for (i = BAR_1; i < PCI_STD_NUM_BARS; i++) { if (pci_resource_len(pdev, i) == 0) continue; if (pci_resource_flags(pdev, i) & IORESOURCE_IO) { @@ -1018,27 +993,22 @@ static int e1000_probe(struct pci_dev *pdev, const struct pci_device_id *ent) */ pci_using_dac = 0; if ((hw->bus_type == e1000_bus_type_pcix) && - !dma_set_mask(&pdev->dev, DMA_BIT_MASK(64))) { - /* according to DMA-API-HOWTO, coherent calls will always - * succeed if the set call did - */ - dma_set_coherent_mask(&pdev->dev, DMA_BIT_MASK(64)); + !dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64))) { pci_using_dac = 1; } else { - err = dma_set_mask(&pdev->dev, DMA_BIT_MASK(32)); + err = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(32)); if (err) { pr_err("No usable DMA config, aborting\n"); goto err_dma; } - dma_set_coherent_mask(&pdev->dev, DMA_BIT_MASK(32)); } netdev->netdev_ops = &e1000_netdev_ops; e1000_set_ethtool_ops(netdev); netdev->watchdog_timeo = 5 * HZ; - netif_napi_add(netdev, &adapter->napi, e1000_clean, 64); + netif_napi_add(netdev, &adapter->napi, e1000_clean); - strncpy(netdev->name, pci_name(pdev), sizeof(netdev->name) - 1); + strscpy(netdev->name, pci_name(pdev), sizeof(netdev->name)); adapter->bd_number = cards_found; @@ -1052,7 +1022,7 @@ static int e1000_probe(struct pci_dev *pdev, const struct pci_device_id *ent) if (hw->mac_type == e1000_ce4100) { hw->ce4100_gbe_mdio_base_virt = ioremap(pci_resource_start(pdev, BAR_1), - pci_resource_len(pdev, BAR_1)); + pci_resource_len(pdev, BAR_1)); if (!hw->ce4100_gbe_mdio_base_virt) goto err_mdio_ioremap; @@ -1086,7 +1056,14 @@ static int e1000_probe(struct pci_dev *pdev, const struct pci_device_id *ent) NETIF_F_HW_CSUM | NETIF_F_SG); - netdev->priv_flags |= IFF_UNICAST_FLT; + /* Do not set IFF_UNICAST_FLT for VMWare's 82545EM */ + if (hw->device_id != E1000_DEV_ID_82545EM_COPPER || + hw->subsystem_vendor_id != PCI_VENDOR_ID_VMWARE) + netdev->priv_flags |= IFF_UNICAST_FLT; + + /* MTU range: 46 - 16110 */ + netdev->min_mtu = ETH_ZLEN - ETH_HLEN; + netdev->max_mtu = MAX_JUMBO_FRAME_SIZE - (ETH_HLEN + ETH_FCS_LEN); adapter->en_mng_pt = e1000_enable_mng_pass_thru(hw); @@ -1119,8 +1096,8 @@ static int e1000_probe(struct pci_dev *pdev, const struct pci_device_id *ent) if (e1000_read_mac_addr(hw)) e_err(probe, "EEPROM Read Error\n"); } - /* don't block initalization here due to bad MAC address */ - memcpy(netdev->dev_addr, hw->mac_addr, netdev->addr_len); + /* don't block initialization here due to bad MAC address */ + eth_hw_addr_set(netdev, hw->mac_addr); if (!is_valid_ether_addr(netdev->dev_addr)) e_err(probe, "Invalid MAC Address\n"); @@ -1151,12 +1128,12 @@ static int e1000_probe(struct pci_dev *pdev, const struct pci_device_id *ent) break; case e1000_82546: case e1000_82546_rev_3: - if (er32(STATUS) & E1000_STATUS_FUNC_1){ + if (er32(STATUS) & E1000_STATUS_FUNC_1) { e1000_read_eeprom(hw, EEPROM_INIT_CONTROL3_PORT_B, 1, &eeprom_data); break; } - /* Fall Through */ + fallthrough; default: e1000_read_eeprom(hw, EEPROM_INIT_CONTROL3_PORT_A, 1, &eeprom_data); @@ -1202,13 +1179,13 @@ static int e1000_probe(struct pci_dev *pdev, const struct pci_device_id *ent) for (i = 0; i < 32; i++) { hw->phy_addr = i; e1000_read_phy_reg(hw, PHY_ID2, &tmp); - if (tmp == 0 || tmp == 0xFF) { - if (i == 31) - goto err_eeprom; - continue; - } else + + if (tmp != 0 && tmp != 0xFF) break; } + + if (i >= 32) + goto err_eeprom; } /* reset the hardware with the new settings */ @@ -1253,11 +1230,13 @@ err_mdio_ioremap: iounmap(hw->ce4100_gbe_mdio_base_virt); iounmap(hw->hw_addr); err_ioremap: + disable_dev = !test_and_set_bit(__E1000_DISABLED, &adapter->flags); free_netdev(netdev); err_alloc_etherdev: pci_release_selected_regions(pdev, bars); err_pci_reg: - pci_disable_device(pdev); + if (!adapter || disable_dev) + pci_disable_device(pdev); return err; } @@ -1266,7 +1245,7 @@ err_pci_reg: * @pdev: PCI device information struct * * e1000_remove is called by the PCI subsystem to alert the driver - * that it should release a PCI device. The could be caused by a + * that it should release a PCI device. That could be caused by a * Hot-Plug event, or because the driver is going to be removed from * memory. **/ @@ -1275,12 +1254,17 @@ static void e1000_remove(struct pci_dev *pdev) struct net_device *netdev = pci_get_drvdata(pdev); struct e1000_adapter *adapter = netdev_priv(netdev); struct e1000_hw *hw = &adapter->hw; + bool disable_dev; e1000_down_and_stop(adapter); e1000_release_manageability(adapter); unregister_netdev(netdev); + /* Only kill reset task if adapter is not resetting */ + if (!test_bit(__E1000_RESETTING, &adapter->flags)) + cancel_work_sync(&adapter->reset_task); + e1000_phy_hw_reset(hw); kfree(adapter->tx_ring); @@ -1293,9 +1277,11 @@ static void e1000_remove(struct pci_dev *pdev) iounmap(hw->flash_address); pci_release_selected_regions(pdev, adapter->bars); + disable_dev = !test_and_set_bit(__E1000_DISABLED, &adapter->flags); free_netdev(netdev); - pci_disable_device(pdev); + if (disable_dev) + pci_disable_device(pdev); } /** @@ -1321,7 +1307,6 @@ static int e1000_sw_init(struct e1000_adapter *adapter) e1000_irq_disable(adapter); spin_lock_init(&adapter->stats_lock); - mutex_init(&adapter->mutex); set_bit(__E1000_DOWN, &adapter->flags); @@ -1338,12 +1323,12 @@ static int e1000_sw_init(struct e1000_adapter *adapter) static int e1000_alloc_queues(struct e1000_adapter *adapter) { adapter->tx_ring = kcalloc(adapter->num_tx_queues, - sizeof(struct e1000_tx_ring), GFP_KERNEL); + sizeof(struct e1000_tx_ring), GFP_KERNEL); if (!adapter->tx_ring) return -ENOMEM; adapter->rx_ring = kcalloc(adapter->num_rx_queues, - sizeof(struct e1000_rx_ring), GFP_KERNEL); + sizeof(struct e1000_rx_ring), GFP_KERNEL); if (!adapter->rx_ring) { kfree(adapter->tx_ring); return -ENOMEM; @@ -1364,7 +1349,7 @@ static int e1000_alloc_queues(struct e1000_adapter *adapter) * handler is registered with the OS, the watchdog task is started, * and the stack is notified that the interface is ready. **/ -static int e1000_open(struct net_device *netdev) +int e1000_open(struct net_device *netdev) { struct e1000_adapter *adapter = netdev_priv(netdev); struct e1000_hw *hw = &adapter->hw; @@ -1408,7 +1393,10 @@ static int e1000_open(struct net_device *netdev) /* From here on the code is the same as e1000_up() */ clear_bit(__E1000_DOWN, &adapter->flags); + netif_napi_set_irq(&adapter->napi, adapter->pdev->irq); napi_enable(&adapter->napi); + netif_queue_set_napi(netdev, 0, NETDEV_QUEUE_TYPE_RX, &adapter->napi); + netif_queue_set_napi(netdev, 0, NETDEV_QUEUE_TYPE_TX, &adapter->napi); e1000_irq_enable(adapter); @@ -1441,12 +1429,21 @@ err_setup_tx: * needs to be disabled. A global MAC reset is issued to stop the * hardware, and all transmit and receive resources are freed. **/ -static int e1000_close(struct net_device *netdev) +int e1000_close(struct net_device *netdev) { struct e1000_adapter *adapter = netdev_priv(netdev); struct e1000_hw *hw = &adapter->hw; + int count = E1000_CHECK_RESET_COUNT; + + while (test_and_set_bit(__E1000_RESETTING, &adapter->flags) && count--) + usleep_range(10000, 20000); + + WARN_ON(count < 0); + + /* signal that we're down so that the reset task will no longer run */ + set_bit(__E1000_DOWN, &adapter->flags); + clear_bit(__E1000_RESETTING, &adapter->flags); - WARN_ON(test_bit(__E1000_RESETTING, &adapter->flags)); e1000_down(adapter); e1000_power_down_phy(adapter); e1000_free_irq(adapter); @@ -1486,7 +1483,7 @@ static bool e1000_check_64k_bound(struct e1000_adapter *adapter, void *start, if (hw->mac_type == e1000_82545 || hw->mac_type == e1000_ce4100 || hw->mac_type == e1000_82546) { - return ((begin ^ (end - 1)) >> 16) != 0 ? false : true; + return ((begin ^ (end - 1)) >> 16) == 0; } return true; @@ -1505,7 +1502,7 @@ static int e1000_setup_tx_resources(struct e1000_adapter *adapter, struct pci_dev *pdev = adapter->pdev; int size; - size = sizeof(struct e1000_buffer) * txdr->count; + size = sizeof(struct e1000_tx_buffer) * txdr->count; txdr->buffer_info = vzalloc(size); if (!txdr->buffer_info) return -ENOMEM; @@ -1695,7 +1692,7 @@ static int e1000_setup_rx_resources(struct e1000_adapter *adapter, struct pci_dev *pdev = adapter->pdev; int size, desc_len; - size = sizeof(struct e1000_buffer) * rxdr->count; + size = sizeof(struct e1000_rx_buffer) * rxdr->count; rxdr->buffer_info = vzalloc(size); if (!rxdr->buffer_info) return -ENOMEM; @@ -1811,20 +1808,20 @@ static void e1000_setup_rctl(struct e1000_adapter *adapter) rctl &= ~E1000_RCTL_SZ_4096; rctl |= E1000_RCTL_BSEX; switch (adapter->rx_buffer_len) { - case E1000_RXBUFFER_2048: - default: - rctl |= E1000_RCTL_SZ_2048; - rctl &= ~E1000_RCTL_BSEX; - break; - case E1000_RXBUFFER_4096: - rctl |= E1000_RCTL_SZ_4096; - break; - case E1000_RXBUFFER_8192: - rctl |= E1000_RCTL_SZ_8192; - break; - case E1000_RXBUFFER_16384: - rctl |= E1000_RCTL_SZ_16384; - break; + case E1000_RXBUFFER_2048: + default: + rctl |= E1000_RCTL_SZ_2048; + rctl &= ~E1000_RCTL_BSEX; + break; + case E1000_RXBUFFER_4096: + rctl |= E1000_RCTL_SZ_4096; + break; + case E1000_RXBUFFER_8192: + rctl |= E1000_RCTL_SZ_8192; + break; + case E1000_RXBUFFER_16384: + rctl |= E1000_RCTL_SZ_16384; + break; } /* This is useful for sniffing bad packets. */ @@ -1861,12 +1858,12 @@ static void e1000_configure_rx(struct e1000_adapter *adapter) if (adapter->netdev->mtu > ETH_DATA_LEN) { rdlen = adapter->rx_ring[0].count * - sizeof(struct e1000_rx_desc); + sizeof(struct e1000_rx_desc); adapter->clean_rx = e1000_clean_jumbo_rx_irq; adapter->alloc_rx_buf = e1000_alloc_jumbo_rx_buffers; } else { rdlen = adapter->rx_ring[0].count * - sizeof(struct e1000_rx_desc); + sizeof(struct e1000_rx_desc); adapter->clean_rx = e1000_clean_rx_irq; adapter->alloc_rx_buf = e1000_alloc_rx_buffers; } @@ -1955,8 +1952,10 @@ void e1000_free_all_tx_resources(struct e1000_adapter *adapter) e1000_free_tx_resources(adapter, &adapter->tx_ring[i]); } -static void e1000_unmap_and_free_tx_resource(struct e1000_adapter *adapter, - struct e1000_buffer *buffer_info) +static void +e1000_unmap_and_free_tx_resource(struct e1000_adapter *adapter, + struct e1000_tx_buffer *buffer_info, + int budget) { if (buffer_info->dma) { if (buffer_info->mapped_as_page) @@ -1969,7 +1968,7 @@ static void e1000_unmap_and_free_tx_resource(struct e1000_adapter *adapter, buffer_info->dma = 0; } if (buffer_info->skb) { - dev_kfree_skb_any(buffer_info->skb); + napi_consume_skb(buffer_info->skb, budget); buffer_info->skb = NULL; } buffer_info->time_stamp = 0; @@ -1985,7 +1984,7 @@ static void e1000_clean_tx_ring(struct e1000_adapter *adapter, struct e1000_tx_ring *tx_ring) { struct e1000_hw *hw = &adapter->hw; - struct e1000_buffer *buffer_info; + struct e1000_tx_buffer *buffer_info; unsigned long size; unsigned int i; @@ -1993,11 +1992,11 @@ static void e1000_clean_tx_ring(struct e1000_adapter *adapter, for (i = 0; i < tx_ring->count; i++) { buffer_info = &tx_ring->buffer_info[i]; - e1000_unmap_and_free_tx_resource(adapter, buffer_info); + e1000_unmap_and_free_tx_resource(adapter, buffer_info, 0); } netdev_reset_queue(adapter->netdev); - size = sizeof(struct e1000_buffer) * tx_ring->count; + size = sizeof(struct e1000_tx_buffer) * tx_ring->count; memset(tx_ring->buffer_info, 0, size); /* Zero out the descriptor ring */ @@ -2061,6 +2060,23 @@ void e1000_free_all_rx_resources(struct e1000_adapter *adapter) e1000_free_rx_resources(adapter, &adapter->rx_ring[i]); } +#define E1000_HEADROOM (NET_SKB_PAD + NET_IP_ALIGN) +static unsigned int e1000_frag_len(const struct e1000_adapter *a) +{ + return SKB_DATA_ALIGN(a->rx_buffer_len + E1000_HEADROOM) + + SKB_DATA_ALIGN(sizeof(struct skb_shared_info)); +} + +static void *e1000_alloc_frag(const struct e1000_adapter *a) +{ + unsigned int len = e1000_frag_len(a); + u8 *data = netdev_alloc_frag(len); + + if (likely(data)) + data += E1000_HEADROOM; + return data; +} + /** * e1000_clean_rx_ring - Free Rx Buffers per Queue * @adapter: board private structure @@ -2070,44 +2086,42 @@ static void e1000_clean_rx_ring(struct e1000_adapter *adapter, struct e1000_rx_ring *rx_ring) { struct e1000_hw *hw = &adapter->hw; - struct e1000_buffer *buffer_info; + struct e1000_rx_buffer *buffer_info; struct pci_dev *pdev = adapter->pdev; unsigned long size; unsigned int i; - /* Free all the Rx ring sk_buffs */ + /* Free all the Rx netfrags */ for (i = 0; i < rx_ring->count; i++) { buffer_info = &rx_ring->buffer_info[i]; - if (buffer_info->dma && - adapter->clean_rx == e1000_clean_rx_irq) { - dma_unmap_single(&pdev->dev, buffer_info->dma, - buffer_info->length, - DMA_FROM_DEVICE); - } else if (buffer_info->dma && - adapter->clean_rx == e1000_clean_jumbo_rx_irq) { - dma_unmap_page(&pdev->dev, buffer_info->dma, - buffer_info->length, - DMA_FROM_DEVICE); + if (adapter->clean_rx == e1000_clean_rx_irq) { + if (buffer_info->dma) + dma_unmap_single(&pdev->dev, buffer_info->dma, + adapter->rx_buffer_len, + DMA_FROM_DEVICE); + if (buffer_info->rxbuf.data) { + skb_free_frag(buffer_info->rxbuf.data); + buffer_info->rxbuf.data = NULL; + } + } else if (adapter->clean_rx == e1000_clean_jumbo_rx_irq) { + if (buffer_info->dma) + dma_unmap_page(&pdev->dev, buffer_info->dma, + adapter->rx_buffer_len, + DMA_FROM_DEVICE); + if (buffer_info->rxbuf.page) { + put_page(buffer_info->rxbuf.page); + buffer_info->rxbuf.page = NULL; + } } buffer_info->dma = 0; - if (buffer_info->page) { - put_page(buffer_info->page); - buffer_info->page = NULL; - } - if (buffer_info->skb) { - dev_kfree_skb(buffer_info->skb); - buffer_info->skb = NULL; - } } /* there also may be some cached data from a chained receive */ - if (rx_ring->rx_skb_top) { - dev_kfree_skb(rx_ring->rx_skb_top); - rx_ring->rx_skb_top = NULL; - } + napi_free_frags(&adapter->napi); + rx_ring->rx_skb_top = NULL; - size = sizeof(struct e1000_buffer) * rx_ring->count; + size = sizeof(struct e1000_rx_buffer) * rx_ring->count; memset(rx_ring->buffer_info, 0, size); /* Zero out the descriptor ring */ @@ -2197,7 +2211,7 @@ static int e1000_set_mac(struct net_device *netdev, void *p) if (hw->mac_type == e1000_82542_rev2_0) e1000_enter_82542_rst(adapter); - memcpy(netdev->dev_addr, addr->sa_data, netdev->addr_len); + eth_hw_addr_set(netdev, addr->sa_data); memcpy(hw->mac_addr, addr->sa_data, netdev->addr_len); e1000_rar_set(hw, hw->mac_addr, 0); @@ -2330,11 +2344,8 @@ static void e1000_update_phy_info_task(struct work_struct *work) struct e1000_adapter *adapter = container_of(work, struct e1000_adapter, phy_info_task.work); - if (test_bit(__E1000_DOWN, &adapter->flags)) - return; - mutex_lock(&adapter->mutex); + e1000_phy_get_info(&adapter->hw, &adapter->phy_info); - mutex_unlock(&adapter->mutex); } /** @@ -2350,9 +2361,6 @@ static void e1000_82547_tx_fifo_stall_task(struct work_struct *work) struct net_device *netdev = adapter->netdev; u32 tctl; - if (test_bit(__E1000_DOWN, &adapter->flags)) - return; - mutex_lock(&adapter->mutex); if (atomic_read(&adapter->tx_fifo_stall)) { if ((er32(TDT) == er32(TDH)) && (er32(TDFT) == er32(TDFH)) && @@ -2373,7 +2381,6 @@ static void e1000_82547_tx_fifo_stall_task(struct work_struct *work) schedule_delayed_work(&adapter->fifo_stall_task, 1); } } - mutex_unlock(&adapter->mutex); } bool e1000_has_link(struct e1000_adapter *adapter) @@ -2427,10 +2434,6 @@ static void e1000_watchdog(struct work_struct *work) struct e1000_tx_ring *txdr = adapter->tx_ring; u32 link, tctl; - if (test_bit(__E1000_DOWN, &adapter->flags)) - return; - - mutex_lock(&adapter->mutex); link = e1000_has_link(adapter); if ((netif_carrier_ok(netdev)) && link) goto link_up; @@ -2438,7 +2441,6 @@ static void e1000_watchdog(struct work_struct *work) if (link) { if (!netif_carrier_ok(netdev)) { u32 ctrl; - bool txb2b = true; /* update snapshot of PHY registers on LSC */ e1000_get_speed_and_duplex(hw, &adapter->link_speed, @@ -2460,11 +2462,9 @@ static void e1000_watchdog(struct work_struct *work) adapter->tx_timeout_factor = 1; switch (adapter->link_speed) { case SPEED_10: - txb2b = false; adapter->tx_timeout_factor = 16; break; case SPEED_100: - txb2b = false; /* maybe add some timeout factor ? */ break; } @@ -2521,7 +2521,7 @@ link_up: adapter->tx_timeout_count++; schedule_work(&adapter->reset_task); /* exit immediately since reset is imminent */ - goto unlock; + return; } } @@ -2549,9 +2549,6 @@ link_up: /* Reschedule the task */ if (!test_bit(__E1000_DOWN, &adapter->flags)) schedule_delayed_work(&adapter->watchdog_task, 2 * HZ); - -unlock: - mutex_unlock(&adapter->mutex); } enum latency_range { @@ -2637,7 +2634,6 @@ static void e1000_set_itr(struct e1000_adapter *adapter) /* for non-gigabit speeds, just fix the interrupt rate at 4000 */ if (unlikely(adapter->link_speed != SPEED_1000)) { - current_itr = 0; new_itr = 4000; goto set_itr_now; } @@ -2696,26 +2692,26 @@ set_itr_now: #define E1000_TX_FLAGS_VLAN_SHIFT 16 static int e1000_tso(struct e1000_adapter *adapter, - struct e1000_tx_ring *tx_ring, struct sk_buff *skb) + struct e1000_tx_ring *tx_ring, struct sk_buff *skb, + __be16 protocol) { struct e1000_context_desc *context_desc; - struct e1000_buffer *buffer_info; + struct e1000_tx_buffer *buffer_info; unsigned int i; u32 cmd_length = 0; u16 ipcse = 0, tucse, mss; u8 ipcss, ipcso, tucss, tucso, hdr_len; - int err; if (skb_is_gso(skb)) { - if (skb_header_cloned(skb)) { - err = pskb_expand_head(skb, 0, 0, GFP_ATOMIC); - if (err) - return err; - } + int err; - hdr_len = skb_transport_offset(skb) + tcp_hdrlen(skb); + err = skb_cow_head(skb, 0); + if (err < 0) + return err; + + hdr_len = skb_tcp_all_headers(skb); mss = skb_shinfo(skb)->gso_size; - if (skb->protocol == htons(ETH_P_IP)) { + if (protocol == htons(ETH_P_IP)) { struct iphdr *iph = ip_hdr(skb); iph->tot_len = 0; iph->check = 0; @@ -2725,12 +2721,8 @@ static int e1000_tso(struct e1000_adapter *adapter, 0); cmd_length = E1000_TXD_CMD_IP; ipcse = skb_transport_offset(skb) - 1; - } else if (skb->protocol == htons(ETH_P_IPV6)) { - ipv6_hdr(skb)->payload_len = 0; - tcp_hdr(skb)->check = - ~csum_ipv6_magic(&ipv6_hdr(skb)->saddr, - &ipv6_hdr(skb)->daddr, - 0, IPPROTO_TCP, 0); + } else if (skb_is_gso_v6(skb)) { + tcp_v6_gso_csum_prep(skb); ipcse = 0; } ipcss = skb_network_offset(skb); @@ -2759,7 +2751,9 @@ static int e1000_tso(struct e1000_adapter *adapter, buffer_info->time_stamp = jiffies; buffer_info->next_to_watch = i; - if (++i == tx_ring->count) i = 0; + if (++i == tx_ring->count) + i = 0; + tx_ring->next_to_use = i; return true; @@ -2768,10 +2762,11 @@ static int e1000_tso(struct e1000_adapter *adapter, } static bool e1000_tx_csum(struct e1000_adapter *adapter, - struct e1000_tx_ring *tx_ring, struct sk_buff *skb) + struct e1000_tx_ring *tx_ring, struct sk_buff *skb, + __be16 protocol) { struct e1000_context_desc *context_desc; - struct e1000_buffer *buffer_info; + struct e1000_tx_buffer *buffer_info; unsigned int i; u8 css; u32 cmd_len = E1000_TXD_CMD_DEXT; @@ -2779,7 +2774,7 @@ static bool e1000_tx_csum(struct e1000_adapter *adapter, if (skb->ip_summed != CHECKSUM_PARTIAL) return false; - switch (skb->protocol) { + switch (protocol) { case cpu_to_be16(ETH_P_IP): if (ip_hdr(skb)->protocol == IPPROTO_TCP) cmd_len |= E1000_TXD_CMD_TCP; @@ -2813,7 +2808,9 @@ static bool e1000_tx_csum(struct e1000_adapter *adapter, buffer_info->time_stamp = jiffies; buffer_info->next_to_watch = i; - if (unlikely(++i == tx_ring->count)) i = 0; + if (unlikely(++i == tx_ring->count)) + i = 0; + tx_ring->next_to_use = i; return true; @@ -2830,7 +2827,7 @@ static int e1000_tx_map(struct e1000_adapter *adapter, { struct e1000_hw *hw = &adapter->hw; struct pci_dev *pdev = adapter->pdev; - struct e1000_buffer *buffer_info; + struct e1000_tx_buffer *buffer_info; unsigned int len = skb_headlen(skb); unsigned int offset = 0, size, count = 0, i; unsigned int f, bytecount, segs; @@ -2862,8 +2859,8 @@ static int e1000_tx_map(struct e1000_adapter *adapter, * packet is smaller than 2048 - 16 - 16 (or 2016) bytes */ if (unlikely((hw->bus_type == e1000_bus_type_pcix) && - (size > 2015) && count == 0)) - size = 2015; + (size > 2015) && count == 0)) + size = 2015; /* Workaround for potential 82544 hang in PCI-X. Avoid * terminating buffers within evenly-aligned dwords. @@ -2895,9 +2892,8 @@ static int e1000_tx_map(struct e1000_adapter *adapter, } for (f = 0; f < nr_frags; f++) { - const struct skb_frag_struct *frag; + const skb_frag_t *frag = &skb_shinfo(skb)->frags[f]; - frag = &skb_shinfo(skb)->frags[f]; len = skb_frag_size(frag); offset = 0; @@ -2960,11 +2956,11 @@ dma_error: count--; while (count--) { - if (i==0) + if (i == 0) i += tx_ring->count; i--; buffer_info = &tx_ring->buffer_info[i]; - e1000_unmap_and_free_tx_resource(adapter, buffer_info); + e1000_unmap_and_free_tx_resource(adapter, buffer_info, 0); } return 0; @@ -2974,9 +2970,8 @@ static void e1000_tx_queue(struct e1000_adapter *adapter, struct e1000_tx_ring *tx_ring, int tx_flags, int count) { - struct e1000_hw *hw = &adapter->hw; struct e1000_tx_desc *tx_desc = NULL; - struct e1000_buffer *buffer_info; + struct e1000_tx_buffer *buffer_info; u32 txd_upper = 0, txd_lower = E1000_TXD_CMD_IFCS; unsigned int i; @@ -3011,7 +3006,8 @@ static void e1000_tx_queue(struct e1000_adapter *adapter, tx_desc->lower.data = cpu_to_le32(txd_lower | buffer_info->length); tx_desc->upper.data = cpu_to_le32(txd_upper); - if (unlikely(++i == tx_ring->count)) i = 0; + if (unlikely(++i == tx_ring->count)) + i = 0; } tx_desc->lower.data |= cpu_to_le32(adapter->txd_cmd); @@ -3025,14 +3021,9 @@ static void e1000_tx_queue(struct e1000_adapter *adapter, * applicable for weak-ordered memory model archs, * such as IA-64). */ - wmb(); + dma_wmb(); tx_ring->next_to_use = i; - writel(i, hw->hw_addr + tx_ring->tdt); - /* we need this if more than one processor can write to our tail - * at a time, it synchronizes IO on IA64/Altix systems - */ - mmiowb(); } /* 82547 workaround to avoid controller hang in half-duplex environment. @@ -3104,7 +3095,7 @@ static int e1000_maybe_stop_tx(struct net_device *netdev, return __e1000_maybe_stop_tx(netdev, size); } -#define TXD_USE_COUNT(S, X) (((S) >> (X)) + 1 ) +#define TXD_USE_COUNT(S, X) (((S) + ((1 << (X)) - 1)) >> (X)) static netdev_tx_t e1000_xmit_frame(struct sk_buff *skb, struct net_device *netdev) { @@ -3120,6 +3111,7 @@ static netdev_tx_t e1000_xmit_frame(struct sk_buff *skb, int count = 0; int tso; unsigned int f; + __be16 protocol = vlan_get_protocol(skb); /* This goes back to the question of how to logically map a Tx queue * to a flow. Right now, performance is impacted slightly negatively @@ -3128,21 +3120,12 @@ static netdev_tx_t e1000_xmit_frame(struct sk_buff *skb, */ tx_ring = adapter->tx_ring; - if (unlikely(skb->len <= 0)) { - dev_kfree_skb_any(skb); - return NETDEV_TX_OK; - } - /* On PCI/PCI-X HW, if packet size is less than ETH_ZLEN, * packets may get corrupted during padding by HW. * To WA this issue, pad all small packets manually. */ - if (skb->len < ETH_ZLEN) { - if (skb_pad(skb, ETH_ZLEN - skb->len)) - return NETDEV_TX_OK; - skb->len = ETH_ZLEN; - skb_set_tail_pointer(skb, ETH_ZLEN); - } + if (eth_skb_pad(skb)) + return NETDEV_TX_OK; mss = skb_shinfo(skb)->gso_size; /* The controller does a simple calculation to @@ -3157,11 +3140,12 @@ static netdev_tx_t e1000_xmit_frame(struct sk_buff *skb, max_per_txd = min(mss << 2, max_per_txd); max_txd_pwr = fls(max_per_txd) - 1; - hdr_len = skb_transport_offset(skb) + tcp_hdrlen(skb); + hdr_len = skb_tcp_all_headers(skb); if (skb->data_len && hdr_len == len) { switch (hw->mac_type) { + case e1000_82544: { unsigned int pull_size; - case e1000_82544: + /* Make sure we have room to chop off 4 bytes, * and that the end alignment will work out to * this hardware's requirements @@ -3172,7 +3156,6 @@ static netdev_tx_t e1000_xmit_frame(struct sk_buff *skb, if ((unsigned long)(skb_tail_pointer(skb) - 1) & 4) break; - /* fall through */ pull_size = min((unsigned int)4, skb->data_len); if (!__pskb_pull_tail(skb, pull_size)) { e_err(drv, "__pskb_pull_tail " @@ -3182,6 +3165,7 @@ static netdev_tx_t e1000_xmit_frame(struct sk_buff *skb, } len = skb_headlen(skb); break; + } default: /* do nothing */ break; @@ -3231,14 +3215,15 @@ static netdev_tx_t e1000_xmit_frame(struct sk_buff *skb, return NETDEV_TX_BUSY; } - if (vlan_tx_tag_present(skb)) { + if (skb_vlan_tag_present(skb)) { tx_flags |= E1000_TX_FLAGS_VLAN; - tx_flags |= (vlan_tx_tag_get(skb) << E1000_TX_FLAGS_VLAN_SHIFT); + tx_flags |= (skb_vlan_tag_get(skb) << + E1000_TX_FLAGS_VLAN_SHIFT); } first = tx_ring->next_to_use; - tso = e1000_tso(adapter, tx_ring, skb); + tso = e1000_tso(adapter, tx_ring, skb, protocol); if (tso < 0) { dev_kfree_skb_any(skb); return NETDEV_TX_OK; @@ -3248,10 +3233,10 @@ static netdev_tx_t e1000_xmit_frame(struct sk_buff *skb, if (likely(hw->mac_type != e1000_82544)) tx_ring->last_tx_tso = true; tx_flags |= E1000_TX_FLAGS_TSO; - } else if (likely(e1000_tx_csum(adapter, tx_ring, skb))) + } else if (likely(e1000_tx_csum(adapter, tx_ring, skb, protocol))) tx_flags |= E1000_TX_FLAGS_CSUM; - if (likely(skb->protocol == htons(ETH_P_IP))) + if (protocol == htons(ETH_P_IP)) tx_flags |= E1000_TX_FLAGS_IPV4; if (unlikely(skb->no_fcs)) @@ -3261,13 +3246,34 @@ static netdev_tx_t e1000_xmit_frame(struct sk_buff *skb, nr_frags, mss); if (count) { + /* The descriptors needed is higher than other Intel drivers + * due to a number of workarounds. The breakdown is below: + * Data descriptors: MAX_SKB_FRAGS + 1 + * Context Descriptor: 1 + * Keep head from touching tail: 2 + * Workarounds: 3 + */ + int desc_needed = MAX_SKB_FRAGS + 7; + netdev_sent_queue(netdev, skb->len); skb_tx_timestamp(skb); e1000_tx_queue(adapter, tx_ring, tx_flags, count); + + /* 82544 potentially requires twice as many data descriptors + * in order to guarantee buffers don't end on evenly-aligned + * dwords + */ + if (adapter->pcix_82544) + desc_needed += MAX_SKB_FRAGS + 1; + /* Make sure there is space in the ring for the next send. */ - e1000_maybe_stop_tx(netdev, tx_ring, MAX_SKB_FRAGS + 2); + e1000_maybe_stop_tx(netdev, tx_ring, desc_needed); + if (!netdev_xmit_more() || + netif_xmit_stopped(netdev_get_tx_queue(netdev, 0))) { + writel(tx_ring->next_to_use, hw->hw_addr + tx_ring->tdt); + } } else { dev_kfree_skb_any(skb); tx_ring->buffer_info[first].time_stamp = 0; @@ -3398,7 +3404,7 @@ static void e1000_dump(struct e1000_adapter *adapter) for (i = 0; tx_ring->desc && (i < tx_ring->count); i++) { struct e1000_tx_desc *tx_desc = E1000_TX_DESC(*tx_ring, i); - struct e1000_buffer *buffer_info = &tx_ring->buffer_info[i]; + struct e1000_tx_buffer *buffer_info = &tx_ring->buffer_info[i]; struct my_u { __le64 a; __le64 b; }; struct my_u *u = (struct my_u *)tx_desc; const char *type; @@ -3440,7 +3446,7 @@ rx_ring_summary: for (i = 0; rx_ring->desc && (i < rx_ring->count); i++) { struct e1000_rx_desc *rx_desc = E1000_RX_DESC(*rx_ring, i); - struct e1000_buffer *buffer_info = &rx_ring->buffer_info[i]; + struct e1000_rx_buffer *buffer_info = &rx_ring->buffer_info[i]; struct my_u { __le64 a; __le64 b; }; struct my_u *u = (struct my_u *)rx_desc; const char *type; @@ -3454,7 +3460,7 @@ rx_ring_summary: pr_info("R[0x%03X] %016llX %016llX %016llX %p %s\n", i, le64_to_cpu(u->a), le64_to_cpu(u->b), - (u64)buffer_info->dma, buffer_info->skb, type); + (u64)buffer_info->dma, buffer_info->rxbuf.data, type); } /* for */ /* dump the descriptor caches */ @@ -3485,8 +3491,9 @@ exit: /** * e1000_tx_timeout - Respond to a Tx Hang * @netdev: network interface device structure + * @txqueue: number of the Tx queue that hung (unused) **/ -static void e1000_tx_timeout(struct net_device *netdev) +static void e1000_tx_timeout(struct net_device *netdev, unsigned int __always_unused txqueue) { struct e1000_adapter *adapter = netdev_priv(netdev); @@ -3500,23 +3507,10 @@ static void e1000_reset_task(struct work_struct *work) struct e1000_adapter *adapter = container_of(work, struct e1000_adapter, reset_task); - if (test_bit(__E1000_DOWN, &adapter->flags)) - return; e_err(drv, "Reset adapter\n"); - e1000_reinit_safe(adapter); -} - -/** - * e1000_get_stats - Get System Network Statistics - * @netdev: network interface device structure - * - * Returns the address of the device statistics structure. - * The statistics are actually updated from the watchdog. - **/ -static struct net_device_stats *e1000_get_stats(struct net_device *netdev) -{ - /* only return the current stats */ - return &netdev->stats; + rtnl_lock(); + e1000_reinit_locked(adapter); + rtnl_unlock(); } /** @@ -3530,13 +3524,7 @@ static int e1000_change_mtu(struct net_device *netdev, int new_mtu) { struct e1000_adapter *adapter = netdev_priv(netdev); struct e1000_hw *hw = &adapter->hw; - int max_frame = new_mtu + ENET_HEADER_SIZE + ETHERNET_FCS_SIZE; - - if ((max_frame < MINIMUM_ETHERNET_FRAME_SIZE) || - (max_frame > MAX_JUMBO_FRAME_SIZE)) { - e_err(probe, "Invalid MTU setting\n"); - return -EINVAL; - } + int max_frame = new_mtu + ETH_HLEN + ETH_FCS_LEN; /* Adapter-specific max frame size limits. */ switch (hw->mac_type) { @@ -3555,8 +3543,11 @@ static int e1000_change_mtu(struct net_device *netdev, int new_mtu) msleep(1); /* e1000_down has a dependency on max_frame_size */ hw->max_frame_size = max_frame; - if (netif_running(netdev)) + if (netif_running(netdev)) { + /* prevent buffers from being reallocated */ + adapter->alloc_rx_buf = e1000_alloc_dummy_rx_buffers; e1000_down(adapter); + } /* NOTE: netdev_alloc_skb reserves 16 bytes, and typically NET_IP_ALIGN * means we reserve 2 more, this pushes us to allocate from the next @@ -3581,9 +3572,9 @@ static int e1000_change_mtu(struct net_device *netdev, int new_mtu) (max_frame == MAXIMUM_ETHERNET_VLAN_SIZE))) adapter->rx_buffer_len = MAXIMUM_ETHERNET_VLAN_SIZE; - pr_info("%s changing MTU from %d to %d\n", - netdev->name, netdev->mtu, new_mtu); - netdev->mtu = new_mtu; + netdev_dbg(netdev, "changing MTU from %d to %d\n", + netdev->mtu, new_mtu); + WRITE_ONCE(netdev->mtu, new_mtu); if (netif_running(netdev)) e1000_up(adapter); @@ -3801,7 +3792,8 @@ static irqreturn_t e1000_intr(int irq, void *data) /** * e1000_clean - NAPI Rx polling callback - * @adapter: board private structure + * @napi: napi struct containing references to driver info + * @budget: budget given to driver for receive packets **/ static int e1000_clean(struct napi_struct *napi, int budget) { @@ -3813,14 +3805,15 @@ static int e1000_clean(struct napi_struct *napi, int budget) adapter->clean_rx(adapter, &adapter->rx_ring[0], &work_done, budget); - if (!tx_clean_complete) - work_done = budget; + if (!tx_clean_complete || work_done == budget) + return budget; - /* If budget not fully consumed, exit the polling mode */ - if (work_done < budget) { + /* Exit the polling mode, but don't re-enable interrupts if stack might + * poll us due to busy-polling + */ + if (likely(napi_complete_done(napi, work_done))) { if (likely(adapter->itr_setting & 3)) e1000_set_itr(adapter); - napi_complete(napi); if (!test_bit(__E1000_DOWN, &adapter->flags)) e1000_irq_enable(adapter); } @@ -3831,6 +3824,7 @@ static int e1000_clean(struct napi_struct *napi, int budget) /** * e1000_clean_tx_irq - Reclaim resources after transmit completes * @adapter: board private structure + * @tx_ring: ring to clean **/ static bool e1000_clean_tx_irq(struct e1000_adapter *adapter, struct e1000_tx_ring *tx_ring) @@ -3838,10 +3832,10 @@ static bool e1000_clean_tx_irq(struct e1000_adapter *adapter, struct e1000_hw *hw = &adapter->hw; struct net_device *netdev = adapter->netdev; struct e1000_tx_desc *tx_desc, *eop_desc; - struct e1000_buffer *buffer_info; + struct e1000_tx_buffer *buffer_info; unsigned int i, eop; unsigned int count = 0; - unsigned int total_tx_bytes=0, total_tx_packets=0; + unsigned int total_tx_bytes = 0, total_tx_packets = 0; unsigned int bytes_compl = 0, pkts_compl = 0; i = tx_ring->next_to_clean; @@ -3851,7 +3845,7 @@ static bool e1000_clean_tx_irq(struct e1000_adapter *adapter, while ((eop_desc->upper.data & cpu_to_le32(E1000_TXD_STAT_DD)) && (count < tx_ring->count)) { bool cleaned = false; - rmb(); /* read buffer_info after eop_desc */ + dma_rmb(); /* read buffer_info after eop_desc */ for ( ; !cleaned; count++) { tx_desc = E1000_TX_DESC(*tx_ring, i); buffer_info = &tx_ring->buffer_info[i]; @@ -3866,17 +3860,22 @@ static bool e1000_clean_tx_irq(struct e1000_adapter *adapter, } } - e1000_unmap_and_free_tx_resource(adapter, buffer_info); + e1000_unmap_and_free_tx_resource(adapter, buffer_info, + 64); tx_desc->upper.data = 0; - if (unlikely(++i == tx_ring->count)) i = 0; + if (unlikely(++i == tx_ring->count)) + i = 0; } eop = tx_ring->buffer_info[i].next_to_watch; eop_desc = E1000_TX_DESC(*tx_ring, eop); } - tx_ring->next_to_clean = i; + /* Synchronize with E1000_DESC_UNUSED called from e1000_xmit_frame, + * which will reuse the cleaned buffers. + */ + smp_store_release(&tx_ring->next_to_clean, i); netdev_completed_queue(netdev, pkts_compl, bytes_compl); @@ -3917,8 +3916,7 @@ static bool e1000_clean_tx_irq(struct e1000_adapter *adapter, " next_to_watch <%x>\n" " jiffies <%lx>\n" " next_to_watch.status <%x>\n", - (unsigned long)((tx_ring - adapter->tx_ring) / - sizeof(struct e1000_tx_ring)), + (unsigned long)(tx_ring - adapter->tx_ring), readl(hw->hw_addr + tx_ring->tdh), readl(hw->hw_addr + tx_ring->tdt), tx_ring->next_to_use, @@ -3943,7 +3941,7 @@ static bool e1000_clean_tx_irq(struct e1000_adapter *adapter, * @adapter: board private structure * @status_err: receive descriptor status and error fields * @csum: receive descriptor csum field - * @sk_buff: socket buffer with received data + * @skb: socket buffer with received data **/ static void e1000_rx_checksum(struct e1000_adapter *adapter, u32 status_err, u32 csum, struct sk_buff *skb) @@ -3955,9 +3953,11 @@ static void e1000_rx_checksum(struct e1000_adapter *adapter, u32 status_err, skb_checksum_none_assert(skb); /* 82543 or newer only */ - if (unlikely(hw->mac_type < e1000_82543)) return; + if (unlikely(hw->mac_type < e1000_82543)) + return; /* Ignore Checksum bit is set */ - if (unlikely(status & E1000_RXD_STAT_IXSM)) return; + if (unlikely(status & E1000_RXD_STAT_IXSM)) + return; /* TCP/UDP checksum error bit is set */ if (unlikely(errors & E1000_RXD_ERR_TCPE)) { /* let the stack verify checksum errors */ @@ -3977,12 +3977,15 @@ static void e1000_rx_checksum(struct e1000_adapter *adapter, u32 status_err, } /** - * e1000_consume_page - helper function + * e1000_consume_page - helper function for jumbo Rx path + * @bi: software descriptor shadow data + * @skb: skb being modified + * @length: length of data being added **/ -static void e1000_consume_page(struct e1000_buffer *bi, struct sk_buff *skb, +static void e1000_consume_page(struct e1000_rx_buffer *bi, struct sk_buff *skb, u16 length) { - bi->page = NULL; + bi->rxbuf.page = NULL; skb->len += length; skb->data_len += length; skb->truesize += PAGE_SIZE; @@ -4009,6 +4012,114 @@ static void e1000_receive_skb(struct e1000_adapter *adapter, u8 status, } /** + * e1000_tbi_adjust_stats + * @hw: Struct containing variables accessed by shared code + * @stats: point to stats struct + * @frame_len: The length of the frame in question + * @mac_addr: The Ethernet destination address of the frame in question + * + * Adjusts the statistic counters when a frame is accepted by TBI_ACCEPT + */ +static void e1000_tbi_adjust_stats(struct e1000_hw *hw, + struct e1000_hw_stats *stats, + u32 frame_len, const u8 *mac_addr) +{ + u64 carry_bit; + + /* First adjust the frame length. */ + frame_len--; + /* We need to adjust the statistics counters, since the hardware + * counters overcount this packet as a CRC error and undercount + * the packet as a good packet + */ + /* This packet should not be counted as a CRC error. */ + stats->crcerrs--; + /* This packet does count as a Good Packet Received. */ + stats->gprc++; + + /* Adjust the Good Octets received counters */ + carry_bit = 0x80000000 & stats->gorcl; + stats->gorcl += frame_len; + /* If the high bit of Gorcl (the low 32 bits of the Good Octets + * Received Count) was one before the addition, + * AND it is zero after, then we lost the carry out, + * need to add one to Gorch (Good Octets Received Count High). + * This could be simplified if all environments supported + * 64-bit integers. + */ + if (carry_bit && ((stats->gorcl & 0x80000000) == 0)) + stats->gorch++; + /* Is this a broadcast or multicast? Check broadcast first, + * since the test for a multicast frame will test positive on + * a broadcast frame. + */ + if (is_broadcast_ether_addr(mac_addr)) + stats->bprc++; + else if (is_multicast_ether_addr(mac_addr)) + stats->mprc++; + + if (frame_len == hw->max_frame_size) { + /* In this case, the hardware has overcounted the number of + * oversize frames. + */ + if (stats->roc > 0) + stats->roc--; + } + + /* Adjust the bin counters when the extra byte put the frame in the + * wrong bin. Remember that the frame_len was adjusted above. + */ + if (frame_len == 64) { + stats->prc64++; + stats->prc127--; + } else if (frame_len == 127) { + stats->prc127++; + stats->prc255--; + } else if (frame_len == 255) { + stats->prc255++; + stats->prc511--; + } else if (frame_len == 511) { + stats->prc511++; + stats->prc1023--; + } else if (frame_len == 1023) { + stats->prc1023++; + stats->prc1522--; + } else if (frame_len == 1522) { + stats->prc1522++; + } +} + +static bool e1000_tbi_should_accept(struct e1000_adapter *adapter, + u8 status, u8 errors, + u32 length, const u8 *data) +{ + struct e1000_hw *hw = &adapter->hw; + u8 last_byte = *(data + length - 1); + + if (TBI_ACCEPT(hw, status, errors, length, last_byte)) { + unsigned long irq_flags; + + spin_lock_irqsave(&adapter->stats_lock, irq_flags); + e1000_tbi_adjust_stats(hw, &adapter->stats, length, data); + spin_unlock_irqrestore(&adapter->stats_lock, irq_flags); + + return true; + } + + return false; +} + +static struct sk_buff *e1000_alloc_rx_skb(struct e1000_adapter *adapter, + unsigned int bufsz) +{ + struct sk_buff *skb = napi_alloc_skb(&adapter->napi, bufsz); + + if (unlikely(!skb)) + adapter->alloc_rx_buff_failed++; + return skb; +} + +/** * e1000_clean_jumbo_rx_irq - Send received data up the network stack; legacy * @adapter: board private structure * @rx_ring: ring to clean @@ -4022,17 +4133,15 @@ static bool e1000_clean_jumbo_rx_irq(struct e1000_adapter *adapter, struct e1000_rx_ring *rx_ring, int *work_done, int work_to_do) { - struct e1000_hw *hw = &adapter->hw; struct net_device *netdev = adapter->netdev; struct pci_dev *pdev = adapter->pdev; struct e1000_rx_desc *rx_desc, *next_rxd; - struct e1000_buffer *buffer_info, *next_buffer; - unsigned long irq_flags; + struct e1000_rx_buffer *buffer_info, *next_buffer; u32 length; unsigned int i; int cleaned_count = 0; bool cleaned = false; - unsigned int total_rx_bytes=0, total_rx_packets=0; + unsigned int total_rx_bytes = 0, total_rx_packets = 0; i = rx_ring->next_to_clean; rx_desc = E1000_RX_DESC(*rx_ring, i); @@ -4045,13 +4154,13 @@ static bool e1000_clean_jumbo_rx_irq(struct e1000_adapter *adapter, if (*work_done >= work_to_do) break; (*work_done)++; - rmb(); /* read descriptor and rx_buffer_info after status DD */ + dma_rmb(); /* read descriptor and rx_buffer_info after status DD */ status = rx_desc->status; - skb = buffer_info->skb; - buffer_info->skb = NULL; - if (++i == rx_ring->count) i = 0; + if (++i == rx_ring->count) + i = 0; + next_rxd = E1000_RX_DESC(*rx_ring, i); prefetch(next_rxd); @@ -4060,7 +4169,7 @@ static bool e1000_clean_jumbo_rx_irq(struct e1000_adapter *adapter, cleaned = true; cleaned_count++; dma_unmap_page(&pdev->dev, buffer_info->dma, - buffer_info->length, DMA_FROM_DEVICE); + adapter->rx_buffer_len, DMA_FROM_DEVICE); buffer_info->dma = 0; length = le16_to_cpu(rx_desc->length); @@ -4068,30 +4177,19 @@ static bool e1000_clean_jumbo_rx_irq(struct e1000_adapter *adapter, /* errors is only valid for DD + EOP descriptors */ if (unlikely((status & E1000_RXD_STAT_EOP) && (rx_desc->errors & E1000_RXD_ERR_FRAME_ERR_MASK))) { - u8 *mapped; - u8 last_byte; - - mapped = page_address(buffer_info->page); - last_byte = *(mapped + length - 1); - if (TBI_ACCEPT(hw, status, rx_desc->errors, length, - last_byte)) { - spin_lock_irqsave(&adapter->stats_lock, - irq_flags); - e1000_tbi_adjust_stats(hw, &adapter->stats, - length, mapped); - spin_unlock_irqrestore(&adapter->stats_lock, - irq_flags); + u8 *mapped = page_address(buffer_info->rxbuf.page); + + if (e1000_tbi_should_accept(adapter, status, + rx_desc->errors, + length, mapped)) { length--; + } else if (netdev->features & NETIF_F_RXALL) { + goto process_skb; } else { - if (netdev->features & NETIF_F_RXALL) - goto process_skb; - /* recycle both page and skb */ - buffer_info->skb = skb; /* an error means any chain goes out the window * too */ - if (rx_ring->rx_skb_top) - dev_kfree_skb(rx_ring->rx_skb_top); + dev_kfree_skb(rx_ring->rx_skb_top); rx_ring->rx_skb_top = NULL; goto next_desc; } @@ -4103,16 +4201,18 @@ process_skb: /* this descriptor is only the beginning (or middle) */ if (!rxtop) { /* this is the beginning of a chain */ - rxtop = skb; - skb_fill_page_desc(rxtop, 0, buffer_info->page, + rxtop = napi_get_frags(&adapter->napi); + if (!rxtop) + break; + + skb_fill_page_desc(rxtop, 0, + buffer_info->rxbuf.page, 0, length); } else { /* this is the middle of a chain */ skb_fill_page_desc(rxtop, skb_shinfo(rxtop)->nr_frags, - buffer_info->page, 0, length); - /* re-use the skb, only consumed the page */ - buffer_info->skb = skb; + buffer_info->rxbuf.page, 0, length); } e1000_consume_page(buffer_info, rxtop, length); goto next_desc; @@ -4121,32 +4221,48 @@ process_skb: /* end of the chain */ skb_fill_page_desc(rxtop, skb_shinfo(rxtop)->nr_frags, - buffer_info->page, 0, length); - /* re-use the current skb, we only consumed the - * page - */ - buffer_info->skb = skb; + buffer_info->rxbuf.page, 0, length); skb = rxtop; rxtop = NULL; e1000_consume_page(buffer_info, skb, length); } else { + struct page *p; /* no chain, got EOP, this buf is the packet * copybreak to save the put_page/alloc_page */ - if (length <= copybreak && - skb_tailroom(skb) >= length) { - u8 *vaddr; - vaddr = kmap_atomic(buffer_info->page); - memcpy(skb_tail_pointer(skb), vaddr, - length); - kunmap_atomic(vaddr); + p = buffer_info->rxbuf.page; + if (length <= copybreak) { + if (likely(!(netdev->features & NETIF_F_RXFCS))) + length -= 4; + skb = e1000_alloc_rx_skb(adapter, + length); + if (!skb) + break; + + memcpy(skb_tail_pointer(skb), + page_address(p), length); + /* re-use the page, so don't erase - * buffer_info->page + * buffer_info->rxbuf.page */ skb_put(skb, length); + e1000_rx_checksum(adapter, + status | rx_desc->errors << 24, + le16_to_cpu(rx_desc->csum), skb); + + total_rx_bytes += skb->len; + total_rx_packets++; + + e1000_receive_skb(adapter, status, + rx_desc->special, skb); + goto next_desc; } else { - skb_fill_page_desc(skb, 0, - buffer_info->page, 0, + skb = napi_get_frags(&adapter->napi); + if (!skb) { + adapter->alloc_rx_buff_failed++; + break; + } + skb_fill_page_desc(skb, 0, p, 0, length); e1000_consume_page(buffer_info, skb, length); @@ -4165,14 +4281,14 @@ process_skb: pskb_trim(skb, skb->len - 4); total_rx_packets++; - /* eth type trans needs skb->data to point to something */ - if (!pskb_may_pull(skb, ETH_HLEN)) { - e_err(drv, "pskb_may_pull failed.\n"); - dev_kfree_skb(skb); - goto next_desc; + if (status & E1000_RXD_STAT_VP) { + __le16 vlan = rx_desc->special; + u16 vid = le16_to_cpu(vlan) & E1000_RXD_SPC_VLAN_MASK; + + __vlan_hwaccel_put_tag(skb, htons(ETH_P_8021Q), vid); } - e1000_receive_skb(adapter, status, rx_desc->special, skb); + napi_gro_frags(&adapter->napi); next_desc: rx_desc->status = 0; @@ -4203,25 +4319,25 @@ next_desc: /* this should improve performance for small packets with large amounts * of reassembly being done in the stack */ -static void e1000_check_copybreak(struct net_device *netdev, - struct e1000_buffer *buffer_info, - u32 length, struct sk_buff **skb) +static struct sk_buff *e1000_copybreak(struct e1000_adapter *adapter, + struct e1000_rx_buffer *buffer_info, + u32 length, const void *data) { - struct sk_buff *new_skb; + struct sk_buff *skb; if (length > copybreak) - return; + return NULL; - new_skb = netdev_alloc_skb_ip_align(netdev, length); - if (!new_skb) - return; + skb = e1000_alloc_rx_skb(adapter, length); + if (!skb) + return NULL; + + dma_sync_single_for_cpu(&adapter->pdev->dev, buffer_info->dma, + length, DMA_FROM_DEVICE); - skb_copy_to_linear_data_offset(new_skb, -NET_IP_ALIGN, - (*skb)->data - NET_IP_ALIGN, - length + NET_IP_ALIGN); - /* save the skb in buffer_info as good */ - buffer_info->skb = *skb; - *skb = new_skb; + skb_put_data(skb, data, length); + + return skb; } /** @@ -4235,17 +4351,15 @@ static bool e1000_clean_rx_irq(struct e1000_adapter *adapter, struct e1000_rx_ring *rx_ring, int *work_done, int work_to_do) { - struct e1000_hw *hw = &adapter->hw; struct net_device *netdev = adapter->netdev; struct pci_dev *pdev = adapter->pdev; struct e1000_rx_desc *rx_desc, *next_rxd; - struct e1000_buffer *buffer_info, *next_buffer; - unsigned long flags; + struct e1000_rx_buffer *buffer_info, *next_buffer; u32 length; unsigned int i; int cleaned_count = 0; bool cleaned = false; - unsigned int total_rx_bytes=0, total_rx_packets=0; + unsigned int total_rx_bytes = 0, total_rx_packets = 0; i = rx_ring->next_to_clean; rx_desc = E1000_RX_DESC(*rx_ring, i); @@ -4253,20 +4367,40 @@ static bool e1000_clean_rx_irq(struct e1000_adapter *adapter, while (rx_desc->status & E1000_RXD_STAT_DD) { struct sk_buff *skb; + u8 *data; u8 status; if (*work_done >= work_to_do) break; (*work_done)++; - rmb(); /* read descriptor and rx_buffer_info after status DD */ + dma_rmb(); /* read descriptor and rx_buffer_info after status DD */ status = rx_desc->status; - skb = buffer_info->skb; - buffer_info->skb = NULL; + length = le16_to_cpu(rx_desc->length); - prefetch(skb->data - NET_IP_ALIGN); + data = buffer_info->rxbuf.data; + prefetch(data); + skb = e1000_copybreak(adapter, buffer_info, length, data); + if (!skb) { + unsigned int frag_len = e1000_frag_len(adapter); + + skb = napi_build_skb(data - E1000_HEADROOM, frag_len); + if (!skb) { + adapter->alloc_rx_buff_failed++; + break; + } + + skb_reserve(skb, E1000_HEADROOM); + dma_unmap_single(&pdev->dev, buffer_info->dma, + adapter->rx_buffer_len, + DMA_FROM_DEVICE); + buffer_info->dma = 0; + buffer_info->rxbuf.data = NULL; + } + + if (++i == rx_ring->count) + i = 0; - if (++i == rx_ring->count) i = 0; next_rxd = E1000_RX_DESC(*rx_ring, i); prefetch(next_rxd); @@ -4274,11 +4408,7 @@ static bool e1000_clean_rx_irq(struct e1000_adapter *adapter, cleaned = true; cleaned_count++; - dma_unmap_single(&pdev->dev, buffer_info->dma, - buffer_info->length, DMA_FROM_DEVICE); - buffer_info->dma = 0; - length = le16_to_cpu(rx_desc->length); /* !EOP means multiple descriptors were used to store a single * packet, if thats the case we need to toss it. In fact, we * to toss every packet with the EOP bit clear and the next @@ -4290,29 +4420,22 @@ static bool e1000_clean_rx_irq(struct e1000_adapter *adapter, if (adapter->discarding) { /* All receives must fit into a single buffer */ - e_dbg("Receive packet consumed multiple buffers\n"); - /* recycle */ - buffer_info->skb = skb; + netdev_dbg(netdev, "Receive packet consumed multiple buffers\n"); + dev_kfree_skb(skb); if (status & E1000_RXD_STAT_EOP) adapter->discarding = false; goto next_desc; } if (unlikely(rx_desc->errors & E1000_RXD_ERR_FRAME_ERR_MASK)) { - u8 last_byte = *(skb->data + length - 1); - if (TBI_ACCEPT(hw, status, rx_desc->errors, length, - last_byte)) { - spin_lock_irqsave(&adapter->stats_lock, flags); - e1000_tbi_adjust_stats(hw, &adapter->stats, - length, skb->data); - spin_unlock_irqrestore(&adapter->stats_lock, - flags); + if (e1000_tbi_should_accept(adapter, status, + rx_desc->errors, + length, data)) { length--; + } else if (netdev->features & NETIF_F_RXALL) { + goto process_skb; } else { - if (netdev->features & NETIF_F_RXALL) - goto process_skb; - /* recycle */ - buffer_info->skb = skb; + dev_kfree_skb(skb); goto next_desc; } } @@ -4327,9 +4450,10 @@ process_skb: */ length -= 4; - e1000_check_copybreak(netdev, buffer_info, length, &skb); - - skb_put(skb, length); + if (buffer_info->rxbuf.data == NULL) + skb_put(skb, length); + else /* copybreak skb */ + skb_trim(skb, length); /* Receive Checksum Offload */ e1000_rx_checksum(adapter, @@ -4375,38 +4499,19 @@ static void e1000_alloc_jumbo_rx_buffers(struct e1000_adapter *adapter, struct e1000_rx_ring *rx_ring, int cleaned_count) { - struct net_device *netdev = adapter->netdev; struct pci_dev *pdev = adapter->pdev; struct e1000_rx_desc *rx_desc; - struct e1000_buffer *buffer_info; - struct sk_buff *skb; + struct e1000_rx_buffer *buffer_info; unsigned int i; - unsigned int bufsz = 256 - 16 /*for skb_reserve */ ; i = rx_ring->next_to_use; buffer_info = &rx_ring->buffer_info[i]; while (cleaned_count--) { - skb = buffer_info->skb; - if (skb) { - skb_trim(skb, 0); - goto check_page; - } - - skb = netdev_alloc_skb_ip_align(netdev, bufsz); - if (unlikely(!skb)) { - /* Better luck next round */ - adapter->alloc_rx_buff_failed++; - break; - } - - buffer_info->skb = skb; - buffer_info->length = adapter->rx_buffer_len; -check_page: /* allocate a new page if necessary */ - if (!buffer_info->page) { - buffer_info->page = alloc_page(GFP_ATOMIC); - if (unlikely(!buffer_info->page)) { + if (!buffer_info->rxbuf.page) { + buffer_info->rxbuf.page = alloc_page(GFP_ATOMIC); + if (unlikely(!buffer_info->rxbuf.page)) { adapter->alloc_rx_buff_failed++; break; } @@ -4414,17 +4519,15 @@ check_page: if (!buffer_info->dma) { buffer_info->dma = dma_map_page(&pdev->dev, - buffer_info->page, 0, - buffer_info->length, + buffer_info->rxbuf.page, 0, + adapter->rx_buffer_len, DMA_FROM_DEVICE); if (dma_mapping_error(&pdev->dev, buffer_info->dma)) { - put_page(buffer_info->page); - dev_kfree_skb(skb); - buffer_info->page = NULL; - buffer_info->skb = NULL; + put_page(buffer_info->rxbuf.page); + buffer_info->rxbuf.page = NULL; buffer_info->dma = 0; adapter->alloc_rx_buff_failed++; - break; /* while !buffer_info->skb */ + break; } } @@ -4446,7 +4549,7 @@ check_page: * applicable for weak-ordered memory model archs, * such as IA-64). */ - wmb(); + dma_wmb(); writel(i, adapter->hw.hw_addr + rx_ring->rdt); } } @@ -4454,17 +4557,17 @@ check_page: /** * e1000_alloc_rx_buffers - Replace used receive buffers; legacy & extended * @adapter: address of board private structure + * @rx_ring: pointer to ring struct + * @cleaned_count: number of new Rx buffers to try to allocate **/ static void e1000_alloc_rx_buffers(struct e1000_adapter *adapter, struct e1000_rx_ring *rx_ring, int cleaned_count) { struct e1000_hw *hw = &adapter->hw; - struct net_device *netdev = adapter->netdev; struct pci_dev *pdev = adapter->pdev; struct e1000_rx_desc *rx_desc; - struct e1000_buffer *buffer_info; - struct sk_buff *skb; + struct e1000_rx_buffer *buffer_info; unsigned int i; unsigned int bufsz = adapter->rx_buffer_len; @@ -4472,57 +4575,52 @@ static void e1000_alloc_rx_buffers(struct e1000_adapter *adapter, buffer_info = &rx_ring->buffer_info[i]; while (cleaned_count--) { - skb = buffer_info->skb; - if (skb) { - skb_trim(skb, 0); - goto map_skb; - } + void *data; + + if (buffer_info->rxbuf.data) + goto skip; - skb = netdev_alloc_skb_ip_align(netdev, bufsz); - if (unlikely(!skb)) { + data = e1000_alloc_frag(adapter); + if (!data) { /* Better luck next round */ adapter->alloc_rx_buff_failed++; break; } /* Fix for errata 23, can't cross 64kB boundary */ - if (!e1000_check_64k_bound(adapter, skb->data, bufsz)) { - struct sk_buff *oldskb = skb; + if (!e1000_check_64k_bound(adapter, data, bufsz)) { + void *olddata = data; e_err(rx_err, "skb align check failed: %u bytes at " - "%p\n", bufsz, skb->data); + "%p\n", bufsz, data); /* Try again, without freeing the previous */ - skb = netdev_alloc_skb_ip_align(netdev, bufsz); + data = e1000_alloc_frag(adapter); /* Failed allocation, critical failure */ - if (!skb) { - dev_kfree_skb(oldskb); + if (!data) { + skb_free_frag(olddata); adapter->alloc_rx_buff_failed++; break; } - if (!e1000_check_64k_bound(adapter, skb->data, bufsz)) { + if (!e1000_check_64k_bound(adapter, data, bufsz)) { /* give up */ - dev_kfree_skb(skb); - dev_kfree_skb(oldskb); + skb_free_frag(data); + skb_free_frag(olddata); adapter->alloc_rx_buff_failed++; - break; /* while !buffer_info->skb */ + break; } /* Use new allocation */ - dev_kfree_skb(oldskb); + skb_free_frag(olddata); } - buffer_info->skb = skb; - buffer_info->length = adapter->rx_buffer_len; -map_skb: buffer_info->dma = dma_map_single(&pdev->dev, - skb->data, - buffer_info->length, + data, + adapter->rx_buffer_len, DMA_FROM_DEVICE); if (dma_mapping_error(&pdev->dev, buffer_info->dma)) { - dev_kfree_skb(skb); - buffer_info->skb = NULL; + skb_free_frag(data); buffer_info->dma = 0; adapter->alloc_rx_buff_failed++; - break; /* while !buffer_info->skb */ + break; } /* XXX if it was allocated cleanly it will never map to a @@ -4536,17 +4634,20 @@ map_skb: e_err(rx_err, "dma align check failed: %u bytes at " "%p\n", adapter->rx_buffer_len, (void *)(unsigned long)buffer_info->dma); - dev_kfree_skb(skb); - buffer_info->skb = NULL; dma_unmap_single(&pdev->dev, buffer_info->dma, adapter->rx_buffer_len, DMA_FROM_DEVICE); + + skb_free_frag(data); + buffer_info->rxbuf.data = NULL; buffer_info->dma = 0; adapter->alloc_rx_buff_failed++; - break; /* while !buffer_info->skb */ + break; } + buffer_info->rxbuf.data = data; + skip: rx_desc = E1000_RX_DESC(*rx_ring, i); rx_desc->buffer_addr = cpu_to_le64(buffer_info->dma); @@ -4565,14 +4666,14 @@ map_skb: * applicable for weak-ordered memory model archs, * such as IA-64). */ - wmb(); + dma_wmb(); writel(i, hw->hw_addr + rx_ring->rdt); } } /** * e1000_smartspeed - Workaround for SmartSpeed on 82541 and 82547 controllers. - * @adapter: + * @adapter: address of board private structure **/ static void e1000_smartspeed(struct e1000_adapter *adapter) { @@ -4589,9 +4690,11 @@ static void e1000_smartspeed(struct e1000_adapter *adapter) * we assume back-to-back */ e1000_read_phy_reg(hw, PHY_1000T_STATUS, &phy_status); - if (!(phy_status & SR_1000T_MS_CONFIG_FAULT)) return; + if (!(phy_status & SR_1000T_MS_CONFIG_FAULT)) + return; e1000_read_phy_reg(hw, PHY_1000T_STATUS, &phy_status); - if (!(phy_status & SR_1000T_MS_CONFIG_FAULT)) return; + if (!(phy_status & SR_1000T_MS_CONFIG_FAULT)) + return; e1000_read_phy_reg(hw, PHY_1000T_CTRL, &phy_ctrl); if (phy_ctrl & CR_1000T_MS_ENABLE) { phy_ctrl &= ~CR_1000T_MS_ENABLE; @@ -4626,10 +4729,10 @@ static void e1000_smartspeed(struct e1000_adapter *adapter) } /** - * e1000_ioctl - - * @netdev: - * @ifreq: - * @cmd: + * e1000_ioctl - handle ioctl calls + * @netdev: pointer to our netdev + * @ifr: pointer to interface request structure + * @cmd: ioctl data **/ static int e1000_ioctl(struct net_device *netdev, struct ifreq *ifr, int cmd) { @@ -4645,9 +4748,9 @@ static int e1000_ioctl(struct net_device *netdev, struct ifreq *ifr, int cmd) /** * e1000_mii_ioctl - - * @netdev: - * @ifreq: - * @cmd: + * @netdev: pointer to our netdev + * @ifr: pointer to interface request structure + * @cmd: ioctl data **/ static int e1000_mii_ioctl(struct net_device *netdev, struct ifreq *ifr, int cmd) @@ -4962,23 +5065,21 @@ static int __e1000_shutdown(struct pci_dev *pdev, bool *enable_wake) struct e1000_hw *hw = &adapter->hw; u32 ctrl, ctrl_ext, rctl, status; u32 wufc = adapter->wol; -#ifdef CONFIG_PM - int retval = 0; -#endif netif_device_detach(netdev); if (netif_running(netdev)) { + int count = E1000_CHECK_RESET_COUNT; + + while (test_bit(__E1000_RESETTING, &adapter->flags) && count--) + usleep_range(10000, 20000); + WARN_ON(test_bit(__E1000_RESETTING, &adapter->flags)); + rtnl_lock(); e1000_down(adapter); + rtnl_unlock(); } -#ifdef CONFIG_PM - retval = pci_save_state(pdev); - if (retval) - return retval; -#endif - status = er32(STATUS); if (status & E1000_STATUS_LU) wufc &= ~E1000_WUFC_LNKC; @@ -5033,42 +5134,32 @@ static int __e1000_shutdown(struct pci_dev *pdev, bool *enable_wake) if (netif_running(netdev)) e1000_free_irq(adapter); - pci_disable_device(pdev); + if (!test_and_set_bit(__E1000_DISABLED, &adapter->flags)) + pci_disable_device(pdev); return 0; } -#ifdef CONFIG_PM -static int e1000_suspend(struct pci_dev *pdev, pm_message_t state) +static int e1000_suspend(struct device *dev) { int retval; + struct pci_dev *pdev = to_pci_dev(dev); bool wake; retval = __e1000_shutdown(pdev, &wake); - if (retval) - return retval; - - if (wake) { - pci_prepare_to_sleep(pdev); - } else { - pci_wake_from_d3(pdev, false); - pci_set_power_state(pdev, PCI_D3hot); - } + device_set_wakeup_enable(dev, wake); - return 0; + return retval; } -static int e1000_resume(struct pci_dev *pdev) +static int e1000_resume(struct device *dev) { + struct pci_dev *pdev = to_pci_dev(dev); struct net_device *netdev = pci_get_drvdata(pdev); struct e1000_adapter *adapter = netdev_priv(netdev); struct e1000_hw *hw = &adapter->hw; u32 err; - pci_set_power_state(pdev, PCI_D0); - pci_restore_state(pdev); - pci_save_state(pdev); - if (adapter->need_ioport) err = pci_enable_device(pdev); else @@ -5077,6 +5168,10 @@ static int e1000_resume(struct pci_dev *pdev) pr_err("Cannot enable PCI device from suspend\n"); return err; } + + /* flush memory to make sure state is correct */ + smp_mb__before_atomic(); + clear_bit(__E1000_DISABLED, &adapter->flags); pci_set_master(pdev); pci_enable_wake(pdev, PCI_D3hot, 0); @@ -5101,7 +5196,6 @@ static int e1000_resume(struct pci_dev *pdev) return 0; } -#endif static void e1000_shutdown(struct pci_dev *pdev) { @@ -5124,8 +5218,8 @@ static void e1000_netpoll(struct net_device *netdev) { struct e1000_adapter *adapter = netdev_priv(netdev); - disable_irq(adapter->pdev->irq); - e1000_intr(adapter->pdev->irq, netdev); + if (disable_hardirq(adapter->pdev->irq)) + e1000_intr(adapter->pdev->irq, netdev); enable_irq(adapter->pdev->irq); } #endif @@ -5144,16 +5238,22 @@ static pci_ers_result_t e1000_io_error_detected(struct pci_dev *pdev, struct net_device *netdev = pci_get_drvdata(pdev); struct e1000_adapter *adapter = netdev_priv(netdev); + rtnl_lock(); netif_device_detach(netdev); - if (state == pci_channel_io_perm_failure) + if (state == pci_channel_io_perm_failure) { + rtnl_unlock(); return PCI_ERS_RESULT_DISCONNECT; + } if (netif_running(netdev)) e1000_down(adapter); - pci_disable_device(pdev); - /* Request a slot slot reset. */ + if (!test_and_set_bit(__E1000_DISABLED, &adapter->flags)) + pci_disable_device(pdev); + rtnl_unlock(); + + /* Request a slot reset. */ return PCI_ERS_RESULT_NEED_RESET; } @@ -5179,6 +5279,10 @@ static pci_ers_result_t e1000_io_slot_reset(struct pci_dev *pdev) pr_err("Cannot re-enable PCI device after reset.\n"); return PCI_ERS_RESULT_DISCONNECT; } + + /* flush memory to make sure state is correct */ + smp_mb__before_atomic(); + clear_bit(__E1000_DISABLED, &adapter->flags); pci_set_master(pdev); pci_enable_wake(pdev, PCI_D3hot, 0); |