internal.h 21 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683
  1. /******************************************************************************
  2. *
  3. * Copyright(c) 2003 - 2015 Intel Corporation. All rights reserved.
  4. * Copyright(c) 2013 - 2015 Intel Mobile Communications GmbH
  5. * Copyright(c) 2016 Intel Deutschland GmbH
  6. *
  7. * Portions of this file are derived from the ipw3945 project, as well
  8. * as portions of the ieee80211 subsystem header files.
  9. *
  10. * This program is free software; you can redistribute it and/or modify it
  11. * under the terms of version 2 of the GNU General Public License as
  12. * published by the Free Software Foundation.
  13. *
  14. * This program is distributed in the hope that it will be useful, but WITHOUT
  15. * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  16. * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
  17. * more details.
  18. *
  19. * You should have received a copy of the GNU General Public License along with
  20. * this program; if not, write to the Free Software Foundation, Inc.,
  21. * 51 Franklin Street, Fifth Floor, Boston, MA 02110, USA
  22. *
  23. * The full GNU General Public License is included in this distribution in the
  24. * file called LICENSE.
  25. *
  26. * Contact Information:
  27. * Intel Linux Wireless <linuxwifi@intel.com>
  28. * Intel Corporation, 5200 N.E. Elam Young Parkway, Hillsboro, OR 97124-6497
  29. *
  30. *****************************************************************************/
  31. #ifndef __iwl_trans_int_pcie_h__
  32. #define __iwl_trans_int_pcie_h__
  33. #include <linux/spinlock.h>
  34. #include <linux/interrupt.h>
  35. #include <linux/skbuff.h>
  36. #include <linux/wait.h>
  37. #include <linux/pci.h>
  38. #include <linux/timer.h>
  39. #include "iwl-fh.h"
  40. #include "iwl-csr.h"
  41. #include "iwl-trans.h"
  42. #include "iwl-debug.h"
  43. #include "iwl-io.h"
  44. #include "iwl-op-mode.h"
  45. /* We need 2 entries for the TX command and header, and another one might
  46. * be needed for potential data in the SKB's head. The remaining ones can
  47. * be used for frags.
  48. */
  49. #define IWL_PCIE_MAX_FRAGS (IWL_NUM_OF_TBS - 3)
  50. /*
  51. * RX related structures and functions
  52. */
  53. #define RX_NUM_QUEUES 1
  54. #define RX_POST_REQ_ALLOC 2
  55. #define RX_CLAIM_REQ_ALLOC 8
  56. #define RX_PENDING_WATERMARK 16
  57. struct iwl_host_cmd;
  58. /*This file includes the declaration that are internal to the
  59. * trans_pcie layer */
  60. /**
  61. * struct iwl_rx_mem_buffer
  62. * @page_dma: bus address of rxb page
  63. * @page: driver's pointer to the rxb page
  64. * @vid: index of this rxb in the global table
  65. */
  66. struct iwl_rx_mem_buffer {
  67. dma_addr_t page_dma;
  68. struct page *page;
  69. u16 vid;
  70. struct list_head list;
  71. };
  72. /**
  73. * struct isr_statistics - interrupt statistics
  74. *
  75. */
  76. struct isr_statistics {
  77. u32 hw;
  78. u32 sw;
  79. u32 err_code;
  80. u32 sch;
  81. u32 alive;
  82. u32 rfkill;
  83. u32 ctkill;
  84. u32 wakeup;
  85. u32 rx;
  86. u32 tx;
  87. u32 unhandled;
  88. };
  89. /**
  90. * struct iwl_rxq - Rx queue
  91. * @id: queue index
  92. * @bd: driver's pointer to buffer of receive buffer descriptors (rbd).
  93. * Address size is 32 bit in pre-9000 devices and 64 bit in 9000 devices.
  94. * @bd_dma: bus address of buffer of receive buffer descriptors (rbd)
  95. * @ubd: driver's pointer to buffer of used receive buffer descriptors (rbd)
  96. * @ubd_dma: physical address of buffer of used receive buffer descriptors (rbd)
  97. * @read: Shared index to newest available Rx buffer
  98. * @write: Shared index to oldest written Rx packet
  99. * @free_count: Number of pre-allocated buffers in rx_free
  100. * @used_count: Number of RBDs handled to allocator to use for allocation
  101. * @write_actual:
  102. * @rx_free: list of RBDs with allocated RB ready for use
  103. * @rx_used: list of RBDs with no RB attached
  104. * @need_update: flag to indicate we need to update read/write index
  105. * @rb_stts: driver's pointer to receive buffer status
  106. * @rb_stts_dma: bus address of receive buffer status
  107. * @lock:
  108. * @queue: actual rx queue. Not used for multi-rx queue.
  109. *
  110. * NOTE: rx_free and rx_used are used as a FIFO for iwl_rx_mem_buffers
  111. */
  112. struct iwl_rxq {
  113. int id;
  114. void *bd;
  115. dma_addr_t bd_dma;
  116. __le32 *used_bd;
  117. dma_addr_t used_bd_dma;
  118. u32 read;
  119. u32 write;
  120. u32 free_count;
  121. u32 used_count;
  122. u32 write_actual;
  123. u32 queue_size;
  124. struct list_head rx_free;
  125. struct list_head rx_used;
  126. bool need_update;
  127. struct iwl_rb_status *rb_stts;
  128. dma_addr_t rb_stts_dma;
  129. spinlock_t lock;
  130. struct napi_struct napi;
  131. struct iwl_rx_mem_buffer *queue[RX_QUEUE_SIZE];
  132. };
  133. /**
  134. * struct iwl_rb_allocator - Rx allocator
  135. * @req_pending: number of requests the allcator had not processed yet
  136. * @req_ready: number of requests honored and ready for claiming
  137. * @rbd_allocated: RBDs with pages allocated and ready to be handled to
  138. * the queue. This is a list of &struct iwl_rx_mem_buffer
  139. * @rbd_empty: RBDs with no page attached for allocator use. This is a list
  140. * of &struct iwl_rx_mem_buffer
  141. * @lock: protects the rbd_allocated and rbd_empty lists
  142. * @alloc_wq: work queue for background calls
  143. * @rx_alloc: work struct for background calls
  144. */
  145. struct iwl_rb_allocator {
  146. atomic_t req_pending;
  147. atomic_t req_ready;
  148. struct list_head rbd_allocated;
  149. struct list_head rbd_empty;
  150. spinlock_t lock;
  151. struct workqueue_struct *alloc_wq;
  152. struct work_struct rx_alloc;
  153. };
  154. struct iwl_dma_ptr {
  155. dma_addr_t dma;
  156. void *addr;
  157. size_t size;
  158. };
  159. /**
  160. * iwl_queue_inc_wrap - increment queue index, wrap back to beginning
  161. * @index -- current index
  162. */
  163. static inline int iwl_queue_inc_wrap(int index)
  164. {
  165. return ++index & (TFD_QUEUE_SIZE_MAX - 1);
  166. }
  167. /**
  168. * iwl_queue_dec_wrap - decrement queue index, wrap back to end
  169. * @index -- current index
  170. */
  171. static inline int iwl_queue_dec_wrap(int index)
  172. {
  173. return --index & (TFD_QUEUE_SIZE_MAX - 1);
  174. }
  175. struct iwl_cmd_meta {
  176. /* only for SYNC commands, iff the reply skb is wanted */
  177. struct iwl_host_cmd *source;
  178. u32 flags;
  179. };
  180. /*
  181. * Generic queue structure
  182. *
  183. * Contains common data for Rx and Tx queues.
  184. *
  185. * Note the difference between TFD_QUEUE_SIZE_MAX and n_window: the hardware
  186. * always assumes 256 descriptors, so TFD_QUEUE_SIZE_MAX is always 256 (unless
  187. * there might be HW changes in the future). For the normal TX
  188. * queues, n_window, which is the size of the software queue data
  189. * is also 256; however, for the command queue, n_window is only
  190. * 32 since we don't need so many commands pending. Since the HW
  191. * still uses 256 BDs for DMA though, TFD_QUEUE_SIZE_MAX stays 256. As a result,
  192. * the software buffers (in the variables @meta, @txb in struct
  193. * iwl_txq) only have 32 entries, while the HW buffers (@tfds in
  194. * the same struct) have 256.
  195. * This means that we end up with the following:
  196. * HW entries: | 0 | ... | N * 32 | ... | N * 32 + 31 | ... | 255 |
  197. * SW entries: | 0 | ... | 31 |
  198. * where N is a number between 0 and 7. This means that the SW
  199. * data is a window overlayed over the HW queue.
  200. */
  201. struct iwl_queue {
  202. int write_ptr; /* 1-st empty entry (index) host_w*/
  203. int read_ptr; /* last used entry (index) host_r*/
  204. /* use for monitoring and recovering the stuck queue */
  205. dma_addr_t dma_addr; /* physical addr for BD's */
  206. int n_window; /* safe queue window */
  207. u32 id;
  208. int low_mark; /* low watermark, resume queue if free
  209. * space more than this */
  210. int high_mark; /* high watermark, stop queue if free
  211. * space less than this */
  212. };
  213. #define TFD_TX_CMD_SLOTS 256
  214. #define TFD_CMD_SLOTS 32
  215. /*
  216. * The FH will write back to the first TB only, so we need
  217. * to copy some data into the buffer regardless of whether
  218. * it should be mapped or not. This indicates how big the
  219. * first TB must be to include the scratch buffer. Since
  220. * the scratch is 4 bytes at offset 12, it's 16 now. If we
  221. * make it bigger then allocations will be bigger and copy
  222. * slower, so that's probably not useful.
  223. */
  224. #define IWL_HCMD_SCRATCHBUF_SIZE 16
  225. struct iwl_pcie_txq_entry {
  226. struct iwl_device_cmd *cmd;
  227. struct sk_buff *skb;
  228. /* buffer to free after command completes */
  229. const void *free_buf;
  230. struct iwl_cmd_meta meta;
  231. };
  232. struct iwl_pcie_txq_scratch_buf {
  233. struct iwl_cmd_header hdr;
  234. u8 buf[8];
  235. __le32 scratch;
  236. };
  237. /**
  238. * struct iwl_txq - Tx Queue for DMA
  239. * @q: generic Rx/Tx queue descriptor
  240. * @tfds: transmit frame descriptors (DMA memory)
  241. * @scratchbufs: start of command headers, including scratch buffers, for
  242. * the writeback -- this is DMA memory and an array holding one buffer
  243. * for each command on the queue
  244. * @scratchbufs_dma: DMA address for the scratchbufs start
  245. * @entries: transmit entries (driver state)
  246. * @lock: queue lock
  247. * @stuck_timer: timer that fires if queue gets stuck
  248. * @trans_pcie: pointer back to transport (for timer)
  249. * @need_update: indicates need to update read/write index
  250. * @active: stores if queue is active
  251. * @ampdu: true if this queue is an ampdu queue for an specific RA/TID
  252. * @wd_timeout: queue watchdog timeout (jiffies) - per queue
  253. * @frozen: tx stuck queue timer is frozen
  254. * @frozen_expiry_remainder: remember how long until the timer fires
  255. *
  256. * A Tx queue consists of circular buffer of BDs (a.k.a. TFDs, transmit frame
  257. * descriptors) and required locking structures.
  258. */
  259. struct iwl_txq {
  260. struct iwl_queue q;
  261. struct iwl_tfd *tfds;
  262. struct iwl_pcie_txq_scratch_buf *scratchbufs;
  263. dma_addr_t scratchbufs_dma;
  264. struct iwl_pcie_txq_entry *entries;
  265. spinlock_t lock;
  266. unsigned long frozen_expiry_remainder;
  267. struct timer_list stuck_timer;
  268. struct iwl_trans_pcie *trans_pcie;
  269. bool need_update;
  270. bool frozen;
  271. u8 active;
  272. bool ampdu;
  273. bool block;
  274. unsigned long wd_timeout;
  275. struct sk_buff_head overflow_q;
  276. };
  277. static inline dma_addr_t
  278. iwl_pcie_get_scratchbuf_dma(struct iwl_txq *txq, int idx)
  279. {
  280. return txq->scratchbufs_dma +
  281. sizeof(struct iwl_pcie_txq_scratch_buf) * idx;
  282. }
  283. struct iwl_tso_hdr_page {
  284. struct page *page;
  285. u8 *pos;
  286. };
  287. /**
  288. * struct iwl_trans_pcie - PCIe transport specific data
  289. * @rxq: all the RX queue data
  290. * @rx_pool: initial pool of iwl_rx_mem_buffer for all the queues
  291. * @global_table: table mapping received VID from hw to rxb
  292. * @rba: allocator for RX replenishing
  293. * @drv - pointer to iwl_drv
  294. * @trans: pointer to the generic transport area
  295. * @scd_base_addr: scheduler sram base address in SRAM
  296. * @scd_bc_tbls: pointer to the byte count table of the scheduler
  297. * @kw: keep warm address
  298. * @pci_dev: basic pci-network driver stuff
  299. * @hw_base: pci hardware address support
  300. * @ucode_write_complete: indicates that the ucode has been copied.
  301. * @ucode_write_waitq: wait queue for uCode load
  302. * @cmd_queue - command queue number
  303. * @rx_buf_size: Rx buffer size
  304. * @bc_table_dword: true if the BC table expects DWORD (as opposed to bytes)
  305. * @scd_set_active: should the transport configure the SCD for HCMD queue
  306. * @wide_cmd_header: true when ucode supports wide command header format
  307. * @sw_csum_tx: if true, then the transport will compute the csum of the TXed
  308. * frame.
  309. * @rx_page_order: page order for receive buffer size
  310. * @reg_lock: protect hw register access
  311. * @mutex: to protect stop_device / start_fw / start_hw
  312. * @cmd_in_flight: true when we have a host command in flight
  313. * @fw_mon_phys: physical address of the buffer for the firmware monitor
  314. * @fw_mon_page: points to the first page of the buffer for the firmware monitor
  315. * @fw_mon_size: size of the buffer for the firmware monitor
  316. * @msix_entries: array of MSI-X entries
  317. * @msix_enabled: true if managed to enable MSI-X
  318. * @allocated_vector: the number of interrupt vector allocated by the OS
  319. * @default_irq_num: default irq for non rx interrupt
  320. * @fh_init_mask: initial unmasked fh causes
  321. * @hw_init_mask: initial unmasked hw causes
  322. * @fh_mask: current unmasked fh causes
  323. * @hw_mask: current unmasked hw causes
  324. */
  325. struct iwl_trans_pcie {
  326. struct iwl_rxq *rxq;
  327. struct iwl_rx_mem_buffer rx_pool[RX_POOL_SIZE];
  328. struct iwl_rx_mem_buffer *global_table[MQ_RX_TABLE_SIZE];
  329. struct iwl_rb_allocator rba;
  330. struct iwl_trans *trans;
  331. struct iwl_drv *drv;
  332. struct net_device napi_dev;
  333. struct __percpu iwl_tso_hdr_page *tso_hdr_page;
  334. /* INT ICT Table */
  335. __le32 *ict_tbl;
  336. dma_addr_t ict_tbl_dma;
  337. int ict_index;
  338. bool use_ict;
  339. bool is_down;
  340. struct isr_statistics isr_stats;
  341. spinlock_t irq_lock;
  342. struct mutex mutex;
  343. u32 inta_mask;
  344. u32 scd_base_addr;
  345. struct iwl_dma_ptr scd_bc_tbls;
  346. struct iwl_dma_ptr kw;
  347. struct iwl_txq *txq;
  348. unsigned long queue_used[BITS_TO_LONGS(IWL_MAX_HW_QUEUES)];
  349. unsigned long queue_stopped[BITS_TO_LONGS(IWL_MAX_HW_QUEUES)];
  350. /* PCI bus related data */
  351. struct pci_dev *pci_dev;
  352. void __iomem *hw_base;
  353. bool ucode_write_complete;
  354. wait_queue_head_t ucode_write_waitq;
  355. wait_queue_head_t wait_command_queue;
  356. wait_queue_head_t d0i3_waitq;
  357. u8 cmd_queue;
  358. u8 cmd_fifo;
  359. unsigned int cmd_q_wdg_timeout;
  360. u8 n_no_reclaim_cmds;
  361. u8 no_reclaim_cmds[MAX_NO_RECLAIM_CMDS];
  362. enum iwl_amsdu_size rx_buf_size;
  363. bool bc_table_dword;
  364. bool scd_set_active;
  365. bool wide_cmd_header;
  366. bool sw_csum_tx;
  367. u32 rx_page_order;
  368. /*protect hw register */
  369. spinlock_t reg_lock;
  370. bool cmd_hold_nic_awake;
  371. bool ref_cmd_in_flight;
  372. /* protect ref counter */
  373. spinlock_t ref_lock;
  374. u32 ref_count;
  375. dma_addr_t fw_mon_phys;
  376. struct page *fw_mon_page;
  377. u32 fw_mon_size;
  378. struct msix_entry msix_entries[IWL_MAX_RX_HW_QUEUES];
  379. bool msix_enabled;
  380. u32 allocated_vector;
  381. u32 default_irq_num;
  382. u32 fh_init_mask;
  383. u32 hw_init_mask;
  384. u32 fh_mask;
  385. u32 hw_mask;
  386. };
  387. static inline struct iwl_trans_pcie *
  388. IWL_TRANS_GET_PCIE_TRANS(struct iwl_trans *trans)
  389. {
  390. return (void *)trans->trans_specific;
  391. }
  392. static inline struct iwl_trans *
  393. iwl_trans_pcie_get_trans(struct iwl_trans_pcie *trans_pcie)
  394. {
  395. return container_of((void *)trans_pcie, struct iwl_trans,
  396. trans_specific);
  397. }
  398. /*
  399. * Convention: trans API functions: iwl_trans_pcie_XXX
  400. * Other functions: iwl_pcie_XXX
  401. */
  402. struct iwl_trans *iwl_trans_pcie_alloc(struct pci_dev *pdev,
  403. const struct pci_device_id *ent,
  404. const struct iwl_cfg *cfg);
  405. void iwl_trans_pcie_free(struct iwl_trans *trans);
  406. /*****************************************************
  407. * RX
  408. ******************************************************/
  409. int iwl_pcie_rx_init(struct iwl_trans *trans);
  410. irqreturn_t iwl_pcie_msix_isr(int irq, void *data);
  411. irqreturn_t iwl_pcie_irq_handler(int irq, void *dev_id);
  412. irqreturn_t iwl_pcie_irq_msix_handler(int irq, void *dev_id);
  413. irqreturn_t iwl_pcie_irq_rx_msix_handler(int irq, void *dev_id);
  414. int iwl_pcie_rx_stop(struct iwl_trans *trans);
  415. void iwl_pcie_rx_free(struct iwl_trans *trans);
  416. /*****************************************************
  417. * ICT - interrupt handling
  418. ******************************************************/
  419. irqreturn_t iwl_pcie_isr(int irq, void *data);
  420. int iwl_pcie_alloc_ict(struct iwl_trans *trans);
  421. void iwl_pcie_free_ict(struct iwl_trans *trans);
  422. void iwl_pcie_reset_ict(struct iwl_trans *trans);
  423. void iwl_pcie_disable_ict(struct iwl_trans *trans);
  424. /*****************************************************
  425. * TX / HCMD
  426. ******************************************************/
  427. int iwl_pcie_tx_init(struct iwl_trans *trans);
  428. void iwl_pcie_tx_start(struct iwl_trans *trans, u32 scd_base_addr);
  429. int iwl_pcie_tx_stop(struct iwl_trans *trans);
  430. void iwl_pcie_tx_free(struct iwl_trans *trans);
  431. void iwl_trans_pcie_txq_enable(struct iwl_trans *trans, int queue, u16 ssn,
  432. const struct iwl_trans_txq_scd_cfg *cfg,
  433. unsigned int wdg_timeout);
  434. void iwl_trans_pcie_txq_disable(struct iwl_trans *trans, int queue,
  435. bool configure_scd);
  436. int iwl_trans_pcie_tx(struct iwl_trans *trans, struct sk_buff *skb,
  437. struct iwl_device_cmd *dev_cmd, int txq_id);
  438. void iwl_pcie_txq_check_wrptrs(struct iwl_trans *trans);
  439. int iwl_trans_pcie_send_hcmd(struct iwl_trans *trans, struct iwl_host_cmd *cmd);
  440. void iwl_pcie_hcmd_complete(struct iwl_trans *trans,
  441. struct iwl_rx_cmd_buffer *rxb);
  442. void iwl_trans_pcie_reclaim(struct iwl_trans *trans, int txq_id, int ssn,
  443. struct sk_buff_head *skbs);
  444. void iwl_trans_pcie_tx_reset(struct iwl_trans *trans);
  445. void iwl_trans_pcie_ref(struct iwl_trans *trans);
  446. void iwl_trans_pcie_unref(struct iwl_trans *trans);
  447. static inline u16 iwl_pcie_tfd_tb_get_len(struct iwl_tfd *tfd, u8 idx)
  448. {
  449. struct iwl_tfd_tb *tb = &tfd->tbs[idx];
  450. return le16_to_cpu(tb->hi_n_len) >> 4;
  451. }
  452. /*****************************************************
  453. * Error handling
  454. ******************************************************/
  455. void iwl_pcie_dump_csr(struct iwl_trans *trans);
  456. /*****************************************************
  457. * Helpers
  458. ******************************************************/
  459. static inline void iwl_disable_interrupts(struct iwl_trans *trans)
  460. {
  461. struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans);
  462. clear_bit(STATUS_INT_ENABLED, &trans->status);
  463. if (!trans_pcie->msix_enabled) {
  464. /* disable interrupts from uCode/NIC to host */
  465. iwl_write32(trans, CSR_INT_MASK, 0x00000000);
  466. /* acknowledge/clear/reset any interrupts still pending
  467. * from uCode or flow handler (Rx/Tx DMA) */
  468. iwl_write32(trans, CSR_INT, 0xffffffff);
  469. iwl_write32(trans, CSR_FH_INT_STATUS, 0xffffffff);
  470. } else {
  471. /* disable all the interrupt we might use */
  472. iwl_write32(trans, CSR_MSIX_FH_INT_MASK_AD,
  473. trans_pcie->fh_init_mask);
  474. iwl_write32(trans, CSR_MSIX_HW_INT_MASK_AD,
  475. trans_pcie->hw_init_mask);
  476. }
  477. IWL_DEBUG_ISR(trans, "Disabled interrupts\n");
  478. }
  479. static inline void iwl_enable_interrupts(struct iwl_trans *trans)
  480. {
  481. struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans);
  482. IWL_DEBUG_ISR(trans, "Enabling interrupts\n");
  483. set_bit(STATUS_INT_ENABLED, &trans->status);
  484. if (!trans_pcie->msix_enabled) {
  485. trans_pcie->inta_mask = CSR_INI_SET_MASK;
  486. iwl_write32(trans, CSR_INT_MASK, trans_pcie->inta_mask);
  487. } else {
  488. /*
  489. * fh/hw_mask keeps all the unmasked causes.
  490. * Unlike msi, in msix cause is enabled when it is unset.
  491. */
  492. trans_pcie->hw_mask = trans_pcie->hw_init_mask;
  493. trans_pcie->fh_mask = trans_pcie->fh_init_mask;
  494. iwl_write32(trans, CSR_MSIX_FH_INT_MASK_AD,
  495. ~trans_pcie->fh_mask);
  496. iwl_write32(trans, CSR_MSIX_HW_INT_MASK_AD,
  497. ~trans_pcie->hw_mask);
  498. }
  499. }
  500. static inline void iwl_enable_hw_int_msk_msix(struct iwl_trans *trans, u32 msk)
  501. {
  502. struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans);
  503. iwl_write32(trans, CSR_MSIX_HW_INT_MASK_AD, ~msk);
  504. trans_pcie->hw_mask = msk;
  505. }
  506. static inline void iwl_enable_fh_int_msk_msix(struct iwl_trans *trans, u32 msk)
  507. {
  508. struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans);
  509. iwl_write32(trans, CSR_MSIX_FH_INT_MASK_AD, ~msk);
  510. trans_pcie->fh_mask = msk;
  511. }
  512. static inline void iwl_enable_fw_load_int(struct iwl_trans *trans)
  513. {
  514. struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans);
  515. IWL_DEBUG_ISR(trans, "Enabling FW load interrupt\n");
  516. if (!trans_pcie->msix_enabled) {
  517. trans_pcie->inta_mask = CSR_INT_BIT_FH_TX;
  518. iwl_write32(trans, CSR_INT_MASK, trans_pcie->inta_mask);
  519. } else {
  520. iwl_write32(trans, CSR_MSIX_HW_INT_MASK_AD,
  521. trans_pcie->hw_init_mask);
  522. iwl_enable_fh_int_msk_msix(trans,
  523. MSIX_FH_INT_CAUSES_D2S_CH0_NUM);
  524. }
  525. }
  526. static inline void iwl_enable_rfkill_int(struct iwl_trans *trans)
  527. {
  528. struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans);
  529. IWL_DEBUG_ISR(trans, "Enabling rfkill interrupt\n");
  530. if (!trans_pcie->msix_enabled) {
  531. trans_pcie->inta_mask = CSR_INT_BIT_RF_KILL;
  532. iwl_write32(trans, CSR_INT_MASK, trans_pcie->inta_mask);
  533. } else {
  534. iwl_write32(trans, CSR_MSIX_FH_INT_MASK_AD,
  535. trans_pcie->fh_init_mask);
  536. iwl_enable_hw_int_msk_msix(trans,
  537. MSIX_HW_INT_CAUSES_REG_RF_KILL);
  538. }
  539. }
  540. static inline void iwl_wake_queue(struct iwl_trans *trans,
  541. struct iwl_txq *txq)
  542. {
  543. struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans);
  544. if (test_and_clear_bit(txq->q.id, trans_pcie->queue_stopped)) {
  545. IWL_DEBUG_TX_QUEUES(trans, "Wake hwq %d\n", txq->q.id);
  546. iwl_op_mode_queue_not_full(trans->op_mode, txq->q.id);
  547. }
  548. }
  549. static inline void iwl_stop_queue(struct iwl_trans *trans,
  550. struct iwl_txq *txq)
  551. {
  552. struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans);
  553. if (!test_and_set_bit(txq->q.id, trans_pcie->queue_stopped)) {
  554. iwl_op_mode_queue_full(trans->op_mode, txq->q.id);
  555. IWL_DEBUG_TX_QUEUES(trans, "Stop hwq %d\n", txq->q.id);
  556. } else
  557. IWL_DEBUG_TX_QUEUES(trans, "hwq %d already stopped\n",
  558. txq->q.id);
  559. }
  560. static inline bool iwl_queue_used(const struct iwl_queue *q, int i)
  561. {
  562. return q->write_ptr >= q->read_ptr ?
  563. (i >= q->read_ptr && i < q->write_ptr) :
  564. !(i < q->read_ptr && i >= q->write_ptr);
  565. }
  566. static inline u8 get_cmd_index(struct iwl_queue *q, u32 index)
  567. {
  568. return index & (q->n_window - 1);
  569. }
  570. static inline bool iwl_is_rfkill_set(struct iwl_trans *trans)
  571. {
  572. return !(iwl_read32(trans, CSR_GP_CNTRL) &
  573. CSR_GP_CNTRL_REG_FLAG_HW_RF_KILL_SW);
  574. }
  575. static inline void __iwl_trans_pcie_set_bits_mask(struct iwl_trans *trans,
  576. u32 reg, u32 mask, u32 value)
  577. {
  578. u32 v;
  579. #ifdef CONFIG_IWLWIFI_DEBUG
  580. WARN_ON_ONCE(value & ~mask);
  581. #endif
  582. v = iwl_read32(trans, reg);
  583. v &= ~mask;
  584. v |= value;
  585. iwl_write32(trans, reg, v);
  586. }
  587. static inline void __iwl_trans_pcie_clear_bit(struct iwl_trans *trans,
  588. u32 reg, u32 mask)
  589. {
  590. __iwl_trans_pcie_set_bits_mask(trans, reg, mask, 0);
  591. }
  592. static inline void __iwl_trans_pcie_set_bit(struct iwl_trans *trans,
  593. u32 reg, u32 mask)
  594. {
  595. __iwl_trans_pcie_set_bits_mask(trans, reg, mask, mask);
  596. }
  597. void iwl_trans_pcie_rf_kill(struct iwl_trans *trans, bool state);
  598. #ifdef CONFIG_IWLWIFI_DEBUGFS
  599. int iwl_trans_pcie_dbgfs_register(struct iwl_trans *trans);
  600. #else
  601. static inline int iwl_trans_pcie_dbgfs_register(struct iwl_trans *trans)
  602. {
  603. return 0;
  604. }
  605. #endif
  606. int iwl_pci_fw_exit_d0i3(struct iwl_trans *trans);
  607. int iwl_pci_fw_enter_d0i3(struct iwl_trans *trans);
  608. #endif /* __iwl_trans_int_pcie_h__ */