pci-ioda.c 86 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975976977978979980981982983984985986987988989990991992993994995996997998999100010011002100310041005100610071008100910101011101210131014101510161017101810191020102110221023102410251026102710281029103010311032103310341035103610371038103910401041104210431044104510461047104810491050105110521053105410551056105710581059106010611062106310641065106610671068106910701071107210731074107510761077107810791080108110821083108410851086108710881089109010911092109310941095109610971098109911001101110211031104110511061107110811091110111111121113111411151116111711181119112011211122112311241125112611271128112911301131113211331134113511361137113811391140114111421143114411451146114711481149115011511152115311541155115611571158115911601161116211631164116511661167116811691170117111721173117411751176117711781179118011811182118311841185118611871188118911901191119211931194119511961197119811991200120112021203120412051206120712081209121012111212121312141215121612171218121912201221122212231224122512261227122812291230123112321233123412351236123712381239124012411242124312441245124612471248124912501251125212531254125512561257125812591260126112621263126412651266126712681269127012711272127312741275127612771278127912801281128212831284128512861287128812891290129112921293129412951296129712981299130013011302130313041305130613071308130913101311131213131314131513161317131813191320132113221323132413251326132713281329133013311332133313341335133613371338133913401341134213431344134513461347134813491350135113521353135413551356135713581359136013611362136313641365136613671368136913701371137213731374137513761377137813791380138113821383138413851386138713881389139013911392139313941395139613971398139914001401140214031404140514061407140814091410141114121413141414151416141714181419142014211422142314241425142614271428142914301431143214331434143514361437143814391440144114421443144414451446144714481449145014511452145314541455145614571458145914601461146214631464146514661467146814691470147114721473147414751476147714781479148014811482148314841485148614871488148914901491149214931494149514961497149814991500150115021503150415051506150715081509151015111512151315141515151615171518151915201521152215231524152515261527152815291530153115321533153415351536153715381539154015411542154315441545154615471548154915501551155215531554155515561557155815591560156115621563156415651566156715681569157015711572157315741575157615771578157915801581158215831584158515861587158815891590159115921593159415951596159715981599160016011602160316041605160616071608160916101611161216131614161516161617161816191620162116221623162416251626162716281629163016311632163316341635163616371638163916401641164216431644164516461647164816491650165116521653165416551656165716581659166016611662166316641665166616671668166916701671167216731674167516761677167816791680168116821683168416851686168716881689169016911692169316941695169616971698169917001701170217031704170517061707170817091710171117121713171417151716171717181719172017211722172317241725172617271728172917301731173217331734173517361737173817391740174117421743174417451746174717481749175017511752175317541755175617571758175917601761176217631764176517661767176817691770177117721773177417751776177717781779178017811782178317841785178617871788178917901791179217931794179517961797179817991800180118021803180418051806180718081809181018111812181318141815181618171818181918201821182218231824182518261827182818291830183118321833183418351836183718381839184018411842184318441845184618471848184918501851185218531854185518561857185818591860186118621863186418651866186718681869187018711872187318741875187618771878187918801881188218831884188518861887188818891890189118921893189418951896189718981899190019011902190319041905190619071908190919101911191219131914191519161917191819191920192119221923192419251926192719281929193019311932193319341935193619371938193919401941194219431944194519461947194819491950195119521953195419551956195719581959196019611962196319641965196619671968196919701971197219731974197519761977197819791980198119821983198419851986198719881989199019911992199319941995199619971998199920002001200220032004200520062007200820092010201120122013201420152016201720182019202020212022202320242025202620272028202920302031203220332034203520362037203820392040204120422043204420452046204720482049205020512052205320542055205620572058205920602061206220632064206520662067206820692070207120722073207420752076207720782079208020812082208320842085208620872088208920902091209220932094209520962097209820992100210121022103210421052106210721082109211021112112211321142115211621172118211921202121212221232124212521262127212821292130213121322133213421352136213721382139214021412142214321442145214621472148214921502151215221532154215521562157215821592160216121622163216421652166216721682169217021712172217321742175217621772178217921802181218221832184218521862187218821892190219121922193219421952196219721982199220022012202220322042205220622072208220922102211221222132214221522162217221822192220222122222223222422252226222722282229223022312232223322342235223622372238223922402241224222432244224522462247224822492250225122522253225422552256225722582259226022612262226322642265226622672268226922702271227222732274227522762277227822792280228122822283228422852286228722882289229022912292229322942295229622972298229923002301230223032304230523062307230823092310231123122313231423152316231723182319232023212322232323242325232623272328232923302331233223332334233523362337233823392340234123422343234423452346234723482349235023512352235323542355235623572358235923602361236223632364236523662367236823692370237123722373237423752376237723782379238023812382238323842385238623872388238923902391239223932394239523962397239823992400240124022403240424052406240724082409241024112412241324142415241624172418241924202421242224232424242524262427242824292430243124322433243424352436243724382439244024412442244324442445244624472448244924502451245224532454245524562457245824592460246124622463246424652466246724682469247024712472247324742475247624772478247924802481248224832484248524862487248824892490249124922493249424952496249724982499250025012502250325042505250625072508250925102511251225132514251525162517251825192520252125222523252425252526252725282529253025312532253325342535253625372538253925402541254225432544254525462547254825492550255125522553255425552556255725582559256025612562256325642565256625672568256925702571257225732574257525762577257825792580258125822583258425852586258725882589259025912592259325942595259625972598259926002601260226032604260526062607260826092610261126122613261426152616261726182619262026212622262326242625262626272628262926302631263226332634263526362637263826392640264126422643264426452646264726482649265026512652265326542655265626572658265926602661266226632664266526662667266826692670267126722673267426752676267726782679268026812682268326842685268626872688268926902691269226932694269526962697269826992700270127022703270427052706270727082709271027112712271327142715271627172718271927202721272227232724272527262727272827292730273127322733273427352736273727382739274027412742274327442745274627472748274927502751275227532754275527562757275827592760276127622763276427652766276727682769277027712772277327742775277627772778277927802781278227832784278527862787278827892790279127922793279427952796279727982799280028012802280328042805280628072808280928102811281228132814281528162817281828192820282128222823282428252826282728282829283028312832283328342835283628372838283928402841284228432844284528462847284828492850285128522853285428552856285728582859286028612862286328642865286628672868286928702871287228732874287528762877287828792880288128822883288428852886288728882889289028912892289328942895289628972898289929002901290229032904290529062907290829092910291129122913291429152916291729182919292029212922292329242925292629272928292929302931293229332934293529362937293829392940294129422943294429452946294729482949295029512952295329542955295629572958295929602961296229632964296529662967296829692970297129722973297429752976297729782979298029812982298329842985298629872988298929902991299229932994299529962997299829993000300130023003300430053006300730083009301030113012301330143015301630173018301930203021302230233024302530263027302830293030303130323033303430353036303730383039304030413042304330443045304630473048304930503051305230533054305530563057305830593060306130623063306430653066306730683069307030713072307330743075307630773078307930803081308230833084308530863087308830893090309130923093309430953096309730983099310031013102310331043105310631073108310931103111311231133114311531163117311831193120312131223123312431253126312731283129313031313132313331343135313631373138313931403141314231433144314531463147314831493150315131523153315431553156315731583159316031613162316331643165316631673168316931703171317231733174317531763177317831793180318131823183318431853186318731883189319031913192319331943195319631973198319932003201320232033204320532063207320832093210321132123213321432153216321732183219322032213222322332243225322632273228322932303231323232333234323532363237323832393240324132423243324432453246324732483249325032513252325332543255325632573258325932603261326232633264326532663267326832693270327132723273327432753276
  1. /*
  2. * Support PCI/PCIe on PowerNV platforms
  3. *
  4. * Copyright 2011 Benjamin Herrenschmidt, IBM Corp.
  5. *
  6. * This program is free software; you can redistribute it and/or
  7. * modify it under the terms of the GNU General Public License
  8. * as published by the Free Software Foundation; either version
  9. * 2 of the License, or (at your option) any later version.
  10. */
  11. #undef DEBUG
  12. #include <linux/kernel.h>
  13. #include <linux/pci.h>
  14. #include <linux/crash_dump.h>
  15. #include <linux/debugfs.h>
  16. #include <linux/delay.h>
  17. #include <linux/string.h>
  18. #include <linux/init.h>
  19. #include <linux/bootmem.h>
  20. #include <linux/irq.h>
  21. #include <linux/io.h>
  22. #include <linux/msi.h>
  23. #include <linux/memblock.h>
  24. #include <linux/iommu.h>
  25. #include <linux/rculist.h>
  26. #include <linux/sizes.h>
  27. #include <asm/sections.h>
  28. #include <asm/io.h>
  29. #include <asm/prom.h>
  30. #include <asm/pci-bridge.h>
  31. #include <asm/machdep.h>
  32. #include <asm/msi_bitmap.h>
  33. #include <asm/ppc-pci.h>
  34. #include <asm/opal.h>
  35. #include <asm/iommu.h>
  36. #include <asm/tce.h>
  37. #include <asm/xics.h>
  38. #include <asm/debug.h>
  39. #include <asm/firmware.h>
  40. #include <asm/pnv-pci.h>
  41. #include <asm/mmzone.h>
  42. #include <misc/cxl-base.h>
  43. #include "powernv.h"
  44. #include "pci.h"
  45. /* 256M DMA window, 4K TCE pages, 8 bytes TCE */
  46. #define TCE32_TABLE_SIZE ((0x10000000 / 0x1000) * 8)
  47. #define POWERNV_IOMMU_DEFAULT_LEVELS 1
  48. #define POWERNV_IOMMU_MAX_LEVELS 5
  49. static void pnv_pci_ioda2_table_free_pages(struct iommu_table *tbl);
  50. static void pe_level_printk(const struct pnv_ioda_pe *pe, const char *level,
  51. const char *fmt, ...)
  52. {
  53. struct va_format vaf;
  54. va_list args;
  55. char pfix[32];
  56. va_start(args, fmt);
  57. vaf.fmt = fmt;
  58. vaf.va = &args;
  59. if (pe->flags & PNV_IODA_PE_DEV)
  60. strlcpy(pfix, dev_name(&pe->pdev->dev), sizeof(pfix));
  61. else if (pe->flags & (PNV_IODA_PE_BUS | PNV_IODA_PE_BUS_ALL))
  62. sprintf(pfix, "%04x:%02x ",
  63. pci_domain_nr(pe->pbus), pe->pbus->number);
  64. #ifdef CONFIG_PCI_IOV
  65. else if (pe->flags & PNV_IODA_PE_VF)
  66. sprintf(pfix, "%04x:%02x:%2x.%d",
  67. pci_domain_nr(pe->parent_dev->bus),
  68. (pe->rid & 0xff00) >> 8,
  69. PCI_SLOT(pe->rid), PCI_FUNC(pe->rid));
  70. #endif /* CONFIG_PCI_IOV*/
  71. printk("%spci %s: [PE# %.3d] %pV",
  72. level, pfix, pe->pe_number, &vaf);
  73. va_end(args);
  74. }
  75. #define pe_err(pe, fmt, ...) \
  76. pe_level_printk(pe, KERN_ERR, fmt, ##__VA_ARGS__)
  77. #define pe_warn(pe, fmt, ...) \
  78. pe_level_printk(pe, KERN_WARNING, fmt, ##__VA_ARGS__)
  79. #define pe_info(pe, fmt, ...) \
  80. pe_level_printk(pe, KERN_INFO, fmt, ##__VA_ARGS__)
  81. static bool pnv_iommu_bypass_disabled __read_mostly;
  82. static int __init iommu_setup(char *str)
  83. {
  84. if (!str)
  85. return -EINVAL;
  86. while (*str) {
  87. if (!strncmp(str, "nobypass", 8)) {
  88. pnv_iommu_bypass_disabled = true;
  89. pr_info("PowerNV: IOMMU bypass window disabled.\n");
  90. break;
  91. }
  92. str += strcspn(str, ",");
  93. if (*str == ',')
  94. str++;
  95. }
  96. return 0;
  97. }
  98. early_param("iommu", iommu_setup);
  99. /*
  100. * stdcix is only supposed to be used in hypervisor real mode as per
  101. * the architecture spec
  102. */
  103. static inline void __raw_rm_writeq(u64 val, volatile void __iomem *paddr)
  104. {
  105. __asm__ __volatile__("stdcix %0,0,%1"
  106. : : "r" (val), "r" (paddr) : "memory");
  107. }
  108. static inline bool pnv_pci_is_mem_pref_64(unsigned long flags)
  109. {
  110. return ((flags & (IORESOURCE_MEM_64 | IORESOURCE_PREFETCH)) ==
  111. (IORESOURCE_MEM_64 | IORESOURCE_PREFETCH));
  112. }
  113. static void pnv_ioda_reserve_pe(struct pnv_phb *phb, int pe_no)
  114. {
  115. if (!(pe_no >= 0 && pe_no < phb->ioda.total_pe)) {
  116. pr_warn("%s: Invalid PE %d on PHB#%x\n",
  117. __func__, pe_no, phb->hose->global_number);
  118. return;
  119. }
  120. if (test_and_set_bit(pe_no, phb->ioda.pe_alloc)) {
  121. pr_warn("%s: PE %d was assigned on PHB#%x\n",
  122. __func__, pe_no, phb->hose->global_number);
  123. return;
  124. }
  125. phb->ioda.pe_array[pe_no].phb = phb;
  126. phb->ioda.pe_array[pe_no].pe_number = pe_no;
  127. }
  128. static int pnv_ioda_alloc_pe(struct pnv_phb *phb)
  129. {
  130. unsigned long pe;
  131. do {
  132. pe = find_next_zero_bit(phb->ioda.pe_alloc,
  133. phb->ioda.total_pe, 0);
  134. if (pe >= phb->ioda.total_pe)
  135. return IODA_INVALID_PE;
  136. } while(test_and_set_bit(pe, phb->ioda.pe_alloc));
  137. phb->ioda.pe_array[pe].phb = phb;
  138. phb->ioda.pe_array[pe].pe_number = pe;
  139. return pe;
  140. }
  141. static void pnv_ioda_free_pe(struct pnv_phb *phb, int pe)
  142. {
  143. WARN_ON(phb->ioda.pe_array[pe].pdev);
  144. memset(&phb->ioda.pe_array[pe], 0, sizeof(struct pnv_ioda_pe));
  145. clear_bit(pe, phb->ioda.pe_alloc);
  146. }
  147. /* The default M64 BAR is shared by all PEs */
  148. static int pnv_ioda2_init_m64(struct pnv_phb *phb)
  149. {
  150. const char *desc;
  151. struct resource *r;
  152. s64 rc;
  153. /* Configure the default M64 BAR */
  154. rc = opal_pci_set_phb_mem_window(phb->opal_id,
  155. OPAL_M64_WINDOW_TYPE,
  156. phb->ioda.m64_bar_idx,
  157. phb->ioda.m64_base,
  158. 0, /* unused */
  159. phb->ioda.m64_size);
  160. if (rc != OPAL_SUCCESS) {
  161. desc = "configuring";
  162. goto fail;
  163. }
  164. /* Enable the default M64 BAR */
  165. rc = opal_pci_phb_mmio_enable(phb->opal_id,
  166. OPAL_M64_WINDOW_TYPE,
  167. phb->ioda.m64_bar_idx,
  168. OPAL_ENABLE_M64_SPLIT);
  169. if (rc != OPAL_SUCCESS) {
  170. desc = "enabling";
  171. goto fail;
  172. }
  173. /* Mark the M64 BAR assigned */
  174. set_bit(phb->ioda.m64_bar_idx, &phb->ioda.m64_bar_alloc);
  175. /*
  176. * Strip off the segment used by the reserved PE, which is
  177. * expected to be 0 or last one of PE capabicity.
  178. */
  179. r = &phb->hose->mem_resources[1];
  180. if (phb->ioda.reserved_pe == 0)
  181. r->start += phb->ioda.m64_segsize;
  182. else if (phb->ioda.reserved_pe == (phb->ioda.total_pe - 1))
  183. r->end -= phb->ioda.m64_segsize;
  184. else
  185. pr_warn(" Cannot strip M64 segment for reserved PE#%d\n",
  186. phb->ioda.reserved_pe);
  187. return 0;
  188. fail:
  189. pr_warn(" Failure %lld %s M64 BAR#%d\n",
  190. rc, desc, phb->ioda.m64_bar_idx);
  191. opal_pci_phb_mmio_enable(phb->opal_id,
  192. OPAL_M64_WINDOW_TYPE,
  193. phb->ioda.m64_bar_idx,
  194. OPAL_DISABLE_M64);
  195. return -EIO;
  196. }
  197. static void pnv_ioda2_reserve_m64_pe(struct pnv_phb *phb)
  198. {
  199. resource_size_t sgsz = phb->ioda.m64_segsize;
  200. struct pci_dev *pdev;
  201. struct resource *r;
  202. int base, step, i;
  203. /*
  204. * Root bus always has full M64 range and root port has
  205. * M64 range used in reality. So we're checking root port
  206. * instead of root bus.
  207. */
  208. list_for_each_entry(pdev, &phb->hose->bus->devices, bus_list) {
  209. for (i = 0; i < PCI_BRIDGE_RESOURCE_NUM; i++) {
  210. r = &pdev->resource[PCI_BRIDGE_RESOURCES + i];
  211. if (!r->parent ||
  212. !pnv_pci_is_mem_pref_64(r->flags))
  213. continue;
  214. base = (r->start - phb->ioda.m64_base) / sgsz;
  215. for (step = 0; step < resource_size(r) / sgsz; step++)
  216. pnv_ioda_reserve_pe(phb, base + step);
  217. }
  218. }
  219. }
  220. static int pnv_ioda2_pick_m64_pe(struct pnv_phb *phb,
  221. struct pci_bus *bus, int all)
  222. {
  223. resource_size_t segsz = phb->ioda.m64_segsize;
  224. struct pci_dev *pdev;
  225. struct resource *r;
  226. struct pnv_ioda_pe *master_pe, *pe;
  227. unsigned long size, *pe_alloc;
  228. bool found;
  229. int start, i, j;
  230. /* Root bus shouldn't use M64 */
  231. if (pci_is_root_bus(bus))
  232. return IODA_INVALID_PE;
  233. /* We support only one M64 window on each bus */
  234. found = false;
  235. pci_bus_for_each_resource(bus, r, i) {
  236. if (r && r->parent &&
  237. pnv_pci_is_mem_pref_64(r->flags)) {
  238. found = true;
  239. break;
  240. }
  241. }
  242. /* No M64 window found ? */
  243. if (!found)
  244. return IODA_INVALID_PE;
  245. /* Allocate bitmap */
  246. size = _ALIGN_UP(phb->ioda.total_pe / 8, sizeof(unsigned long));
  247. pe_alloc = kzalloc(size, GFP_KERNEL);
  248. if (!pe_alloc) {
  249. pr_warn("%s: Out of memory !\n",
  250. __func__);
  251. return IODA_INVALID_PE;
  252. }
  253. /*
  254. * Figure out reserved PE numbers by the PE
  255. * the its child PEs.
  256. */
  257. start = (r->start - phb->ioda.m64_base) / segsz;
  258. for (i = 0; i < resource_size(r) / segsz; i++)
  259. set_bit(start + i, pe_alloc);
  260. if (all)
  261. goto done;
  262. /*
  263. * If the PE doesn't cover all subordinate buses,
  264. * we need subtract from reserved PEs for children.
  265. */
  266. list_for_each_entry(pdev, &bus->devices, bus_list) {
  267. if (!pdev->subordinate)
  268. continue;
  269. pci_bus_for_each_resource(pdev->subordinate, r, i) {
  270. if (!r || !r->parent ||
  271. !pnv_pci_is_mem_pref_64(r->flags))
  272. continue;
  273. start = (r->start - phb->ioda.m64_base) / segsz;
  274. for (j = 0; j < resource_size(r) / segsz ; j++)
  275. clear_bit(start + j, pe_alloc);
  276. }
  277. }
  278. /*
  279. * the current bus might not own M64 window and that's all
  280. * contributed by its child buses. For the case, we needn't
  281. * pick M64 dependent PE#.
  282. */
  283. if (bitmap_empty(pe_alloc, phb->ioda.total_pe)) {
  284. kfree(pe_alloc);
  285. return IODA_INVALID_PE;
  286. }
  287. /*
  288. * Figure out the master PE and put all slave PEs to master
  289. * PE's list to form compound PE.
  290. */
  291. done:
  292. master_pe = NULL;
  293. i = -1;
  294. while ((i = find_next_bit(pe_alloc, phb->ioda.total_pe, i + 1)) <
  295. phb->ioda.total_pe) {
  296. pe = &phb->ioda.pe_array[i];
  297. if (!master_pe) {
  298. pe->flags |= PNV_IODA_PE_MASTER;
  299. INIT_LIST_HEAD(&pe->slaves);
  300. master_pe = pe;
  301. } else {
  302. pe->flags |= PNV_IODA_PE_SLAVE;
  303. pe->master = master_pe;
  304. list_add_tail(&pe->list, &master_pe->slaves);
  305. }
  306. }
  307. kfree(pe_alloc);
  308. return master_pe->pe_number;
  309. }
  310. static void __init pnv_ioda_parse_m64_window(struct pnv_phb *phb)
  311. {
  312. struct pci_controller *hose = phb->hose;
  313. struct device_node *dn = hose->dn;
  314. struct resource *res;
  315. const u32 *r;
  316. u64 pci_addr;
  317. /* FIXME: Support M64 for P7IOC */
  318. if (phb->type != PNV_PHB_IODA2) {
  319. pr_info(" Not support M64 window\n");
  320. return;
  321. }
  322. if (!firmware_has_feature(FW_FEATURE_OPALv3)) {
  323. pr_info(" Firmware too old to support M64 window\n");
  324. return;
  325. }
  326. r = of_get_property(dn, "ibm,opal-m64-window", NULL);
  327. if (!r) {
  328. pr_info(" No <ibm,opal-m64-window> on %s\n",
  329. dn->full_name);
  330. return;
  331. }
  332. res = &hose->mem_resources[1];
  333. res->start = of_translate_address(dn, r + 2);
  334. res->end = res->start + of_read_number(r + 4, 2) - 1;
  335. res->flags = (IORESOURCE_MEM | IORESOURCE_MEM_64 | IORESOURCE_PREFETCH);
  336. pci_addr = of_read_number(r, 2);
  337. hose->mem_offset[1] = res->start - pci_addr;
  338. phb->ioda.m64_size = resource_size(res);
  339. phb->ioda.m64_segsize = phb->ioda.m64_size / phb->ioda.total_pe;
  340. phb->ioda.m64_base = pci_addr;
  341. pr_info(" MEM64 0x%016llx..0x%016llx -> 0x%016llx\n",
  342. res->start, res->end, pci_addr);
  343. /* Use last M64 BAR to cover M64 window */
  344. phb->ioda.m64_bar_idx = 15;
  345. phb->init_m64 = pnv_ioda2_init_m64;
  346. phb->reserve_m64_pe = pnv_ioda2_reserve_m64_pe;
  347. phb->pick_m64_pe = pnv_ioda2_pick_m64_pe;
  348. }
  349. static void pnv_ioda_freeze_pe(struct pnv_phb *phb, int pe_no)
  350. {
  351. struct pnv_ioda_pe *pe = &phb->ioda.pe_array[pe_no];
  352. struct pnv_ioda_pe *slave;
  353. s64 rc;
  354. /* Fetch master PE */
  355. if (pe->flags & PNV_IODA_PE_SLAVE) {
  356. pe = pe->master;
  357. if (WARN_ON(!pe || !(pe->flags & PNV_IODA_PE_MASTER)))
  358. return;
  359. pe_no = pe->pe_number;
  360. }
  361. /* Freeze master PE */
  362. rc = opal_pci_eeh_freeze_set(phb->opal_id,
  363. pe_no,
  364. OPAL_EEH_ACTION_SET_FREEZE_ALL);
  365. if (rc != OPAL_SUCCESS) {
  366. pr_warn("%s: Failure %lld freezing PHB#%x-PE#%x\n",
  367. __func__, rc, phb->hose->global_number, pe_no);
  368. return;
  369. }
  370. /* Freeze slave PEs */
  371. if (!(pe->flags & PNV_IODA_PE_MASTER))
  372. return;
  373. list_for_each_entry(slave, &pe->slaves, list) {
  374. rc = opal_pci_eeh_freeze_set(phb->opal_id,
  375. slave->pe_number,
  376. OPAL_EEH_ACTION_SET_FREEZE_ALL);
  377. if (rc != OPAL_SUCCESS)
  378. pr_warn("%s: Failure %lld freezing PHB#%x-PE#%x\n",
  379. __func__, rc, phb->hose->global_number,
  380. slave->pe_number);
  381. }
  382. }
  383. static int pnv_ioda_unfreeze_pe(struct pnv_phb *phb, int pe_no, int opt)
  384. {
  385. struct pnv_ioda_pe *pe, *slave;
  386. s64 rc;
  387. /* Find master PE */
  388. pe = &phb->ioda.pe_array[pe_no];
  389. if (pe->flags & PNV_IODA_PE_SLAVE) {
  390. pe = pe->master;
  391. WARN_ON(!pe || !(pe->flags & PNV_IODA_PE_MASTER));
  392. pe_no = pe->pe_number;
  393. }
  394. /* Clear frozen state for master PE */
  395. rc = opal_pci_eeh_freeze_clear(phb->opal_id, pe_no, opt);
  396. if (rc != OPAL_SUCCESS) {
  397. pr_warn("%s: Failure %lld clear %d on PHB#%x-PE#%x\n",
  398. __func__, rc, opt, phb->hose->global_number, pe_no);
  399. return -EIO;
  400. }
  401. if (!(pe->flags & PNV_IODA_PE_MASTER))
  402. return 0;
  403. /* Clear frozen state for slave PEs */
  404. list_for_each_entry(slave, &pe->slaves, list) {
  405. rc = opal_pci_eeh_freeze_clear(phb->opal_id,
  406. slave->pe_number,
  407. opt);
  408. if (rc != OPAL_SUCCESS) {
  409. pr_warn("%s: Failure %lld clear %d on PHB#%x-PE#%x\n",
  410. __func__, rc, opt, phb->hose->global_number,
  411. slave->pe_number);
  412. return -EIO;
  413. }
  414. }
  415. return 0;
  416. }
  417. static int pnv_ioda_get_pe_state(struct pnv_phb *phb, int pe_no)
  418. {
  419. struct pnv_ioda_pe *slave, *pe;
  420. u8 fstate, state;
  421. __be16 pcierr;
  422. s64 rc;
  423. /* Sanity check on PE number */
  424. if (pe_no < 0 || pe_no >= phb->ioda.total_pe)
  425. return OPAL_EEH_STOPPED_PERM_UNAVAIL;
  426. /*
  427. * Fetch the master PE and the PE instance might be
  428. * not initialized yet.
  429. */
  430. pe = &phb->ioda.pe_array[pe_no];
  431. if (pe->flags & PNV_IODA_PE_SLAVE) {
  432. pe = pe->master;
  433. WARN_ON(!pe || !(pe->flags & PNV_IODA_PE_MASTER));
  434. pe_no = pe->pe_number;
  435. }
  436. /* Check the master PE */
  437. rc = opal_pci_eeh_freeze_status(phb->opal_id, pe_no,
  438. &state, &pcierr, NULL);
  439. if (rc != OPAL_SUCCESS) {
  440. pr_warn("%s: Failure %lld getting "
  441. "PHB#%x-PE#%x state\n",
  442. __func__, rc,
  443. phb->hose->global_number, pe_no);
  444. return OPAL_EEH_STOPPED_TEMP_UNAVAIL;
  445. }
  446. /* Check the slave PE */
  447. if (!(pe->flags & PNV_IODA_PE_MASTER))
  448. return state;
  449. list_for_each_entry(slave, &pe->slaves, list) {
  450. rc = opal_pci_eeh_freeze_status(phb->opal_id,
  451. slave->pe_number,
  452. &fstate,
  453. &pcierr,
  454. NULL);
  455. if (rc != OPAL_SUCCESS) {
  456. pr_warn("%s: Failure %lld getting "
  457. "PHB#%x-PE#%x state\n",
  458. __func__, rc,
  459. phb->hose->global_number, slave->pe_number);
  460. return OPAL_EEH_STOPPED_TEMP_UNAVAIL;
  461. }
  462. /*
  463. * Override the result based on the ascending
  464. * priority.
  465. */
  466. if (fstate > state)
  467. state = fstate;
  468. }
  469. return state;
  470. }
  471. /* Currently those 2 are only used when MSIs are enabled, this will change
  472. * but in the meantime, we need to protect them to avoid warnings
  473. */
  474. #ifdef CONFIG_PCI_MSI
  475. static struct pnv_ioda_pe *pnv_ioda_get_pe(struct pci_dev *dev)
  476. {
  477. struct pci_controller *hose = pci_bus_to_host(dev->bus);
  478. struct pnv_phb *phb = hose->private_data;
  479. struct pci_dn *pdn = pci_get_pdn(dev);
  480. if (!pdn)
  481. return NULL;
  482. if (pdn->pe_number == IODA_INVALID_PE)
  483. return NULL;
  484. return &phb->ioda.pe_array[pdn->pe_number];
  485. }
  486. #endif /* CONFIG_PCI_MSI */
  487. static int pnv_ioda_set_one_peltv(struct pnv_phb *phb,
  488. struct pnv_ioda_pe *parent,
  489. struct pnv_ioda_pe *child,
  490. bool is_add)
  491. {
  492. const char *desc = is_add ? "adding" : "removing";
  493. uint8_t op = is_add ? OPAL_ADD_PE_TO_DOMAIN :
  494. OPAL_REMOVE_PE_FROM_DOMAIN;
  495. struct pnv_ioda_pe *slave;
  496. long rc;
  497. /* Parent PE affects child PE */
  498. rc = opal_pci_set_peltv(phb->opal_id, parent->pe_number,
  499. child->pe_number, op);
  500. if (rc != OPAL_SUCCESS) {
  501. pe_warn(child, "OPAL error %ld %s to parent PELTV\n",
  502. rc, desc);
  503. return -ENXIO;
  504. }
  505. if (!(child->flags & PNV_IODA_PE_MASTER))
  506. return 0;
  507. /* Compound case: parent PE affects slave PEs */
  508. list_for_each_entry(slave, &child->slaves, list) {
  509. rc = opal_pci_set_peltv(phb->opal_id, parent->pe_number,
  510. slave->pe_number, op);
  511. if (rc != OPAL_SUCCESS) {
  512. pe_warn(slave, "OPAL error %ld %s to parent PELTV\n",
  513. rc, desc);
  514. return -ENXIO;
  515. }
  516. }
  517. return 0;
  518. }
  519. static int pnv_ioda_set_peltv(struct pnv_phb *phb,
  520. struct pnv_ioda_pe *pe,
  521. bool is_add)
  522. {
  523. struct pnv_ioda_pe *slave;
  524. struct pci_dev *pdev = NULL;
  525. int ret;
  526. /*
  527. * Clear PE frozen state. If it's master PE, we need
  528. * clear slave PE frozen state as well.
  529. */
  530. if (is_add) {
  531. opal_pci_eeh_freeze_clear(phb->opal_id, pe->pe_number,
  532. OPAL_EEH_ACTION_CLEAR_FREEZE_ALL);
  533. if (pe->flags & PNV_IODA_PE_MASTER) {
  534. list_for_each_entry(slave, &pe->slaves, list)
  535. opal_pci_eeh_freeze_clear(phb->opal_id,
  536. slave->pe_number,
  537. OPAL_EEH_ACTION_CLEAR_FREEZE_ALL);
  538. }
  539. }
  540. /*
  541. * Associate PE in PELT. We need add the PE into the
  542. * corresponding PELT-V as well. Otherwise, the error
  543. * originated from the PE might contribute to other
  544. * PEs.
  545. */
  546. ret = pnv_ioda_set_one_peltv(phb, pe, pe, is_add);
  547. if (ret)
  548. return ret;
  549. /* For compound PEs, any one affects all of them */
  550. if (pe->flags & PNV_IODA_PE_MASTER) {
  551. list_for_each_entry(slave, &pe->slaves, list) {
  552. ret = pnv_ioda_set_one_peltv(phb, slave, pe, is_add);
  553. if (ret)
  554. return ret;
  555. }
  556. }
  557. if (pe->flags & (PNV_IODA_PE_BUS_ALL | PNV_IODA_PE_BUS))
  558. pdev = pe->pbus->self;
  559. else if (pe->flags & PNV_IODA_PE_DEV)
  560. pdev = pe->pdev->bus->self;
  561. #ifdef CONFIG_PCI_IOV
  562. else if (pe->flags & PNV_IODA_PE_VF)
  563. pdev = pe->parent_dev->bus->self;
  564. #endif /* CONFIG_PCI_IOV */
  565. while (pdev) {
  566. struct pci_dn *pdn = pci_get_pdn(pdev);
  567. struct pnv_ioda_pe *parent;
  568. if (pdn && pdn->pe_number != IODA_INVALID_PE) {
  569. parent = &phb->ioda.pe_array[pdn->pe_number];
  570. ret = pnv_ioda_set_one_peltv(phb, parent, pe, is_add);
  571. if (ret)
  572. return ret;
  573. }
  574. pdev = pdev->bus->self;
  575. }
  576. return 0;
  577. }
  578. #ifdef CONFIG_PCI_IOV
  579. static int pnv_ioda_deconfigure_pe(struct pnv_phb *phb, struct pnv_ioda_pe *pe)
  580. {
  581. struct pci_dev *parent;
  582. uint8_t bcomp, dcomp, fcomp;
  583. int64_t rc;
  584. long rid_end, rid;
  585. /* Currently, we just deconfigure VF PE. Bus PE will always there.*/
  586. if (pe->pbus) {
  587. int count;
  588. dcomp = OPAL_IGNORE_RID_DEVICE_NUMBER;
  589. fcomp = OPAL_IGNORE_RID_FUNCTION_NUMBER;
  590. parent = pe->pbus->self;
  591. if (pe->flags & PNV_IODA_PE_BUS_ALL)
  592. count = pe->pbus->busn_res.end - pe->pbus->busn_res.start + 1;
  593. else
  594. count = 1;
  595. switch(count) {
  596. case 1: bcomp = OpalPciBusAll; break;
  597. case 2: bcomp = OpalPciBus7Bits; break;
  598. case 4: bcomp = OpalPciBus6Bits; break;
  599. case 8: bcomp = OpalPciBus5Bits; break;
  600. case 16: bcomp = OpalPciBus4Bits; break;
  601. case 32: bcomp = OpalPciBus3Bits; break;
  602. default:
  603. dev_err(&pe->pbus->dev, "Number of subordinate buses %d unsupported\n",
  604. count);
  605. /* Do an exact match only */
  606. bcomp = OpalPciBusAll;
  607. }
  608. rid_end = pe->rid + (count << 8);
  609. } else {
  610. if (pe->flags & PNV_IODA_PE_VF)
  611. parent = pe->parent_dev;
  612. else
  613. parent = pe->pdev->bus->self;
  614. bcomp = OpalPciBusAll;
  615. dcomp = OPAL_COMPARE_RID_DEVICE_NUMBER;
  616. fcomp = OPAL_COMPARE_RID_FUNCTION_NUMBER;
  617. rid_end = pe->rid + 1;
  618. }
  619. /* Clear the reverse map */
  620. for (rid = pe->rid; rid < rid_end; rid++)
  621. phb->ioda.pe_rmap[rid] = 0;
  622. /* Release from all parents PELT-V */
  623. while (parent) {
  624. struct pci_dn *pdn = pci_get_pdn(parent);
  625. if (pdn && pdn->pe_number != IODA_INVALID_PE) {
  626. rc = opal_pci_set_peltv(phb->opal_id, pdn->pe_number,
  627. pe->pe_number, OPAL_REMOVE_PE_FROM_DOMAIN);
  628. /* XXX What to do in case of error ? */
  629. }
  630. parent = parent->bus->self;
  631. }
  632. opal_pci_eeh_freeze_set(phb->opal_id, pe->pe_number,
  633. OPAL_EEH_ACTION_CLEAR_FREEZE_ALL);
  634. /* Disassociate PE in PELT */
  635. rc = opal_pci_set_peltv(phb->opal_id, pe->pe_number,
  636. pe->pe_number, OPAL_REMOVE_PE_FROM_DOMAIN);
  637. if (rc)
  638. pe_warn(pe, "OPAL error %ld remove self from PELTV\n", rc);
  639. rc = opal_pci_set_pe(phb->opal_id, pe->pe_number, pe->rid,
  640. bcomp, dcomp, fcomp, OPAL_UNMAP_PE);
  641. if (rc)
  642. pe_err(pe, "OPAL error %ld trying to setup PELT table\n", rc);
  643. pe->pbus = NULL;
  644. pe->pdev = NULL;
  645. pe->parent_dev = NULL;
  646. return 0;
  647. }
  648. #endif /* CONFIG_PCI_IOV */
  649. static int pnv_ioda_configure_pe(struct pnv_phb *phb, struct pnv_ioda_pe *pe)
  650. {
  651. struct pci_dev *parent;
  652. uint8_t bcomp, dcomp, fcomp;
  653. long rc, rid_end, rid;
  654. /* Bus validation ? */
  655. if (pe->pbus) {
  656. int count;
  657. dcomp = OPAL_IGNORE_RID_DEVICE_NUMBER;
  658. fcomp = OPAL_IGNORE_RID_FUNCTION_NUMBER;
  659. parent = pe->pbus->self;
  660. if (pe->flags & PNV_IODA_PE_BUS_ALL)
  661. count = pe->pbus->busn_res.end - pe->pbus->busn_res.start + 1;
  662. else
  663. count = 1;
  664. switch(count) {
  665. case 1: bcomp = OpalPciBusAll; break;
  666. case 2: bcomp = OpalPciBus7Bits; break;
  667. case 4: bcomp = OpalPciBus6Bits; break;
  668. case 8: bcomp = OpalPciBus5Bits; break;
  669. case 16: bcomp = OpalPciBus4Bits; break;
  670. case 32: bcomp = OpalPciBus3Bits; break;
  671. default:
  672. dev_err(&pe->pbus->dev, "Number of subordinate buses %d unsupported\n",
  673. count);
  674. /* Do an exact match only */
  675. bcomp = OpalPciBusAll;
  676. }
  677. rid_end = pe->rid + (count << 8);
  678. } else {
  679. #ifdef CONFIG_PCI_IOV
  680. if (pe->flags & PNV_IODA_PE_VF)
  681. parent = pe->parent_dev;
  682. else
  683. #endif /* CONFIG_PCI_IOV */
  684. parent = pe->pdev->bus->self;
  685. bcomp = OpalPciBusAll;
  686. dcomp = OPAL_COMPARE_RID_DEVICE_NUMBER;
  687. fcomp = OPAL_COMPARE_RID_FUNCTION_NUMBER;
  688. rid_end = pe->rid + 1;
  689. }
  690. /*
  691. * Associate PE in PELT. We need add the PE into the
  692. * corresponding PELT-V as well. Otherwise, the error
  693. * originated from the PE might contribute to other
  694. * PEs.
  695. */
  696. rc = opal_pci_set_pe(phb->opal_id, pe->pe_number, pe->rid,
  697. bcomp, dcomp, fcomp, OPAL_MAP_PE);
  698. if (rc) {
  699. pe_err(pe, "OPAL error %ld trying to setup PELT table\n", rc);
  700. return -ENXIO;
  701. }
  702. /* Configure PELTV */
  703. pnv_ioda_set_peltv(phb, pe, true);
  704. /* Setup reverse map */
  705. for (rid = pe->rid; rid < rid_end; rid++)
  706. phb->ioda.pe_rmap[rid] = pe->pe_number;
  707. /* Setup one MVTs on IODA1 */
  708. if (phb->type != PNV_PHB_IODA1) {
  709. pe->mve_number = 0;
  710. goto out;
  711. }
  712. pe->mve_number = pe->pe_number;
  713. rc = opal_pci_set_mve(phb->opal_id, pe->mve_number, pe->pe_number);
  714. if (rc != OPAL_SUCCESS) {
  715. pe_err(pe, "OPAL error %ld setting up MVE %d\n",
  716. rc, pe->mve_number);
  717. pe->mve_number = -1;
  718. } else {
  719. rc = opal_pci_set_mve_enable(phb->opal_id,
  720. pe->mve_number, OPAL_ENABLE_MVE);
  721. if (rc) {
  722. pe_err(pe, "OPAL error %ld enabling MVE %d\n",
  723. rc, pe->mve_number);
  724. pe->mve_number = -1;
  725. }
  726. }
  727. out:
  728. return 0;
  729. }
  730. static void pnv_ioda_link_pe_by_weight(struct pnv_phb *phb,
  731. struct pnv_ioda_pe *pe)
  732. {
  733. struct pnv_ioda_pe *lpe;
  734. list_for_each_entry(lpe, &phb->ioda.pe_dma_list, dma_link) {
  735. if (lpe->dma_weight < pe->dma_weight) {
  736. list_add_tail(&pe->dma_link, &lpe->dma_link);
  737. return;
  738. }
  739. }
  740. list_add_tail(&pe->dma_link, &phb->ioda.pe_dma_list);
  741. }
  742. static unsigned int pnv_ioda_dma_weight(struct pci_dev *dev)
  743. {
  744. /* This is quite simplistic. The "base" weight of a device
  745. * is 10. 0 means no DMA is to be accounted for it.
  746. */
  747. /* If it's a bridge, no DMA */
  748. if (dev->hdr_type != PCI_HEADER_TYPE_NORMAL)
  749. return 0;
  750. /* Reduce the weight of slow USB controllers */
  751. if (dev->class == PCI_CLASS_SERIAL_USB_UHCI ||
  752. dev->class == PCI_CLASS_SERIAL_USB_OHCI ||
  753. dev->class == PCI_CLASS_SERIAL_USB_EHCI)
  754. return 3;
  755. /* Increase the weight of RAID (includes Obsidian) */
  756. if ((dev->class >> 8) == PCI_CLASS_STORAGE_RAID)
  757. return 15;
  758. /* Default */
  759. return 10;
  760. }
  761. #ifdef CONFIG_PCI_IOV
  762. static int pnv_pci_vf_resource_shift(struct pci_dev *dev, int offset)
  763. {
  764. struct pci_dn *pdn = pci_get_pdn(dev);
  765. int i;
  766. struct resource *res, res2;
  767. resource_size_t size;
  768. u16 num_vfs;
  769. if (!dev->is_physfn)
  770. return -EINVAL;
  771. /*
  772. * "offset" is in VFs. The M64 windows are sized so that when they
  773. * are segmented, each segment is the same size as the IOV BAR.
  774. * Each segment is in a separate PE, and the high order bits of the
  775. * address are the PE number. Therefore, each VF's BAR is in a
  776. * separate PE, and changing the IOV BAR start address changes the
  777. * range of PEs the VFs are in.
  778. */
  779. num_vfs = pdn->num_vfs;
  780. for (i = 0; i < PCI_SRIOV_NUM_BARS; i++) {
  781. res = &dev->resource[i + PCI_IOV_RESOURCES];
  782. if (!res->flags || !res->parent)
  783. continue;
  784. if (!pnv_pci_is_mem_pref_64(res->flags))
  785. continue;
  786. /*
  787. * The actual IOV BAR range is determined by the start address
  788. * and the actual size for num_vfs VFs BAR. This check is to
  789. * make sure that after shifting, the range will not overlap
  790. * with another device.
  791. */
  792. size = pci_iov_resource_size(dev, i + PCI_IOV_RESOURCES);
  793. res2.flags = res->flags;
  794. res2.start = res->start + (size * offset);
  795. res2.end = res2.start + (size * num_vfs) - 1;
  796. if (res2.end > res->end) {
  797. dev_err(&dev->dev, "VF BAR%d: %pR would extend past %pR (trying to enable %d VFs shifted by %d)\n",
  798. i, &res2, res, num_vfs, offset);
  799. return -EBUSY;
  800. }
  801. }
  802. /*
  803. * After doing so, there would be a "hole" in the /proc/iomem when
  804. * offset is a positive value. It looks like the device return some
  805. * mmio back to the system, which actually no one could use it.
  806. */
  807. for (i = 0; i < PCI_SRIOV_NUM_BARS; i++) {
  808. res = &dev->resource[i + PCI_IOV_RESOURCES];
  809. if (!res->flags || !res->parent)
  810. continue;
  811. if (!pnv_pci_is_mem_pref_64(res->flags))
  812. continue;
  813. size = pci_iov_resource_size(dev, i + PCI_IOV_RESOURCES);
  814. res2 = *res;
  815. res->start += size * offset;
  816. dev_info(&dev->dev, "VF BAR%d: %pR shifted to %pR (enabling %d VFs shifted by %d)\n",
  817. i, &res2, res, num_vfs, offset);
  818. pci_update_resource(dev, i + PCI_IOV_RESOURCES);
  819. }
  820. return 0;
  821. }
  822. #endif /* CONFIG_PCI_IOV */
  823. #if 0
  824. static struct pnv_ioda_pe *pnv_ioda_setup_dev_PE(struct pci_dev *dev)
  825. {
  826. struct pci_controller *hose = pci_bus_to_host(dev->bus);
  827. struct pnv_phb *phb = hose->private_data;
  828. struct pci_dn *pdn = pci_get_pdn(dev);
  829. struct pnv_ioda_pe *pe;
  830. int pe_num;
  831. if (!pdn) {
  832. pr_err("%s: Device tree node not associated properly\n",
  833. pci_name(dev));
  834. return NULL;
  835. }
  836. if (pdn->pe_number != IODA_INVALID_PE)
  837. return NULL;
  838. /* PE#0 has been pre-set */
  839. if (dev->bus->number == 0)
  840. pe_num = 0;
  841. else
  842. pe_num = pnv_ioda_alloc_pe(phb);
  843. if (pe_num == IODA_INVALID_PE) {
  844. pr_warning("%s: Not enough PE# available, disabling device\n",
  845. pci_name(dev));
  846. return NULL;
  847. }
  848. /* NOTE: We get only one ref to the pci_dev for the pdn, not for the
  849. * pointer in the PE data structure, both should be destroyed at the
  850. * same time. However, this needs to be looked at more closely again
  851. * once we actually start removing things (Hotplug, SR-IOV, ...)
  852. *
  853. * At some point we want to remove the PDN completely anyways
  854. */
  855. pe = &phb->ioda.pe_array[pe_num];
  856. pci_dev_get(dev);
  857. pdn->pcidev = dev;
  858. pdn->pe_number = pe_num;
  859. pe->pdev = dev;
  860. pe->pbus = NULL;
  861. pe->tce32_seg = -1;
  862. pe->mve_number = -1;
  863. pe->rid = dev->bus->number << 8 | pdn->devfn;
  864. pe_info(pe, "Associated device to PE\n");
  865. if (pnv_ioda_configure_pe(phb, pe)) {
  866. /* XXX What do we do here ? */
  867. if (pe_num)
  868. pnv_ioda_free_pe(phb, pe_num);
  869. pdn->pe_number = IODA_INVALID_PE;
  870. pe->pdev = NULL;
  871. pci_dev_put(dev);
  872. return NULL;
  873. }
  874. /* Assign a DMA weight to the device */
  875. pe->dma_weight = pnv_ioda_dma_weight(dev);
  876. if (pe->dma_weight != 0) {
  877. phb->ioda.dma_weight += pe->dma_weight;
  878. phb->ioda.dma_pe_count++;
  879. }
  880. /* Link the PE */
  881. pnv_ioda_link_pe_by_weight(phb, pe);
  882. return pe;
  883. }
  884. #endif /* Useful for SRIOV case */
  885. static void pnv_ioda_setup_same_PE(struct pci_bus *bus, struct pnv_ioda_pe *pe)
  886. {
  887. struct pci_dev *dev;
  888. list_for_each_entry(dev, &bus->devices, bus_list) {
  889. struct pci_dn *pdn = pci_get_pdn(dev);
  890. if (pdn == NULL) {
  891. pr_warn("%s: No device node associated with device !\n",
  892. pci_name(dev));
  893. continue;
  894. }
  895. pdn->pe_number = pe->pe_number;
  896. pe->dma_weight += pnv_ioda_dma_weight(dev);
  897. if ((pe->flags & PNV_IODA_PE_BUS_ALL) && dev->subordinate)
  898. pnv_ioda_setup_same_PE(dev->subordinate, pe);
  899. }
  900. }
  901. /*
  902. * There're 2 types of PCI bus sensitive PEs: One that is compromised of
  903. * single PCI bus. Another one that contains the primary PCI bus and its
  904. * subordinate PCI devices and buses. The second type of PE is normally
  905. * orgiriated by PCIe-to-PCI bridge or PLX switch downstream ports.
  906. */
  907. static void pnv_ioda_setup_bus_PE(struct pci_bus *bus, int all)
  908. {
  909. struct pci_controller *hose = pci_bus_to_host(bus);
  910. struct pnv_phb *phb = hose->private_data;
  911. struct pnv_ioda_pe *pe;
  912. int pe_num = IODA_INVALID_PE;
  913. /* Check if PE is determined by M64 */
  914. if (phb->pick_m64_pe)
  915. pe_num = phb->pick_m64_pe(phb, bus, all);
  916. /* The PE number isn't pinned by M64 */
  917. if (pe_num == IODA_INVALID_PE)
  918. pe_num = pnv_ioda_alloc_pe(phb);
  919. if (pe_num == IODA_INVALID_PE) {
  920. pr_warning("%s: Not enough PE# available for PCI bus %04x:%02x\n",
  921. __func__, pci_domain_nr(bus), bus->number);
  922. return;
  923. }
  924. pe = &phb->ioda.pe_array[pe_num];
  925. pe->flags |= (all ? PNV_IODA_PE_BUS_ALL : PNV_IODA_PE_BUS);
  926. pe->pbus = bus;
  927. pe->pdev = NULL;
  928. pe->tce32_seg = -1;
  929. pe->mve_number = -1;
  930. pe->rid = bus->busn_res.start << 8;
  931. pe->dma_weight = 0;
  932. if (all)
  933. pe_info(pe, "Secondary bus %d..%d associated with PE#%d\n",
  934. bus->busn_res.start, bus->busn_res.end, pe_num);
  935. else
  936. pe_info(pe, "Secondary bus %d associated with PE#%d\n",
  937. bus->busn_res.start, pe_num);
  938. if (pnv_ioda_configure_pe(phb, pe)) {
  939. /* XXX What do we do here ? */
  940. if (pe_num)
  941. pnv_ioda_free_pe(phb, pe_num);
  942. pe->pbus = NULL;
  943. return;
  944. }
  945. /* Associate it with all child devices */
  946. pnv_ioda_setup_same_PE(bus, pe);
  947. /* Put PE to the list */
  948. list_add_tail(&pe->list, &phb->ioda.pe_list);
  949. /* Account for one DMA PE if at least one DMA capable device exist
  950. * below the bridge
  951. */
  952. if (pe->dma_weight != 0) {
  953. phb->ioda.dma_weight += pe->dma_weight;
  954. phb->ioda.dma_pe_count++;
  955. }
  956. /* Link the PE */
  957. pnv_ioda_link_pe_by_weight(phb, pe);
  958. }
  959. static void pnv_ioda_setup_PEs(struct pci_bus *bus)
  960. {
  961. struct pci_dev *dev;
  962. pnv_ioda_setup_bus_PE(bus, 0);
  963. list_for_each_entry(dev, &bus->devices, bus_list) {
  964. if (dev->subordinate) {
  965. if (pci_pcie_type(dev) == PCI_EXP_TYPE_PCI_BRIDGE)
  966. pnv_ioda_setup_bus_PE(dev->subordinate, 1);
  967. else
  968. pnv_ioda_setup_PEs(dev->subordinate);
  969. }
  970. }
  971. }
  972. /*
  973. * Configure PEs so that the downstream PCI buses and devices
  974. * could have their associated PE#. Unfortunately, we didn't
  975. * figure out the way to identify the PLX bridge yet. So we
  976. * simply put the PCI bus and the subordinate behind the root
  977. * port to PE# here. The game rule here is expected to be changed
  978. * as soon as we can detected PLX bridge correctly.
  979. */
  980. static void pnv_pci_ioda_setup_PEs(void)
  981. {
  982. struct pci_controller *hose, *tmp;
  983. struct pnv_phb *phb;
  984. list_for_each_entry_safe(hose, tmp, &hose_list, list_node) {
  985. phb = hose->private_data;
  986. /* M64 layout might affect PE allocation */
  987. if (phb->reserve_m64_pe)
  988. phb->reserve_m64_pe(phb);
  989. pnv_ioda_setup_PEs(hose->bus);
  990. }
  991. }
  992. #ifdef CONFIG_PCI_IOV
  993. static int pnv_pci_vf_release_m64(struct pci_dev *pdev)
  994. {
  995. struct pci_bus *bus;
  996. struct pci_controller *hose;
  997. struct pnv_phb *phb;
  998. struct pci_dn *pdn;
  999. int i, j;
  1000. bus = pdev->bus;
  1001. hose = pci_bus_to_host(bus);
  1002. phb = hose->private_data;
  1003. pdn = pci_get_pdn(pdev);
  1004. for (i = 0; i < PCI_SRIOV_NUM_BARS; i++)
  1005. for (j = 0; j < M64_PER_IOV; j++) {
  1006. if (pdn->m64_wins[i][j] == IODA_INVALID_M64)
  1007. continue;
  1008. opal_pci_phb_mmio_enable(phb->opal_id,
  1009. OPAL_M64_WINDOW_TYPE, pdn->m64_wins[i][j], 0);
  1010. clear_bit(pdn->m64_wins[i][j], &phb->ioda.m64_bar_alloc);
  1011. pdn->m64_wins[i][j] = IODA_INVALID_M64;
  1012. }
  1013. return 0;
  1014. }
  1015. static int pnv_pci_vf_assign_m64(struct pci_dev *pdev, u16 num_vfs)
  1016. {
  1017. struct pci_bus *bus;
  1018. struct pci_controller *hose;
  1019. struct pnv_phb *phb;
  1020. struct pci_dn *pdn;
  1021. unsigned int win;
  1022. struct resource *res;
  1023. int i, j;
  1024. int64_t rc;
  1025. int total_vfs;
  1026. resource_size_t size, start;
  1027. int pe_num;
  1028. int vf_groups;
  1029. int vf_per_group;
  1030. bus = pdev->bus;
  1031. hose = pci_bus_to_host(bus);
  1032. phb = hose->private_data;
  1033. pdn = pci_get_pdn(pdev);
  1034. total_vfs = pci_sriov_get_totalvfs(pdev);
  1035. /* Initialize the m64_wins to IODA_INVALID_M64 */
  1036. for (i = 0; i < PCI_SRIOV_NUM_BARS; i++)
  1037. for (j = 0; j < M64_PER_IOV; j++)
  1038. pdn->m64_wins[i][j] = IODA_INVALID_M64;
  1039. if (pdn->m64_per_iov == M64_PER_IOV) {
  1040. vf_groups = (num_vfs <= M64_PER_IOV) ? num_vfs: M64_PER_IOV;
  1041. vf_per_group = (num_vfs <= M64_PER_IOV)? 1:
  1042. roundup_pow_of_two(num_vfs) / pdn->m64_per_iov;
  1043. } else {
  1044. vf_groups = 1;
  1045. vf_per_group = 1;
  1046. }
  1047. for (i = 0; i < PCI_SRIOV_NUM_BARS; i++) {
  1048. res = &pdev->resource[i + PCI_IOV_RESOURCES];
  1049. if (!res->flags || !res->parent)
  1050. continue;
  1051. if (!pnv_pci_is_mem_pref_64(res->flags))
  1052. continue;
  1053. for (j = 0; j < vf_groups; j++) {
  1054. do {
  1055. win = find_next_zero_bit(&phb->ioda.m64_bar_alloc,
  1056. phb->ioda.m64_bar_idx + 1, 0);
  1057. if (win >= phb->ioda.m64_bar_idx + 1)
  1058. goto m64_failed;
  1059. } while (test_and_set_bit(win, &phb->ioda.m64_bar_alloc));
  1060. pdn->m64_wins[i][j] = win;
  1061. if (pdn->m64_per_iov == M64_PER_IOV) {
  1062. size = pci_iov_resource_size(pdev,
  1063. PCI_IOV_RESOURCES + i);
  1064. size = size * vf_per_group;
  1065. start = res->start + size * j;
  1066. } else {
  1067. size = resource_size(res);
  1068. start = res->start;
  1069. }
  1070. /* Map the M64 here */
  1071. if (pdn->m64_per_iov == M64_PER_IOV) {
  1072. pe_num = pdn->offset + j;
  1073. rc = opal_pci_map_pe_mmio_window(phb->opal_id,
  1074. pe_num, OPAL_M64_WINDOW_TYPE,
  1075. pdn->m64_wins[i][j], 0);
  1076. }
  1077. rc = opal_pci_set_phb_mem_window(phb->opal_id,
  1078. OPAL_M64_WINDOW_TYPE,
  1079. pdn->m64_wins[i][j],
  1080. start,
  1081. 0, /* unused */
  1082. size);
  1083. if (rc != OPAL_SUCCESS) {
  1084. dev_err(&pdev->dev, "Failed to map M64 window #%d: %lld\n",
  1085. win, rc);
  1086. goto m64_failed;
  1087. }
  1088. if (pdn->m64_per_iov == M64_PER_IOV)
  1089. rc = opal_pci_phb_mmio_enable(phb->opal_id,
  1090. OPAL_M64_WINDOW_TYPE, pdn->m64_wins[i][j], 2);
  1091. else
  1092. rc = opal_pci_phb_mmio_enable(phb->opal_id,
  1093. OPAL_M64_WINDOW_TYPE, pdn->m64_wins[i][j], 1);
  1094. if (rc != OPAL_SUCCESS) {
  1095. dev_err(&pdev->dev, "Failed to enable M64 window #%d: %llx\n",
  1096. win, rc);
  1097. goto m64_failed;
  1098. }
  1099. }
  1100. }
  1101. return 0;
  1102. m64_failed:
  1103. pnv_pci_vf_release_m64(pdev);
  1104. return -EBUSY;
  1105. }
  1106. static long pnv_pci_ioda2_unset_window(struct iommu_table_group *table_group,
  1107. int num);
  1108. static void pnv_pci_ioda2_set_bypass(struct pnv_ioda_pe *pe, bool enable);
  1109. static void pnv_pci_ioda2_release_dma_pe(struct pci_dev *dev, struct pnv_ioda_pe *pe)
  1110. {
  1111. struct iommu_table *tbl;
  1112. int64_t rc;
  1113. tbl = pe->table_group.tables[0];
  1114. rc = pnv_pci_ioda2_unset_window(&pe->table_group, 0);
  1115. if (rc)
  1116. pe_warn(pe, "OPAL error %ld release DMA window\n", rc);
  1117. pnv_pci_ioda2_set_bypass(pe, false);
  1118. if (pe->table_group.group) {
  1119. iommu_group_put(pe->table_group.group);
  1120. BUG_ON(pe->table_group.group);
  1121. }
  1122. pnv_pci_ioda2_table_free_pages(tbl);
  1123. iommu_free_table(tbl, of_node_full_name(dev->dev.of_node));
  1124. }
  1125. static void pnv_ioda_release_vf_PE(struct pci_dev *pdev, u16 num_vfs)
  1126. {
  1127. struct pci_bus *bus;
  1128. struct pci_controller *hose;
  1129. struct pnv_phb *phb;
  1130. struct pnv_ioda_pe *pe, *pe_n;
  1131. struct pci_dn *pdn;
  1132. u16 vf_index;
  1133. int64_t rc;
  1134. bus = pdev->bus;
  1135. hose = pci_bus_to_host(bus);
  1136. phb = hose->private_data;
  1137. pdn = pci_get_pdn(pdev);
  1138. if (!pdev->is_physfn)
  1139. return;
  1140. if (pdn->m64_per_iov == M64_PER_IOV && num_vfs > M64_PER_IOV) {
  1141. int vf_group;
  1142. int vf_per_group;
  1143. int vf_index1;
  1144. vf_per_group = roundup_pow_of_two(num_vfs) / pdn->m64_per_iov;
  1145. for (vf_group = 0; vf_group < M64_PER_IOV; vf_group++)
  1146. for (vf_index = vf_group * vf_per_group;
  1147. vf_index < (vf_group + 1) * vf_per_group &&
  1148. vf_index < num_vfs;
  1149. vf_index++)
  1150. for (vf_index1 = vf_group * vf_per_group;
  1151. vf_index1 < (vf_group + 1) * vf_per_group &&
  1152. vf_index1 < num_vfs;
  1153. vf_index1++){
  1154. rc = opal_pci_set_peltv(phb->opal_id,
  1155. pdn->offset + vf_index,
  1156. pdn->offset + vf_index1,
  1157. OPAL_REMOVE_PE_FROM_DOMAIN);
  1158. if (rc)
  1159. dev_warn(&pdev->dev, "%s: Failed to unlink same group PE#%d(%lld)\n",
  1160. __func__,
  1161. pdn->offset + vf_index1, rc);
  1162. }
  1163. }
  1164. list_for_each_entry_safe(pe, pe_n, &phb->ioda.pe_list, list) {
  1165. if (pe->parent_dev != pdev)
  1166. continue;
  1167. pnv_pci_ioda2_release_dma_pe(pdev, pe);
  1168. /* Remove from list */
  1169. mutex_lock(&phb->ioda.pe_list_mutex);
  1170. list_del(&pe->list);
  1171. mutex_unlock(&phb->ioda.pe_list_mutex);
  1172. pnv_ioda_deconfigure_pe(phb, pe);
  1173. pnv_ioda_free_pe(phb, pe->pe_number);
  1174. }
  1175. }
  1176. void pnv_pci_sriov_disable(struct pci_dev *pdev)
  1177. {
  1178. struct pci_bus *bus;
  1179. struct pci_controller *hose;
  1180. struct pnv_phb *phb;
  1181. struct pci_dn *pdn;
  1182. struct pci_sriov *iov;
  1183. u16 num_vfs;
  1184. bus = pdev->bus;
  1185. hose = pci_bus_to_host(bus);
  1186. phb = hose->private_data;
  1187. pdn = pci_get_pdn(pdev);
  1188. iov = pdev->sriov;
  1189. num_vfs = pdn->num_vfs;
  1190. /* Release VF PEs */
  1191. pnv_ioda_release_vf_PE(pdev, num_vfs);
  1192. if (phb->type == PNV_PHB_IODA2) {
  1193. if (pdn->m64_per_iov == 1)
  1194. pnv_pci_vf_resource_shift(pdev, -pdn->offset);
  1195. /* Release M64 windows */
  1196. pnv_pci_vf_release_m64(pdev);
  1197. /* Release PE numbers */
  1198. bitmap_clear(phb->ioda.pe_alloc, pdn->offset, num_vfs);
  1199. pdn->offset = 0;
  1200. }
  1201. }
  1202. static void pnv_pci_ioda2_setup_dma_pe(struct pnv_phb *phb,
  1203. struct pnv_ioda_pe *pe);
  1204. static void pnv_ioda_setup_vf_PE(struct pci_dev *pdev, u16 num_vfs)
  1205. {
  1206. struct pci_bus *bus;
  1207. struct pci_controller *hose;
  1208. struct pnv_phb *phb;
  1209. struct pnv_ioda_pe *pe;
  1210. int pe_num;
  1211. u16 vf_index;
  1212. struct pci_dn *pdn;
  1213. int64_t rc;
  1214. bus = pdev->bus;
  1215. hose = pci_bus_to_host(bus);
  1216. phb = hose->private_data;
  1217. pdn = pci_get_pdn(pdev);
  1218. if (!pdev->is_physfn)
  1219. return;
  1220. /* Reserve PE for each VF */
  1221. for (vf_index = 0; vf_index < num_vfs; vf_index++) {
  1222. pe_num = pdn->offset + vf_index;
  1223. pe = &phb->ioda.pe_array[pe_num];
  1224. pe->pe_number = pe_num;
  1225. pe->phb = phb;
  1226. pe->flags = PNV_IODA_PE_VF;
  1227. pe->pbus = NULL;
  1228. pe->parent_dev = pdev;
  1229. pe->tce32_seg = -1;
  1230. pe->mve_number = -1;
  1231. pe->rid = (pci_iov_virtfn_bus(pdev, vf_index) << 8) |
  1232. pci_iov_virtfn_devfn(pdev, vf_index);
  1233. pe_info(pe, "VF %04d:%02d:%02d.%d associated with PE#%d\n",
  1234. hose->global_number, pdev->bus->number,
  1235. PCI_SLOT(pci_iov_virtfn_devfn(pdev, vf_index)),
  1236. PCI_FUNC(pci_iov_virtfn_devfn(pdev, vf_index)), pe_num);
  1237. if (pnv_ioda_configure_pe(phb, pe)) {
  1238. /* XXX What do we do here ? */
  1239. if (pe_num)
  1240. pnv_ioda_free_pe(phb, pe_num);
  1241. pe->pdev = NULL;
  1242. continue;
  1243. }
  1244. /* Put PE to the list */
  1245. mutex_lock(&phb->ioda.pe_list_mutex);
  1246. list_add_tail(&pe->list, &phb->ioda.pe_list);
  1247. mutex_unlock(&phb->ioda.pe_list_mutex);
  1248. pnv_pci_ioda2_setup_dma_pe(phb, pe);
  1249. }
  1250. if (pdn->m64_per_iov == M64_PER_IOV && num_vfs > M64_PER_IOV) {
  1251. int vf_group;
  1252. int vf_per_group;
  1253. int vf_index1;
  1254. vf_per_group = roundup_pow_of_two(num_vfs) / pdn->m64_per_iov;
  1255. for (vf_group = 0; vf_group < M64_PER_IOV; vf_group++) {
  1256. for (vf_index = vf_group * vf_per_group;
  1257. vf_index < (vf_group + 1) * vf_per_group &&
  1258. vf_index < num_vfs;
  1259. vf_index++) {
  1260. for (vf_index1 = vf_group * vf_per_group;
  1261. vf_index1 < (vf_group + 1) * vf_per_group &&
  1262. vf_index1 < num_vfs;
  1263. vf_index1++) {
  1264. rc = opal_pci_set_peltv(phb->opal_id,
  1265. pdn->offset + vf_index,
  1266. pdn->offset + vf_index1,
  1267. OPAL_ADD_PE_TO_DOMAIN);
  1268. if (rc)
  1269. dev_warn(&pdev->dev, "%s: Failed to link same group PE#%d(%lld)\n",
  1270. __func__,
  1271. pdn->offset + vf_index1, rc);
  1272. }
  1273. }
  1274. }
  1275. }
  1276. }
  1277. int pnv_pci_sriov_enable(struct pci_dev *pdev, u16 num_vfs)
  1278. {
  1279. struct pci_bus *bus;
  1280. struct pci_controller *hose;
  1281. struct pnv_phb *phb;
  1282. struct pci_dn *pdn;
  1283. int ret;
  1284. bus = pdev->bus;
  1285. hose = pci_bus_to_host(bus);
  1286. phb = hose->private_data;
  1287. pdn = pci_get_pdn(pdev);
  1288. if (phb->type == PNV_PHB_IODA2) {
  1289. /* Calculate available PE for required VFs */
  1290. mutex_lock(&phb->ioda.pe_alloc_mutex);
  1291. pdn->offset = bitmap_find_next_zero_area(
  1292. phb->ioda.pe_alloc, phb->ioda.total_pe,
  1293. 0, num_vfs, 0);
  1294. if (pdn->offset >= phb->ioda.total_pe) {
  1295. mutex_unlock(&phb->ioda.pe_alloc_mutex);
  1296. dev_info(&pdev->dev, "Failed to enable VF%d\n", num_vfs);
  1297. pdn->offset = 0;
  1298. return -EBUSY;
  1299. }
  1300. bitmap_set(phb->ioda.pe_alloc, pdn->offset, num_vfs);
  1301. pdn->num_vfs = num_vfs;
  1302. mutex_unlock(&phb->ioda.pe_alloc_mutex);
  1303. /* Assign M64 window accordingly */
  1304. ret = pnv_pci_vf_assign_m64(pdev, num_vfs);
  1305. if (ret) {
  1306. dev_info(&pdev->dev, "Not enough M64 window resources\n");
  1307. goto m64_failed;
  1308. }
  1309. /*
  1310. * When using one M64 BAR to map one IOV BAR, we need to shift
  1311. * the IOV BAR according to the PE# allocated to the VFs.
  1312. * Otherwise, the PE# for the VF will conflict with others.
  1313. */
  1314. if (pdn->m64_per_iov == 1) {
  1315. ret = pnv_pci_vf_resource_shift(pdev, pdn->offset);
  1316. if (ret)
  1317. goto m64_failed;
  1318. }
  1319. }
  1320. /* Setup VF PEs */
  1321. pnv_ioda_setup_vf_PE(pdev, num_vfs);
  1322. return 0;
  1323. m64_failed:
  1324. bitmap_clear(phb->ioda.pe_alloc, pdn->offset, num_vfs);
  1325. pdn->offset = 0;
  1326. return ret;
  1327. }
  1328. int pcibios_sriov_disable(struct pci_dev *pdev)
  1329. {
  1330. pnv_pci_sriov_disable(pdev);
  1331. /* Release PCI data */
  1332. remove_dev_pci_data(pdev);
  1333. return 0;
  1334. }
  1335. int pcibios_sriov_enable(struct pci_dev *pdev, u16 num_vfs)
  1336. {
  1337. /* Allocate PCI data */
  1338. add_dev_pci_data(pdev);
  1339. pnv_pci_sriov_enable(pdev, num_vfs);
  1340. return 0;
  1341. }
  1342. #endif /* CONFIG_PCI_IOV */
  1343. static void pnv_pci_ioda_dma_dev_setup(struct pnv_phb *phb, struct pci_dev *pdev)
  1344. {
  1345. struct pci_dn *pdn = pci_get_pdn(pdev);
  1346. struct pnv_ioda_pe *pe;
  1347. /*
  1348. * The function can be called while the PE#
  1349. * hasn't been assigned. Do nothing for the
  1350. * case.
  1351. */
  1352. if (!pdn || pdn->pe_number == IODA_INVALID_PE)
  1353. return;
  1354. pe = &phb->ioda.pe_array[pdn->pe_number];
  1355. WARN_ON(get_dma_ops(&pdev->dev) != &dma_iommu_ops);
  1356. set_iommu_table_base(&pdev->dev, pe->table_group.tables[0]);
  1357. /*
  1358. * Note: iommu_add_device() will fail here as
  1359. * for physical PE: the device is already added by now;
  1360. * for virtual PE: sysfs entries are not ready yet and
  1361. * tce_iommu_bus_notifier will add the device to a group later.
  1362. */
  1363. }
  1364. static int pnv_pci_ioda_dma_set_mask(struct pci_dev *pdev, u64 dma_mask)
  1365. {
  1366. struct pci_controller *hose = pci_bus_to_host(pdev->bus);
  1367. struct pnv_phb *phb = hose->private_data;
  1368. struct pci_dn *pdn = pci_get_pdn(pdev);
  1369. struct pnv_ioda_pe *pe;
  1370. uint64_t top;
  1371. bool bypass = false;
  1372. if (WARN_ON(!pdn || pdn->pe_number == IODA_INVALID_PE))
  1373. return -ENODEV;;
  1374. pe = &phb->ioda.pe_array[pdn->pe_number];
  1375. if (pe->tce_bypass_enabled) {
  1376. top = pe->tce_bypass_base + memblock_end_of_DRAM() - 1;
  1377. bypass = (dma_mask >= top);
  1378. }
  1379. if (bypass) {
  1380. dev_info(&pdev->dev, "Using 64-bit DMA iommu bypass\n");
  1381. set_dma_ops(&pdev->dev, &dma_direct_ops);
  1382. set_dma_offset(&pdev->dev, pe->tce_bypass_base);
  1383. } else {
  1384. dev_info(&pdev->dev, "Using 32-bit DMA via iommu\n");
  1385. set_dma_ops(&pdev->dev, &dma_iommu_ops);
  1386. set_iommu_table_base(&pdev->dev, pe->table_group.tables[0]);
  1387. }
  1388. *pdev->dev.dma_mask = dma_mask;
  1389. return 0;
  1390. }
  1391. static u64 pnv_pci_ioda_dma_get_required_mask(struct pnv_phb *phb,
  1392. struct pci_dev *pdev)
  1393. {
  1394. struct pci_dn *pdn = pci_get_pdn(pdev);
  1395. struct pnv_ioda_pe *pe;
  1396. u64 end, mask;
  1397. if (WARN_ON(!pdn || pdn->pe_number == IODA_INVALID_PE))
  1398. return 0;
  1399. pe = &phb->ioda.pe_array[pdn->pe_number];
  1400. if (!pe->tce_bypass_enabled)
  1401. return __dma_get_required_mask(&pdev->dev);
  1402. end = pe->tce_bypass_base + memblock_end_of_DRAM();
  1403. mask = 1ULL << (fls64(end) - 1);
  1404. mask += mask - 1;
  1405. return mask;
  1406. }
  1407. static void pnv_ioda_setup_bus_dma(struct pnv_ioda_pe *pe,
  1408. struct pci_bus *bus)
  1409. {
  1410. struct pci_dev *dev;
  1411. list_for_each_entry(dev, &bus->devices, bus_list) {
  1412. set_iommu_table_base(&dev->dev, pe->table_group.tables[0]);
  1413. iommu_add_device(&dev->dev);
  1414. if ((pe->flags & PNV_IODA_PE_BUS_ALL) && dev->subordinate)
  1415. pnv_ioda_setup_bus_dma(pe, dev->subordinate);
  1416. }
  1417. }
  1418. static void pnv_pci_ioda1_tce_invalidate(struct iommu_table *tbl,
  1419. unsigned long index, unsigned long npages, bool rm)
  1420. {
  1421. struct iommu_table_group_link *tgl = list_first_entry_or_null(
  1422. &tbl->it_group_list, struct iommu_table_group_link,
  1423. next);
  1424. struct pnv_ioda_pe *pe = container_of(tgl->table_group,
  1425. struct pnv_ioda_pe, table_group);
  1426. __be64 __iomem *invalidate = rm ?
  1427. (__be64 __iomem *)pe->phb->ioda.tce_inval_reg_phys :
  1428. pe->phb->ioda.tce_inval_reg;
  1429. unsigned long start, end, inc;
  1430. const unsigned shift = tbl->it_page_shift;
  1431. start = __pa(((__be64 *)tbl->it_base) + index - tbl->it_offset);
  1432. end = __pa(((__be64 *)tbl->it_base) + index - tbl->it_offset +
  1433. npages - 1);
  1434. /* BML uses this case for p6/p7/galaxy2: Shift addr and put in node */
  1435. if (tbl->it_busno) {
  1436. start <<= shift;
  1437. end <<= shift;
  1438. inc = 128ull << shift;
  1439. start |= tbl->it_busno;
  1440. end |= tbl->it_busno;
  1441. } else if (tbl->it_type & TCE_PCI_SWINV_PAIR) {
  1442. /* p7ioc-style invalidation, 2 TCEs per write */
  1443. start |= (1ull << 63);
  1444. end |= (1ull << 63);
  1445. inc = 16;
  1446. } else {
  1447. /* Default (older HW) */
  1448. inc = 128;
  1449. }
  1450. end |= inc - 1; /* round up end to be different than start */
  1451. mb(); /* Ensure above stores are visible */
  1452. while (start <= end) {
  1453. if (rm)
  1454. __raw_rm_writeq(cpu_to_be64(start), invalidate);
  1455. else
  1456. __raw_writeq(cpu_to_be64(start), invalidate);
  1457. start += inc;
  1458. }
  1459. /*
  1460. * The iommu layer will do another mb() for us on build()
  1461. * and we don't care on free()
  1462. */
  1463. }
  1464. static int pnv_ioda1_tce_build(struct iommu_table *tbl, long index,
  1465. long npages, unsigned long uaddr,
  1466. enum dma_data_direction direction,
  1467. struct dma_attrs *attrs)
  1468. {
  1469. int ret = pnv_tce_build(tbl, index, npages, uaddr, direction,
  1470. attrs);
  1471. if (!ret && (tbl->it_type & TCE_PCI_SWINV_CREATE))
  1472. pnv_pci_ioda1_tce_invalidate(tbl, index, npages, false);
  1473. return ret;
  1474. }
  1475. #ifdef CONFIG_IOMMU_API
  1476. static int pnv_ioda1_tce_xchg(struct iommu_table *tbl, long index,
  1477. unsigned long *hpa, enum dma_data_direction *direction)
  1478. {
  1479. long ret = pnv_tce_xchg(tbl, index, hpa, direction);
  1480. if (!ret && (tbl->it_type &
  1481. (TCE_PCI_SWINV_CREATE | TCE_PCI_SWINV_FREE)))
  1482. pnv_pci_ioda1_tce_invalidate(tbl, index, 1, false);
  1483. return ret;
  1484. }
  1485. #endif
  1486. static void pnv_ioda1_tce_free(struct iommu_table *tbl, long index,
  1487. long npages)
  1488. {
  1489. pnv_tce_free(tbl, index, npages);
  1490. if (tbl->it_type & TCE_PCI_SWINV_FREE)
  1491. pnv_pci_ioda1_tce_invalidate(tbl, index, npages, false);
  1492. }
  1493. static struct iommu_table_ops pnv_ioda1_iommu_ops = {
  1494. .set = pnv_ioda1_tce_build,
  1495. #ifdef CONFIG_IOMMU_API
  1496. .exchange = pnv_ioda1_tce_xchg,
  1497. #endif
  1498. .clear = pnv_ioda1_tce_free,
  1499. .get = pnv_tce_get,
  1500. };
  1501. static inline void pnv_pci_ioda2_tce_invalidate_entire(struct pnv_ioda_pe *pe)
  1502. {
  1503. /* 01xb - invalidate TCEs that match the specified PE# */
  1504. unsigned long val = (0x4ull << 60) | (pe->pe_number & 0xFF);
  1505. struct pnv_phb *phb = pe->phb;
  1506. if (!phb->ioda.tce_inval_reg)
  1507. return;
  1508. mb(); /* Ensure above stores are visible */
  1509. __raw_writeq(cpu_to_be64(val), phb->ioda.tce_inval_reg);
  1510. }
  1511. static void pnv_pci_ioda2_do_tce_invalidate(unsigned pe_number, bool rm,
  1512. __be64 __iomem *invalidate, unsigned shift,
  1513. unsigned long index, unsigned long npages)
  1514. {
  1515. unsigned long start, end, inc;
  1516. /* We'll invalidate DMA address in PE scope */
  1517. start = 0x2ull << 60;
  1518. start |= (pe_number & 0xFF);
  1519. end = start;
  1520. /* Figure out the start, end and step */
  1521. start |= (index << shift);
  1522. end |= ((index + npages - 1) << shift);
  1523. inc = (0x1ull << shift);
  1524. mb();
  1525. while (start <= end) {
  1526. if (rm)
  1527. __raw_rm_writeq(cpu_to_be64(start), invalidate);
  1528. else
  1529. __raw_writeq(cpu_to_be64(start), invalidate);
  1530. start += inc;
  1531. }
  1532. }
  1533. static void pnv_pci_ioda2_tce_invalidate(struct iommu_table *tbl,
  1534. unsigned long index, unsigned long npages, bool rm)
  1535. {
  1536. struct iommu_table_group_link *tgl;
  1537. list_for_each_entry_rcu(tgl, &tbl->it_group_list, next) {
  1538. struct pnv_ioda_pe *pe = container_of(tgl->table_group,
  1539. struct pnv_ioda_pe, table_group);
  1540. __be64 __iomem *invalidate = rm ?
  1541. (__be64 __iomem *)pe->phb->ioda.tce_inval_reg_phys :
  1542. pe->phb->ioda.tce_inval_reg;
  1543. pnv_pci_ioda2_do_tce_invalidate(pe->pe_number, rm,
  1544. invalidate, tbl->it_page_shift,
  1545. index, npages);
  1546. }
  1547. }
  1548. static int pnv_ioda2_tce_build(struct iommu_table *tbl, long index,
  1549. long npages, unsigned long uaddr,
  1550. enum dma_data_direction direction,
  1551. struct dma_attrs *attrs)
  1552. {
  1553. int ret = pnv_tce_build(tbl, index, npages, uaddr, direction,
  1554. attrs);
  1555. if (!ret && (tbl->it_type & TCE_PCI_SWINV_CREATE))
  1556. pnv_pci_ioda2_tce_invalidate(tbl, index, npages, false);
  1557. return ret;
  1558. }
  1559. #ifdef CONFIG_IOMMU_API
  1560. static int pnv_ioda2_tce_xchg(struct iommu_table *tbl, long index,
  1561. unsigned long *hpa, enum dma_data_direction *direction)
  1562. {
  1563. long ret = pnv_tce_xchg(tbl, index, hpa, direction);
  1564. if (!ret && (tbl->it_type &
  1565. (TCE_PCI_SWINV_CREATE | TCE_PCI_SWINV_FREE)))
  1566. pnv_pci_ioda2_tce_invalidate(tbl, index, 1, false);
  1567. return ret;
  1568. }
  1569. #endif
  1570. static void pnv_ioda2_tce_free(struct iommu_table *tbl, long index,
  1571. long npages)
  1572. {
  1573. pnv_tce_free(tbl, index, npages);
  1574. if (tbl->it_type & TCE_PCI_SWINV_FREE)
  1575. pnv_pci_ioda2_tce_invalidate(tbl, index, npages, false);
  1576. }
  1577. static void pnv_ioda2_table_free(struct iommu_table *tbl)
  1578. {
  1579. pnv_pci_ioda2_table_free_pages(tbl);
  1580. iommu_free_table(tbl, "pnv");
  1581. }
  1582. static struct iommu_table_ops pnv_ioda2_iommu_ops = {
  1583. .set = pnv_ioda2_tce_build,
  1584. #ifdef CONFIG_IOMMU_API
  1585. .exchange = pnv_ioda2_tce_xchg,
  1586. #endif
  1587. .clear = pnv_ioda2_tce_free,
  1588. .get = pnv_tce_get,
  1589. .free = pnv_ioda2_table_free,
  1590. };
  1591. static void pnv_pci_ioda_setup_dma_pe(struct pnv_phb *phb,
  1592. struct pnv_ioda_pe *pe, unsigned int base,
  1593. unsigned int segs)
  1594. {
  1595. struct page *tce_mem = NULL;
  1596. struct iommu_table *tbl;
  1597. unsigned int i;
  1598. int64_t rc;
  1599. void *addr;
  1600. /* XXX FIXME: Handle 64-bit only DMA devices */
  1601. /* XXX FIXME: Provide 64-bit DMA facilities & non-4K TCE tables etc.. */
  1602. /* XXX FIXME: Allocate multi-level tables on PHB3 */
  1603. /* We shouldn't already have a 32-bit DMA associated */
  1604. if (WARN_ON(pe->tce32_seg >= 0))
  1605. return;
  1606. tbl = pnv_pci_table_alloc(phb->hose->node);
  1607. iommu_register_group(&pe->table_group, phb->hose->global_number,
  1608. pe->pe_number);
  1609. pnv_pci_link_table_and_group(phb->hose->node, 0, tbl, &pe->table_group);
  1610. /* Grab a 32-bit TCE table */
  1611. pe->tce32_seg = base;
  1612. pe_info(pe, " Setting up 32-bit TCE table at %08x..%08x\n",
  1613. (base << 28), ((base + segs) << 28) - 1);
  1614. /* XXX Currently, we allocate one big contiguous table for the
  1615. * TCEs. We only really need one chunk per 256M of TCE space
  1616. * (ie per segment) but that's an optimization for later, it
  1617. * requires some added smarts with our get/put_tce implementation
  1618. */
  1619. tce_mem = alloc_pages_node(phb->hose->node, GFP_KERNEL,
  1620. get_order(TCE32_TABLE_SIZE * segs));
  1621. if (!tce_mem) {
  1622. pe_err(pe, " Failed to allocate a 32-bit TCE memory\n");
  1623. goto fail;
  1624. }
  1625. addr = page_address(tce_mem);
  1626. memset(addr, 0, TCE32_TABLE_SIZE * segs);
  1627. /* Configure HW */
  1628. for (i = 0; i < segs; i++) {
  1629. rc = opal_pci_map_pe_dma_window(phb->opal_id,
  1630. pe->pe_number,
  1631. base + i, 1,
  1632. __pa(addr) + TCE32_TABLE_SIZE * i,
  1633. TCE32_TABLE_SIZE, 0x1000);
  1634. if (rc) {
  1635. pe_err(pe, " Failed to configure 32-bit TCE table,"
  1636. " err %ld\n", rc);
  1637. goto fail;
  1638. }
  1639. }
  1640. /* Setup linux iommu table */
  1641. pnv_pci_setup_iommu_table(tbl, addr, TCE32_TABLE_SIZE * segs,
  1642. base << 28, IOMMU_PAGE_SHIFT_4K);
  1643. /* OPAL variant of P7IOC SW invalidated TCEs */
  1644. if (phb->ioda.tce_inval_reg)
  1645. tbl->it_type |= (TCE_PCI_SWINV_CREATE |
  1646. TCE_PCI_SWINV_FREE |
  1647. TCE_PCI_SWINV_PAIR);
  1648. tbl->it_ops = &pnv_ioda1_iommu_ops;
  1649. pe->table_group.tce32_start = tbl->it_offset << tbl->it_page_shift;
  1650. pe->table_group.tce32_size = tbl->it_size << tbl->it_page_shift;
  1651. iommu_init_table(tbl, phb->hose->node);
  1652. if (pe->flags & PNV_IODA_PE_DEV) {
  1653. /*
  1654. * Setting table base here only for carrying iommu_group
  1655. * further down to let iommu_add_device() do the job.
  1656. * pnv_pci_ioda_dma_dev_setup will override it later anyway.
  1657. */
  1658. set_iommu_table_base(&pe->pdev->dev, tbl);
  1659. iommu_add_device(&pe->pdev->dev);
  1660. } else if (pe->flags & (PNV_IODA_PE_BUS | PNV_IODA_PE_BUS_ALL))
  1661. pnv_ioda_setup_bus_dma(pe, pe->pbus);
  1662. return;
  1663. fail:
  1664. /* XXX Failure: Try to fallback to 64-bit only ? */
  1665. if (pe->tce32_seg >= 0)
  1666. pe->tce32_seg = -1;
  1667. if (tce_mem)
  1668. __free_pages(tce_mem, get_order(TCE32_TABLE_SIZE * segs));
  1669. if (tbl) {
  1670. pnv_pci_unlink_table_and_group(tbl, &pe->table_group);
  1671. iommu_free_table(tbl, "pnv");
  1672. }
  1673. }
  1674. static long pnv_pci_ioda2_set_window(struct iommu_table_group *table_group,
  1675. int num, struct iommu_table *tbl)
  1676. {
  1677. struct pnv_ioda_pe *pe = container_of(table_group, struct pnv_ioda_pe,
  1678. table_group);
  1679. struct pnv_phb *phb = pe->phb;
  1680. int64_t rc;
  1681. const unsigned long size = tbl->it_indirect_levels ?
  1682. tbl->it_level_size : tbl->it_size;
  1683. const __u64 start_addr = tbl->it_offset << tbl->it_page_shift;
  1684. const __u64 win_size = tbl->it_size << tbl->it_page_shift;
  1685. pe_info(pe, "Setting up window#%d %llx..%llx pg=%x\n", num,
  1686. start_addr, start_addr + win_size - 1,
  1687. IOMMU_PAGE_SIZE(tbl));
  1688. /*
  1689. * Map TCE table through TVT. The TVE index is the PE number
  1690. * shifted by 1 bit for 32-bits DMA space.
  1691. */
  1692. rc = opal_pci_map_pe_dma_window(phb->opal_id,
  1693. pe->pe_number,
  1694. (pe->pe_number << 1) + num,
  1695. tbl->it_indirect_levels + 1,
  1696. __pa(tbl->it_base),
  1697. size << 3,
  1698. IOMMU_PAGE_SIZE(tbl));
  1699. if (rc) {
  1700. pe_err(pe, "Failed to configure TCE table, err %ld\n", rc);
  1701. return rc;
  1702. }
  1703. pnv_pci_link_table_and_group(phb->hose->node, num,
  1704. tbl, &pe->table_group);
  1705. pnv_pci_ioda2_tce_invalidate_entire(pe);
  1706. return 0;
  1707. }
  1708. static void pnv_pci_ioda2_set_bypass(struct pnv_ioda_pe *pe, bool enable)
  1709. {
  1710. uint16_t window_id = (pe->pe_number << 1 ) + 1;
  1711. int64_t rc;
  1712. pe_info(pe, "%sabling 64-bit DMA bypass\n", enable ? "En" : "Dis");
  1713. if (enable) {
  1714. phys_addr_t top = memblock_end_of_DRAM();
  1715. top = roundup_pow_of_two(top);
  1716. rc = opal_pci_map_pe_dma_window_real(pe->phb->opal_id,
  1717. pe->pe_number,
  1718. window_id,
  1719. pe->tce_bypass_base,
  1720. top);
  1721. } else {
  1722. rc = opal_pci_map_pe_dma_window_real(pe->phb->opal_id,
  1723. pe->pe_number,
  1724. window_id,
  1725. pe->tce_bypass_base,
  1726. 0);
  1727. }
  1728. if (rc)
  1729. pe_err(pe, "OPAL error %lld configuring bypass window\n", rc);
  1730. else
  1731. pe->tce_bypass_enabled = enable;
  1732. }
  1733. static long pnv_pci_ioda2_table_alloc_pages(int nid, __u64 bus_offset,
  1734. __u32 page_shift, __u64 window_size, __u32 levels,
  1735. struct iommu_table *tbl);
  1736. static long pnv_pci_ioda2_create_table(struct iommu_table_group *table_group,
  1737. int num, __u32 page_shift, __u64 window_size, __u32 levels,
  1738. struct iommu_table **ptbl)
  1739. {
  1740. struct pnv_ioda_pe *pe = container_of(table_group, struct pnv_ioda_pe,
  1741. table_group);
  1742. int nid = pe->phb->hose->node;
  1743. __u64 bus_offset = num ? pe->tce_bypass_base : table_group->tce32_start;
  1744. long ret;
  1745. struct iommu_table *tbl;
  1746. tbl = pnv_pci_table_alloc(nid);
  1747. if (!tbl)
  1748. return -ENOMEM;
  1749. ret = pnv_pci_ioda2_table_alloc_pages(nid,
  1750. bus_offset, page_shift, window_size,
  1751. levels, tbl);
  1752. if (ret) {
  1753. iommu_free_table(tbl, "pnv");
  1754. return ret;
  1755. }
  1756. tbl->it_ops = &pnv_ioda2_iommu_ops;
  1757. if (pe->phb->ioda.tce_inval_reg)
  1758. tbl->it_type |= (TCE_PCI_SWINV_CREATE | TCE_PCI_SWINV_FREE);
  1759. *ptbl = tbl;
  1760. return 0;
  1761. }
  1762. static long pnv_pci_ioda2_setup_default_config(struct pnv_ioda_pe *pe)
  1763. {
  1764. struct iommu_table *tbl = NULL;
  1765. long rc;
  1766. rc = pnv_pci_ioda2_create_table(&pe->table_group, 0,
  1767. IOMMU_PAGE_SHIFT_4K,
  1768. pe->table_group.tce32_size,
  1769. POWERNV_IOMMU_DEFAULT_LEVELS, &tbl);
  1770. if (rc) {
  1771. pe_err(pe, "Failed to create 32-bit TCE table, err %ld",
  1772. rc);
  1773. return rc;
  1774. }
  1775. iommu_init_table(tbl, pe->phb->hose->node);
  1776. rc = pnv_pci_ioda2_set_window(&pe->table_group, 0, tbl);
  1777. if (rc) {
  1778. pe_err(pe, "Failed to configure 32-bit TCE table, err %ld\n",
  1779. rc);
  1780. pnv_ioda2_table_free(tbl);
  1781. return rc;
  1782. }
  1783. if (!pnv_iommu_bypass_disabled)
  1784. pnv_pci_ioda2_set_bypass(pe, true);
  1785. /* OPAL variant of PHB3 invalidated TCEs */
  1786. if (pe->phb->ioda.tce_inval_reg)
  1787. tbl->it_type |= (TCE_PCI_SWINV_CREATE | TCE_PCI_SWINV_FREE);
  1788. /*
  1789. * Setting table base here only for carrying iommu_group
  1790. * further down to let iommu_add_device() do the job.
  1791. * pnv_pci_ioda_dma_dev_setup will override it later anyway.
  1792. */
  1793. if (pe->flags & PNV_IODA_PE_DEV)
  1794. set_iommu_table_base(&pe->pdev->dev, tbl);
  1795. return 0;
  1796. }
  1797. #if defined(CONFIG_IOMMU_API) || defined(CONFIG_PCI_IOV)
  1798. static long pnv_pci_ioda2_unset_window(struct iommu_table_group *table_group,
  1799. int num)
  1800. {
  1801. struct pnv_ioda_pe *pe = container_of(table_group, struct pnv_ioda_pe,
  1802. table_group);
  1803. struct pnv_phb *phb = pe->phb;
  1804. long ret;
  1805. pe_info(pe, "Removing DMA window #%d\n", num);
  1806. ret = opal_pci_map_pe_dma_window(phb->opal_id, pe->pe_number,
  1807. (pe->pe_number << 1) + num,
  1808. 0/* levels */, 0/* table address */,
  1809. 0/* table size */, 0/* page size */);
  1810. if (ret)
  1811. pe_warn(pe, "Unmapping failed, ret = %ld\n", ret);
  1812. else
  1813. pnv_pci_ioda2_tce_invalidate_entire(pe);
  1814. pnv_pci_unlink_table_and_group(table_group->tables[num], table_group);
  1815. return ret;
  1816. }
  1817. #endif
  1818. #ifdef CONFIG_IOMMU_API
  1819. static unsigned long pnv_pci_ioda2_get_table_size(__u32 page_shift,
  1820. __u64 window_size, __u32 levels)
  1821. {
  1822. unsigned long bytes = 0;
  1823. const unsigned window_shift = ilog2(window_size);
  1824. unsigned entries_shift = window_shift - page_shift;
  1825. unsigned table_shift = entries_shift + 3;
  1826. unsigned long tce_table_size = max(0x1000UL, 1UL << table_shift);
  1827. unsigned long direct_table_size;
  1828. if (!levels || (levels > POWERNV_IOMMU_MAX_LEVELS) ||
  1829. (window_size > memory_hotplug_max()) ||
  1830. !is_power_of_2(window_size))
  1831. return 0;
  1832. /* Calculate a direct table size from window_size and levels */
  1833. entries_shift = (entries_shift + levels - 1) / levels;
  1834. table_shift = entries_shift + 3;
  1835. table_shift = max_t(unsigned, table_shift, PAGE_SHIFT);
  1836. direct_table_size = 1UL << table_shift;
  1837. for ( ; levels; --levels) {
  1838. bytes += _ALIGN_UP(tce_table_size, direct_table_size);
  1839. tce_table_size /= direct_table_size;
  1840. tce_table_size <<= 3;
  1841. tce_table_size = _ALIGN_UP(tce_table_size, direct_table_size);
  1842. }
  1843. return bytes;
  1844. }
  1845. static void pnv_ioda2_take_ownership(struct iommu_table_group *table_group)
  1846. {
  1847. struct pnv_ioda_pe *pe = container_of(table_group, struct pnv_ioda_pe,
  1848. table_group);
  1849. /* Store @tbl as pnv_pci_ioda2_unset_window() resets it */
  1850. struct iommu_table *tbl = pe->table_group.tables[0];
  1851. pnv_pci_ioda2_set_bypass(pe, false);
  1852. pnv_pci_ioda2_unset_window(&pe->table_group, 0);
  1853. pnv_ioda2_table_free(tbl);
  1854. }
  1855. static void pnv_ioda2_release_ownership(struct iommu_table_group *table_group)
  1856. {
  1857. struct pnv_ioda_pe *pe = container_of(table_group, struct pnv_ioda_pe,
  1858. table_group);
  1859. pnv_pci_ioda2_setup_default_config(pe);
  1860. }
  1861. static struct iommu_table_group_ops pnv_pci_ioda2_ops = {
  1862. .get_table_size = pnv_pci_ioda2_get_table_size,
  1863. .create_table = pnv_pci_ioda2_create_table,
  1864. .set_window = pnv_pci_ioda2_set_window,
  1865. .unset_window = pnv_pci_ioda2_unset_window,
  1866. .take_ownership = pnv_ioda2_take_ownership,
  1867. .release_ownership = pnv_ioda2_release_ownership,
  1868. };
  1869. #endif
  1870. static void pnv_pci_ioda_setup_opal_tce_kill(struct pnv_phb *phb)
  1871. {
  1872. const __be64 *swinvp;
  1873. /* OPAL variant of PHB3 invalidated TCEs */
  1874. swinvp = of_get_property(phb->hose->dn, "ibm,opal-tce-kill", NULL);
  1875. if (!swinvp)
  1876. return;
  1877. phb->ioda.tce_inval_reg_phys = be64_to_cpup(swinvp);
  1878. phb->ioda.tce_inval_reg = ioremap(phb->ioda.tce_inval_reg_phys, 8);
  1879. }
  1880. static __be64 *pnv_pci_ioda2_table_do_alloc_pages(int nid, unsigned shift,
  1881. unsigned levels, unsigned long limit,
  1882. unsigned long *current_offset, unsigned long *total_allocated)
  1883. {
  1884. struct page *tce_mem = NULL;
  1885. __be64 *addr, *tmp;
  1886. unsigned order = max_t(unsigned, shift, PAGE_SHIFT) - PAGE_SHIFT;
  1887. unsigned long allocated = 1UL << (order + PAGE_SHIFT);
  1888. unsigned entries = 1UL << (shift - 3);
  1889. long i;
  1890. tce_mem = alloc_pages_node(nid, GFP_KERNEL, order);
  1891. if (!tce_mem) {
  1892. pr_err("Failed to allocate a TCE memory, order=%d\n", order);
  1893. return NULL;
  1894. }
  1895. addr = page_address(tce_mem);
  1896. memset(addr, 0, allocated);
  1897. *total_allocated += allocated;
  1898. --levels;
  1899. if (!levels) {
  1900. *current_offset += allocated;
  1901. return addr;
  1902. }
  1903. for (i = 0; i < entries; ++i) {
  1904. tmp = pnv_pci_ioda2_table_do_alloc_pages(nid, shift,
  1905. levels, limit, current_offset, total_allocated);
  1906. if (!tmp)
  1907. break;
  1908. addr[i] = cpu_to_be64(__pa(tmp) |
  1909. TCE_PCI_READ | TCE_PCI_WRITE);
  1910. if (*current_offset >= limit)
  1911. break;
  1912. }
  1913. return addr;
  1914. }
  1915. static void pnv_pci_ioda2_table_do_free_pages(__be64 *addr,
  1916. unsigned long size, unsigned level);
  1917. static long pnv_pci_ioda2_table_alloc_pages(int nid, __u64 bus_offset,
  1918. __u32 page_shift, __u64 window_size, __u32 levels,
  1919. struct iommu_table *tbl)
  1920. {
  1921. void *addr;
  1922. unsigned long offset = 0, level_shift, total_allocated = 0;
  1923. const unsigned window_shift = ilog2(window_size);
  1924. unsigned entries_shift = window_shift - page_shift;
  1925. unsigned table_shift = max_t(unsigned, entries_shift + 3, PAGE_SHIFT);
  1926. const unsigned long tce_table_size = 1UL << table_shift;
  1927. if (!levels || (levels > POWERNV_IOMMU_MAX_LEVELS))
  1928. return -EINVAL;
  1929. if ((window_size > memory_hotplug_max()) || !is_power_of_2(window_size))
  1930. return -EINVAL;
  1931. /* Adjust direct table size from window_size and levels */
  1932. entries_shift = (entries_shift + levels - 1) / levels;
  1933. level_shift = entries_shift + 3;
  1934. level_shift = max_t(unsigned, level_shift, PAGE_SHIFT);
  1935. /* Allocate TCE table */
  1936. addr = pnv_pci_ioda2_table_do_alloc_pages(nid, level_shift,
  1937. levels, tce_table_size, &offset, &total_allocated);
  1938. /* addr==NULL means that the first level allocation failed */
  1939. if (!addr)
  1940. return -ENOMEM;
  1941. /*
  1942. * First level was allocated but some lower level failed as
  1943. * we did not allocate as much as we wanted,
  1944. * release partially allocated table.
  1945. */
  1946. if (offset < tce_table_size) {
  1947. pnv_pci_ioda2_table_do_free_pages(addr,
  1948. 1ULL << (level_shift - 3), levels - 1);
  1949. return -ENOMEM;
  1950. }
  1951. /* Setup linux iommu table */
  1952. pnv_pci_setup_iommu_table(tbl, addr, tce_table_size, bus_offset,
  1953. page_shift);
  1954. tbl->it_level_size = 1ULL << (level_shift - 3);
  1955. tbl->it_indirect_levels = levels - 1;
  1956. tbl->it_allocated_size = total_allocated;
  1957. pr_devel("Created TCE table: ws=%08llx ts=%lx @%08llx\n",
  1958. window_size, tce_table_size, bus_offset);
  1959. return 0;
  1960. }
  1961. static void pnv_pci_ioda2_table_do_free_pages(__be64 *addr,
  1962. unsigned long size, unsigned level)
  1963. {
  1964. const unsigned long addr_ul = (unsigned long) addr &
  1965. ~(TCE_PCI_READ | TCE_PCI_WRITE);
  1966. if (level) {
  1967. long i;
  1968. u64 *tmp = (u64 *) addr_ul;
  1969. for (i = 0; i < size; ++i) {
  1970. unsigned long hpa = be64_to_cpu(tmp[i]);
  1971. if (!(hpa & (TCE_PCI_READ | TCE_PCI_WRITE)))
  1972. continue;
  1973. pnv_pci_ioda2_table_do_free_pages(__va(hpa), size,
  1974. level - 1);
  1975. }
  1976. }
  1977. free_pages(addr_ul, get_order(size << 3));
  1978. }
  1979. static void pnv_pci_ioda2_table_free_pages(struct iommu_table *tbl)
  1980. {
  1981. const unsigned long size = tbl->it_indirect_levels ?
  1982. tbl->it_level_size : tbl->it_size;
  1983. if (!tbl->it_size)
  1984. return;
  1985. pnv_pci_ioda2_table_do_free_pages((__be64 *)tbl->it_base, size,
  1986. tbl->it_indirect_levels);
  1987. }
  1988. static void pnv_pci_ioda2_setup_dma_pe(struct pnv_phb *phb,
  1989. struct pnv_ioda_pe *pe)
  1990. {
  1991. int64_t rc;
  1992. /* We shouldn't already have a 32-bit DMA associated */
  1993. if (WARN_ON(pe->tce32_seg >= 0))
  1994. return;
  1995. /* TVE #1 is selected by PCI address bit 59 */
  1996. pe->tce_bypass_base = 1ull << 59;
  1997. iommu_register_group(&pe->table_group, phb->hose->global_number,
  1998. pe->pe_number);
  1999. /* The PE will reserve all possible 32-bits space */
  2000. pe->tce32_seg = 0;
  2001. pe_info(pe, "Setting up 32-bit TCE table at 0..%08x\n",
  2002. phb->ioda.m32_pci_base);
  2003. /* Setup linux iommu table */
  2004. pe->table_group.tce32_start = 0;
  2005. pe->table_group.tce32_size = phb->ioda.m32_pci_base;
  2006. pe->table_group.max_dynamic_windows_supported =
  2007. IOMMU_TABLE_GROUP_MAX_TABLES;
  2008. pe->table_group.max_levels = POWERNV_IOMMU_MAX_LEVELS;
  2009. pe->table_group.pgsizes = SZ_4K | SZ_64K | SZ_16M;
  2010. #ifdef CONFIG_IOMMU_API
  2011. pe->table_group.ops = &pnv_pci_ioda2_ops;
  2012. #endif
  2013. rc = pnv_pci_ioda2_setup_default_config(pe);
  2014. if (rc) {
  2015. if (pe->tce32_seg >= 0)
  2016. pe->tce32_seg = -1;
  2017. return;
  2018. }
  2019. if (pe->flags & PNV_IODA_PE_DEV)
  2020. iommu_add_device(&pe->pdev->dev);
  2021. else if (pe->flags & (PNV_IODA_PE_BUS | PNV_IODA_PE_BUS_ALL))
  2022. pnv_ioda_setup_bus_dma(pe, pe->pbus);
  2023. }
  2024. static void pnv_ioda_setup_dma(struct pnv_phb *phb)
  2025. {
  2026. struct pci_controller *hose = phb->hose;
  2027. unsigned int residual, remaining, segs, tw, base;
  2028. struct pnv_ioda_pe *pe;
  2029. /* If we have more PE# than segments available, hand out one
  2030. * per PE until we run out and let the rest fail. If not,
  2031. * then we assign at least one segment per PE, plus more based
  2032. * on the amount of devices under that PE
  2033. */
  2034. if (phb->ioda.dma_pe_count > phb->ioda.tce32_count)
  2035. residual = 0;
  2036. else
  2037. residual = phb->ioda.tce32_count -
  2038. phb->ioda.dma_pe_count;
  2039. pr_info("PCI: Domain %04x has %ld available 32-bit DMA segments\n",
  2040. hose->global_number, phb->ioda.tce32_count);
  2041. pr_info("PCI: %d PE# for a total weight of %d\n",
  2042. phb->ioda.dma_pe_count, phb->ioda.dma_weight);
  2043. pnv_pci_ioda_setup_opal_tce_kill(phb);
  2044. /* Walk our PE list and configure their DMA segments, hand them
  2045. * out one base segment plus any residual segments based on
  2046. * weight
  2047. */
  2048. remaining = phb->ioda.tce32_count;
  2049. tw = phb->ioda.dma_weight;
  2050. base = 0;
  2051. list_for_each_entry(pe, &phb->ioda.pe_dma_list, dma_link) {
  2052. if (!pe->dma_weight)
  2053. continue;
  2054. if (!remaining) {
  2055. pe_warn(pe, "No DMA32 resources available\n");
  2056. continue;
  2057. }
  2058. segs = 1;
  2059. if (residual) {
  2060. segs += ((pe->dma_weight * residual) + (tw / 2)) / tw;
  2061. if (segs > remaining)
  2062. segs = remaining;
  2063. }
  2064. /*
  2065. * For IODA2 compliant PHB3, we needn't care about the weight.
  2066. * The all available 32-bits DMA space will be assigned to
  2067. * the specific PE.
  2068. */
  2069. if (phb->type == PNV_PHB_IODA1) {
  2070. pe_info(pe, "DMA weight %d, assigned %d DMA32 segments\n",
  2071. pe->dma_weight, segs);
  2072. pnv_pci_ioda_setup_dma_pe(phb, pe, base, segs);
  2073. } else {
  2074. pe_info(pe, "Assign DMA32 space\n");
  2075. segs = 0;
  2076. pnv_pci_ioda2_setup_dma_pe(phb, pe);
  2077. }
  2078. remaining -= segs;
  2079. base += segs;
  2080. }
  2081. }
  2082. #ifdef CONFIG_PCI_MSI
  2083. static void pnv_ioda2_msi_eoi(struct irq_data *d)
  2084. {
  2085. unsigned int hw_irq = (unsigned int)irqd_to_hwirq(d);
  2086. struct irq_chip *chip = irq_data_get_irq_chip(d);
  2087. struct pnv_phb *phb = container_of(chip, struct pnv_phb,
  2088. ioda.irq_chip);
  2089. int64_t rc;
  2090. rc = opal_pci_msi_eoi(phb->opal_id, hw_irq);
  2091. WARN_ON_ONCE(rc);
  2092. icp_native_eoi(d);
  2093. }
  2094. static void set_msi_irq_chip(struct pnv_phb *phb, unsigned int virq)
  2095. {
  2096. struct irq_data *idata;
  2097. struct irq_chip *ichip;
  2098. if (phb->type != PNV_PHB_IODA2)
  2099. return;
  2100. if (!phb->ioda.irq_chip_init) {
  2101. /*
  2102. * First time we setup an MSI IRQ, we need to setup the
  2103. * corresponding IRQ chip to route correctly.
  2104. */
  2105. idata = irq_get_irq_data(virq);
  2106. ichip = irq_data_get_irq_chip(idata);
  2107. phb->ioda.irq_chip_init = 1;
  2108. phb->ioda.irq_chip = *ichip;
  2109. phb->ioda.irq_chip.irq_eoi = pnv_ioda2_msi_eoi;
  2110. }
  2111. irq_set_chip(virq, &phb->ioda.irq_chip);
  2112. }
  2113. #ifdef CONFIG_CXL_BASE
  2114. struct device_node *pnv_pci_get_phb_node(struct pci_dev *dev)
  2115. {
  2116. struct pci_controller *hose = pci_bus_to_host(dev->bus);
  2117. return of_node_get(hose->dn);
  2118. }
  2119. EXPORT_SYMBOL(pnv_pci_get_phb_node);
  2120. int pnv_phb_to_cxl_mode(struct pci_dev *dev, uint64_t mode)
  2121. {
  2122. struct pci_controller *hose = pci_bus_to_host(dev->bus);
  2123. struct pnv_phb *phb = hose->private_data;
  2124. struct pnv_ioda_pe *pe;
  2125. int rc;
  2126. pe = pnv_ioda_get_pe(dev);
  2127. if (!pe)
  2128. return -ENODEV;
  2129. pe_info(pe, "Switching PHB to CXL\n");
  2130. rc = opal_pci_set_phb_cxl_mode(phb->opal_id, mode, pe->pe_number);
  2131. if (rc)
  2132. dev_err(&dev->dev, "opal_pci_set_phb_cxl_mode failed: %i\n", rc);
  2133. return rc;
  2134. }
  2135. EXPORT_SYMBOL(pnv_phb_to_cxl_mode);
  2136. /* Find PHB for cxl dev and allocate MSI hwirqs?
  2137. * Returns the absolute hardware IRQ number
  2138. */
  2139. int pnv_cxl_alloc_hwirqs(struct pci_dev *dev, int num)
  2140. {
  2141. struct pci_controller *hose = pci_bus_to_host(dev->bus);
  2142. struct pnv_phb *phb = hose->private_data;
  2143. int hwirq = msi_bitmap_alloc_hwirqs(&phb->msi_bmp, num);
  2144. if (hwirq < 0) {
  2145. dev_warn(&dev->dev, "Failed to find a free MSI\n");
  2146. return -ENOSPC;
  2147. }
  2148. return phb->msi_base + hwirq;
  2149. }
  2150. EXPORT_SYMBOL(pnv_cxl_alloc_hwirqs);
  2151. void pnv_cxl_release_hwirqs(struct pci_dev *dev, int hwirq, int num)
  2152. {
  2153. struct pci_controller *hose = pci_bus_to_host(dev->bus);
  2154. struct pnv_phb *phb = hose->private_data;
  2155. msi_bitmap_free_hwirqs(&phb->msi_bmp, hwirq - phb->msi_base, num);
  2156. }
  2157. EXPORT_SYMBOL(pnv_cxl_release_hwirqs);
  2158. void pnv_cxl_release_hwirq_ranges(struct cxl_irq_ranges *irqs,
  2159. struct pci_dev *dev)
  2160. {
  2161. struct pci_controller *hose = pci_bus_to_host(dev->bus);
  2162. struct pnv_phb *phb = hose->private_data;
  2163. int i, hwirq;
  2164. for (i = 1; i < CXL_IRQ_RANGES; i++) {
  2165. if (!irqs->range[i])
  2166. continue;
  2167. pr_devel("cxl release irq range 0x%x: offset: 0x%lx limit: %ld\n",
  2168. i, irqs->offset[i],
  2169. irqs->range[i]);
  2170. hwirq = irqs->offset[i] - phb->msi_base;
  2171. msi_bitmap_free_hwirqs(&phb->msi_bmp, hwirq,
  2172. irqs->range[i]);
  2173. }
  2174. }
  2175. EXPORT_SYMBOL(pnv_cxl_release_hwirq_ranges);
  2176. int pnv_cxl_alloc_hwirq_ranges(struct cxl_irq_ranges *irqs,
  2177. struct pci_dev *dev, int num)
  2178. {
  2179. struct pci_controller *hose = pci_bus_to_host(dev->bus);
  2180. struct pnv_phb *phb = hose->private_data;
  2181. int i, hwirq, try;
  2182. memset(irqs, 0, sizeof(struct cxl_irq_ranges));
  2183. /* 0 is reserved for the multiplexed PSL DSI interrupt */
  2184. for (i = 1; i < CXL_IRQ_RANGES && num; i++) {
  2185. try = num;
  2186. while (try) {
  2187. hwirq = msi_bitmap_alloc_hwirqs(&phb->msi_bmp, try);
  2188. if (hwirq >= 0)
  2189. break;
  2190. try /= 2;
  2191. }
  2192. if (!try)
  2193. goto fail;
  2194. irqs->offset[i] = phb->msi_base + hwirq;
  2195. irqs->range[i] = try;
  2196. pr_devel("cxl alloc irq range 0x%x: offset: 0x%lx limit: %li\n",
  2197. i, irqs->offset[i], irqs->range[i]);
  2198. num -= try;
  2199. }
  2200. if (num)
  2201. goto fail;
  2202. return 0;
  2203. fail:
  2204. pnv_cxl_release_hwirq_ranges(irqs, dev);
  2205. return -ENOSPC;
  2206. }
  2207. EXPORT_SYMBOL(pnv_cxl_alloc_hwirq_ranges);
  2208. int pnv_cxl_get_irq_count(struct pci_dev *dev)
  2209. {
  2210. struct pci_controller *hose = pci_bus_to_host(dev->bus);
  2211. struct pnv_phb *phb = hose->private_data;
  2212. return phb->msi_bmp.irq_count;
  2213. }
  2214. EXPORT_SYMBOL(pnv_cxl_get_irq_count);
  2215. int pnv_cxl_ioda_msi_setup(struct pci_dev *dev, unsigned int hwirq,
  2216. unsigned int virq)
  2217. {
  2218. struct pci_controller *hose = pci_bus_to_host(dev->bus);
  2219. struct pnv_phb *phb = hose->private_data;
  2220. unsigned int xive_num = hwirq - phb->msi_base;
  2221. struct pnv_ioda_pe *pe;
  2222. int rc;
  2223. if (!(pe = pnv_ioda_get_pe(dev)))
  2224. return -ENODEV;
  2225. /* Assign XIVE to PE */
  2226. rc = opal_pci_set_xive_pe(phb->opal_id, pe->pe_number, xive_num);
  2227. if (rc) {
  2228. pe_warn(pe, "%s: OPAL error %d setting msi_base 0x%x "
  2229. "hwirq 0x%x XIVE 0x%x PE\n",
  2230. pci_name(dev), rc, phb->msi_base, hwirq, xive_num);
  2231. return -EIO;
  2232. }
  2233. set_msi_irq_chip(phb, virq);
  2234. return 0;
  2235. }
  2236. EXPORT_SYMBOL(pnv_cxl_ioda_msi_setup);
  2237. #endif
  2238. static int pnv_pci_ioda_msi_setup(struct pnv_phb *phb, struct pci_dev *dev,
  2239. unsigned int hwirq, unsigned int virq,
  2240. unsigned int is_64, struct msi_msg *msg)
  2241. {
  2242. struct pnv_ioda_pe *pe = pnv_ioda_get_pe(dev);
  2243. unsigned int xive_num = hwirq - phb->msi_base;
  2244. __be32 data;
  2245. int rc;
  2246. /* No PE assigned ? bail out ... no MSI for you ! */
  2247. if (pe == NULL)
  2248. return -ENXIO;
  2249. /* Check if we have an MVE */
  2250. if (pe->mve_number < 0)
  2251. return -ENXIO;
  2252. /* Force 32-bit MSI on some broken devices */
  2253. if (dev->no_64bit_msi)
  2254. is_64 = 0;
  2255. /* Assign XIVE to PE */
  2256. rc = opal_pci_set_xive_pe(phb->opal_id, pe->pe_number, xive_num);
  2257. if (rc) {
  2258. pr_warn("%s: OPAL error %d setting XIVE %d PE\n",
  2259. pci_name(dev), rc, xive_num);
  2260. return -EIO;
  2261. }
  2262. if (is_64) {
  2263. __be64 addr64;
  2264. rc = opal_get_msi_64(phb->opal_id, pe->mve_number, xive_num, 1,
  2265. &addr64, &data);
  2266. if (rc) {
  2267. pr_warn("%s: OPAL error %d getting 64-bit MSI data\n",
  2268. pci_name(dev), rc);
  2269. return -EIO;
  2270. }
  2271. msg->address_hi = be64_to_cpu(addr64) >> 32;
  2272. msg->address_lo = be64_to_cpu(addr64) & 0xfffffffful;
  2273. } else {
  2274. __be32 addr32;
  2275. rc = opal_get_msi_32(phb->opal_id, pe->mve_number, xive_num, 1,
  2276. &addr32, &data);
  2277. if (rc) {
  2278. pr_warn("%s: OPAL error %d getting 32-bit MSI data\n",
  2279. pci_name(dev), rc);
  2280. return -EIO;
  2281. }
  2282. msg->address_hi = 0;
  2283. msg->address_lo = be32_to_cpu(addr32);
  2284. }
  2285. msg->data = be32_to_cpu(data);
  2286. set_msi_irq_chip(phb, virq);
  2287. pr_devel("%s: %s-bit MSI on hwirq %x (xive #%d),"
  2288. " address=%x_%08x data=%x PE# %d\n",
  2289. pci_name(dev), is_64 ? "64" : "32", hwirq, xive_num,
  2290. msg->address_hi, msg->address_lo, data, pe->pe_number);
  2291. return 0;
  2292. }
  2293. static void pnv_pci_init_ioda_msis(struct pnv_phb *phb)
  2294. {
  2295. unsigned int count;
  2296. const __be32 *prop = of_get_property(phb->hose->dn,
  2297. "ibm,opal-msi-ranges", NULL);
  2298. if (!prop) {
  2299. /* BML Fallback */
  2300. prop = of_get_property(phb->hose->dn, "msi-ranges", NULL);
  2301. }
  2302. if (!prop)
  2303. return;
  2304. phb->msi_base = be32_to_cpup(prop);
  2305. count = be32_to_cpup(prop + 1);
  2306. if (msi_bitmap_alloc(&phb->msi_bmp, count, phb->hose->dn)) {
  2307. pr_err("PCI %d: Failed to allocate MSI bitmap !\n",
  2308. phb->hose->global_number);
  2309. return;
  2310. }
  2311. phb->msi_setup = pnv_pci_ioda_msi_setup;
  2312. phb->msi32_support = 1;
  2313. pr_info(" Allocated bitmap for %d MSIs (base IRQ 0x%x)\n",
  2314. count, phb->msi_base);
  2315. }
  2316. #else
  2317. static void pnv_pci_init_ioda_msis(struct pnv_phb *phb) { }
  2318. #endif /* CONFIG_PCI_MSI */
  2319. #ifdef CONFIG_PCI_IOV
  2320. static void pnv_pci_ioda_fixup_iov_resources(struct pci_dev *pdev)
  2321. {
  2322. struct pci_controller *hose;
  2323. struct pnv_phb *phb;
  2324. struct resource *res;
  2325. int i;
  2326. resource_size_t size;
  2327. struct pci_dn *pdn;
  2328. int mul, total_vfs;
  2329. if (!pdev->is_physfn || pdev->is_added)
  2330. return;
  2331. hose = pci_bus_to_host(pdev->bus);
  2332. phb = hose->private_data;
  2333. pdn = pci_get_pdn(pdev);
  2334. pdn->vfs_expanded = 0;
  2335. total_vfs = pci_sriov_get_totalvfs(pdev);
  2336. pdn->m64_per_iov = 1;
  2337. mul = phb->ioda.total_pe;
  2338. for (i = 0; i < PCI_SRIOV_NUM_BARS; i++) {
  2339. res = &pdev->resource[i + PCI_IOV_RESOURCES];
  2340. if (!res->flags || res->parent)
  2341. continue;
  2342. if (!pnv_pci_is_mem_pref_64(res->flags)) {
  2343. dev_warn(&pdev->dev, " non M64 VF BAR%d: %pR\n",
  2344. i, res);
  2345. continue;
  2346. }
  2347. size = pci_iov_resource_size(pdev, i + PCI_IOV_RESOURCES);
  2348. /* bigger than 64M */
  2349. if (size > (1 << 26)) {
  2350. dev_info(&pdev->dev, "PowerNV: VF BAR%d: %pR IOV size is bigger than 64M, roundup power2\n",
  2351. i, res);
  2352. pdn->m64_per_iov = M64_PER_IOV;
  2353. mul = roundup_pow_of_two(total_vfs);
  2354. break;
  2355. }
  2356. }
  2357. for (i = 0; i < PCI_SRIOV_NUM_BARS; i++) {
  2358. res = &pdev->resource[i + PCI_IOV_RESOURCES];
  2359. if (!res->flags || res->parent)
  2360. continue;
  2361. if (!pnv_pci_is_mem_pref_64(res->flags)) {
  2362. dev_warn(&pdev->dev, "Skipping expanding VF BAR%d: %pR\n",
  2363. i, res);
  2364. continue;
  2365. }
  2366. dev_dbg(&pdev->dev, " Fixing VF BAR%d: %pR to\n", i, res);
  2367. size = pci_iov_resource_size(pdev, i + PCI_IOV_RESOURCES);
  2368. res->end = res->start + size * mul - 1;
  2369. dev_dbg(&pdev->dev, " %pR\n", res);
  2370. dev_info(&pdev->dev, "VF BAR%d: %pR (expanded to %d VFs for PE alignment)",
  2371. i, res, mul);
  2372. }
  2373. pdn->vfs_expanded = mul;
  2374. }
  2375. #endif /* CONFIG_PCI_IOV */
  2376. /*
  2377. * This function is supposed to be called on basis of PE from top
  2378. * to bottom style. So the the I/O or MMIO segment assigned to
  2379. * parent PE could be overrided by its child PEs if necessary.
  2380. */
  2381. static void pnv_ioda_setup_pe_seg(struct pci_controller *hose,
  2382. struct pnv_ioda_pe *pe)
  2383. {
  2384. struct pnv_phb *phb = hose->private_data;
  2385. struct pci_bus_region region;
  2386. struct resource *res;
  2387. int i, index;
  2388. int rc;
  2389. /*
  2390. * NOTE: We only care PCI bus based PE for now. For PCI
  2391. * device based PE, for example SRIOV sensitive VF should
  2392. * be figured out later.
  2393. */
  2394. BUG_ON(!(pe->flags & (PNV_IODA_PE_BUS | PNV_IODA_PE_BUS_ALL)));
  2395. pci_bus_for_each_resource(pe->pbus, res, i) {
  2396. if (!res || !res->flags ||
  2397. res->start > res->end)
  2398. continue;
  2399. if (res->flags & IORESOURCE_IO) {
  2400. region.start = res->start - phb->ioda.io_pci_base;
  2401. region.end = res->end - phb->ioda.io_pci_base;
  2402. index = region.start / phb->ioda.io_segsize;
  2403. while (index < phb->ioda.total_pe &&
  2404. region.start <= region.end) {
  2405. phb->ioda.io_segmap[index] = pe->pe_number;
  2406. rc = opal_pci_map_pe_mmio_window(phb->opal_id,
  2407. pe->pe_number, OPAL_IO_WINDOW_TYPE, 0, index);
  2408. if (rc != OPAL_SUCCESS) {
  2409. pr_err("%s: OPAL error %d when mapping IO "
  2410. "segment #%d to PE#%d\n",
  2411. __func__, rc, index, pe->pe_number);
  2412. break;
  2413. }
  2414. region.start += phb->ioda.io_segsize;
  2415. index++;
  2416. }
  2417. } else if ((res->flags & IORESOURCE_MEM) &&
  2418. !pnv_pci_is_mem_pref_64(res->flags)) {
  2419. region.start = res->start -
  2420. hose->mem_offset[0] -
  2421. phb->ioda.m32_pci_base;
  2422. region.end = res->end -
  2423. hose->mem_offset[0] -
  2424. phb->ioda.m32_pci_base;
  2425. index = region.start / phb->ioda.m32_segsize;
  2426. while (index < phb->ioda.total_pe &&
  2427. region.start <= region.end) {
  2428. phb->ioda.m32_segmap[index] = pe->pe_number;
  2429. rc = opal_pci_map_pe_mmio_window(phb->opal_id,
  2430. pe->pe_number, OPAL_M32_WINDOW_TYPE, 0, index);
  2431. if (rc != OPAL_SUCCESS) {
  2432. pr_err("%s: OPAL error %d when mapping M32 "
  2433. "segment#%d to PE#%d",
  2434. __func__, rc, index, pe->pe_number);
  2435. break;
  2436. }
  2437. region.start += phb->ioda.m32_segsize;
  2438. index++;
  2439. }
  2440. }
  2441. }
  2442. }
  2443. static void pnv_pci_ioda_setup_seg(void)
  2444. {
  2445. struct pci_controller *tmp, *hose;
  2446. struct pnv_phb *phb;
  2447. struct pnv_ioda_pe *pe;
  2448. list_for_each_entry_safe(hose, tmp, &hose_list, list_node) {
  2449. phb = hose->private_data;
  2450. list_for_each_entry(pe, &phb->ioda.pe_list, list) {
  2451. pnv_ioda_setup_pe_seg(hose, pe);
  2452. }
  2453. }
  2454. }
  2455. static void pnv_pci_ioda_setup_DMA(void)
  2456. {
  2457. struct pci_controller *hose, *tmp;
  2458. struct pnv_phb *phb;
  2459. list_for_each_entry_safe(hose, tmp, &hose_list, list_node) {
  2460. pnv_ioda_setup_dma(hose->private_data);
  2461. /* Mark the PHB initialization done */
  2462. phb = hose->private_data;
  2463. phb->initialized = 1;
  2464. }
  2465. }
  2466. static void pnv_pci_ioda_create_dbgfs(void)
  2467. {
  2468. #ifdef CONFIG_DEBUG_FS
  2469. struct pci_controller *hose, *tmp;
  2470. struct pnv_phb *phb;
  2471. char name[16];
  2472. list_for_each_entry_safe(hose, tmp, &hose_list, list_node) {
  2473. phb = hose->private_data;
  2474. sprintf(name, "PCI%04x", hose->global_number);
  2475. phb->dbgfs = debugfs_create_dir(name, powerpc_debugfs_root);
  2476. if (!phb->dbgfs)
  2477. pr_warning("%s: Error on creating debugfs on PHB#%x\n",
  2478. __func__, hose->global_number);
  2479. }
  2480. #endif /* CONFIG_DEBUG_FS */
  2481. }
  2482. static void pnv_pci_ioda_fixup(void)
  2483. {
  2484. pnv_pci_ioda_setup_PEs();
  2485. pnv_pci_ioda_setup_seg();
  2486. pnv_pci_ioda_setup_DMA();
  2487. pnv_pci_ioda_create_dbgfs();
  2488. #ifdef CONFIG_EEH
  2489. eeh_init();
  2490. eeh_addr_cache_build();
  2491. #endif
  2492. }
  2493. /*
  2494. * Returns the alignment for I/O or memory windows for P2P
  2495. * bridges. That actually depends on how PEs are segmented.
  2496. * For now, we return I/O or M32 segment size for PE sensitive
  2497. * P2P bridges. Otherwise, the default values (4KiB for I/O,
  2498. * 1MiB for memory) will be returned.
  2499. *
  2500. * The current PCI bus might be put into one PE, which was
  2501. * create against the parent PCI bridge. For that case, we
  2502. * needn't enlarge the alignment so that we can save some
  2503. * resources.
  2504. */
  2505. static resource_size_t pnv_pci_window_alignment(struct pci_bus *bus,
  2506. unsigned long type)
  2507. {
  2508. struct pci_dev *bridge;
  2509. struct pci_controller *hose = pci_bus_to_host(bus);
  2510. struct pnv_phb *phb = hose->private_data;
  2511. int num_pci_bridges = 0;
  2512. bridge = bus->self;
  2513. while (bridge) {
  2514. if (pci_pcie_type(bridge) == PCI_EXP_TYPE_PCI_BRIDGE) {
  2515. num_pci_bridges++;
  2516. if (num_pci_bridges >= 2)
  2517. return 1;
  2518. }
  2519. bridge = bridge->bus->self;
  2520. }
  2521. /* We fail back to M32 if M64 isn't supported */
  2522. if (phb->ioda.m64_segsize &&
  2523. pnv_pci_is_mem_pref_64(type))
  2524. return phb->ioda.m64_segsize;
  2525. if (type & IORESOURCE_MEM)
  2526. return phb->ioda.m32_segsize;
  2527. return phb->ioda.io_segsize;
  2528. }
  2529. #ifdef CONFIG_PCI_IOV
  2530. static resource_size_t pnv_pci_iov_resource_alignment(struct pci_dev *pdev,
  2531. int resno)
  2532. {
  2533. struct pci_dn *pdn = pci_get_pdn(pdev);
  2534. resource_size_t align, iov_align;
  2535. iov_align = resource_size(&pdev->resource[resno]);
  2536. if (iov_align)
  2537. return iov_align;
  2538. align = pci_iov_resource_size(pdev, resno);
  2539. if (pdn->vfs_expanded)
  2540. return pdn->vfs_expanded * align;
  2541. return align;
  2542. }
  2543. #endif /* CONFIG_PCI_IOV */
  2544. /* Prevent enabling devices for which we couldn't properly
  2545. * assign a PE
  2546. */
  2547. static bool pnv_pci_enable_device_hook(struct pci_dev *dev)
  2548. {
  2549. struct pci_controller *hose = pci_bus_to_host(dev->bus);
  2550. struct pnv_phb *phb = hose->private_data;
  2551. struct pci_dn *pdn;
  2552. /* The function is probably called while the PEs have
  2553. * not be created yet. For example, resource reassignment
  2554. * during PCI probe period. We just skip the check if
  2555. * PEs isn't ready.
  2556. */
  2557. if (!phb->initialized)
  2558. return true;
  2559. pdn = pci_get_pdn(dev);
  2560. if (!pdn || pdn->pe_number == IODA_INVALID_PE)
  2561. return false;
  2562. return true;
  2563. }
  2564. static u32 pnv_ioda_bdfn_to_pe(struct pnv_phb *phb, struct pci_bus *bus,
  2565. u32 devfn)
  2566. {
  2567. return phb->ioda.pe_rmap[(bus->number << 8) | devfn];
  2568. }
  2569. static void pnv_pci_ioda_shutdown(struct pci_controller *hose)
  2570. {
  2571. struct pnv_phb *phb = hose->private_data;
  2572. opal_pci_reset(phb->opal_id, OPAL_RESET_PCI_IODA_TABLE,
  2573. OPAL_ASSERT_RESET);
  2574. }
  2575. static const struct pci_controller_ops pnv_pci_ioda_controller_ops = {
  2576. .dma_dev_setup = pnv_pci_dma_dev_setup,
  2577. #ifdef CONFIG_PCI_MSI
  2578. .setup_msi_irqs = pnv_setup_msi_irqs,
  2579. .teardown_msi_irqs = pnv_teardown_msi_irqs,
  2580. #endif
  2581. .enable_device_hook = pnv_pci_enable_device_hook,
  2582. .window_alignment = pnv_pci_window_alignment,
  2583. .reset_secondary_bus = pnv_pci_reset_secondary_bus,
  2584. .dma_set_mask = pnv_pci_ioda_dma_set_mask,
  2585. .shutdown = pnv_pci_ioda_shutdown,
  2586. };
  2587. static void __init pnv_pci_init_ioda_phb(struct device_node *np,
  2588. u64 hub_id, int ioda_type)
  2589. {
  2590. struct pci_controller *hose;
  2591. struct pnv_phb *phb;
  2592. unsigned long size, m32map_off, pemap_off, iomap_off = 0;
  2593. const __be64 *prop64;
  2594. const __be32 *prop32;
  2595. int len;
  2596. u64 phb_id;
  2597. void *aux;
  2598. long rc;
  2599. pr_info("Initializing IODA%d OPAL PHB %s\n", ioda_type, np->full_name);
  2600. prop64 = of_get_property(np, "ibm,opal-phbid", NULL);
  2601. if (!prop64) {
  2602. pr_err(" Missing \"ibm,opal-phbid\" property !\n");
  2603. return;
  2604. }
  2605. phb_id = be64_to_cpup(prop64);
  2606. pr_debug(" PHB-ID : 0x%016llx\n", phb_id);
  2607. phb = memblock_virt_alloc(sizeof(struct pnv_phb), 0);
  2608. /* Allocate PCI controller */
  2609. phb->hose = hose = pcibios_alloc_controller(np);
  2610. if (!phb->hose) {
  2611. pr_err(" Can't allocate PCI controller for %s\n",
  2612. np->full_name);
  2613. memblock_free(__pa(phb), sizeof(struct pnv_phb));
  2614. return;
  2615. }
  2616. spin_lock_init(&phb->lock);
  2617. prop32 = of_get_property(np, "bus-range", &len);
  2618. if (prop32 && len == 8) {
  2619. hose->first_busno = be32_to_cpu(prop32[0]);
  2620. hose->last_busno = be32_to_cpu(prop32[1]);
  2621. } else {
  2622. pr_warn(" Broken <bus-range> on %s\n", np->full_name);
  2623. hose->first_busno = 0;
  2624. hose->last_busno = 0xff;
  2625. }
  2626. hose->private_data = phb;
  2627. phb->hub_id = hub_id;
  2628. phb->opal_id = phb_id;
  2629. phb->type = ioda_type;
  2630. mutex_init(&phb->ioda.pe_alloc_mutex);
  2631. /* Detect specific models for error handling */
  2632. if (of_device_is_compatible(np, "ibm,p7ioc-pciex"))
  2633. phb->model = PNV_PHB_MODEL_P7IOC;
  2634. else if (of_device_is_compatible(np, "ibm,power8-pciex"))
  2635. phb->model = PNV_PHB_MODEL_PHB3;
  2636. else
  2637. phb->model = PNV_PHB_MODEL_UNKNOWN;
  2638. /* Parse 32-bit and IO ranges (if any) */
  2639. pci_process_bridge_OF_ranges(hose, np, !hose->global_number);
  2640. /* Get registers */
  2641. phb->regs = of_iomap(np, 0);
  2642. if (phb->regs == NULL)
  2643. pr_err(" Failed to map registers !\n");
  2644. /* Initialize more IODA stuff */
  2645. phb->ioda.total_pe = 1;
  2646. prop32 = of_get_property(np, "ibm,opal-num-pes", NULL);
  2647. if (prop32)
  2648. phb->ioda.total_pe = be32_to_cpup(prop32);
  2649. prop32 = of_get_property(np, "ibm,opal-reserved-pe", NULL);
  2650. if (prop32)
  2651. phb->ioda.reserved_pe = be32_to_cpup(prop32);
  2652. /* Parse 64-bit MMIO range */
  2653. pnv_ioda_parse_m64_window(phb);
  2654. phb->ioda.m32_size = resource_size(&hose->mem_resources[0]);
  2655. /* FW Has already off top 64k of M32 space (MSI space) */
  2656. phb->ioda.m32_size += 0x10000;
  2657. phb->ioda.m32_segsize = phb->ioda.m32_size / phb->ioda.total_pe;
  2658. phb->ioda.m32_pci_base = hose->mem_resources[0].start - hose->mem_offset[0];
  2659. phb->ioda.io_size = hose->pci_io_size;
  2660. phb->ioda.io_segsize = phb->ioda.io_size / phb->ioda.total_pe;
  2661. phb->ioda.io_pci_base = 0; /* XXX calculate this ? */
  2662. /* Allocate aux data & arrays. We don't have IO ports on PHB3 */
  2663. size = _ALIGN_UP(phb->ioda.total_pe / 8, sizeof(unsigned long));
  2664. m32map_off = size;
  2665. size += phb->ioda.total_pe * sizeof(phb->ioda.m32_segmap[0]);
  2666. if (phb->type == PNV_PHB_IODA1) {
  2667. iomap_off = size;
  2668. size += phb->ioda.total_pe * sizeof(phb->ioda.io_segmap[0]);
  2669. }
  2670. pemap_off = size;
  2671. size += phb->ioda.total_pe * sizeof(struct pnv_ioda_pe);
  2672. aux = memblock_virt_alloc(size, 0);
  2673. phb->ioda.pe_alloc = aux;
  2674. phb->ioda.m32_segmap = aux + m32map_off;
  2675. if (phb->type == PNV_PHB_IODA1)
  2676. phb->ioda.io_segmap = aux + iomap_off;
  2677. phb->ioda.pe_array = aux + pemap_off;
  2678. set_bit(phb->ioda.reserved_pe, phb->ioda.pe_alloc);
  2679. INIT_LIST_HEAD(&phb->ioda.pe_dma_list);
  2680. INIT_LIST_HEAD(&phb->ioda.pe_list);
  2681. mutex_init(&phb->ioda.pe_list_mutex);
  2682. /* Calculate how many 32-bit TCE segments we have */
  2683. phb->ioda.tce32_count = phb->ioda.m32_pci_base >> 28;
  2684. #if 0 /* We should really do that ... */
  2685. rc = opal_pci_set_phb_mem_window(opal->phb_id,
  2686. window_type,
  2687. window_num,
  2688. starting_real_address,
  2689. starting_pci_address,
  2690. segment_size);
  2691. #endif
  2692. pr_info(" %03d (%03d) PE's M32: 0x%x [segment=0x%x]\n",
  2693. phb->ioda.total_pe, phb->ioda.reserved_pe,
  2694. phb->ioda.m32_size, phb->ioda.m32_segsize);
  2695. if (phb->ioda.m64_size)
  2696. pr_info(" M64: 0x%lx [segment=0x%lx]\n",
  2697. phb->ioda.m64_size, phb->ioda.m64_segsize);
  2698. if (phb->ioda.io_size)
  2699. pr_info(" IO: 0x%x [segment=0x%x]\n",
  2700. phb->ioda.io_size, phb->ioda.io_segsize);
  2701. phb->hose->ops = &pnv_pci_ops;
  2702. phb->get_pe_state = pnv_ioda_get_pe_state;
  2703. phb->freeze_pe = pnv_ioda_freeze_pe;
  2704. phb->unfreeze_pe = pnv_ioda_unfreeze_pe;
  2705. /* Setup RID -> PE mapping function */
  2706. phb->bdfn_to_pe = pnv_ioda_bdfn_to_pe;
  2707. /* Setup TCEs */
  2708. phb->dma_dev_setup = pnv_pci_ioda_dma_dev_setup;
  2709. phb->dma_get_required_mask = pnv_pci_ioda_dma_get_required_mask;
  2710. /* Setup MSI support */
  2711. pnv_pci_init_ioda_msis(phb);
  2712. /*
  2713. * We pass the PCI probe flag PCI_REASSIGN_ALL_RSRC here
  2714. * to let the PCI core do resource assignment. It's supposed
  2715. * that the PCI core will do correct I/O and MMIO alignment
  2716. * for the P2P bridge bars so that each PCI bus (excluding
  2717. * the child P2P bridges) can form individual PE.
  2718. */
  2719. ppc_md.pcibios_fixup = pnv_pci_ioda_fixup;
  2720. hose->controller_ops = pnv_pci_ioda_controller_ops;
  2721. #ifdef CONFIG_PCI_IOV
  2722. ppc_md.pcibios_fixup_sriov = pnv_pci_ioda_fixup_iov_resources;
  2723. ppc_md.pcibios_iov_resource_alignment = pnv_pci_iov_resource_alignment;
  2724. #endif
  2725. pci_add_flags(PCI_REASSIGN_ALL_RSRC);
  2726. /* Reset IODA tables to a clean state */
  2727. rc = opal_pci_reset(phb_id, OPAL_RESET_PCI_IODA_TABLE, OPAL_ASSERT_RESET);
  2728. if (rc)
  2729. pr_warning(" OPAL Error %ld performing IODA table reset !\n", rc);
  2730. /* If we're running in kdump kerenl, the previous kerenl never
  2731. * shutdown PCI devices correctly. We already got IODA table
  2732. * cleaned out. So we have to issue PHB reset to stop all PCI
  2733. * transactions from previous kerenl.
  2734. */
  2735. if (is_kdump_kernel()) {
  2736. pr_info(" Issue PHB reset ...\n");
  2737. pnv_eeh_phb_reset(hose, EEH_RESET_FUNDAMENTAL);
  2738. pnv_eeh_phb_reset(hose, EEH_RESET_DEACTIVATE);
  2739. }
  2740. /* Remove M64 resource if we can't configure it successfully */
  2741. if (!phb->init_m64 || phb->init_m64(phb))
  2742. hose->mem_resources[1].flags = 0;
  2743. }
  2744. void __init pnv_pci_init_ioda2_phb(struct device_node *np)
  2745. {
  2746. pnv_pci_init_ioda_phb(np, 0, PNV_PHB_IODA2);
  2747. }
  2748. void __init pnv_pci_init_ioda_hub(struct device_node *np)
  2749. {
  2750. struct device_node *phbn;
  2751. const __be64 *prop64;
  2752. u64 hub_id;
  2753. pr_info("Probing IODA IO-Hub %s\n", np->full_name);
  2754. prop64 = of_get_property(np, "ibm,opal-hubid", NULL);
  2755. if (!prop64) {
  2756. pr_err(" Missing \"ibm,opal-hubid\" property !\n");
  2757. return;
  2758. }
  2759. hub_id = be64_to_cpup(prop64);
  2760. pr_devel(" HUB-ID : 0x%016llx\n", hub_id);
  2761. /* Count child PHBs */
  2762. for_each_child_of_node(np, phbn) {
  2763. /* Look for IODA1 PHBs */
  2764. if (of_device_is_compatible(phbn, "ibm,ioda-phb"))
  2765. pnv_pci_init_ioda_phb(phbn, hub_id, PNV_PHB_IODA1);
  2766. }
  2767. }