rt.c 49 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975976977978979980981982983984985986987988989990991992993994995996997998999100010011002100310041005100610071008100910101011101210131014101510161017101810191020102110221023102410251026102710281029103010311032103310341035103610371038103910401041104210431044104510461047104810491050105110521053105410551056105710581059106010611062106310641065106610671068106910701071107210731074107510761077107810791080108110821083108410851086108710881089109010911092109310941095109610971098109911001101110211031104110511061107110811091110111111121113111411151116111711181119112011211122112311241125112611271128112911301131113211331134113511361137113811391140114111421143114411451146114711481149115011511152115311541155115611571158115911601161116211631164116511661167116811691170117111721173117411751176117711781179118011811182118311841185118611871188118911901191119211931194119511961197119811991200120112021203120412051206120712081209121012111212121312141215121612171218121912201221122212231224122512261227122812291230123112321233123412351236123712381239124012411242124312441245124612471248124912501251125212531254125512561257125812591260126112621263126412651266126712681269127012711272127312741275127612771278127912801281128212831284128512861287128812891290129112921293129412951296129712981299130013011302130313041305130613071308130913101311131213131314131513161317131813191320132113221323132413251326132713281329133013311332133313341335133613371338133913401341134213431344134513461347134813491350135113521353135413551356135713581359136013611362136313641365136613671368136913701371137213731374137513761377137813791380138113821383138413851386138713881389139013911392139313941395139613971398139914001401140214031404140514061407140814091410141114121413141414151416141714181419142014211422142314241425142614271428142914301431143214331434143514361437143814391440144114421443144414451446144714481449145014511452145314541455145614571458145914601461146214631464146514661467146814691470147114721473147414751476147714781479148014811482148314841485148614871488148914901491149214931494149514961497149814991500150115021503150415051506150715081509151015111512151315141515151615171518151915201521152215231524152515261527152815291530153115321533153415351536153715381539154015411542154315441545154615471548154915501551155215531554155515561557155815591560156115621563156415651566156715681569157015711572157315741575157615771578157915801581158215831584158515861587158815891590159115921593159415951596159715981599160016011602160316041605160616071608160916101611161216131614161516161617161816191620162116221623162416251626162716281629163016311632163316341635163616371638163916401641164216431644164516461647164816491650165116521653165416551656165716581659166016611662166316641665166616671668166916701671167216731674167516761677167816791680168116821683168416851686168716881689169016911692169316941695169616971698169917001701170217031704170517061707170817091710171117121713171417151716171717181719172017211722172317241725172617271728172917301731173217331734173517361737173817391740174117421743174417451746174717481749175017511752175317541755175617571758175917601761176217631764176517661767176817691770177117721773177417751776177717781779178017811782178317841785178617871788178917901791179217931794179517961797179817991800180118021803180418051806180718081809181018111812181318141815181618171818181918201821182218231824182518261827182818291830183118321833183418351836183718381839184018411842184318441845184618471848184918501851185218531854185518561857185818591860186118621863186418651866186718681869187018711872187318741875187618771878187918801881188218831884188518861887188818891890189118921893189418951896189718981899190019011902190319041905190619071908190919101911191219131914191519161917191819191920192119221923192419251926192719281929193019311932193319341935193619371938193919401941194219431944194519461947194819491950195119521953195419551956195719581959196019611962196319641965196619671968196919701971197219731974197519761977197819791980198119821983198419851986198719881989199019911992199319941995199619971998199920002001200220032004200520062007200820092010201120122013201420152016201720182019202020212022202320242025202620272028202920302031203220332034203520362037203820392040204120422043204420452046204720482049205020512052205320542055205620572058205920602061206220632064206520662067206820692070207120722073207420752076207720782079208020812082208320842085208620872088208920902091209220932094209520962097209820992100210121022103210421052106210721082109211021112112211321142115211621172118211921202121212221232124212521262127212821292130213121322133213421352136213721382139214021412142214321442145214621472148214921502151215221532154215521562157215821592160216121622163216421652166216721682169
  1. /*
  2. * Real-Time Scheduling Class (mapped to the SCHED_FIFO and SCHED_RR
  3. * policies)
  4. */
  5. #include "sched.h"
  6. #include <linux/slab.h>
  7. int sched_rr_timeslice = RR_TIMESLICE;
  8. static int do_sched_rt_period_timer(struct rt_bandwidth *rt_b, int overrun);
  9. struct rt_bandwidth def_rt_bandwidth;
  10. static enum hrtimer_restart sched_rt_period_timer(struct hrtimer *timer)
  11. {
  12. struct rt_bandwidth *rt_b =
  13. container_of(timer, struct rt_bandwidth, rt_period_timer);
  14. ktime_t now;
  15. int overrun;
  16. int idle = 0;
  17. for (;;) {
  18. now = hrtimer_cb_get_time(timer);
  19. overrun = hrtimer_forward(timer, now, rt_b->rt_period);
  20. if (!overrun)
  21. break;
  22. idle = do_sched_rt_period_timer(rt_b, overrun);
  23. }
  24. return idle ? HRTIMER_NORESTART : HRTIMER_RESTART;
  25. }
  26. void init_rt_bandwidth(struct rt_bandwidth *rt_b, u64 period, u64 runtime)
  27. {
  28. rt_b->rt_period = ns_to_ktime(period);
  29. rt_b->rt_runtime = runtime;
  30. raw_spin_lock_init(&rt_b->rt_runtime_lock);
  31. hrtimer_init(&rt_b->rt_period_timer,
  32. CLOCK_MONOTONIC, HRTIMER_MODE_REL);
  33. rt_b->rt_period_timer.function = sched_rt_period_timer;
  34. }
  35. static void start_rt_bandwidth(struct rt_bandwidth *rt_b)
  36. {
  37. if (!rt_bandwidth_enabled() || rt_b->rt_runtime == RUNTIME_INF)
  38. return;
  39. if (hrtimer_active(&rt_b->rt_period_timer))
  40. return;
  41. raw_spin_lock(&rt_b->rt_runtime_lock);
  42. start_bandwidth_timer(&rt_b->rt_period_timer, rt_b->rt_period);
  43. raw_spin_unlock(&rt_b->rt_runtime_lock);
  44. }
  45. void init_rt_rq(struct rt_rq *rt_rq, struct rq *rq)
  46. {
  47. struct rt_prio_array *array;
  48. int i;
  49. array = &rt_rq->active;
  50. for (i = 0; i < MAX_RT_PRIO; i++) {
  51. INIT_LIST_HEAD(array->queue + i);
  52. __clear_bit(i, array->bitmap);
  53. }
  54. /* delimiter for bitsearch: */
  55. __set_bit(MAX_RT_PRIO, array->bitmap);
  56. #if defined CONFIG_SMP
  57. rt_rq->highest_prio.curr = MAX_RT_PRIO;
  58. rt_rq->highest_prio.next = MAX_RT_PRIO;
  59. rt_rq->rt_nr_migratory = 0;
  60. rt_rq->overloaded = 0;
  61. plist_head_init(&rt_rq->pushable_tasks);
  62. #endif
  63. /* We start is dequeued state, because no RT tasks are queued */
  64. rt_rq->rt_queued = 0;
  65. rt_rq->rt_time = 0;
  66. rt_rq->rt_throttled = 0;
  67. rt_rq->rt_runtime = 0;
  68. raw_spin_lock_init(&rt_rq->rt_runtime_lock);
  69. }
  70. #ifdef CONFIG_RT_GROUP_SCHED
  71. static void destroy_rt_bandwidth(struct rt_bandwidth *rt_b)
  72. {
  73. hrtimer_cancel(&rt_b->rt_period_timer);
  74. }
  75. #define rt_entity_is_task(rt_se) (!(rt_se)->my_q)
  76. static inline struct task_struct *rt_task_of(struct sched_rt_entity *rt_se)
  77. {
  78. #ifdef CONFIG_SCHED_DEBUG
  79. WARN_ON_ONCE(!rt_entity_is_task(rt_se));
  80. #endif
  81. return container_of(rt_se, struct task_struct, rt);
  82. }
  83. static inline struct rq *rq_of_rt_rq(struct rt_rq *rt_rq)
  84. {
  85. return rt_rq->rq;
  86. }
  87. static inline struct rt_rq *rt_rq_of_se(struct sched_rt_entity *rt_se)
  88. {
  89. return rt_se->rt_rq;
  90. }
  91. static inline struct rq *rq_of_rt_se(struct sched_rt_entity *rt_se)
  92. {
  93. struct rt_rq *rt_rq = rt_se->rt_rq;
  94. return rt_rq->rq;
  95. }
  96. void free_rt_sched_group(struct task_group *tg)
  97. {
  98. int i;
  99. if (tg->rt_se)
  100. destroy_rt_bandwidth(&tg->rt_bandwidth);
  101. for_each_possible_cpu(i) {
  102. if (tg->rt_rq)
  103. kfree(tg->rt_rq[i]);
  104. if (tg->rt_se)
  105. kfree(tg->rt_se[i]);
  106. }
  107. kfree(tg->rt_rq);
  108. kfree(tg->rt_se);
  109. }
  110. void init_tg_rt_entry(struct task_group *tg, struct rt_rq *rt_rq,
  111. struct sched_rt_entity *rt_se, int cpu,
  112. struct sched_rt_entity *parent)
  113. {
  114. struct rq *rq = cpu_rq(cpu);
  115. rt_rq->highest_prio.curr = MAX_RT_PRIO;
  116. rt_rq->rt_nr_boosted = 0;
  117. rt_rq->rq = rq;
  118. rt_rq->tg = tg;
  119. tg->rt_rq[cpu] = rt_rq;
  120. tg->rt_se[cpu] = rt_se;
  121. if (!rt_se)
  122. return;
  123. if (!parent)
  124. rt_se->rt_rq = &rq->rt;
  125. else
  126. rt_se->rt_rq = parent->my_q;
  127. rt_se->my_q = rt_rq;
  128. rt_se->parent = parent;
  129. INIT_LIST_HEAD(&rt_se->run_list);
  130. }
  131. int alloc_rt_sched_group(struct task_group *tg, struct task_group *parent)
  132. {
  133. struct rt_rq *rt_rq;
  134. struct sched_rt_entity *rt_se;
  135. int i;
  136. tg->rt_rq = kzalloc(sizeof(rt_rq) * nr_cpu_ids, GFP_KERNEL);
  137. if (!tg->rt_rq)
  138. goto err;
  139. tg->rt_se = kzalloc(sizeof(rt_se) * nr_cpu_ids, GFP_KERNEL);
  140. if (!tg->rt_se)
  141. goto err;
  142. init_rt_bandwidth(&tg->rt_bandwidth,
  143. ktime_to_ns(def_rt_bandwidth.rt_period), 0);
  144. for_each_possible_cpu(i) {
  145. rt_rq = kzalloc_node(sizeof(struct rt_rq),
  146. GFP_KERNEL, cpu_to_node(i));
  147. if (!rt_rq)
  148. goto err;
  149. rt_se = kzalloc_node(sizeof(struct sched_rt_entity),
  150. GFP_KERNEL, cpu_to_node(i));
  151. if (!rt_se)
  152. goto err_free_rq;
  153. init_rt_rq(rt_rq, cpu_rq(i));
  154. rt_rq->rt_runtime = tg->rt_bandwidth.rt_runtime;
  155. init_tg_rt_entry(tg, rt_rq, rt_se, i, parent->rt_se[i]);
  156. }
  157. return 1;
  158. err_free_rq:
  159. kfree(rt_rq);
  160. err:
  161. return 0;
  162. }
  163. #else /* CONFIG_RT_GROUP_SCHED */
  164. #define rt_entity_is_task(rt_se) (1)
  165. static inline struct task_struct *rt_task_of(struct sched_rt_entity *rt_se)
  166. {
  167. return container_of(rt_se, struct task_struct, rt);
  168. }
  169. static inline struct rq *rq_of_rt_rq(struct rt_rq *rt_rq)
  170. {
  171. return container_of(rt_rq, struct rq, rt);
  172. }
  173. static inline struct rq *rq_of_rt_se(struct sched_rt_entity *rt_se)
  174. {
  175. struct task_struct *p = rt_task_of(rt_se);
  176. return task_rq(p);
  177. }
  178. static inline struct rt_rq *rt_rq_of_se(struct sched_rt_entity *rt_se)
  179. {
  180. struct rq *rq = rq_of_rt_se(rt_se);
  181. return &rq->rt;
  182. }
  183. void free_rt_sched_group(struct task_group *tg) { }
  184. int alloc_rt_sched_group(struct task_group *tg, struct task_group *parent)
  185. {
  186. return 1;
  187. }
  188. #endif /* CONFIG_RT_GROUP_SCHED */
  189. #ifdef CONFIG_SMP
  190. static int pull_rt_task(struct rq *this_rq);
  191. static inline bool need_pull_rt_task(struct rq *rq, struct task_struct *prev)
  192. {
  193. /* Try to pull RT tasks here if we lower this rq's prio */
  194. return rq->rt.highest_prio.curr > prev->prio;
  195. }
  196. static inline int rt_overloaded(struct rq *rq)
  197. {
  198. return atomic_read(&rq->rd->rto_count);
  199. }
  200. static inline void rt_set_overload(struct rq *rq)
  201. {
  202. if (!rq->online)
  203. return;
  204. cpumask_set_cpu(rq->cpu, rq->rd->rto_mask);
  205. /*
  206. * Make sure the mask is visible before we set
  207. * the overload count. That is checked to determine
  208. * if we should look at the mask. It would be a shame
  209. * if we looked at the mask, but the mask was not
  210. * updated yet.
  211. *
  212. * Matched by the barrier in pull_rt_task().
  213. */
  214. smp_wmb();
  215. atomic_inc(&rq->rd->rto_count);
  216. }
  217. static inline void rt_clear_overload(struct rq *rq)
  218. {
  219. if (!rq->online)
  220. return;
  221. /* the order here really doesn't matter */
  222. atomic_dec(&rq->rd->rto_count);
  223. cpumask_clear_cpu(rq->cpu, rq->rd->rto_mask);
  224. }
  225. static void update_rt_migration(struct rt_rq *rt_rq)
  226. {
  227. if (rt_rq->rt_nr_migratory && rt_rq->rt_nr_total > 1) {
  228. if (!rt_rq->overloaded) {
  229. rt_set_overload(rq_of_rt_rq(rt_rq));
  230. rt_rq->overloaded = 1;
  231. }
  232. } else if (rt_rq->overloaded) {
  233. rt_clear_overload(rq_of_rt_rq(rt_rq));
  234. rt_rq->overloaded = 0;
  235. }
  236. }
  237. static void inc_rt_migration(struct sched_rt_entity *rt_se, struct rt_rq *rt_rq)
  238. {
  239. struct task_struct *p;
  240. if (!rt_entity_is_task(rt_se))
  241. return;
  242. p = rt_task_of(rt_se);
  243. rt_rq = &rq_of_rt_rq(rt_rq)->rt;
  244. rt_rq->rt_nr_total++;
  245. if (p->nr_cpus_allowed > 1)
  246. rt_rq->rt_nr_migratory++;
  247. update_rt_migration(rt_rq);
  248. }
  249. static void dec_rt_migration(struct sched_rt_entity *rt_se, struct rt_rq *rt_rq)
  250. {
  251. struct task_struct *p;
  252. if (!rt_entity_is_task(rt_se))
  253. return;
  254. p = rt_task_of(rt_se);
  255. rt_rq = &rq_of_rt_rq(rt_rq)->rt;
  256. rt_rq->rt_nr_total--;
  257. if (p->nr_cpus_allowed > 1)
  258. rt_rq->rt_nr_migratory--;
  259. update_rt_migration(rt_rq);
  260. }
  261. static inline int has_pushable_tasks(struct rq *rq)
  262. {
  263. return !plist_head_empty(&rq->rt.pushable_tasks);
  264. }
  265. static inline void set_post_schedule(struct rq *rq)
  266. {
  267. /*
  268. * We detect this state here so that we can avoid taking the RQ
  269. * lock again later if there is no need to push
  270. */
  271. rq->post_schedule = has_pushable_tasks(rq);
  272. }
  273. static void enqueue_pushable_task(struct rq *rq, struct task_struct *p)
  274. {
  275. plist_del(&p->pushable_tasks, &rq->rt.pushable_tasks);
  276. plist_node_init(&p->pushable_tasks, p->prio);
  277. plist_add(&p->pushable_tasks, &rq->rt.pushable_tasks);
  278. /* Update the highest prio pushable task */
  279. if (p->prio < rq->rt.highest_prio.next)
  280. rq->rt.highest_prio.next = p->prio;
  281. }
  282. static void dequeue_pushable_task(struct rq *rq, struct task_struct *p)
  283. {
  284. plist_del(&p->pushable_tasks, &rq->rt.pushable_tasks);
  285. /* Update the new highest prio pushable task */
  286. if (has_pushable_tasks(rq)) {
  287. p = plist_first_entry(&rq->rt.pushable_tasks,
  288. struct task_struct, pushable_tasks);
  289. rq->rt.highest_prio.next = p->prio;
  290. } else
  291. rq->rt.highest_prio.next = MAX_RT_PRIO;
  292. }
  293. #else
  294. static inline void enqueue_pushable_task(struct rq *rq, struct task_struct *p)
  295. {
  296. }
  297. static inline void dequeue_pushable_task(struct rq *rq, struct task_struct *p)
  298. {
  299. }
  300. static inline
  301. void inc_rt_migration(struct sched_rt_entity *rt_se, struct rt_rq *rt_rq)
  302. {
  303. }
  304. static inline
  305. void dec_rt_migration(struct sched_rt_entity *rt_se, struct rt_rq *rt_rq)
  306. {
  307. }
  308. static inline bool need_pull_rt_task(struct rq *rq, struct task_struct *prev)
  309. {
  310. return false;
  311. }
  312. static inline int pull_rt_task(struct rq *this_rq)
  313. {
  314. return 0;
  315. }
  316. static inline void set_post_schedule(struct rq *rq)
  317. {
  318. }
  319. #endif /* CONFIG_SMP */
  320. static void enqueue_top_rt_rq(struct rt_rq *rt_rq);
  321. static void dequeue_top_rt_rq(struct rt_rq *rt_rq);
  322. static inline int on_rt_rq(struct sched_rt_entity *rt_se)
  323. {
  324. return !list_empty(&rt_se->run_list);
  325. }
  326. #ifdef CONFIG_RT_GROUP_SCHED
  327. static inline u64 sched_rt_runtime(struct rt_rq *rt_rq)
  328. {
  329. if (!rt_rq->tg)
  330. return RUNTIME_INF;
  331. return rt_rq->rt_runtime;
  332. }
  333. static inline u64 sched_rt_period(struct rt_rq *rt_rq)
  334. {
  335. return ktime_to_ns(rt_rq->tg->rt_bandwidth.rt_period);
  336. }
  337. typedef struct task_group *rt_rq_iter_t;
  338. static inline struct task_group *next_task_group(struct task_group *tg)
  339. {
  340. do {
  341. tg = list_entry_rcu(tg->list.next,
  342. typeof(struct task_group), list);
  343. } while (&tg->list != &task_groups && task_group_is_autogroup(tg));
  344. if (&tg->list == &task_groups)
  345. tg = NULL;
  346. return tg;
  347. }
  348. #define for_each_rt_rq(rt_rq, iter, rq) \
  349. for (iter = container_of(&task_groups, typeof(*iter), list); \
  350. (iter = next_task_group(iter)) && \
  351. (rt_rq = iter->rt_rq[cpu_of(rq)]);)
  352. #define for_each_sched_rt_entity(rt_se) \
  353. for (; rt_se; rt_se = rt_se->parent)
  354. static inline struct rt_rq *group_rt_rq(struct sched_rt_entity *rt_se)
  355. {
  356. return rt_se->my_q;
  357. }
  358. static void enqueue_rt_entity(struct sched_rt_entity *rt_se, bool head);
  359. static void dequeue_rt_entity(struct sched_rt_entity *rt_se);
  360. static void sched_rt_rq_enqueue(struct rt_rq *rt_rq)
  361. {
  362. struct task_struct *curr = rq_of_rt_rq(rt_rq)->curr;
  363. struct rq *rq = rq_of_rt_rq(rt_rq);
  364. struct sched_rt_entity *rt_se;
  365. int cpu = cpu_of(rq);
  366. rt_se = rt_rq->tg->rt_se[cpu];
  367. if (rt_rq->rt_nr_running) {
  368. if (!rt_se)
  369. enqueue_top_rt_rq(rt_rq);
  370. else if (!on_rt_rq(rt_se))
  371. enqueue_rt_entity(rt_se, false);
  372. if (rt_rq->highest_prio.curr < curr->prio)
  373. resched_curr(rq);
  374. }
  375. }
  376. static void sched_rt_rq_dequeue(struct rt_rq *rt_rq)
  377. {
  378. struct sched_rt_entity *rt_se;
  379. int cpu = cpu_of(rq_of_rt_rq(rt_rq));
  380. rt_se = rt_rq->tg->rt_se[cpu];
  381. if (!rt_se)
  382. dequeue_top_rt_rq(rt_rq);
  383. else if (on_rt_rq(rt_se))
  384. dequeue_rt_entity(rt_se);
  385. }
  386. static inline int rt_rq_throttled(struct rt_rq *rt_rq)
  387. {
  388. return rt_rq->rt_throttled && !rt_rq->rt_nr_boosted;
  389. }
  390. static int rt_se_boosted(struct sched_rt_entity *rt_se)
  391. {
  392. struct rt_rq *rt_rq = group_rt_rq(rt_se);
  393. struct task_struct *p;
  394. if (rt_rq)
  395. return !!rt_rq->rt_nr_boosted;
  396. p = rt_task_of(rt_se);
  397. return p->prio != p->normal_prio;
  398. }
  399. #ifdef CONFIG_SMP
  400. static inline const struct cpumask *sched_rt_period_mask(void)
  401. {
  402. return this_rq()->rd->span;
  403. }
  404. #else
  405. static inline const struct cpumask *sched_rt_period_mask(void)
  406. {
  407. return cpu_online_mask;
  408. }
  409. #endif
  410. static inline
  411. struct rt_rq *sched_rt_period_rt_rq(struct rt_bandwidth *rt_b, int cpu)
  412. {
  413. return container_of(rt_b, struct task_group, rt_bandwidth)->rt_rq[cpu];
  414. }
  415. static inline struct rt_bandwidth *sched_rt_bandwidth(struct rt_rq *rt_rq)
  416. {
  417. return &rt_rq->tg->rt_bandwidth;
  418. }
  419. #else /* !CONFIG_RT_GROUP_SCHED */
  420. static inline u64 sched_rt_runtime(struct rt_rq *rt_rq)
  421. {
  422. return rt_rq->rt_runtime;
  423. }
  424. static inline u64 sched_rt_period(struct rt_rq *rt_rq)
  425. {
  426. return ktime_to_ns(def_rt_bandwidth.rt_period);
  427. }
  428. typedef struct rt_rq *rt_rq_iter_t;
  429. #define for_each_rt_rq(rt_rq, iter, rq) \
  430. for ((void) iter, rt_rq = &rq->rt; rt_rq; rt_rq = NULL)
  431. #define for_each_sched_rt_entity(rt_se) \
  432. for (; rt_se; rt_se = NULL)
  433. static inline struct rt_rq *group_rt_rq(struct sched_rt_entity *rt_se)
  434. {
  435. return NULL;
  436. }
  437. static inline void sched_rt_rq_enqueue(struct rt_rq *rt_rq)
  438. {
  439. struct rq *rq = rq_of_rt_rq(rt_rq);
  440. if (!rt_rq->rt_nr_running)
  441. return;
  442. enqueue_top_rt_rq(rt_rq);
  443. resched_curr(rq);
  444. }
  445. static inline void sched_rt_rq_dequeue(struct rt_rq *rt_rq)
  446. {
  447. dequeue_top_rt_rq(rt_rq);
  448. }
  449. static inline int rt_rq_throttled(struct rt_rq *rt_rq)
  450. {
  451. return rt_rq->rt_throttled;
  452. }
  453. static inline const struct cpumask *sched_rt_period_mask(void)
  454. {
  455. return cpu_online_mask;
  456. }
  457. static inline
  458. struct rt_rq *sched_rt_period_rt_rq(struct rt_bandwidth *rt_b, int cpu)
  459. {
  460. return &cpu_rq(cpu)->rt;
  461. }
  462. static inline struct rt_bandwidth *sched_rt_bandwidth(struct rt_rq *rt_rq)
  463. {
  464. return &def_rt_bandwidth;
  465. }
  466. #endif /* CONFIG_RT_GROUP_SCHED */
  467. bool sched_rt_bandwidth_account(struct rt_rq *rt_rq)
  468. {
  469. struct rt_bandwidth *rt_b = sched_rt_bandwidth(rt_rq);
  470. return (hrtimer_active(&rt_b->rt_period_timer) ||
  471. rt_rq->rt_time < rt_b->rt_runtime);
  472. }
  473. #ifdef CONFIG_SMP
  474. /*
  475. * We ran out of runtime, see if we can borrow some from our neighbours.
  476. */
  477. static int do_balance_runtime(struct rt_rq *rt_rq)
  478. {
  479. struct rt_bandwidth *rt_b = sched_rt_bandwidth(rt_rq);
  480. struct root_domain *rd = rq_of_rt_rq(rt_rq)->rd;
  481. int i, weight, more = 0;
  482. u64 rt_period;
  483. weight = cpumask_weight(rd->span);
  484. raw_spin_lock(&rt_b->rt_runtime_lock);
  485. rt_period = ktime_to_ns(rt_b->rt_period);
  486. for_each_cpu(i, rd->span) {
  487. struct rt_rq *iter = sched_rt_period_rt_rq(rt_b, i);
  488. s64 diff;
  489. if (iter == rt_rq)
  490. continue;
  491. raw_spin_lock(&iter->rt_runtime_lock);
  492. /*
  493. * Either all rqs have inf runtime and there's nothing to steal
  494. * or __disable_runtime() below sets a specific rq to inf to
  495. * indicate its been disabled and disalow stealing.
  496. */
  497. if (iter->rt_runtime == RUNTIME_INF)
  498. goto next;
  499. /*
  500. * From runqueues with spare time, take 1/n part of their
  501. * spare time, but no more than our period.
  502. */
  503. diff = iter->rt_runtime - iter->rt_time;
  504. if (diff > 0) {
  505. diff = div_u64((u64)diff, weight);
  506. if (rt_rq->rt_runtime + diff > rt_period)
  507. diff = rt_period - rt_rq->rt_runtime;
  508. iter->rt_runtime -= diff;
  509. rt_rq->rt_runtime += diff;
  510. more = 1;
  511. if (rt_rq->rt_runtime == rt_period) {
  512. raw_spin_unlock(&iter->rt_runtime_lock);
  513. break;
  514. }
  515. }
  516. next:
  517. raw_spin_unlock(&iter->rt_runtime_lock);
  518. }
  519. raw_spin_unlock(&rt_b->rt_runtime_lock);
  520. return more;
  521. }
  522. /*
  523. * Ensure this RQ takes back all the runtime it lend to its neighbours.
  524. */
  525. static void __disable_runtime(struct rq *rq)
  526. {
  527. struct root_domain *rd = rq->rd;
  528. rt_rq_iter_t iter;
  529. struct rt_rq *rt_rq;
  530. if (unlikely(!scheduler_running))
  531. return;
  532. for_each_rt_rq(rt_rq, iter, rq) {
  533. struct rt_bandwidth *rt_b = sched_rt_bandwidth(rt_rq);
  534. s64 want;
  535. int i;
  536. raw_spin_lock(&rt_b->rt_runtime_lock);
  537. raw_spin_lock(&rt_rq->rt_runtime_lock);
  538. /*
  539. * Either we're all inf and nobody needs to borrow, or we're
  540. * already disabled and thus have nothing to do, or we have
  541. * exactly the right amount of runtime to take out.
  542. */
  543. if (rt_rq->rt_runtime == RUNTIME_INF ||
  544. rt_rq->rt_runtime == rt_b->rt_runtime)
  545. goto balanced;
  546. raw_spin_unlock(&rt_rq->rt_runtime_lock);
  547. /*
  548. * Calculate the difference between what we started out with
  549. * and what we current have, that's the amount of runtime
  550. * we lend and now have to reclaim.
  551. */
  552. want = rt_b->rt_runtime - rt_rq->rt_runtime;
  553. /*
  554. * Greedy reclaim, take back as much as we can.
  555. */
  556. for_each_cpu(i, rd->span) {
  557. struct rt_rq *iter = sched_rt_period_rt_rq(rt_b, i);
  558. s64 diff;
  559. /*
  560. * Can't reclaim from ourselves or disabled runqueues.
  561. */
  562. if (iter == rt_rq || iter->rt_runtime == RUNTIME_INF)
  563. continue;
  564. raw_spin_lock(&iter->rt_runtime_lock);
  565. if (want > 0) {
  566. diff = min_t(s64, iter->rt_runtime, want);
  567. iter->rt_runtime -= diff;
  568. want -= diff;
  569. } else {
  570. iter->rt_runtime -= want;
  571. want -= want;
  572. }
  573. raw_spin_unlock(&iter->rt_runtime_lock);
  574. if (!want)
  575. break;
  576. }
  577. raw_spin_lock(&rt_rq->rt_runtime_lock);
  578. /*
  579. * We cannot be left wanting - that would mean some runtime
  580. * leaked out of the system.
  581. */
  582. BUG_ON(want);
  583. balanced:
  584. /*
  585. * Disable all the borrow logic by pretending we have inf
  586. * runtime - in which case borrowing doesn't make sense.
  587. */
  588. rt_rq->rt_runtime = RUNTIME_INF;
  589. rt_rq->rt_throttled = 0;
  590. raw_spin_unlock(&rt_rq->rt_runtime_lock);
  591. raw_spin_unlock(&rt_b->rt_runtime_lock);
  592. /* Make rt_rq available for pick_next_task() */
  593. sched_rt_rq_enqueue(rt_rq);
  594. }
  595. }
  596. static void __enable_runtime(struct rq *rq)
  597. {
  598. rt_rq_iter_t iter;
  599. struct rt_rq *rt_rq;
  600. if (unlikely(!scheduler_running))
  601. return;
  602. /*
  603. * Reset each runqueue's bandwidth settings
  604. */
  605. for_each_rt_rq(rt_rq, iter, rq) {
  606. struct rt_bandwidth *rt_b = sched_rt_bandwidth(rt_rq);
  607. raw_spin_lock(&rt_b->rt_runtime_lock);
  608. raw_spin_lock(&rt_rq->rt_runtime_lock);
  609. rt_rq->rt_runtime = rt_b->rt_runtime;
  610. rt_rq->rt_time = 0;
  611. rt_rq->rt_throttled = 0;
  612. raw_spin_unlock(&rt_rq->rt_runtime_lock);
  613. raw_spin_unlock(&rt_b->rt_runtime_lock);
  614. }
  615. }
  616. static int balance_runtime(struct rt_rq *rt_rq)
  617. {
  618. int more = 0;
  619. if (!sched_feat(RT_RUNTIME_SHARE))
  620. return more;
  621. if (rt_rq->rt_time > rt_rq->rt_runtime) {
  622. raw_spin_unlock(&rt_rq->rt_runtime_lock);
  623. more = do_balance_runtime(rt_rq);
  624. raw_spin_lock(&rt_rq->rt_runtime_lock);
  625. }
  626. return more;
  627. }
  628. #else /* !CONFIG_SMP */
  629. static inline int balance_runtime(struct rt_rq *rt_rq)
  630. {
  631. return 0;
  632. }
  633. #endif /* CONFIG_SMP */
  634. static int do_sched_rt_period_timer(struct rt_bandwidth *rt_b, int overrun)
  635. {
  636. int i, idle = 1, throttled = 0;
  637. const struct cpumask *span;
  638. span = sched_rt_period_mask();
  639. #ifdef CONFIG_RT_GROUP_SCHED
  640. /*
  641. * FIXME: isolated CPUs should really leave the root task group,
  642. * whether they are isolcpus or were isolated via cpusets, lest
  643. * the timer run on a CPU which does not service all runqueues,
  644. * potentially leaving other CPUs indefinitely throttled. If
  645. * isolation is really required, the user will turn the throttle
  646. * off to kill the perturbations it causes anyway. Meanwhile,
  647. * this maintains functionality for boot and/or troubleshooting.
  648. */
  649. if (rt_b == &root_task_group.rt_bandwidth)
  650. span = cpu_online_mask;
  651. #endif
  652. for_each_cpu(i, span) {
  653. int enqueue = 0;
  654. struct rt_rq *rt_rq = sched_rt_period_rt_rq(rt_b, i);
  655. struct rq *rq = rq_of_rt_rq(rt_rq);
  656. raw_spin_lock(&rq->lock);
  657. if (rt_rq->rt_time) {
  658. u64 runtime;
  659. raw_spin_lock(&rt_rq->rt_runtime_lock);
  660. if (rt_rq->rt_throttled)
  661. balance_runtime(rt_rq);
  662. runtime = rt_rq->rt_runtime;
  663. rt_rq->rt_time -= min(rt_rq->rt_time, overrun*runtime);
  664. if (rt_rq->rt_throttled && rt_rq->rt_time < runtime) {
  665. rt_rq->rt_throttled = 0;
  666. enqueue = 1;
  667. /*
  668. * When we're idle and a woken (rt) task is
  669. * throttled check_preempt_curr() will set
  670. * skip_update and the time between the wakeup
  671. * and this unthrottle will get accounted as
  672. * 'runtime'.
  673. */
  674. if (rt_rq->rt_nr_running && rq->curr == rq->idle)
  675. rq_clock_skip_update(rq, false);
  676. }
  677. if (rt_rq->rt_time || rt_rq->rt_nr_running)
  678. idle = 0;
  679. raw_spin_unlock(&rt_rq->rt_runtime_lock);
  680. } else if (rt_rq->rt_nr_running) {
  681. idle = 0;
  682. if (!rt_rq_throttled(rt_rq))
  683. enqueue = 1;
  684. }
  685. if (rt_rq->rt_throttled)
  686. throttled = 1;
  687. if (enqueue)
  688. sched_rt_rq_enqueue(rt_rq);
  689. raw_spin_unlock(&rq->lock);
  690. }
  691. if (!throttled && (!rt_bandwidth_enabled() || rt_b->rt_runtime == RUNTIME_INF))
  692. return 1;
  693. return idle;
  694. }
  695. static inline int rt_se_prio(struct sched_rt_entity *rt_se)
  696. {
  697. #ifdef CONFIG_RT_GROUP_SCHED
  698. struct rt_rq *rt_rq = group_rt_rq(rt_se);
  699. if (rt_rq)
  700. return rt_rq->highest_prio.curr;
  701. #endif
  702. return rt_task_of(rt_se)->prio;
  703. }
  704. static int sched_rt_runtime_exceeded(struct rt_rq *rt_rq)
  705. {
  706. u64 runtime = sched_rt_runtime(rt_rq);
  707. if (rt_rq->rt_throttled)
  708. return rt_rq_throttled(rt_rq);
  709. if (runtime >= sched_rt_period(rt_rq))
  710. return 0;
  711. balance_runtime(rt_rq);
  712. runtime = sched_rt_runtime(rt_rq);
  713. if (runtime == RUNTIME_INF)
  714. return 0;
  715. if (rt_rq->rt_time > runtime) {
  716. struct rt_bandwidth *rt_b = sched_rt_bandwidth(rt_rq);
  717. /*
  718. * Don't actually throttle groups that have no runtime assigned
  719. * but accrue some time due to boosting.
  720. */
  721. if (likely(rt_b->rt_runtime)) {
  722. rt_rq->rt_throttled = 1;
  723. printk_deferred_once("sched: RT throttling activated\n");
  724. } else {
  725. /*
  726. * In case we did anyway, make it go away,
  727. * replenishment is a joke, since it will replenish us
  728. * with exactly 0 ns.
  729. */
  730. rt_rq->rt_time = 0;
  731. }
  732. if (rt_rq_throttled(rt_rq)) {
  733. sched_rt_rq_dequeue(rt_rq);
  734. return 1;
  735. }
  736. }
  737. return 0;
  738. }
  739. /*
  740. * Update the current task's runtime statistics. Skip current tasks that
  741. * are not in our scheduling class.
  742. */
  743. static void update_curr_rt(struct rq *rq)
  744. {
  745. struct task_struct *curr = rq->curr;
  746. struct sched_rt_entity *rt_se = &curr->rt;
  747. u64 delta_exec;
  748. if (curr->sched_class != &rt_sched_class)
  749. return;
  750. delta_exec = rq_clock_task(rq) - curr->se.exec_start;
  751. if (unlikely((s64)delta_exec <= 0))
  752. return;
  753. schedstat_set(curr->se.statistics.exec_max,
  754. max(curr->se.statistics.exec_max, delta_exec));
  755. curr->se.sum_exec_runtime += delta_exec;
  756. account_group_exec_runtime(curr, delta_exec);
  757. curr->se.exec_start = rq_clock_task(rq);
  758. cpuacct_charge(curr, delta_exec);
  759. sched_rt_avg_update(rq, delta_exec);
  760. if (!rt_bandwidth_enabled())
  761. return;
  762. for_each_sched_rt_entity(rt_se) {
  763. struct rt_rq *rt_rq = rt_rq_of_se(rt_se);
  764. if (sched_rt_runtime(rt_rq) != RUNTIME_INF) {
  765. raw_spin_lock(&rt_rq->rt_runtime_lock);
  766. rt_rq->rt_time += delta_exec;
  767. if (sched_rt_runtime_exceeded(rt_rq))
  768. resched_curr(rq);
  769. raw_spin_unlock(&rt_rq->rt_runtime_lock);
  770. }
  771. }
  772. }
  773. static void
  774. dequeue_top_rt_rq(struct rt_rq *rt_rq)
  775. {
  776. struct rq *rq = rq_of_rt_rq(rt_rq);
  777. BUG_ON(&rq->rt != rt_rq);
  778. if (!rt_rq->rt_queued)
  779. return;
  780. BUG_ON(!rq->nr_running);
  781. sub_nr_running(rq, rt_rq->rt_nr_running);
  782. rt_rq->rt_queued = 0;
  783. }
  784. static void
  785. enqueue_top_rt_rq(struct rt_rq *rt_rq)
  786. {
  787. struct rq *rq = rq_of_rt_rq(rt_rq);
  788. BUG_ON(&rq->rt != rt_rq);
  789. if (rt_rq->rt_queued)
  790. return;
  791. if (rt_rq_throttled(rt_rq) || !rt_rq->rt_nr_running)
  792. return;
  793. add_nr_running(rq, rt_rq->rt_nr_running);
  794. rt_rq->rt_queued = 1;
  795. }
  796. #if defined CONFIG_SMP
  797. static void
  798. inc_rt_prio_smp(struct rt_rq *rt_rq, int prio, int prev_prio)
  799. {
  800. struct rq *rq = rq_of_rt_rq(rt_rq);
  801. #ifdef CONFIG_RT_GROUP_SCHED
  802. /*
  803. * Change rq's cpupri only if rt_rq is the top queue.
  804. */
  805. if (&rq->rt != rt_rq)
  806. return;
  807. #endif
  808. if (rq->online && prio < prev_prio)
  809. cpupri_set(&rq->rd->cpupri, rq->cpu, prio);
  810. }
  811. static void
  812. dec_rt_prio_smp(struct rt_rq *rt_rq, int prio, int prev_prio)
  813. {
  814. struct rq *rq = rq_of_rt_rq(rt_rq);
  815. #ifdef CONFIG_RT_GROUP_SCHED
  816. /*
  817. * Change rq's cpupri only if rt_rq is the top queue.
  818. */
  819. if (&rq->rt != rt_rq)
  820. return;
  821. #endif
  822. if (rq->online && rt_rq->highest_prio.curr != prev_prio)
  823. cpupri_set(&rq->rd->cpupri, rq->cpu, rt_rq->highest_prio.curr);
  824. }
  825. #else /* CONFIG_SMP */
  826. static inline
  827. void inc_rt_prio_smp(struct rt_rq *rt_rq, int prio, int prev_prio) {}
  828. static inline
  829. void dec_rt_prio_smp(struct rt_rq *rt_rq, int prio, int prev_prio) {}
  830. #endif /* CONFIG_SMP */
  831. #if defined CONFIG_SMP || defined CONFIG_RT_GROUP_SCHED
  832. static void
  833. inc_rt_prio(struct rt_rq *rt_rq, int prio)
  834. {
  835. int prev_prio = rt_rq->highest_prio.curr;
  836. if (prio < prev_prio)
  837. rt_rq->highest_prio.curr = prio;
  838. inc_rt_prio_smp(rt_rq, prio, prev_prio);
  839. }
  840. static void
  841. dec_rt_prio(struct rt_rq *rt_rq, int prio)
  842. {
  843. int prev_prio = rt_rq->highest_prio.curr;
  844. if (rt_rq->rt_nr_running) {
  845. WARN_ON(prio < prev_prio);
  846. /*
  847. * This may have been our highest task, and therefore
  848. * we may have some recomputation to do
  849. */
  850. if (prio == prev_prio) {
  851. struct rt_prio_array *array = &rt_rq->active;
  852. rt_rq->highest_prio.curr =
  853. sched_find_first_bit(array->bitmap);
  854. }
  855. } else
  856. rt_rq->highest_prio.curr = MAX_RT_PRIO;
  857. dec_rt_prio_smp(rt_rq, prio, prev_prio);
  858. }
  859. #else
  860. static inline void inc_rt_prio(struct rt_rq *rt_rq, int prio) {}
  861. static inline void dec_rt_prio(struct rt_rq *rt_rq, int prio) {}
  862. #endif /* CONFIG_SMP || CONFIG_RT_GROUP_SCHED */
  863. #ifdef CONFIG_RT_GROUP_SCHED
  864. static void
  865. inc_rt_group(struct sched_rt_entity *rt_se, struct rt_rq *rt_rq)
  866. {
  867. if (rt_se_boosted(rt_se))
  868. rt_rq->rt_nr_boosted++;
  869. if (rt_rq->tg)
  870. start_rt_bandwidth(&rt_rq->tg->rt_bandwidth);
  871. }
  872. static void
  873. dec_rt_group(struct sched_rt_entity *rt_se, struct rt_rq *rt_rq)
  874. {
  875. if (rt_se_boosted(rt_se))
  876. rt_rq->rt_nr_boosted--;
  877. WARN_ON(!rt_rq->rt_nr_running && rt_rq->rt_nr_boosted);
  878. }
  879. #else /* CONFIG_RT_GROUP_SCHED */
  880. static void
  881. inc_rt_group(struct sched_rt_entity *rt_se, struct rt_rq *rt_rq)
  882. {
  883. start_rt_bandwidth(&def_rt_bandwidth);
  884. }
  885. static inline
  886. void dec_rt_group(struct sched_rt_entity *rt_se, struct rt_rq *rt_rq) {}
  887. #endif /* CONFIG_RT_GROUP_SCHED */
  888. static inline
  889. unsigned int rt_se_nr_running(struct sched_rt_entity *rt_se)
  890. {
  891. struct rt_rq *group_rq = group_rt_rq(rt_se);
  892. if (group_rq)
  893. return group_rq->rt_nr_running;
  894. else
  895. return 1;
  896. }
  897. static inline
  898. void inc_rt_tasks(struct sched_rt_entity *rt_se, struct rt_rq *rt_rq)
  899. {
  900. int prio = rt_se_prio(rt_se);
  901. WARN_ON(!rt_prio(prio));
  902. rt_rq->rt_nr_running += rt_se_nr_running(rt_se);
  903. inc_rt_prio(rt_rq, prio);
  904. inc_rt_migration(rt_se, rt_rq);
  905. inc_rt_group(rt_se, rt_rq);
  906. }
  907. static inline
  908. void dec_rt_tasks(struct sched_rt_entity *rt_se, struct rt_rq *rt_rq)
  909. {
  910. WARN_ON(!rt_prio(rt_se_prio(rt_se)));
  911. WARN_ON(!rt_rq->rt_nr_running);
  912. rt_rq->rt_nr_running -= rt_se_nr_running(rt_se);
  913. dec_rt_prio(rt_rq, rt_se_prio(rt_se));
  914. dec_rt_migration(rt_se, rt_rq);
  915. dec_rt_group(rt_se, rt_rq);
  916. }
  917. static void __enqueue_rt_entity(struct sched_rt_entity *rt_se, bool head)
  918. {
  919. struct rt_rq *rt_rq = rt_rq_of_se(rt_se);
  920. struct rt_prio_array *array = &rt_rq->active;
  921. struct rt_rq *group_rq = group_rt_rq(rt_se);
  922. struct list_head *queue = array->queue + rt_se_prio(rt_se);
  923. /*
  924. * Don't enqueue the group if its throttled, or when empty.
  925. * The latter is a consequence of the former when a child group
  926. * get throttled and the current group doesn't have any other
  927. * active members.
  928. */
  929. if (group_rq && (rt_rq_throttled(group_rq) || !group_rq->rt_nr_running))
  930. return;
  931. if (head)
  932. list_add(&rt_se->run_list, queue);
  933. else
  934. list_add_tail(&rt_se->run_list, queue);
  935. __set_bit(rt_se_prio(rt_se), array->bitmap);
  936. inc_rt_tasks(rt_se, rt_rq);
  937. }
  938. static void __dequeue_rt_entity(struct sched_rt_entity *rt_se)
  939. {
  940. struct rt_rq *rt_rq = rt_rq_of_se(rt_se);
  941. struct rt_prio_array *array = &rt_rq->active;
  942. list_del_init(&rt_se->run_list);
  943. if (list_empty(array->queue + rt_se_prio(rt_se)))
  944. __clear_bit(rt_se_prio(rt_se), array->bitmap);
  945. dec_rt_tasks(rt_se, rt_rq);
  946. }
  947. /*
  948. * Because the prio of an upper entry depends on the lower
  949. * entries, we must remove entries top - down.
  950. */
  951. static void dequeue_rt_stack(struct sched_rt_entity *rt_se)
  952. {
  953. struct sched_rt_entity *back = NULL;
  954. for_each_sched_rt_entity(rt_se) {
  955. rt_se->back = back;
  956. back = rt_se;
  957. }
  958. dequeue_top_rt_rq(rt_rq_of_se(back));
  959. for (rt_se = back; rt_se; rt_se = rt_se->back) {
  960. if (on_rt_rq(rt_se))
  961. __dequeue_rt_entity(rt_se);
  962. }
  963. }
  964. static void enqueue_rt_entity(struct sched_rt_entity *rt_se, bool head)
  965. {
  966. struct rq *rq = rq_of_rt_se(rt_se);
  967. dequeue_rt_stack(rt_se);
  968. for_each_sched_rt_entity(rt_se)
  969. __enqueue_rt_entity(rt_se, head);
  970. enqueue_top_rt_rq(&rq->rt);
  971. }
  972. static void dequeue_rt_entity(struct sched_rt_entity *rt_se)
  973. {
  974. struct rq *rq = rq_of_rt_se(rt_se);
  975. dequeue_rt_stack(rt_se);
  976. for_each_sched_rt_entity(rt_se) {
  977. struct rt_rq *rt_rq = group_rt_rq(rt_se);
  978. if (rt_rq && rt_rq->rt_nr_running)
  979. __enqueue_rt_entity(rt_se, false);
  980. }
  981. enqueue_top_rt_rq(&rq->rt);
  982. }
  983. /*
  984. * Adding/removing a task to/from a priority array:
  985. */
  986. static void
  987. enqueue_task_rt(struct rq *rq, struct task_struct *p, int flags)
  988. {
  989. struct sched_rt_entity *rt_se = &p->rt;
  990. if (flags & ENQUEUE_WAKEUP)
  991. rt_se->timeout = 0;
  992. enqueue_rt_entity(rt_se, flags & ENQUEUE_HEAD);
  993. if (!task_current(rq, p) && p->nr_cpus_allowed > 1)
  994. enqueue_pushable_task(rq, p);
  995. }
  996. static void dequeue_task_rt(struct rq *rq, struct task_struct *p, int flags)
  997. {
  998. struct sched_rt_entity *rt_se = &p->rt;
  999. update_curr_rt(rq);
  1000. dequeue_rt_entity(rt_se);
  1001. dequeue_pushable_task(rq, p);
  1002. }
  1003. /*
  1004. * Put task to the head or the end of the run list without the overhead of
  1005. * dequeue followed by enqueue.
  1006. */
  1007. static void
  1008. requeue_rt_entity(struct rt_rq *rt_rq, struct sched_rt_entity *rt_se, int head)
  1009. {
  1010. if (on_rt_rq(rt_se)) {
  1011. struct rt_prio_array *array = &rt_rq->active;
  1012. struct list_head *queue = array->queue + rt_se_prio(rt_se);
  1013. if (head)
  1014. list_move(&rt_se->run_list, queue);
  1015. else
  1016. list_move_tail(&rt_se->run_list, queue);
  1017. }
  1018. }
  1019. static void requeue_task_rt(struct rq *rq, struct task_struct *p, int head)
  1020. {
  1021. struct sched_rt_entity *rt_se = &p->rt;
  1022. struct rt_rq *rt_rq;
  1023. for_each_sched_rt_entity(rt_se) {
  1024. rt_rq = rt_rq_of_se(rt_se);
  1025. requeue_rt_entity(rt_rq, rt_se, head);
  1026. }
  1027. }
  1028. static void yield_task_rt(struct rq *rq)
  1029. {
  1030. requeue_task_rt(rq, rq->curr, 0);
  1031. }
  1032. #ifdef CONFIG_SMP
  1033. static int find_lowest_rq(struct task_struct *task);
  1034. static int
  1035. select_task_rq_rt(struct task_struct *p, int cpu, int sd_flag, int flags)
  1036. {
  1037. struct task_struct *curr;
  1038. struct rq *rq;
  1039. /* For anything but wake ups, just return the task_cpu */
  1040. if (sd_flag != SD_BALANCE_WAKE && sd_flag != SD_BALANCE_FORK)
  1041. goto out;
  1042. rq = cpu_rq(cpu);
  1043. rcu_read_lock();
  1044. curr = ACCESS_ONCE(rq->curr); /* unlocked access */
  1045. /*
  1046. * If the current task on @p's runqueue is an RT task, then
  1047. * try to see if we can wake this RT task up on another
  1048. * runqueue. Otherwise simply start this RT task
  1049. * on its current runqueue.
  1050. *
  1051. * We want to avoid overloading runqueues. If the woken
  1052. * task is a higher priority, then it will stay on this CPU
  1053. * and the lower prio task should be moved to another CPU.
  1054. * Even though this will probably make the lower prio task
  1055. * lose its cache, we do not want to bounce a higher task
  1056. * around just because it gave up its CPU, perhaps for a
  1057. * lock?
  1058. *
  1059. * For equal prio tasks, we just let the scheduler sort it out.
  1060. *
  1061. * Otherwise, just let it ride on the affined RQ and the
  1062. * post-schedule router will push the preempted task away
  1063. *
  1064. * This test is optimistic, if we get it wrong the load-balancer
  1065. * will have to sort it out.
  1066. */
  1067. if (curr && unlikely(rt_task(curr)) &&
  1068. (curr->nr_cpus_allowed < 2 ||
  1069. curr->prio <= p->prio)) {
  1070. int target = find_lowest_rq(p);
  1071. /*
  1072. * Don't bother moving it if the destination CPU is
  1073. * not running a lower priority task.
  1074. */
  1075. if (target != -1 &&
  1076. p->prio < cpu_rq(target)->rt.highest_prio.curr)
  1077. cpu = target;
  1078. }
  1079. rcu_read_unlock();
  1080. out:
  1081. return cpu;
  1082. }
  1083. static void check_preempt_equal_prio(struct rq *rq, struct task_struct *p)
  1084. {
  1085. /*
  1086. * Current can't be migrated, useless to reschedule,
  1087. * let's hope p can move out.
  1088. */
  1089. if (rq->curr->nr_cpus_allowed == 1 ||
  1090. !cpupri_find(&rq->rd->cpupri, rq->curr, NULL))
  1091. return;
  1092. /*
  1093. * p is migratable, so let's not schedule it and
  1094. * see if it is pushed or pulled somewhere else.
  1095. */
  1096. if (p->nr_cpus_allowed != 1
  1097. && cpupri_find(&rq->rd->cpupri, p, NULL))
  1098. return;
  1099. /*
  1100. * There appears to be other cpus that can accept
  1101. * current and none to run 'p', so lets reschedule
  1102. * to try and push current away:
  1103. */
  1104. requeue_task_rt(rq, p, 1);
  1105. resched_curr(rq);
  1106. }
  1107. #endif /* CONFIG_SMP */
  1108. /*
  1109. * Preempt the current task with a newly woken task if needed:
  1110. */
  1111. static void check_preempt_curr_rt(struct rq *rq, struct task_struct *p, int flags)
  1112. {
  1113. if (p->prio < rq->curr->prio) {
  1114. resched_curr(rq);
  1115. return;
  1116. }
  1117. #ifdef CONFIG_SMP
  1118. /*
  1119. * If:
  1120. *
  1121. * - the newly woken task is of equal priority to the current task
  1122. * - the newly woken task is non-migratable while current is migratable
  1123. * - current will be preempted on the next reschedule
  1124. *
  1125. * we should check to see if current can readily move to a different
  1126. * cpu. If so, we will reschedule to allow the push logic to try
  1127. * to move current somewhere else, making room for our non-migratable
  1128. * task.
  1129. */
  1130. if (p->prio == rq->curr->prio && !test_tsk_need_resched(rq->curr))
  1131. check_preempt_equal_prio(rq, p);
  1132. #endif
  1133. }
  1134. static struct sched_rt_entity *pick_next_rt_entity(struct rq *rq,
  1135. struct rt_rq *rt_rq)
  1136. {
  1137. struct rt_prio_array *array = &rt_rq->active;
  1138. struct sched_rt_entity *next = NULL;
  1139. struct list_head *queue;
  1140. int idx;
  1141. idx = sched_find_first_bit(array->bitmap);
  1142. BUG_ON(idx >= MAX_RT_PRIO);
  1143. queue = array->queue + idx;
  1144. next = list_entry(queue->next, struct sched_rt_entity, run_list);
  1145. return next;
  1146. }
  1147. static struct task_struct *_pick_next_task_rt(struct rq *rq)
  1148. {
  1149. struct sched_rt_entity *rt_se;
  1150. struct task_struct *p;
  1151. struct rt_rq *rt_rq = &rq->rt;
  1152. do {
  1153. rt_se = pick_next_rt_entity(rq, rt_rq);
  1154. BUG_ON(!rt_se);
  1155. rt_rq = group_rt_rq(rt_se);
  1156. } while (rt_rq);
  1157. p = rt_task_of(rt_se);
  1158. p->se.exec_start = rq_clock_task(rq);
  1159. return p;
  1160. }
  1161. static struct task_struct *
  1162. pick_next_task_rt(struct rq *rq, struct task_struct *prev)
  1163. {
  1164. struct task_struct *p;
  1165. struct rt_rq *rt_rq = &rq->rt;
  1166. if (need_pull_rt_task(rq, prev)) {
  1167. pull_rt_task(rq);
  1168. /*
  1169. * pull_rt_task() can drop (and re-acquire) rq->lock; this
  1170. * means a dl or stop task can slip in, in which case we need
  1171. * to re-start task selection.
  1172. */
  1173. if (unlikely((rq->stop && task_on_rq_queued(rq->stop)) ||
  1174. rq->dl.dl_nr_running))
  1175. return RETRY_TASK;
  1176. }
  1177. /*
  1178. * We may dequeue prev's rt_rq in put_prev_task().
  1179. * So, we update time before rt_nr_running check.
  1180. */
  1181. if (prev->sched_class == &rt_sched_class)
  1182. update_curr_rt(rq);
  1183. if (!rt_rq->rt_queued)
  1184. return NULL;
  1185. put_prev_task(rq, prev);
  1186. p = _pick_next_task_rt(rq);
  1187. /* The running task is never eligible for pushing */
  1188. dequeue_pushable_task(rq, p);
  1189. set_post_schedule(rq);
  1190. return p;
  1191. }
  1192. static void put_prev_task_rt(struct rq *rq, struct task_struct *p)
  1193. {
  1194. update_curr_rt(rq);
  1195. /*
  1196. * The previous task needs to be made eligible for pushing
  1197. * if it is still active
  1198. */
  1199. if (on_rt_rq(&p->rt) && p->nr_cpus_allowed > 1)
  1200. enqueue_pushable_task(rq, p);
  1201. }
  1202. #ifdef CONFIG_SMP
  1203. /* Only try algorithms three times */
  1204. #define RT_MAX_TRIES 3
  1205. static int pick_rt_task(struct rq *rq, struct task_struct *p, int cpu)
  1206. {
  1207. if (!task_running(rq, p) &&
  1208. cpumask_test_cpu(cpu, tsk_cpus_allowed(p)))
  1209. return 1;
  1210. return 0;
  1211. }
  1212. /*
  1213. * Return the highest pushable rq's task, which is suitable to be executed
  1214. * on the cpu, NULL otherwise
  1215. */
  1216. static struct task_struct *pick_highest_pushable_task(struct rq *rq, int cpu)
  1217. {
  1218. struct plist_head *head = &rq->rt.pushable_tasks;
  1219. struct task_struct *p;
  1220. if (!has_pushable_tasks(rq))
  1221. return NULL;
  1222. plist_for_each_entry(p, head, pushable_tasks) {
  1223. if (pick_rt_task(rq, p, cpu))
  1224. return p;
  1225. }
  1226. return NULL;
  1227. }
  1228. static DEFINE_PER_CPU(cpumask_var_t, local_cpu_mask);
  1229. static int find_lowest_rq(struct task_struct *task)
  1230. {
  1231. struct sched_domain *sd;
  1232. struct cpumask *lowest_mask = this_cpu_cpumask_var_ptr(local_cpu_mask);
  1233. int this_cpu = smp_processor_id();
  1234. int cpu = task_cpu(task);
  1235. /* Make sure the mask is initialized first */
  1236. if (unlikely(!lowest_mask))
  1237. return -1;
  1238. if (task->nr_cpus_allowed == 1)
  1239. return -1; /* No other targets possible */
  1240. if (!cpupri_find(&task_rq(task)->rd->cpupri, task, lowest_mask))
  1241. return -1; /* No targets found */
  1242. /*
  1243. * At this point we have built a mask of cpus representing the
  1244. * lowest priority tasks in the system. Now we want to elect
  1245. * the best one based on our affinity and topology.
  1246. *
  1247. * We prioritize the last cpu that the task executed on since
  1248. * it is most likely cache-hot in that location.
  1249. */
  1250. if (cpumask_test_cpu(cpu, lowest_mask))
  1251. return cpu;
  1252. /*
  1253. * Otherwise, we consult the sched_domains span maps to figure
  1254. * out which cpu is logically closest to our hot cache data.
  1255. */
  1256. if (!cpumask_test_cpu(this_cpu, lowest_mask))
  1257. this_cpu = -1; /* Skip this_cpu opt if not among lowest */
  1258. rcu_read_lock();
  1259. for_each_domain(cpu, sd) {
  1260. if (sd->flags & SD_WAKE_AFFINE) {
  1261. int best_cpu;
  1262. /*
  1263. * "this_cpu" is cheaper to preempt than a
  1264. * remote processor.
  1265. */
  1266. if (this_cpu != -1 &&
  1267. cpumask_test_cpu(this_cpu, sched_domain_span(sd))) {
  1268. rcu_read_unlock();
  1269. return this_cpu;
  1270. }
  1271. best_cpu = cpumask_first_and(lowest_mask,
  1272. sched_domain_span(sd));
  1273. if (best_cpu < nr_cpu_ids) {
  1274. rcu_read_unlock();
  1275. return best_cpu;
  1276. }
  1277. }
  1278. }
  1279. rcu_read_unlock();
  1280. /*
  1281. * And finally, if there were no matches within the domains
  1282. * just give the caller *something* to work with from the compatible
  1283. * locations.
  1284. */
  1285. if (this_cpu != -1)
  1286. return this_cpu;
  1287. cpu = cpumask_any(lowest_mask);
  1288. if (cpu < nr_cpu_ids)
  1289. return cpu;
  1290. return -1;
  1291. }
  1292. /* Will lock the rq it finds */
  1293. static struct rq *find_lock_lowest_rq(struct task_struct *task, struct rq *rq)
  1294. {
  1295. struct rq *lowest_rq = NULL;
  1296. int tries;
  1297. int cpu;
  1298. for (tries = 0; tries < RT_MAX_TRIES; tries++) {
  1299. cpu = find_lowest_rq(task);
  1300. if ((cpu == -1) || (cpu == rq->cpu))
  1301. break;
  1302. lowest_rq = cpu_rq(cpu);
  1303. if (lowest_rq->rt.highest_prio.curr <= task->prio) {
  1304. /*
  1305. * Target rq has tasks of equal or higher priority,
  1306. * retrying does not release any lock and is unlikely
  1307. * to yield a different result.
  1308. */
  1309. lowest_rq = NULL;
  1310. break;
  1311. }
  1312. /* if the prio of this runqueue changed, try again */
  1313. if (double_lock_balance(rq, lowest_rq)) {
  1314. /*
  1315. * We had to unlock the run queue. In
  1316. * the mean time, task could have
  1317. * migrated already or had its affinity changed.
  1318. * Also make sure that it wasn't scheduled on its rq.
  1319. */
  1320. if (unlikely(task_rq(task) != rq ||
  1321. !cpumask_test_cpu(lowest_rq->cpu,
  1322. tsk_cpus_allowed(task)) ||
  1323. task_running(rq, task) ||
  1324. !task_on_rq_queued(task))) {
  1325. double_unlock_balance(rq, lowest_rq);
  1326. lowest_rq = NULL;
  1327. break;
  1328. }
  1329. }
  1330. /* If this rq is still suitable use it. */
  1331. if (lowest_rq->rt.highest_prio.curr > task->prio)
  1332. break;
  1333. /* try again */
  1334. double_unlock_balance(rq, lowest_rq);
  1335. lowest_rq = NULL;
  1336. }
  1337. return lowest_rq;
  1338. }
  1339. static struct task_struct *pick_next_pushable_task(struct rq *rq)
  1340. {
  1341. struct task_struct *p;
  1342. if (!has_pushable_tasks(rq))
  1343. return NULL;
  1344. p = plist_first_entry(&rq->rt.pushable_tasks,
  1345. struct task_struct, pushable_tasks);
  1346. BUG_ON(rq->cpu != task_cpu(p));
  1347. BUG_ON(task_current(rq, p));
  1348. BUG_ON(p->nr_cpus_allowed <= 1);
  1349. BUG_ON(!task_on_rq_queued(p));
  1350. BUG_ON(!rt_task(p));
  1351. return p;
  1352. }
  1353. /*
  1354. * If the current CPU has more than one RT task, see if the non
  1355. * running task can migrate over to a CPU that is running a task
  1356. * of lesser priority.
  1357. */
  1358. static int push_rt_task(struct rq *rq)
  1359. {
  1360. struct task_struct *next_task;
  1361. struct rq *lowest_rq;
  1362. int ret = 0;
  1363. if (!rq->rt.overloaded)
  1364. return 0;
  1365. next_task = pick_next_pushable_task(rq);
  1366. if (!next_task)
  1367. return 0;
  1368. retry:
  1369. if (unlikely(next_task == rq->curr)) {
  1370. WARN_ON(1);
  1371. return 0;
  1372. }
  1373. /*
  1374. * It's possible that the next_task slipped in of
  1375. * higher priority than current. If that's the case
  1376. * just reschedule current.
  1377. */
  1378. if (unlikely(next_task->prio < rq->curr->prio)) {
  1379. resched_curr(rq);
  1380. return 0;
  1381. }
  1382. /* We might release rq lock */
  1383. get_task_struct(next_task);
  1384. /* find_lock_lowest_rq locks the rq if found */
  1385. lowest_rq = find_lock_lowest_rq(next_task, rq);
  1386. if (!lowest_rq) {
  1387. struct task_struct *task;
  1388. /*
  1389. * find_lock_lowest_rq releases rq->lock
  1390. * so it is possible that next_task has migrated.
  1391. *
  1392. * We need to make sure that the task is still on the same
  1393. * run-queue and is also still the next task eligible for
  1394. * pushing.
  1395. */
  1396. task = pick_next_pushable_task(rq);
  1397. if (task_cpu(next_task) == rq->cpu && task == next_task) {
  1398. /*
  1399. * The task hasn't migrated, and is still the next
  1400. * eligible task, but we failed to find a run-queue
  1401. * to push it to. Do not retry in this case, since
  1402. * other cpus will pull from us when ready.
  1403. */
  1404. goto out;
  1405. }
  1406. if (!task)
  1407. /* No more tasks, just exit */
  1408. goto out;
  1409. /*
  1410. * Something has shifted, try again.
  1411. */
  1412. put_task_struct(next_task);
  1413. next_task = task;
  1414. goto retry;
  1415. }
  1416. deactivate_task(rq, next_task, 0);
  1417. set_task_cpu(next_task, lowest_rq->cpu);
  1418. activate_task(lowest_rq, next_task, 0);
  1419. ret = 1;
  1420. resched_curr(lowest_rq);
  1421. double_unlock_balance(rq, lowest_rq);
  1422. out:
  1423. put_task_struct(next_task);
  1424. return ret;
  1425. }
  1426. static void push_rt_tasks(struct rq *rq)
  1427. {
  1428. /* push_rt_task will return true if it moved an RT */
  1429. while (push_rt_task(rq))
  1430. ;
  1431. }
  1432. static int pull_rt_task(struct rq *this_rq)
  1433. {
  1434. int this_cpu = this_rq->cpu, ret = 0, cpu;
  1435. struct task_struct *p;
  1436. struct rq *src_rq;
  1437. if (likely(!rt_overloaded(this_rq)))
  1438. return 0;
  1439. /*
  1440. * Match the barrier from rt_set_overloaded; this guarantees that if we
  1441. * see overloaded we must also see the rto_mask bit.
  1442. */
  1443. smp_rmb();
  1444. for_each_cpu(cpu, this_rq->rd->rto_mask) {
  1445. if (this_cpu == cpu)
  1446. continue;
  1447. src_rq = cpu_rq(cpu);
  1448. /*
  1449. * Don't bother taking the src_rq->lock if the next highest
  1450. * task is known to be lower-priority than our current task.
  1451. * This may look racy, but if this value is about to go
  1452. * logically higher, the src_rq will push this task away.
  1453. * And if its going logically lower, we do not care
  1454. */
  1455. if (src_rq->rt.highest_prio.next >=
  1456. this_rq->rt.highest_prio.curr)
  1457. continue;
  1458. /*
  1459. * We can potentially drop this_rq's lock in
  1460. * double_lock_balance, and another CPU could
  1461. * alter this_rq
  1462. */
  1463. double_lock_balance(this_rq, src_rq);
  1464. /*
  1465. * We can pull only a task, which is pushable
  1466. * on its rq, and no others.
  1467. */
  1468. p = pick_highest_pushable_task(src_rq, this_cpu);
  1469. /*
  1470. * Do we have an RT task that preempts
  1471. * the to-be-scheduled task?
  1472. */
  1473. if (p && (p->prio < this_rq->rt.highest_prio.curr)) {
  1474. WARN_ON(p == src_rq->curr);
  1475. WARN_ON(!task_on_rq_queued(p));
  1476. /*
  1477. * There's a chance that p is higher in priority
  1478. * than what's currently running on its cpu.
  1479. * This is just that p is wakeing up and hasn't
  1480. * had a chance to schedule. We only pull
  1481. * p if it is lower in priority than the
  1482. * current task on the run queue
  1483. */
  1484. if (p->prio < src_rq->curr->prio)
  1485. goto skip;
  1486. ret = 1;
  1487. deactivate_task(src_rq, p, 0);
  1488. set_task_cpu(p, this_cpu);
  1489. activate_task(this_rq, p, 0);
  1490. /*
  1491. * We continue with the search, just in
  1492. * case there's an even higher prio task
  1493. * in another runqueue. (low likelihood
  1494. * but possible)
  1495. */
  1496. }
  1497. skip:
  1498. double_unlock_balance(this_rq, src_rq);
  1499. }
  1500. return ret;
  1501. }
  1502. static void post_schedule_rt(struct rq *rq)
  1503. {
  1504. push_rt_tasks(rq);
  1505. }
  1506. /*
  1507. * If we are not running and we are not going to reschedule soon, we should
  1508. * try to push tasks away now
  1509. */
  1510. static void task_woken_rt(struct rq *rq, struct task_struct *p)
  1511. {
  1512. if (!task_running(rq, p) &&
  1513. !test_tsk_need_resched(rq->curr) &&
  1514. has_pushable_tasks(rq) &&
  1515. p->nr_cpus_allowed > 1 &&
  1516. (dl_task(rq->curr) || rt_task(rq->curr)) &&
  1517. (rq->curr->nr_cpus_allowed < 2 ||
  1518. rq->curr->prio <= p->prio))
  1519. push_rt_tasks(rq);
  1520. }
  1521. static void set_cpus_allowed_rt(struct task_struct *p,
  1522. const struct cpumask *new_mask)
  1523. {
  1524. struct rq *rq;
  1525. int weight;
  1526. BUG_ON(!rt_task(p));
  1527. if (!task_on_rq_queued(p))
  1528. return;
  1529. weight = cpumask_weight(new_mask);
  1530. /*
  1531. * Only update if the process changes its state from whether it
  1532. * can migrate or not.
  1533. */
  1534. if ((p->nr_cpus_allowed > 1) == (weight > 1))
  1535. return;
  1536. rq = task_rq(p);
  1537. /*
  1538. * The process used to be able to migrate OR it can now migrate
  1539. */
  1540. if (weight <= 1) {
  1541. if (!task_current(rq, p))
  1542. dequeue_pushable_task(rq, p);
  1543. BUG_ON(!rq->rt.rt_nr_migratory);
  1544. rq->rt.rt_nr_migratory--;
  1545. } else {
  1546. if (!task_current(rq, p))
  1547. enqueue_pushable_task(rq, p);
  1548. rq->rt.rt_nr_migratory++;
  1549. }
  1550. update_rt_migration(&rq->rt);
  1551. }
  1552. /* Assumes rq->lock is held */
  1553. static void rq_online_rt(struct rq *rq)
  1554. {
  1555. if (rq->rt.overloaded)
  1556. rt_set_overload(rq);
  1557. __enable_runtime(rq);
  1558. cpupri_set(&rq->rd->cpupri, rq->cpu, rq->rt.highest_prio.curr);
  1559. }
  1560. /* Assumes rq->lock is held */
  1561. static void rq_offline_rt(struct rq *rq)
  1562. {
  1563. if (rq->rt.overloaded)
  1564. rt_clear_overload(rq);
  1565. __disable_runtime(rq);
  1566. cpupri_set(&rq->rd->cpupri, rq->cpu, CPUPRI_INVALID);
  1567. }
  1568. /*
  1569. * When switch from the rt queue, we bring ourselves to a position
  1570. * that we might want to pull RT tasks from other runqueues.
  1571. */
  1572. static void switched_from_rt(struct rq *rq, struct task_struct *p)
  1573. {
  1574. /*
  1575. * If there are other RT tasks then we will reschedule
  1576. * and the scheduling of the other RT tasks will handle
  1577. * the balancing. But if we are the last RT task
  1578. * we may need to handle the pulling of RT tasks
  1579. * now.
  1580. */
  1581. if (!task_on_rq_queued(p) || rq->rt.rt_nr_running)
  1582. return;
  1583. if (pull_rt_task(rq))
  1584. resched_curr(rq);
  1585. }
  1586. void __init init_sched_rt_class(void)
  1587. {
  1588. unsigned int i;
  1589. for_each_possible_cpu(i) {
  1590. zalloc_cpumask_var_node(&per_cpu(local_cpu_mask, i),
  1591. GFP_KERNEL, cpu_to_node(i));
  1592. }
  1593. }
  1594. #endif /* CONFIG_SMP */
  1595. /*
  1596. * When switching a task to RT, we may overload the runqueue
  1597. * with RT tasks. In this case we try to push them off to
  1598. * other runqueues.
  1599. */
  1600. static void switched_to_rt(struct rq *rq, struct task_struct *p)
  1601. {
  1602. int check_resched = 1;
  1603. /*
  1604. * If we are already running, then there's nothing
  1605. * that needs to be done. But if we are not running
  1606. * we may need to preempt the current running task.
  1607. * If that current running task is also an RT task
  1608. * then see if we can move to another run queue.
  1609. */
  1610. if (task_on_rq_queued(p) && rq->curr != p) {
  1611. #ifdef CONFIG_SMP
  1612. if (p->nr_cpus_allowed > 1 && rq->rt.overloaded &&
  1613. /* Don't resched if we changed runqueues */
  1614. push_rt_task(rq) && rq != task_rq(p))
  1615. check_resched = 0;
  1616. #endif /* CONFIG_SMP */
  1617. if (check_resched && p->prio < rq->curr->prio)
  1618. resched_curr(rq);
  1619. }
  1620. }
  1621. /*
  1622. * Priority of the task has changed. This may cause
  1623. * us to initiate a push or pull.
  1624. */
  1625. static void
  1626. prio_changed_rt(struct rq *rq, struct task_struct *p, int oldprio)
  1627. {
  1628. if (!task_on_rq_queued(p))
  1629. return;
  1630. if (rq->curr == p) {
  1631. #ifdef CONFIG_SMP
  1632. /*
  1633. * If our priority decreases while running, we
  1634. * may need to pull tasks to this runqueue.
  1635. */
  1636. if (oldprio < p->prio)
  1637. pull_rt_task(rq);
  1638. /*
  1639. * If there's a higher priority task waiting to run
  1640. * then reschedule. Note, the above pull_rt_task
  1641. * can release the rq lock and p could migrate.
  1642. * Only reschedule if p is still on the same runqueue.
  1643. */
  1644. if (p->prio > rq->rt.highest_prio.curr && rq->curr == p)
  1645. resched_curr(rq);
  1646. #else
  1647. /* For UP simply resched on drop of prio */
  1648. if (oldprio < p->prio)
  1649. resched_curr(rq);
  1650. #endif /* CONFIG_SMP */
  1651. } else {
  1652. /*
  1653. * This task is not running, but if it is
  1654. * greater than the current running task
  1655. * then reschedule.
  1656. */
  1657. if (p->prio < rq->curr->prio)
  1658. resched_curr(rq);
  1659. }
  1660. }
  1661. static void watchdog(struct rq *rq, struct task_struct *p)
  1662. {
  1663. unsigned long soft, hard;
  1664. /* max may change after cur was read, this will be fixed next tick */
  1665. soft = task_rlimit(p, RLIMIT_RTTIME);
  1666. hard = task_rlimit_max(p, RLIMIT_RTTIME);
  1667. if (soft != RLIM_INFINITY) {
  1668. unsigned long next;
  1669. if (p->rt.watchdog_stamp != jiffies) {
  1670. p->rt.timeout++;
  1671. p->rt.watchdog_stamp = jiffies;
  1672. }
  1673. next = DIV_ROUND_UP(min(soft, hard), USEC_PER_SEC/HZ);
  1674. if (p->rt.timeout > next)
  1675. p->cputime_expires.sched_exp = p->se.sum_exec_runtime;
  1676. }
  1677. }
  1678. static void task_tick_rt(struct rq *rq, struct task_struct *p, int queued)
  1679. {
  1680. struct sched_rt_entity *rt_se = &p->rt;
  1681. update_curr_rt(rq);
  1682. watchdog(rq, p);
  1683. /*
  1684. * RR tasks need a special form of timeslice management.
  1685. * FIFO tasks have no timeslices.
  1686. */
  1687. if (p->policy != SCHED_RR)
  1688. return;
  1689. if (--p->rt.time_slice)
  1690. return;
  1691. p->rt.time_slice = sched_rr_timeslice;
  1692. /*
  1693. * Requeue to the end of queue if we (and all of our ancestors) are not
  1694. * the only element on the queue
  1695. */
  1696. for_each_sched_rt_entity(rt_se) {
  1697. if (rt_se->run_list.prev != rt_se->run_list.next) {
  1698. requeue_task_rt(rq, p, 0);
  1699. resched_curr(rq);
  1700. return;
  1701. }
  1702. }
  1703. }
  1704. static void set_curr_task_rt(struct rq *rq)
  1705. {
  1706. struct task_struct *p = rq->curr;
  1707. p->se.exec_start = rq_clock_task(rq);
  1708. /* The running task is never eligible for pushing */
  1709. dequeue_pushable_task(rq, p);
  1710. }
  1711. static unsigned int get_rr_interval_rt(struct rq *rq, struct task_struct *task)
  1712. {
  1713. /*
  1714. * Time slice is 0 for SCHED_FIFO tasks
  1715. */
  1716. if (task->policy == SCHED_RR)
  1717. return sched_rr_timeslice;
  1718. else
  1719. return 0;
  1720. }
  1721. const struct sched_class rt_sched_class = {
  1722. .next = &fair_sched_class,
  1723. .enqueue_task = enqueue_task_rt,
  1724. .dequeue_task = dequeue_task_rt,
  1725. .yield_task = yield_task_rt,
  1726. .check_preempt_curr = check_preempt_curr_rt,
  1727. .pick_next_task = pick_next_task_rt,
  1728. .put_prev_task = put_prev_task_rt,
  1729. #ifdef CONFIG_SMP
  1730. .select_task_rq = select_task_rq_rt,
  1731. .set_cpus_allowed = set_cpus_allowed_rt,
  1732. .rq_online = rq_online_rt,
  1733. .rq_offline = rq_offline_rt,
  1734. .post_schedule = post_schedule_rt,
  1735. .task_woken = task_woken_rt,
  1736. .switched_from = switched_from_rt,
  1737. #endif
  1738. .set_curr_task = set_curr_task_rt,
  1739. .task_tick = task_tick_rt,
  1740. .get_rr_interval = get_rr_interval_rt,
  1741. .prio_changed = prio_changed_rt,
  1742. .switched_to = switched_to_rt,
  1743. .update_curr = update_curr_rt,
  1744. };
  1745. #ifdef CONFIG_SCHED_DEBUG
  1746. extern void print_rt_rq(struct seq_file *m, int cpu, struct rt_rq *rt_rq);
  1747. void print_rt_stats(struct seq_file *m, int cpu)
  1748. {
  1749. rt_rq_iter_t iter;
  1750. struct rt_rq *rt_rq;
  1751. rcu_read_lock();
  1752. for_each_rt_rq(rt_rq, iter, cpu_rq(cpu))
  1753. print_rt_rq(m, cpu, rt_rq);
  1754. rcu_read_unlock();
  1755. }
  1756. #endif /* CONFIG_SCHED_DEBUG */