|
@@ -1,5 +1,7 @@
|
|
|
/* SPDX-License-Identifier: GPL-2.0 */
|
|
|
-
|
|
|
+/*
|
|
|
+ * Scheduler internal types and methods:
|
|
|
+ */
|
|
|
#include <linux/sched.h>
|
|
|
#include <linux/sched/autogroup.h>
|
|
|
#include <linux/sched/sysctl.h>
|
|
@@ -79,11 +81,11 @@ static inline void cpu_load_update_active(struct rq *this_rq) { }
|
|
|
* and does not change the user-interface for setting shares/weights.
|
|
|
*
|
|
|
* We increase resolution only if we have enough bits to allow this increased
|
|
|
- * resolution (i.e. 64bit). The costs for increasing resolution when 32bit are
|
|
|
- * pretty high and the returns do not justify the increased costs.
|
|
|
+ * resolution (i.e. 64-bit). The costs for increasing resolution when 32-bit
|
|
|
+ * are pretty high and the returns do not justify the increased costs.
|
|
|
*
|
|
|
- * Really only required when CONFIG_FAIR_GROUP_SCHED is also set, but to
|
|
|
- * increase coverage and consistency always enable it on 64bit platforms.
|
|
|
+ * Really only required when CONFIG_FAIR_GROUP_SCHED=y is also set, but to
|
|
|
+ * increase coverage and consistency always enable it on 64-bit platforms.
|
|
|
*/
|
|
|
#ifdef CONFIG_64BIT
|
|
|
# define NICE_0_LOAD_SHIFT (SCHED_FIXEDPOINT_SHIFT + SCHED_FIXEDPOINT_SHIFT)
|
|
@@ -111,16 +113,12 @@ static inline void cpu_load_update_active(struct rq *this_rq) { }
|
|
|
* 10 -> just above 1us
|
|
|
* 9 -> just above 0.5us
|
|
|
*/
|
|
|
-#define DL_SCALE (10)
|
|
|
-
|
|
|
-/*
|
|
|
- * These are the 'tuning knobs' of the scheduler:
|
|
|
- */
|
|
|
+#define DL_SCALE 10
|
|
|
|
|
|
/*
|
|
|
- * single value that denotes runtime == period, ie unlimited time.
|
|
|
+ * Single value that denotes runtime == period, ie unlimited time.
|
|
|
*/
|
|
|
-#define RUNTIME_INF ((u64)~0ULL)
|
|
|
+#define RUNTIME_INF ((u64)~0ULL)
|
|
|
|
|
|
static inline int idle_policy(int policy)
|
|
|
{
|
|
@@ -235,9 +233,9 @@ void __dl_clear_params(struct task_struct *p);
|
|
|
* control.
|
|
|
*/
|
|
|
struct dl_bandwidth {
|
|
|
- raw_spinlock_t dl_runtime_lock;
|
|
|
- u64 dl_runtime;
|
|
|
- u64 dl_period;
|
|
|
+ raw_spinlock_t dl_runtime_lock;
|
|
|
+ u64 dl_runtime;
|
|
|
+ u64 dl_period;
|
|
|
};
|
|
|
|
|
|
static inline int dl_bandwidth_enabled(void)
|
|
@@ -246,8 +244,9 @@ static inline int dl_bandwidth_enabled(void)
|
|
|
}
|
|
|
|
|
|
struct dl_bw {
|
|
|
- raw_spinlock_t lock;
|
|
|
- u64 bw, total_bw;
|
|
|
+ raw_spinlock_t lock;
|
|
|
+ u64 bw;
|
|
|
+ u64 total_bw;
|
|
|
};
|
|
|
|
|
|
static inline void __dl_update(struct dl_bw *dl_b, s64 bw);
|
|
@@ -273,20 +272,17 @@ bool __dl_overflow(struct dl_bw *dl_b, int cpus, u64 old_bw, u64 new_bw)
|
|
|
dl_b->bw * cpus < dl_b->total_bw - old_bw + new_bw;
|
|
|
}
|
|
|
|
|
|
-void dl_change_utilization(struct task_struct *p, u64 new_bw);
|
|
|
+extern void dl_change_utilization(struct task_struct *p, u64 new_bw);
|
|
|
extern void init_dl_bw(struct dl_bw *dl_b);
|
|
|
-extern int sched_dl_global_validate(void);
|
|
|
+extern int sched_dl_global_validate(void);
|
|
|
extern void sched_dl_do_global(void);
|
|
|
-extern int sched_dl_overflow(struct task_struct *p, int policy,
|
|
|
- const struct sched_attr *attr);
|
|
|
+extern int sched_dl_overflow(struct task_struct *p, int policy, const struct sched_attr *attr);
|
|
|
extern void __setparam_dl(struct task_struct *p, const struct sched_attr *attr);
|
|
|
extern void __getparam_dl(struct task_struct *p, struct sched_attr *attr);
|
|
|
extern bool __checkparam_dl(const struct sched_attr *attr);
|
|
|
extern bool dl_param_changed(struct task_struct *p, const struct sched_attr *attr);
|
|
|
-extern int dl_task_can_attach(struct task_struct *p,
|
|
|
- const struct cpumask *cs_cpus_allowed);
|
|
|
-extern int dl_cpuset_cpumask_can_shrink(const struct cpumask *cur,
|
|
|
- const struct cpumask *trial);
|
|
|
+extern int dl_task_can_attach(struct task_struct *p, const struct cpumask *cs_cpus_allowed);
|
|
|
+extern int dl_cpuset_cpumask_can_shrink(const struct cpumask *cur, const struct cpumask *trial);
|
|
|
extern bool dl_cpu_busy(unsigned int cpu);
|
|
|
|
|
|
#ifdef CONFIG_CGROUP_SCHED
|
|
@@ -300,32 +296,36 @@ extern struct list_head task_groups;
|
|
|
|
|
|
struct cfs_bandwidth {
|
|
|
#ifdef CONFIG_CFS_BANDWIDTH
|
|
|
- raw_spinlock_t lock;
|
|
|
- ktime_t period;
|
|
|
- u64 quota, runtime;
|
|
|
- s64 hierarchical_quota;
|
|
|
- u64 runtime_expires;
|
|
|
-
|
|
|
- int idle, period_active;
|
|
|
- struct hrtimer period_timer, slack_timer;
|
|
|
- struct list_head throttled_cfs_rq;
|
|
|
-
|
|
|
- /* statistics */
|
|
|
- int nr_periods, nr_throttled;
|
|
|
- u64 throttled_time;
|
|
|
+ raw_spinlock_t lock;
|
|
|
+ ktime_t period;
|
|
|
+ u64 quota;
|
|
|
+ u64 runtime;
|
|
|
+ s64 hierarchical_quota;
|
|
|
+ u64 runtime_expires;
|
|
|
+
|
|
|
+ int idle;
|
|
|
+ int period_active;
|
|
|
+ struct hrtimer period_timer;
|
|
|
+ struct hrtimer slack_timer;
|
|
|
+ struct list_head throttled_cfs_rq;
|
|
|
+
|
|
|
+ /* Statistics: */
|
|
|
+ int nr_periods;
|
|
|
+ int nr_throttled;
|
|
|
+ u64 throttled_time;
|
|
|
#endif
|
|
|
};
|
|
|
|
|
|
-/* task group related information */
|
|
|
+/* Task group related information */
|
|
|
struct task_group {
|
|
|
struct cgroup_subsys_state css;
|
|
|
|
|
|
#ifdef CONFIG_FAIR_GROUP_SCHED
|
|
|
- /* schedulable entities of this group on each cpu */
|
|
|
- struct sched_entity **se;
|
|
|
- /* runqueue "owned" by this group on each cpu */
|
|
|
- struct cfs_rq **cfs_rq;
|
|
|
- unsigned long shares;
|
|
|
+ /* schedulable entities of this group on each CPU */
|
|
|
+ struct sched_entity **se;
|
|
|
+ /* runqueue "owned" by this group on each CPU */
|
|
|
+ struct cfs_rq **cfs_rq;
|
|
|
+ unsigned long shares;
|
|
|
|
|
|
#ifdef CONFIG_SMP
|
|
|
/*
|
|
@@ -333,29 +333,29 @@ struct task_group {
|
|
|
* it in its own cacheline separated from the fields above which
|
|
|
* will also be accessed at each tick.
|
|
|
*/
|
|
|
- atomic_long_t load_avg ____cacheline_aligned;
|
|
|
+ atomic_long_t load_avg ____cacheline_aligned;
|
|
|
#endif
|
|
|
#endif
|
|
|
|
|
|
#ifdef CONFIG_RT_GROUP_SCHED
|
|
|
- struct sched_rt_entity **rt_se;
|
|
|
- struct rt_rq **rt_rq;
|
|
|
+ struct sched_rt_entity **rt_se;
|
|
|
+ struct rt_rq **rt_rq;
|
|
|
|
|
|
- struct rt_bandwidth rt_bandwidth;
|
|
|
+ struct rt_bandwidth rt_bandwidth;
|
|
|
#endif
|
|
|
|
|
|
- struct rcu_head rcu;
|
|
|
- struct list_head list;
|
|
|
+ struct rcu_head rcu;
|
|
|
+ struct list_head list;
|
|
|
|
|
|
- struct task_group *parent;
|
|
|
- struct list_head siblings;
|
|
|
- struct list_head children;
|
|
|
+ struct task_group *parent;
|
|
|
+ struct list_head siblings;
|
|
|
+ struct list_head children;
|
|
|
|
|
|
#ifdef CONFIG_SCHED_AUTOGROUP
|
|
|
- struct autogroup *autogroup;
|
|
|
+ struct autogroup *autogroup;
|
|
|
#endif
|
|
|
|
|
|
- struct cfs_bandwidth cfs_bandwidth;
|
|
|
+ struct cfs_bandwidth cfs_bandwidth;
|
|
|
};
|
|
|
|
|
|
#ifdef CONFIG_FAIR_GROUP_SCHED
|
|
@@ -369,8 +369,8 @@ struct task_group {
|
|
|
* (The default weight is 1024 - so there's no practical
|
|
|
* limitation from this.)
|
|
|
*/
|
|
|
-#define MIN_SHARES (1UL << 1)
|
|
|
-#define MAX_SHARES (1UL << 18)
|
|
|
+#define MIN_SHARES (1UL << 1)
|
|
|
+#define MAX_SHARES (1UL << 18)
|
|
|
#endif
|
|
|
|
|
|
typedef int (*tg_visitor)(struct task_group *, void *);
|
|
@@ -443,35 +443,39 @@ struct cfs_bandwidth { };
|
|
|
|
|
|
/* CFS-related fields in a runqueue */
|
|
|
struct cfs_rq {
|
|
|
- struct load_weight load;
|
|
|
- unsigned long runnable_weight;
|
|
|
- unsigned int nr_running, h_nr_running;
|
|
|
+ struct load_weight load;
|
|
|
+ unsigned long runnable_weight;
|
|
|
+ unsigned int nr_running;
|
|
|
+ unsigned int h_nr_running;
|
|
|
|
|
|
- u64 exec_clock;
|
|
|
- u64 min_vruntime;
|
|
|
+ u64 exec_clock;
|
|
|
+ u64 min_vruntime;
|
|
|
#ifndef CONFIG_64BIT
|
|
|
- u64 min_vruntime_copy;
|
|
|
+ u64 min_vruntime_copy;
|
|
|
#endif
|
|
|
|
|
|
- struct rb_root_cached tasks_timeline;
|
|
|
+ struct rb_root_cached tasks_timeline;
|
|
|
|
|
|
/*
|
|
|
* 'curr' points to currently running entity on this cfs_rq.
|
|
|
* It is set to NULL otherwise (i.e when none are currently running).
|
|
|
*/
|
|
|
- struct sched_entity *curr, *next, *last, *skip;
|
|
|
+ struct sched_entity *curr;
|
|
|
+ struct sched_entity *next;
|
|
|
+ struct sched_entity *last;
|
|
|
+ struct sched_entity *skip;
|
|
|
|
|
|
#ifdef CONFIG_SCHED_DEBUG
|
|
|
- unsigned int nr_spread_over;
|
|
|
+ unsigned int nr_spread_over;
|
|
|
#endif
|
|
|
|
|
|
#ifdef CONFIG_SMP
|
|
|
/*
|
|
|
* CFS load tracking
|
|
|
*/
|
|
|
- struct sched_avg avg;
|
|
|
+ struct sched_avg avg;
|
|
|
#ifndef CONFIG_64BIT
|
|
|
- u64 load_last_update_time_copy;
|
|
|
+ u64 load_last_update_time_copy;
|
|
|
#endif
|
|
|
struct {
|
|
|
raw_spinlock_t lock ____cacheline_aligned;
|
|
@@ -482,9 +486,9 @@ struct cfs_rq {
|
|
|
} removed;
|
|
|
|
|
|
#ifdef CONFIG_FAIR_GROUP_SCHED
|
|
|
- unsigned long tg_load_avg_contrib;
|
|
|
- long propagate;
|
|
|
- long prop_runnable_sum;
|
|
|
+ unsigned long tg_load_avg_contrib;
|
|
|
+ long propagate;
|
|
|
+ long prop_runnable_sum;
|
|
|
|
|
|
/*
|
|
|
* h_load = weight * f(tg)
|
|
@@ -492,36 +496,38 @@ struct cfs_rq {
|
|
|
* Where f(tg) is the recursive weight fraction assigned to
|
|
|
* this group.
|
|
|
*/
|
|
|
- unsigned long h_load;
|
|
|
- u64 last_h_load_update;
|
|
|
- struct sched_entity *h_load_next;
|
|
|
+ unsigned long h_load;
|
|
|
+ u64 last_h_load_update;
|
|
|
+ struct sched_entity *h_load_next;
|
|
|
#endif /* CONFIG_FAIR_GROUP_SCHED */
|
|
|
#endif /* CONFIG_SMP */
|
|
|
|
|
|
#ifdef CONFIG_FAIR_GROUP_SCHED
|
|
|
- struct rq *rq; /* cpu runqueue to which this cfs_rq is attached */
|
|
|
+ struct rq *rq; /* CPU runqueue to which this cfs_rq is attached */
|
|
|
|
|
|
/*
|
|
|
* leaf cfs_rqs are those that hold tasks (lowest schedulable entity in
|
|
|
* a hierarchy). Non-leaf lrqs hold other higher schedulable entities
|
|
|
* (like users, containers etc.)
|
|
|
*
|
|
|
- * leaf_cfs_rq_list ties together list of leaf cfs_rq's in a cpu. This
|
|
|
- * list is used during load balance.
|
|
|
+ * leaf_cfs_rq_list ties together list of leaf cfs_rq's in a CPU.
|
|
|
+ * This list is used during load balance.
|
|
|
*/
|
|
|
- int on_list;
|
|
|
- struct list_head leaf_cfs_rq_list;
|
|
|
- struct task_group *tg; /* group that "owns" this runqueue */
|
|
|
+ int on_list;
|
|
|
+ struct list_head leaf_cfs_rq_list;
|
|
|
+ struct task_group *tg; /* group that "owns" this runqueue */
|
|
|
|
|
|
#ifdef CONFIG_CFS_BANDWIDTH
|
|
|
- int runtime_enabled;
|
|
|
- u64 runtime_expires;
|
|
|
- s64 runtime_remaining;
|
|
|
-
|
|
|
- u64 throttled_clock, throttled_clock_task;
|
|
|
- u64 throttled_clock_task_time;
|
|
|
- int throttled, throttle_count;
|
|
|
- struct list_head throttled_list;
|
|
|
+ int runtime_enabled;
|
|
|
+ u64 runtime_expires;
|
|
|
+ s64 runtime_remaining;
|
|
|
+
|
|
|
+ u64 throttled_clock;
|
|
|
+ u64 throttled_clock_task;
|
|
|
+ u64 throttled_clock_task_time;
|
|
|
+ int throttled;
|
|
|
+ int throttle_count;
|
|
|
+ struct list_head throttled_list;
|
|
|
#endif /* CONFIG_CFS_BANDWIDTH */
|
|
|
#endif /* CONFIG_FAIR_GROUP_SCHED */
|
|
|
};
|
|
@@ -538,45 +544,45 @@ static inline int rt_bandwidth_enabled(void)
|
|
|
|
|
|
/* Real-Time classes' related field in a runqueue: */
|
|
|
struct rt_rq {
|
|
|
- struct rt_prio_array active;
|
|
|
- unsigned int rt_nr_running;
|
|
|
- unsigned int rr_nr_running;
|
|
|
+ struct rt_prio_array active;
|
|
|
+ unsigned int rt_nr_running;
|
|
|
+ unsigned int rr_nr_running;
|
|
|
#if defined CONFIG_SMP || defined CONFIG_RT_GROUP_SCHED
|
|
|
struct {
|
|
|
- int curr; /* highest queued rt task prio */
|
|
|
+ int curr; /* highest queued rt task prio */
|
|
|
#ifdef CONFIG_SMP
|
|
|
- int next; /* next highest */
|
|
|
+ int next; /* next highest */
|
|
|
#endif
|
|
|
} highest_prio;
|
|
|
#endif
|
|
|
#ifdef CONFIG_SMP
|
|
|
- unsigned long rt_nr_migratory;
|
|
|
- unsigned long rt_nr_total;
|
|
|
- int overloaded;
|
|
|
- struct plist_head pushable_tasks;
|
|
|
+ unsigned long rt_nr_migratory;
|
|
|
+ unsigned long rt_nr_total;
|
|
|
+ int overloaded;
|
|
|
+ struct plist_head pushable_tasks;
|
|
|
#endif /* CONFIG_SMP */
|
|
|
- int rt_queued;
|
|
|
+ int rt_queued;
|
|
|
|
|
|
- int rt_throttled;
|
|
|
- u64 rt_time;
|
|
|
- u64 rt_runtime;
|
|
|
+ int rt_throttled;
|
|
|
+ u64 rt_time;
|
|
|
+ u64 rt_runtime;
|
|
|
/* Nests inside the rq lock: */
|
|
|
- raw_spinlock_t rt_runtime_lock;
|
|
|
+ raw_spinlock_t rt_runtime_lock;
|
|
|
|
|
|
#ifdef CONFIG_RT_GROUP_SCHED
|
|
|
- unsigned long rt_nr_boosted;
|
|
|
+ unsigned long rt_nr_boosted;
|
|
|
|
|
|
- struct rq *rq;
|
|
|
- struct task_group *tg;
|
|
|
+ struct rq *rq;
|
|
|
+ struct task_group *tg;
|
|
|
#endif
|
|
|
};
|
|
|
|
|
|
/* Deadline class' related fields in a runqueue */
|
|
|
struct dl_rq {
|
|
|
/* runqueue is an rbtree, ordered by deadline */
|
|
|
- struct rb_root_cached root;
|
|
|
+ struct rb_root_cached root;
|
|
|
|
|
|
- unsigned long dl_nr_running;
|
|
|
+ unsigned long dl_nr_running;
|
|
|
|
|
|
#ifdef CONFIG_SMP
|
|
|
/*
|
|
@@ -586,28 +592,28 @@ struct dl_rq {
|
|
|
* should migrate somewhere else.
|
|
|
*/
|
|
|
struct {
|
|
|
- u64 curr;
|
|
|
- u64 next;
|
|
|
+ u64 curr;
|
|
|
+ u64 next;
|
|
|
} earliest_dl;
|
|
|
|
|
|
- unsigned long dl_nr_migratory;
|
|
|
- int overloaded;
|
|
|
+ unsigned long dl_nr_migratory;
|
|
|
+ int overloaded;
|
|
|
|
|
|
/*
|
|
|
* Tasks on this rq that can be pushed away. They are kept in
|
|
|
* an rb-tree, ordered by tasks' deadlines, with caching
|
|
|
* of the leftmost (earliest deadline) element.
|
|
|
*/
|
|
|
- struct rb_root_cached pushable_dl_tasks_root;
|
|
|
+ struct rb_root_cached pushable_dl_tasks_root;
|
|
|
#else
|
|
|
- struct dl_bw dl_bw;
|
|
|
+ struct dl_bw dl_bw;
|
|
|
#endif
|
|
|
/*
|
|
|
* "Active utilization" for this runqueue: increased when a
|
|
|
* task wakes up (becomes TASK_RUNNING) and decreased when a
|
|
|
* task blocks
|
|
|
*/
|
|
|
- u64 running_bw;
|
|
|
+ u64 running_bw;
|
|
|
|
|
|
/*
|
|
|
* Utilization of the tasks "assigned" to this runqueue (including
|
|
@@ -618,14 +624,14 @@ struct dl_rq {
|
|
|
* This is needed to compute the "inactive utilization" for the
|
|
|
* runqueue (inactive utilization = this_bw - running_bw).
|
|
|
*/
|
|
|
- u64 this_bw;
|
|
|
- u64 extra_bw;
|
|
|
+ u64 this_bw;
|
|
|
+ u64 extra_bw;
|
|
|
|
|
|
/*
|
|
|
* Inverse of the fraction of CPU utilization that can be reclaimed
|
|
|
* by the GRUB algorithm.
|
|
|
*/
|
|
|
- u64 bw_ratio;
|
|
|
+ u64 bw_ratio;
|
|
|
};
|
|
|
|
|
|
#ifdef CONFIG_SMP
|
|
@@ -638,51 +644,51 @@ static inline bool sched_asym_prefer(int a, int b)
|
|
|
/*
|
|
|
* We add the notion of a root-domain which will be used to define per-domain
|
|
|
* variables. Each exclusive cpuset essentially defines an island domain by
|
|
|
- * fully partitioning the member cpus from any other cpuset. Whenever a new
|
|
|
+ * fully partitioning the member CPUs from any other cpuset. Whenever a new
|
|
|
* exclusive cpuset is created, we also create and attach a new root-domain
|
|
|
* object.
|
|
|
*
|
|
|
*/
|
|
|
struct root_domain {
|
|
|
- atomic_t refcount;
|
|
|
- atomic_t rto_count;
|
|
|
- struct rcu_head rcu;
|
|
|
- cpumask_var_t span;
|
|
|
- cpumask_var_t online;
|
|
|
+ atomic_t refcount;
|
|
|
+ atomic_t rto_count;
|
|
|
+ struct rcu_head rcu;
|
|
|
+ cpumask_var_t span;
|
|
|
+ cpumask_var_t online;
|
|
|
|
|
|
/* Indicate more than one runnable task for any CPU */
|
|
|
- bool overload;
|
|
|
+ bool overload;
|
|
|
|
|
|
/*
|
|
|
* The bit corresponding to a CPU gets set here if such CPU has more
|
|
|
* than one runnable -deadline task (as it is below for RT tasks).
|
|
|
*/
|
|
|
- cpumask_var_t dlo_mask;
|
|
|
- atomic_t dlo_count;
|
|
|
- struct dl_bw dl_bw;
|
|
|
- struct cpudl cpudl;
|
|
|
+ cpumask_var_t dlo_mask;
|
|
|
+ atomic_t dlo_count;
|
|
|
+ struct dl_bw dl_bw;
|
|
|
+ struct cpudl cpudl;
|
|
|
|
|
|
#ifdef HAVE_RT_PUSH_IPI
|
|
|
/*
|
|
|
* For IPI pull requests, loop across the rto_mask.
|
|
|
*/
|
|
|
- struct irq_work rto_push_work;
|
|
|
- raw_spinlock_t rto_lock;
|
|
|
+ struct irq_work rto_push_work;
|
|
|
+ raw_spinlock_t rto_lock;
|
|
|
/* These are only updated and read within rto_lock */
|
|
|
- int rto_loop;
|
|
|
- int rto_cpu;
|
|
|
+ int rto_loop;
|
|
|
+ int rto_cpu;
|
|
|
/* These atomics are updated outside of a lock */
|
|
|
- atomic_t rto_loop_next;
|
|
|
- atomic_t rto_loop_start;
|
|
|
+ atomic_t rto_loop_next;
|
|
|
+ atomic_t rto_loop_start;
|
|
|
#endif
|
|
|
/*
|
|
|
* The "RT overload" flag: it gets set if a CPU has more than
|
|
|
* one runnable RT task.
|
|
|
*/
|
|
|
- cpumask_var_t rto_mask;
|
|
|
- struct cpupri cpupri;
|
|
|
+ cpumask_var_t rto_mask;
|
|
|
+ struct cpupri cpupri;
|
|
|
|
|
|
- unsigned long max_cpu_capacity;
|
|
|
+ unsigned long max_cpu_capacity;
|
|
|
};
|
|
|
|
|
|
extern struct root_domain def_root_domain;
|
|
@@ -708,39 +714,39 @@ extern void rto_push_irq_work_func(struct irq_work *work);
|
|
|
*/
|
|
|
struct rq {
|
|
|
/* runqueue lock: */
|
|
|
- raw_spinlock_t lock;
|
|
|
+ raw_spinlock_t lock;
|
|
|
|
|
|
/*
|
|
|
* nr_running and cpu_load should be in the same cacheline because
|
|
|
* remote CPUs use both these fields when doing load calculation.
|
|
|
*/
|
|
|
- unsigned int nr_running;
|
|
|
+ unsigned int nr_running;
|
|
|
#ifdef CONFIG_NUMA_BALANCING
|
|
|
- unsigned int nr_numa_running;
|
|
|
- unsigned int nr_preferred_running;
|
|
|
+ unsigned int nr_numa_running;
|
|
|
+ unsigned int nr_preferred_running;
|
|
|
#endif
|
|
|
#define CPU_LOAD_IDX_MAX 5
|
|
|
- unsigned long cpu_load[CPU_LOAD_IDX_MAX];
|
|
|
+ unsigned long cpu_load[CPU_LOAD_IDX_MAX];
|
|
|
#ifdef CONFIG_NO_HZ_COMMON
|
|
|
#ifdef CONFIG_SMP
|
|
|
- unsigned long last_load_update_tick;
|
|
|
+ unsigned long last_load_update_tick;
|
|
|
#endif /* CONFIG_SMP */
|
|
|
- unsigned long nohz_flags;
|
|
|
+ unsigned long nohz_flags;
|
|
|
#endif /* CONFIG_NO_HZ_COMMON */
|
|
|
|
|
|
- /* capture load from *all* tasks on this cpu: */
|
|
|
- struct load_weight load;
|
|
|
- unsigned long nr_load_updates;
|
|
|
- u64 nr_switches;
|
|
|
+ /* capture load from *all* tasks on this CPU: */
|
|
|
+ struct load_weight load;
|
|
|
+ unsigned long nr_load_updates;
|
|
|
+ u64 nr_switches;
|
|
|
|
|
|
- struct cfs_rq cfs;
|
|
|
- struct rt_rq rt;
|
|
|
- struct dl_rq dl;
|
|
|
+ struct cfs_rq cfs;
|
|
|
+ struct rt_rq rt;
|
|
|
+ struct dl_rq dl;
|
|
|
|
|
|
#ifdef CONFIG_FAIR_GROUP_SCHED
|
|
|
- /* list of leaf cfs_rq on this cpu: */
|
|
|
- struct list_head leaf_cfs_rq_list;
|
|
|
- struct list_head *tmp_alone_branch;
|
|
|
+ /* list of leaf cfs_rq on this CPU: */
|
|
|
+ struct list_head leaf_cfs_rq_list;
|
|
|
+ struct list_head *tmp_alone_branch;
|
|
|
#endif /* CONFIG_FAIR_GROUP_SCHED */
|
|
|
|
|
|
/*
|
|
@@ -749,94 +755,98 @@ struct rq {
|
|
|
* one CPU and if it got migrated afterwards it may decrease
|
|
|
* it on another CPU. Always updated under the runqueue lock:
|
|
|
*/
|
|
|
- unsigned long nr_uninterruptible;
|
|
|
+ unsigned long nr_uninterruptible;
|
|
|
|
|
|
- struct task_struct *curr, *idle, *stop;
|
|
|
- unsigned long next_balance;
|
|
|
- struct mm_struct *prev_mm;
|
|
|
+ struct task_struct *curr;
|
|
|
+ struct task_struct *idle;
|
|
|
+ struct task_struct *stop;
|
|
|
+ unsigned long next_balance;
|
|
|
+ struct mm_struct *prev_mm;
|
|
|
|
|
|
- unsigned int clock_update_flags;
|
|
|
- u64 clock;
|
|
|
- u64 clock_task;
|
|
|
+ unsigned int clock_update_flags;
|
|
|
+ u64 clock;
|
|
|
+ u64 clock_task;
|
|
|
|
|
|
- atomic_t nr_iowait;
|
|
|
+ atomic_t nr_iowait;
|
|
|
|
|
|
#ifdef CONFIG_SMP
|
|
|
- struct root_domain *rd;
|
|
|
- struct sched_domain *sd;
|
|
|
+ struct root_domain *rd;
|
|
|
+ struct sched_domain *sd;
|
|
|
+
|
|
|
+ unsigned long cpu_capacity;
|
|
|
+ unsigned long cpu_capacity_orig;
|
|
|
|
|
|
- unsigned long cpu_capacity;
|
|
|
- unsigned long cpu_capacity_orig;
|
|
|
+ struct callback_head *balance_callback;
|
|
|
|
|
|
- struct callback_head *balance_callback;
|
|
|
+ unsigned char idle_balance;
|
|
|
|
|
|
- unsigned char idle_balance;
|
|
|
/* For active balancing */
|
|
|
- int active_balance;
|
|
|
- int push_cpu;
|
|
|
- struct cpu_stop_work active_balance_work;
|
|
|
- /* cpu of this runqueue: */
|
|
|
- int cpu;
|
|
|
- int online;
|
|
|
+ int active_balance;
|
|
|
+ int push_cpu;
|
|
|
+ struct cpu_stop_work active_balance_work;
|
|
|
+
|
|
|
+ /* CPU of this runqueue: */
|
|
|
+ int cpu;
|
|
|
+ int online;
|
|
|
|
|
|
struct list_head cfs_tasks;
|
|
|
|
|
|
- u64 rt_avg;
|
|
|
- u64 age_stamp;
|
|
|
- u64 idle_stamp;
|
|
|
- u64 avg_idle;
|
|
|
+ u64 rt_avg;
|
|
|
+ u64 age_stamp;
|
|
|
+ u64 idle_stamp;
|
|
|
+ u64 avg_idle;
|
|
|
|
|
|
/* This is used to determine avg_idle's max value */
|
|
|
- u64 max_idle_balance_cost;
|
|
|
+ u64 max_idle_balance_cost;
|
|
|
#endif
|
|
|
|
|
|
#ifdef CONFIG_IRQ_TIME_ACCOUNTING
|
|
|
- u64 prev_irq_time;
|
|
|
+ u64 prev_irq_time;
|
|
|
#endif
|
|
|
#ifdef CONFIG_PARAVIRT
|
|
|
- u64 prev_steal_time;
|
|
|
+ u64 prev_steal_time;
|
|
|
#endif
|
|
|
#ifdef CONFIG_PARAVIRT_TIME_ACCOUNTING
|
|
|
- u64 prev_steal_time_rq;
|
|
|
+ u64 prev_steal_time_rq;
|
|
|
#endif
|
|
|
|
|
|
/* calc_load related fields */
|
|
|
- unsigned long calc_load_update;
|
|
|
- long calc_load_active;
|
|
|
+ unsigned long calc_load_update;
|
|
|
+ long calc_load_active;
|
|
|
|
|
|
#ifdef CONFIG_SCHED_HRTICK
|
|
|
#ifdef CONFIG_SMP
|
|
|
- int hrtick_csd_pending;
|
|
|
- call_single_data_t hrtick_csd;
|
|
|
+ int hrtick_csd_pending;
|
|
|
+ call_single_data_t hrtick_csd;
|
|
|
#endif
|
|
|
- struct hrtimer hrtick_timer;
|
|
|
+ struct hrtimer hrtick_timer;
|
|
|
#endif
|
|
|
|
|
|
#ifdef CONFIG_SCHEDSTATS
|
|
|
/* latency stats */
|
|
|
- struct sched_info rq_sched_info;
|
|
|
- unsigned long long rq_cpu_time;
|
|
|
+ struct sched_info rq_sched_info;
|
|
|
+ unsigned long long rq_cpu_time;
|
|
|
/* could above be rq->cfs_rq.exec_clock + rq->rt_rq.rt_runtime ? */
|
|
|
|
|
|
/* sys_sched_yield() stats */
|
|
|
- unsigned int yld_count;
|
|
|
+ unsigned int yld_count;
|
|
|
|
|
|
/* schedule() stats */
|
|
|
- unsigned int sched_count;
|
|
|
- unsigned int sched_goidle;
|
|
|
+ unsigned int sched_count;
|
|
|
+ unsigned int sched_goidle;
|
|
|
|
|
|
/* try_to_wake_up() stats */
|
|
|
- unsigned int ttwu_count;
|
|
|
- unsigned int ttwu_local;
|
|
|
+ unsigned int ttwu_count;
|
|
|
+ unsigned int ttwu_local;
|
|
|
#endif
|
|
|
|
|
|
#ifdef CONFIG_SMP
|
|
|
- struct llist_head wake_list;
|
|
|
+ struct llist_head wake_list;
|
|
|
#endif
|
|
|
|
|
|
#ifdef CONFIG_CPU_IDLE
|
|
|
/* Must be inspected within a rcu lock section */
|
|
|
- struct cpuidle_state *idle_state;
|
|
|
+ struct cpuidle_state *idle_state;
|
|
|
#endif
|
|
|
};
|
|
|
|
|
@@ -902,9 +912,9 @@ static inline u64 __rq_clock_broken(struct rq *rq)
|
|
|
* one position though, because the next rq_unpin_lock() will shift it
|
|
|
* back.
|
|
|
*/
|
|
|
-#define RQCF_REQ_SKIP 0x01
|
|
|
-#define RQCF_ACT_SKIP 0x02
|
|
|
-#define RQCF_UPDATED 0x04
|
|
|
+#define RQCF_REQ_SKIP 0x01
|
|
|
+#define RQCF_ACT_SKIP 0x02
|
|
|
+#define RQCF_UPDATED 0x04
|
|
|
|
|
|
static inline void assert_clock_updated(struct rq *rq)
|
|
|
{
|
|
@@ -1057,12 +1067,12 @@ extern void sched_ttwu_pending(void);
|
|
|
|
|
|
/**
|
|
|
* highest_flag_domain - Return highest sched_domain containing flag.
|
|
|
- * @cpu: The cpu whose highest level of sched domain is to
|
|
|
+ * @cpu: The CPU whose highest level of sched domain is to
|
|
|
* be returned.
|
|
|
* @flag: The flag to check for the highest sched_domain
|
|
|
- * for the given cpu.
|
|
|
+ * for the given CPU.
|
|
|
*
|
|
|
- * Returns the highest sched_domain of a cpu which contains the given flag.
|
|
|
+ * Returns the highest sched_domain of a CPU which contains the given flag.
|
|
|
*/
|
|
|
static inline struct sched_domain *highest_flag_domain(int cpu, int flag)
|
|
|
{
|
|
@@ -1097,30 +1107,30 @@ DECLARE_PER_CPU(struct sched_domain *, sd_numa);
|
|
|
DECLARE_PER_CPU(struct sched_domain *, sd_asym);
|
|
|
|
|
|
struct sched_group_capacity {
|
|
|
- atomic_t ref;
|
|
|
+ atomic_t ref;
|
|
|
/*
|
|
|
* CPU capacity of this group, SCHED_CAPACITY_SCALE being max capacity
|
|
|
* for a single CPU.
|
|
|
*/
|
|
|
- unsigned long capacity;
|
|
|
- unsigned long min_capacity; /* Min per-CPU capacity in group */
|
|
|
- unsigned long next_update;
|
|
|
- int imbalance; /* XXX unrelated to capacity but shared group state */
|
|
|
+ unsigned long capacity;
|
|
|
+ unsigned long min_capacity; /* Min per-CPU capacity in group */
|
|
|
+ unsigned long next_update;
|
|
|
+ int imbalance; /* XXX unrelated to capacity but shared group state */
|
|
|
|
|
|
#ifdef CONFIG_SCHED_DEBUG
|
|
|
- int id;
|
|
|
+ int id;
|
|
|
#endif
|
|
|
|
|
|
- unsigned long cpumask[0]; /* balance mask */
|
|
|
+ unsigned long cpumask[0]; /* Balance mask */
|
|
|
};
|
|
|
|
|
|
struct sched_group {
|
|
|
- struct sched_group *next; /* Must be a circular list */
|
|
|
- atomic_t ref;
|
|
|
+ struct sched_group *next; /* Must be a circular list */
|
|
|
+ atomic_t ref;
|
|
|
|
|
|
- unsigned int group_weight;
|
|
|
+ unsigned int group_weight;
|
|
|
struct sched_group_capacity *sgc;
|
|
|
- int asym_prefer_cpu; /* cpu of highest priority in group */
|
|
|
+ int asym_prefer_cpu; /* CPU of highest priority in group */
|
|
|
|
|
|
/*
|
|
|
* The CPUs this group covers.
|
|
@@ -1129,7 +1139,7 @@ struct sched_group {
|
|
|
* by attaching extra space to the end of the structure,
|
|
|
* depending on how many CPUs the kernel has booted up with)
|
|
|
*/
|
|
|
- unsigned long cpumask[0];
|
|
|
+ unsigned long cpumask[0];
|
|
|
};
|
|
|
|
|
|
static inline struct cpumask *sched_group_span(struct sched_group *sg)
|
|
@@ -1146,8 +1156,8 @@ static inline struct cpumask *group_balance_mask(struct sched_group *sg)
|
|
|
}
|
|
|
|
|
|
/**
|
|
|
- * group_first_cpu - Returns the first cpu in the cpumask of a sched_group.
|
|
|
- * @group: The group whose first cpu is to be returned.
|
|
|
+ * group_first_cpu - Returns the first CPU in the cpumask of a sched_group.
|
|
|
+ * @group: The group whose first CPU is to be returned.
|
|
|
*/
|
|
|
static inline unsigned int group_first_cpu(struct sched_group *group)
|
|
|
{
|
|
@@ -1357,9 +1367,9 @@ static inline int task_on_rq_migrating(struct task_struct *p)
|
|
|
/*
|
|
|
* wake flags
|
|
|
*/
|
|
|
-#define WF_SYNC 0x01 /* waker goes to sleep after wakeup */
|
|
|
-#define WF_FORK 0x02 /* child wakeup after fork */
|
|
|
-#define WF_MIGRATED 0x4 /* internal use, task got migrated */
|
|
|
+#define WF_SYNC 0x01 /* Waker goes to sleep after wakeup */
|
|
|
+#define WF_FORK 0x02 /* Child wakeup after fork */
|
|
|
+#define WF_MIGRATED 0x4 /* Internal use, task got migrated */
|
|
|
|
|
|
/*
|
|
|
* To aid in avoiding the subversion of "niceness" due to uneven distribution
|
|
@@ -1370,11 +1380,11 @@ static inline int task_on_rq_migrating(struct task_struct *p)
|
|
|
* slice expiry etc.
|
|
|
*/
|
|
|
|
|
|
-#define WEIGHT_IDLEPRIO 3
|
|
|
-#define WMULT_IDLEPRIO 1431655765
|
|
|
+#define WEIGHT_IDLEPRIO 3
|
|
|
+#define WMULT_IDLEPRIO 1431655765
|
|
|
|
|
|
-extern const int sched_prio_to_weight[40];
|
|
|
-extern const u32 sched_prio_to_wmult[40];
|
|
|
+extern const int sched_prio_to_weight[40];
|
|
|
+extern const u32 sched_prio_to_wmult[40];
|
|
|
|
|
|
/*
|
|
|
* {de,en}queue flags:
|
|
@@ -1396,9 +1406,9 @@ extern const u32 sched_prio_to_wmult[40];
|
|
|
*/
|
|
|
|
|
|
#define DEQUEUE_SLEEP 0x01
|
|
|
-#define DEQUEUE_SAVE 0x02 /* matches ENQUEUE_RESTORE */
|
|
|
-#define DEQUEUE_MOVE 0x04 /* matches ENQUEUE_MOVE */
|
|
|
-#define DEQUEUE_NOCLOCK 0x08 /* matches ENQUEUE_NOCLOCK */
|
|
|
+#define DEQUEUE_SAVE 0x02 /* Matches ENQUEUE_RESTORE */
|
|
|
+#define DEQUEUE_MOVE 0x04 /* Matches ENQUEUE_MOVE */
|
|
|
+#define DEQUEUE_NOCLOCK 0x08 /* Matches ENQUEUE_NOCLOCK */
|
|
|
|
|
|
#define ENQUEUE_WAKEUP 0x01
|
|
|
#define ENQUEUE_RESTORE 0x02
|
|
@@ -1420,10 +1430,10 @@ struct sched_class {
|
|
|
|
|
|
void (*enqueue_task) (struct rq *rq, struct task_struct *p, int flags);
|
|
|
void (*dequeue_task) (struct rq *rq, struct task_struct *p, int flags);
|
|
|
- void (*yield_task) (struct rq *rq);
|
|
|
- bool (*yield_to_task) (struct rq *rq, struct task_struct *p, bool preempt);
|
|
|
+ void (*yield_task) (struct rq *rq);
|
|
|
+ bool (*yield_to_task)(struct rq *rq, struct task_struct *p, bool preempt);
|
|
|
|
|
|
- void (*check_preempt_curr) (struct rq *rq, struct task_struct *p, int flags);
|
|
|
+ void (*check_preempt_curr)(struct rq *rq, struct task_struct *p, int flags);
|
|
|
|
|
|
/*
|
|
|
* It is the responsibility of the pick_next_task() method that will
|
|
@@ -1433,16 +1443,16 @@ struct sched_class {
|
|
|
* May return RETRY_TASK when it finds a higher prio class has runnable
|
|
|
* tasks.
|
|
|
*/
|
|
|
- struct task_struct * (*pick_next_task) (struct rq *rq,
|
|
|
- struct task_struct *prev,
|
|
|
- struct rq_flags *rf);
|
|
|
- void (*put_prev_task) (struct rq *rq, struct task_struct *p);
|
|
|
+ struct task_struct * (*pick_next_task)(struct rq *rq,
|
|
|
+ struct task_struct *prev,
|
|
|
+ struct rq_flags *rf);
|
|
|
+ void (*put_prev_task)(struct rq *rq, struct task_struct *p);
|
|
|
|
|
|
#ifdef CONFIG_SMP
|
|
|
int (*select_task_rq)(struct task_struct *p, int task_cpu, int sd_flag, int flags);
|
|
|
void (*migrate_task_rq)(struct task_struct *p);
|
|
|
|
|
|
- void (*task_woken) (struct rq *this_rq, struct task_struct *task);
|
|
|
+ void (*task_woken)(struct rq *this_rq, struct task_struct *task);
|
|
|
|
|
|
void (*set_cpus_allowed)(struct task_struct *p,
|
|
|
const struct cpumask *newmask);
|
|
@@ -1451,31 +1461,31 @@ struct sched_class {
|
|
|
void (*rq_offline)(struct rq *rq);
|
|
|
#endif
|
|
|
|
|
|
- void (*set_curr_task) (struct rq *rq);
|
|
|
- void (*task_tick) (struct rq *rq, struct task_struct *p, int queued);
|
|
|
- void (*task_fork) (struct task_struct *p);
|
|
|
- void (*task_dead) (struct task_struct *p);
|
|
|
+ void (*set_curr_task)(struct rq *rq);
|
|
|
+ void (*task_tick)(struct rq *rq, struct task_struct *p, int queued);
|
|
|
+ void (*task_fork)(struct task_struct *p);
|
|
|
+ void (*task_dead)(struct task_struct *p);
|
|
|
|
|
|
/*
|
|
|
* The switched_from() call is allowed to drop rq->lock, therefore we
|
|
|
* cannot assume the switched_from/switched_to pair is serliazed by
|
|
|
* rq->lock. They are however serialized by p->pi_lock.
|
|
|
*/
|
|
|
- void (*switched_from) (struct rq *this_rq, struct task_struct *task);
|
|
|
- void (*switched_to) (struct rq *this_rq, struct task_struct *task);
|
|
|
+ void (*switched_from)(struct rq *this_rq, struct task_struct *task);
|
|
|
+ void (*switched_to) (struct rq *this_rq, struct task_struct *task);
|
|
|
void (*prio_changed) (struct rq *this_rq, struct task_struct *task,
|
|
|
- int oldprio);
|
|
|
+ int oldprio);
|
|
|
|
|
|
- unsigned int (*get_rr_interval) (struct rq *rq,
|
|
|
- struct task_struct *task);
|
|
|
+ unsigned int (*get_rr_interval)(struct rq *rq,
|
|
|
+ struct task_struct *task);
|
|
|
|
|
|
- void (*update_curr) (struct rq *rq);
|
|
|
+ void (*update_curr)(struct rq *rq);
|
|
|
|
|
|
-#define TASK_SET_GROUP 0
|
|
|
-#define TASK_MOVE_GROUP 1
|
|
|
+#define TASK_SET_GROUP 0
|
|
|
+#define TASK_MOVE_GROUP 1
|
|
|
|
|
|
#ifdef CONFIG_FAIR_GROUP_SCHED
|
|
|
- void (*task_change_group) (struct task_struct *p, int type);
|
|
|
+ void (*task_change_group)(struct task_struct *p, int type);
|
|
|
#endif
|
|
|
};
|
|
|
|
|
@@ -1524,6 +1534,7 @@ static inline void idle_set_state(struct rq *rq,
|
|
|
static inline struct cpuidle_state *idle_get_state(struct rq *rq)
|
|
|
{
|
|
|
SCHED_WARN_ON(!rcu_read_lock_held());
|
|
|
+
|
|
|
return rq->idle_state;
|
|
|
}
|
|
|
#else
|
|
@@ -1562,9 +1573,9 @@ extern void init_dl_task_timer(struct sched_dl_entity *dl_se);
|
|
|
extern void init_dl_inactive_task_timer(struct sched_dl_entity *dl_se);
|
|
|
extern void init_dl_rq_bw_ratio(struct dl_rq *dl_rq);
|
|
|
|
|
|
-#define BW_SHIFT 20
|
|
|
-#define BW_UNIT (1 << BW_SHIFT)
|
|
|
-#define RATIO_SHIFT 8
|
|
|
+#define BW_SHIFT 20
|
|
|
+#define BW_UNIT (1 << BW_SHIFT)
|
|
|
+#define RATIO_SHIFT 8
|
|
|
unsigned long to_ratio(u64 period, u64 runtime);
|
|
|
|
|
|
extern void init_entity_runnable_average(struct sched_entity *se);
|
|
@@ -1814,8 +1825,8 @@ static inline int _double_lock_balance(struct rq *this_rq, struct rq *busiest)
|
|
|
/*
|
|
|
* Unfair double_lock_balance: Optimizes throughput at the expense of
|
|
|
* latency by eliminating extra atomic operations when the locks are
|
|
|
- * already in proper order on entry. This favors lower cpu-ids and will
|
|
|
- * grant the double lock to lower cpus over higher ids under contention,
|
|
|
+ * already in proper order on entry. This favors lower CPU-ids and will
|
|
|
+ * grant the double lock to lower CPUs over higher ids under contention,
|
|
|
* regardless of entry order into the function.
|
|
|
*/
|
|
|
static inline int _double_lock_balance(struct rq *this_rq, struct rq *busiest)
|
|
@@ -1847,7 +1858,7 @@ static inline int _double_lock_balance(struct rq *this_rq, struct rq *busiest)
|
|
|
static inline int double_lock_balance(struct rq *this_rq, struct rq *busiest)
|
|
|
{
|
|
|
if (unlikely(!irqs_disabled())) {
|
|
|
- /* printk() doesn't work good under rq->lock */
|
|
|
+ /* printk() doesn't work well under rq->lock */
|
|
|
raw_spin_unlock(&this_rq->lock);
|
|
|
BUG_ON(1);
|
|
|
}
|
|
@@ -2106,15 +2117,14 @@ static inline void cpufreq_update_util(struct rq *rq, unsigned int flags) {}
|
|
|
#endif /* CONFIG_CPU_FREQ */
|
|
|
|
|
|
#ifdef arch_scale_freq_capacity
|
|
|
-#ifndef arch_scale_freq_invariant
|
|
|
-#define arch_scale_freq_invariant() (true)
|
|
|
-#endif
|
|
|
-#else /* arch_scale_freq_capacity */
|
|
|
-#define arch_scale_freq_invariant() (false)
|
|
|
+# ifndef arch_scale_freq_invariant
|
|
|
+# define arch_scale_freq_invariant() true
|
|
|
+# endif
|
|
|
+#else
|
|
|
+# define arch_scale_freq_invariant() false
|
|
|
#endif
|
|
|
|
|
|
#ifdef CONFIG_CPU_FREQ_GOV_SCHEDUTIL
|
|
|
-
|
|
|
static inline unsigned long cpu_util_dl(struct rq *rq)
|
|
|
{
|
|
|
return (rq->dl.running_bw * SCHED_CAPACITY_SCALE) >> BW_SHIFT;
|
|
@@ -2124,5 +2134,4 @@ static inline unsigned long cpu_util_cfs(struct rq *rq)
|
|
|
{
|
|
|
return rq->cfs.avg.util_avg;
|
|
|
}
|
|
|
-
|
|
|
#endif
|