/drivers/cpufreq/cpufreq.c
C | 1898 lines | 1257 code | 318 blank | 323 comment | 209 complexity | 6d008d17f7f9d519e53c8ccbf96d9c1b MD5 | raw file
Possible License(s): GPL-2.0, LGPL-2.0, AGPL-1.0
1/*
2 * linux/drivers/cpufreq/cpufreq.c
3 *
4 * Copyright (C) 2001 Russell King
5 * (C) 2002 - 2003 Dominik Brodowski <linux@brodo.de>
6 *
7 * Oct 2005 - Ashok Raj <ashok.raj@intel.com>
8 * Added handling for CPU hotplug
9 * Feb 2006 - Jacob Shin <jacob.shin@amd.com>
10 * Fix handling for CPU hotplug -- affected CPUs
11 *
12 * This program is free software; you can redistribute it and/or modify
13 * it under the terms of the GNU General Public License version 2 as
14 * published by the Free Software Foundation.
15 *
16 */
17
18#include <linux/kernel.h>
19#include <linux/module.h>
20#include <linux/init.h>
21#include <linux/notifier.h>
22#include <linux/cpufreq.h>
23#include <linux/delay.h>
24#include <linux/interrupt.h>
25#include <linux/spinlock.h>
26#include <linux/device.h>
27#include <linux/slab.h>
28#include <linux/cpu.h>
29#include <linux/completion.h>
30#include <linux/mutex.h>
31#include <linux/syscore_ops.h>
32
33#include <trace/events/power.h>
34
35/**
36 * The "cpufreq driver" - the arch- or hardware-dependent low
37 * level driver of CPUFreq support, and its spinlock. This lock
38 * also protects the cpufreq_cpu_data array.
39 */
40static struct cpufreq_driver *cpufreq_driver;
41static DEFINE_PER_CPU(struct cpufreq_policy *, cpufreq_cpu_data);
42#ifdef CONFIG_HOTPLUG_CPU
43/* This one keeps track of the previously set governor of a removed CPU */
44static DEFINE_PER_CPU(char[CPUFREQ_NAME_LEN], cpufreq_cpu_governor);
45#endif
46static DEFINE_SPINLOCK(cpufreq_driver_lock);
47
48/*
49 * cpu_policy_rwsem is a per CPU reader-writer semaphore designed to cure
50 * all cpufreq/hotplug/workqueue/etc related lock issues.
51 *
52 * The rules for this semaphore:
53 * - Any routine that wants to read from the policy structure will
54 * do a down_read on this semaphore.
55 * - Any routine that will write to the policy structure and/or may take away
56 * the policy altogether (eg. CPU hotplug), will hold this lock in write
57 * mode before doing so.
58 *
59 * Additional rules:
60 * - All holders of the lock should check to make sure that the CPU they
61 * are concerned with are online after they get the lock.
62 * - Governor routines that can be called in cpufreq hotplug path should not
63 * take this sem as top level hotplug notifier handler takes this.
64 * - Lock should not be held across
65 * __cpufreq_governor(data, CPUFREQ_GOV_STOP);
66 */
67static DEFINE_PER_CPU(int, cpufreq_policy_cpu);
68static DEFINE_PER_CPU(struct rw_semaphore, cpu_policy_rwsem);
69
70#define lock_policy_rwsem(mode, cpu) \
71static int lock_policy_rwsem_##mode \
72(int cpu) \
73{ \
74 int policy_cpu = per_cpu(cpufreq_policy_cpu, cpu); \
75 BUG_ON(policy_cpu == -1); \
76 down_##mode(&per_cpu(cpu_policy_rwsem, policy_cpu)); \
77 if (unlikely(!cpu_online(cpu))) { \
78 up_##mode(&per_cpu(cpu_policy_rwsem, policy_cpu)); \
79 return -1; \
80 } \
81 \
82 return 0; \
83}
84
85lock_policy_rwsem(read, cpu);
86
87lock_policy_rwsem(write, cpu);
88
89static void unlock_policy_rwsem_read(int cpu)
90{
91 int policy_cpu = per_cpu(cpufreq_policy_cpu, cpu);
92 BUG_ON(policy_cpu == -1);
93 up_read(&per_cpu(cpu_policy_rwsem, policy_cpu));
94}
95
96static void unlock_policy_rwsem_write(int cpu)
97{
98 int policy_cpu = per_cpu(cpufreq_policy_cpu, cpu);
99 BUG_ON(policy_cpu == -1);
100 up_write(&per_cpu(cpu_policy_rwsem, policy_cpu));
101}
102
103
104/* internal prototypes */
105static int __cpufreq_governor(struct cpufreq_policy *policy,
106 unsigned int event);
107static unsigned int __cpufreq_get(unsigned int cpu);
108static void handle_update(struct work_struct *work);
109
110/**
111 * Two notifier lists: the "policy" list is involved in the
112 * validation process for a new CPU frequency policy; the
113 * "transition" list for kernel code that needs to handle
114 * changes to devices when the CPU clock speed changes.
115 * The mutex locks both lists.
116 */
117static BLOCKING_NOTIFIER_HEAD(cpufreq_policy_notifier_list);
118static struct srcu_notifier_head cpufreq_transition_notifier_list;
119
120static bool init_cpufreq_transition_notifier_list_called;
121static int __init init_cpufreq_transition_notifier_list(void)
122{
123 srcu_init_notifier_head(&cpufreq_transition_notifier_list);
124 init_cpufreq_transition_notifier_list_called = true;
125 return 0;
126}
127pure_initcall(init_cpufreq_transition_notifier_list);
128
129static LIST_HEAD(cpufreq_governor_list);
130static DEFINE_MUTEX(cpufreq_governor_mutex);
131
132struct cpufreq_policy *cpufreq_cpu_get(unsigned int cpu)
133{
134 struct cpufreq_policy *data;
135 unsigned long flags;
136
137 if (cpu >= nr_cpu_ids)
138 goto err_out;
139
140 /* get the cpufreq driver */
141 spin_lock_irqsave(&cpufreq_driver_lock, flags);
142
143 if (!cpufreq_driver)
144 goto err_out_unlock;
145
146 if (!try_module_get(cpufreq_driver->owner))
147 goto err_out_unlock;
148
149
150 /* get the CPU */
151 data = per_cpu(cpufreq_cpu_data, cpu);
152
153 if (!data)
154 goto err_out_put_module;
155
156 if (!kobject_get(&data->kobj))
157 goto err_out_put_module;
158
159 spin_unlock_irqrestore(&cpufreq_driver_lock, flags);
160 return data;
161
162err_out_put_module:
163 module_put(cpufreq_driver->owner);
164err_out_unlock:
165 spin_unlock_irqrestore(&cpufreq_driver_lock, flags);
166err_out:
167 return NULL;
168}
169EXPORT_SYMBOL_GPL(cpufreq_cpu_get);
170
171
172void cpufreq_cpu_put(struct cpufreq_policy *data)
173{
174 kobject_put(&data->kobj);
175 module_put(cpufreq_driver->owner);
176}
177EXPORT_SYMBOL_GPL(cpufreq_cpu_put);
178
179
180/*********************************************************************
181 * EXTERNALLY AFFECTING FREQUENCY CHANGES *
182 *********************************************************************/
183
184/**
185 * adjust_jiffies - adjust the system "loops_per_jiffy"
186 *
187 * This function alters the system "loops_per_jiffy" for the clock
188 * speed change. Note that loops_per_jiffy cannot be updated on SMP
189 * systems as each CPU might be scaled differently. So, use the arch
190 * per-CPU loops_per_jiffy value wherever possible.
191 */
192#ifndef CONFIG_SMP
193static unsigned long l_p_j_ref;
194static unsigned int l_p_j_ref_freq;
195
196static void adjust_jiffies(unsigned long val, struct cpufreq_freqs *ci)
197{
198 if (ci->flags & CPUFREQ_CONST_LOOPS)
199 return;
200
201 if (!l_p_j_ref_freq) {
202 l_p_j_ref = loops_per_jiffy;
203 l_p_j_ref_freq = ci->old;
204 pr_debug("saving %lu as reference value for loops_per_jiffy; "
205 "freq is %u kHz\n", l_p_j_ref, l_p_j_ref_freq);
206 }
207 if ((val == CPUFREQ_PRECHANGE && ci->old < ci->new) ||
208 (val == CPUFREQ_POSTCHANGE && ci->old > ci->new) ||
209 (val == CPUFREQ_RESUMECHANGE || val == CPUFREQ_SUSPENDCHANGE)) {
210 loops_per_jiffy = cpufreq_scale(l_p_j_ref, l_p_j_ref_freq,
211 ci->new);
212 pr_debug("scaling loops_per_jiffy to %lu "
213 "for frequency %u kHz\n", loops_per_jiffy, ci->new);
214 }
215}
216#else
217static inline void adjust_jiffies(unsigned long val, struct cpufreq_freqs *ci)
218{
219 return;
220}
221#endif
222
223
224/**
225 * cpufreq_notify_transition - call notifier chain and adjust_jiffies
226 * on frequency transition.
227 *
228 * This function calls the transition notifiers and the "adjust_jiffies"
229 * function. It is called twice on all CPU frequency changes that have
230 * external effects.
231 */
232void cpufreq_notify_transition(struct cpufreq_freqs *freqs, unsigned int state)
233{
234 struct cpufreq_policy *policy;
235
236 BUG_ON(irqs_disabled());
237
238 freqs->flags = cpufreq_driver->flags;
239 pr_debug("notification %u of frequency transition to %u kHz\n",
240 state, freqs->new);
241
242 policy = per_cpu(cpufreq_cpu_data, freqs->cpu);
243 switch (state) {
244
245 case CPUFREQ_PRECHANGE:
246 /* detect if the driver reported a value as "old frequency"
247 * which is not equal to what the cpufreq core thinks is
248 * "old frequency".
249 */
250 if (!(cpufreq_driver->flags & CPUFREQ_CONST_LOOPS)) {
251 if ((policy) && (policy->cpu == freqs->cpu) &&
252 (policy->cur) && (policy->cur != freqs->old)) {
253 pr_debug("Warning: CPU frequency is"
254 " %u, cpufreq assumed %u kHz.\n",
255 freqs->old, policy->cur);
256 freqs->old = policy->cur;
257 }
258 }
259 srcu_notifier_call_chain(&cpufreq_transition_notifier_list,
260 CPUFREQ_PRECHANGE, freqs);
261 adjust_jiffies(CPUFREQ_PRECHANGE, freqs);
262 break;
263
264 case CPUFREQ_POSTCHANGE:
265 adjust_jiffies(CPUFREQ_POSTCHANGE, freqs);
266 pr_debug("FREQ: %lu - CPU: %lu", (unsigned long)freqs->new,
267 (unsigned long)freqs->cpu);
268 trace_power_frequency(POWER_PSTATE, freqs->new, freqs->cpu);
269 trace_cpu_frequency(freqs->new, freqs->cpu);
270 srcu_notifier_call_chain(&cpufreq_transition_notifier_list,
271 CPUFREQ_POSTCHANGE, freqs);
272 if (likely(policy) && likely(policy->cpu == freqs->cpu))
273 policy->cur = freqs->new;
274 break;
275 }
276}
277EXPORT_SYMBOL_GPL(cpufreq_notify_transition);
278
279
280
281/*********************************************************************
282 * SYSFS INTERFACE *
283 *********************************************************************/
284
285static struct cpufreq_governor *__find_governor(const char *str_governor)
286{
287 struct cpufreq_governor *t;
288
289 list_for_each_entry(t, &cpufreq_governor_list, governor_list)
290 if (!strnicmp(str_governor, t->name, CPUFREQ_NAME_LEN))
291 return t;
292
293 return NULL;
294}
295
296/**
297 * cpufreq_parse_governor - parse a governor string
298 */
299static int cpufreq_parse_governor(char *str_governor, unsigned int *policy,
300 struct cpufreq_governor **governor)
301{
302 int err = -EINVAL;
303
304 if (!cpufreq_driver)
305 goto out;
306
307 if (cpufreq_driver->setpolicy) {
308 if (!strnicmp(str_governor, "performance", CPUFREQ_NAME_LEN)) {
309 *policy = CPUFREQ_POLICY_PERFORMANCE;
310 err = 0;
311 } else if (!strnicmp(str_governor, "powersave",
312 CPUFREQ_NAME_LEN)) {
313 *policy = CPUFREQ_POLICY_POWERSAVE;
314 err = 0;
315 }
316 } else if (cpufreq_driver->target) {
317 struct cpufreq_governor *t;
318
319 mutex_lock(&cpufreq_governor_mutex);
320
321 t = __find_governor(str_governor);
322
323 if (t == NULL) {
324 int ret;
325
326 mutex_unlock(&cpufreq_governor_mutex);
327 ret = request_module("cpufreq_%s", str_governor);
328 mutex_lock(&cpufreq_governor_mutex);
329
330 if (ret == 0)
331 t = __find_governor(str_governor);
332 }
333
334 if (t != NULL) {
335 *governor = t;
336 err = 0;
337 }
338
339 mutex_unlock(&cpufreq_governor_mutex);
340 }
341out:
342 return err;
343}
344
345
346/**
347 * cpufreq_per_cpu_attr_read() / show_##file_name() -
348 * print out cpufreq information
349 *
350 * Write out information from cpufreq_driver->policy[cpu]; object must be
351 * "unsigned int".
352 */
353
354#define show_one(file_name, object) \
355static ssize_t show_##file_name \
356(struct cpufreq_policy *policy, char *buf) \
357{ \
358 return sprintf(buf, "%u\n", policy->object); \
359}
360
361show_one(cpuinfo_min_freq, cpuinfo.min_freq);
362show_one(cpuinfo_max_freq, cpuinfo.max_freq);
363show_one(cpuinfo_transition_latency, cpuinfo.transition_latency);
364show_one(scaling_min_freq, min);
365show_one(scaling_max_freq, max);
366show_one(scaling_cur_freq, cur);
367
368static int __cpufreq_set_policy(struct cpufreq_policy *data,
369 struct cpufreq_policy *policy);
370
371/**
372 * cpufreq_per_cpu_attr_write() / store_##file_name() - sysfs write access
373 */
374#define store_one(file_name, object) \
375static ssize_t store_##file_name \
376(struct cpufreq_policy *policy, const char *buf, size_t count) \
377{ \
378 unsigned int ret = -EINVAL; \
379 struct cpufreq_policy new_policy; \
380 \
381 ret = cpufreq_get_policy(&new_policy, policy->cpu); \
382 if (ret) \
383 return -EINVAL; \
384 \
385 ret = sscanf(buf, "%u", &new_policy.object); \
386 if (ret != 1) \
387 return -EINVAL; \
388 \
389 ret = __cpufreq_set_policy(policy, &new_policy); \
390 policy->user_policy.object = policy->object; \
391 \
392 return ret ? ret : count; \
393}
394
395store_one(scaling_min_freq, min);
396store_one(scaling_max_freq, max);
397
398/**
399 * show_cpuinfo_cur_freq - current CPU frequency as detected by hardware
400 */
401static ssize_t show_cpuinfo_cur_freq(struct cpufreq_policy *policy,
402 char *buf)
403{
404 unsigned int cur_freq = __cpufreq_get(policy->cpu);
405 if (!cur_freq)
406 return sprintf(buf, "<unknown>");
407 return sprintf(buf, "%u\n", cur_freq);
408}
409
410
411/**
412 * show_scaling_governor - show the current policy for the specified CPU
413 */
414static ssize_t show_scaling_governor(struct cpufreq_policy *policy, char *buf)
415{
416 if (policy->policy == CPUFREQ_POLICY_POWERSAVE)
417 return sprintf(buf, "powersave\n");
418 else if (policy->policy == CPUFREQ_POLICY_PERFORMANCE)
419 return sprintf(buf, "performance\n");
420 else if (policy->governor)
421 return scnprintf(buf, CPUFREQ_NAME_LEN, "%s\n",
422 policy->governor->name);
423 return -EINVAL;
424}
425
426
427/**
428 * store_scaling_governor - store policy for the specified CPU
429 */
430static ssize_t store_scaling_governor(struct cpufreq_policy *policy,
431 const char *buf, size_t count)
432{
433 unsigned int ret = -EINVAL;
434 char str_governor[16];
435 struct cpufreq_policy new_policy;
436
437 ret = cpufreq_get_policy(&new_policy, policy->cpu);
438 if (ret)
439 return ret;
440
441 ret = sscanf(buf, "%15s", str_governor);
442 if (ret != 1)
443 return -EINVAL;
444
445 if (cpufreq_parse_governor(str_governor, &new_policy.policy,
446 &new_policy.governor))
447 return -EINVAL;
448
449 /* Do not use cpufreq_set_policy here or the user_policy.max
450 will be wrongly overridden */
451 ret = __cpufreq_set_policy(policy, &new_policy);
452
453 policy->user_policy.policy = policy->policy;
454 policy->user_policy.governor = policy->governor;
455
456 if (ret)
457 return ret;
458 else
459 return count;
460}
461
462/**
463 * show_scaling_driver - show the cpufreq driver currently loaded
464 */
465static ssize_t show_scaling_driver(struct cpufreq_policy *policy, char *buf)
466{
467 return scnprintf(buf, CPUFREQ_NAME_LEN, "%s\n", cpufreq_driver->name);
468}
469
470/**
471 * show_scaling_available_governors - show the available CPUfreq governors
472 */
473static ssize_t show_scaling_available_governors(struct cpufreq_policy *policy,
474 char *buf)
475{
476 ssize_t i = 0;
477 struct cpufreq_governor *t;
478
479 if (!cpufreq_driver->target) {
480 i += sprintf(buf, "performance powersave");
481 goto out;
482 }
483
484 list_for_each_entry(t, &cpufreq_governor_list, governor_list) {
485 if (i >= (ssize_t) ((PAGE_SIZE / sizeof(char))
486 - (CPUFREQ_NAME_LEN + 2)))
487 goto out;
488 i += scnprintf(&buf[i], CPUFREQ_NAME_LEN, "%s ", t->name);
489 }
490out:
491 i += sprintf(&buf[i], "\n");
492 return i;
493}
494
495static ssize_t show_cpus(const struct cpumask *mask, char *buf)
496{
497 ssize_t i = 0;
498 unsigned int cpu;
499
500 for_each_cpu(cpu, mask) {
501 if (i)
502 i += scnprintf(&buf[i], (PAGE_SIZE - i - 2), " ");
503 i += scnprintf(&buf[i], (PAGE_SIZE - i - 2), "%u", cpu);
504 if (i >= (PAGE_SIZE - 5))
505 break;
506 }
507 i += sprintf(&buf[i], "\n");
508 return i;
509}
510
511/**
512 * show_related_cpus - show the CPUs affected by each transition even if
513 * hw coordination is in use
514 */
515static ssize_t show_related_cpus(struct cpufreq_policy *policy, char *buf)
516{
517 if (cpumask_empty(policy->related_cpus))
518 return show_cpus(policy->cpus, buf);
519 return show_cpus(policy->related_cpus, buf);
520}
521
522/**
523 * show_affected_cpus - show the CPUs affected by each transition
524 */
525static ssize_t show_affected_cpus(struct cpufreq_policy *policy, char *buf)
526{
527 return show_cpus(policy->cpus, buf);
528}
529
530static ssize_t store_scaling_setspeed(struct cpufreq_policy *policy,
531 const char *buf, size_t count)
532{
533 unsigned int freq = 0;
534 unsigned int ret;
535
536 if (!policy->governor || !policy->governor->store_setspeed)
537 return -EINVAL;
538
539 ret = sscanf(buf, "%u", &freq);
540 if (ret != 1)
541 return -EINVAL;
542
543 policy->governor->store_setspeed(policy, freq);
544
545 return count;
546}
547
548static ssize_t show_scaling_setspeed(struct cpufreq_policy *policy, char *buf)
549{
550 if (!policy->governor || !policy->governor->show_setspeed)
551 return sprintf(buf, "<unsupported>\n");
552
553 return policy->governor->show_setspeed(policy, buf);
554}
555
556/**
557 * show_scaling_driver - show the current cpufreq HW/BIOS limitation
558 */
559static ssize_t show_bios_limit(struct cpufreq_policy *policy, char *buf)
560{
561 unsigned int limit;
562 int ret;
563 if (cpufreq_driver->bios_limit) {
564 ret = cpufreq_driver->bios_limit(policy->cpu, &limit);
565 if (!ret)
566 return sprintf(buf, "%u\n", limit);
567 }
568 return sprintf(buf, "%u\n", policy->cpuinfo.max_freq);
569}
570
571cpufreq_freq_attr_ro_perm(cpuinfo_cur_freq, 0400);
572cpufreq_freq_attr_ro(cpuinfo_min_freq);
573cpufreq_freq_attr_ro(cpuinfo_max_freq);
574cpufreq_freq_attr_ro(cpuinfo_transition_latency);
575cpufreq_freq_attr_ro(scaling_available_governors);
576cpufreq_freq_attr_ro(scaling_driver);
577cpufreq_freq_attr_ro(scaling_cur_freq);
578cpufreq_freq_attr_ro(bios_limit);
579cpufreq_freq_attr_ro(related_cpus);
580cpufreq_freq_attr_ro(affected_cpus);
581cpufreq_freq_attr_rw(scaling_min_freq);
582cpufreq_freq_attr_rw(scaling_max_freq);
583cpufreq_freq_attr_rw(scaling_governor);
584cpufreq_freq_attr_rw(scaling_setspeed);
585
586static struct attribute *default_attrs[] = {
587 &cpuinfo_min_freq.attr,
588 &cpuinfo_max_freq.attr,
589 &cpuinfo_transition_latency.attr,
590 &scaling_min_freq.attr,
591 &scaling_max_freq.attr,
592 &affected_cpus.attr,
593 &related_cpus.attr,
594 &scaling_governor.attr,
595 &scaling_driver.attr,
596 &scaling_available_governors.attr,
597 &scaling_setspeed.attr,
598 NULL
599};
600
601struct kobject *cpufreq_global_kobject;
602EXPORT_SYMBOL(cpufreq_global_kobject);
603
604#define to_policy(k) container_of(k, struct cpufreq_policy, kobj)
605#define to_attr(a) container_of(a, struct freq_attr, attr)
606
607static ssize_t show(struct kobject *kobj, struct attribute *attr, char *buf)
608{
609 struct cpufreq_policy *policy = to_policy(kobj);
610 struct freq_attr *fattr = to_attr(attr);
611 ssize_t ret = -EINVAL;
612 policy = cpufreq_cpu_get(policy->cpu);
613 if (!policy)
614 goto no_policy;
615
616 if (lock_policy_rwsem_read(policy->cpu) < 0)
617 goto fail;
618
619 if (fattr->show)
620 ret = fattr->show(policy, buf);
621 else
622 ret = -EIO;
623
624 unlock_policy_rwsem_read(policy->cpu);
625fail:
626 cpufreq_cpu_put(policy);
627no_policy:
628 return ret;
629}
630
631static ssize_t store(struct kobject *kobj, struct attribute *attr,
632 const char *buf, size_t count)
633{
634 struct cpufreq_policy *policy = to_policy(kobj);
635 struct freq_attr *fattr = to_attr(attr);
636 ssize_t ret = -EINVAL;
637 policy = cpufreq_cpu_get(policy->cpu);
638 if (!policy)
639 goto no_policy;
640
641 if (lock_policy_rwsem_write(policy->cpu) < 0)
642 goto fail;
643
644 if (fattr->store)
645 ret = fattr->store(policy, buf, count);
646 else
647 ret = -EIO;
648
649 unlock_policy_rwsem_write(policy->cpu);
650fail:
651 cpufreq_cpu_put(policy);
652no_policy:
653 return ret;
654}
655
656static void cpufreq_sysfs_release(struct kobject *kobj)
657{
658 struct cpufreq_policy *policy = to_policy(kobj);
659 pr_debug("last reference is dropped\n");
660 complete(&policy->kobj_unregister);
661}
662
663static const struct sysfs_ops sysfs_ops = {
664 .show = show,
665 .store = store,
666};
667
668static struct kobj_type ktype_cpufreq = {
669 .sysfs_ops = &sysfs_ops,
670 .default_attrs = default_attrs,
671 .release = cpufreq_sysfs_release,
672};
673
674/*
675 * Returns:
676 * Negative: Failure
677 * 0: Success
678 * Positive: When we have a managed CPU and the sysfs got symlinked
679 */
680static int cpufreq_add_dev_policy(unsigned int cpu,
681 struct cpufreq_policy *policy,
682 struct sys_device *sys_dev)
683{
684 int ret = 0;
685#ifdef CONFIG_SMP
686 unsigned long flags;
687 unsigned int j;
688#ifdef CONFIG_HOTPLUG_CPU
689 struct cpufreq_governor *gov;
690
691 gov = __find_governor(per_cpu(cpufreq_cpu_governor, cpu));
692 if (gov) {
693 policy->governor = gov;
694 pr_debug("Restoring governor %s for cpu %d\n",
695 policy->governor->name, cpu);
696 }
697#endif
698
699 for_each_cpu(j, policy->cpus) {
700 struct cpufreq_policy *managed_policy;
701
702 if (cpu == j)
703 continue;
704
705 /* Check for existing affected CPUs.
706 * They may not be aware of it due to CPU Hotplug.
707 * cpufreq_cpu_put is called when the device is removed
708 * in __cpufreq_remove_dev()
709 */
710 managed_policy = cpufreq_cpu_get(j);
711 if (unlikely(managed_policy)) {
712
713 /* Set proper policy_cpu */
714 unlock_policy_rwsem_write(cpu);
715 per_cpu(cpufreq_policy_cpu, cpu) = managed_policy->cpu;
716
717 if (lock_policy_rwsem_write(cpu) < 0) {
718 /* Should not go through policy unlock path */
719 if (cpufreq_driver->exit)
720 cpufreq_driver->exit(policy);
721 cpufreq_cpu_put(managed_policy);
722 return -EBUSY;
723 }
724
725 spin_lock_irqsave(&cpufreq_driver_lock, flags);
726 cpumask_copy(managed_policy->cpus, policy->cpus);
727 per_cpu(cpufreq_cpu_data, cpu) = managed_policy;
728 spin_unlock_irqrestore(&cpufreq_driver_lock, flags);
729
730 pr_debug("CPU already managed, adding link\n");
731 ret = sysfs_create_link(&sys_dev->kobj,
732 &managed_policy->kobj,
733 "cpufreq");
734 if (ret)
735 cpufreq_cpu_put(managed_policy);
736 /*
737 * Success. We only needed to be added to the mask.
738 * Call driver->exit() because only the cpu parent of
739 * the kobj needed to call init().
740 */
741 if (cpufreq_driver->exit)
742 cpufreq_driver->exit(policy);
743
744 if (!ret)
745 return 1;
746 else
747 return ret;
748 }
749 }
750#endif
751 return ret;
752}
753
754
755/* symlink affected CPUs */
756static int cpufreq_add_dev_symlink(unsigned int cpu,
757 struct cpufreq_policy *policy)
758{
759 unsigned int j;
760 int ret = 0;
761
762 for_each_cpu(j, policy->cpus) {
763 struct cpufreq_policy *managed_policy;
764 struct sys_device *cpu_sys_dev;
765
766 if (j == cpu)
767 continue;
768 if (!cpu_online(j))
769 continue;
770
771 pr_debug("CPU %u already managed, adding link\n", j);
772 managed_policy = cpufreq_cpu_get(cpu);
773 cpu_sys_dev = get_cpu_sysdev(j);
774 ret = sysfs_create_link(&cpu_sys_dev->kobj, &policy->kobj,
775 "cpufreq");
776 if (ret) {
777 cpufreq_cpu_put(managed_policy);
778 return ret;
779 }
780 }
781 return ret;
782}
783
784static int cpufreq_add_dev_interface(unsigned int cpu,
785 struct cpufreq_policy *policy,
786 struct sys_device *sys_dev)
787{
788 struct cpufreq_policy new_policy;
789 struct freq_attr **drv_attr;
790 unsigned long flags;
791 int ret = 0;
792 unsigned int j;
793
794 /* prepare interface data */
795 ret = kobject_init_and_add(&policy->kobj, &ktype_cpufreq,
796 &sys_dev->kobj, "cpufreq");
797 if (ret)
798 return ret;
799
800 /* set up files for this cpu device */
801 drv_attr = cpufreq_driver->attr;
802 while ((drv_attr) && (*drv_attr)) {
803 ret = sysfs_create_file(&policy->kobj, &((*drv_attr)->attr));
804 if (ret)
805 goto err_out_kobj_put;
806 drv_attr++;
807 }
808 if (cpufreq_driver->get) {
809 ret = sysfs_create_file(&policy->kobj, &cpuinfo_cur_freq.attr);
810 if (ret)
811 goto err_out_kobj_put;
812 }
813 if (cpufreq_driver->target) {
814 ret = sysfs_create_file(&policy->kobj, &scaling_cur_freq.attr);
815 if (ret)
816 goto err_out_kobj_put;
817 }
818 if (cpufreq_driver->bios_limit) {
819 ret = sysfs_create_file(&policy->kobj, &bios_limit.attr);
820 if (ret)
821 goto err_out_kobj_put;
822 }
823
824 spin_lock_irqsave(&cpufreq_driver_lock, flags);
825 for_each_cpu(j, policy->cpus) {
826 if (!cpu_online(j))
827 continue;
828 per_cpu(cpufreq_cpu_data, j) = policy;
829 per_cpu(cpufreq_policy_cpu, j) = policy->cpu;
830 }
831 spin_unlock_irqrestore(&cpufreq_driver_lock, flags);
832
833 ret = cpufreq_add_dev_symlink(cpu, policy);
834 if (ret)
835 goto err_out_kobj_put;
836
837 memcpy(&new_policy, policy, sizeof(struct cpufreq_policy));
838 /* assure that the starting sequence is run in __cpufreq_set_policy */
839 policy->governor = NULL;
840
841 /* set default policy */
842 ret = __cpufreq_set_policy(policy, &new_policy);
843 policy->user_policy.policy = policy->policy;
844 policy->user_policy.governor = policy->governor;
845
846 if (ret) {
847 pr_debug("setting policy failed\n");
848 if (cpufreq_driver->exit)
849 cpufreq_driver->exit(policy);
850 }
851 return ret;
852
853err_out_kobj_put:
854 kobject_put(&policy->kobj);
855 wait_for_completion(&policy->kobj_unregister);
856 return ret;
857}
858
859
860/**
861 * cpufreq_add_dev - add a CPU device
862 *
863 * Adds the cpufreq interface for a CPU device.
864 *
865 * The Oracle says: try running cpufreq registration/unregistration concurrently
866 * with with cpu hotplugging and all hell will break loose. Tried to clean this
867 * mess up, but more thorough testing is needed. - Mathieu
868 */
869static int cpufreq_add_dev(struct sys_device *sys_dev)
870{
871 unsigned int cpu = sys_dev->id;
872 int ret = 0, found = 0;
873 struct cpufreq_policy *policy;
874 unsigned long flags;
875 unsigned int j;
876#ifdef CONFIG_HOTPLUG_CPU
877 int sibling;
878#endif
879
880 if (cpu_is_offline(cpu))
881 return 0;
882
883 pr_debug("adding CPU %u\n", cpu);
884
885#ifdef CONFIG_SMP
886 /* check whether a different CPU already registered this
887 * CPU because it is in the same boat. */
888 policy = cpufreq_cpu_get(cpu);
889 if (unlikely(policy)) {
890 cpufreq_cpu_put(policy);
891 return 0;
892 }
893#endif
894
895 if (!try_module_get(cpufreq_driver->owner)) {
896 ret = -EINVAL;
897 goto module_out;
898 }
899
900 ret = -ENOMEM;
901 policy = kzalloc(sizeof(struct cpufreq_policy), GFP_KERNEL);
902 if (!policy)
903 goto nomem_out;
904
905 if (!alloc_cpumask_var(&policy->cpus, GFP_KERNEL))
906 goto err_free_policy;
907
908 if (!zalloc_cpumask_var(&policy->related_cpus, GFP_KERNEL))
909 goto err_free_cpumask;
910
911 policy->cpu = cpu;
912 cpumask_copy(policy->cpus, cpumask_of(cpu));
913
914 /* Initially set CPU itself as the policy_cpu */
915 per_cpu(cpufreq_policy_cpu, cpu) = cpu;
916 ret = (lock_policy_rwsem_write(cpu) < 0);
917 WARN_ON(ret);
918
919 init_completion(&policy->kobj_unregister);
920 INIT_WORK(&policy->update, handle_update);
921
922 /* Set governor before ->init, so that driver could check it */
923#ifdef CONFIG_HOTPLUG_CPU
924 for_each_online_cpu(sibling) {
925 struct cpufreq_policy *cp = per_cpu(cpufreq_cpu_data, sibling);
926 if (cp && cp->governor &&
927 (cpumask_test_cpu(cpu, cp->related_cpus))) {
928 policy->governor = cp->governor;
929 found = 1;
930 break;
931 }
932 }
933#endif
934 if (!found)
935 policy->governor = CPUFREQ_DEFAULT_GOVERNOR;
936
937 /* call driver. From then on the cpufreq must be able
938 * to accept all calls to ->verify and ->setpolicy for this CPU
939 */
940 ret = cpufreq_driver->init(policy);
941 if (ret) {
942 pr_debug("initialization failed\n");
943 goto err_unlock_policy;
944 }
945 policy->user_policy.min = policy->min;
946 policy->user_policy.max = policy->max;
947
948 blocking_notifier_call_chain(&cpufreq_policy_notifier_list,
949 CPUFREQ_START, policy);
950
951 ret = cpufreq_add_dev_policy(cpu, policy, sys_dev);
952 if (ret) {
953 if (ret > 0)
954 /* This is a managed cpu, symlink created,
955 exit with 0 */
956 ret = 0;
957 goto err_unlock_policy;
958 }
959
960 ret = cpufreq_add_dev_interface(cpu, policy, sys_dev);
961 if (ret)
962 goto err_out_unregister;
963
964 unlock_policy_rwsem_write(cpu);
965
966 kobject_uevent(&policy->kobj, KOBJ_ADD);
967 module_put(cpufreq_driver->owner);
968 pr_debug("initialization complete\n");
969
970 return 0;
971
972
973err_out_unregister:
974 spin_lock_irqsave(&cpufreq_driver_lock, flags);
975 for_each_cpu(j, policy->cpus)
976 per_cpu(cpufreq_cpu_data, j) = NULL;
977 spin_unlock_irqrestore(&cpufreq_driver_lock, flags);
978
979 kobject_put(&policy->kobj);
980 wait_for_completion(&policy->kobj_unregister);
981
982err_unlock_policy:
983 unlock_policy_rwsem_write(cpu);
984 free_cpumask_var(policy->related_cpus);
985err_free_cpumask:
986 free_cpumask_var(policy->cpus);
987err_free_policy:
988 kfree(policy);
989nomem_out:
990 module_put(cpufreq_driver->owner);
991module_out:
992 return ret;
993}
994
995
996/**
997 * __cpufreq_remove_dev - remove a CPU device
998 *
999 * Removes the cpufreq interface for a CPU device.
1000 * Caller should already have policy_rwsem in write mode for this CPU.
1001 * This routine frees the rwsem before returning.
1002 */
1003static int __cpufreq_remove_dev(struct sys_device *sys_dev)
1004{
1005 unsigned int cpu = sys_dev->id;
1006 unsigned long flags;
1007 struct cpufreq_policy *data;
1008 struct kobject *kobj;
1009 struct completion *cmp;
1010#ifdef CONFIG_SMP
1011 struct sys_device *cpu_sys_dev;
1012 unsigned int j;
1013#endif
1014
1015 pr_debug("unregistering CPU %u\n", cpu);
1016
1017 spin_lock_irqsave(&cpufreq_driver_lock, flags);
1018 data = per_cpu(cpufreq_cpu_data, cpu);
1019
1020 if (!data) {
1021 spin_unlock_irqrestore(&cpufreq_driver_lock, flags);
1022 unlock_policy_rwsem_write(cpu);
1023 return -EINVAL;
1024 }
1025 per_cpu(cpufreq_cpu_data, cpu) = NULL;
1026
1027
1028#ifdef CONFIG_SMP
1029 /* if this isn't the CPU which is the parent of the kobj, we
1030 * only need to unlink, put and exit
1031 */
1032 if (unlikely(cpu != data->cpu)) {
1033 pr_debug("removing link\n");
1034 cpumask_clear_cpu(cpu, data->cpus);
1035 spin_unlock_irqrestore(&cpufreq_driver_lock, flags);
1036 kobj = &sys_dev->kobj;
1037 cpufreq_cpu_put(data);
1038 unlock_policy_rwsem_write(cpu);
1039 sysfs_remove_link(kobj, "cpufreq");
1040 return 0;
1041 }
1042#endif
1043
1044#ifdef CONFIG_SMP
1045
1046#ifdef CONFIG_HOTPLUG_CPU
1047 strncpy(per_cpu(cpufreq_cpu_governor, cpu), data->governor->name,
1048 CPUFREQ_NAME_LEN);
1049#endif
1050
1051 /* if we have other CPUs still registered, we need to unlink them,
1052 * or else wait_for_completion below will lock up. Clean the
1053 * per_cpu(cpufreq_cpu_data) while holding the lock, and remove
1054 * the sysfs links afterwards.
1055 */
1056 if (unlikely(cpumask_weight(data->cpus) > 1)) {
1057 for_each_cpu(j, data->cpus) {
1058 if (j == cpu)
1059 continue;
1060 per_cpu(cpufreq_cpu_data, j) = NULL;
1061 }
1062 }
1063
1064 spin_unlock_irqrestore(&cpufreq_driver_lock, flags);
1065
1066 if (unlikely(cpumask_weight(data->cpus) > 1)) {
1067 for_each_cpu(j, data->cpus) {
1068 if (j == cpu)
1069 continue;
1070 pr_debug("removing link for cpu %u\n", j);
1071#ifdef CONFIG_HOTPLUG_CPU
1072 strncpy(per_cpu(cpufreq_cpu_governor, j),
1073 data->governor->name, CPUFREQ_NAME_LEN);
1074#endif
1075 cpu_sys_dev = get_cpu_sysdev(j);
1076 kobj = &cpu_sys_dev->kobj;
1077 unlock_policy_rwsem_write(cpu);
1078 sysfs_remove_link(kobj, "cpufreq");
1079 lock_policy_rwsem_write(cpu);
1080 cpufreq_cpu_put(data);
1081 }
1082 }
1083#else
1084 spin_unlock_irqrestore(&cpufreq_driver_lock, flags);
1085#endif
1086
1087 if (cpufreq_driver->target)
1088 __cpufreq_governor(data, CPUFREQ_GOV_STOP);
1089
1090 kobj = &data->kobj;
1091 cmp = &data->kobj_unregister;
1092 unlock_policy_rwsem_write(cpu);
1093 kobject_put(kobj);
1094
1095 /* we need to make sure that the underlying kobj is actually
1096 * not referenced anymore by anybody before we proceed with
1097 * unloading.
1098 */
1099 pr_debug("waiting for dropping of refcount\n");
1100 wait_for_completion(cmp);
1101 pr_debug("wait complete\n");
1102
1103 lock_policy_rwsem_write(cpu);
1104 if (cpufreq_driver->exit)
1105 cpufreq_driver->exit(data);
1106 unlock_policy_rwsem_write(cpu);
1107
1108#ifdef CONFIG_HOTPLUG_CPU
1109 /* when the CPU which is the parent of the kobj is hotplugged
1110 * offline, check for siblings, and create cpufreq sysfs interface
1111 * and symlinks
1112 */
1113 if (unlikely(cpumask_weight(data->cpus) > 1)) {
1114 /* first sibling now owns the new sysfs dir */
1115 cpumask_clear_cpu(cpu, data->cpus);
1116 cpufreq_add_dev(get_cpu_sysdev(cpumask_first(data->cpus)));
1117
1118 /* finally remove our own symlink */
1119 lock_policy_rwsem_write(cpu);
1120 __cpufreq_remove_dev(sys_dev);
1121 }
1122#endif
1123
1124 free_cpumask_var(data->related_cpus);
1125 free_cpumask_var(data->cpus);
1126 kfree(data);
1127
1128 return 0;
1129}
1130
1131
1132static int cpufreq_remove_dev(struct sys_device *sys_dev)
1133{
1134 unsigned int cpu = sys_dev->id;
1135 int retval;
1136
1137 if (cpu_is_offline(cpu))
1138 return 0;
1139
1140 if (unlikely(lock_policy_rwsem_write(cpu)))
1141 BUG();
1142
1143 retval = __cpufreq_remove_dev(sys_dev);
1144 return retval;
1145}
1146
1147
1148static void handle_update(struct work_struct *work)
1149{
1150 struct cpufreq_policy *policy =
1151 container_of(work, struct cpufreq_policy, update);
1152 unsigned int cpu = policy->cpu;
1153 pr_debug("handle_update for cpu %u called\n", cpu);
1154 cpufreq_update_policy(cpu);
1155}
1156
1157/**
1158 * cpufreq_out_of_sync - If actual and saved CPU frequency differs, we're in deep trouble.
1159 * @cpu: cpu number
1160 * @old_freq: CPU frequency the kernel thinks the CPU runs at
1161 * @new_freq: CPU frequency the CPU actually runs at
1162 *
1163 * We adjust to current frequency first, and need to clean up later.
1164 * So either call to cpufreq_update_policy() or schedule handle_update()).
1165 */
1166static void cpufreq_out_of_sync(unsigned int cpu, unsigned int old_freq,
1167 unsigned int new_freq)
1168{
1169 struct cpufreq_freqs freqs;
1170
1171 pr_debug("Warning: CPU frequency out of sync: cpufreq and timing "
1172 "core thinks of %u, is %u kHz.\n", old_freq, new_freq);
1173
1174 freqs.cpu = cpu;
1175 freqs.old = old_freq;
1176 freqs.new = new_freq;
1177 cpufreq_notify_transition(&freqs, CPUFREQ_PRECHANGE);
1178 cpufreq_notify_transition(&freqs, CPUFREQ_POSTCHANGE);
1179}
1180
1181
1182/**
1183 * cpufreq_quick_get - get the CPU frequency (in kHz) from policy->cur
1184 * @cpu: CPU number
1185 *
1186 * This is the last known freq, without actually getting it from the driver.
1187 * Return value will be same as what is shown in scaling_cur_freq in sysfs.
1188 */
1189unsigned int cpufreq_quick_get(unsigned int cpu)
1190{
1191 struct cpufreq_policy *policy = cpufreq_cpu_get(cpu);
1192 unsigned int ret_freq = 0;
1193
1194 if (policy) {
1195 ret_freq = policy->cur;
1196 cpufreq_cpu_put(policy);
1197 }
1198
1199 return ret_freq;
1200}
1201EXPORT_SYMBOL(cpufreq_quick_get);
1202
1203
1204static unsigned int __cpufreq_get(unsigned int cpu)
1205{
1206 struct cpufreq_policy *policy = per_cpu(cpufreq_cpu_data, cpu);
1207 unsigned int ret_freq = 0;
1208
1209 if (!cpufreq_driver->get)
1210 return ret_freq;
1211
1212 ret_freq = cpufreq_driver->get(cpu);
1213
1214 if (ret_freq && policy->cur &&
1215 !(cpufreq_driver->flags & CPUFREQ_CONST_LOOPS)) {
1216 /* verify no discrepancy between actual and
1217 saved value exists */
1218 if (unlikely(ret_freq != policy->cur)) {
1219 cpufreq_out_of_sync(cpu, policy->cur, ret_freq);
1220 schedule_work(&policy->update);
1221 }
1222 }
1223
1224 return ret_freq;
1225}
1226
1227/**
1228 * cpufreq_get - get the current CPU frequency (in kHz)
1229 * @cpu: CPU number
1230 *
1231 * Get the CPU current (static) CPU frequency
1232 */
1233unsigned int cpufreq_get(unsigned int cpu)
1234{
1235 unsigned int ret_freq = 0;
1236 struct cpufreq_policy *policy = cpufreq_cpu_get(cpu);
1237
1238 if (!policy)
1239 goto out;
1240
1241 if (unlikely(lock_policy_rwsem_read(cpu)))
1242 goto out_policy;
1243
1244 ret_freq = __cpufreq_get(cpu);
1245
1246 unlock_policy_rwsem_read(cpu);
1247
1248out_policy:
1249 cpufreq_cpu_put(policy);
1250out:
1251 return ret_freq;
1252}
1253EXPORT_SYMBOL(cpufreq_get);
1254
1255static struct sysdev_driver cpufreq_sysdev_driver = {
1256 .add = cpufreq_add_dev,
1257 .remove = cpufreq_remove_dev,
1258};
1259
1260
1261/**
1262 * cpufreq_bp_suspend - Prepare the boot CPU for system suspend.
1263 *
1264 * This function is only executed for the boot processor. The other CPUs
1265 * have been put offline by means of CPU hotplug.
1266 */
1267static int cpufreq_bp_suspend(void)
1268{
1269 int ret = 0;
1270
1271 int cpu = smp_processor_id();
1272 struct cpufreq_policy *cpu_policy;
1273
1274 pr_debug("suspending cpu %u\n", cpu);
1275
1276 /* If there's no policy for the boot CPU, we have nothing to do. */
1277 cpu_policy = cpufreq_cpu_get(cpu);
1278 if (!cpu_policy)
1279 return 0;
1280
1281 if (cpufreq_driver->suspend) {
1282 ret = cpufreq_driver->suspend(cpu_policy);
1283 if (ret)
1284 printk(KERN_ERR "cpufreq: suspend failed in ->suspend "
1285 "step on CPU %u\n", cpu_policy->cpu);
1286 }
1287
1288 cpufreq_cpu_put(cpu_policy);
1289 return ret;
1290}
1291
1292/**
1293 * cpufreq_bp_resume - Restore proper frequency handling of the boot CPU.
1294 *
1295 * 1.) resume CPUfreq hardware support (cpufreq_driver->resume())
1296 * 2.) schedule call cpufreq_update_policy() ASAP as interrupts are
1297 * restored. It will verify that the current freq is in sync with
1298 * what we believe it to be. This is a bit later than when it
1299 * should be, but nonethteless it's better than calling
1300 * cpufreq_driver->get() here which might re-enable interrupts...
1301 *
1302 * This function is only executed for the boot CPU. The other CPUs have not
1303 * been turned on yet.
1304 */
1305static void cpufreq_bp_resume(void)
1306{
1307 int ret = 0;
1308
1309 int cpu = smp_processor_id();
1310 struct cpufreq_policy *cpu_policy;
1311
1312 pr_debug("resuming cpu %u\n", cpu);
1313
1314 /* If there's no policy for the boot CPU, we have nothing to do. */
1315 cpu_policy = cpufreq_cpu_get(cpu);
1316 if (!cpu_policy)
1317 return;
1318
1319 if (cpufreq_driver->resume) {
1320 ret = cpufreq_driver->resume(cpu_policy);
1321 if (ret) {
1322 printk(KERN_ERR "cpufreq: resume failed in ->resume "
1323 "step on CPU %u\n", cpu_policy->cpu);
1324 goto fail;
1325 }
1326 }
1327
1328 schedule_work(&cpu_policy->update);
1329
1330fail:
1331 cpufreq_cpu_put(cpu_policy);
1332}
1333
1334static struct syscore_ops cpufreq_syscore_ops = {
1335 .suspend = cpufreq_bp_suspend,
1336 .resume = cpufreq_bp_resume,
1337};
1338
1339
1340/*********************************************************************
1341 * NOTIFIER LISTS INTERFACE *
1342 *********************************************************************/
1343
1344/**
1345 * cpufreq_register_notifier - register a driver with cpufreq
1346 * @nb: notifier function to register
1347 * @list: CPUFREQ_TRANSITION_NOTIFIER or CPUFREQ_POLICY_NOTIFIER
1348 *
1349 * Add a driver to one of two lists: either a list of drivers that
1350 * are notified about clock rate changes (once before and once after
1351 * the transition), or a list of drivers that are notified about
1352 * changes in cpufreq policy.
1353 *
1354 * This function may sleep, and has the same return conditions as
1355 * blocking_notifier_chain_register.
1356 */
1357int cpufreq_register_notifier(struct notifier_block *nb, unsigned int list)
1358{
1359 int ret;
1360
1361 WARN_ON(!init_cpufreq_transition_notifier_list_called);
1362
1363 switch (list) {
1364 case CPUFREQ_TRANSITION_NOTIFIER:
1365 ret = srcu_notifier_chain_register(
1366 &cpufreq_transition_notifier_list, nb);
1367 break;
1368 case CPUFREQ_POLICY_NOTIFIER:
1369 ret = blocking_notifier_chain_register(
1370 &cpufreq_policy_notifier_list, nb);
1371 break;
1372 default:
1373 ret = -EINVAL;
1374 }
1375
1376 return ret;
1377}
1378EXPORT_SYMBOL(cpufreq_register_notifier);
1379
1380
1381/**
1382 * cpufreq_unregister_notifier - unregister a driver with cpufreq
1383 * @nb: notifier block to be unregistered
1384 * @list: CPUFREQ_TRANSITION_NOTIFIER or CPUFREQ_POLICY_NOTIFIER
1385 *
1386 * Remove a driver from the CPU frequency notifier list.
1387 *
1388 * This function may sleep, and has the same return conditions as
1389 * blocking_notifier_chain_unregister.
1390 */
1391int cpufreq_unregister_notifier(struct notifier_block *nb, unsigned int list)
1392{
1393 int ret;
1394
1395 switch (list) {
1396 case CPUFREQ_TRANSITION_NOTIFIER:
1397 ret = srcu_notifier_chain_unregister(
1398 &cpufreq_transition_notifier_list, nb);
1399 break;
1400 case CPUFREQ_POLICY_NOTIFIER:
1401 ret = blocking_notifier_chain_unregister(
1402 &cpufreq_policy_notifier_list, nb);
1403 break;
1404 default:
1405 ret = -EINVAL;
1406 }
1407
1408 return ret;
1409}
1410EXPORT_SYMBOL(cpufreq_unregister_notifier);
1411
1412
1413/*********************************************************************
1414 * GOVERNORS *
1415 *********************************************************************/
1416
1417
1418int __cpufreq_driver_target(struct cpufreq_policy *policy,
1419 unsigned int target_freq,
1420 unsigned int relation)
1421{
1422 int retval = -EINVAL;
1423
1424 pr_debug("target for CPU %u: %u kHz, relation %u\n", policy->cpu,
1425 target_freq, relation);
1426 if (cpu_online(policy->cpu) && cpufreq_driver->target)
1427 retval = cpufreq_driver->target(policy, target_freq, relation);
1428
1429 return retval;
1430}
1431EXPORT_SYMBOL_GPL(__cpufreq_driver_target);
1432
1433int cpufreq_driver_target(struct cpufreq_policy *policy,
1434 unsigned int target_freq,
1435 unsigned int relation)
1436{
1437 int ret = -EINVAL;
1438
1439 policy = cpufreq_cpu_get(policy->cpu);
1440 if (!policy)
1441 goto no_policy;
1442
1443 if (unlikely(lock_policy_rwsem_write(policy->cpu)))
1444 goto fail;
1445
1446 ret = __cpufreq_driver_target(policy, target_freq, relation);
1447
1448 unlock_policy_rwsem_write(policy->cpu);
1449
1450fail:
1451 cpufreq_cpu_put(policy);
1452no_policy:
1453 return ret;
1454}
1455EXPORT_SYMBOL_GPL(cpufreq_driver_target);
1456
1457int __cpufreq_driver_getavg(struct cpufreq_policy *policy, unsigned int cpu)
1458{
1459 int ret = 0;
1460
1461 policy = cpufreq_cpu_get(policy->cpu);
1462 if (!policy)
1463 return -EINVAL;
1464
1465 if (cpu_online(cpu) && cpufreq_driver->getavg)
1466 ret = cpufreq_driver->getavg(policy, cpu);
1467
1468 cpufreq_cpu_put(policy);
1469 return ret;
1470}
1471EXPORT_SYMBOL_GPL(__cpufreq_driver_getavg);
1472
1473/*
1474 * when "event" is CPUFREQ_GOV_LIMITS
1475 */
1476
1477static int __cpufreq_governor(struct cpufreq_policy *policy,
1478 unsigned int event)
1479{
1480 int ret;
1481
1482 /* Only must be defined when default governor is known to have latency
1483 restrictions, like e.g. conservative or ondemand.
1484 That this is the case is already ensured in Kconfig
1485 */
1486#ifdef CONFIG_CPU_FREQ_GOV_PERFORMANCE
1487 struct cpufreq_governor *gov = &cpufreq_gov_performance;
1488#else
1489 struct cpufreq_governor *gov = NULL;
1490#endif
1491
1492 if (policy->governor->max_transition_latency &&
1493 policy->cpuinfo.transition_latency >
1494 policy->governor->max_transition_latency) {
1495 if (!gov)
1496 return -EINVAL;
1497 else {
1498 printk(KERN_WARNING "%s governor failed, too long"
1499 " transition latency of HW, fallback"
1500 " to %s governor\n",
1501 policy->governor->name,
1502 gov->name);
1503 policy->governor = gov;
1504 }
1505 }
1506
1507 if (!try_module_get(policy->governor->owner))
1508 return -EINVAL;
1509
1510 pr_debug("__cpufreq_governor for CPU %u, event %u\n",
1511 policy->cpu, event);
1512 ret = policy->governor->governor(policy, event);
1513
1514 /* we keep one module reference alive for
1515 each CPU governed by this CPU */
1516 if ((event != CPUFREQ_GOV_START) || ret)
1517 module_put(policy->governor->owner);
1518 if ((event == CPUFREQ_GOV_STOP) && !ret)
1519 module_put(policy->governor->owner);
1520
1521 return ret;
1522}
1523
1524
1525int cpufreq_register_governor(struct cpufreq_governor *governor)
1526{
1527 int err;
1528
1529 if (!governor)
1530 return -EINVAL;
1531
1532 mutex_lock(&cpufreq_governor_mutex);
1533
1534 err = -EBUSY;
1535 if (__find_governor(governor->name) == NULL) {
1536 err = 0;
1537 list_add(&governor->governor_list, &cpufreq_governor_list);
1538 }
1539
1540 mutex_unlock(&cpufreq_governor_mutex);
1541 return err;
1542}
1543EXPORT_SYMBOL_GPL(cpufreq_register_governor);
1544
1545
1546void cpufreq_unregister_governor(struct cpufreq_governor *governor)
1547{
1548#ifdef CONFIG_HOTPLUG_CPU
1549 int cpu;
1550#endif
1551
1552 if (!governor)
1553 return;
1554
1555#ifdef CONFIG_HOTPLUG_CPU
1556 for_each_present_cpu(cpu) {
1557 if (cpu_online(cpu))
1558 continue;
1559 if (!strcmp(per_cpu(cpufreq_cpu_governor, cpu), governor->name))
1560 strcpy(per_cpu(cpufreq_cpu_governor, cpu), "\0");
1561 }
1562#endif
1563
1564 mutex_lock(&cpufreq_governor_mutex);
1565 list_del(&governor->governor_list);
1566 mutex_unlock(&cpufreq_governor_mutex);
1567 return;
1568}
1569EXPORT_SYMBOL_GPL(cpufreq_unregister_governor);
1570
1571
1572
1573/*********************************************************************
1574 * POLICY INTERFACE *
1575 *********************************************************************/
1576
1577/**
1578 * cpufreq_get_policy - get the current cpufreq_policy
1579 * @policy: struct cpufreq_policy into which the current cpufreq_policy
1580 * is written
1581 *
1582 * Reads the current cpufreq policy.
1583 */
1584int cpufreq_get_policy(struct cpufreq_policy *policy, unsigned int cpu)
1585{
1586 struct cpufreq_policy *cpu_policy;
1587 if (!policy)
1588 return -EINVAL;
1589
1590 cpu_policy = cpufreq_cpu_get(cpu);
1591 if (!cpu_policy)
1592 return -EINVAL;
1593
1594 memcpy(policy, cpu_policy, sizeof(struct cpufreq_policy));
1595
1596 cpufreq_cpu_put(cpu_policy);
1597 return 0;
1598}
1599EXPORT_SYMBOL(cpufreq_get_policy);
1600
1601
1602/*
1603 * data : current policy.
1604 * policy : policy to be set.
1605 */
1606static int __cpufreq_set_policy(struct cpufreq_policy *data,
1607 struct cpufreq_policy *policy)
1608{
1609 int ret = 0;
1610
1611 pr_debug("setting new policy for CPU %u: %u - %u kHz\n", policy->cpu,
1612 policy->min, policy->max);
1613
1614 memcpy(&policy->cpuinfo, &data->cpuinfo,
1615 sizeof(struct cpufreq_cpuinfo));
1616
1617 if (policy->min > data->max || policy->max < data->min) {
1618 ret = -EINVAL;
1619 goto error_out;
1620 }
1621
1622 /* verify the cpu speed can be set within this limit */
1623 ret = cpufreq_driver->verify(policy);
1624 if (ret)
1625 goto error_out;
1626
1627 /* adjust if necessary - all reasons */
1628 blocking_notifier_call_chain(&cpufreq_policy_notifier_list,
1629 CPUFREQ_ADJUST, policy);
1630
1631 /* adjust if necessary - hardware incompatibility*/
1632 blocking_notifier_call_chain(&cpufreq_policy_notifier_list,
1633 CPUFREQ_INCOMPATIBLE, policy);
1634
1635 /* verify the cpu speed can be set within this limit,
1636 which might be different to the first one */
1637 ret = cpufreq_driver->verify(policy);
1638 if (ret)
1639 goto error_out;
1640
1641 /* notification of the new policy */
1642 blocking_notifier_call_chain(&cpufreq_policy_notifier_list,
1643 CPUFREQ_NOTIFY, policy);
1644
1645 data->min = policy->min;
1646 data->max = policy->max;
1647
1648 pr_debug("new min and max freqs are %u - %u kHz\n",
1649 data->min, data->max);
1650
1651 if (cpufreq_driver->setpolicy) {
1652 data->policy = policy->policy;
1653 pr_debug("setting range\n");
1654 ret = cpufreq_driver->setpolicy(policy);
1655 } else {
1656 if (policy->governor != data->governor) {
1657 /* save old, working values */
1658 struct cpufreq_governor *old_gov = data->governor;
1659
1660 pr_debug("governor switch\n");
1661
1662 /* end old governor */
1663 if (data->governor)
1664 __cpufreq_governor(data, CPUFREQ_GOV_STOP);
1665
1666 /* start new governor */
1667 data->governor = policy->governor;
1668 if (__cpufreq_governor(data, CPUFREQ_GOV_START)) {
1669 /* new governor failed, so re-start old one */
1670 pr_debug("starting governor %s failed\n",
1671 data->governor->name);
1672 if (old_gov) {
1673 data->governor = old_gov;
1674 __cpufreq_governor(data,
1675 CPUFREQ_GOV_START);
1676 }
1677 ret = -EINVAL;
1678 goto error_out;
1679 }
1680 /* might be a policy change, too, so fall through */
1681 }
1682 pr_debug("governor: change or update limits\n");
1683 __cpufreq_governor(data, CPUFREQ_GOV_LIMITS);
1684 }
1685
1686error_out:
1687 return ret;
1688}
1689
1690/**
1691 * cpufreq_update_policy - re-evaluate an existing cpufreq policy
1692 * @cpu: CPU which shall be re-evaluated
1693 *
1694 * Useful for policy notifiers which have different necessities
1695 * at different times.
1696 */
1697int cpufreq_update_policy(unsigned int cpu)
1698{
1699 struct cpufreq_policy *data = cpufreq_cpu_get(cpu);
1700 struct cpufreq_policy policy;
1701 int ret;
1702
1703 if (!data) {
1704 ret = -ENODEV;
1705 goto no_policy;
1706 }
1707
1708 if (unlikely(lock_policy_rwsem_write(cpu))) {
1709 ret = -EINVAL;
1710 goto fail;
1711 }
1712
1713 pr_debug("updating policy for CPU %u\n", cpu);
1714 memcpy(&policy, data, sizeof(struct cpufreq_policy));
1715 policy.min = data->user_policy.min;
1716 policy.max = data->user_policy.max;
1717 policy.policy = data->user_policy.policy;
1718 policy.governor = data->user_policy.governor;
1719
1720 /* BIOS might change freq behind our back
1721 -> ask driver for current freq and notify governors about a change */
1722 if (cpufreq_driver->get) {
1723 policy.cur = cpufreq_driver->get(cpu);
1724 if (!data->cur) {
1725 pr_debug("Driver did not initialize current freq");
1726 data->cur = policy.cur;
1727 } else {
1728 if (data->cur != policy.cur)
1729 cpufreq_out_of_sync(cpu, data->cur,
1730 policy.cur);
1731 }
1732 }
1733
1734 ret = __cpufreq_set_policy(data, &policy);
1735
1736 unlock_policy_rwsem_write(cpu);
1737
1738fail:
1739 cpufreq_cpu_put(data);
1740no_policy:
1741 return ret;
1742}
1743EXPORT_SYMBOL(cpufreq_update_policy);
1744
1745static int __cpuinit cpufreq_cpu_callback(struct notifier_block *nfb,
1746 unsigned long action, void *hcpu)
1747{
1748 unsigned int cpu = (unsigned long)hcpu;
1749 struct sys_device *sys_dev;
1750
1751 sys_dev = get_cpu_sysdev(cpu);
1752 if (sys_dev) {
1753 switch (action) {
1754 case CPU_ONLINE:
1755 case CPU_ONLINE_FROZEN:
1756 cpufreq_add_dev(sys_dev);
1757 break;
1758 case CPU_DOWN_PREPARE:
1759 case CPU_DOWN_PREPARE_FROZEN:
1760 if (unlikely(lock_policy_rwsem_write(cpu)))
1761 BUG();
1762
1763 __cpufreq_remove_dev(sys_dev);
1764 break;
1765 case CPU_DOWN_FAILED:
1766 case CPU_DOWN_FAILED_FROZEN:
1767 cpufreq_add_dev(sys_dev);
1768 break;
1769 }
1770 }
1771 return NOTIFY_OK;
1772}
1773
1774static struct notifier_block __refdata cpufreq_cpu_notifier = {
1775 .notifier_call = cpufreq_cpu_callback,
1776};
1777
1778/*********************************************************************
1779 * REGISTER / UNREGISTER CPUFREQ DRIVER *
1780 *********************************************************************/
1781
1782/**
1783 * cpufreq_register_driver - register a CPU Frequency driver
1784 * @driver_data: A struct cpufreq_driver containing the values#
1785 * submitted by the CPU Frequency driver.
1786 *
1787 * Registers a CPU Frequency driver to this core code. This code
1788 * returns zero on success, -EBUSY when another driver got here first
1789 * (and isn't unregistered in the meantime).
1790 *
1791 */
1792int cpufreq_register_driver(struct cpufreq_driver *driver_data)
1793{
1794 unsigned long flags;
1795 int ret;
1796
1797 if (!driver_data || !driver_data->verify || !driver_data->init ||
1798 ((!driver_data->setpolicy) && (!driver_data->target)))
1799 return -EINVAL;
1800
1801 pr_debug("trying to register driver %s\n", driver_data->name);
1802
1803 if (driver_data->setpolicy)
1804 driver_data->flags |= CPUFREQ_CONST_LOOPS;
1805
1806 spin_lock_irqsave(&cpufreq_driver_lock, flags);
1807 if (cpufreq_driver) {
1808 spin_unlock_irqrestore(&cpufreq_driver_lock, flags);
1809 return -EBUSY;
1810 }
1811 cpufreq_driver = driver_data;
1812 spin_unlock_irqrestore(&cpufreq_driver_lock, flags);
1813
1814 ret = sysdev_driver_register(&cpu_sysdev_class,
1815 &cpufreq_sysdev_driver);
1816 if (ret)
1817 goto err_null_driver;
1818
1819 if (!(cpufreq_driver->flags & CPUFREQ_STICKY)) {
1820 int i;
1821 ret = -ENODEV;
1822
1823 /* check for at least one working CPU */
1824 for (i = 0; i < nr_cpu_ids; i++)
1825 if (cpu_possible(i) && per_cpu(cpufreq_cpu_data, i)) {
1826 ret = 0;
1827 break;
1828 }
1829
1830 /* if all ->init() calls failed, unregister */
1831 if (ret) {
1832 pr_debug("no CPU initialized for driver %s\n",
1833 driver_data->name);
1834 goto err_sysdev_unreg;
1835 }
1836 }
1837
1838 register_hotcpu_notifier(&cpufreq_cpu_notifier);
1839 pr_debug("driver %s up and running\n", driver_data->name);
1840
1841 return 0;
1842err_sysdev_unreg:
1843 sysdev_driver_unregister(&cpu_sysdev_class,
1844 &cpufreq_sysdev_driver);
1845err_null_driver:
1846 spin_lock_irqsave(&cpufreq_driver_lock, flags);
1847 cpufreq_driver = NULL;
1848 spin_unlock_irqrestore(&cpufreq_driver_lock, flags);
1849 return ret;
1850}
1851EXPORT_SYMBOL_GPL(cpufreq_register_driver);
1852
1853
1854/**
1855 * cpufreq_unregister_driver - unregister the current CPUFreq driver
1856 *
1857 * Unregister the current CPUFreq driver. Only call this if you have
1858 * the right to do so, i.e. if you have succeeded in initialising before!
1859 * Returns zero if successful, and -EINVAL if the cpufreq_driver is
1860 * currently not initialised.
1861 */
1862int cpufreq_unregister_driver(struct cpufreq_driver *driver)
1863{
1864 unsigned long flags;
1865
1866 if (!cpufreq_driver || (driver != cpufreq_driver))
1867 return -EINVAL;
1868
1869 pr_debug("unregistering driver %s\n", driver->name);
1870
1871 sysdev_driver_unregister(&cpu_sysdev_class, &cpufreq_sysdev_driver);
1872 unregister_hotcpu_notifier(&cpufreq_cpu_notifier);
1873
1874 spin_lock_irqsave(&cpufreq_driver_lock, flags);
1875 cpufreq_driver = NULL;
1876 spin_unlock_irqrestore(&cpufreq_driver_lock, flags);
1877
1878 return 0;
1879}
1880EXPORT_SYMBOL_GPL(cpufreq_unregister_driver);
1881
1882static int __init cpufreq_core_init(void)
1883{
1884 int cpu;
1885
1886 for_each_possible_cpu(cpu) {
1887 per_cpu(cpufreq_policy_cpu, cpu) = -1;
1888 init_rwsem(&per_cpu(cpu_policy_rwsem, cpu));
1889 }
1890
1891 cpufreq_global_kobject = kobject_create_and_add("cpufreq",
1892 &cpu_sysdev_class.kset.kobj);
1893 BUG_ON(!cpufreq_global_kobject);
1894 register_syscore_ops(&cpufreq_syscore_ops);
1895
1896 return 0;
1897}
1898core_initcall(cpufreq_core_init);