PageRenderTime 76ms CodeModel.GetById 40ms RepoModel.GetById 0ms app.codeStats 0ms

/arch/s390/numa/mode_emu.c

https://github.com/tytso/ext4
C | 577 lines | 368 code | 67 blank | 142 comment | 55 complexity | 5df4b015bf892527302d167ce54fc60c MD5 | raw file
Possible License(s): GPL-2.0
  1. // SPDX-License-Identifier: GPL-2.0
  2. /*
  3. * NUMA support for s390
  4. *
  5. * NUMA emulation (aka fake NUMA) distributes the available memory to nodes
  6. * without using real topology information about the physical memory of the
  7. * machine.
  8. *
  9. * It distributes the available CPUs to nodes while respecting the original
  10. * machine topology information. This is done by trying to avoid to separate
  11. * CPUs which reside on the same book or even on the same MC.
  12. *
  13. * Because the current Linux scheduler code requires a stable cpu to node
  14. * mapping, cores are pinned to nodes when the first CPU thread is set online.
  15. *
  16. * Copyright IBM Corp. 2015
  17. */
  18. #define KMSG_COMPONENT "numa_emu"
  19. #define pr_fmt(fmt) KMSG_COMPONENT ": " fmt
  20. #include <linux/kernel.h>
  21. #include <linux/cpumask.h>
  22. #include <linux/memblock.h>
  23. #include <linux/node.h>
  24. #include <linux/memory.h>
  25. #include <linux/slab.h>
  26. #include <asm/smp.h>
  27. #include <asm/topology.h>
  28. #include "numa_mode.h"
  29. #include "toptree.h"
  30. /* Distances between the different system components */
  31. #define DIST_EMPTY 0
  32. #define DIST_CORE 1
  33. #define DIST_MC 2
  34. #define DIST_BOOK 3
  35. #define DIST_DRAWER 4
  36. #define DIST_MAX 5
  37. /* Node distance reported to common code */
  38. #define EMU_NODE_DIST 10
  39. /* Node ID for free (not yet pinned) cores */
  40. #define NODE_ID_FREE -1
  41. /* Different levels of toptree */
  42. enum toptree_level {CORE, MC, BOOK, DRAWER, NODE, TOPOLOGY};
  43. /* The two toptree IDs */
  44. enum {TOPTREE_ID_PHYS, TOPTREE_ID_NUMA};
  45. /* Number of NUMA nodes */
  46. static int emu_nodes = 1;
  47. /* NUMA stripe size */
  48. static unsigned long emu_size;
  49. /*
  50. * Node to core pinning information updates are protected by
  51. * "sched_domains_mutex".
  52. */
  53. static struct {
  54. s32 to_node_id[CONFIG_NR_CPUS]; /* Pinned core to node mapping */
  55. int total; /* Total number of pinned cores */
  56. int per_node_target; /* Cores per node without extra cores */
  57. int per_node[MAX_NUMNODES]; /* Number of cores pinned to node */
  58. } *emu_cores;
  59. /*
  60. * Pin a core to a node
  61. */
  62. static void pin_core_to_node(int core_id, int node_id)
  63. {
  64. if (emu_cores->to_node_id[core_id] == NODE_ID_FREE) {
  65. emu_cores->per_node[node_id]++;
  66. emu_cores->to_node_id[core_id] = node_id;
  67. emu_cores->total++;
  68. } else {
  69. WARN_ON(emu_cores->to_node_id[core_id] != node_id);
  70. }
  71. }
  72. /*
  73. * Number of pinned cores of a node
  74. */
  75. static int cores_pinned(struct toptree *node)
  76. {
  77. return emu_cores->per_node[node->id];
  78. }
  79. /*
  80. * ID of the node where the core is pinned (or NODE_ID_FREE)
  81. */
  82. static int core_pinned_to_node_id(struct toptree *core)
  83. {
  84. return emu_cores->to_node_id[core->id];
  85. }
  86. /*
  87. * Number of cores in the tree that are not yet pinned
  88. */
  89. static int cores_free(struct toptree *tree)
  90. {
  91. struct toptree *core;
  92. int count = 0;
  93. toptree_for_each(core, tree, CORE) {
  94. if (core_pinned_to_node_id(core) == NODE_ID_FREE)
  95. count++;
  96. }
  97. return count;
  98. }
  99. /*
  100. * Return node of core
  101. */
  102. static struct toptree *core_node(struct toptree *core)
  103. {
  104. return core->parent->parent->parent->parent;
  105. }
  106. /*
  107. * Return drawer of core
  108. */
  109. static struct toptree *core_drawer(struct toptree *core)
  110. {
  111. return core->parent->parent->parent;
  112. }
  113. /*
  114. * Return book of core
  115. */
  116. static struct toptree *core_book(struct toptree *core)
  117. {
  118. return core->parent->parent;
  119. }
  120. /*
  121. * Return mc of core
  122. */
  123. static struct toptree *core_mc(struct toptree *core)
  124. {
  125. return core->parent;
  126. }
  127. /*
  128. * Distance between two cores
  129. */
  130. static int dist_core_to_core(struct toptree *core1, struct toptree *core2)
  131. {
  132. if (core_drawer(core1)->id != core_drawer(core2)->id)
  133. return DIST_DRAWER;
  134. if (core_book(core1)->id != core_book(core2)->id)
  135. return DIST_BOOK;
  136. if (core_mc(core1)->id != core_mc(core2)->id)
  137. return DIST_MC;
  138. /* Same core or sibling on same MC */
  139. return DIST_CORE;
  140. }
  141. /*
  142. * Distance of a node to a core
  143. */
  144. static int dist_node_to_core(struct toptree *node, struct toptree *core)
  145. {
  146. struct toptree *core_node;
  147. int dist_min = DIST_MAX;
  148. toptree_for_each(core_node, node, CORE)
  149. dist_min = min(dist_min, dist_core_to_core(core_node, core));
  150. return dist_min == DIST_MAX ? DIST_EMPTY : dist_min;
  151. }
  152. /*
  153. * Unify will delete empty nodes, therefore recreate nodes.
  154. */
  155. static void toptree_unify_tree(struct toptree *tree)
  156. {
  157. int nid;
  158. toptree_unify(tree);
  159. for (nid = 0; nid < emu_nodes; nid++)
  160. toptree_get_child(tree, nid);
  161. }
  162. /*
  163. * Find the best/nearest node for a given core and ensure that no node
  164. * gets more than "emu_cores->per_node_target + extra" cores.
  165. */
  166. static struct toptree *node_for_core(struct toptree *numa, struct toptree *core,
  167. int extra)
  168. {
  169. struct toptree *node, *node_best = NULL;
  170. int dist_cur, dist_best, cores_target;
  171. cores_target = emu_cores->per_node_target + extra;
  172. dist_best = DIST_MAX;
  173. node_best = NULL;
  174. toptree_for_each(node, numa, NODE) {
  175. /* Already pinned cores must use their nodes */
  176. if (core_pinned_to_node_id(core) == node->id) {
  177. node_best = node;
  178. break;
  179. }
  180. /* Skip nodes that already have enough cores */
  181. if (cores_pinned(node) >= cores_target)
  182. continue;
  183. dist_cur = dist_node_to_core(node, core);
  184. if (dist_cur < dist_best) {
  185. dist_best = dist_cur;
  186. node_best = node;
  187. }
  188. }
  189. return node_best;
  190. }
  191. /*
  192. * Find the best node for each core with respect to "extra" core count
  193. */
  194. static void toptree_to_numa_single(struct toptree *numa, struct toptree *phys,
  195. int extra)
  196. {
  197. struct toptree *node, *core, *tmp;
  198. toptree_for_each_safe(core, tmp, phys, CORE) {
  199. node = node_for_core(numa, core, extra);
  200. if (!node)
  201. return;
  202. toptree_move(core, node);
  203. pin_core_to_node(core->id, node->id);
  204. }
  205. }
  206. /*
  207. * Move structures of given level to specified NUMA node
  208. */
  209. static void move_level_to_numa_node(struct toptree *node, struct toptree *phys,
  210. enum toptree_level level, bool perfect)
  211. {
  212. int cores_free, cores_target = emu_cores->per_node_target;
  213. struct toptree *cur, *tmp;
  214. toptree_for_each_safe(cur, tmp, phys, level) {
  215. cores_free = cores_target - toptree_count(node, CORE);
  216. if (perfect) {
  217. if (cores_free == toptree_count(cur, CORE))
  218. toptree_move(cur, node);
  219. } else {
  220. if (cores_free >= toptree_count(cur, CORE))
  221. toptree_move(cur, node);
  222. }
  223. }
  224. }
  225. /*
  226. * Move structures of a given level to NUMA nodes. If "perfect" is specified
  227. * move only perfectly fitting structures. Otherwise move also smaller
  228. * than needed structures.
  229. */
  230. static void move_level_to_numa(struct toptree *numa, struct toptree *phys,
  231. enum toptree_level level, bool perfect)
  232. {
  233. struct toptree *node;
  234. toptree_for_each(node, numa, NODE)
  235. move_level_to_numa_node(node, phys, level, perfect);
  236. }
  237. /*
  238. * For the first run try to move the big structures
  239. */
  240. static void toptree_to_numa_first(struct toptree *numa, struct toptree *phys)
  241. {
  242. struct toptree *core;
  243. /* Always try to move perfectly fitting structures first */
  244. move_level_to_numa(numa, phys, DRAWER, true);
  245. move_level_to_numa(numa, phys, DRAWER, false);
  246. move_level_to_numa(numa, phys, BOOK, true);
  247. move_level_to_numa(numa, phys, BOOK, false);
  248. move_level_to_numa(numa, phys, MC, true);
  249. move_level_to_numa(numa, phys, MC, false);
  250. /* Now pin all the moved cores */
  251. toptree_for_each(core, numa, CORE)
  252. pin_core_to_node(core->id, core_node(core)->id);
  253. }
  254. /*
  255. * Allocate new topology and create required nodes
  256. */
  257. static struct toptree *toptree_new(int id, int nodes)
  258. {
  259. struct toptree *tree;
  260. int nid;
  261. tree = toptree_alloc(TOPOLOGY, id);
  262. if (!tree)
  263. goto fail;
  264. for (nid = 0; nid < nodes; nid++) {
  265. if (!toptree_get_child(tree, nid))
  266. goto fail;
  267. }
  268. return tree;
  269. fail:
  270. panic("NUMA emulation could not allocate topology");
  271. }
  272. /*
  273. * Allocate and initialize core to node mapping
  274. */
  275. static void __ref create_core_to_node_map(void)
  276. {
  277. int i;
  278. emu_cores = memblock_alloc(sizeof(*emu_cores), 8);
  279. if (!emu_cores)
  280. panic("%s: Failed to allocate %zu bytes align=0x%x\n",
  281. __func__, sizeof(*emu_cores), 8);
  282. for (i = 0; i < ARRAY_SIZE(emu_cores->to_node_id); i++)
  283. emu_cores->to_node_id[i] = NODE_ID_FREE;
  284. }
  285. /*
  286. * Move cores from physical topology into NUMA target topology
  287. * and try to keep as much of the physical topology as possible.
  288. */
  289. static struct toptree *toptree_to_numa(struct toptree *phys)
  290. {
  291. static int first = 1;
  292. struct toptree *numa;
  293. int cores_total;
  294. cores_total = emu_cores->total + cores_free(phys);
  295. emu_cores->per_node_target = cores_total / emu_nodes;
  296. numa = toptree_new(TOPTREE_ID_NUMA, emu_nodes);
  297. if (first) {
  298. toptree_to_numa_first(numa, phys);
  299. first = 0;
  300. }
  301. toptree_to_numa_single(numa, phys, 0);
  302. toptree_to_numa_single(numa, phys, 1);
  303. toptree_unify_tree(numa);
  304. WARN_ON(cpumask_weight(&phys->mask));
  305. return numa;
  306. }
  307. /*
  308. * Create a toptree out of the physical topology that we got from the hypervisor
  309. */
  310. static struct toptree *toptree_from_topology(void)
  311. {
  312. struct toptree *phys, *node, *drawer, *book, *mc, *core;
  313. struct cpu_topology_s390 *top;
  314. int cpu;
  315. phys = toptree_new(TOPTREE_ID_PHYS, 1);
  316. for_each_cpu(cpu, &cpus_with_topology) {
  317. top = &cpu_topology[cpu];
  318. node = toptree_get_child(phys, 0);
  319. drawer = toptree_get_child(node, top->drawer_id);
  320. book = toptree_get_child(drawer, top->book_id);
  321. mc = toptree_get_child(book, top->socket_id);
  322. core = toptree_get_child(mc, smp_get_base_cpu(cpu));
  323. if (!drawer || !book || !mc || !core)
  324. panic("NUMA emulation could not allocate memory");
  325. cpumask_set_cpu(cpu, &core->mask);
  326. toptree_update_mask(mc);
  327. }
  328. return phys;
  329. }
  330. /*
  331. * Add toptree core to topology and create correct CPU masks
  332. */
  333. static void topology_add_core(struct toptree *core)
  334. {
  335. struct cpu_topology_s390 *top;
  336. int cpu;
  337. for_each_cpu(cpu, &core->mask) {
  338. top = &cpu_topology[cpu];
  339. cpumask_copy(&top->thread_mask, &core->mask);
  340. cpumask_copy(&top->core_mask, &core_mc(core)->mask);
  341. cpumask_copy(&top->book_mask, &core_book(core)->mask);
  342. cpumask_copy(&top->drawer_mask, &core_drawer(core)->mask);
  343. cpumask_set_cpu(cpu, &node_to_cpumask_map[core_node(core)->id]);
  344. top->node_id = core_node(core)->id;
  345. }
  346. }
  347. /*
  348. * Apply toptree to topology and create CPU masks
  349. */
  350. static void toptree_to_topology(struct toptree *numa)
  351. {
  352. struct toptree *core;
  353. int i;
  354. /* Clear all node masks */
  355. for (i = 0; i < MAX_NUMNODES; i++)
  356. cpumask_clear(&node_to_cpumask_map[i]);
  357. /* Rebuild all masks */
  358. toptree_for_each(core, numa, CORE)
  359. topology_add_core(core);
  360. }
  361. /*
  362. * Show the node to core mapping
  363. */
  364. static void print_node_to_core_map(void)
  365. {
  366. int nid, cid;
  367. if (!numa_debug_enabled)
  368. return;
  369. printk(KERN_DEBUG "NUMA node to core mapping\n");
  370. for (nid = 0; nid < emu_nodes; nid++) {
  371. printk(KERN_DEBUG " node %3d: ", nid);
  372. for (cid = 0; cid < ARRAY_SIZE(emu_cores->to_node_id); cid++) {
  373. if (emu_cores->to_node_id[cid] == nid)
  374. printk(KERN_CONT "%d ", cid);
  375. }
  376. printk(KERN_CONT "\n");
  377. }
  378. }
  379. static void pin_all_possible_cpus(void)
  380. {
  381. int core_id, node_id, cpu;
  382. static int initialized;
  383. if (initialized)
  384. return;
  385. print_node_to_core_map();
  386. node_id = 0;
  387. for_each_possible_cpu(cpu) {
  388. core_id = smp_get_base_cpu(cpu);
  389. if (emu_cores->to_node_id[core_id] != NODE_ID_FREE)
  390. continue;
  391. pin_core_to_node(core_id, node_id);
  392. cpu_topology[cpu].node_id = node_id;
  393. node_id = (node_id + 1) % emu_nodes;
  394. }
  395. print_node_to_core_map();
  396. initialized = 1;
  397. }
  398. /*
  399. * Transfer physical topology into a NUMA topology and modify CPU masks
  400. * according to the NUMA topology.
  401. *
  402. * Must be called with "sched_domains_mutex" lock held.
  403. */
  404. static void emu_update_cpu_topology(void)
  405. {
  406. struct toptree *phys, *numa;
  407. if (emu_cores == NULL)
  408. create_core_to_node_map();
  409. phys = toptree_from_topology();
  410. numa = toptree_to_numa(phys);
  411. toptree_free(phys);
  412. toptree_to_topology(numa);
  413. toptree_free(numa);
  414. pin_all_possible_cpus();
  415. }
  416. /*
  417. * If emu_size is not set, use CONFIG_EMU_SIZE. Then round to minimum
  418. * alignment (needed for memory hotplug).
  419. */
  420. static unsigned long emu_setup_size_adjust(unsigned long size)
  421. {
  422. unsigned long size_new;
  423. size = size ? : CONFIG_EMU_SIZE;
  424. size_new = roundup(size, memory_block_size_bytes());
  425. if (size_new == size)
  426. return size;
  427. pr_warn("Increasing memory stripe size from %ld MB to %ld MB\n",
  428. size >> 20, size_new >> 20);
  429. return size_new;
  430. }
  431. /*
  432. * If we have not enough memory for the specified nodes, reduce the node count.
  433. */
  434. static int emu_setup_nodes_adjust(int nodes)
  435. {
  436. int nodes_max;
  437. nodes_max = memblock.memory.total_size / emu_size;
  438. nodes_max = max(nodes_max, 1);
  439. if (nodes_max >= nodes)
  440. return nodes;
  441. pr_warn("Not enough memory for %d nodes, reducing node count\n", nodes);
  442. return nodes_max;
  443. }
  444. /*
  445. * Early emu setup
  446. */
  447. static void emu_setup(void)
  448. {
  449. int nid;
  450. emu_size = emu_setup_size_adjust(emu_size);
  451. emu_nodes = emu_setup_nodes_adjust(emu_nodes);
  452. for (nid = 0; nid < emu_nodes; nid++)
  453. node_set(nid, node_possible_map);
  454. pr_info("Creating %d nodes with memory stripe size %ld MB\n",
  455. emu_nodes, emu_size >> 20);
  456. }
  457. /*
  458. * Return node id for given page number
  459. */
  460. static int emu_pfn_to_nid(unsigned long pfn)
  461. {
  462. return (pfn / (emu_size >> PAGE_SHIFT)) % emu_nodes;
  463. }
  464. /*
  465. * Return stripe size
  466. */
  467. static unsigned long emu_align(void)
  468. {
  469. return emu_size;
  470. }
  471. /*
  472. * Return distance between two nodes
  473. */
  474. static int emu_distance(int node1, int node2)
  475. {
  476. return (node1 != node2) * EMU_NODE_DIST;
  477. }
  478. /*
  479. * Define callbacks for generic s390 NUMA infrastructure
  480. */
  481. const struct numa_mode numa_mode_emu = {
  482. .name = "emu",
  483. .setup = emu_setup,
  484. .update_cpu_topology = emu_update_cpu_topology,
  485. .__pfn_to_nid = emu_pfn_to_nid,
  486. .align = emu_align,
  487. .distance = emu_distance,
  488. };
  489. /*
  490. * Kernel parameter: emu_nodes=<n>
  491. */
  492. static int __init early_parse_emu_nodes(char *p)
  493. {
  494. int count;
  495. if (!p || kstrtoint(p, 0, &count) != 0 || count <= 0)
  496. return 0;
  497. emu_nodes = min(count, MAX_NUMNODES);
  498. return 0;
  499. }
  500. early_param("emu_nodes", early_parse_emu_nodes);
  501. /*
  502. * Kernel parameter: emu_size=[<n>[k|M|G|T]]
  503. */
  504. static int __init early_parse_emu_size(char *p)
  505. {
  506. if (p)
  507. emu_size = memparse(p, NULL);
  508. return 0;
  509. }
  510. early_param("emu_size", early_parse_emu_size);