/arch/x86/kernel/pci-dma.c

  1#include <linux/dma-mapping.h>
  2#include <linux/dma-debug.h>
  3#include <linux/dmar.h>
  4#include <linux/bootmem.h>
  5#include <linux/gfp.h>
  6#include <linux/pci.h>
  7#include <linux/kmemleak.h>
  8
  9#include <asm/proto.h>
 10#include <asm/dma.h>
 11#include <asm/iommu.h>
 12#include <asm/gart.h>
 13#include <asm/calgary.h>
 14#include <asm/amd_iommu.h>
 15#include <asm/x86_init.h>
 16
 17static int forbid_dac __read_mostly;
 18
 19struct dma_map_ops *dma_ops = &nommu_dma_ops;
 20EXPORT_SYMBOL(dma_ops);
 21
 22static int iommu_sac_force __read_mostly;
 23
 24#ifdef CONFIG_IOMMU_DEBUG
 25int panic_on_overflow __read_mostly = 1;
 26int force_iommu __read_mostly = 1;
 27#else
 28int panic_on_overflow __read_mostly = 0;
 29int force_iommu __read_mostly = 0;
 30#endif
 31
 32int iommu_merge __read_mostly = 0;
 33
 34int no_iommu __read_mostly;
 35/* Set this to 1 if there is a HW IOMMU in the system */
 36int iommu_detected __read_mostly = 0;
 37
 38/*
 39 * This variable becomes 1 if iommu=pt is passed on the kernel command line.
 40 * If this variable is 1, IOMMU implementations do no DMA translation for
 41 * devices and allow every device to access to whole physical memory. This is
 42 * useful if a user wants to use an IOMMU only for KVM device assignment to
 43 * guests and not for driver dma translation.
 44 */
 45int iommu_pass_through __read_mostly;
 46
 47/* Dummy device used for NULL arguments (normally ISA). */
 48struct device x86_dma_fallback_dev = {
 49	.init_name = "fallback device",
 50	.coherent_dma_mask = ISA_DMA_BIT_MASK,
 51	.dma_mask = &x86_dma_fallback_dev.coherent_dma_mask,
 52};
 53EXPORT_SYMBOL(x86_dma_fallback_dev);
 54
 55/* Number of entries preallocated for DMA-API debugging */
 56#define PREALLOC_DMA_DEBUG_ENTRIES       32768
 57
 58int dma_set_mask(struct device *dev, u64 mask)
 59{
 60	if (!dev->dma_mask || !dma_supported(dev, mask))
 61		return -EIO;
 62
 63	*dev->dma_mask = mask;
 64
 65	return 0;
 66}
 67EXPORT_SYMBOL(dma_set_mask);
 68
 69#if defined(CONFIG_X86_64) && !defined(CONFIG_NUMA)
 70static __initdata void *dma32_bootmem_ptr;
 71static unsigned long dma32_bootmem_size __initdata = (128ULL<<20);
 72
 73static int __init parse_dma32_size_opt(char *p)
 74{
 75	if (!p)
 76		return -EINVAL;
 77	dma32_bootmem_size = memparse(p, &p);
 78	return 0;
 79}
 80early_param("dma32_size", parse_dma32_size_opt);
 81
 82void __init dma32_reserve_bootmem(void)
 83{
 84	unsigned long size, align;
 85	if (max_pfn <= MAX_DMA32_PFN)
 86		return;
 87
 88	/*
 89	 * check aperture_64.c allocate_aperture() for reason about
 90	 * using 512M as goal
 91	 */
 92	align = 64ULL<<20;
 93	size = roundup(dma32_bootmem_size, align);
 94	dma32_bootmem_ptr = __alloc_bootmem_nopanic(size, align,
 95				 512ULL<<20);
 96	/*
 97	 * Kmemleak should not scan this block as it may not be mapped via the
 98	 * kernel direct mapping.
 99	 */
100	kmemleak_ignore(dma32_bootmem_ptr);
101	if (dma32_bootmem_ptr)
102		dma32_bootmem_size = size;
103	else
104		dma32_bootmem_size = 0;
105}
106static void __init dma32_free_bootmem(void)
107{
108
109	if (max_pfn <= MAX_DMA32_PFN)
110		return;
111
112	if (!dma32_bootmem_ptr)
113		return;
114
115	free_bootmem(__pa(dma32_bootmem_ptr), dma32_bootmem_size);
116
117	dma32_bootmem_ptr = NULL;
118	dma32_bootmem_size = 0;
119}
120#else
121void __init dma32_reserve_bootmem(void)
122{
123}
124static void __init dma32_free_bootmem(void)
125{
126}
127
128#endif
129
130void __init pci_iommu_alloc(void)
131{
132	/* free the range so iommu could get some range less than 4G */
133	dma32_free_bootmem();
134
135	if (pci_swiotlb_detect())
136		goto out;
137
138	gart_iommu_hole_init();
139
140	detect_calgary();
141
142	detect_intel_iommu();
143
144	/* needs to be called after gart_iommu_hole_init */
145	amd_iommu_detect();
146out:
147	pci_swiotlb_init();
148}
149
150void *dma_generic_alloc_coherent(struct device *dev, size_t size,
151				 dma_addr_t *dma_addr, gfp_t flag)
152{
153	unsigned long dma_mask;
154	struct page *page;
155	dma_addr_t addr;
156
157	dma_mask = dma_alloc_coherent_mask(dev, flag);
158
159	flag |= __GFP_ZERO;
160again:
161	page = alloc_pages_node(dev_to_node(dev), flag, get_order(size));
162	if (!page)
163		return NULL;
164
165	addr = page_to_phys(page);
166	if (addr + size > dma_mask) {
167		__free_pages(page, get_order(size));
168
169		if (dma_mask < DMA_BIT_MASK(32) && !(flag & GFP_DMA)) {
170			flag = (flag & ~GFP_DMA32) | GFP_DMA;
171			goto again;
172		}
173
174		return NULL;
175	}
176
177	*dma_addr = addr;
178	return page_address(page);
179}
180
181/*
182 * See <Documentation/x86_64/boot-options.txt> for the iommu kernel parameter
183 * documentation.
184 */
185static __init int iommu_setup(char *p)
186{
187	iommu_merge = 1;
188
189	if (!p)
190		return -EINVAL;
191
192	while (*p) {
193		if (!strncmp(p, "off", 3))
194			no_iommu = 1;
195		/* gart_parse_options has more force support */
196		if (!strncmp(p, "force", 5))
197			force_iommu = 1;
198		if (!strncmp(p, "noforce", 7)) {
199			iommu_merge = 0;
200			force_iommu = 0;
201		}
202
203		if (!strncmp(p, "biomerge", 8)) {
204			iommu_merge = 1;
205			force_iommu = 1;
206		}
207		if (!strncmp(p, "panic", 5))
208			panic_on_overflow = 1;
209		if (!strncmp(p, "nopanic", 7))
210			panic_on_overflow = 0;
211		if (!strncmp(p, "merge", 5)) {
212			iommu_merge = 1;
213			force_iommu = 1;
214		}
215		if (!strncmp(p, "nomerge", 7))
216			iommu_merge = 0;
217		if (!strncmp(p, "forcesac", 8))
218			iommu_sac_force = 1;
219		if (!strncmp(p, "allowdac", 8))
220			forbid_dac = 0;
221		if (!strncmp(p, "nodac", 5))
222			forbid_dac = 1;
223		if (!strncmp(p, "usedac", 6)) {
224			forbid_dac = -1;
225			return 1;
226		}
227#ifdef CONFIG_SWIOTLB
228		if (!strncmp(p, "soft", 4))
229			swiotlb = 1;
230#endif
231		if (!strncmp(p, "pt", 2))
232			iommu_pass_through = 1;
233
234		gart_parse_options(p);
235
236#ifdef CONFIG_CALGARY_IOMMU
237		if (!strncmp(p, "calgary", 7))
238			use_calgary = 1;
239#endif /* CONFIG_CALGARY_IOMMU */
240
241		p += strcspn(p, ",");
242		if (*p == ',')
243			++p;
244	}
245	return 0;
246}
247early_param("iommu", iommu_setup);
248
249int dma_supported(struct device *dev, u64 mask)
250{
251	struct dma_map_ops *ops = get_dma_ops(dev);
252
253#ifdef CONFIG_PCI
254	if (mask > 0xffffffff && forbid_dac > 0) {
255		dev_info(dev, "PCI: Disallowing DAC for device\n");
256		return 0;
257	}
258#endif
259
260	if (ops->dma_supported)
261		return ops->dma_supported(dev, mask);
262
263	/* Copied from i386. Doesn't make much sense, because it will
264	   only work for pci_alloc_coherent.
265	   The caller just has to use GFP_DMA in this case. */
266	if (mask < DMA_BIT_MASK(24))
267		return 0;
268
269	/* Tell the device to use SAC when IOMMU force is on.  This
270	   allows the driver to use cheaper accesses in some cases.
271
272	   Problem with this is that if we overflow the IOMMU area and
273	   return DAC as fallback address the device may not handle it
274	   correctly.
275
276	   As a special case some controllers have a 39bit address
277	   mode that is as efficient as 32bit (aic79xx). Don't force
278	   SAC for these.  Assume all masks <= 40 bits are of this
279	   type. Normally this doesn't make any difference, but gives
280	   more gentle handling of IOMMU overflow. */
281	if (iommu_sac_force && (mask >= DMA_BIT_MASK(40))) {
282		dev_info(dev, "Force SAC with mask %Lx\n", mask);
283		return 0;
284	}
285
286	return 1;
287}
288EXPORT_SYMBOL(dma_supported);
289
290static int __init pci_iommu_init(void)
291{
292	dma_debug_init(PREALLOC_DMA_DEBUG_ENTRIES);
293
294#ifdef CONFIG_PCI
295	dma_debug_add_bus(&pci_bus_type);
296#endif
297	x86_init.iommu.iommu_init();
298
299	if (swiotlb) {
300		printk(KERN_INFO "PCI-DMA: "
301		       "Using software bounce buffering for IO (SWIOTLB)\n");
302		swiotlb_print_info();
303	} else
304		swiotlb_free();
305
306	return 0;
307}
308/* Must execute after PCI subsystem */
309rootfs_initcall(pci_iommu_init);
310
311#ifdef CONFIG_PCI
312/* Many VIA bridges seem to corrupt data for DAC. Disable it here */
313
314static __devinit void via_no_dac(struct pci_dev *dev)
315{
316	if ((dev->class >> 8) == PCI_CLASS_BRIDGE_PCI && forbid_dac == 0) {
317		dev_info(&dev->dev, "disabling DAC on VIA PCI bridge\n");
318		forbid_dac = 1;
319	}
320}
321DECLARE_PCI_FIXUP_FINAL(PCI_VENDOR_ID_VIA, PCI_ANY_ID, via_no_dac);
322#endif