patch-2.4.13 linux/arch/alpha/kernel/pci_iommu.c
Next file: linux/arch/alpha/kernel/srm_env.c
Previous file: linux/arch/alpha/kernel/pci_impl.h
Back to the patch index
Back to the overall index
- Lines: 541
- Date:
Fri Oct 12 15:35:53 2001
- Orig file:
v2.4.12/linux/arch/alpha/kernel/pci_iommu.c
- Orig date:
Sun Sep 23 11:40:55 2001
diff -u --recursive --new-file v2.4.12/linux/arch/alpha/kernel/pci_iommu.c linux/arch/alpha/kernel/pci_iommu.c
@@ -17,17 +17,18 @@
#define DEBUG_ALLOC 0
#if DEBUG_ALLOC > 0
-# define DBGA(args...) printk(KERN_DEBUG ##args)
+# define DBGA(args...) printk(KERN_DEBUG args)
#else
# define DBGA(args...)
#endif
#if DEBUG_ALLOC > 1
-# define DBGA2(args...) printk(KERN_DEBUG ##args)
+# define DBGA2(args...) printk(KERN_DEBUG args)
#else
# define DBGA2(args...)
#endif
#define DEBUG_NODIRECT 0
+#define DEBUG_FORCEDAC 0
static inline unsigned long
@@ -43,6 +44,18 @@
}
+/* Return the minimum of MAX or the first power of two larger
+ than main memory. */
+
+unsigned long
+size_for_memory(unsigned long max)
+{
+ unsigned long mem = max_low_pfn << PAGE_SHIFT;
+ if (mem < max)
+ max = 1UL << ceil_log2(mem);
+ return max;
+}
+
struct pci_iommu_arena *
iommu_arena_new(struct pci_controller *hose, dma_addr_t base,
unsigned long window_size, unsigned long align)
@@ -163,8 +176,9 @@
Once the device is given the dma address, the device owns this memory
until either pci_unmap_single or pci_dma_sync_single is performed. */
-dma_addr_t
-pci_map_single(struct pci_dev *pdev, void *cpu_addr, long size, int direction)
+static dma_addr_t
+pci_map_single_1(struct pci_dev *pdev, void *cpu_addr, size_t size,
+ int dac_allowed)
{
struct pci_controller *hose = pdev ? pdev->sysdata : pci_isa_hose;
dma_addr_t max_dma = pdev ? pdev->dma_mask : 0x00ffffff;
@@ -173,10 +187,7 @@
unsigned long paddr;
dma_addr_t ret;
- if (direction == PCI_DMA_NONE)
- BUG();
-
- paddr = virt_to_phys(cpu_addr);
+ paddr = __pa(cpu_addr);
#if !DEBUG_NODIRECT
/* First check to see if we can use the direct map window. */
@@ -184,13 +195,23 @@
&& paddr + size <= __direct_map_size) {
ret = paddr + __direct_map_base;
- DBGA2("pci_map_single: [%p,%lx] -> direct %x from %p\n",
+ DBGA2("pci_map_single: [%p,%lx] -> direct %lx from %p\n",
cpu_addr, size, ret, __builtin_return_address(0));
return ret;
}
#endif
+ /* Next, use DAC if selected earlier. */
+ if (dac_allowed) {
+ ret = paddr + alpha_mv.pci_dac_offset;
+
+ DBGA2("pci_map_single: [%p,%lx] -> DAC %lx from %p\n",
+ cpu_addr, size, ret, __builtin_return_address(0));
+
+ return ret;
+ }
+
/* If the machine doesn't define a pci_tbi routine, we have to
assume it doesn't support sg mapping. */
if (! alpha_mv.mv_pci_tbi) {
@@ -217,12 +238,30 @@
ret = arena->dma_base + dma_ofs * PAGE_SIZE;
ret += (unsigned long)cpu_addr & ~PAGE_MASK;
- DBGA("pci_map_single: [%p,%lx] np %ld -> sg %x from %p\n",
- cpu_addr, size, npages, ret, __builtin_return_address(0));
+ DBGA2("pci_map_single: [%p,%lx] np %ld -> sg %lx from %p\n",
+ cpu_addr, size, npages, ret, __builtin_return_address(0));
return ret;
}
+dma_addr_t
+pci_map_single(struct pci_dev *pdev, void *cpu_addr, size_t size, int dir)
+{
+ if (dir == PCI_DMA_NONE)
+ BUG();
+ return pci_map_single_1(pdev, cpu_addr, size,
+ (pdev->dma_mask >> 32) != 0);
+}
+
+dma_addr_t
+pci_map_page(struct pci_dev *pdev, struct page *page, unsigned long offset,
+ size_t size, int dir)
+{
+ if (dir == PCI_DMA_NONE)
+ BUG();
+ return pci_map_single_1(pdev, (char *)page_address(page) + offset,
+ size, (pdev->dma_mask >> 32) != 0);
+}
/* Unmap a single streaming mode DMA translation. The DMA_ADDR and
SIZE must match what was provided for in a previous pci_map_single
@@ -231,7 +270,7 @@
wrote there. */
void
-pci_unmap_single(struct pci_dev *pdev, dma_addr_t dma_addr, long size,
+pci_unmap_single(struct pci_dev *pdev, dma_addr_t dma_addr, size_t size,
int direction)
{
unsigned long flags;
@@ -242,17 +281,21 @@
if (direction == PCI_DMA_NONE)
BUG();
-#if !DEBUG_NODIRECT
if (dma_addr >= __direct_map_base
&& dma_addr < __direct_map_base + __direct_map_size) {
/* Nothing to do. */
- DBGA2("pci_unmap_single: direct [%x,%lx] from %p\n",
+ DBGA2("pci_unmap_single: direct [%lx,%lx] from %p\n",
dma_addr, size, __builtin_return_address(0));
return;
}
-#endif
+
+ if (dma_addr > 0xffffffff) {
+ DBGA2("pci64_unmap_single: DAC [%lx,%lx] from %p\n",
+ dma_addr, size, __builtin_return_address(0));
+ return;
+ }
arena = hose->sg_pci;
if (!arena || dma_addr < arena->dma_base)
@@ -260,7 +303,7 @@
dma_ofs = (dma_addr - arena->dma_base) >> PAGE_SHIFT;
if (dma_ofs * PAGE_SIZE >= arena->size) {
- printk(KERN_ERR "Bogus pci_unmap_single: dma_addr %x "
+ printk(KERN_ERR "Bogus pci_unmap_single: dma_addr %lx "
" base %x size %x\n", dma_addr, arena->dma_base,
arena->size);
return;
@@ -273,21 +316,24 @@
iommu_arena_free(arena, dma_ofs, npages);
-
- /*
- If we're freeing ptes above the `next_entry' pointer (they
+ /* If we're freeing ptes above the `next_entry' pointer (they
may have snuck back into the TLB since the last wrap flush),
- we need to flush the TLB before reallocating the latter.
- */
+ we need to flush the TLB before reallocating the latter. */
if (dma_ofs >= arena->next_entry)
alpha_mv.mv_pci_tbi(hose, dma_addr, dma_addr + size - 1);
spin_unlock_irqrestore(&arena->lock, flags);
- DBGA("pci_unmap_single: sg [%x,%lx] np %ld from %p\n",
- dma_addr, size, npages, __builtin_return_address(0));
+ DBGA2("pci_unmap_single: sg [%lx,%lx] np %ld from %p\n",
+ dma_addr, size, npages, __builtin_return_address(0));
}
+void
+pci_unmap_page(struct pci_dev *pdev, dma_addr_t dma_addr,
+ size_t size, int direction)
+{
+ pci_unmap_single(pdev, dma_addr, size, direction);
+}
/* Allocate and map kernel buffer using consistent mode DMA for PCI
device. Returns non-NULL cpu-view pointer to the buffer if
@@ -295,7 +341,7 @@
else DMA_ADDRP is undefined. */
void *
-pci_alloc_consistent(struct pci_dev *pdev, long size, dma_addr_t *dma_addrp)
+pci_alloc_consistent(struct pci_dev *pdev, size_t size, dma_addr_t *dma_addrp)
{
void *cpu_addr;
long order = get_order(size);
@@ -311,8 +357,7 @@
}
memset(cpu_addr, 0, size);
- *dma_addrp = pci_map_single(pdev, cpu_addr, size,
- PCI_DMA_BIDIRECTIONAL);
+ *dma_addrp = pci_map_single_1(pdev, cpu_addr, size, 0);
if (*dma_addrp == 0) {
free_pages((unsigned long)cpu_addr, order);
return NULL;
@@ -324,7 +369,6 @@
return cpu_addr;
}
-
/* Free and unmap a consistent DMA buffer. CPU_ADDR and DMA_ADDR must
be values that were returned from pci_alloc_consistent. SIZE must
be the same as what as passed into pci_alloc_consistent.
@@ -332,7 +376,7 @@
DMA_ADDR past this call are illegal. */
void
-pci_free_consistent(struct pci_dev *pdev, long size, void *cpu_addr,
+pci_free_consistent(struct pci_dev *pdev, size_t size, void *cpu_addr,
dma_addr_t dma_addr)
{
pci_unmap_single(pdev, dma_addr, size, PCI_DMA_BIDIRECTIONAL);
@@ -352,27 +396,35 @@
Write dma_length of each leader with the combined lengths of
the mergable followers. */
+#define SG_ENT_VIRT_ADDRESS(SG) \
+ ((SG)->address \
+ ? (SG)->address \
+ : page_address((SG)->page) + (SG)->offset)
+
+#define SG_ENT_PHYS_ADDRESS(SG) \
+ __pa(SG_ENT_VIRT_ADDRESS(SG))
+
static void
sg_classify(struct scatterlist *sg, struct scatterlist *end, int virt_ok)
{
- unsigned long next_vaddr;
+ unsigned long next_paddr;
struct scatterlist *leader;
long leader_flag, leader_length;
leader = sg;
leader_flag = 0;
leader_length = leader->length;
- next_vaddr = (unsigned long)leader->address + leader_length;
+ next_paddr = SG_ENT_PHYS_ADDRESS(leader) + leader_length;
for (++sg; sg < end; ++sg) {
unsigned long addr, len;
- addr = (unsigned long) sg->address;
+ addr = SG_ENT_PHYS_ADDRESS(sg);
len = sg->length;
- if (next_vaddr == addr) {
+ if (next_paddr == addr) {
sg->dma_address = -1;
leader_length += len;
- } else if (((next_vaddr | addr) & ~PAGE_MASK) == 0 && virt_ok) {
+ } else if (((next_paddr | addr) & ~PAGE_MASK) == 0 && virt_ok) {
sg->dma_address = -2;
leader_flag = 1;
leader_length += len;
@@ -384,7 +436,7 @@
leader_length = len;
}
- next_vaddr = addr + len;
+ next_paddr = addr + len;
}
leader->dma_address = leader_flag;
@@ -397,9 +449,9 @@
static inline int
sg_fill(struct scatterlist *leader, struct scatterlist *end,
struct scatterlist *out, struct pci_iommu_arena *arena,
- dma_addr_t max_dma)
+ dma_addr_t max_dma, int dac_allowed)
{
- unsigned long paddr = virt_to_phys(leader->address);
+ unsigned long paddr = SG_ENT_PHYS_ADDRESS(leader);
long size = leader->dma_length;
struct scatterlist *sg;
unsigned long *ptes;
@@ -414,13 +466,24 @@
out->dma_address = paddr + __direct_map_base;
out->dma_length = size;
- DBGA(" sg_fill: [%p,%lx] -> direct %x\n",
- leader->address, size, out->dma_address);
+ DBGA(" sg_fill: [%p,%lx] -> direct %lx\n",
+ __va(paddr), size, out->dma_address);
return 0;
}
#endif
+ /* If physically contiguous and DAC is available, use it. */
+ if (leader->dma_address == 0 && dac_allowed) {
+ out->dma_address = paddr + alpha_mv.pci_dac_offset;
+ out->dma_length = size;
+
+ DBGA(" sg_fill: [%p,%lx] -> DAC %lx\n",
+ __va(paddr), size, out->dma_address);
+
+ return 0;
+ }
+
/* Otherwise, we'll use the iommu to make the pages virtually
contiguous. */
@@ -433,17 +496,16 @@
return -1;
/* Otherwise, break up the remaining virtually contiguous
- hunks into individual direct maps. */
+ hunks into individual direct maps and retry. */
sg_classify(leader, end, 0);
- /* Retry. */
- return sg_fill(leader, end, out, arena, max_dma);
+ return sg_fill(leader, end, out, arena, max_dma, dac_allowed);
}
out->dma_address = arena->dma_base + dma_ofs*PAGE_SIZE + paddr;
out->dma_length = size;
- DBGA(" sg_fill: [%p,%lx] -> sg %x np %ld\n",
- leader->address, size, out->dma_address, npages);
+ DBGA(" sg_fill: [%p,%lx] -> sg %lx np %ld\n",
+ __va(paddr), size, out->dma_address, npages);
/* All virtually contiguous. We need to find the length of each
physically contiguous subsegment to fill in the ptes. */
@@ -455,7 +517,7 @@
#endif
size = sg->length;
- paddr = virt_to_phys(sg->address);
+ paddr = SG_ENT_PHYS_ADDRESS(sg);
while (sg+1 < end && (int) sg[1].dma_address == -1) {
size += sg[1].length;
@@ -470,11 +532,11 @@
#if DEBUG_ALLOC > 0
DBGA(" (%ld) [%p,%x] np %ld\n",
- last_sg - leader, last_sg->address,
+ last_sg - leader, SG_ENT_VIRT_ADDRESS(last_sg),
last_sg->length, npages);
while (++last_sg <= sg) {
DBGA(" (%ld) [%p,%x] cont\n",
- last_sg - leader, last_sg->address,
+ last_sg - leader, SG_ENT_VIRT_ADDRESS(last_sg),
last_sg->length);
}
#endif
@@ -491,15 +553,19 @@
struct pci_controller *hose;
struct pci_iommu_arena *arena;
dma_addr_t max_dma;
+ int dac_allowed;
if (direction == PCI_DMA_NONE)
BUG();
+ dac_allowed = ((pdev->dma_mask >> 32) != 0);
+
/* Fast path single entry scatterlists. */
if (nents == 1) {
sg->dma_length = sg->length;
sg->dma_address
- = pci_map_single(pdev, sg->address, sg->length, direction);
+ = pci_map_single_1(pdev, SG_ENT_VIRT_ADDRESS(sg),
+ sg->length, dac_allowed);
return sg->dma_address != 0;
}
@@ -527,7 +593,7 @@
for (out = sg; sg < end; ++sg) {
if ((int) sg->dma_address < 0)
continue;
- if (sg_fill(sg, end, out, arena, max_dma) < 0)
+ if (sg_fill(sg, end, out, arena, max_dma, dac_allowed) < 0)
goto error;
out++;
}
@@ -542,7 +608,7 @@
return out - start;
-error:
+ error:
printk(KERN_WARNING "pci_map_sg failed: "
"could not allocate dma page tables\n");
@@ -553,7 +619,6 @@
return 0;
}
-
/* Unmap a set of streaming mode DMA translations. Again, cpu read
rules concerning calls here are the same as for pci_unmap_single()
above. */
@@ -586,7 +651,8 @@
spin_lock_irqsave(&arena->lock, flags);
for (end = sg + nents; sg < end; ++sg) {
- unsigned long addr, size;
+ dma64_addr_t addr;
+ size_t size;
long npages, ofs;
dma_addr_t tend;
@@ -595,7 +661,13 @@
if (!size)
break;
-#if !DEBUG_NODIRECT
+ if (addr > 0xffffffff) {
+ /* It's a DAC address -- nothing to do. */
+ DBGA(" (%ld) DAC [%lx,%lx]\n",
+ sg - end + nents, addr, size);
+ continue;
+ }
+
if (addr >= __direct_map_base
&& addr < __direct_map_base + __direct_map_size) {
/* Nothing to do. */
@@ -603,7 +675,6 @@
sg - end + nents, addr, size);
continue;
}
-#endif
DBGA(" (%ld) sg [%lx,%lx]\n",
sg - end + nents, addr, size);
@@ -617,29 +688,27 @@
if (fend < tend) fend = tend;
}
- /*
- If we're freeing ptes above the `next_entry' pointer (they
+ /* If we're freeing ptes above the `next_entry' pointer (they
may have snuck back into the TLB since the last wrap flush),
- we need to flush the TLB before reallocating the latter.
- */
+ we need to flush the TLB before reallocating the latter. */
if ((fend - arena->dma_base) >> PAGE_SHIFT >= arena->next_entry)
alpha_mv.mv_pci_tbi(hose, fbeg, fend);
spin_unlock_irqrestore(&arena->lock, flags);
- DBGA("pci_unmap_sg: %d entries\n", nents - (end - sg));
+ DBGA("pci_unmap_sg: %ld entries\n", nents - (end - sg));
}
+
/* Return whether the given PCI device DMA address mask can be
supported properly. */
int
-pci_dma_supported(struct pci_dev *pdev, dma_addr_t mask)
+pci_dma_supported(struct pci_dev *pdev, u64 mask)
{
struct pci_controller *hose;
struct pci_iommu_arena *arena;
-#if !DEBUG_NODIRECT
/* If there exists a direct map, and the mask fits either
MAX_DMA_ADDRESS defined such that GFP_DMA does something
useful, or the total system memory as shifted by the
@@ -648,7 +717,6 @@
&& (__direct_map_base + MAX_DMA_ADDRESS-IDENT_ADDR-1 <= mask
|| __direct_map_base + (max_low_pfn<<PAGE_SHIFT)-1 <= mask))
return 1;
-#endif
/* Check that we have a scatter-gather arena that fits. */
hose = pdev ? pdev->sysdata : pci_isa_hose;
@@ -758,4 +826,50 @@
p[i] = IOMMU_RESERVED_PTE;
return 0;
+}
+
+/* True if the machine supports DAC addressing, and DEV can
+ make use of it given MASK. */
+
+int
+pci_dac_dma_supported(struct pci_dev *dev, u64 mask)
+{
+ dma64_addr_t dac_offset = alpha_mv.pci_dac_offset;
+ int ok = 1;
+
+ /* If this is not set, the machine doesn't support DAC at all. */
+ if (dac_offset == 0)
+ ok = 0;
+
+ /* The device has to be able to address our DAC bit. */
+ if ((dac_offset & dev->dma_mask) != dac_offset)
+ ok = 0;
+
+ /* If both conditions above are met, we are fine. */
+ DBGA("pci_dac_dma_supported %s from %p\n",
+ ok ? "yes" : "no", __builtin_return_address(0));
+
+ return ok;
+}
+
+dma64_addr_t
+pci_dac_page_to_dma(struct pci_dev *pdev, struct page *page,
+ unsigned long offset, int direction)
+{
+ return (alpha_mv.pci_dac_offset
+ + __pa(page_address(page))
+ + (dma64_addr_t) offset);
+}
+
+struct page *
+pci_dac_dma_to_page(struct pci_dev *pdev, dma64_addr_t dma_addr)
+{
+ unsigned long paddr = (dma_addr & PAGE_MASK) - alpha_mv.pci_dac_offset;
+ return virt_to_page(__va(paddr));
+}
+
+unsigned long
+pci_dac_dma_to_offset(struct pci_dev *pdev, dma64_addr_t dma_addr)
+{
+ return (dma_addr & ~PAGE_MASK);
}
FUNET's LINUX-ADM group, linux-adm@nic.funet.fi
TCL-scripts by Sam Shen (who was at: slshen@lbl.gov)