| 123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365 | 
							- /*
 
-  *  linux/arch/arm/mm/mmu.c
 
-  *
 
-  *  Copyright (C) 1995-2005 Russell King
 
-  *
 
-  * This program is free software; you can redistribute it and/or modify
 
-  * it under the terms of the GNU General Public License version 2 as
 
-  * published by the Free Software Foundation.
 
-  */
 
- #include <linux/module.h>
 
- #include <linux/kernel.h>
 
- #include <linux/errno.h>
 
- #include <linux/init.h>
 
- #include <linux/mman.h>
 
- #include <linux/nodemask.h>
 
- #include <linux/memblock.h>
 
- #include <linux/fs.h>
 
- #include <linux/vmalloc.h>
 
- #include <linux/sizes.h>
 
- #include <asm/cp15.h>
 
- #include <asm/cputype.h>
 
- #include <asm/sections.h>
 
- #include <asm/cachetype.h>
 
- #include <asm/setup.h>
 
- #include <asm/smp_plat.h>
 
- #include <asm/tlb.h>
 
- #include <asm/highmem.h>
 
- #include <asm/system_info.h>
 
- #include <asm/traps.h>
 
- #include <asm/mach/arch.h>
 
- #include <asm/mach/map.h>
 
- #include <asm/mach/pci.h>
 
- #include "mm.h"
 
- /*
 
-  * empty_zero_page is a special page that is used for
 
-  * zero-initialized data and COW.
 
-  */
 
- struct page *empty_zero_page;
 
- EXPORT_SYMBOL(empty_zero_page);
 
- /*
 
-  * The pmd table for the upper-most set of pages.
 
-  */
 
- pmd_t *top_pmd;
 
- #define CPOLICY_UNCACHED	0
 
- #define CPOLICY_BUFFERED	1
 
- #define CPOLICY_WRITETHROUGH	2
 
- #define CPOLICY_WRITEBACK	3
 
- #define CPOLICY_WRITEALLOC	4
 
- static unsigned int cachepolicy __initdata = CPOLICY_WRITEBACK;
 
- static unsigned int ecc_mask __initdata = 0;
 
- pgprot_t pgprot_user;
 
- pgprot_t pgprot_kernel;
 
- EXPORT_SYMBOL(pgprot_user);
 
- EXPORT_SYMBOL(pgprot_kernel);
 
- struct cachepolicy {
 
- 	const char	policy[16];
 
- 	unsigned int	cr_mask;
 
- 	pmdval_t	pmd;
 
- 	pteval_t	pte;
 
- };
 
- static struct cachepolicy cache_policies[] __initdata = {
 
- 	{
 
- 		.policy		= "uncached",
 
- 		.cr_mask	= CR_W|CR_C,
 
- 		.pmd		= PMD_SECT_UNCACHED,
 
- 		.pte		= L_PTE_MT_UNCACHED,
 
- 	}, {
 
- 		.policy		= "buffered",
 
- 		.cr_mask	= CR_C,
 
- 		.pmd		= PMD_SECT_BUFFERED,
 
- 		.pte		= L_PTE_MT_BUFFERABLE,
 
- 	}, {
 
- 		.policy		= "writethrough",
 
- 		.cr_mask	= 0,
 
- 		.pmd		= PMD_SECT_WT,
 
- 		.pte		= L_PTE_MT_WRITETHROUGH,
 
- 	}, {
 
- 		.policy		= "writeback",
 
- 		.cr_mask	= 0,
 
- 		.pmd		= PMD_SECT_WB,
 
- 		.pte		= L_PTE_MT_WRITEBACK,
 
- 	}, {
 
- 		.policy		= "writealloc",
 
- 		.cr_mask	= 0,
 
- 		.pmd		= PMD_SECT_WBWA,
 
- 		.pte		= L_PTE_MT_WRITEALLOC,
 
- 	}
 
- };
 
- /*
 
-  * These are useful for identifying cache coherency
 
-  * problems by allowing the cache or the cache and
 
-  * writebuffer to be turned off.  (Note: the write
 
-  * buffer should not be on and the cache off).
 
-  */
 
- static int __init early_cachepolicy(char *p)
 
- {
 
- 	int i;
 
- 	for (i = 0; i < ARRAY_SIZE(cache_policies); i++) {
 
- 		int len = strlen(cache_policies[i].policy);
 
- 		if (memcmp(p, cache_policies[i].policy, len) == 0) {
 
- 			cachepolicy = i;
 
- 			cr_alignment &= ~cache_policies[i].cr_mask;
 
- 			cr_no_alignment &= ~cache_policies[i].cr_mask;
 
- 			break;
 
- 		}
 
- 	}
 
- 	if (i == ARRAY_SIZE(cache_policies))
 
- 		printk(KERN_ERR "ERROR: unknown or unsupported cache policy\n");
 
- 	/*
 
- 	 * This restriction is partly to do with the way we boot; it is
 
- 	 * unpredictable to have memory mapped using two different sets of
 
- 	 * memory attributes (shared, type, and cache attribs).  We can not
 
- 	 * change these attributes once the initial assembly has setup the
 
- 	 * page tables.
 
- 	 */
 
- 	if (cpu_architecture() >= CPU_ARCH_ARMv6) {
 
- 		printk(KERN_WARNING "Only cachepolicy=writeback supported on ARMv6 and later\n");
 
- 		cachepolicy = CPOLICY_WRITEBACK;
 
- 	}
 
- 	flush_cache_all();
 
- 	set_cr(cr_alignment);
 
- 	return 0;
 
- }
 
- early_param("cachepolicy", early_cachepolicy);
 
- static int __init early_nocache(char *__unused)
 
- {
 
- 	char *p = "buffered";
 
- 	printk(KERN_WARNING "nocache is deprecated; use cachepolicy=%s\n", p);
 
- 	early_cachepolicy(p);
 
- 	return 0;
 
- }
 
- early_param("nocache", early_nocache);
 
- static int __init early_nowrite(char *__unused)
 
- {
 
- 	char *p = "uncached";
 
- 	printk(KERN_WARNING "nowb is deprecated; use cachepolicy=%s\n", p);
 
- 	early_cachepolicy(p);
 
- 	return 0;
 
- }
 
- early_param("nowb", early_nowrite);
 
- #ifndef CONFIG_ARM_LPAE
 
- static int __init early_ecc(char *p)
 
- {
 
- 	if (memcmp(p, "on", 2) == 0)
 
- 		ecc_mask = PMD_PROTECTION;
 
- 	else if (memcmp(p, "off", 3) == 0)
 
- 		ecc_mask = 0;
 
- 	return 0;
 
- }
 
- early_param("ecc", early_ecc);
 
- #endif
 
- static int __init noalign_setup(char *__unused)
 
- {
 
- 	cr_alignment &= ~CR_A;
 
- 	cr_no_alignment &= ~CR_A;
 
- 	set_cr(cr_alignment);
 
- 	return 1;
 
- }
 
- __setup("noalign", noalign_setup);
 
- #ifndef CONFIG_SMP
 
- void adjust_cr(unsigned long mask, unsigned long set)
 
- {
 
- 	unsigned long flags;
 
- 	mask &= ~CR_A;
 
- 	set &= mask;
 
- 	local_irq_save(flags);
 
- 	cr_no_alignment = (cr_no_alignment & ~mask) | set;
 
- 	cr_alignment = (cr_alignment & ~mask) | set;
 
- 	set_cr((get_cr() & ~mask) | set);
 
- 	local_irq_restore(flags);
 
- }
 
- #endif
 
- #define PROT_PTE_DEVICE		L_PTE_PRESENT|L_PTE_YOUNG|L_PTE_DIRTY|L_PTE_XN
 
- #define PROT_SECT_DEVICE	PMD_TYPE_SECT|PMD_SECT_AP_WRITE
 
- static struct mem_type mem_types[] = {
 
- 	[MT_DEVICE] = {		  /* Strongly ordered / ARMv6 shared device */
 
- 		.prot_pte	= PROT_PTE_DEVICE | L_PTE_MT_DEV_SHARED |
 
- 				  L_PTE_SHARED,
 
- 		.prot_l1	= PMD_TYPE_TABLE,
 
- 		.prot_sect	= PROT_SECT_DEVICE | PMD_SECT_S,
 
- 		.domain		= DOMAIN_IO,
 
- 	},
 
- 	[MT_DEVICE_NONSHARED] = { /* ARMv6 non-shared device */
 
- 		.prot_pte	= PROT_PTE_DEVICE | L_PTE_MT_DEV_NONSHARED,
 
- 		.prot_l1	= PMD_TYPE_TABLE,
 
- 		.prot_sect	= PROT_SECT_DEVICE,
 
- 		.domain		= DOMAIN_IO,
 
- 	},
 
- 	[MT_DEVICE_CACHED] = {	  /* ioremap_cached */
 
- 		.prot_pte	= PROT_PTE_DEVICE | L_PTE_MT_DEV_CACHED,
 
- 		.prot_l1	= PMD_TYPE_TABLE,
 
- 		.prot_sect	= PROT_SECT_DEVICE | PMD_SECT_WB,
 
- 		.domain		= DOMAIN_IO,
 
- 	},
 
- 	[MT_DEVICE_WC] = {	/* ioremap_wc */
 
- 		.prot_pte	= PROT_PTE_DEVICE | L_PTE_MT_DEV_WC,
 
- 		.prot_l1	= PMD_TYPE_TABLE,
 
- 		.prot_sect	= PROT_SECT_DEVICE,
 
- 		.domain		= DOMAIN_IO,
 
- 	},
 
- 	[MT_UNCACHED] = {
 
- 		.prot_pte	= PROT_PTE_DEVICE,
 
- 		.prot_l1	= PMD_TYPE_TABLE,
 
- 		.prot_sect	= PMD_TYPE_SECT | PMD_SECT_XN,
 
- 		.domain		= DOMAIN_IO,
 
- 	},
 
- 	[MT_CACHECLEAN] = {
 
- 		.prot_sect = PMD_TYPE_SECT | PMD_SECT_XN,
 
- 		.domain    = DOMAIN_KERNEL,
 
- 	},
 
- #ifndef CONFIG_ARM_LPAE
 
- 	[MT_MINICLEAN] = {
 
- 		.prot_sect = PMD_TYPE_SECT | PMD_SECT_XN | PMD_SECT_MINICACHE,
 
- 		.domain    = DOMAIN_KERNEL,
 
- 	},
 
- #endif
 
- 	[MT_LOW_VECTORS] = {
 
- 		.prot_pte  = L_PTE_PRESENT | L_PTE_YOUNG | L_PTE_DIRTY |
 
- 				L_PTE_RDONLY,
 
- 		.prot_l1   = PMD_TYPE_TABLE,
 
- 		.domain    = DOMAIN_USER,
 
- 	},
 
- 	[MT_HIGH_VECTORS] = {
 
- 		.prot_pte  = L_PTE_PRESENT | L_PTE_YOUNG | L_PTE_DIRTY |
 
- 				L_PTE_USER | L_PTE_RDONLY,
 
- 		.prot_l1   = PMD_TYPE_TABLE,
 
- 		.domain    = DOMAIN_USER,
 
- 	},
 
- 	[MT_MEMORY] = {
 
- 		.prot_pte  = L_PTE_PRESENT | L_PTE_YOUNG | L_PTE_DIRTY,
 
- 		.prot_l1   = PMD_TYPE_TABLE,
 
- 		.prot_sect = PMD_TYPE_SECT | PMD_SECT_AP_WRITE,
 
- 		.domain    = DOMAIN_KERNEL,
 
- 	},
 
- 	[MT_ROM] = {
 
- 		.prot_sect = PMD_TYPE_SECT,
 
- 		.domain    = DOMAIN_KERNEL,
 
- 	},
 
- 	[MT_MEMORY_NONCACHED] = {
 
- 		.prot_pte  = L_PTE_PRESENT | L_PTE_YOUNG | L_PTE_DIRTY |
 
- 				L_PTE_MT_BUFFERABLE,
 
- 		.prot_l1   = PMD_TYPE_TABLE,
 
- 		.prot_sect = PMD_TYPE_SECT | PMD_SECT_AP_WRITE,
 
- 		.domain    = DOMAIN_KERNEL,
 
- 	},
 
- 	[MT_MEMORY_DTCM] = {
 
- 		.prot_pte  = L_PTE_PRESENT | L_PTE_YOUNG | L_PTE_DIRTY |
 
- 				L_PTE_XN,
 
- 		.prot_l1   = PMD_TYPE_TABLE,
 
- 		.prot_sect = PMD_TYPE_SECT | PMD_SECT_XN,
 
- 		.domain    = DOMAIN_KERNEL,
 
- 	},
 
- 	[MT_MEMORY_ITCM] = {
 
- 		.prot_pte  = L_PTE_PRESENT | L_PTE_YOUNG | L_PTE_DIRTY,
 
- 		.prot_l1   = PMD_TYPE_TABLE,
 
- 		.domain    = DOMAIN_KERNEL,
 
- 	},
 
- 	[MT_MEMORY_SO] = {
 
- 		.prot_pte  = L_PTE_PRESENT | L_PTE_YOUNG | L_PTE_DIRTY |
 
- 				L_PTE_MT_UNCACHED | L_PTE_XN,
 
- 		.prot_l1   = PMD_TYPE_TABLE,
 
- 		.prot_sect = PMD_TYPE_SECT | PMD_SECT_AP_WRITE | PMD_SECT_S |
 
- 				PMD_SECT_UNCACHED | PMD_SECT_XN,
 
- 		.domain    = DOMAIN_KERNEL,
 
- 	},
 
- 	[MT_MEMORY_DMA_READY] = {
 
- 		.prot_pte  = L_PTE_PRESENT | L_PTE_YOUNG | L_PTE_DIRTY,
 
- 		.prot_l1   = PMD_TYPE_TABLE,
 
- 		.domain    = DOMAIN_KERNEL,
 
- 	},
 
- };
 
- const struct mem_type *get_mem_type(unsigned int type)
 
- {
 
- 	return type < ARRAY_SIZE(mem_types) ? &mem_types[type] : NULL;
 
- }
 
- EXPORT_SYMBOL(get_mem_type);
 
- /*
 
-  * Adjust the PMD section entries according to the CPU in use.
 
-  */
 
- static void __init build_mem_type_table(void)
 
- {
 
- 	struct cachepolicy *cp;
 
- 	unsigned int cr = get_cr();
 
- 	pteval_t user_pgprot, kern_pgprot, vecs_pgprot;
 
- 	int cpu_arch = cpu_architecture();
 
- 	int i;
 
- 	if (cpu_arch < CPU_ARCH_ARMv6) {
 
- #if defined(CONFIG_CPU_DCACHE_DISABLE)
 
- 		if (cachepolicy > CPOLICY_BUFFERED)
 
- 			cachepolicy = CPOLICY_BUFFERED;
 
- #elif defined(CONFIG_CPU_DCACHE_WRITETHROUGH)
 
- 		if (cachepolicy > CPOLICY_WRITETHROUGH)
 
- 			cachepolicy = CPOLICY_WRITETHROUGH;
 
- #endif
 
- 	}
 
- 	if (cpu_arch < CPU_ARCH_ARMv5) {
 
- 		if (cachepolicy >= CPOLICY_WRITEALLOC)
 
- 			cachepolicy = CPOLICY_WRITEBACK;
 
- 		ecc_mask = 0;
 
- 	}
 
- 	if (is_smp())
 
- 		cachepolicy = CPOLICY_WRITEALLOC;
 
- 	/*
 
- 	 * Strip out features not present on earlier architectures.
 
- 	 * Pre-ARMv5 CPUs don't have TEX bits.  Pre-ARMv6 CPUs or those
 
- 	 * without extended page tables don't have the 'Shared' bit.
 
- 	 */
 
- 	if (cpu_arch < CPU_ARCH_ARMv5)
 
- 		for (i = 0; i < ARRAY_SIZE(mem_types); i++)
 
- 			mem_types[i].prot_sect &= ~PMD_SECT_TEX(7);
 
- 	if ((cpu_arch < CPU_ARCH_ARMv6 || !(cr & CR_XP)) && !cpu_is_xsc3())
 
- 		for (i = 0; i < ARRAY_SIZE(mem_types); i++)
 
- 			mem_types[i].prot_sect &= ~PMD_SECT_S;
 
- 	/*
 
- 	 * ARMv5 and lower, bit 4 must be set for page tables (was: cache
 
- 	 * "update-able on write" bit on ARM610).  However, Xscale and
 
- 	 * Xscale3 require this bit to be cleared.
 
- 	 */
 
- 	if (cpu_is_xscale() || cpu_is_xsc3()) {
 
- 		for (i = 0; i < ARRAY_SIZE(mem_types); i++) {
 
- 			mem_types[i].prot_sect &= ~PMD_BIT4;
 
- 			mem_types[i].prot_l1 &= ~PMD_BIT4;
 
- 		}
 
- 	} else if (cpu_arch < CPU_ARCH_ARMv6) {
 
- 		for (i = 0; i < ARRAY_SIZE(mem_types); i++) {
 
- 			if (mem_types[i].prot_l1)
 
- 				mem_types[i].prot_l1 |= PMD_BIT4;
 
- 			if (mem_types[i].prot_sect)
 
- 				mem_types[i].prot_sect |= PMD_BIT4;
 
- 		}
 
- 	}
 
- 	/*
 
- 	 * Mark the device areas according to the CPU/architecture.
 
 
  |