diff --git a/arch/arm/include/asm/io.h b/arch/arm/include/asm/io.h index 05b281e4..92ce1f96 100644 --- a/arch/arm/include/asm/io.h +++ b/arch/arm/include/asm/io.h @@ -185,6 +185,12 @@ extern void _memset_io(volatile void __iomem *, int, size_t); #define readsw(p,d,l) __raw_readsw(__mem_pci(p),d,l) #define readsl(p,d,l) __raw_readsl(__mem_pci(p),d,l) +#define writeb_relaxed(v,c) ((void)__raw_writeb(v,__mem_pci(c))) +#define writew_relaxed(v,c) ((void)__raw_writew((__force u16) \ + cpu_to_le16(v),__mem_pci(c))) +#define writel_relaxed(v,c) ((void)__raw_writel((__force u32) \ + cpu_to_le32(v),__mem_pci(c))) + #define writeb(v,c) __raw_writeb(v,__mem_pci(c)) #define writew(v,c) __raw_writew((__force __u16) \ cpu_to_le16(v),__mem_pci(c)) diff --git a/arch/arm/mach-msm/board-htcleo.c b/arch/arm/mach-msm/board-htcleo.c index dcc7c990..8b5f90be 100644 --- a/arch/arm/mach-msm/board-htcleo.c +++ b/arch/arm/mach-msm/board-htcleo.c @@ -59,6 +59,10 @@ #include #include +#ifdef CONFIG_MSM_KGSL +#include +#endif + #include #include "board-htcleo.h" @@ -790,19 +794,83 @@ static struct platform_device msm_kgsl_device = .num_resources = ARRAY_SIZE(msm_kgsl_resources), }; +#ifdef CONFIG_MSM_KGSL +/* start kgsl */ +static struct resource kgsl_3d0_resources[] = { + { + .name = KGSL_3D0_REG_MEMORY, + .start = 0xA0000000, + .end = 0xA001ffff, + .flags = IORESOURCE_MEM, + }, + { + .name = KGSL_3D0_IRQ, + .start = INT_GRAPHICS, + .end = INT_GRAPHICS, + .flags = IORESOURCE_IRQ, + }, +}; + +static struct kgsl_device_platform_data kgsl_3d0_pdata = { + .pwr_data = { + .pwrlevel = { + { + .gpu_freq = 0, + .bus_freq = 128000000, + }, + }, + .init_level = 0, + .num_levels = 1, + .set_grp_async = NULL, + .idle_timeout = HZ/5, + }, + .clk = { + .name = { + .clk = "grp_clk", + }, + }, + .imem_clk_name = { + .clk = "imem_clk", + }, +}; + +struct platform_device msm_kgsl_3d0 = { + .name = "kgsl-3d0", + .id = 0, + .num_resources = ARRAY_SIZE(kgsl_3d0_resources), + .resource = kgsl_3d0_resources, + .dev = { + .platform_data = &kgsl_3d0_pdata, + }, +}; +/* end kgsl */ +#endif + /////////////////////////////////////////////////////////////////////// // Memory /////////////////////////////////////////////////////////////////////// static struct android_pmem_platform_data mdp_pmem_pdata = { .name = "pmem", + .start = MSM_PMEM_MDP_BASE, + .size = MSM_PMEM_MDP_SIZE, +#ifdef CONFIG_MSM_KGSL + .allocator_type = PMEM_ALLOCATORTYPE_BITMAP, +#else .no_allocator = 0, +#endif .cached = 1, }; static struct android_pmem_platform_data android_pmem_adsp_pdata = { .name = "pmem_adsp", + .start = MSM_PMEM_ADSP_BASE, + .size = MSM_PMEM_ADSP_SIZE, +#ifdef CONFIG_MSM_KGSL + .allocator_type = PMEM_ALLOCATORTYPE_BITMAP, +#else .no_allocator = 0, +#endif .cached = 1, }; @@ -811,7 +879,11 @@ static struct android_pmem_platform_data android_pmem_venc_pdata = { .name = "pmem_venc", .start = MSM_PMEM_VENC_BASE, .size = MSM_PMEM_VENC_SIZE, +#ifdef CONFIG_MSM_KGSL + .allocator_type = PMEM_ALLOCATORTYPE_BITMAP, +#else .no_allocator = 0, +#endif .cached = 1, }; @@ -945,7 +1017,11 @@ static struct platform_device *devices[] __initdata = &msm_device_i2c, &ds2746_battery_pdev, &htc_battery_pdev, +#ifdef CONFIG_MSM_KGSL + &msm_kgsl_3d0, +#else &msm_kgsl_device, +#endif &msm_camera_sensor_s5k3e2fx, &htcleo_flashlight_device, &qsd_device_spi, diff --git a/arch/arm/mach-msm/board-htcleo.h b/arch/arm/mach-msm/board-htcleo.h index 4504550b..d3777317 100644 --- a/arch/arm/mach-msm/board-htcleo.h +++ b/arch/arm/mach-msm/board-htcleo.h @@ -38,6 +38,12 @@ #define MSM_FB_BASE MSM_PMEM_SMI_BASE #define MSM_FB_SIZE 0x00600000 +#define MSM_PMEM_MDP_BASE 0x3B700000 +#define MSM_PMEM_MDP_SIZE 0x02000000 + +#define MSM_PMEM_ADSP_BASE 0x3D700000 +#define MSM_PMEM_ADSP_SIZE 0x02900000 + #define MSM_GPU_PHYS_BASE (MSM_PMEM_SMI_BASE + MSM_FB_SIZE) #define MSM_GPU_PHYS_SIZE 0x00800000 /* #define MSM_GPU_PHYS_SIZE 0x00300000 */ @@ -54,8 +60,6 @@ #define MSM_PMEM_SF_SIZE 0x02000000 -#define MSM_PMEM_ADSP_SIZE 0x02196000 - /* MSM_RAM_CONSOLE uses the last 0x00040000 of EBI memory, defined in msm_iomap.h #define MSM_RAM_CONSOLE_SIZE 0x00040000 #define MSM_RAM_CONSOLE_BASE (MSM_EBI1_BANK0_BASE + MSM_EBI1_BANK0_SIZE - MSM_RAM_CONSOLE_SIZE) //0x2FFC0000 diff --git a/arch/arm/mach-msm/include/mach/msm_bus.h b/arch/arm/mach-msm/include/mach/msm_bus.h new file mode 100755 index 00000000..095af3a6 --- /dev/null +++ b/arch/arm/mach-msm/include/mach/msm_bus.h @@ -0,0 +1,134 @@ +/* Copyright (c) 2010-2011, Code Aurora Forum. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, and the entire permission notice in its entirety, + * including the disclaimer of warranties. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * 3. The name of the author may not be used to endorse or promote + * products derived from this software without specific prior + * written permission. + * + * ALTERNATIVELY, this product may be distributed under the terms of + * the GNU General Public License, version 2, in which case the provisions + * of the GPL version 2 are required INSTEAD OF the BSD license. + * + * THIS SOFTWARE IS PROVIDED ``AS IS'' AND ANY EXPRESS OR IMPLIED + * WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE, ALL OF + * WHICH ARE HEREBY DISCLAIMED. IN NO EVENT SHALL THE AUTHOR BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT + * OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR + * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF + * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE + * USE OF THIS SOFTWARE, EVEN IF NOT ADVISED OF THE POSSIBILITY OF SUCH + * DAMAGE. + */ + +#ifndef _ARCH_ARM_MACH_MSM_BUS_H +#define _ARCH_ARM_MACH_MSM_BUS_H + +#include +#include + +/* + * Macros for clients to convert their data to ib and ab + * Ws : Time window over which to transfer the data in SECONDS + * Bs : Size of the data block in bytes + * Per : Recurrence period + * Tb : Throughput bandwidth to prevent stalling + * R : Ratio of actual bandwidth used to Tb + * Ib : Instantaneous bandwidth + * Ab : Arbitrated bandwidth + * + * IB_RECURRBLOCK and AB_RECURRBLOCK: + * These are used if the requirement is to transfer a + * recurring block of data over a known time window. + * + * IB_THROUGHPUTBW and AB_THROUGHPUTBW: + * These are used for CPU style masters. Here the requirement + * is to have minimum throughput bandwidth available to avoid + * stalling. + */ +#define IB_RECURRBLOCK(Ws, Bs) ((Ws) == 0 ? 0 : ((Bs)/(Ws))) +#define AB_RECURRBLOCK(Ws, Per) ((Ws) == 0 ? 0 : ((Bs)/(Per))) +#define IB_THROUGHPUTBW(Tb) (Tb) +#define AB_THROUGHPUTBW(Tb, R) ((Tb) * (R)) + +struct msm_bus_vectors { + int src; /* Master */ + int dst; /* Slave */ + unsigned int ab; /* Arbitrated bandwidth */ + unsigned int ib; /* Instantaneous bandwidth */ +}; + +struct msm_bus_paths { + int num_paths; + struct msm_bus_vectors *vectors; +}; + +struct msm_bus_scale_pdata { + struct msm_bus_paths *usecase; + int num_usecases; + const char *name; + /* + * If the active_only flag is set to 1, the BW request is applied + * only when at least one CPU is active (powered on). If the flag + * is set to 0, then the BW request is always applied irrespective + * of the CPU state. + */ + unsigned int active_only; +}; + +/* Scaling APIs */ + +/* + * This function returns a handle to the client. This should be used to + * call msm_bus_scale_client_update_request. + * The function returns 0 if bus driver is unable to register a client + */ + +#ifdef CONFIG_MSM_BUS_SCALING +uint32_t msm_bus_scale_register_client(struct msm_bus_scale_pdata *pdata); +int msm_bus_scale_client_update_request(uint32_t cl, unsigned int index); +void msm_bus_scale_unregister_client(uint32_t cl); +/* AXI Port configuration APIs */ +int msm_bus_axi_porthalt(int master_port); +int msm_bus_axi_portunhalt(int master_port); + +#else +static inline uint32_t +msm_bus_scale_register_client(struct msm_bus_scale_pdata *pdata) +{ + return 1; +} + +static inline int +msm_bus_scale_client_update_request(uint32_t cl, unsigned int index) +{ + return 0; +} + +static inline void +msm_bus_scale_unregister_client(uint32_t cl) +{ +} + +static inline int msm_bus_axi_porthalt(int master_port) +{ + return 0; +} + +static inline int msm_bus_axi_portunhalt(int master_port) +{ + return 0; +} +#endif + +#endif /*_ARCH_ARM_MACH_MSM_BUS_H*/ diff --git a/drivers/gpu/Makefile b/drivers/gpu/Makefile index 30879df3..417debe0 100644 --- a/drivers/gpu/Makefile +++ b/drivers/gpu/Makefile @@ -1 +1,2 @@ obj-y += drm/ vga/ +obj-$(CONFIG_MSM_KGSL) += msm/ diff --git a/drivers/gpu/msm/Kconfig b/drivers/gpu/msm/Kconfig new file mode 100644 index 00000000..64cbc304 --- /dev/null +++ b/drivers/gpu/msm/Kconfig @@ -0,0 +1,105 @@ +config MSM_KGSL + tristate "MSM 3D Graphics driver" + default n + depends on ARCH_MSM && !ARCH_MSM7X00A && !ARCH_MSM7X25 + select GENERIC_ALLOCATOR + select FW_LOADER + ---help--- + 3D graphics driver. Required to use hardware accelerated + OpenGL ES 2.0 and 1.1. + +config MSM_KGSL_CFF_DUMP + bool "Enable KGSL Common File Format (CFF) Dump Feature [Use with caution]" + default n + depends on MSM_KGSL + select RELAY + ---help--- + This is an analysis and diagnostic feature only, and should only be + turned on during KGSL GPU diagnostics and will slow down the KGSL + performance sigificantly, hence *do not use in production builds*. + When enabled, CFF Dump is on at boot. It can be turned off at runtime + via 'echo 0 > /d/kgsl/cff_dump'. The log can be captured via + /d/kgsl-cff/cpu[0|1]. + +config MSM_KGSL_CFF_DUMP_NO_CONTEXT_MEM_DUMP + bool "When selected will disable KGSL CFF Dump for context switches" + default n + depends on MSM_KGSL_CFF_DUMP + ---help--- + Dumping all the memory for every context switch can produce quite + huge log files, to reduce this, turn this feature on. + +config MSM_KGSL_PSTMRTMDMP_CP_STAT_NO_DETAIL + bool "Disable human readable CP_STAT fields in post-mortem dump" + default n + depends on MSM_KGSL + ---help--- + For a more compact kernel log the human readable output of + CP_STAT can be turned off with this option. + +config MSM_KGSL_PSTMRTMDMP_NO_IB_DUMP + bool "Disable dumping current IB1 and IB2 in post-mortem dump" + default n + depends on MSM_KGSL + ---help--- + For a more compact kernel log the IB1 and IB2 embedded dump + can be turned off with this option. Some IB dumps take up + so much space that vital other information gets cut from the + post-mortem dump. + +config MSM_KGSL_PSTMRTMDMP_RB_HEX + bool "Use hex version for ring-buffer in post-mortem dump" + default n + depends on MSM_KGSL + ---help--- + Use hex version for the ring-buffer in the post-mortem dump, instead + of the human readable version. + +config MSM_KGSL_2D + tristate "MSM 2D graphics driver. Required for OpenVG" + default y + depends on MSM_KGSL && !ARCH_MSM7X27 && !ARCH_MSM7X27A && !(ARCH_QSD8X50 && !MSM_SOC_REV_A) + +config MSM_KGSL_DRM + bool "Build a DRM interface for the MSM_KGSL driver" + depends on MSM_KGSL && DRM + +config MSM_KGSL_MMU + bool "Enable the GPU MMU in the MSM_KGSL driver" + depends on MSM_KGSL && MMU && !MSM_KGSL_CFF_DUMP + default y + +config KGSL_PER_PROCESS_PAGE_TABLE + bool "Enable Per Process page tables for the KGSL driver" + default n + depends on MSM_KGSL_MMU && !MSM_KGSL_DRM + ---help--- + The MMU will use per process pagetables when enabled. + +config MSM_KGSL_PAGE_TABLE_SIZE + hex "Size of pagetables" + default 0xFFF0000 + depends on MSM_KGSL_MMU + ---help--- + Sets the pagetable size used by the MMU. The max value + is 0xFFF0000 or (256M - 64K). + +config MSM_KGSL_PAGE_TABLE_COUNT + int "Minimum of concurrent pagetables to support" + default 8 + depends on KGSL_PER_PROCESS_PAGE_TABLE + ---help--- + Specify the number of pagetables to allocate at init time + This is the number of concurrent processes that are guaranteed to + to run at any time. Additional processes can be created dynamically + assuming there is enough contiguous memory to allocate the pagetable. + +config MSM_KGSL_MMU_PAGE_FAULT + bool "Force the GPU MMU to page fault for unmapped regions" + default y + depends on MSM_KGSL_MMU + +config MSM_KGSL_DISABLE_SHADOW_WRITES + bool "Disable register shadow writes for context switches" + default n + depends on MSM_KGSL diff --git a/drivers/gpu/msm/Makefile b/drivers/gpu/msm/Makefile new file mode 100644 index 00000000..c905bfec --- /dev/null +++ b/drivers/gpu/msm/Makefile @@ -0,0 +1,30 @@ +ccflags-y := -Iinclude/drm + +msm_kgsl_core-y = \ + kgsl.o \ + kgsl_sharedmem.o \ + kgsl_pwrctrl.o \ + kgsl_pwrscale.o + +msm_kgsl_core-$(CONFIG_DEBUG_FS) += kgsl_debugfs.o +msm_kgsl_core-$(CONFIG_MSM_KGSL_MMU) += kgsl_mmu.o +msm_kgsl_core-$(CONFIG_MSM_KGSL_CFF_DUMP) += kgsl_cffdump.o +msm_kgsl_core-$(CONFIG_MSM_KGSL_DRM) += kgsl_drm.o + +msm_adreno-y += \ + adreno_ringbuffer.o \ + adreno_drawctxt.o \ + adreno_postmortem.o \ + adreno.o + +msm_adreno-$(CONFIG_DEBUG_FS) += adreno_debugfs.o + +msm_z180-y += z180.o + +msm_kgsl_core-objs = $(msm_kgsl_core-y) +msm_adreno-objs = $(msm_adreno-y) +msm_z180-objs = $(msm_z180-y) + +obj-$(CONFIG_MSM_KGSL) += msm_kgsl_core.o +obj-$(CONFIG_MSM_KGSL) += msm_adreno.o +obj-$(CONFIG_MSM_KGSL_2D) += msm_z180.o diff --git a/drivers/gpu/msm/a200_reg.h b/drivers/gpu/msm/a200_reg.h new file mode 100644 index 00000000..4df6e14c --- /dev/null +++ b/drivers/gpu/msm/a200_reg.h @@ -0,0 +1,448 @@ +/* Copyright (c) 2002,2007-2011, Code Aurora Forum. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are + * met: + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following + * disclaimer in the documentation and/or other materials provided + * with the distribution. + * * Neither the name of Code Aurora Forum, Inc. nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED "AS IS" AND ANY EXPRESS OR IMPLIED + * WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS + * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR + * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, + * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE + * OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN + * IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + */ +#ifndef __A200_REG_H +#define __A200_REG_H + +enum VGT_EVENT_TYPE { + VS_DEALLOC = 0, + PS_DEALLOC = 1, + VS_DONE_TS = 2, + PS_DONE_TS = 3, + CACHE_FLUSH_TS = 4, + CONTEXT_DONE = 5, + CACHE_FLUSH = 6, + VIZQUERY_START = 7, + VIZQUERY_END = 8, + SC_WAIT_WC = 9, + RST_PIX_CNT = 13, + RST_VTX_CNT = 14, + TILE_FLUSH = 15, + CACHE_FLUSH_AND_INV_TS_EVENT = 20, + ZPASS_DONE = 21, + CACHE_FLUSH_AND_INV_EVENT = 22, + PERFCOUNTER_START = 23, + PERFCOUNTER_STOP = 24, + VS_FETCH_DONE = 27, + FACENESS_FLUSH = 28, +}; + +enum COLORFORMATX { + COLORX_4_4_4_4 = 0, + COLORX_1_5_5_5 = 1, + COLORX_5_6_5 = 2, + COLORX_8 = 3, + COLORX_8_8 = 4, + COLORX_8_8_8_8 = 5, + COLORX_S8_8_8_8 = 6, + COLORX_16_FLOAT = 7, + COLORX_16_16_FLOAT = 8, + COLORX_16_16_16_16_FLOAT = 9, + COLORX_32_FLOAT = 10, + COLORX_32_32_FLOAT = 11, + COLORX_32_32_32_32_FLOAT = 12, + COLORX_2_3_3 = 13, + COLORX_8_8_8 = 14, +}; + +enum SURFACEFORMAT { + FMT_1_REVERSE = 0, + FMT_1 = 1, + FMT_8 = 2, + FMT_1_5_5_5 = 3, + FMT_5_6_5 = 4, + FMT_6_5_5 = 5, + FMT_8_8_8_8 = 6, + FMT_2_10_10_10 = 7, + FMT_8_A = 8, + FMT_8_B = 9, + FMT_8_8 = 10, + FMT_Cr_Y1_Cb_Y0 = 11, + FMT_Y1_Cr_Y0_Cb = 12, + FMT_5_5_5_1 = 13, + FMT_8_8_8_8_A = 14, + FMT_4_4_4_4 = 15, + FMT_10_11_11 = 16, + FMT_11_11_10 = 17, + FMT_DXT1 = 18, + FMT_DXT2_3 = 19, + FMT_DXT4_5 = 20, + FMT_24_8 = 22, + FMT_24_8_FLOAT = 23, + FMT_16 = 24, + FMT_16_16 = 25, + FMT_16_16_16_16 = 26, + FMT_16_EXPAND = 27, + FMT_16_16_EXPAND = 28, + FMT_16_16_16_16_EXPAND = 29, + FMT_16_FLOAT = 30, + FMT_16_16_FLOAT = 31, + FMT_16_16_16_16_FLOAT = 32, + FMT_32 = 33, + FMT_32_32 = 34, + FMT_32_32_32_32 = 35, + FMT_32_FLOAT = 36, + FMT_32_32_FLOAT = 37, + FMT_32_32_32_32_FLOAT = 38, + FMT_32_AS_8 = 39, + FMT_32_AS_8_8 = 40, + FMT_16_MPEG = 41, + FMT_16_16_MPEG = 42, + FMT_8_INTERLACED = 43, + FMT_32_AS_8_INTERLACED = 44, + FMT_32_AS_8_8_INTERLACED = 45, + FMT_16_INTERLACED = 46, + FMT_16_MPEG_INTERLACED = 47, + FMT_16_16_MPEG_INTERLACED = 48, + FMT_DXN = 49, + FMT_8_8_8_8_AS_16_16_16_16 = 50, + FMT_DXT1_AS_16_16_16_16 = 51, + FMT_DXT2_3_AS_16_16_16_16 = 52, + FMT_DXT4_5_AS_16_16_16_16 = 53, + FMT_2_10_10_10_AS_16_16_16_16 = 54, + FMT_10_11_11_AS_16_16_16_16 = 55, + FMT_11_11_10_AS_16_16_16_16 = 56, + FMT_32_32_32_FLOAT = 57, + FMT_DXT3A = 58, + FMT_DXT5A = 59, + FMT_CTX1 = 60, + FMT_DXT3A_AS_1_1_1_1 = 61 +}; + +#define REG_PERF_MODE_CNT 0x0 +#define REG_PERF_STATE_RESET 0x0 +#define REG_PERF_STATE_ENABLE 0x1 +#define REG_PERF_STATE_FREEZE 0x2 + +#define RB_EDRAM_INFO_EDRAM_SIZE_SIZE 4 +#define RB_EDRAM_INFO_EDRAM_MAPPING_MODE_SIZE 2 +#define RB_EDRAM_INFO_UNUSED0_SIZE 8 +#define RB_EDRAM_INFO_EDRAM_RANGE_SIZE 18 + +struct rb_edram_info_t { + unsigned int edram_size:RB_EDRAM_INFO_EDRAM_SIZE_SIZE; + unsigned int edram_mapping_mode:RB_EDRAM_INFO_EDRAM_MAPPING_MODE_SIZE; + unsigned int unused0:RB_EDRAM_INFO_UNUSED0_SIZE; + unsigned int edram_range:RB_EDRAM_INFO_EDRAM_RANGE_SIZE; +}; + +union reg_rb_edram_info { + unsigned int val; + struct rb_edram_info_t f; +}; + +#define RBBM_READ_ERROR_UNUSED0_SIZE 2 +#define RBBM_READ_ERROR_READ_ADDRESS_SIZE 15 +#define RBBM_READ_ERROR_UNUSED1_SIZE 13 +#define RBBM_READ_ERROR_READ_REQUESTER_SIZE 1 +#define RBBM_READ_ERROR_READ_ERROR_SIZE 1 + +struct rbbm_read_error_t { + unsigned int unused0:RBBM_READ_ERROR_UNUSED0_SIZE; + unsigned int read_address:RBBM_READ_ERROR_READ_ADDRESS_SIZE; + unsigned int unused1:RBBM_READ_ERROR_UNUSED1_SIZE; + unsigned int read_requester:RBBM_READ_ERROR_READ_REQUESTER_SIZE; + unsigned int read_error:RBBM_READ_ERROR_READ_ERROR_SIZE; +}; + +union rbbm_read_error_u { + unsigned int val:32; + struct rbbm_read_error_t f; +}; + +#define CP_RB_CNTL_RB_BUFSZ_SIZE 6 +#define CP_RB_CNTL_UNUSED0_SIZE 2 +#define CP_RB_CNTL_RB_BLKSZ_SIZE 6 +#define CP_RB_CNTL_UNUSED1_SIZE 2 +#define CP_RB_CNTL_BUF_SWAP_SIZE 2 +#define CP_RB_CNTL_UNUSED2_SIZE 2 +#define CP_RB_CNTL_RB_POLL_EN_SIZE 1 +#define CP_RB_CNTL_UNUSED3_SIZE 6 +#define CP_RB_CNTL_RB_NO_UPDATE_SIZE 1 +#define CP_RB_CNTL_UNUSED4_SIZE 3 +#define CP_RB_CNTL_RB_RPTR_WR_ENA_SIZE 1 + +struct cp_rb_cntl_t { + unsigned int rb_bufsz:CP_RB_CNTL_RB_BUFSZ_SIZE; + unsigned int unused0:CP_RB_CNTL_UNUSED0_SIZE; + unsigned int rb_blksz:CP_RB_CNTL_RB_BLKSZ_SIZE; + unsigned int unused1:CP_RB_CNTL_UNUSED1_SIZE; + unsigned int buf_swap:CP_RB_CNTL_BUF_SWAP_SIZE; + unsigned int unused2:CP_RB_CNTL_UNUSED2_SIZE; + unsigned int rb_poll_en:CP_RB_CNTL_RB_POLL_EN_SIZE; + unsigned int unused3:CP_RB_CNTL_UNUSED3_SIZE; + unsigned int rb_no_update:CP_RB_CNTL_RB_NO_UPDATE_SIZE; + unsigned int unused4:CP_RB_CNTL_UNUSED4_SIZE; + unsigned int rb_rptr_wr_ena:CP_RB_CNTL_RB_RPTR_WR_ENA_SIZE; +}; + +union reg_cp_rb_cntl { + unsigned int val:32; + struct cp_rb_cntl_t f; +}; + +#define RB_COLOR_INFO__COLOR_FORMAT_MASK 0x0000000fL +#define RB_COPY_DEST_INFO__COPY_DEST_FORMAT__SHIFT 0x00000004 + + +#define SQ_INT_CNTL__PS_WATCHDOG_MASK 0x00000001L +#define SQ_INT_CNTL__VS_WATCHDOG_MASK 0x00000002L + +#define RBBM_INT_CNTL__RDERR_INT_MASK 0x00000001L +#define RBBM_INT_CNTL__DISPLAY_UPDATE_INT_MASK 0x00000002L +#define RBBM_INT_CNTL__GUI_IDLE_INT_MASK 0x00080000L + +#define RBBM_STATUS__CMDFIFO_AVAIL_MASK 0x0000001fL +#define RBBM_STATUS__TC_BUSY_MASK 0x00000020L +#define RBBM_STATUS__HIRQ_PENDING_MASK 0x00000100L +#define RBBM_STATUS__CPRQ_PENDING_MASK 0x00000200L +#define RBBM_STATUS__CFRQ_PENDING_MASK 0x00000400L +#define RBBM_STATUS__PFRQ_PENDING_MASK 0x00000800L +#define RBBM_STATUS__VGT_BUSY_NO_DMA_MASK 0x00001000L +#define RBBM_STATUS__RBBM_WU_BUSY_MASK 0x00004000L +#define RBBM_STATUS__CP_NRT_BUSY_MASK 0x00010000L +#define RBBM_STATUS__MH_BUSY_MASK 0x00040000L +#define RBBM_STATUS__MH_COHERENCY_BUSY_MASK 0x00080000L +#define RBBM_STATUS__SX_BUSY_MASK 0x00200000L +#define RBBM_STATUS__TPC_BUSY_MASK 0x00400000L +#define RBBM_STATUS__SC_CNTX_BUSY_MASK 0x01000000L +#define RBBM_STATUS__PA_BUSY_MASK 0x02000000L +#define RBBM_STATUS__VGT_BUSY_MASK 0x04000000L +#define RBBM_STATUS__SQ_CNTX17_BUSY_MASK 0x08000000L +#define RBBM_STATUS__SQ_CNTX0_BUSY_MASK 0x10000000L +#define RBBM_STATUS__RB_CNTX_BUSY_MASK 0x40000000L +#define RBBM_STATUS__GUI_ACTIVE_MASK 0x80000000L + +#define CP_INT_CNTL__SW_INT_MASK 0x00080000L +#define CP_INT_CNTL__T0_PACKET_IN_IB_MASK 0x00800000L +#define CP_INT_CNTL__OPCODE_ERROR_MASK 0x01000000L +#define CP_INT_CNTL__PROTECTED_MODE_ERROR_MASK 0x02000000L +#define CP_INT_CNTL__RESERVED_BIT_ERROR_MASK 0x04000000L +#define CP_INT_CNTL__IB_ERROR_MASK 0x08000000L +#define CP_INT_CNTL__IB2_INT_MASK 0x20000000L +#define CP_INT_CNTL__IB1_INT_MASK 0x40000000L +#define CP_INT_CNTL__RB_INT_MASK 0x80000000L + +#define MASTER_INT_SIGNAL__MH_INT_STAT 0x00000020L +#define MASTER_INT_SIGNAL__SQ_INT_STAT 0x04000000L +#define MASTER_INT_SIGNAL__CP_INT_STAT 0x40000000L +#define MASTER_INT_SIGNAL__RBBM_INT_STAT 0x80000000L + +#define RB_EDRAM_INFO__EDRAM_SIZE_MASK 0x0000000fL +#define RB_EDRAM_INFO__EDRAM_RANGE_MASK 0xffffc000L + +#define MH_ARBITER_CONFIG__SAME_PAGE_GRANULARITY__SHIFT 0x00000006 +#define MH_ARBITER_CONFIG__L1_ARB_ENABLE__SHIFT 0x00000007 +#define MH_ARBITER_CONFIG__L1_ARB_HOLD_ENABLE__SHIFT 0x00000008 +#define MH_ARBITER_CONFIG__L2_ARB_CONTROL__SHIFT 0x00000009 +#define MH_ARBITER_CONFIG__PAGE_SIZE__SHIFT 0x0000000a +#define MH_ARBITER_CONFIG__TC_REORDER_ENABLE__SHIFT 0x0000000d +#define MH_ARBITER_CONFIG__TC_ARB_HOLD_ENABLE__SHIFT 0x0000000e +#define MH_ARBITER_CONFIG__IN_FLIGHT_LIMIT_ENABLE__SHIFT 0x0000000f +#define MH_ARBITER_CONFIG__IN_FLIGHT_LIMIT__SHIFT 0x00000010 +#define MH_ARBITER_CONFIG__CP_CLNT_ENABLE__SHIFT 0x00000016 +#define MH_ARBITER_CONFIG__VGT_CLNT_ENABLE__SHIFT 0x00000017 +#define MH_ARBITER_CONFIG__TC_CLNT_ENABLE__SHIFT 0x00000018 +#define MH_ARBITER_CONFIG__RB_CLNT_ENABLE__SHIFT 0x00000019 +#define MH_ARBITER_CONFIG__PA_CLNT_ENABLE__SHIFT 0x0000001a + +#define MH_MMU_CONFIG__RB_W_CLNT_BEHAVIOR__SHIFT 0x00000004 +#define MH_MMU_CONFIG__CP_W_CLNT_BEHAVIOR__SHIFT 0x00000006 +#define MH_MMU_CONFIG__CP_R0_CLNT_BEHAVIOR__SHIFT 0x00000008 +#define MH_MMU_CONFIG__CP_R1_CLNT_BEHAVIOR__SHIFT 0x0000000a +#define MH_MMU_CONFIG__CP_R2_CLNT_BEHAVIOR__SHIFT 0x0000000c +#define MH_MMU_CONFIG__CP_R3_CLNT_BEHAVIOR__SHIFT 0x0000000e +#define MH_MMU_CONFIG__CP_R4_CLNT_BEHAVIOR__SHIFT 0x00000010 +#define MH_MMU_CONFIG__VGT_R0_CLNT_BEHAVIOR__SHIFT 0x00000012 +#define MH_MMU_CONFIG__VGT_R1_CLNT_BEHAVIOR__SHIFT 0x00000014 +#define MH_MMU_CONFIG__TC_R_CLNT_BEHAVIOR__SHIFT 0x00000016 +#define MH_MMU_CONFIG__PA_W_CLNT_BEHAVIOR__SHIFT 0x00000018 + +#define CP_RB_CNTL__RB_BUFSZ__SHIFT 0x00000000 +#define CP_RB_CNTL__RB_BLKSZ__SHIFT 0x00000008 +#define CP_RB_CNTL__RB_POLL_EN__SHIFT 0x00000014 +#define CP_RB_CNTL__RB_NO_UPDATE__SHIFT 0x0000001b + +#define RB_COLOR_INFO__COLOR_FORMAT__SHIFT 0x00000000 +#define RB_EDRAM_INFO__EDRAM_MAPPING_MODE__SHIFT 0x00000004 +#define RB_EDRAM_INFO__EDRAM_RANGE__SHIFT 0x0000000e + +#define REG_CP_CSQ_IB1_STAT 0x01FE +#define REG_CP_CSQ_IB2_STAT 0x01FF +#define REG_CP_CSQ_RB_STAT 0x01FD +#define REG_CP_DEBUG 0x01FC +#define REG_CP_IB1_BASE 0x0458 +#define REG_CP_IB1_BUFSZ 0x0459 +#define REG_CP_IB2_BASE 0x045A +#define REG_CP_IB2_BUFSZ 0x045B +#define REG_CP_INT_ACK 0x01F4 +#define REG_CP_INT_CNTL 0x01F2 +#define REG_CP_INT_STATUS 0x01F3 +#define REG_CP_ME_CNTL 0x01F6 +#define REG_CP_ME_RAM_DATA 0x01FA +#define REG_CP_ME_RAM_WADDR 0x01F8 +#define REG_CP_ME_STATUS 0x01F7 +#define REG_CP_PFP_UCODE_ADDR 0x00C0 +#define REG_CP_PFP_UCODE_DATA 0x00C1 +#define REG_CP_QUEUE_THRESHOLDS 0x01D5 +#define REG_CP_RB_BASE 0x01C0 +#define REG_CP_RB_CNTL 0x01C1 +#define REG_CP_RB_RPTR 0x01C4 +#define REG_CP_RB_RPTR_ADDR 0x01C3 +#define REG_CP_RB_RPTR_WR 0x01C7 +#define REG_CP_RB_WPTR 0x01C5 +#define REG_CP_RB_WPTR_BASE 0x01C8 +#define REG_CP_RB_WPTR_DELAY 0x01C6 +#define REG_CP_STAT 0x047F +#define REG_CP_STATE_DEBUG_DATA 0x01ED +#define REG_CP_STATE_DEBUG_INDEX 0x01EC +#define REG_CP_ST_BASE 0x044D +#define REG_CP_ST_BUFSZ 0x044E + +#define REG_CP_PERFMON_CNTL 0x0444 +#define REG_CP_PERFCOUNTER_SELECT 0x0445 +#define REG_CP_PERFCOUNTER_LO 0x0446 +#define REG_CP_PERFCOUNTER_HI 0x0447 + +#define REG_RBBM_PERFCOUNTER1_SELECT 0x0395 +#define REG_RBBM_PERFCOUNTER1_HI 0x0398 +#define REG_RBBM_PERFCOUNTER1_LO 0x0397 + +#define REG_MASTER_INT_SIGNAL 0x03B7 + +#define REG_MH_ARBITER_CONFIG 0x0A40 +#define REG_MH_INTERRUPT_CLEAR 0x0A44 +#define REG_MH_INTERRUPT_MASK 0x0A42 +#define REG_MH_INTERRUPT_STATUS 0x0A43 +#define REG_MH_MMU_CONFIG 0x0040 +#define REG_MH_MMU_INVALIDATE 0x0045 +#define REG_MH_MMU_MPU_BASE 0x0046 +#define REG_MH_MMU_MPU_END 0x0047 +#define REG_MH_MMU_PAGE_FAULT 0x0043 +#define REG_MH_MMU_PT_BASE 0x0042 +#define REG_MH_MMU_TRAN_ERROR 0x0044 +#define REG_MH_MMU_VA_RANGE 0x0041 +#define REG_MH_CLNT_INTF_CTRL_CONFIG1 0x0A54 +#define REG_MH_CLNT_INTF_CTRL_CONFIG2 0x0A55 + +#define REG_PA_CL_VPORT_XSCALE 0x210F +#define REG_PA_CL_VPORT_ZOFFSET 0x2114 +#define REG_PA_CL_VPORT_ZSCALE 0x2113 +#define REG_PA_CL_CLIP_CNTL 0x2204 +#define REG_PA_CL_VTE_CNTL 0x2206 +#define REG_PA_SC_AA_MASK 0x2312 +#define REG_PA_SC_LINE_CNTL 0x2300 +#define REG_PA_SC_SCREEN_SCISSOR_BR 0x200F +#define REG_PA_SC_SCREEN_SCISSOR_TL 0x200E +#define REG_PA_SC_VIZ_QUERY 0x2293 +#define REG_PA_SC_VIZ_QUERY_STATUS 0x0C44 +#define REG_PA_SC_WINDOW_OFFSET 0x2080 +#define REG_PA_SC_WINDOW_SCISSOR_BR 0x2082 +#define REG_PA_SC_WINDOW_SCISSOR_TL 0x2081 +#define REG_PA_SU_FACE_DATA 0x0C86 +#define REG_PA_SU_POINT_SIZE 0x2280 +#define REG_PA_SU_LINE_CNTL 0x2282 +#define REG_PA_SU_POLY_OFFSET_BACK_OFFSET 0x2383 +#define REG_PA_SU_POLY_OFFSET_FRONT_SCALE 0x2380 +#define REG_PA_SU_SC_MODE_CNTL 0x2205 + +#define REG_PC_INDEX_OFFSET 0x2102 + +#define REG_RBBM_CNTL 0x003B +#define REG_RBBM_INT_ACK 0x03B6 +#define REG_RBBM_INT_CNTL 0x03B4 +#define REG_RBBM_INT_STATUS 0x03B5 +#define REG_RBBM_PATCH_RELEASE 0x0001 +#define REG_RBBM_PERIPHID1 0x03F9 +#define REG_RBBM_PERIPHID2 0x03FA +#define REG_RBBM_DEBUG 0x039B +#define REG_RBBM_DEBUG_OUT 0x03A0 +#define REG_RBBM_DEBUG_CNTL 0x03A1 +#define REG_RBBM_PM_OVERRIDE1 0x039C +#define REG_RBBM_PM_OVERRIDE2 0x039D +#define REG_RBBM_READ_ERROR 0x03B3 +#define REG_RBBM_SOFT_RESET 0x003C +#define REG_RBBM_STATUS 0x05D0 + +#define REG_RB_COLORCONTROL 0x2202 +#define REG_RB_COLOR_DEST_MASK 0x2326 +#define REG_RB_COLOR_MASK 0x2104 +#define REG_RB_COPY_CONTROL 0x2318 +#define REG_RB_DEPTHCONTROL 0x2200 +#define REG_RB_EDRAM_INFO 0x0F02 +#define REG_RB_MODECONTROL 0x2208 +#define REG_RB_SURFACE_INFO 0x2000 +#define REG_RB_SAMPLE_POS 0x220a + +#define REG_SCRATCH_ADDR 0x01DD +#define REG_SCRATCH_REG0 0x0578 +#define REG_SCRATCH_REG2 0x057A +#define REG_SCRATCH_UMSK 0x01DC + +#define REG_SQ_CF_BOOLEANS 0x4900 +#define REG_SQ_CF_LOOP 0x4908 +#define REG_SQ_GPR_MANAGEMENT 0x0D00 +#define REG_SQ_INST_STORE_MANAGMENT 0x0D02 +#define REG_SQ_INT_ACK 0x0D36 +#define REG_SQ_INT_CNTL 0x0D34 +#define REG_SQ_INT_STATUS 0x0D35 +#define REG_SQ_PROGRAM_CNTL 0x2180 +#define REG_SQ_PS_PROGRAM 0x21F6 +#define REG_SQ_VS_PROGRAM 0x21F7 +#define REG_SQ_WRAPPING_0 0x2183 +#define REG_SQ_WRAPPING_1 0x2184 + +#define REG_VGT_ENHANCE 0x2294 +#define REG_VGT_INDX_OFFSET 0x2102 +#define REG_VGT_MAX_VTX_INDX 0x2100 +#define REG_VGT_MIN_VTX_INDX 0x2101 + +#define REG_TP0_CHICKEN 0x0E1E +#define REG_TC_CNTL_STATUS 0x0E00 +#define REG_PA_SC_AA_CONFIG 0x2301 +#define REG_VGT_VERTEX_REUSE_BLOCK_CNTL 0x2316 +#define REG_SQ_INTERPOLATOR_CNTL 0x2182 +#define REG_RB_DEPTH_INFO 0x2002 +#define REG_COHER_DEST_BASE_0 0x2006 +#define REG_RB_FOG_COLOR 0x2109 +#define REG_RB_STENCILREFMASK_BF 0x210C +#define REG_PA_SC_LINE_STIPPLE 0x2283 +#define REG_SQ_PS_CONST 0x2308 +#define REG_RB_DEPTH_CLEAR 0x231D +#define REG_RB_SAMPLE_COUNT_CTL 0x2324 +#define REG_SQ_CONSTANT_0 0x4000 +#define REG_SQ_FETCH_0 0x4800 + +#define REG_MH_AXI_ERROR 0xA45 +#define REG_MH_DEBUG_CTRL 0xA4E +#define REG_MH_DEBUG_DATA 0xA4F +#define REG_COHER_BASE_PM4 0xA2A +#define REG_COHER_STATUS_PM4 0xA2B +#define REG_COHER_SIZE_PM4 0xA29 + +#endif /* __A200_REG_H */ diff --git a/drivers/gpu/msm/a220_reg.h b/drivers/gpu/msm/a220_reg.h new file mode 100644 index 00000000..9542a9ba --- /dev/null +++ b/drivers/gpu/msm/a220_reg.h @@ -0,0 +1,39 @@ +/* Copyright (c) 2010-2011, Code Aurora Forum. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are + * met: + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following + * disclaimer in the documentation and/or other materials provided + * with the distribution. + * * Neither the name of Code Aurora Forum, Inc. nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED "AS IS" AND ANY EXPRESS OR IMPLIED + * WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS + * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR + * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, + * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE + * OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN + * IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + */ +#ifndef __A205_REG_H +#define __A205_REG_H + +#define REG_LEIA_PC_INDX_OFFSET REG_VGT_INDX_OFFSET +#define REG_LEIA_PC_VERTEX_REUSE_BLOCK_CNTL REG_VGT_VERTEX_REUSE_BLOCK_CNTL +#define REG_LEIA_PC_MAX_VTX_INDX REG_VGT_MAX_VTX_INDX +#define REG_LEIA_GRAS_CONTROL 0x2210 +#define REG_LEIA_VSC_BIN_SIZE 0x0C01 +#define REG_LEIA_VSC_PIPE_DATA_LENGTH_7 0x0C1D + +#endif /*__A205_REG_H */ diff --git a/drivers/gpu/msm/adreno.c b/drivers/gpu/msm/adreno.c new file mode 100644 index 00000000..db3d9782 --- /dev/null +++ b/drivers/gpu/msm/adreno.c @@ -0,0 +1,1364 @@ +/* Copyright (c) 2002,2007-2011, Code Aurora Forum. All rights reserved. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 and + * only version 2 as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + */ +#include +#include +#include +#include +#include + +#include "kgsl.h" +#include "kgsl_pwrscale.h" +#include "kgsl_cffdump.h" + +#include "adreno.h" +#include "adreno_pm4types.h" +#include "adreno_debugfs.h" +#include "adreno_postmortem.h" + +#include "a200_reg.h" + +#define DRIVER_VERSION_MAJOR 3 +#define DRIVER_VERSION_MINOR 1 + +#define GSL_RBBM_INT_MASK \ + (RBBM_INT_CNTL__RDERR_INT_MASK | \ + RBBM_INT_CNTL__DISPLAY_UPDATE_INT_MASK) + +/* Adreno MH arbiter config*/ +#define ADRENO_CFG_MHARB \ + (0x10 \ + | (0 << MH_ARBITER_CONFIG__SAME_PAGE_GRANULARITY__SHIFT) \ + | (1 << MH_ARBITER_CONFIG__L1_ARB_ENABLE__SHIFT) \ + | (1 << MH_ARBITER_CONFIG__L1_ARB_HOLD_ENABLE__SHIFT) \ + | (0 << MH_ARBITER_CONFIG__L2_ARB_CONTROL__SHIFT) \ + | (1 << MH_ARBITER_CONFIG__PAGE_SIZE__SHIFT) \ + | (1 << MH_ARBITER_CONFIG__TC_REORDER_ENABLE__SHIFT) \ + | (1 << MH_ARBITER_CONFIG__TC_ARB_HOLD_ENABLE__SHIFT) \ + | (0 << MH_ARBITER_CONFIG__IN_FLIGHT_LIMIT_ENABLE__SHIFT) \ + | (0x8 << MH_ARBITER_CONFIG__IN_FLIGHT_LIMIT__SHIFT) \ + | (1 << MH_ARBITER_CONFIG__CP_CLNT_ENABLE__SHIFT) \ + | (1 << MH_ARBITER_CONFIG__VGT_CLNT_ENABLE__SHIFT) \ + | (1 << MH_ARBITER_CONFIG__TC_CLNT_ENABLE__SHIFT) \ + | (1 << MH_ARBITER_CONFIG__RB_CLNT_ENABLE__SHIFT) \ + | (1 << MH_ARBITER_CONFIG__PA_CLNT_ENABLE__SHIFT)) + +#define ADRENO_MMU_CONFIG \ + (0x01 \ + | (MMU_CONFIG << MH_MMU_CONFIG__RB_W_CLNT_BEHAVIOR__SHIFT) \ + | (MMU_CONFIG << MH_MMU_CONFIG__CP_W_CLNT_BEHAVIOR__SHIFT) \ + | (MMU_CONFIG << MH_MMU_CONFIG__CP_R0_CLNT_BEHAVIOR__SHIFT) \ + | (MMU_CONFIG << MH_MMU_CONFIG__CP_R1_CLNT_BEHAVIOR__SHIFT) \ + | (MMU_CONFIG << MH_MMU_CONFIG__CP_R2_CLNT_BEHAVIOR__SHIFT) \ + | (MMU_CONFIG << MH_MMU_CONFIG__CP_R3_CLNT_BEHAVIOR__SHIFT) \ + | (MMU_CONFIG << MH_MMU_CONFIG__CP_R4_CLNT_BEHAVIOR__SHIFT) \ + | (MMU_CONFIG << MH_MMU_CONFIG__VGT_R0_CLNT_BEHAVIOR__SHIFT) \ + | (MMU_CONFIG << MH_MMU_CONFIG__VGT_R1_CLNT_BEHAVIOR__SHIFT) \ + | (MMU_CONFIG << MH_MMU_CONFIG__TC_R_CLNT_BEHAVIOR__SHIFT) \ + | (MMU_CONFIG << MH_MMU_CONFIG__PA_W_CLNT_BEHAVIOR__SHIFT)) + +/* max msecs to wait for gpu to finish its operation(s) */ +#define MAX_WAITGPU_SECS (HZ + HZ/2) + +static struct adreno_device device_3d0 = { + .dev = { + .name = DEVICE_3D0_NAME, + .id = KGSL_DEVICE_3D0, + .ver_major = DRIVER_VERSION_MAJOR, + .ver_minor = DRIVER_VERSION_MINOR, + .mmu = { + .config = ADRENO_MMU_CONFIG, + /* turn off memory protection unit by setting + acceptable physical address range to include + all pages. */ + .mpu_base = 0x00000000, + .mpu_range = 0xFFFFF000, + .reg = { + .config = REG_MH_MMU_CONFIG, + .mpu_base = REG_MH_MMU_MPU_BASE, + .mpu_end = REG_MH_MMU_MPU_END, + .va_range = REG_MH_MMU_VA_RANGE, + .pt_page = REG_MH_MMU_PT_BASE, + .page_fault = REG_MH_MMU_PAGE_FAULT, + .tran_error = REG_MH_MMU_TRAN_ERROR, + .invalidate = REG_MH_MMU_INVALIDATE, + .interrupt_mask = REG_MH_INTERRUPT_MASK, + .interrupt_status = REG_MH_INTERRUPT_STATUS, + .interrupt_clear = REG_MH_INTERRUPT_CLEAR, + .axi_error = REG_MH_AXI_ERROR, + }, + }, + .pwrctrl = { + .regulator_name = "fs_gfx3d", + .irq_name = KGSL_3D0_IRQ, + .src_clk_name = "grp_src_clk", + }, + .mutex = __MUTEX_INITIALIZER(device_3d0.dev.mutex), + .state = KGSL_STATE_INIT, + .active_cnt = 0, + .iomemname = KGSL_3D0_REG_MEMORY, + }, + .gmemspace = { + .gpu_base = 0, + .sizebytes = SZ_256K, + }, + .pfp_fw = NULL, + .pm4_fw = NULL, + .mharb = ADRENO_CFG_MHARB, +}; + +static void __devinit adreno_getfunctable(struct kgsl_functable *ftbl); + +static int adreno_gmeminit(struct adreno_device *adreno_dev) +{ + struct kgsl_device *device = &adreno_dev->dev; + union reg_rb_edram_info rb_edram_info; + unsigned int gmem_size; + unsigned int edram_value = 0; + + /* make sure edram range is aligned to size */ + BUG_ON(adreno_dev->gmemspace.gpu_base & + (adreno_dev->gmemspace.sizebytes - 1)); + + /* get edram_size value equivalent */ + gmem_size = (adreno_dev->gmemspace.sizebytes >> 14); + while (gmem_size >>= 1) + edram_value++; + + rb_edram_info.val = 0; + + rb_edram_info.f.edram_size = edram_value; + if (!adreno_is_a220(adreno_dev)) + rb_edram_info.f.edram_mapping_mode = 0; /* EDRAM_MAP_UPPER */ + + /* must be aligned to size */ + rb_edram_info.f.edram_range = (adreno_dev->gmemspace.gpu_base >> 14); + + adreno_regwrite(device, REG_RB_EDRAM_INFO, rb_edram_info.val); + + return 0; +} + +static int adreno_gmemclose(struct kgsl_device *device) +{ + adreno_regwrite(device, REG_RB_EDRAM_INFO, 0x00000000); + + return 0; +} + +static void adreno_rbbm_intrcallback(struct kgsl_device *device) +{ + unsigned int status = 0; + unsigned int rderr = 0; + + adreno_regread_isr(device, REG_RBBM_INT_STATUS, &status); + + if (status & RBBM_INT_CNTL__RDERR_INT_MASK) { + union rbbm_read_error_u rerr; + adreno_regread_isr(device, REG_RBBM_READ_ERROR, &rderr); + rerr.val = rderr; + if (rerr.f.read_address == REG_CP_INT_STATUS && + rerr.f.read_error && + rerr.f.read_requester) + KGSL_DRV_WARN(device, + "rbbm read error interrupt: %08x\n", rderr); + else + KGSL_DRV_CRIT(device, + "rbbm read error interrupt: %08x\n", rderr); + } else if (status & RBBM_INT_CNTL__DISPLAY_UPDATE_INT_MASK) { + KGSL_DRV_INFO(device, "rbbm display update interrupt\n"); + } else if (status & RBBM_INT_CNTL__GUI_IDLE_INT_MASK) { + KGSL_DRV_INFO(device, "rbbm gui idle interrupt\n"); + } else { + KGSL_CMD_WARN(device, + "bad bits in REG_CP_INT_STATUS %08x\n", status); + } + + status &= GSL_RBBM_INT_MASK; + adreno_regwrite_isr(device, REG_RBBM_INT_ACK, status); +} + +irqreturn_t adreno_isr(int irq, void *data) +{ + irqreturn_t result = IRQ_NONE; + struct kgsl_device *device; + unsigned int status; + + device = (struct kgsl_device *) data; + + BUG_ON(device == NULL); + BUG_ON(device->regspace.sizebytes == 0); + BUG_ON(device->regspace.mmio_virt_base == 0); + + adreno_regread_isr(device, REG_MASTER_INT_SIGNAL, &status); + + if (status & MASTER_INT_SIGNAL__MH_INT_STAT) { + kgsl_mh_intrcallback(device); + result = IRQ_HANDLED; + } + + if (status & MASTER_INT_SIGNAL__CP_INT_STAT) { + kgsl_cp_intrcallback(device); + result = IRQ_HANDLED; + } + + if (status & MASTER_INT_SIGNAL__RBBM_INT_STAT) { + adreno_rbbm_intrcallback(device); + result = IRQ_HANDLED; + } + + if (device->requested_state == KGSL_STATE_NONE) { + if (device->pwrctrl.nap_allowed == true) { + device->requested_state = KGSL_STATE_NAP; + queue_work(device->work_queue, &device->idle_check_ws); + } else if (device->pwrctrl.idle_pass == true) { + queue_work(device->work_queue, &device->idle_check_ws); + } + } + + /* Reset the time-out in our idle timer */ + mod_timer(&device->idle_timer, + jiffies + device->pwrctrl.interval_timeout); + return result; +} + +static int adreno_cleanup_pt(struct kgsl_device *device, + struct kgsl_pagetable *pagetable) +{ + struct adreno_device *adreno_dev = ADRENO_DEVICE(device); + struct adreno_ringbuffer *rb = &adreno_dev->ringbuffer; + + kgsl_mmu_unmap(pagetable, &rb->buffer_desc); + + kgsl_mmu_unmap(pagetable, &rb->memptrs_desc); + + kgsl_mmu_unmap(pagetable, &device->memstore); + + kgsl_mmu_unmap(pagetable, &device->mmu.dummyspace); + + return 0; +} + +static int adreno_setup_pt(struct kgsl_device *device, + struct kgsl_pagetable *pagetable) +{ + int result = 0; + struct adreno_device *adreno_dev = ADRENO_DEVICE(device); + struct adreno_ringbuffer *rb = &adreno_dev->ringbuffer; + + BUG_ON(rb->buffer_desc.physaddr == 0); + BUG_ON(rb->memptrs_desc.physaddr == 0); + BUG_ON(device->memstore.physaddr == 0); +#ifdef CONFIG_MSM_KGSL_MMU + BUG_ON(device->mmu.dummyspace.physaddr == 0); +#endif + result = kgsl_mmu_map_global(pagetable, &rb->buffer_desc, + GSL_PT_PAGE_RV); + if (result) + goto error; + + result = kgsl_mmu_map_global(pagetable, &rb->memptrs_desc, + GSL_PT_PAGE_RV | GSL_PT_PAGE_WV); + if (result) + goto unmap_buffer_desc; + + result = kgsl_mmu_map_global(pagetable, &device->memstore, + GSL_PT_PAGE_RV | GSL_PT_PAGE_WV); + if (result) + goto unmap_memptrs_desc; + + result = kgsl_mmu_map_global(pagetable, &device->mmu.dummyspace, + GSL_PT_PAGE_RV | GSL_PT_PAGE_WV); + if (result) + goto unmap_memstore_desc; + + return result; + +unmap_memstore_desc: + kgsl_mmu_unmap(pagetable, &device->memstore); + +unmap_memptrs_desc: + kgsl_mmu_unmap(pagetable, &rb->memptrs_desc); + +unmap_buffer_desc: + kgsl_mmu_unmap(pagetable, &rb->buffer_desc); + +error: + return result; +} + +static int adreno_setstate(struct kgsl_device *device, uint32_t flags) +{ + struct adreno_device *adreno_dev = ADRENO_DEVICE(device); + unsigned int link[32]; + unsigned int *cmds = &link[0]; + int sizedwords = 0; + unsigned int mh_mmu_invalidate = 0x00000003; /*invalidate all and tc */ + +#ifndef CONFIG_MSM_KGSL_MMU + return 0; +#endif + /* if possible, set via command stream, + * otherwise set via direct register writes + */ + if (adreno_dev->drawctxt_active) { + if (flags & KGSL_MMUFLAGS_PTUPDATE) { + /* wait for graphics pipe to be idle */ + *cmds++ = pm4_type3_packet(PM4_WAIT_FOR_IDLE, 1); + *cmds++ = 0x00000000; + + /* set page table base */ + *cmds++ = pm4_type0_packet(REG_MH_MMU_PT_BASE, 1); + *cmds++ = device->mmu.hwpagetable->base.gpuaddr; + sizedwords += 4; + } + + if (flags & KGSL_MMUFLAGS_TLBFLUSH) { + if (!(flags & KGSL_MMUFLAGS_PTUPDATE)) { + *cmds++ = pm4_type3_packet(PM4_WAIT_FOR_IDLE, + 1); + *cmds++ = 0x00000000; + sizedwords += 2; + } + *cmds++ = pm4_type0_packet(REG_MH_MMU_INVALIDATE, 1); + *cmds++ = mh_mmu_invalidate; + sizedwords += 2; + } + + if (flags & KGSL_MMUFLAGS_PTUPDATE && + !adreno_is_a220(adreno_dev)) { + /* HW workaround: to resolve MMU page fault interrupts + * caused by the VGT.It prevents the CP PFP from filling + * the VGT DMA request fifo too early,thereby ensuring + * that the VGT will not fetch vertex/bin data until + * after the page table base register has been updated. + * + * Two null DRAW_INDX_BIN packets are inserted right + * after the page table base update, followed by a + * wait for idle. The null packets will fill up the + * VGT DMA request fifo and prevent any further + * vertex/bin updates from occurring until the wait + * has finished. */ + *cmds++ = pm4_type3_packet(PM4_SET_CONSTANT, 2); + *cmds++ = (0x4 << 16) | + (REG_PA_SU_SC_MODE_CNTL - 0x2000); + *cmds++ = 0; /* disable faceness generation */ + *cmds++ = pm4_type3_packet(PM4_SET_BIN_BASE_OFFSET, 1); + *cmds++ = device->mmu.dummyspace.gpuaddr; + *cmds++ = pm4_type3_packet(PM4_DRAW_INDX_BIN, 6); + *cmds++ = 0; /* viz query info */ + *cmds++ = 0x0003C004; /* draw indicator */ + *cmds++ = 0; /* bin base */ + *cmds++ = 3; /* bin size */ + *cmds++ = device->mmu.dummyspace.gpuaddr; /* dma base */ + *cmds++ = 6; /* dma size */ + *cmds++ = pm4_type3_packet(PM4_DRAW_INDX_BIN, 6); + *cmds++ = 0; /* viz query info */ + *cmds++ = 0x0003C004; /* draw indicator */ + *cmds++ = 0; /* bin base */ + *cmds++ = 3; /* bin size */ + /* dma base */ + *cmds++ = device->mmu.dummyspace.gpuaddr; + *cmds++ = 6; /* dma size */ + *cmds++ = pm4_type3_packet(PM4_WAIT_FOR_IDLE, 1); + *cmds++ = 0x00000000; + sizedwords += 21; + } + + if (flags & (KGSL_MMUFLAGS_PTUPDATE | KGSL_MMUFLAGS_TLBFLUSH)) { + *cmds++ = pm4_type3_packet(PM4_INVALIDATE_STATE, 1); + *cmds++ = 0x7fff; /* invalidate all base pointers */ + sizedwords += 2; + } + + adreno_ringbuffer_issuecmds(device, KGSL_CMD_FLAGS_PMODE, + &link[0], sizedwords); + } else { + if (flags & KGSL_MMUFLAGS_PTUPDATE) { + adreno_idle(device, KGSL_TIMEOUT_DEFAULT); + adreno_regwrite(device, REG_MH_MMU_PT_BASE, + device->mmu.hwpagetable->base.gpuaddr); + } + + if (flags & KGSL_MMUFLAGS_TLBFLUSH) { + adreno_regwrite(device, REG_MH_MMU_INVALIDATE, + mh_mmu_invalidate); + } + } + + return 0; +} + +static unsigned int +adreno_getchipid(struct kgsl_device *device) +{ + /* XXX: drewis edit: only for 8x50 */ + unsigned int chipid = 0; + unsigned int coreid, majorid, minorid, patchid, revid; + + adreno_regread(device, REG_RBBM_PERIPHID1, &coreid); + adreno_regread(device, REG_RBBM_PERIPHID2, &majorid); + adreno_regread(device, REG_RBBM_PATCH_RELEASE, &revid); + + chipid = (coreid & 0xF) << 24; + + chipid |= ((majorid >> 4) & 0xF) << 16; + + minorid = ((revid >> 0) & 0xFF); + + patchid = 1; + + chipid |= (minorid << 8) | patchid; + + return chipid; +} + +/* all chipid fields are 8 bits wide so 256 won't occur in a real chipid */ +#define DONT_CARE 256 +static const struct { + unsigned int core; + unsigned int major; + unsigned int minor; + enum adreno_gpurev gpurev; +} gpurev_table[] = { + /* major and minor may be DONT_CARE, but core must not be */ + {0, 2, DONT_CARE, ADRENO_REV_A200}, + {0, 1, 0, ADRENO_REV_A205}, + {2, 1, DONT_CARE, ADRENO_REV_A220}, + {2, 2, DONT_CARE, ADRENO_REV_A225}, +}; + +static inline bool _rev_match(unsigned int id, unsigned int entry) +{ + return (entry == DONT_CARE || entry == id); +} +#undef DONT_CARE + +enum adreno_gpurev adreno_get_rev(struct adreno_device *adreno_dev) +{ + enum adreno_gpurev gpurev = ADRENO_REV_UNKNOWN; + unsigned int i, core, major, minor; + core = (adreno_dev->chip_id >> 24) & 0xff; + major = (adreno_dev->chip_id >> 16) & 0xff; + minor = (adreno_dev->chip_id >> 8) & 0xff; + + for (i = 0; i < ARRAY_SIZE(gpurev_table); i++) { + if (core == gpurev_table[i].core && + _rev_match(major, gpurev_table[i].major) && + _rev_match(minor, gpurev_table[i].minor)) { + gpurev = gpurev_table[i].gpurev; + break; + } + } + return gpurev; +} + +static int __devinit +adreno_probe(struct platform_device *pdev) +{ + struct kgsl_device *device; + struct adreno_device *adreno_dev; + int status = -EINVAL; + + device = (struct kgsl_device *)pdev->id_entry->driver_data; + adreno_dev = ADRENO_DEVICE(device); + device->parentdev = &pdev->dev; + + init_completion(&device->recovery_gate); + + adreno_getfunctable(&device->ftbl); + + status = adreno_ringbuffer_init(device); + if (status != 0) + goto error; + + status = kgsl_device_platform_probe(device, adreno_isr); + if (status) + goto error_close_rb; + + adreno_debugfs_init(device); + + device->flags &= ~KGSL_FLAGS_SOFT_RESET; + return 0; + +error_close_rb: + adreno_ringbuffer_close(&adreno_dev->ringbuffer); +error: + device->parentdev = NULL; + return status; +} + +static int __devexit adreno_remove(struct platform_device *pdev) +{ + struct kgsl_device *device; + struct adreno_device *adreno_dev; + + device = (struct kgsl_device *)pdev->id_entry->driver_data; + adreno_dev = ADRENO_DEVICE(device); + + adreno_ringbuffer_close(&adreno_dev->ringbuffer); + kgsl_device_platform_remove(device); + + return 0; +} + +static int adreno_start(struct kgsl_device *device, unsigned int init_ram) +{ + int status = -EINVAL; + struct adreno_device *adreno_dev = ADRENO_DEVICE(device); + int init_reftimestamp = 0x7fffffff; + + device->state = KGSL_STATE_INIT; + device->requested_state = KGSL_STATE_NONE; + + /* Power up the device */ + kgsl_pwrctrl_enable(device); + + if (kgsl_mmu_start(device)) + goto error_clk_off; + + adreno_dev->chip_id = adreno_getchipid(device); + + /*We need to make sure all blocks are powered up and clocked before + *issuing a soft reset. The overrides will then be turned off (set to 0) + */ + adreno_regwrite(device, REG_RBBM_PM_OVERRIDE1, 0xfffffffe); + if (adreno_dev->chip_id == CHIP_REV_251) + adreno_regwrite(device, REG_RBBM_PM_OVERRIDE2, 0x000000ff); + else + adreno_regwrite(device, REG_RBBM_PM_OVERRIDE2, 0xffffffff); + + /* Only reset CP block if all blocks have previously been reset */ + if (!(device->flags & KGSL_FLAGS_SOFT_RESET) || + !adreno_is_a220(adreno_dev)) { + adreno_regwrite(device, REG_RBBM_SOFT_RESET, 0xFFFFFFFF); + device->flags |= KGSL_FLAGS_SOFT_RESET; + } else + adreno_regwrite(device, REG_RBBM_SOFT_RESET, 0x00000001); + + /* The core is in an indeterminate state until the reset completes + * after 30ms. + */ + msleep(30); + + adreno_regwrite(device, REG_RBBM_SOFT_RESET, 0x00000000); + + adreno_regwrite(device, REG_RBBM_CNTL, 0x00004442); + + adreno_regwrite(device, REG_MH_ARBITER_CONFIG, + adreno_dev->mharb); + + if (!adreno_is_a220(adreno_dev)) { + adreno_regwrite(device, + REG_MH_CLNT_INTF_CTRL_CONFIG1, 0x00030f27); + adreno_regwrite(device, + REG_MH_CLNT_INTF_CTRL_CONFIG2, 0x00472747); + } + + /* Remove 1k boundary check in z470 to avoid GPU hang. + Notice that, this solution won't work if both EBI and SMI are used */ + if (adreno_is_a220(adreno_dev)) { + adreno_regwrite(device, REG_MH_CLNT_INTF_CTRL_CONFIG1, + 0x00032f07); + } + + adreno_regwrite(device, REG_SQ_VS_PROGRAM, 0x00000000); + adreno_regwrite(device, REG_SQ_PS_PROGRAM, 0x00000000); + + adreno_regwrite(device, REG_RBBM_PM_OVERRIDE1, 0); + if (!adreno_is_a220(adreno_dev)) + adreno_regwrite(device, REG_RBBM_PM_OVERRIDE2, 0); + else + adreno_regwrite(device, REG_RBBM_PM_OVERRIDE2, 0x80); + + kgsl_sharedmem_writel(&device->memstore, + KGSL_DEVICE_MEMSTORE_OFFSET(ref_wait_ts), + init_reftimestamp); + + adreno_regwrite(device, REG_RBBM_DEBUG, 0x000C0000); + + adreno_regwrite(device, REG_RBBM_INT_CNTL, GSL_RBBM_INT_MASK); + + /* make sure SQ interrupts are disabled */ + adreno_regwrite(device, REG_SQ_INT_CNTL, 0); + + if (adreno_is_a220(adreno_dev)) + adreno_dev->gmemspace.sizebytes = SZ_512K; + else + adreno_dev->gmemspace.sizebytes = SZ_256K; + adreno_gmeminit(adreno_dev); + + kgsl_pwrctrl_irq(device, KGSL_PWRFLAGS_ON); + + status = adreno_ringbuffer_start(&adreno_dev->ringbuffer, init_ram); + if (status != 0) + goto error_irq_off; + + mod_timer(&device->idle_timer, jiffies + FIRST_TIMEOUT); + return status; + +error_irq_off: + kgsl_pwrctrl_irq(device, KGSL_PWRFLAGS_OFF); +error_clk_off: + kgsl_pwrctrl_disable(device); + kgsl_mmu_stop(device); + + return status; +} + +static int adreno_stop(struct kgsl_device *device) +{ + struct adreno_device *adreno_dev = ADRENO_DEVICE(device); + del_timer(&device->idle_timer); + adreno_regwrite(device, REG_RBBM_INT_CNTL, 0); + + adreno_dev->drawctxt_active = NULL; + + adreno_ringbuffer_stop(&adreno_dev->ringbuffer); + + adreno_gmemclose(device); + + kgsl_mmu_stop(device); + + /* Disable the clocks before the power rail. */ + kgsl_pwrctrl_irq(device, KGSL_PWRFLAGS_OFF); + + /* Power down the device */ + kgsl_pwrctrl_disable(device); + + return 0; +} + +static int +adreno_recover_hang(struct kgsl_device *device) +{ + int ret; + unsigned int *rb_buffer; + struct adreno_device *adreno_dev = ADRENO_DEVICE(device); + struct adreno_ringbuffer *rb = &adreno_dev->ringbuffer; + unsigned int timestamp; + unsigned int num_rb_contents; + unsigned int bad_context; + unsigned int reftimestamp; + unsigned int enable_ts; + unsigned int soptimestamp; + unsigned int eoptimestamp; + struct adreno_context *drawctxt; + + KGSL_DRV_ERR(device, "Starting recovery from 3D GPU hang....\n"); + rb_buffer = vmalloc(rb->buffer_desc.size); + if (!rb_buffer) { + KGSL_MEM_ERR(device, + "Failed to allocate memory for recovery: %x\n", + rb->buffer_desc.size); + return -ENOMEM; + } + /* Extract valid contents from rb which can stil be executed after + * hang */ + ret = adreno_ringbuffer_extract(rb, rb_buffer, &num_rb_contents); + if (ret) + goto done; + timestamp = rb->timestamp; + KGSL_DRV_ERR(device, "Last issued timestamp: %x\n", timestamp); + kgsl_sharedmem_readl(&device->memstore, &bad_context, + KGSL_DEVICE_MEMSTORE_OFFSET(current_context)); + kgsl_sharedmem_readl(&device->memstore, &reftimestamp, + KGSL_DEVICE_MEMSTORE_OFFSET(ref_wait_ts)); + kgsl_sharedmem_readl(&device->memstore, &enable_ts, + KGSL_DEVICE_MEMSTORE_OFFSET(ts_cmp_enable)); + kgsl_sharedmem_readl(&device->memstore, &soptimestamp, + KGSL_DEVICE_MEMSTORE_OFFSET(soptimestamp)); + kgsl_sharedmem_readl(&device->memstore, &eoptimestamp, + KGSL_DEVICE_MEMSTORE_OFFSET(eoptimestamp)); + /* Make sure memory is synchronized before restarting the GPU */ + mb(); + KGSL_CTXT_ERR(device, + "Context that caused a GPU hang: %x\n", bad_context); + /* restart device */ + ret = adreno_stop(device); + if (ret) + goto done; + ret = adreno_start(device, true); + if (ret) + goto done; + KGSL_DRV_ERR(device, "Device has been restarted after hang\n"); + /* Restore timestamp states */ + kgsl_sharedmem_writel(&device->memstore, + KGSL_DEVICE_MEMSTORE_OFFSET(soptimestamp), + soptimestamp); + kgsl_sharedmem_writel(&device->memstore, + KGSL_DEVICE_MEMSTORE_OFFSET(eoptimestamp), + eoptimestamp); + kgsl_sharedmem_writel(&device->memstore, + KGSL_DEVICE_MEMSTORE_OFFSET(soptimestamp), + soptimestamp); + if (num_rb_contents) { + kgsl_sharedmem_writel(&device->memstore, + KGSL_DEVICE_MEMSTORE_OFFSET(ref_wait_ts), + reftimestamp); + kgsl_sharedmem_writel(&device->memstore, + KGSL_DEVICE_MEMSTORE_OFFSET(ts_cmp_enable), + enable_ts); + } + /* Make sure all writes are posted before the GPU reads them */ + wmb(); + /* Mark the invalid context so no more commands are accepted from + * that context */ + + drawctxt = (struct adreno_context *) bad_context; + + KGSL_CTXT_ERR(device, + "Context that caused a GPU hang: %x\n", bad_context); + + drawctxt->flags |= CTXT_FLAGS_GPU_HANG; + + /* Restore valid commands in ringbuffer */ + adreno_ringbuffer_restore(rb, rb_buffer, num_rb_contents); + rb->timestamp = timestamp; +done: + vfree(rb_buffer); + return ret; +} + +static int +adreno_dump_and_recover(struct kgsl_device *device) +{ + static int recovery; + int result = -ETIMEDOUT; + + if (device->state == KGSL_STATE_HUNG) + goto done; + if (device->state == KGSL_STATE_DUMP_AND_RECOVER && !recovery) { + mutex_unlock(&device->mutex); + wait_for_completion(&device->recovery_gate); + mutex_lock(&device->mutex); + if (!(device->state & KGSL_STATE_HUNG)) + /* recovery success */ + result = 0; + } else { + INIT_COMPLETION(device->recovery_gate); + /* Detected a hang - trigger an automatic dump */ + adreno_postmortem_dump(device, 0); + if (!recovery) { + recovery = 1; + result = adreno_recover_hang(device); + if (result) + device->state = KGSL_STATE_HUNG; + recovery = 0; + complete_all(&device->recovery_gate); + } else + KGSL_DRV_ERR(device, + "Cannot recover from another hang while " + "recovering from a hang\n"); + } +done: + return result; +} + +static int adreno_getproperty(struct kgsl_device *device, + enum kgsl_property_type type, + void *value, + unsigned int sizebytes) +{ + int status = -EINVAL; + struct adreno_device *adreno_dev = ADRENO_DEVICE(device); + + switch (type) { + case KGSL_PROP_DEVICE_INFO: + { + struct kgsl_devinfo devinfo; + + if (sizebytes != sizeof(devinfo)) { + status = -EINVAL; + break; + } + + memset(&devinfo, 0, sizeof(devinfo)); + devinfo.device_id = device->id+1; + devinfo.chip_id = adreno_dev->chip_id; + devinfo.mmu_enabled = kgsl_mmu_enabled(); + devinfo.gpu_id = adreno_get_rev(adreno_dev); + devinfo.gmem_gpubaseaddr = adreno_dev->gmemspace. + gpu_base; + devinfo.gmem_sizebytes = adreno_dev->gmemspace. + sizebytes; + + if (copy_to_user(value, &devinfo, sizeof(devinfo)) != + 0) { + status = -EFAULT; + break; + } + status = 0; + } + break; + case KGSL_PROP_DEVICE_SHADOW: + { + struct kgsl_shadowprop shadowprop; + + if (sizebytes != sizeof(shadowprop)) { + status = -EINVAL; + break; + } + memset(&shadowprop, 0, sizeof(shadowprop)); + if (device->memstore.hostptr) { + /*NOTE: with mmu enabled, gpuaddr doesn't mean + * anything to mmap(). + */ + shadowprop.gpuaddr = device->memstore.physaddr; + shadowprop.size = device->memstore.size; + /* GSL needs this to be set, even if it + appears to be meaningless */ + shadowprop.flags = KGSL_FLAGS_INITIALIZED; + } + if (copy_to_user(value, &shadowprop, + sizeof(shadowprop))) { + status = -EFAULT; + break; + } + status = 0; + } + break; + case KGSL_PROP_MMU_ENABLE: + { +#ifdef CONFIG_MSM_KGSL_MMU + int mmuProp = 1; +#else + int mmuProp = 0; +#endif + if (sizebytes != sizeof(int)) { + status = -EINVAL; + break; + } + if (copy_to_user(value, &mmuProp, sizeof(mmuProp))) { + status = -EFAULT; + break; + } + status = 0; + } + break; + case KGSL_PROP_INTERRUPT_WAITS: + { + int int_waits = 1; + if (sizebytes != sizeof(int)) { + status = -EINVAL; + break; + } + if (copy_to_user(value, &int_waits, sizeof(int))) { + status = -EFAULT; + break; + } + status = 0; + } + break; + default: + status = -EINVAL; + } + + return status; +} + +/* Caller must hold the device mutex. */ +int adreno_idle(struct kgsl_device *device, unsigned int timeout) +{ + struct adreno_device *adreno_dev = ADRENO_DEVICE(device); + struct adreno_ringbuffer *rb = &adreno_dev->ringbuffer; + unsigned int rbbm_status; + unsigned long wait_time = jiffies + MAX_WAITGPU_SECS; + + kgsl_cffdump_regpoll(device->id, REG_RBBM_STATUS << 2, + 0x00000000, 0x80000000); + /* first, wait until the CP has consumed all the commands in + * the ring buffer + */ +retry: + if (rb->flags & KGSL_FLAGS_STARTED) { + do { + GSL_RB_GET_READPTR(rb, &rb->rptr); + if (time_after(jiffies, wait_time)) { + KGSL_DRV_ERR(device, "rptr: %x, wptr: %x\n", + rb->rptr, rb->wptr); + goto err; + } + } while (rb->rptr != rb->wptr); + } + + /* now, wait for the GPU to finish its operations */ + wait_time = jiffies + MAX_WAITGPU_SECS; + while (time_before(jiffies, wait_time)) { + adreno_regread(device, REG_RBBM_STATUS, &rbbm_status); + if (rbbm_status == 0x110) + return 0; + } + +err: + KGSL_DRV_ERR(device, "spun too long waiting for RB to idle\n"); + if (!adreno_dump_and_recover(device)) { + wait_time = jiffies + MAX_WAITGPU_SECS; + goto retry; + } + return -ETIMEDOUT; +} + +static unsigned int adreno_isidle(struct kgsl_device *device) +{ + int status = false; + struct adreno_device *adreno_dev = ADRENO_DEVICE(device); + struct adreno_ringbuffer *rb = &adreno_dev->ringbuffer; + unsigned int rbbm_status; + + if (rb->flags & KGSL_FLAGS_STARTED) { + /* Is the ring buffer is empty? */ + GSL_RB_GET_READPTR(rb, &rb->rptr); + if (!device->active_cnt && (rb->rptr == rb->wptr)) { + /* Is the core idle? */ + adreno_regread(device, REG_RBBM_STATUS, + &rbbm_status); + if (rbbm_status == 0x110) + status = true; + } + } else { + KGSL_DRV_ERR(device, "ringbuffer not started\n"); + BUG(); + } + return status; +} + + +/******************************************************************/ +/* Caller must hold the driver mutex. */ +static int adreno_resume_context(struct kgsl_device *device) +{ + int status = 0; + struct adreno_device *adreno_dev = ADRENO_DEVICE(device); + + if (device->pwrctrl.suspended_ctxt != NULL) { + adreno_drawctxt_switch(adreno_dev, + device->pwrctrl.suspended_ctxt, 0); + status = adreno_idle(device, 0); + + } + + return status; +} + +/******************************************************************/ +/* Caller must hold the device mutex. */ +static int adreno_suspend_context(struct kgsl_device *device) +{ + int status = 0; + struct adreno_device *adreno_dev = ADRENO_DEVICE(device); + + /* save ctxt ptr and switch to NULL ctxt */ + device->pwrctrl.suspended_ctxt = adreno_dev->drawctxt_active; + if (device->pwrctrl.suspended_ctxt != NULL) { + adreno_drawctxt_switch(adreno_dev, NULL, 0); + status = adreno_idle(device, KGSL_TIMEOUT_DEFAULT); + } + + return status; +} + +uint8_t *kgsl_sharedmem_convertaddr(struct kgsl_device *device, + unsigned int pt_base, unsigned int gpuaddr, unsigned int *size) +{ + uint8_t *result = NULL; + struct kgsl_mem_entry *entry; + struct kgsl_process_private *priv; + struct adreno_device *adreno_dev = ADRENO_DEVICE(device); + struct adreno_ringbuffer *ringbuffer = &adreno_dev->ringbuffer; + + if (kgsl_gpuaddr_in_memdesc(&ringbuffer->buffer_desc, gpuaddr)) { + return kgsl_gpuaddr_to_vaddr(&ringbuffer->buffer_desc, + gpuaddr, size); + } + + if (kgsl_gpuaddr_in_memdesc(&ringbuffer->memptrs_desc, gpuaddr)) { + return kgsl_gpuaddr_to_vaddr(&ringbuffer->memptrs_desc, + gpuaddr, size); + } + + if (kgsl_gpuaddr_in_memdesc(&device->memstore, gpuaddr)) { + return kgsl_gpuaddr_to_vaddr(&device->memstore, + gpuaddr, size); + } + + mutex_lock(&kgsl_driver.process_mutex); + list_for_each_entry(priv, &kgsl_driver.process_list, list) { + if (pt_base != 0 + && priv->pagetable + && priv->pagetable->base.gpuaddr != pt_base) { + continue; + } + + spin_lock(&priv->mem_lock); + entry = kgsl_sharedmem_find_region(priv, gpuaddr, + sizeof(unsigned int)); + if (entry) { + result = kgsl_gpuaddr_to_vaddr(&entry->memdesc, + gpuaddr, size); + spin_unlock(&priv->mem_lock); + mutex_unlock(&kgsl_driver.process_mutex); + return result; + } + spin_unlock(&priv->mem_lock); + } + mutex_unlock(&kgsl_driver.process_mutex); + + BUG_ON(!mutex_is_locked(&device->mutex)); + list_for_each_entry(entry, &device->memqueue, list) { + if (kgsl_gpuaddr_in_memdesc(&entry->memdesc, gpuaddr)) { + result = kgsl_gpuaddr_to_vaddr(&entry->memdesc, + gpuaddr, size); + break; + } + + } + return result; +} + +static void _adreno_regread(struct kgsl_device *device, + unsigned int offsetwords, + unsigned int *value) +{ + unsigned int *reg; + BUG_ON(offsetwords*sizeof(uint32_t) >= device->regspace.sizebytes); + reg = (unsigned int *)(device->regspace.mmio_virt_base + + (offsetwords << 2)); + /*ensure this read finishes before the next one. + * i.e. act like normal readl() */ + *value = __raw_readl(reg); + rmb(); +} + +void adreno_regread(struct kgsl_device *device, unsigned int offsetwords, + unsigned int *value) +{ + kgsl_pre_hwaccess(device); + _adreno_regread(device, offsetwords, value); +} + +void adreno_regread_isr(struct kgsl_device *device, + unsigned int offsetwords, + unsigned int *value) +{ + _adreno_regread(device, offsetwords, value); +} + +static void _adreno_regwrite(struct kgsl_device *device, + unsigned int offsetwords, + unsigned int value) +{ + unsigned int *reg; + + BUG_ON(offsetwords*sizeof(uint32_t) >= device->regspace.sizebytes); + + kgsl_cffdump_regwrite(device->id, offsetwords << 2, value); + reg = (unsigned int *)(device->regspace.mmio_virt_base + + (offsetwords << 2)); + + /*ensure previous writes post before this one, + * i.e. act like normal writel() */ + wmb(); + __raw_writel(value, reg); +} + +void adreno_regwrite(struct kgsl_device *device, unsigned int offsetwords, + unsigned int value) +{ + kgsl_pre_hwaccess(device); + _adreno_regwrite(device, offsetwords, value); +} + +void adreno_regwrite_isr(struct kgsl_device *device, + unsigned int offsetwords, + unsigned int value) +{ + _adreno_regwrite(device, offsetwords, value); +} + +static int kgsl_check_interrupt_timestamp(struct kgsl_device *device, + unsigned int timestamp) +{ + int status; + unsigned int ref_ts, enableflag; + + status = kgsl_check_timestamp(device, timestamp); + if (!status) { + mutex_lock(&device->mutex); + kgsl_sharedmem_readl(&device->memstore, &enableflag, + KGSL_DEVICE_MEMSTORE_OFFSET(ts_cmp_enable)); + mb(); + + if (enableflag) { + kgsl_sharedmem_readl(&device->memstore, &ref_ts, + KGSL_DEVICE_MEMSTORE_OFFSET(ref_wait_ts)); + mb(); + if (timestamp_cmp(ref_ts, timestamp)) { + kgsl_sharedmem_writel(&device->memstore, + KGSL_DEVICE_MEMSTORE_OFFSET(ref_wait_ts), + timestamp); + wmb(); + } + } else { + unsigned int cmds[2]; + kgsl_sharedmem_writel(&device->memstore, + KGSL_DEVICE_MEMSTORE_OFFSET(ref_wait_ts), + timestamp); + enableflag = 1; + kgsl_sharedmem_writel(&device->memstore, + KGSL_DEVICE_MEMSTORE_OFFSET(ts_cmp_enable), + enableflag); + wmb(); + /* submit a dummy packet so that even if all + * commands upto timestamp get executed we will still + * get an interrupt */ + cmds[0] = pm4_type3_packet(PM4_NOP, 1); + cmds[1] = 0; + adreno_ringbuffer_issuecmds(device, 0, &cmds[0], 2); + } + mutex_unlock(&device->mutex); + } + + return status; +} + +/* + wait_io_event_interruptible_timeout checks for the exit condition before + placing a process in wait q. For conditional interrupts we expect the + process to already be in its wait q when its exit condition checking + function is called. +*/ +#define kgsl_wait_io_event_interruptible_timeout(wq, condition, timeout)\ +({ \ + long __ret = timeout; \ + __wait_io_event_interruptible_timeout(wq, condition, __ret); \ + __ret; \ +}) + +/* MUST be called with the device mutex held */ +static int adreno_waittimestamp(struct kgsl_device *device, + unsigned int timestamp, + unsigned int msecs) +{ + long status = 0; + struct adreno_device *adreno_dev = ADRENO_DEVICE(device); + + if (timestamp != adreno_dev->ringbuffer.timestamp && + timestamp_cmp(timestamp, + adreno_dev->ringbuffer.timestamp)) { + KGSL_DRV_ERR(device, "Cannot wait for invalid ts: %x, " + "rb->timestamp: %x\n", + timestamp, adreno_dev->ringbuffer.timestamp); + status = -EINVAL; + goto done; + } + if (!kgsl_check_timestamp(device, timestamp)) { + mutex_unlock(&device->mutex); + /* We need to make sure that the process is placed in wait-q + * before its condition is called */ + status = kgsl_wait_io_event_interruptible_timeout( + device->wait_queue, + kgsl_check_interrupt_timestamp(device, + timestamp), msecs_to_jiffies(msecs)); + mutex_lock(&device->mutex); + + if (status > 0) + status = 0; + else if (status == 0) { + if (!kgsl_check_timestamp(device, timestamp)) { + status = -ETIMEDOUT; + KGSL_DRV_ERR(device, + "Device hang detected while waiting " + "for timestamp: %x, last " + "submitted(rb->timestamp): %x, wptr: " + "%x\n", timestamp, + adreno_dev->ringbuffer.timestamp, + adreno_dev->ringbuffer.wptr); + if (!adreno_dump_and_recover(device)) { + /* wait for idle after recovery as the + * timestamp that this process wanted + * to wait on may be invalid */ + if (!adreno_idle(device, + KGSL_TIMEOUT_DEFAULT)) + status = 0; + } + } + } + } + +done: + return (int)status; +} + +static unsigned int adreno_readtimestamp(struct kgsl_device *device, + enum kgsl_timestamp_type type) +{ + unsigned int timestamp = 0; + + if (type == KGSL_TIMESTAMP_CONSUMED) + adreno_regread(device, REG_CP_TIMESTAMP, ×tamp); + else if (type == KGSL_TIMESTAMP_RETIRED) + kgsl_sharedmem_readl(&device->memstore, ×tamp, + KGSL_DEVICE_MEMSTORE_OFFSET(eoptimestamp)); + rmb(); + + return timestamp; +} + +static long adreno_ioctl(struct kgsl_device_private *dev_priv, + unsigned int cmd, void *data) +{ + int result = 0; + struct kgsl_drawctxt_set_bin_base_offset *binbase; + struct kgsl_context *context; + + switch (cmd) { + case IOCTL_KGSL_DRAWCTXT_SET_BIN_BASE_OFFSET: + binbase = data; + + context = kgsl_find_context(dev_priv, binbase->drawctxt_id); + if (context) { + result = adreno_drawctxt_set_bin_base_offset( + dev_priv->device, + context, + binbase->offset); + } else { + result = -EINVAL; + KGSL_DRV_ERR(dev_priv->device, + "invalid drawctxt drawctxt_id %d " + "device_id=%d\n", + binbase->drawctxt_id, dev_priv->device->id); + } + break; + + default: + KGSL_DRV_INFO(dev_priv->device, + "invalid ioctl code %08x\n", cmd); + result = -EINVAL; + break; + } + return result; + +} + +static inline s64 adreno_ticks_to_us(u32 ticks, u32 gpu_freq) +{ + gpu_freq /= 1000000; + return ticks / gpu_freq; +} + +static void adreno_power_stats(struct kgsl_device *device, + struct kgsl_power_stats *stats) +{ + unsigned int reg; + struct kgsl_pwrctrl *pwr = &device->pwrctrl; + + /* In order to calculate idle you have to have run the algorithm * + * at least once to get a start time. */ + if (pwr->time != 0) { + s64 tmp; + /* Stop the performance moniter and read the current * + * busy cycles. */ + adreno_regwrite(device, + REG_CP_PERFMON_CNTL, + REG_PERF_MODE_CNT | + REG_PERF_STATE_FREEZE); + adreno_regread(device, REG_RBBM_PERFCOUNTER1_LO, ®); + tmp = ktime_to_us(ktime_get()); + stats->total_time = tmp - pwr->time; + pwr->time = tmp; + stats->busy_time = adreno_ticks_to_us(reg, device->pwrctrl. + pwrlevels[device->pwrctrl.active_pwrlevel]. + gpu_freq); + + adreno_regwrite(device, + REG_CP_PERFMON_CNTL, + REG_PERF_MODE_CNT | + REG_PERF_STATE_RESET); + } else { + stats->total_time = 0; + stats->busy_time = 0; + pwr->time = ktime_to_us(ktime_get()); + } + + /* re-enable the performance moniters */ + adreno_regread(device, REG_RBBM_PM_OVERRIDE2, ®); + adreno_regwrite(device, REG_RBBM_PM_OVERRIDE2, (reg | 0x40)); + adreno_regwrite(device, REG_RBBM_PERFCOUNTER1_SELECT, 0x1); + adreno_regwrite(device, + REG_CP_PERFMON_CNTL, + REG_PERF_MODE_CNT | REG_PERF_STATE_ENABLE); +} + +static void __devinit adreno_getfunctable(struct kgsl_functable *ftbl) +{ + if (ftbl == NULL) + return; + ftbl->device_regread = adreno_regread; + ftbl->device_regwrite = adreno_regwrite; + ftbl->device_regread_isr = adreno_regread_isr; + ftbl->device_regwrite_isr = adreno_regwrite_isr; + ftbl->device_setstate = adreno_setstate; + ftbl->device_idle = adreno_idle; + ftbl->device_isidle = adreno_isidle; + ftbl->device_suspend_context = adreno_suspend_context; + ftbl->device_resume_context = adreno_resume_context; + ftbl->device_start = adreno_start; + ftbl->device_stop = adreno_stop; + ftbl->device_getproperty = adreno_getproperty; + ftbl->device_waittimestamp = adreno_waittimestamp; + ftbl->device_readtimestamp = adreno_readtimestamp; + ftbl->device_issueibcmds = adreno_ringbuffer_issueibcmds; + ftbl->device_drawctxt_create = adreno_drawctxt_create; + ftbl->device_drawctxt_destroy = adreno_drawctxt_destroy; + ftbl->device_ioctl = adreno_ioctl; + ftbl->device_setup_pt = adreno_setup_pt; + ftbl->device_cleanup_pt = adreno_cleanup_pt; + ftbl->device_power_stats = adreno_power_stats; +} + +static struct platform_device_id adreno_id_table[] = { + { DEVICE_3D0_NAME, (kernel_ulong_t)&device_3d0.dev, }, + { }, +}; +MODULE_DEVICE_TABLE(platform, adreno_id_table); + +static struct platform_driver adreno_platform_driver = { + .probe = adreno_probe, + .remove = __devexit_p(adreno_remove), + .suspend = kgsl_suspend_driver, + .resume = kgsl_resume_driver, + .id_table = adreno_id_table, + .driver = { + .owner = THIS_MODULE, + .name = DEVICE_3D_NAME, + .pm = &kgsl_pm_ops, + } +}; + +static int __init kgsl_3d_init(void) +{ + return platform_driver_register(&adreno_platform_driver); +} + +static void __exit kgsl_3d_exit(void) +{ + platform_driver_unregister(&adreno_platform_driver); +} + +module_init(kgsl_3d_init); +module_exit(kgsl_3d_exit); + +MODULE_DESCRIPTION("3D Graphics driver"); +MODULE_VERSION("1.2"); +MODULE_LICENSE("GPL v2"); +MODULE_ALIAS("platform:kgsl_3d"); diff --git a/drivers/gpu/msm/adreno.h b/drivers/gpu/msm/adreno.h new file mode 100644 index 00000000..74b3a845 --- /dev/null +++ b/drivers/gpu/msm/adreno.h @@ -0,0 +1,121 @@ +/* Copyright (c) 2008-2011, Code Aurora Forum. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are + * met: + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following + * disclaimer in the documentation and/or other materials provided + * with the distribution. + * * Neither the name of Code Aurora Forum, Inc. nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED "AS IS" AND ANY EXPRESS OR IMPLIED + * WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS + * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR + * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, + * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE + * OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN + * IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + */ +#ifndef __ADRENO_H +#define __ADRENO_H + +#include "adreno_drawctxt.h" +#include "adreno_ringbuffer.h" + +#define DEVICE_3D_NAME "kgsl-3d" +#define DEVICE_3D0_NAME "kgsl-3d0" + +#define ADRENO_DEVICE(device) \ + KGSL_CONTAINER_OF(device, struct adreno_device, dev) + +/* Flags to control command packet settings */ +#define KGSL_CMD_FLAGS_PMODE 0x00000001 +#define KGSL_CMD_FLAGS_NO_TS_CMP 0x00000002 +#define KGSL_CMD_FLAGS_NOT_KERNEL_CMD 0x00000004 + +/* Command identifiers */ +#define KGSL_CONTEXT_TO_MEM_IDENTIFIER 0xDEADBEEF +#define KGSL_CMD_IDENTIFIER 0xFEEDFACE + +struct adreno_device { + struct kgsl_device dev; /* Must be first field in this struct */ + unsigned int chip_id; + struct kgsl_memregion gmemspace; + struct adreno_context *drawctxt_active; + wait_queue_head_t ib1_wq; + unsigned int *pfp_fw; + size_t pfp_fw_size; + unsigned int *pm4_fw; + size_t pm4_fw_size; + struct adreno_ringbuffer ringbuffer; + unsigned int mharb; +}; + +int adreno_idle(struct kgsl_device *device, unsigned int timeout); +void adreno_regread(struct kgsl_device *device, unsigned int offsetwords, + unsigned int *value); +void adreno_regwrite(struct kgsl_device *device, unsigned int offsetwords, + unsigned int value); +void adreno_regread_isr(struct kgsl_device *device, + unsigned int offsetwords, + unsigned int *value); +void adreno_regwrite_isr(struct kgsl_device *device, + unsigned int offsetwords, + unsigned int value); + +uint8_t *kgsl_sharedmem_convertaddr(struct kgsl_device *device, + unsigned int pt_base, unsigned int gpuaddr, unsigned int *size); + +enum adreno_gpurev { + ADRENO_REV_UNKNOWN = 0, + ADRENO_REV_A200 = 200, + ADRENO_REV_A205 = 205, + ADRENO_REV_A220 = 220, + ADRENO_REV_A225 = 225, +}; + +enum adreno_gpurev adreno_get_rev(struct adreno_device *adreno_dev); + +static inline int adreno_is_a200(struct adreno_device *adreno_dev) +{ + return (adreno_get_rev(adreno_dev) == ADRENO_REV_A200); +} + +static inline int adreno_is_a205(struct adreno_device *adreno_dev) +{ + return (adreno_get_rev(adreno_dev) == ADRENO_REV_A200); +} + +static inline int adreno_is_a20x(struct adreno_device *adreno_dev) +{ + enum adreno_gpurev rev = adreno_get_rev(adreno_dev); + return (rev == ADRENO_REV_A200 || rev == ADRENO_REV_A205); +} + +static inline int adreno_is_a220(struct adreno_device *adreno_dev) +{ + return (adreno_get_rev(adreno_dev) == ADRENO_REV_A220); +} + +static inline int adreno_is_a225(struct adreno_device *adreno_dev) +{ + return (adreno_get_rev(adreno_dev) == ADRENO_REV_A225); +} + +static inline int adreno_is_a22x(struct adreno_device *adreno_dev) +{ + enum adreno_gpurev rev = adreno_get_rev(adreno_dev); + return (rev == ADRENO_REV_A220 || rev == ADRENO_REV_A225); +} + +#endif /*__ADRENO_H */ diff --git a/drivers/gpu/msm/adreno_debugfs.c b/drivers/gpu/msm/adreno_debugfs.c new file mode 100644 index 00000000..487e1f7f --- /dev/null +++ b/drivers/gpu/msm/adreno_debugfs.c @@ -0,0 +1,450 @@ +/* Copyright (c) 2002,2008-2011, Code Aurora Forum. All rights reserved. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 and + * only version 2 as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + */ + +#include +#include +#include + +#include "kgsl.h" +#include "adreno_postmortem.h" +#include "adreno.h" + +#include "a200_reg.h" + +unsigned int kgsl_cff_dump_enable; +int kgsl_pm_regs_enabled; + +static uint32_t kgsl_ib_base; +static uint32_t kgsl_ib_size; + +static struct dentry *pm_d_debugfs; + +static int pm_dump_set(void *data, u64 val) +{ + struct kgsl_device *device = data; + + if (val) { + mutex_lock(&device->mutex); + adreno_postmortem_dump(device, 1); + mutex_unlock(&device->mutex); + } + + return 0; +} + +DEFINE_SIMPLE_ATTRIBUTE(pm_dump_fops, + NULL, + pm_dump_set, "%llu\n"); + +static int pm_regs_enabled_set(void *data, u64 val) +{ + kgsl_pm_regs_enabled = val ? 1 : 0; + return 0; +} + +static int pm_regs_enabled_get(void *data, u64 *val) +{ + *val = kgsl_pm_regs_enabled; + return 0; +} + +DEFINE_SIMPLE_ATTRIBUTE(pm_regs_enabled_fops, + pm_regs_enabled_get, + pm_regs_enabled_set, "%llu\n"); + + +static int kgsl_cff_dump_enable_set(void *data, u64 val) +{ +#ifdef CONFIG_MSM_KGSL_CFF_DUMP + kgsl_cff_dump_enable = (val != 0); + return 0; +#else + return -EINVAL; +#endif +} + +static int kgsl_cff_dump_enable_get(void *data, u64 *val) +{ + *val = kgsl_cff_dump_enable; + return 0; +} + +DEFINE_SIMPLE_ATTRIBUTE(kgsl_cff_dump_enable_fops, kgsl_cff_dump_enable_get, + kgsl_cff_dump_enable_set, "%llu\n"); + +static int kgsl_dbgfs_open(struct inode *inode, struct file *file) +{ + file->f_mode &= ~(FMODE_PREAD | FMODE_PWRITE); + file->private_data = inode->i_private; + return 0; +} + +static int kgsl_dbgfs_release(struct inode *inode, struct file *file) +{ + return 0; +} + +static int kgsl_hex_dump(const char *prefix, int c, uint8_t *data, + int rowc, int linec, char __user *buff) +{ + int ss; + /* Prefix of 20 chars max, 32 bytes per row, in groups of four - that's + * 8 groups at 8 chars per group plus a space, plus new-line, plus + * ending character */ + char linebuf[20 + 64 + 1 + 1]; + + ss = snprintf(linebuf, sizeof(linebuf), prefix, c); + hex_dump_to_buffer(data, linec, rowc, 4, linebuf+ss, + sizeof(linebuf)-ss, 0); + strlcat(linebuf, "\n", sizeof(linebuf)); + linebuf[sizeof(linebuf)-1] = 0; + ss = strlen(linebuf); + if (copy_to_user(buff, linebuf, ss+1)) + return -EFAULT; + return ss; +} + +static ssize_t kgsl_ib_dump_read( + struct file *file, + char __user *buff, + size_t buff_count, + loff_t *ppos) +{ + int i, count = kgsl_ib_size, remaining, pos = 0, tot = 0, ss; + struct kgsl_device *device = file->private_data; + const int rowc = 32; + unsigned int pt_base, ib_memsize; + uint8_t *base_addr; + char linebuf[80]; + + if (!ppos || !device || !kgsl_ib_base) + return 0; + + kgsl_regread(device, REG_MH_MMU_PT_BASE, &pt_base); + base_addr = kgsl_sharedmem_convertaddr(device, pt_base, kgsl_ib_base, + &ib_memsize); + + if (!base_addr) + return 0; + + pr_info("%s ppos=%ld, buff_count=%d, count=%d\n", __func__, (long)*ppos, + buff_count, count); + ss = snprintf(linebuf, sizeof(linebuf), "IB: base=%08x(%08x" + "), size=%d, memsize=%d\n", kgsl_ib_base, + (uint32_t)base_addr, kgsl_ib_size, ib_memsize); + if (*ppos == 0) { + if (copy_to_user(buff, linebuf, ss+1)) + return -EFAULT; + tot += ss; + buff += ss; + *ppos += ss; + } + pos += ss; + remaining = count; + for (i = 0; i < count; i += rowc) { + int linec = min(remaining, rowc); + + remaining -= rowc; + ss = kgsl_hex_dump("IB: %05x: ", i, base_addr, rowc, linec, + buff); + if (ss < 0) + return ss; + + if (pos >= *ppos) { + if (tot+ss >= buff_count) { + ss = copy_to_user(buff, "", 1); + return tot; + } + tot += ss; + buff += ss; + *ppos += ss; + } + pos += ss; + base_addr += linec; + } + + return tot; +} + +static ssize_t kgsl_ib_dump_write( + struct file *file, + const char __user *buff, + size_t count, + loff_t *ppos) +{ + char local_buff[64]; + + if (count >= sizeof(local_buff)) + return -EFAULT; + + if (copy_from_user(local_buff, buff, count)) + return -EFAULT; + + local_buff[count] = 0; /* end of string */ + sscanf(local_buff, "%x %d", &kgsl_ib_base, &kgsl_ib_size); + + pr_info("%s: base=%08X size=%d\n", __func__, kgsl_ib_base, + kgsl_ib_size); + + return count; +} + +static const struct file_operations kgsl_ib_dump_fops = { + .open = kgsl_dbgfs_open, + .release = kgsl_dbgfs_release, + .read = kgsl_ib_dump_read, + .write = kgsl_ib_dump_write, +}; + +static int kgsl_regread_nolock(struct kgsl_device *device, + unsigned int offsetwords, unsigned int *value) +{ + unsigned int *reg; + + if (offsetwords*sizeof(uint32_t) >= device->regspace.sizebytes) { + KGSL_DRV_ERR(device, "invalid offset %d\n", offsetwords); + return -ERANGE; + } + + reg = (unsigned int *)(device->regspace.mmio_virt_base + + (offsetwords << 2)); + *value = __raw_readl(reg); + return 0; +} + +#define KGSL_ISTORE_START 0x5000 +#define KGSL_ISTORE_LENGTH 0x600 +static ssize_t kgsl_istore_read( + struct file *file, + char __user *buff, + size_t buff_count, + loff_t *ppos) +{ + int i, count = KGSL_ISTORE_LENGTH, remaining, pos = 0, tot = 0; + struct kgsl_device *device = file->private_data; + const int rowc = 8; + + if (!ppos || !device) + return 0; + + remaining = count; + for (i = 0; i < count; i += rowc) { + unsigned int vals[rowc]; + int j, ss; + int linec = min(remaining, rowc); + remaining -= rowc; + + if (pos >= *ppos) { + for (j = 0; j < linec; ++j) + kgsl_regread_nolock(device, + KGSL_ISTORE_START+i+j, vals+j); + } else + memset(vals, 0, sizeof(vals)); + + ss = kgsl_hex_dump("IS: %04x: ", i, (uint8_t *)vals, rowc*4, + linec*4, buff); + if (ss < 0) + return ss; + + if (pos >= *ppos) { + if (tot+ss >= buff_count) + return tot; + tot += ss; + buff += ss; + *ppos += ss; + } + pos += ss; + } + + return tot; +} + +static const struct file_operations kgsl_istore_fops = { + .open = kgsl_dbgfs_open, + .release = kgsl_dbgfs_release, + .read = kgsl_istore_read, + .llseek = default_llseek, +}; + +typedef void (*reg_read_init_t)(struct kgsl_device *device); +typedef void (*reg_read_fill_t)(struct kgsl_device *device, int i, + unsigned int *vals, int linec); +static ssize_t kgsl_reg_read(struct kgsl_device *device, int count, + reg_read_init_t reg_read_init, + reg_read_fill_t reg_read_fill, const char *prefix, char __user *buff, + loff_t *ppos) +{ + int i, remaining; + const int rowc = 8; + + if (!ppos || *ppos || !device) + return 0; + + mutex_lock(&device->mutex); + reg_read_init(device); + remaining = count; + for (i = 0; i < count; i += rowc) { + unsigned int vals[rowc]; + int ss; + int linec = min(remaining, rowc); + remaining -= rowc; + + reg_read_fill(device, i, vals, linec); + ss = kgsl_hex_dump(prefix, i, (uint8_t *)vals, rowc*4, linec*4, + buff); + if (ss < 0) { + mutex_unlock(&device->mutex); + return ss; + } + buff += ss; + *ppos += ss; + } + mutex_unlock(&device->mutex); + + return *ppos; +} + + +static void kgsl_sx_reg_read_init(struct kgsl_device *device) +{ + kgsl_regwrite(device, REG_RBBM_PM_OVERRIDE2, 0xFF); + kgsl_regwrite(device, REG_RBBM_DEBUG_CNTL, 0); +} + +static void kgsl_sx_reg_read_fill(struct kgsl_device *device, int i, + unsigned int *vals, int linec) +{ + int j; + + for (j = 0; j < linec; ++j) { + kgsl_regwrite(device, REG_RBBM_DEBUG_CNTL, 0x1B00 | i); + kgsl_regread(device, REG_RBBM_DEBUG_OUT, vals+j); + } +} + +static ssize_t kgsl_sx_debug_read( + struct file *file, + char __user *buff, + size_t buff_count, + loff_t *ppos) +{ + struct kgsl_device *device = file->private_data; + return kgsl_reg_read(device, 0x1B, kgsl_sx_reg_read_init, + kgsl_sx_reg_read_fill, "SX: %02x: ", buff, ppos); +} + +static const struct file_operations kgsl_sx_debug_fops = { + .open = kgsl_dbgfs_open, + .release = kgsl_dbgfs_release, + .read = kgsl_sx_debug_read, +}; + +static void kgsl_cp_reg_read_init(struct kgsl_device *device) +{ + kgsl_regwrite(device, REG_RBBM_DEBUG_CNTL, 0); +} + +static void kgsl_cp_reg_read_fill(struct kgsl_device *device, int i, + unsigned int *vals, int linec) +{ + int j; + + for (j = 0; j < linec; ++j) { + kgsl_regwrite(device, REG_RBBM_DEBUG_CNTL, 0x1628); + kgsl_regread(device, REG_RBBM_DEBUG_OUT, vals+j); + msleep(100); + } +} + +static ssize_t kgsl_cp_debug_read( + struct file *file, + char __user *buff, + size_t buff_count, + loff_t *ppos) +{ + struct kgsl_device *device = file->private_data; + return kgsl_reg_read(device, 20, kgsl_cp_reg_read_init, + kgsl_cp_reg_read_fill, + "CP: %02x: ", buff, ppos); +} + +static const struct file_operations kgsl_cp_debug_fops = { + .open = kgsl_dbgfs_open, + .release = kgsl_dbgfs_release, + .read = kgsl_cp_debug_read, +}; + +static void kgsl_mh_reg_read_init(struct kgsl_device *device) +{ + kgsl_regwrite(device, REG_RBBM_DEBUG_CNTL, 0); +} + +static void kgsl_mh_reg_read_fill(struct kgsl_device *device, int i, + unsigned int *vals, int linec) +{ + int j; + + for (j = 0; j < linec; ++j) { + kgsl_regwrite(device, REG_MH_DEBUG_CTRL, i+j); + kgsl_regread(device, REG_MH_DEBUG_DATA, vals+j); + } +} + +static ssize_t kgsl_mh_debug_read( + struct file *file, + char __user *buff, + size_t buff_count, + loff_t *ppos) +{ + struct kgsl_device *device = file->private_data; + return kgsl_reg_read(device, 0x40, kgsl_mh_reg_read_init, + kgsl_mh_reg_read_fill, + "MH: %02x: ", buff, ppos); +} + +static const struct file_operations kgsl_mh_debug_fops = { + .open = kgsl_dbgfs_open, + .release = kgsl_dbgfs_release, + .read = kgsl_mh_debug_read, +}; + +void adreno_debugfs_init(struct kgsl_device *device) +{ + if (!device->d_debugfs || IS_ERR(device->d_debugfs)) + return; + + debugfs_create_file("ib_dump", 0600, device->d_debugfs, device, + &kgsl_ib_dump_fops); + debugfs_create_file("istore", 0400, device->d_debugfs, device, + &kgsl_istore_fops); + debugfs_create_file("sx_debug", 0400, device->d_debugfs, device, + &kgsl_sx_debug_fops); + debugfs_create_file("cp_debug", 0400, device->d_debugfs, device, + &kgsl_cp_debug_fops); + debugfs_create_file("mh_debug", 0400, device->d_debugfs, device, + &kgsl_mh_debug_fops); + debugfs_create_file("cff_dump", 0644, device->d_debugfs, device, + &kgsl_cff_dump_enable_fops); + + /* Create post mortem control files */ + + pm_d_debugfs = debugfs_create_dir("postmortem", device->d_debugfs); + + if (IS_ERR(pm_d_debugfs)) + return; + + debugfs_create_file("dump", 0600, pm_d_debugfs, device, + &pm_dump_fops); + debugfs_create_file("regs_enabled", 0644, pm_d_debugfs, device, + &pm_regs_enabled_fops); +} diff --git a/drivers/gpu/msm/adreno_debugfs.h b/drivers/gpu/msm/adreno_debugfs.h new file mode 100644 index 00000000..680eb849 --- /dev/null +++ b/drivers/gpu/msm/adreno_debugfs.h @@ -0,0 +1,56 @@ +/* Copyright (c) 2002,2008-2011, Code Aurora Forum. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are + * met: + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following + * disclaimer in the documentation and/or other materials provided + * with the distribution. + * * Neither the name of Code Aurora Forum, Inc. nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED "AS IS" AND ANY EXPRESS OR IMPLIED + * WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS + * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR + * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, + * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE + * OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN + * IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + */ +#ifndef __ADRENO_DEBUGFS_H +#define __ADRENO_DEBUGFS_H + +#ifdef CONFIG_DEBUG_FS + +int adreno_debugfs_init(struct kgsl_device *device); + +extern int kgsl_pm_regs_enabled; + +static inline int kgsl_pmregs_enabled(void) +{ + return kgsl_pm_regs_enabled; +} + +#else +static inline int adreno_debugfs_init(struct kgsl_device *device) +{ + return 0; +} + +static inline int kgsl_pmregs_enabled(void) +{ + /* If debugfs is turned off, then always print registers */ + return 1; +} +#endif + +#endif /* __ADRENO_DEBUGFS_H */ diff --git a/drivers/gpu/msm/adreno_drawctxt.c b/drivers/gpu/msm/adreno_drawctxt.c new file mode 100644 index 00000000..ab28b0d2 --- /dev/null +++ b/drivers/gpu/msm/adreno_drawctxt.c @@ -0,0 +1,1681 @@ +/* Copyright (c) 2002,2007-2011, Code Aurora Forum. All rights reserved. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 and + * only version 2 as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + */ +#include + +#include "kgsl.h" + +#include "adreno.h" +#include "adreno_pm4types.h" +#include "adreno_drawctxt.h" + +/* + * + * Memory Map for Register, Constant & Instruction Shadow, and Command Buffers + * (34.5KB) + * + * +---------------------+------------+-------------+---+---------------------+ + * | ALU Constant Shadow | Reg Shadow | C&V Buffers |Tex| Shader Instr Shadow | + * +---------------------+------------+-------------+---+---------------------+ + * ________________________________/ \____________________ + * / | + * +--------------+-----------+------+-----------+------------------------+ + * | Restore Regs | Save Regs | Quad | Gmem Save | Gmem Restore | unused | + * +--------------+-----------+------+-----------+------------------------+ + * + * 8K - ALU Constant Shadow (8K aligned) + * 4K - H/W Register Shadow (8K aligned) + * 4K - Command and Vertex Buffers + * - Indirect command buffer : Const/Reg restore + * - includes Loop & Bool const shadows + * - Indirect command buffer : Const/Reg save + * - Quad vertices & texture coordinates + * - Indirect command buffer : Gmem save + * - Indirect command buffer : Gmem restore + * - Unused (padding to 8KB boundary) + * <1K - Texture Constant Shadow (768 bytes) (8K aligned) + * 18K - Shader Instruction Shadow + * - 6K vertex (32 byte aligned) + * - 6K pixel (32 byte aligned) + * - 6K shared (32 byte aligned) + * + * Note: Reading constants into a shadow, one at a time using REG_TO_MEM, takes + * 3 DWORDS per DWORD transfered, plus 1 DWORD for the shadow, for a total of + * 16 bytes per constant. If the texture constants were transfered this way, + * the Command & Vertex Buffers section would extend past the 16K boundary. + * By moving the texture constant shadow area to start at 16KB boundary, we + * only require approximately 40 bytes more memory, but are able to use the + * LOAD_CONSTANT_CONTEXT shadowing feature for the textures, speeding up + * context switching. + * + * [Using LOAD_CONSTANT_CONTEXT shadowing feature for the Loop and/or Bool + * constants would require an additional 8KB each, for alignment.] + * + */ + +/* Constants */ + +#define ALU_CONSTANTS 2048 /* DWORDS */ +#define NUM_REGISTERS 1024 /* DWORDS */ +#ifdef CONFIG_MSM_KGSL_DISABLE_SHADOW_WRITES +#define CMD_BUFFER_LEN 9216 /* DWORDS */ +#else +#define CMD_BUFFER_LEN 3072 /* DWORDS */ +#endif +#define TEX_CONSTANTS (32*6) /* DWORDS */ +#define BOOL_CONSTANTS 8 /* DWORDS */ +#define LOOP_CONSTANTS 56 /* DWORDS */ +#define SHADER_INSTRUCT_LOG2 9U /* 2^n == SHADER_INSTRUCTIONS */ + +#if defined(PM4_IM_STORE) +/* 96-bit instructions */ +#define SHADER_INSTRUCT (1<= 0x10000) { + exp += 16; + u >>= 16; + } + if (u >= 0x100) { + exp += 8; + u >>= 8; + } + if (u >= 0x10) { + exp += 4; + u >>= 4; + } + if (u >= 0x4) { + exp += 2; + u >>= 2; + } + if (u >= 0x2) { + exp += 1; + u >>= 1; + } + + /* Calculate fraction */ + if (23 > exp) + frac = (uintval & (~(1 << exp))) << (23 - exp); + + /* Exp is biased by 127 and shifted 23 bits */ + exp = (exp + 127) << 23; + + return exp | frac; +} + +/* context save (gmem -> sys) */ + +/* pre-compiled vertex shader program +* +* attribute vec4 P; +* void main(void) +* { +* gl_Position = P; +* } +*/ +#define GMEM2SYS_VTX_PGM_LEN 0x12 + +static unsigned int gmem2sys_vtx_pgm[GMEM2SYS_VTX_PGM_LEN] = { + 0x00011003, 0x00001000, 0xc2000000, + 0x00001004, 0x00001000, 0xc4000000, + 0x00001005, 0x00002000, 0x00000000, + 0x1cb81000, 0x00398a88, 0x00000003, + 0x140f803e, 0x00000000, 0xe2010100, + 0x14000000, 0x00000000, 0xe2000000 +}; + +/* pre-compiled fragment shader program +* +* precision highp float; +* uniform vec4 clear_color; +* void main(void) +* { +* gl_FragColor = clear_color; +* } +*/ + +#define GMEM2SYS_FRAG_PGM_LEN 0x0c + +static unsigned int gmem2sys_frag_pgm[GMEM2SYS_FRAG_PGM_LEN] = { + 0x00000000, 0x1002c400, 0x10000000, + 0x00001003, 0x00002000, 0x00000000, + 0x140f8000, 0x00000000, 0x22000000, + 0x14000000, 0x00000000, 0xe2000000 +}; + +/* context restore (sys -> gmem) */ +/* pre-compiled vertex shader program +* +* attribute vec4 position; +* attribute vec4 texcoord; +* varying vec4 texcoord0; +* void main() +* { +* gl_Position = position; +* texcoord0 = texcoord; +* } +*/ + +#define SYS2GMEM_VTX_PGM_LEN 0x18 + +static unsigned int sys2gmem_vtx_pgm[SYS2GMEM_VTX_PGM_LEN] = { + 0x00052003, 0x00001000, 0xc2000000, 0x00001005, + 0x00001000, 0xc4000000, 0x00001006, 0x10071000, + 0x20000000, 0x18981000, 0x0039ba88, 0x00000003, + 0x12982000, 0x40257b08, 0x00000002, 0x140f803e, + 0x00000000, 0xe2010100, 0x140f8000, 0x00000000, + 0xe2020200, 0x14000000, 0x00000000, 0xe2000000 +}; + +/* pre-compiled fragment shader program +* +* precision mediump float; +* uniform sampler2D tex0; +* varying vec4 texcoord0; +* void main() +* { +* gl_FragColor = texture2D(tex0, texcoord0.xy); +* } +*/ + +#define SYS2GMEM_FRAG_PGM_LEN 0x0f + +static unsigned int sys2gmem_frag_pgm[SYS2GMEM_FRAG_PGM_LEN] = { + 0x00011002, 0x00001000, 0xc4000000, 0x00001003, + 0x10041000, 0x20000000, 0x10000001, 0x1ffff688, + 0x00000002, 0x140f8000, 0x00000000, 0xe2000000, + 0x14000000, 0x00000000, 0xe2000000 +}; + +/* shader texture constants (sysmem -> gmem) */ +#define SYS2GMEM_TEX_CONST_LEN 6 + +static unsigned int sys2gmem_tex_const[SYS2GMEM_TEX_CONST_LEN] = { + /* Texture, FormatXYZW=Unsigned, ClampXYZ=Wrap/Repeat, + * RFMode=ZeroClamp-1, Dim=1:2d + */ + 0x00000002, /* Pitch = TBD */ + + /* Format=6:8888_WZYX, EndianSwap=0:None, ReqSize=0:256bit, DimHi=0, + * NearestClamp=1:OGL Mode + */ + 0x00000800, /* Address[31:12] = TBD */ + + /* Width, Height, EndianSwap=0:None */ + 0, /* Width & Height = TBD */ + + /* NumFormat=0:RF, DstSelXYZW=XYZW, ExpAdj=0, MagFilt=MinFilt=0:Point, + * Mip=2:BaseMap + */ + 0 << 1 | 1 << 4 | 2 << 7 | 3 << 10 | 2 << 23, + + /* VolMag=VolMin=0:Point, MinMipLvl=0, MaxMipLvl=1, LodBiasH=V=0, + * Dim3d=0 + */ + 0, + + /* BorderColor=0:ABGRBlack, ForceBC=0:diable, TriJuice=0, Aniso=0, + * Dim=1:2d, MipPacking=0 + */ + 1 << 9 /* Mip Address[31:12] = TBD */ +}; + +/* quad for copying GMEM to context shadow */ +#define QUAD_LEN 12 + +static unsigned int gmem_copy_quad[QUAD_LEN] = { + 0x00000000, 0x00000000, 0x3f800000, + 0x00000000, 0x00000000, 0x3f800000, + 0x00000000, 0x00000000, 0x3f800000, + 0x00000000, 0x00000000, 0x3f800000 +}; + +#define TEXCOORD_LEN 8 + +static unsigned int gmem_copy_texcoord[TEXCOORD_LEN] = { + 0x00000000, 0x3f800000, + 0x3f800000, 0x3f800000, + 0x00000000, 0x00000000, + 0x3f800000, 0x00000000 +}; + +#define NUM_COLOR_FORMATS 13 + +static enum SURFACEFORMAT surface_format_table[NUM_COLOR_FORMATS] = { + FMT_4_4_4_4, /* COLORX_4_4_4_4 */ + FMT_1_5_5_5, /* COLORX_1_5_5_5 */ + FMT_5_6_5, /* COLORX_5_6_5 */ + FMT_8, /* COLORX_8 */ + FMT_8_8, /* COLORX_8_8 */ + FMT_8_8_8_8, /* COLORX_8_8_8_8 */ + FMT_8_8_8_8, /* COLORX_S8_8_8_8 */ + FMT_16_FLOAT, /* COLORX_16_FLOAT */ + FMT_16_16_FLOAT, /* COLORX_16_16_FLOAT */ + FMT_16_16_16_16_FLOAT, /* COLORX_16_16_16_16_FLOAT */ + FMT_32_FLOAT, /* COLORX_32_FLOAT */ + FMT_32_32_FLOAT, /* COLORX_32_32_FLOAT */ + FMT_32_32_32_32_FLOAT, /* COLORX_32_32_32_32_FLOAT */ +}; + +static unsigned int format2bytesperpixel[NUM_COLOR_FORMATS] = { + 2, /* COLORX_4_4_4_4 */ + 2, /* COLORX_1_5_5_5 */ + 2, /* COLORX_5_6_5 */ + 1, /* COLORX_8 */ + 2, /* COLORX_8_8 8*/ + 4, /* COLORX_8_8_8_8 */ + 4, /* COLORX_S8_8_8_8 */ + 2, /* COLORX_16_FLOAT */ + 4, /* COLORX_16_16_FLOAT */ + 8, /* COLORX_16_16_16_16_FLOAT */ + 4, /* COLORX_32_FLOAT */ + 8, /* COLORX_32_32_FLOAT */ + 16, /* COLORX_32_32_32_32_FLOAT */ +}; + +/* shader linkage info */ +#define SHADER_CONST_ADDR (11 * 6 + 3) + +/* gmem command buffer length */ +#define PM4_REG(reg) ((0x4 << 16) | (GSL_HAL_SUBBLOCK_OFFSET(reg))) + +/* functions */ +static void config_gmemsize(struct gmem_shadow_t *shadow, int gmem_size) +{ + int w = 64, h = 64; /* 16KB surface, minimum */ + + shadow->format = COLORX_8_8_8_8; + /* convert from bytes to 32-bit words */ + gmem_size = (gmem_size + 3) / 4; + + /* find the right surface size, close to a square. */ + while (w * h < gmem_size) + if (w < h) + w *= 2; + else + h *= 2; + + shadow->width = w; + shadow->pitch = w; + shadow->height = h; + shadow->gmem_pitch = shadow->pitch; + + shadow->size = shadow->pitch * shadow->height * 4; +} + +static unsigned int gpuaddr(unsigned int *cmd, struct kgsl_memdesc *memdesc) +{ + return memdesc->gpuaddr + ((char *)cmd - (char *)memdesc->hostptr); +} + +static void +create_ib1(struct adreno_context *drawctxt, unsigned int *cmd, + unsigned int *start, unsigned int *end) +{ + cmd[0] = PM4_HDR_INDIRECT_BUFFER_PFD; + cmd[1] = gpuaddr(start, &drawctxt->gpustate); + cmd[2] = end - start; +} + +static unsigned int *program_shader(unsigned int *cmds, int vtxfrag, + unsigned int *shader_pgm, int dwords) +{ + /* load the patched vertex shader stream */ + *cmds++ = pm4_type3_packet(PM4_IM_LOAD_IMMEDIATE, 2 + dwords); + /* 0=vertex shader, 1=fragment shader */ + *cmds++ = vtxfrag; + /* instruction start & size (in 32-bit words) */ + *cmds++ = ((0 << 16) | dwords); + + memcpy(cmds, shader_pgm, dwords << 2); + cmds += dwords; + + return cmds; +} + +static unsigned int *reg_to_mem(unsigned int *cmds, uint32_t dst, + uint32_t src, int dwords) +{ + while (dwords-- > 0) { + *cmds++ = pm4_type3_packet(PM4_REG_TO_MEM, 2); + *cmds++ = src++; + *cmds++ = dst; + dst += 4; + } + + return cmds; +} + +#ifdef CONFIG_MSM_KGSL_DISABLE_SHADOW_WRITES + +static void build_reg_to_mem_range(unsigned int start, unsigned int end, + unsigned int **cmd, + struct adreno_context *drawctxt) +{ + unsigned int i = start; + + for (i = start; i <= end; i++) { + *(*cmd)++ = pm4_type3_packet(PM4_REG_TO_MEM, 2); + *(*cmd)++ = i; + *(*cmd)++ = + ((drawctxt->gpustate.gpuaddr + REG_OFFSET) & 0xFFFFE000) + + (i - 0x2000) * 4; + } +} + +#endif + +/* chicken restore */ +static unsigned int *build_chicken_restore_cmds( + struct adreno_context *drawctxt, + struct tmp_ctx *ctx) +{ + unsigned int *start = ctx->cmd; + unsigned int *cmds = start; + + *cmds++ = pm4_type3_packet(PM4_WAIT_FOR_IDLE, 1); + *cmds++ = 0; + + *cmds++ = pm4_type0_packet(REG_TP0_CHICKEN, 1); + ctx->chicken_restore = gpuaddr(cmds, &drawctxt->gpustate); + *cmds++ = 0x00000000; + + /* create indirect buffer command for above command sequence */ + create_ib1(drawctxt, drawctxt->chicken_restore, start, cmds); + + return cmds; +} + +/* save h/w regs, alu constants, texture contants, etc. ... +* requires: bool_shadow_gpuaddr, loop_shadow_gpuaddr +*/ +static void build_regsave_cmds(struct adreno_device *adreno_dev, + struct adreno_context *drawctxt, + struct tmp_ctx *ctx) +{ + unsigned int *start = ctx->cmd; + unsigned int *cmd = start; + + *cmd++ = pm4_type3_packet(PM4_WAIT_FOR_IDLE, 1); + *cmd++ = 0; + +#ifdef CONFIG_MSM_KGSL_DISABLE_SHADOW_WRITES + /* Make sure the HW context has the correct register values + * before reading them. */ + *cmd++ = pm4_type3_packet(PM4_CONTEXT_UPDATE, 1); + *cmd++ = 0; +#endif + +#ifdef CONFIG_MSM_KGSL_DISABLE_SHADOW_WRITES + /* Write HW registers into shadow */ + build_reg_to_mem_range(REG_RB_SURFACE_INFO, REG_RB_DEPTH_INFO, + &cmd, drawctxt); + build_reg_to_mem_range(REG_COHER_DEST_BASE_0, + REG_PA_SC_SCREEN_SCISSOR_BR, + &cmd, drawctxt); + build_reg_to_mem_range(REG_PA_SC_WINDOW_OFFSET, + REG_PA_SC_WINDOW_SCISSOR_BR, + &cmd, drawctxt); + if (!adreno_is_a220(adreno_dev)) { + build_reg_to_mem_range(REG_VGT_MAX_VTX_INDX, REG_RB_FOG_COLOR, + &cmd, drawctxt); + } else { + build_reg_to_mem_range(REG_LEIA_PC_MAX_VTX_INDX, + REG_LEIA_PC_INDX_OFFSET, + &cmd, drawctxt); + build_reg_to_mem_range(REG_RB_COLOR_MASK, + REG_RB_FOG_COLOR, + &cmd, drawctxt); + } + build_reg_to_mem_range(REG_RB_STENCILREFMASK_BF, + REG_PA_CL_VPORT_ZOFFSET, + &cmd, drawctxt); + build_reg_to_mem_range(REG_SQ_PROGRAM_CNTL, REG_SQ_WRAPPING_1, + &cmd, drawctxt); + if (!adreno_is_a220(adreno_dev)) { + build_reg_to_mem_range(REG_RB_DEPTHCONTROL, REG_RB_MODECONTROL, + &cmd, drawctxt); + build_reg_to_mem_range(REG_PA_SU_POINT_SIZE, + REG_PA_SC_LINE_STIPPLE, + &cmd, drawctxt); + build_reg_to_mem_range(REG_PA_SC_VIZ_QUERY, REG_PA_SC_VIZ_QUERY, + &cmd, drawctxt); + } else { + build_reg_to_mem_range(REG_RB_DEPTHCONTROL, + REG_RB_COLORCONTROL, + &cmd, drawctxt); + build_reg_to_mem_range(REG_PA_CL_CLIP_CNTL, + REG_PA_CL_VTE_CNTL, + &cmd, drawctxt); + build_reg_to_mem_range(REG_RB_MODECONTROL, + REG_LEIA_GRAS_CONTROL, + &cmd, drawctxt); + build_reg_to_mem_range(REG_PA_SU_POINT_SIZE, + REG_PA_SU_LINE_CNTL, + &cmd, drawctxt); + } + build_reg_to_mem_range(REG_PA_SC_LINE_CNTL, REG_SQ_PS_CONST, + &cmd, drawctxt); + build_reg_to_mem_range(REG_PA_SC_AA_MASK, REG_PA_SC_AA_MASK, + &cmd, drawctxt); + if (!adreno_is_a220(adreno_dev)) { + build_reg_to_mem_range(REG_VGT_VERTEX_REUSE_BLOCK_CNTL, + REG_RB_DEPTH_CLEAR, + &cmd, drawctxt); + } else { + build_reg_to_mem_range(REG_LEIA_PC_VERTEX_REUSE_BLOCK_CNTL, + REG_LEIA_PC_VERTEX_REUSE_BLOCK_CNTL, + &cmd, drawctxt); + build_reg_to_mem_range(REG_RB_COPY_CONTROL, + REG_RB_DEPTH_CLEAR, + &cmd, drawctxt); + } + build_reg_to_mem_range(REG_RB_SAMPLE_COUNT_CTL, + REG_RB_COLOR_DEST_MASK, + &cmd, drawctxt); + build_reg_to_mem_range(REG_PA_SU_POLY_OFFSET_FRONT_SCALE, + REG_PA_SU_POLY_OFFSET_BACK_OFFSET, + &cmd, drawctxt); + + /* Copy ALU constants */ + cmd = + reg_to_mem(cmd, (drawctxt->gpustate.gpuaddr) & 0xFFFFE000, + REG_SQ_CONSTANT_0, ALU_CONSTANTS); + + /* Copy Tex constants */ + cmd = + reg_to_mem(cmd, + (drawctxt->gpustate.gpuaddr + TEX_OFFSET) & 0xFFFFE000, + REG_SQ_FETCH_0, TEX_CONSTANTS); +#else + + /* Insert a wait for idle packet before reading the registers. + * This is to fix a hang/reset seen during stress testing. In this + * hang, CP encountered a timeout reading SQ's boolean constant + * register. There is logic in the HW that blocks reading of this + * register when the SQ block is not idle, which we believe is + * contributing to the hang.*/ + *cmd++ = pm4_type3_packet(PM4_WAIT_FOR_IDLE, 1); + *cmd++ = 0; + + /* H/w registers are already shadowed; just need to disable shadowing + * to prevent corruption. + */ + *cmd++ = pm4_type3_packet(PM4_LOAD_CONSTANT_CONTEXT, 3); + *cmd++ = (drawctxt->gpustate.gpuaddr + REG_OFFSET) & 0xFFFFE000; + *cmd++ = 4 << 16; /* regs, start=0 */ + *cmd++ = 0x0; /* count = 0 */ + + /* ALU constants are already shadowed; just need to disable shadowing + * to prevent corruption. + */ + *cmd++ = pm4_type3_packet(PM4_LOAD_CONSTANT_CONTEXT, 3); + *cmd++ = drawctxt->gpustate.gpuaddr & 0xFFFFE000; + *cmd++ = 0 << 16; /* ALU, start=0 */ + *cmd++ = 0x0; /* count = 0 */ + + /* Tex constants are already shadowed; just need to disable shadowing + * to prevent corruption. + */ + *cmd++ = pm4_type3_packet(PM4_LOAD_CONSTANT_CONTEXT, 3); + *cmd++ = (drawctxt->gpustate.gpuaddr + TEX_OFFSET) & 0xFFFFE000; + *cmd++ = 1 << 16; /* Tex, start=0 */ + *cmd++ = 0x0; /* count = 0 */ +#endif + + /* Need to handle some of the registers separately */ + *cmd++ = pm4_type3_packet(PM4_REG_TO_MEM, 2); + *cmd++ = REG_SQ_GPR_MANAGEMENT; + *cmd++ = ctx->reg_values[0]; + + *cmd++ = pm4_type3_packet(PM4_REG_TO_MEM, 2); + *cmd++ = REG_TP0_CHICKEN; + *cmd++ = ctx->reg_values[1]; + + *cmd++ = pm4_type3_packet(PM4_REG_TO_MEM, 2); + *cmd++ = REG_RBBM_PM_OVERRIDE2; + *cmd++ = ctx->reg_values[2]; + + if (adreno_is_a220(adreno_dev)) { + unsigned int i; + unsigned int j = 3; + for (i = REG_LEIA_VSC_BIN_SIZE; i <= + REG_LEIA_VSC_PIPE_DATA_LENGTH_7; i++) { + *cmd++ = pm4_type3_packet(PM4_REG_TO_MEM, 2); + *cmd++ = i; + *cmd++ = ctx->reg_values[j]; + j++; + } + } + + /* Copy Boolean constants */ + cmd = reg_to_mem(cmd, ctx->bool_shadow, REG_SQ_CF_BOOLEANS, + BOOL_CONSTANTS); + + /* Copy Loop constants */ + cmd = reg_to_mem(cmd, ctx->loop_shadow, REG_SQ_CF_LOOP, LOOP_CONSTANTS); + + /* create indirect buffer command for above command sequence */ + create_ib1(drawctxt, drawctxt->reg_save, start, cmd); + + ctx->cmd = cmd; +} + +/*copy colour, depth, & stencil buffers from graphics memory to system memory*/ +static unsigned int *build_gmem2sys_cmds(struct adreno_device *adreno_dev, + struct adreno_context *drawctxt, + struct tmp_ctx *ctx, + struct gmem_shadow_t *shadow) +{ + unsigned int *cmds = shadow->gmem_save_commands; + unsigned int *start = cmds; + /* Calculate the new offset based on the adjusted base */ + unsigned int bytesperpixel = format2bytesperpixel[shadow->format]; + unsigned int addr = shadow->gmemshadow.gpuaddr; + unsigned int offset = (addr - (addr & 0xfffff000)) / bytesperpixel; + + /* Store TP0_CHICKEN register */ + *cmds++ = pm4_type3_packet(PM4_REG_TO_MEM, 2); + *cmds++ = REG_TP0_CHICKEN; + if (ctx) + *cmds++ = ctx->chicken_restore; + else + cmds++; + + *cmds++ = pm4_type3_packet(PM4_WAIT_FOR_IDLE, 1); + *cmds++ = 0; + + /* Set TP0_CHICKEN to zero */ + *cmds++ = pm4_type0_packet(REG_TP0_CHICKEN, 1); + *cmds++ = 0x00000000; + + /* Set PA_SC_AA_CONFIG to 0 */ + *cmds++ = pm4_type0_packet(REG_PA_SC_AA_CONFIG, 1); + *cmds++ = 0x00000000; + + /* program shader */ + + /* load shader vtx constants ... 5 dwords */ + *cmds++ = pm4_type3_packet(PM4_SET_CONSTANT, 4); + *cmds++ = (0x1 << 16) | SHADER_CONST_ADDR; + *cmds++ = 0; + /* valid(?) vtx constant flag & addr */ + *cmds++ = shadow->quad_vertices.gpuaddr | 0x3; + /* limit = 12 dwords */ + *cmds++ = 0x00000030; + + /* Invalidate L2 cache to make sure vertices are updated */ + *cmds++ = pm4_type0_packet(REG_TC_CNTL_STATUS, 1); + *cmds++ = 0x1; + + *cmds++ = pm4_type3_packet(PM4_SET_CONSTANT, 4); + *cmds++ = PM4_REG(REG_VGT_MAX_VTX_INDX); + *cmds++ = 0x00ffffff; /* REG_VGT_MAX_VTX_INDX */ + *cmds++ = 0x0; /* REG_VGT_MIN_VTX_INDX */ + *cmds++ = 0x00000000; /* REG_VGT_INDX_OFFSET */ + + *cmds++ = pm4_type3_packet(PM4_SET_CONSTANT, 2); + *cmds++ = PM4_REG(REG_PA_SC_AA_MASK); + *cmds++ = 0x0000ffff; /* REG_PA_SC_AA_MASK */ + + /* load the patched vertex shader stream */ + cmds = program_shader(cmds, 0, gmem2sys_vtx_pgm, GMEM2SYS_VTX_PGM_LEN); + + /* Load the patched fragment shader stream */ + cmds = + program_shader(cmds, 1, gmem2sys_frag_pgm, GMEM2SYS_FRAG_PGM_LEN); + + /* SQ_PROGRAM_CNTL / SQ_CONTEXT_MISC */ + *cmds++ = pm4_type3_packet(PM4_SET_CONSTANT, 3); + *cmds++ = PM4_REG(REG_SQ_PROGRAM_CNTL); + if (adreno_is_a220(adreno_dev)) + *cmds++ = 0x10018001; + else + *cmds++ = 0x10010001; + *cmds++ = 0x00000008; + + /* resolve */ + + /* PA_CL_VTE_CNTL */ + *cmds++ = pm4_type3_packet(PM4_SET_CONSTANT, 2); + *cmds++ = PM4_REG(REG_PA_CL_VTE_CNTL); + /* disable X/Y/Z transforms, X/Y/Z are premultiplied by W */ + *cmds++ = 0x00000b00; + + /* program surface info */ + *cmds++ = pm4_type3_packet(PM4_SET_CONSTANT, 3); + *cmds++ = PM4_REG(REG_RB_SURFACE_INFO); + *cmds++ = shadow->gmem_pitch; /* pitch, MSAA = 1 */ + + /* RB_COLOR_INFO Endian=none, Linear, Format=RGBA8888, Swap=0, + * Base=gmem_base + */ + /* gmem base assumed 4K aligned. */ + if (ctx) { + BUG_ON(ctx->gmem_base & 0xFFF); + *cmds++ = + (shadow-> + format << RB_COLOR_INFO__COLOR_FORMAT__SHIFT) | ctx-> + gmem_base; + } else { + unsigned int temp = *cmds; + *cmds++ = (temp & ~RB_COLOR_INFO__COLOR_FORMAT_MASK) | + (shadow->format << RB_COLOR_INFO__COLOR_FORMAT__SHIFT); + } + + /* disable Z */ + *cmds++ = pm4_type3_packet(PM4_SET_CONSTANT, 2); + *cmds++ = PM4_REG(REG_RB_DEPTHCONTROL); + if (adreno_is_a220(adreno_dev)) + *cmds++ = 0x08; + else + *cmds++ = 0; + + /* set REG_PA_SU_SC_MODE_CNTL + * Front_ptype = draw triangles + * Back_ptype = draw triangles + * Provoking vertex = last + */ + *cmds++ = pm4_type3_packet(PM4_SET_CONSTANT, 2); + *cmds++ = PM4_REG(REG_PA_SU_SC_MODE_CNTL); + *cmds++ = 0x00080240; + + /* Use maximum scissor values -- quad vertices already have the + * correct bounds */ + *cmds++ = pm4_type3_packet(PM4_SET_CONSTANT, 3); + *cmds++ = PM4_REG(REG_PA_SC_SCREEN_SCISSOR_TL); + *cmds++ = (0 << 16) | 0; + *cmds++ = (0x1fff << 16) | (0x1fff); + *cmds++ = pm4_type3_packet(PM4_SET_CONSTANT, 3); + *cmds++ = PM4_REG(REG_PA_SC_WINDOW_SCISSOR_TL); + *cmds++ = (unsigned int)((1U << 31) | (0 << 16) | 0); + *cmds++ = (0x1fff << 16) | (0x1fff); + + /* load the viewport so that z scale = clear depth and + * z offset = 0.0f + */ + *cmds++ = pm4_type3_packet(PM4_SET_CONSTANT, 3); + *cmds++ = PM4_REG(REG_PA_CL_VPORT_ZSCALE); + *cmds++ = 0xbf800000; /* -1.0f */ + *cmds++ = 0x0; + + /* load the stencil ref value + * $AAM - do this later + */ + + /* load the COPY state */ + *cmds++ = pm4_type3_packet(PM4_SET_CONSTANT, 6); + *cmds++ = PM4_REG(REG_RB_COPY_CONTROL); + *cmds++ = 0; /* RB_COPY_CONTROL */ + *cmds++ = addr & 0xfffff000; /* RB_COPY_DEST_BASE */ + *cmds++ = shadow->pitch >> 5; /* RB_COPY_DEST_PITCH */ + + /* Endian=none, Linear, Format=RGBA8888,Swap=0,!Dither, + * MaskWrite:R=G=B=A=1 + */ + *cmds++ = 0x0003c008 | + (shadow->format << RB_COPY_DEST_INFO__COPY_DEST_FORMAT__SHIFT); + /* Make sure we stay in offsetx field. */ + BUG_ON(offset & 0xfffff000); + *cmds++ = offset; + + *cmds++ = pm4_type3_packet(PM4_SET_CONSTANT, 2); + *cmds++ = PM4_REG(REG_RB_MODECONTROL); + *cmds++ = 0x6; /* EDRAM copy */ + + if (adreno_is_a220(adreno_dev)) { + *cmds++ = 0xc0043600; /* packet 3 3D_DRAW_INDX_2 */ + *cmds++ = 0x0; + *cmds++ = 0x00004046; /* tristrip */ + *cmds++ = 0x00000004; /* NUM_INDICES */ + *cmds++ = 0x00010000; /* index: 0x00, 0x01 */ + *cmds++ = 0x00030002; /* index: 0x02, 0x03 */ + } else { + /* queue the draw packet */ + *cmds++ = pm4_type3_packet(PM4_DRAW_INDX, 2); + *cmds++ = 0; /* viz query info. */ + /* PrimType=RectList, NumIndices=3, SrcSel=AutoIndex */ + *cmds++ = 0x00030088; + } + + /* create indirect buffer command for above command sequence */ + create_ib1(drawctxt, shadow->gmem_save, start, cmds); + + return cmds; +} + +/* context restore */ + +/*copy colour, depth, & stencil buffers from system memory to graphics memory*/ +static unsigned int *build_sys2gmem_cmds(struct adreno_device *adreno_dev, + struct adreno_context *drawctxt, + struct tmp_ctx *ctx, + struct gmem_shadow_t *shadow) +{ + unsigned int *cmds = shadow->gmem_restore_commands; + unsigned int *start = cmds; + + /* Store TP0_CHICKEN register */ + *cmds++ = pm4_type3_packet(PM4_REG_TO_MEM, 2); + *cmds++ = REG_TP0_CHICKEN; + if (ctx) + *cmds++ = ctx->chicken_restore; + else + cmds++; + + *cmds++ = pm4_type3_packet(PM4_WAIT_FOR_IDLE, 1); + *cmds++ = 0; + + /* Set TP0_CHICKEN to zero */ + *cmds++ = pm4_type0_packet(REG_TP0_CHICKEN, 1); + *cmds++ = 0x00000000; + + /* Set PA_SC_AA_CONFIG to 0 */ + *cmds++ = pm4_type0_packet(REG_PA_SC_AA_CONFIG, 1); + *cmds++ = 0x00000000; + /* shader constants */ + + /* vertex buffer constants */ + *cmds++ = pm4_type3_packet(PM4_SET_CONSTANT, 7); + + *cmds++ = (0x1 << 16) | (9 * 6); + /* valid(?) vtx constant flag & addr */ + *cmds++ = shadow->quad_vertices.gpuaddr | 0x3; + /* limit = 12 dwords */ + *cmds++ = 0x00000030; + /* valid(?) vtx constant flag & addr */ + *cmds++ = shadow->quad_texcoords.gpuaddr | 0x3; + /* limit = 8 dwords */ + *cmds++ = 0x00000020; + *cmds++ = 0; + *cmds++ = 0; + + /* Invalidate L2 cache to make sure vertices are updated */ + *cmds++ = pm4_type0_packet(REG_TC_CNTL_STATUS, 1); + *cmds++ = 0x1; + + cmds = program_shader(cmds, 0, sys2gmem_vtx_pgm, SYS2GMEM_VTX_PGM_LEN); + + /* Load the patched fragment shader stream */ + cmds = + program_shader(cmds, 1, sys2gmem_frag_pgm, SYS2GMEM_FRAG_PGM_LEN); + + /* SQ_PROGRAM_CNTL / SQ_CONTEXT_MISC */ + *cmds++ = pm4_type3_packet(PM4_SET_CONSTANT, 3); + *cmds++ = PM4_REG(REG_SQ_PROGRAM_CNTL); + *cmds++ = 0x10030002; + *cmds++ = 0x00000008; + + *cmds++ = pm4_type3_packet(PM4_SET_CONSTANT, 2); + *cmds++ = PM4_REG(REG_PA_SC_AA_MASK); + *cmds++ = 0x0000ffff; /* REG_PA_SC_AA_MASK */ + + if (!adreno_is_a220(adreno_dev)) { + /* PA_SC_VIZ_QUERY */ + *cmds++ = pm4_type3_packet(PM4_SET_CONSTANT, 2); + *cmds++ = PM4_REG(REG_PA_SC_VIZ_QUERY); + *cmds++ = 0x0; /*REG_PA_SC_VIZ_QUERY */ + } + + /* RB_COLORCONTROL */ + *cmds++ = pm4_type3_packet(PM4_SET_CONSTANT, 2); + *cmds++ = PM4_REG(REG_RB_COLORCONTROL); + *cmds++ = 0x00000c20; + + *cmds++ = pm4_type3_packet(PM4_SET_CONSTANT, 4); + *cmds++ = PM4_REG(REG_VGT_MAX_VTX_INDX); + *cmds++ = 0x00ffffff; /* mmVGT_MAX_VTX_INDX */ + *cmds++ = 0x0; /* mmVGT_MIN_VTX_INDX */ + *cmds++ = 0x00000000; /* mmVGT_INDX_OFFSET */ + + *cmds++ = pm4_type3_packet(PM4_SET_CONSTANT, 3); + *cmds++ = PM4_REG(REG_VGT_VERTEX_REUSE_BLOCK_CNTL); + *cmds++ = 0x00000002; /* mmVGT_VERTEX_REUSE_BLOCK_CNTL */ + *cmds++ = 0x00000002; /* mmVGT_OUT_DEALLOC_CNTL */ + + *cmds++ = pm4_type3_packet(PM4_SET_CONSTANT, 2); + *cmds++ = PM4_REG(REG_SQ_INTERPOLATOR_CNTL); + *cmds++ = 0xffffffff; /* mmSQ_INTERPOLATOR_CNTL */ + + *cmds++ = pm4_type3_packet(PM4_SET_CONSTANT, 2); + *cmds++ = PM4_REG(REG_PA_SC_AA_CONFIG); + *cmds++ = 0x00000000; /* REG_PA_SC_AA_CONFIG */ + + /* set REG_PA_SU_SC_MODE_CNTL + * Front_ptype = draw triangles + * Back_ptype = draw triangles + * Provoking vertex = last + */ + *cmds++ = pm4_type3_packet(PM4_SET_CONSTANT, 2); + *cmds++ = PM4_REG(REG_PA_SU_SC_MODE_CNTL); + *cmds++ = 0x00080240; + + /* texture constants */ + *cmds++ = + pm4_type3_packet(PM4_SET_CONSTANT, (SYS2GMEM_TEX_CONST_LEN + 1)); + *cmds++ = (0x1 << 16) | (0 * 6); + memcpy(cmds, sys2gmem_tex_const, SYS2GMEM_TEX_CONST_LEN << 2); + cmds[0] |= (shadow->pitch >> 5) << 22; + cmds[1] |= + shadow->gmemshadow.gpuaddr | surface_format_table[shadow->format]; + cmds[2] |= (shadow->width - 1) | (shadow->height - 1) << 13; + cmds += SYS2GMEM_TEX_CONST_LEN; + + /* program surface info */ + *cmds++ = pm4_type3_packet(PM4_SET_CONSTANT, 3); + *cmds++ = PM4_REG(REG_RB_SURFACE_INFO); + *cmds++ = shadow->gmem_pitch; /* pitch, MSAA = 1 */ + + /* RB_COLOR_INFO Endian=none, Linear, Format=RGBA8888, Swap=0, + * Base=gmem_base + */ + if (ctx) + *cmds++ = + (shadow-> + format << RB_COLOR_INFO__COLOR_FORMAT__SHIFT) | ctx-> + gmem_base; + else { + unsigned int temp = *cmds; + *cmds++ = (temp & ~RB_COLOR_INFO__COLOR_FORMAT_MASK) | + (shadow->format << RB_COLOR_INFO__COLOR_FORMAT__SHIFT); + } + + /* RB_DEPTHCONTROL */ + *cmds++ = pm4_type3_packet(PM4_SET_CONSTANT, 2); + *cmds++ = PM4_REG(REG_RB_DEPTHCONTROL); + + if (adreno_is_a220(adreno_dev)) + *cmds++ = 8; /* disable Z */ + else + *cmds++ = 0; /* disable Z */ + + /* Use maximum scissor values -- quad vertices already + * have the correct bounds */ + *cmds++ = pm4_type3_packet(PM4_SET_CONSTANT, 3); + *cmds++ = PM4_REG(REG_PA_SC_SCREEN_SCISSOR_TL); + *cmds++ = (0 << 16) | 0; + *cmds++ = ((0x1fff) << 16) | 0x1fff; + *cmds++ = pm4_type3_packet(PM4_SET_CONSTANT, 3); + *cmds++ = PM4_REG(REG_PA_SC_WINDOW_SCISSOR_TL); + *cmds++ = (unsigned int)((1U << 31) | (0 << 16) | 0); + *cmds++ = ((0x1fff) << 16) | 0x1fff; + + *cmds++ = pm4_type3_packet(PM4_SET_CONSTANT, 2); + *cmds++ = PM4_REG(REG_PA_CL_VTE_CNTL); + /* disable X/Y/Z transforms, X/Y/Z are premultiplied by W */ + *cmds++ = 0x00000b00; + + /*load the viewport so that z scale = clear depth and z offset = 0.0f */ + *cmds++ = pm4_type3_packet(PM4_SET_CONSTANT, 3); + *cmds++ = PM4_REG(REG_PA_CL_VPORT_ZSCALE); + *cmds++ = 0xbf800000; + *cmds++ = 0x0; + + *cmds++ = pm4_type3_packet(PM4_SET_CONSTANT, 2); + *cmds++ = PM4_REG(REG_RB_COLOR_MASK); + *cmds++ = 0x0000000f; /* R = G = B = 1:enabled */ + + *cmds++ = pm4_type3_packet(PM4_SET_CONSTANT, 2); + *cmds++ = PM4_REG(REG_RB_COLOR_DEST_MASK); + *cmds++ = 0xffffffff; + + *cmds++ = pm4_type3_packet(PM4_SET_CONSTANT, 3); + *cmds++ = PM4_REG(REG_SQ_WRAPPING_0); + *cmds++ = 0x00000000; + *cmds++ = 0x00000000; + + /* load the stencil ref value + * $AAM - do this later + */ + *cmds++ = pm4_type3_packet(PM4_SET_CONSTANT, 2); + *cmds++ = PM4_REG(REG_RB_MODECONTROL); + /* draw pixels with color and depth/stencil component */ + *cmds++ = 0x4; + + if (adreno_is_a220(adreno_dev)) { + *cmds++ = 0xc0043600; /* packet 3 3D_DRAW_INDX_2 */ + *cmds++ = 0x0; + *cmds++ = 0x00004046; /* tristrip */ + *cmds++ = 0x00000004; /* NUM_INDICES */ + *cmds++ = 0x00010000; /* index: 0x00, 0x01 */ + *cmds++ = 0x00030002; /* index: 0x02, 0x03 */ + } else { + /* queue the draw packet */ + *cmds++ = pm4_type3_packet(PM4_DRAW_INDX, 2); + *cmds++ = 0; /* viz query info. */ + /* PrimType=RectList, NumIndices=3, SrcSel=AutoIndex */ + *cmds++ = 0x00030088; + } + + /* create indirect buffer command for above command sequence */ + create_ib1(drawctxt, shadow->gmem_restore, start, cmds); + + return cmds; +} + +/* restore h/w regs, alu constants, texture constants, etc. ... */ +static unsigned *reg_range(unsigned int *cmd, unsigned int start, + unsigned int end) +{ + *cmd++ = PM4_REG(start); /* h/w regs, start addr */ + *cmd++ = end - start + 1; /* count */ + return cmd; +} + +static void build_regrestore_cmds(struct adreno_device *adreno_dev, + struct adreno_context *drawctxt, + struct tmp_ctx *ctx) +{ + unsigned int *start = ctx->cmd; + unsigned int *cmd = start; + + *cmd++ = pm4_type3_packet(PM4_WAIT_FOR_IDLE, 1); + *cmd++ = 0; + + /* H/W Registers */ + /* deferred pm4_type3_packet(PM4_LOAD_CONSTANT_CONTEXT, ???); */ + cmd++; +#ifdef CONFIG_MSM_KGSL_DISABLE_SHADOW_WRITES + /* Force mismatch */ + *cmd++ = ((drawctxt->gpustate.gpuaddr + REG_OFFSET) & 0xFFFFE000) | 1; +#else + *cmd++ = (drawctxt->gpustate.gpuaddr + REG_OFFSET) & 0xFFFFE000; +#endif + + if (!adreno_is_a220(adreno_dev)) { + cmd = reg_range(cmd, REG_RB_SURFACE_INFO, + REG_PA_SC_SCREEN_SCISSOR_BR); + } else { + cmd = reg_range(cmd, REG_RB_SURFACE_INFO, REG_RB_DEPTH_INFO); + cmd = reg_range(cmd, REG_COHER_DEST_BASE_0, + REG_PA_SC_SCREEN_SCISSOR_BR); + } + cmd = reg_range(cmd, REG_PA_SC_WINDOW_OFFSET, + REG_PA_SC_WINDOW_SCISSOR_BR); + if (!adreno_is_a220(adreno_dev)) { + cmd = reg_range(cmd, REG_VGT_MAX_VTX_INDX, + REG_PA_CL_VPORT_ZOFFSET); + } else { + cmd = reg_range(cmd, REG_LEIA_PC_MAX_VTX_INDX, + REG_LEIA_PC_INDX_OFFSET); + cmd = reg_range(cmd, REG_RB_COLOR_MASK, REG_RB_FOG_COLOR); + cmd = reg_range(cmd, REG_RB_STENCILREFMASK_BF, + REG_PA_CL_VPORT_ZOFFSET); + } + cmd = reg_range(cmd, REG_SQ_PROGRAM_CNTL, REG_SQ_WRAPPING_1); + if (!adreno_is_a220(adreno_dev)) { + cmd = reg_range(cmd, REG_RB_DEPTHCONTROL, REG_RB_MODECONTROL); + cmd = reg_range(cmd, REG_PA_SU_POINT_SIZE, + REG_PA_SC_VIZ_QUERY); /*REG_VGT_ENHANCE */ + cmd = reg_range(cmd, REG_PA_SC_LINE_CNTL, + REG_RB_COLOR_DEST_MASK); + } else { + cmd = reg_range(cmd, REG_RB_DEPTHCONTROL, REG_RB_COLORCONTROL); + cmd = reg_range(cmd, REG_PA_CL_CLIP_CNTL, REG_PA_CL_VTE_CNTL); + cmd = reg_range(cmd, REG_RB_MODECONTROL, REG_LEIA_GRAS_CONTROL); + cmd = reg_range(cmd, REG_PA_SU_POINT_SIZE, REG_PA_SU_LINE_CNTL); + cmd = reg_range(cmd, REG_PA_SC_LINE_CNTL, REG_SQ_PS_CONST); + cmd = reg_range(cmd, REG_PA_SC_AA_MASK, REG_PA_SC_AA_MASK); + cmd = reg_range(cmd, REG_LEIA_PC_VERTEX_REUSE_BLOCK_CNTL, + REG_LEIA_PC_VERTEX_REUSE_BLOCK_CNTL); + cmd = reg_range(cmd, REG_RB_COPY_CONTROL, REG_RB_DEPTH_CLEAR); + cmd = reg_range(cmd, REG_RB_SAMPLE_COUNT_CTL, + REG_RB_COLOR_DEST_MASK); + } + cmd = reg_range(cmd, REG_PA_SU_POLY_OFFSET_FRONT_SCALE, + REG_PA_SU_POLY_OFFSET_BACK_OFFSET); + + /* Now we know how many register blocks we have, we can compute command + * length + */ + start[2] = + pm4_type3_packet(PM4_LOAD_CONSTANT_CONTEXT, (cmd - start) - 3); + /* Enable shadowing for the entire register block. */ +#ifdef CONFIG_MSM_KGSL_DISABLE_SHADOW_WRITES + start[4] |= (0 << 24) | (4 << 16); /* Disable shadowing. */ +#else + start[4] |= (1 << 24) | (4 << 16); +#endif + + /* Need to handle some of the registers separately */ + *cmd++ = pm4_type0_packet(REG_SQ_GPR_MANAGEMENT, 1); + ctx->reg_values[0] = gpuaddr(cmd, &drawctxt->gpustate); + *cmd++ = 0x00040400; + + *cmd++ = pm4_type3_packet(PM4_WAIT_FOR_IDLE, 1); + *cmd++ = 0; + *cmd++ = pm4_type0_packet(REG_TP0_CHICKEN, 1); + ctx->reg_values[1] = gpuaddr(cmd, &drawctxt->gpustate); + *cmd++ = 0x00000000; + + *cmd++ = pm4_type0_packet(REG_RBBM_PM_OVERRIDE2, 1); + ctx->reg_values[2] = gpuaddr(cmd, &drawctxt->gpustate); + if (!adreno_is_a220(adreno_dev)) + *cmd++ = 0x00000000; + else + *cmd++ = 0x80; + + if (adreno_is_a220(adreno_dev)) { + unsigned int i; + unsigned int j = 3; + for (i = REG_LEIA_VSC_BIN_SIZE; i <= + REG_LEIA_VSC_PIPE_DATA_LENGTH_7; i++) { + *cmd++ = pm4_type0_packet(i, 1); + ctx->reg_values[j] = gpuaddr(cmd, &drawctxt->gpustate); + *cmd++ = 0x00000000; + j++; + } + } + + /* ALU Constants */ + *cmd++ = pm4_type3_packet(PM4_LOAD_CONSTANT_CONTEXT, 3); + *cmd++ = drawctxt->gpustate.gpuaddr & 0xFFFFE000; +#ifdef CONFIG_MSM_KGSL_DISABLE_SHADOW_WRITES + *cmd++ = (0 << 24) | (0 << 16) | 0; /* Disable shadowing */ +#else + *cmd++ = (1 << 24) | (0 << 16) | 0; +#endif + *cmd++ = ALU_CONSTANTS; + + /* Texture Constants */ + *cmd++ = pm4_type3_packet(PM4_LOAD_CONSTANT_CONTEXT, 3); + *cmd++ = (drawctxt->gpustate.gpuaddr + TEX_OFFSET) & 0xFFFFE000; +#ifdef CONFIG_MSM_KGSL_DISABLE_SHADOW_WRITES + /* Disable shadowing */ + *cmd++ = (0 << 24) | (1 << 16) | 0; +#else + *cmd++ = (1 << 24) | (1 << 16) | 0; +#endif + *cmd++ = TEX_CONSTANTS; + + /* Boolean Constants */ + *cmd++ = pm4_type3_packet(PM4_SET_CONSTANT, 1 + BOOL_CONSTANTS); + *cmd++ = (2 << 16) | 0; + + /* the next BOOL_CONSTANT dwords is the shadow area for + * boolean constants. + */ + ctx->bool_shadow = gpuaddr(cmd, &drawctxt->gpustate); + cmd += BOOL_CONSTANTS; + + /* Loop Constants */ + *cmd++ = pm4_type3_packet(PM4_SET_CONSTANT, 1 + LOOP_CONSTANTS); + *cmd++ = (3 << 16) | 0; + + /* the next LOOP_CONSTANTS dwords is the shadow area for + * loop constants. + */ + ctx->loop_shadow = gpuaddr(cmd, &drawctxt->gpustate); + cmd += LOOP_CONSTANTS; + + /* create indirect buffer command for above command sequence */ + create_ib1(drawctxt, drawctxt->reg_restore, start, cmd); + + ctx->cmd = cmd; +} + +/* quad for saving/restoring gmem */ +static void set_gmem_copy_quad(struct gmem_shadow_t *shadow) +{ + /* set vertex buffer values */ + gmem_copy_quad[1] = uint2float(shadow->height); + gmem_copy_quad[3] = uint2float(shadow->width); + gmem_copy_quad[4] = uint2float(shadow->height); + gmem_copy_quad[9] = uint2float(shadow->width); + + gmem_copy_quad[0] = uint2float(0); + gmem_copy_quad[6] = uint2float(0); + gmem_copy_quad[7] = uint2float(0); + gmem_copy_quad[10] = uint2float(0); + + memcpy(shadow->quad_vertices.hostptr, gmem_copy_quad, QUAD_LEN << 2); + + memcpy(shadow->quad_texcoords.hostptr, gmem_copy_texcoord, + TEXCOORD_LEN << 2); +} + +/* quad for saving/restoring gmem */ +static void build_quad_vtxbuff(struct adreno_context *drawctxt, + struct tmp_ctx *ctx, struct gmem_shadow_t *shadow) +{ + unsigned int *cmd = ctx->cmd; + + /* quad vertex buffer location (in GPU space) */ + shadow->quad_vertices.hostptr = cmd; + shadow->quad_vertices.gpuaddr = gpuaddr(cmd, &drawctxt->gpustate); + + cmd += QUAD_LEN; + + /* tex coord buffer location (in GPU space) */ + shadow->quad_texcoords.hostptr = cmd; + shadow->quad_texcoords.gpuaddr = gpuaddr(cmd, &drawctxt->gpustate); + + cmd += TEXCOORD_LEN; + + set_gmem_copy_quad(shadow); + + ctx->cmd = cmd; +} + +static void +build_shader_save_restore_cmds(struct adreno_context *drawctxt, + struct tmp_ctx *ctx) +{ + unsigned int *cmd = ctx->cmd; + unsigned int *save, *restore, *fixup; +#if defined(PM4_IM_STORE) + unsigned int *startSizeVtx, *startSizePix, *startSizeShared; +#endif + unsigned int *partition1; + unsigned int *shaderBases, *partition2; + +#if defined(PM4_IM_STORE) + /* compute vertex, pixel and shared instruction shadow GPU addresses */ + ctx->shader_vertex = drawctxt->gpustate.gpuaddr + SHADER_OFFSET; + ctx->shader_pixel = ctx->shader_vertex + SHADER_SHADOW_SIZE; + ctx->shader_shared = ctx->shader_pixel + SHADER_SHADOW_SIZE; +#endif + + /* restore shader partitioning and instructions */ + + restore = cmd; /* start address */ + + /* Invalidate Vertex & Pixel instruction code address and sizes */ + *cmd++ = pm4_type3_packet(PM4_INVALIDATE_STATE, 1); + *cmd++ = 0x00000300; /* 0x100 = Vertex, 0x200 = Pixel */ + + /* Restore previous shader vertex & pixel instruction bases. */ + *cmd++ = pm4_type3_packet(PM4_SET_SHADER_BASES, 1); + shaderBases = cmd++; /* TBD #5: shader bases (from fixup) */ + + /* write the shader partition information to a scratch register */ + *cmd++ = pm4_type0_packet(REG_SQ_INST_STORE_MANAGMENT, 1); + partition1 = cmd++; /* TBD #4a: partition info (from save) */ + +#if defined(PM4_IM_STORE) + /* load vertex shader instructions from the shadow. */ + *cmd++ = pm4_type3_packet(PM4_IM_LOAD, 2); + *cmd++ = ctx->shader_vertex + 0x0; /* 0x0 = Vertex */ + startSizeVtx = cmd++; /* TBD #1: start/size (from save) */ + + /* load pixel shader instructions from the shadow. */ + *cmd++ = pm4_type3_packet(PM4_IM_LOAD, 2); + *cmd++ = ctx->shader_pixel + 0x1; /* 0x1 = Pixel */ + startSizePix = cmd++; /* TBD #2: start/size (from save) */ + + /* load shared shader instructions from the shadow. */ + *cmd++ = pm4_type3_packet(PM4_IM_LOAD, 2); + *cmd++ = ctx->shader_shared + 0x2; /* 0x2 = Shared */ + startSizeShared = cmd++; /* TBD #3: start/size (from save) */ +#endif + + /* create indirect buffer command for above command sequence */ + create_ib1(drawctxt, drawctxt->shader_restore, restore, cmd); + + /* + * fixup SET_SHADER_BASES data + * + * since self-modifying PM4 code is being used here, a seperate + * command buffer is used for this fixup operation, to ensure the + * commands are not read by the PM4 engine before the data fields + * have been written. + */ + + fixup = cmd; /* start address */ + + /* write the shader partition information to a scratch register */ + *cmd++ = pm4_type0_packet(REG_SCRATCH_REG2, 1); + partition2 = cmd++; /* TBD #4b: partition info (from save) */ + + /* mask off unused bits, then OR with shader instruction memory size */ + *cmd++ = pm4_type3_packet(PM4_REG_RMW, 3); + *cmd++ = REG_SCRATCH_REG2; + /* AND off invalid bits. */ + *cmd++ = 0x0FFF0FFF; + /* OR in instruction memory size */ + *cmd++ = (unsigned int)((SHADER_INSTRUCT_LOG2 - 5U) << 29); + + /* write the computed value to the SET_SHADER_BASES data field */ + *cmd++ = pm4_type3_packet(PM4_REG_TO_MEM, 2); + *cmd++ = REG_SCRATCH_REG2; + /* TBD #5: shader bases (to restore) */ + *cmd++ = gpuaddr(shaderBases, &drawctxt->gpustate); + + /* create indirect buffer command for above command sequence */ + create_ib1(drawctxt, drawctxt->shader_fixup, fixup, cmd); + + /* save shader partitioning and instructions */ + + save = cmd; /* start address */ + + *cmd++ = pm4_type3_packet(PM4_WAIT_FOR_IDLE, 1); + *cmd++ = 0; + + /* fetch the SQ_INST_STORE_MANAGMENT register value, + * store the value in the data fields of the SET_CONSTANT commands + * above. + */ + *cmd++ = pm4_type3_packet(PM4_REG_TO_MEM, 2); + *cmd++ = REG_SQ_INST_STORE_MANAGMENT; + /* TBD #4a: partition info (to restore) */ + *cmd++ = gpuaddr(partition1, &drawctxt->gpustate); + *cmd++ = pm4_type3_packet(PM4_REG_TO_MEM, 2); + *cmd++ = REG_SQ_INST_STORE_MANAGMENT; + /* TBD #4b: partition info (to fixup) */ + *cmd++ = gpuaddr(partition2, &drawctxt->gpustate); + +#if defined(PM4_IM_STORE) + + /* store the vertex shader instructions */ + *cmd++ = pm4_type3_packet(PM4_IM_STORE, 2); + *cmd++ = ctx->shader_vertex + 0x0; /* 0x0 = Vertex */ + /* TBD #1: start/size (to restore) */ + *cmd++ = gpuaddr(startSizeVtx, &drawctxt->gpustate); + + /* store the pixel shader instructions */ + *cmd++ = pm4_type3_packet(PM4_IM_STORE, 2); + *cmd++ = ctx->shader_pixel + 0x1; /* 0x1 = Pixel */ + /* TBD #2: start/size (to restore) */ + *cmd++ = gpuaddr(startSizePix, &drawctxt->gpustate); + + /* store the shared shader instructions if vertex base is nonzero */ + + *cmd++ = pm4_type3_packet(PM4_IM_STORE, 2); + *cmd++ = ctx->shader_shared + 0x2; /* 0x2 = Shared */ + /* TBD #3: start/size (to restore) */ + *cmd++ = gpuaddr(startSizeShared, &drawctxt->gpustate); + +#endif + + *cmd++ = pm4_type3_packet(PM4_WAIT_FOR_IDLE, 1); + *cmd++ = 0; + + /* create indirect buffer command for above command sequence */ + create_ib1(drawctxt, drawctxt->shader_save, save, cmd); + + ctx->cmd = cmd; +} + +/* create buffers for saving/restoring registers, constants, & GMEM */ +static int +create_gpustate_shadow(struct kgsl_device *device, + struct adreno_context *drawctxt, + struct tmp_ctx *ctx) +{ + struct adreno_device *adreno_dev = ADRENO_DEVICE(device); + int result; + + /* Allocate vmalloc memory to store the gpustate */ + result = kgsl_sharedmem_vmalloc(&drawctxt->gpustate, + drawctxt->pagetable, CONTEXT_SIZE); + + if (result) + return result; + + drawctxt->flags |= CTXT_FLAGS_STATE_SHADOW; + + /* Blank out h/w register, constant, and command buffer shadows. */ + kgsl_sharedmem_set(&drawctxt->gpustate, 0, 0, CONTEXT_SIZE); + + /* set-up command and vertex buffer pointers */ + ctx->cmd = ctx->start + = (unsigned int *)((char *)drawctxt->gpustate.hostptr + CMD_OFFSET); + + /* build indirect command buffers to save & restore regs/constants */ + adreno_idle(device, KGSL_TIMEOUT_DEFAULT); + build_regrestore_cmds(adreno_dev, drawctxt, ctx); + build_regsave_cmds(adreno_dev, drawctxt, ctx); + + build_shader_save_restore_cmds(drawctxt, ctx); + + kgsl_cache_range_op(&drawctxt->gpustate, + KGSL_CACHE_OP_FLUSH); + + return 0; +} + +/* create buffers for saving/restoring registers, constants, & GMEM */ +static int +create_gmem_shadow(struct adreno_device *adreno_dev, + struct adreno_context *drawctxt, + struct tmp_ctx *ctx) +{ + struct kgsl_device *device = &adreno_dev->dev; + int result; + + config_gmemsize(&drawctxt->context_gmem_shadow, + adreno_dev->gmemspace.sizebytes); + ctx->gmem_base = adreno_dev->gmemspace.gpu_base; + + result = kgsl_sharedmem_vmalloc( + &drawctxt->context_gmem_shadow.gmemshadow, + drawctxt->pagetable, + drawctxt->context_gmem_shadow.size); + + if (result) + return result; + + /* we've allocated the shadow, when swapped out, GMEM must be saved. */ + drawctxt->flags |= CTXT_FLAGS_GMEM_SHADOW | CTXT_FLAGS_GMEM_SAVE; + + /* blank out gmem shadow. */ + kgsl_sharedmem_set(&drawctxt->context_gmem_shadow.gmemshadow, 0, 0, + drawctxt->context_gmem_shadow.size); + + /* build quad vertex buffer */ + build_quad_vtxbuff(drawctxt, ctx, &drawctxt->context_gmem_shadow); + + /* build TP0_CHICKEN register restore command buffer */ + ctx->cmd = build_chicken_restore_cmds(drawctxt, ctx); + + /* build indirect command buffers to save & restore gmem */ + /* Idle because we are reading PM override registers */ + adreno_idle(device, KGSL_TIMEOUT_DEFAULT); + drawctxt->context_gmem_shadow.gmem_save_commands = ctx->cmd; + ctx->cmd = + build_gmem2sys_cmds(adreno_dev, drawctxt, ctx, + &drawctxt->context_gmem_shadow); + drawctxt->context_gmem_shadow.gmem_restore_commands = ctx->cmd; + ctx->cmd = + build_sys2gmem_cmds(adreno_dev, drawctxt, ctx, + &drawctxt->context_gmem_shadow); + + kgsl_cache_range_op(&drawctxt->context_gmem_shadow.gmemshadow, + KGSL_CACHE_OP_FLUSH); + + return 0; +} + +/* create a new drawing context */ + +int +adreno_drawctxt_create(struct kgsl_device_private *dev_priv, uint32_t flags, + struct kgsl_context *context) +{ + struct adreno_context *drawctxt; + struct kgsl_device *device = dev_priv->device; + struct adreno_device *adreno_dev = ADRENO_DEVICE(device); + struct kgsl_pagetable *pagetable = dev_priv->process_priv->pagetable; + struct tmp_ctx ctx; + int ret; + + drawctxt = kzalloc(sizeof(struct adreno_context), GFP_KERNEL); + + if (drawctxt == NULL) + return -ENOMEM; + + drawctxt->pagetable = pagetable; + drawctxt->bin_base_offset = 0; + + ret = create_gpustate_shadow(device, drawctxt, &ctx); + if (ret) + goto err; + + /* Save the shader instruction memory on context switching */ + drawctxt->flags |= CTXT_FLAGS_SHADER_SAVE; + + memset(&drawctxt->context_gmem_shadow.gmemshadow, + 0, sizeof(struct kgsl_memdesc)); + + if (!(flags & KGSL_CONTEXT_NO_GMEM_ALLOC)) { + /* create gmem shadow */ + ret = create_gmem_shadow(adreno_dev, drawctxt, &ctx); + if (ret != 0) + goto err; + } + + BUG_ON(ctx.cmd - ctx.start > CMD_BUFFER_LEN); + + context->devctxt = drawctxt; + return 0; +err: + kgsl_sharedmem_free(&drawctxt->gpustate); + kfree(drawctxt); + return ret; +} + +/* destroy a drawing context */ + +int adreno_drawctxt_destroy(struct kgsl_device *device, + struct kgsl_context *context) +{ + struct adreno_device *adreno_dev = ADRENO_DEVICE(device); + struct adreno_context *drawctxt = context->devctxt; + + if (drawctxt == NULL) + return -EINVAL; + + /* deactivate context */ + if (adreno_dev->drawctxt_active == drawctxt) { + /* no need to save GMEM or shader, the context is + * being destroyed. + */ + drawctxt->flags &= ~(CTXT_FLAGS_GMEM_SAVE | + CTXT_FLAGS_SHADER_SAVE | + CTXT_FLAGS_GMEM_SHADOW | + CTXT_FLAGS_STATE_SHADOW); + + adreno_drawctxt_switch(adreno_dev, NULL, 0); + } + + adreno_idle(device, KGSL_TIMEOUT_DEFAULT); + + kgsl_sharedmem_free(&drawctxt->gpustate); + kgsl_sharedmem_free(&drawctxt->context_gmem_shadow.gmemshadow); + + kfree(drawctxt); + context->devctxt = NULL; + + return 0; +} + +/* set bin base offset */ +int adreno_drawctxt_set_bin_base_offset(struct kgsl_device *device, + struct kgsl_context *context, + unsigned int offset) +{ + struct adreno_context *drawctxt = context->devctxt; + + if (drawctxt == NULL) + return -EINVAL; + + drawctxt->bin_base_offset = offset; + + return 0; +} + +/* switch drawing contexts */ +void +adreno_drawctxt_switch(struct adreno_device *adreno_dev, + struct adreno_context *drawctxt, + unsigned int flags) +{ + struct adreno_context *active_ctxt = + adreno_dev->drawctxt_active; + struct kgsl_device *device = &adreno_dev->dev; + unsigned int cmds[5]; + + if (drawctxt) { + if (flags & KGSL_CONTEXT_SAVE_GMEM) + /* Set the flag in context so that the save is done + * when this context is switched out. */ + drawctxt->flags |= CTXT_FLAGS_GMEM_SAVE; + else + /* Remove GMEM saving flag from the context */ + drawctxt->flags &= ~CTXT_FLAGS_GMEM_SAVE; + } + /* already current? */ + if (active_ctxt == drawctxt) + return; + + KGSL_CTXT_INFO(device, "from %p to %p flags %d\n", + adreno_dev->drawctxt_active, drawctxt, flags); + /* save old context*/ + if (active_ctxt && active_ctxt->flags & CTXT_FLAGS_GPU_HANG) + KGSL_CTXT_WARN(device, + "Current active context has caused gpu hang\n"); + + if (active_ctxt != NULL) { + KGSL_CTXT_INFO(device, + "active_ctxt flags %08x\n", active_ctxt->flags); + /* save registers and constants. */ + adreno_ringbuffer_issuecmds(device, 0, + active_ctxt->reg_save, 3); + + if (active_ctxt->flags & CTXT_FLAGS_SHADER_SAVE) { + /* save shader partitioning and instructions. */ + adreno_ringbuffer_issuecmds(device, + KGSL_CMD_FLAGS_PMODE, + active_ctxt->shader_save, 3); + + /* fixup shader partitioning parameter for + * SET_SHADER_BASES. + */ + adreno_ringbuffer_issuecmds(device, 0, + active_ctxt->shader_fixup, 3); + + active_ctxt->flags |= CTXT_FLAGS_SHADER_RESTORE; + } + + if (active_ctxt->flags & CTXT_FLAGS_GMEM_SAVE + && active_ctxt->flags & CTXT_FLAGS_GMEM_SHADOW) { + /* save gmem. + * (note: changes shader. shader must already be saved.) + */ + adreno_ringbuffer_issuecmds(device, + KGSL_CMD_FLAGS_PMODE, + active_ctxt->context_gmem_shadow.gmem_save, 3); + + /* Restore TP0_CHICKEN */ + adreno_ringbuffer_issuecmds(device, 0, + active_ctxt->chicken_restore, 3); + + active_ctxt->flags |= CTXT_FLAGS_GMEM_RESTORE; + } + } + + adreno_dev->drawctxt_active = drawctxt; + + /* restore new context */ + if (drawctxt != NULL) { + + KGSL_CTXT_INFO(device, + "drawctxt flags %08x\n", drawctxt->flags); + cmds[0] = pm4_nop_packet(1); + cmds[1] = KGSL_CONTEXT_TO_MEM_IDENTIFIER; + cmds[2] = pm4_type3_packet(PM4_MEM_WRITE, 2); + cmds[3] = device->memstore.gpuaddr + + KGSL_DEVICE_MEMSTORE_OFFSET(current_context); + cmds[4] = (unsigned int)adreno_dev->drawctxt_active; + adreno_ringbuffer_issuecmds(device, 0, cmds, 5); + kgsl_mmu_setstate(device, drawctxt->pagetable); + +#ifndef CONFIG_MSM_KGSL_CFF_DUMP_NO_CONTEXT_MEM_DUMP + kgsl_cffdump_syncmem(NULL, &drawctxt->gpustate, + drawctxt->gpustate.gpuaddr, LCC_SHADOW_SIZE + + REG_SHADOW_SIZE + CMD_BUFFER_SIZE + TEX_SHADOW_SIZE, + false); +#endif + + /* restore gmem. + * (note: changes shader. shader must not already be restored.) + */ + if (drawctxt->flags & CTXT_FLAGS_GMEM_RESTORE) { + adreno_ringbuffer_issuecmds(device, + KGSL_CMD_FLAGS_PMODE, + drawctxt->context_gmem_shadow.gmem_restore, 3); + + /* Restore TP0_CHICKEN */ + adreno_ringbuffer_issuecmds(device, 0, + drawctxt->chicken_restore, 3); + + drawctxt->flags &= ~CTXT_FLAGS_GMEM_RESTORE; + } + + /* restore registers and constants. */ + adreno_ringbuffer_issuecmds(device, 0, + drawctxt->reg_restore, 3); + + /* restore shader instructions & partitioning. */ + if (drawctxt->flags & CTXT_FLAGS_SHADER_RESTORE) { + adreno_ringbuffer_issuecmds(device, 0, + drawctxt->shader_restore, 3); + } + + cmds[0] = pm4_type3_packet(PM4_SET_BIN_BASE_OFFSET, 1); + cmds[1] = drawctxt->bin_base_offset; + if (!adreno_is_a220(adreno_dev)) + adreno_ringbuffer_issuecmds(device, 0, cmds, 2); + + } else + kgsl_mmu_setstate(device, device->mmu.defaultpagetable); +} diff --git a/drivers/gpu/msm/adreno_drawctxt.h b/drivers/gpu/msm/adreno_drawctxt.h new file mode 100644 index 00000000..8ea40436 --- /dev/null +++ b/drivers/gpu/msm/adreno_drawctxt.h @@ -0,0 +1,113 @@ +/* Copyright (c) 2002,2007-2011, Code Aurora Forum. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are + * met: + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following + * disclaimer in the documentation and/or other materials provided + * with the distribution. + * * Neither the name of Code Aurora Forum, Inc. nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED "AS IS" AND ANY EXPRESS OR IMPLIED + * WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS + * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR + * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, + * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE + * OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN + * IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + */ +#ifndef __ADRENO_DRAWCTXT_H +#define __ADRENO_DRAWCTXT_H + +#include "a200_reg.h" +#include "a220_reg.h" + +/* Flags */ + +#define CTXT_FLAGS_NOT_IN_USE 0x00000000 +#define CTXT_FLAGS_IN_USE 0x00000001 + +/* state shadow memory allocated */ +#define CTXT_FLAGS_STATE_SHADOW 0x00000010 + +/* gmem shadow memory allocated */ +#define CTXT_FLAGS_GMEM_SHADOW 0x00000100 +/* gmem must be copied to shadow */ +#define CTXT_FLAGS_GMEM_SAVE 0x00000200 +/* gmem can be restored from shadow */ +#define CTXT_FLAGS_GMEM_RESTORE 0x00000400 +/* shader must be copied to shadow */ +#define CTXT_FLAGS_SHADER_SAVE 0x00002000 +/* shader can be restored from shadow */ +#define CTXT_FLAGS_SHADER_RESTORE 0x00004000 +/* Context has caused a GPU hang */ +#define CTXT_FLAGS_GPU_HANG 0x00008000 + +struct kgsl_device; +struct adreno_device; +struct kgsl_device_private; +struct kgsl_context; + +/* draw context */ +struct gmem_shadow_t { + struct kgsl_memdesc gmemshadow; /* Shadow buffer address */ + + /* 256 KB GMEM surface = 4 bytes-per-pixel x 256 pixels/row x + * 256 rows. */ + /* width & height must be a multiples of 32, in case tiled textures + * are used. */ + enum COLORFORMATX format; + unsigned int size; /* Size of surface used to store GMEM */ + unsigned int width; /* Width of surface used to store GMEM */ + unsigned int height; /* Height of surface used to store GMEM */ + unsigned int pitch; /* Pitch of surface used to store GMEM */ + unsigned int gmem_pitch; /* Pitch value used for GMEM */ + unsigned int *gmem_save_commands; + unsigned int *gmem_restore_commands; + unsigned int gmem_save[3]; + unsigned int gmem_restore[3]; + struct kgsl_memdesc quad_vertices; + struct kgsl_memdesc quad_texcoords; +}; + +struct adreno_context { + uint32_t flags; + struct kgsl_pagetable *pagetable; + struct kgsl_memdesc gpustate; + unsigned int reg_save[3]; + unsigned int reg_restore[3]; + unsigned int shader_save[3]; + unsigned int shader_fixup[3]; + unsigned int shader_restore[3]; + unsigned int chicken_restore[3]; + unsigned int bin_base_offset; + /* Information of the GMEM shadow that is created in context create */ + struct gmem_shadow_t context_gmem_shadow; +}; + + +int adreno_drawctxt_create(struct kgsl_device_private *dev_priv, + uint32_t flags, + struct kgsl_context *context); + +int adreno_drawctxt_destroy(struct kgsl_device *device, + struct kgsl_context *context); + +void adreno_drawctxt_switch(struct adreno_device *adreno_dev, + struct adreno_context *drawctxt, + unsigned int flags); +int adreno_drawctxt_set_bin_base_offset(struct kgsl_device *device, + struct kgsl_context *context, + unsigned int offset); + +#endif /* __ADRENO_DRAWCTXT_H */ diff --git a/drivers/gpu/msm/adreno_pm4types.h b/drivers/gpu/msm/adreno_pm4types.h new file mode 100644 index 00000000..246315b6 --- /dev/null +++ b/drivers/gpu/msm/adreno_pm4types.h @@ -0,0 +1,193 @@ +/* Copyright (c) 2002,2007-2011, Code Aurora Forum. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are + * met: + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following + * disclaimer in the documentation and/or other materials provided + * with the distribution. + * * Neither the name of Code Aurora Forum, Inc. nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED "AS IS" AND ANY EXPRESS OR IMPLIED + * WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS + * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR + * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, + * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE + * OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN + * IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + */ +#ifndef __ADRENO_PM4TYPES_H +#define __ADRENO_PM4TYPES_H + + +#define PM4_PKT_MASK 0xc0000000 + +#define PM4_TYPE0_PKT ((unsigned int)0 << 30) +#define PM4_TYPE1_PKT ((unsigned int)1 << 30) +#define PM4_TYPE2_PKT ((unsigned int)2 << 30) +#define PM4_TYPE3_PKT ((unsigned int)3 << 30) + + +/* type3 packets */ +/* initialize CP's micro-engine */ +#define PM4_ME_INIT 0x48 + +/* skip N 32-bit words to get to the next packet */ +#define PM4_NOP 0x10 + +/* indirect buffer dispatch. prefetch parser uses this packet type to determine +* whether to pre-fetch the IB +*/ +#define PM4_INDIRECT_BUFFER 0x3f + +/* indirect buffer dispatch. same as IB, but init is pipelined */ +#define PM4_INDIRECT_BUFFER_PFD 0x37 + +/* wait for the IDLE state of the engine */ +#define PM4_WAIT_FOR_IDLE 0x26 + +/* wait until a register or memory location is a specific value */ +#define PM4_WAIT_REG_MEM 0x3c + +/* wait until a register location is equal to a specific value */ +#define PM4_WAIT_REG_EQ 0x52 + +/* wait until a register location is >= a specific value */ +#define PM4_WAT_REG_GTE 0x53 + +/* wait until a read completes */ +#define PM4_WAIT_UNTIL_READ 0x5c + +/* wait until all base/size writes from an IB_PFD packet have completed */ +#define PM4_WAIT_IB_PFD_COMPLETE 0x5d + +/* register read/modify/write */ +#define PM4_REG_RMW 0x21 + +/* reads register in chip and writes to memory */ +#define PM4_REG_TO_MEM 0x3e + +/* write N 32-bit words to memory */ +#define PM4_MEM_WRITE 0x3d + +/* write CP_PROG_COUNTER value to memory */ +#define PM4_MEM_WRITE_CNTR 0x4f + +/* conditional execution of a sequence of packets */ +#define PM4_COND_EXEC 0x44 + +/* conditional write to memory or register */ +#define PM4_COND_WRITE 0x45 + +/* generate an event that creates a write to memory when completed */ +#define PM4_EVENT_WRITE 0x46 + +/* generate a VS|PS_done event */ +#define PM4_EVENT_WRITE_SHD 0x58 + +/* generate a cache flush done event */ +#define PM4_EVENT_WRITE_CFL 0x59 + +/* generate a z_pass done event */ +#define PM4_EVENT_WRITE_ZPD 0x5b + + +/* initiate fetch of index buffer and draw */ +#define PM4_DRAW_INDX 0x22 + +/* draw using supplied indices in packet */ +#define PM4_DRAW_INDX_2 0x36 + +/* initiate fetch of index buffer and binIDs and draw */ +#define PM4_DRAW_INDX_BIN 0x34 + +/* initiate fetch of bin IDs and draw using supplied indices */ +#define PM4_DRAW_INDX_2_BIN 0x35 + + +/* begin/end initiator for viz query extent processing */ +#define PM4_VIZ_QUERY 0x23 + +/* fetch state sub-blocks and initiate shader code DMAs */ +#define PM4_SET_STATE 0x25 + +/* load constant into chip and to memory */ +#define PM4_SET_CONSTANT 0x2d + +/* load sequencer instruction memory (pointer-based) */ +#define PM4_IM_LOAD 0x27 + +/* load sequencer instruction memory (code embedded in packet) */ +#define PM4_IM_LOAD_IMMEDIATE 0x2b + +/* load constants from a location in memory */ +#define PM4_LOAD_CONSTANT_CONTEXT 0x2e + +/* selective invalidation of state pointers */ +#define PM4_INVALIDATE_STATE 0x3b + + +/* dynamically changes shader instruction memory partition */ +#define PM4_SET_SHADER_BASES 0x4A + +/* sets the 64-bit BIN_MASK register in the PFP */ +#define PM4_SET_BIN_MASK 0x50 + +/* sets the 64-bit BIN_SELECT register in the PFP */ +#define PM4_SET_BIN_SELECT 0x51 + + +/* updates the current context, if needed */ +#define PM4_CONTEXT_UPDATE 0x5e + +/* generate interrupt from the command stream */ +#define PM4_INTERRUPT 0x40 + + +/* copy sequencer instruction memory to system memory */ +#define PM4_IM_STORE 0x2c + +/* program an offset that will added to the BIN_BASE value of + * the 3D_DRAW_INDX_BIN packet */ +#define PM4_SET_BIN_BASE_OFFSET 0x4B + +#define PM4_SET_PROTECTED_MODE 0x5f /* sets the register protection mode */ + + +/* packet header building macros */ +#define pm4_type0_packet(regindx, cnt) \ + (PM4_TYPE0_PKT | (((cnt)-1) << 16) | ((regindx) & 0x7FFF)) + +#define pm4_type0_packet_for_sameregister(regindx, cnt) \ + ((PM4_TYPE0_PKT | (((cnt)-1) << 16) | ((1 << 15) | \ + ((regindx) & 0x7FFF))) + +#define pm4_type1_packet(reg0, reg1) \ + (PM4_TYPE1_PKT | ((reg1) << 12) | (reg0)) + +#define pm4_type3_packet(opcode, cnt) \ + (PM4_TYPE3_PKT | (((cnt)-1) << 16) | (((opcode) & 0xFF) << 8)) + +#define pm4_predicated_type3_packet(opcode, cnt) \ + (PM4_TYPE3_PKT | (((cnt)-1) << 16) | (((opcode) & 0xFF) << 8) | 0x1) + +#define pm4_nop_packet(cnt) \ + (PM4_TYPE3_PKT | (((cnt)-1) << 16) | (PM4_NOP << 8)) + + +/* packet headers */ +#define PM4_HDR_ME_INIT pm4_type3_packet(PM4_ME_INIT, 18) +#define PM4_HDR_INDIRECT_BUFFER_PFD pm4_type3_packet(PM4_INDIRECT_BUFFER_PFD, 2) +#define PM4_HDR_INDIRECT_BUFFER pm4_type3_packet(PM4_INDIRECT_BUFFER, 2) + +#endif /* __ADRENO_PM4TYPES_H */ diff --git a/drivers/gpu/msm/adreno_postmortem.c b/drivers/gpu/msm/adreno_postmortem.c new file mode 100644 index 00000000..4911e937 --- /dev/null +++ b/drivers/gpu/msm/adreno_postmortem.c @@ -0,0 +1,854 @@ +/* Copyright (c) 2010-2011, Code Aurora Forum. All rights reserved. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 and + * only version 2 as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + */ + +#include + +#include "kgsl.h" + +#include "adreno.h" +#include "adreno_pm4types.h" +#include "adreno_ringbuffer.h" +#include "adreno_postmortem.h" +#include "adreno_debugfs.h" +#include "kgsl_cffdump.h" + +#include "a200_reg.h" + +#define INVALID_RB_CMD 0xaaaaaaaa + +struct pm_id_name { + uint32_t id; + char name[9]; +}; + +static const struct pm_id_name pm0_types[] = { + {REG_PA_SC_AA_CONFIG, "RPASCAAC"}, + {REG_RBBM_PM_OVERRIDE2, "RRBBPMO2"}, + {REG_SCRATCH_REG2, "RSCRTRG2"}, + {REG_SQ_GPR_MANAGEMENT, "RSQGPRMN"}, + {REG_SQ_INST_STORE_MANAGMENT, "RSQINSTS"}, + {REG_TC_CNTL_STATUS, "RTCCNTLS"}, + {REG_TP0_CHICKEN, "RTP0CHCK"}, + {REG_CP_TIMESTAMP, "CP_TM_ST"}, +}; + +static const struct pm_id_name pm3_types[] = { + {PM4_COND_EXEC, "CND_EXEC"}, + {PM4_CONTEXT_UPDATE, "CX__UPDT"}, + {PM4_DRAW_INDX, "DRW_NDX_"}, + {PM4_DRAW_INDX_BIN, "DRW_NDXB"}, + {PM4_EVENT_WRITE, "EVENT_WT"}, + {PM4_IM_LOAD, "IN__LOAD"}, + {PM4_IM_LOAD_IMMEDIATE, "IM_LOADI"}, + {PM4_IM_STORE, "IM_STORE"}, + {PM4_INDIRECT_BUFFER, "IND_BUF_"}, + {PM4_INDIRECT_BUFFER_PFD, "IND_BUFP"}, + {PM4_INTERRUPT, "PM4_INTR"}, + {PM4_INVALIDATE_STATE, "INV_STAT"}, + {PM4_LOAD_CONSTANT_CONTEXT, "LD_CN_CX"}, + {PM4_ME_INIT, "ME__INIT"}, + {PM4_NOP, "PM4__NOP"}, + {PM4_REG_RMW, "REG__RMW"}, + {PM4_REG_TO_MEM, "REG2_MEM"}, + {PM4_SET_BIN_BASE_OFFSET, "ST_BIN_O"}, + {PM4_SET_CONSTANT, "ST_CONST"}, + {PM4_SET_PROTECTED_MODE, "ST_PRT_M"}, + {PM4_SET_SHADER_BASES, "ST_SHD_B"}, + {PM4_WAIT_FOR_IDLE, "WAIT4IDL"}, +}; + +/* Offset address pairs: start, end of range to dump (inclusive) */ + +/* GPU < Z470 */ + +static const int a200_registers[] = { + 0x0000, 0x0008, 0x0010, 0x002c, 0x00ec, 0x00f4, + 0x0100, 0x0110, 0x0118, 0x011c, + 0x0700, 0x0704, 0x070c, 0x0720, 0x0754, 0x0764, + 0x0770, 0x0774, 0x07a8, 0x07a8, 0x07b8, 0x07cc, + 0x07d8, 0x07dc, 0x07f0, 0x07fc, 0x0e44, 0x0e48, + 0x0e6c, 0x0e78, 0x0ec8, 0x0ed4, 0x0edc, 0x0edc, + 0x0fe0, 0x0fec, 0x1100, 0x1100, + + 0x110c, 0x1110, 0x112c, 0x112c, 0x1134, 0x113c, + 0x1148, 0x1148, 0x1150, 0x116c, 0x11fc, 0x11fc, + 0x15e0, 0x161c, 0x1724, 0x1724, 0x1740, 0x1740, + 0x1804, 0x1810, 0x1818, 0x1824, 0x182c, 0x1838, + 0x184c, 0x1850, 0x28a4, 0x28ac, 0x28bc, 0x28c4, + 0x2900, 0x290c, 0x2914, 0x2914, 0x2938, 0x293c, + 0x30b0, 0x30b0, 0x30c0, 0x30c0, 0x30e0, 0x30f0, + 0x3100, 0x3100, 0x3110, 0x3110, 0x3200, 0x3218, + 0x3220, 0x3250, 0x3264, 0x3268, 0x3290, 0x3294, + 0x3400, 0x340c, 0x3418, 0x3418, 0x3420, 0x342c, + 0x34d0, 0x34d4, 0x36b8, 0x3704, 0x3720, 0x3750, + 0x3760, 0x3764, 0x3800, 0x3800, 0x3808, 0x3810, + 0x385c, 0x3878, 0x3b00, 0x3b24, 0x3b2c, 0x3b30, + 0x3b40, 0x3b40, 0x3b50, 0x3b5c, 0x3b80, 0x3b88, + 0x3c04, 0x3c08, 0x3c30, 0x3c30, 0x3c38, 0x3c48, + 0x3c98, 0x3ca8, 0x3cb0, 0x3cb0, + + 0x8000, 0x8008, 0x8018, 0x803c, 0x8200, 0x8208, + 0x8400, 0x8424, 0x8430, 0x8450, 0x8600, 0x8610, + 0x87d4, 0x87dc, 0x8800, 0x8820, 0x8a00, 0x8a0c, + 0x8a4c, 0x8a50, 0x8c00, 0x8c20, 0x8c48, 0x8c48, + 0x8c58, 0x8c74, 0x8c90, 0x8c98, 0x8e00, 0x8e0c, + + 0x9000, 0x9008, 0x9018, 0x903c, 0x9200, 0x9208, + 0x9400, 0x9424, 0x9430, 0x9450, 0x9600, 0x9610, + 0x97d4, 0x97dc, 0x9800, 0x9820, 0x9a00, 0x9a0c, + 0x9a4c, 0x9a50, 0x9c00, 0x9c20, 0x9c48, 0x9c48, + 0x9c58, 0x9c74, 0x9c90, 0x9c98, 0x9e00, 0x9e0c, + + 0x10000, 0x1000c, 0x12000, 0x12014, + 0x12400, 0x12400, 0x12420, 0x12420 +}; + +/* GPU = Z470 */ + +static const int a220_registers[] = { + 0x0000, 0x0008, 0x0010, 0x002c, 0x00ec, 0x00f4, + 0x0100, 0x0110, 0x0118, 0x011c, + 0x0700, 0x0704, 0x070c, 0x0720, 0x0754, 0x0764, + 0x0770, 0x0774, 0x07a8, 0x07a8, 0x07b8, 0x07cc, + 0x07d8, 0x07dc, 0x07f0, 0x07fc, 0x0e44, 0x0e48, + 0x0e6c, 0x0e78, 0x0ec8, 0x0ed4, 0x0edc, 0x0edc, + 0x0fe0, 0x0fec, 0x1100, 0x1100, + + 0x110c, 0x1110, 0x112c, 0x112c, 0x1134, 0x113c, + 0x1148, 0x1148, 0x1150, 0x116c, 0x11fc, 0x11fc, + 0x15e0, 0x161c, 0x1724, 0x1724, 0x1740, 0x1740, + 0x1804, 0x1810, 0x1818, 0x1824, 0x182c, 0x1838, + 0x184c, 0x1850, 0x28a4, 0x28ac, 0x28bc, 0x28c4, + 0x2900, 0x2900, 0x2908, 0x290c, 0x2914, 0x2914, + 0x2938, 0x293c, 0x30c0, 0x30c0, 0x30e0, 0x30e4, + 0x30f0, 0x30f0, 0x3200, 0x3204, 0x3220, 0x324c, + 0x3400, 0x340c, 0x3414, 0x3418, 0x3420, 0x342c, + 0x34d0, 0x34d4, 0x36b8, 0x3704, 0x3720, 0x3750, + 0x3760, 0x3764, 0x3800, 0x3800, 0x3808, 0x3810, + 0x385c, 0x3878, 0x3b00, 0x3b24, 0x3b2c, 0x3b30, + 0x3b40, 0x3b40, 0x3b50, 0x3b5c, 0x3b80, 0x3b88, + 0x3c04, 0x3c08, 0x8000, 0x8008, 0x8018, 0x803c, + 0x8200, 0x8208, 0x8400, 0x8408, 0x8410, 0x8424, + 0x8430, 0x8450, 0x8600, 0x8610, 0x87d4, 0x87dc, + 0x8800, 0x8808, 0x8810, 0x8810, 0x8820, 0x8820, + 0x8a00, 0x8a08, 0x8a50, 0x8a50, + 0x8c00, 0x8c20, 0x8c24, 0x8c28, 0x8c48, 0x8c48, + 0x8c58, 0x8c58, 0x8c60, 0x8c74, 0x8c90, 0x8c98, + 0x8e00, 0x8e0c, 0x9000, 0x9008, 0x9018, 0x903c, + 0x9200, 0x9208, 0x9400, 0x9408, 0x9410, 0x9424, + 0x9430, 0x9450, 0x9600, 0x9610, 0x97d4, 0x97dc, + 0x9800, 0x9808, 0x9810, 0x9818, 0x9820, 0x9820, + 0x9a00, 0x9a08, 0x9a50, 0x9a50, 0x9c00, 0x9c20, + 0x9c48, 0x9c48, 0x9c58, 0x9c58, 0x9c60, 0x9c74, + 0x9c90, 0x9c98, 0x9e00, 0x9e0c, + + 0x10000, 0x1000c, 0x12000, 0x12014, + 0x12400, 0x12400, 0x12420, 0x12420 +}; + +static uint32_t adreno_is_pm4_len(uint32_t word) +{ + if (word == INVALID_RB_CMD) + return 0; + + return (word >> 16) & 0x3FFF; +} + +static bool adreno_is_pm4_type(uint32_t word) +{ + int i; + + if (word == INVALID_RB_CMD) + return 1; + + if (adreno_is_pm4_len(word) > 16) + return 0; + + if ((word & (3<<30)) == PM4_TYPE0_PKT) { + for (i = 0; i < ARRAY_SIZE(pm0_types); ++i) { + if ((word & 0x7FFF) == pm0_types[i].id) + return 1; + } + return 0; + } + if ((word & (3<<30)) == PM4_TYPE3_PKT) { + for (i = 0; i < ARRAY_SIZE(pm3_types); ++i) { + if ((word & 0xFFFF) == (pm3_types[i].id << 8)) + return 1; + } + return 0; + } + return 0; +} + +static const char *adreno_pm4_name(uint32_t word) +{ + int i; + + if (word == INVALID_RB_CMD) + return "--------"; + + if ((word & (3<<30)) == PM4_TYPE0_PKT) { + for (i = 0; i < ARRAY_SIZE(pm0_types); ++i) { + if ((word & 0x7FFF) == pm0_types[i].id) + return pm0_types[i].name; + } + return "????????"; + } + if ((word & (3<<30)) == PM4_TYPE3_PKT) { + for (i = 0; i < ARRAY_SIZE(pm3_types); ++i) { + if ((word & 0xFFFF) == (pm3_types[i].id << 8)) + return pm3_types[i].name; + } + return "????????"; + } + return "????????"; +} + +static void adreno_dump_regs(struct kgsl_device *device, + const int *registers, int size) +{ + int range = 0, offset = 0; + + for (range = 0; range < size; range++) { + /* start and end are in dword offsets */ + int start = registers[range * 2] / 4; + int end = registers[range * 2 + 1] / 4; + + unsigned char linebuf[32 * 3 + 2 + 32 + 1]; + int linelen, i; + + for (offset = start; offset <= end; offset += linelen) { + unsigned int regvals[32/4]; + linelen = min(end+1-offset, 32/4); + + for (i = 0; i < linelen; ++i) + kgsl_regread(device, offset+i, regvals+i); + + hex_dump_to_buffer(regvals, linelen*4, 32, 4, + linebuf, sizeof(linebuf), 0); + KGSL_LOG_DUMP(device, + "REG: %5.5X: %s\n", offset<<2, linebuf); + } + } +} + +static void dump_ib(struct kgsl_device *device, char* buffId, uint32_t pt_base, + uint32_t base_offset, uint32_t ib_base, uint32_t ib_size, bool dump) +{ + unsigned int memsize; + uint8_t *base_addr = kgsl_sharedmem_convertaddr(device, pt_base, + ib_base, &memsize); + + if (base_addr && dump) + print_hex_dump(KERN_ERR, buffId, DUMP_PREFIX_OFFSET, + 32, 4, base_addr, ib_size*4, 0); + else + KGSL_LOG_DUMP(device, "%s base:%8.8X ib_size:%d " + "offset:%5.5X%s\n", + buffId, ib_base, ib_size*4, base_offset, + base_addr ? "" : " [Invalid]"); +} + +#define IB_LIST_SIZE 64 +struct ib_list { + int count; + uint32_t bases[IB_LIST_SIZE]; + uint32_t sizes[IB_LIST_SIZE]; + uint32_t offsets[IB_LIST_SIZE]; +}; + +static void dump_ib1(struct kgsl_device *device, uint32_t pt_base, + uint32_t base_offset, + uint32_t ib1_base, uint32_t ib1_size, + struct ib_list *ib_list, bool dump) +{ + int i, j; + uint32_t value; + uint32_t *ib1_addr; + unsigned int memsize; + + dump_ib(device, "IB1:", pt_base, base_offset, ib1_base, + ib1_size, dump); + + /* fetch virtual address for given IB base */ + ib1_addr = (uint32_t *)kgsl_sharedmem_convertaddr(device, pt_base, + ib1_base, &memsize); + if (!ib1_addr) + return; + + for (i = 0; i+3 < ib1_size; ) { + value = ib1_addr[i++]; + if (value == pm4_type3_packet(PM4_INDIRECT_BUFFER_PFD, 2)) { + uint32_t ib2_base = ib1_addr[i++]; + uint32_t ib2_size = ib1_addr[i++]; + + /* find previous match */ + for (j = 0; j < ib_list->count; ++j) + if (ib_list->sizes[j] == ib2_size + && ib_list->bases[j] == ib2_base) + break; + + if (j < ib_list->count || ib_list->count + >= IB_LIST_SIZE) + continue; + + /* store match */ + ib_list->sizes[ib_list->count] = ib2_size; + ib_list->bases[ib_list->count] = ib2_base; + ib_list->offsets[ib_list->count] = i<<2; + ++ib_list->count; + } + } +} + +static void adreno_dump_rb_buffer(const void *buf, size_t len, + char *linebuf, size_t linebuflen, int *argp) +{ + const u32 *ptr4 = buf; + const int ngroups = len; + int lx = 0, j; + bool nxsp = 1; + + for (j = 0; j < ngroups; j++) { + if (*argp < 0) { + lx += scnprintf(linebuf + lx, linebuflen - lx, " <"); + *argp = -*argp; + } else if (nxsp) + lx += scnprintf(linebuf + lx, linebuflen - lx, " "); + else + nxsp = 1; + if (!*argp && adreno_is_pm4_type(ptr4[j])) { + lx += scnprintf(linebuf + lx, linebuflen - lx, + "%s", adreno_pm4_name(ptr4[j])); + *argp = -(adreno_is_pm4_len(ptr4[j])+1); + } else { + lx += scnprintf(linebuf + lx, linebuflen - lx, + "%8.8X", ptr4[j]); + if (*argp > 1) + --*argp; + else if (*argp == 1) { + *argp = 0; + nxsp = 0; + lx += scnprintf(linebuf + lx, linebuflen - lx, + "> "); + } + } + } + linebuf[lx] = '\0'; +} + +static bool adreno_rb_use_hex(void) +{ +#ifdef CONFIG_MSM_KGSL_PSTMRTMDMP_RB_HEX + return 1; +#else + return 0; +#endif +} + +static void adreno_dump_rb(struct kgsl_device *device, const void *buf, + size_t len, int start, int size) +{ + const uint32_t *ptr = buf; + int i, remaining, args = 0; + unsigned char linebuf[32 * 3 + 2 + 32 + 1]; + const int rowsize = 8; + + len >>= 2; + remaining = len; + for (i = 0; i < len; i += rowsize) { + int linelen = min(remaining, rowsize); + remaining -= rowsize; + + if (adreno_rb_use_hex()) + hex_dump_to_buffer(ptr+i, linelen*4, rowsize*4, 4, + linebuf, sizeof(linebuf), 0); + else + adreno_dump_rb_buffer(ptr+i, linelen, linebuf, + sizeof(linebuf), &args); + KGSL_LOG_DUMP(device, + "RB: %4.4X:%s\n", (start+i)%size, linebuf); + } +} + +static bool adreno_ib_dump_enabled(void) +{ +#ifdef CONFIG_MSM_KGSL_PSTMRTMDMP_NO_IB_DUMP + return 0; +#else + return 1; +#endif +} + +struct log_field { + bool show; + const char *display; +}; + +static int adreno_dump_fields_line(struct kgsl_device *device, + const char *start, char *str, int slen, + const struct log_field **lines, + int num) +{ + const struct log_field *l = *lines; + int sptr, count = 0; + + sptr = snprintf(str, slen, "%s", start); + + for ( ; num && sptr < slen; num--, l++) { + int ilen = strlen(l->display); + + if (!l->show) + continue; + + if (count) + ilen += strlen(" | "); + + if (ilen > (slen - sptr)) + break; + + if (count++) + sptr += snprintf(str + sptr, slen - sptr, " | "); + + sptr += snprintf(str + sptr, slen - sptr, "%s", l->display); + } + + KGSL_LOG_DUMP(device, "%s\n", str); + + *lines = l; + return num; +} + +static void adreno_dump_fields(struct kgsl_device *device, + const char *start, const struct log_field *lines, + int num) +{ + char lb[90]; + const char *sstr = start; + + lb[sizeof(lb) - 1] = '\0'; + + while (num) { + int ret = adreno_dump_fields_line(device, sstr, lb, + sizeof(lb) - 1, &lines, num); + + if (ret == num) + break; + + num = ret; + sstr = " "; + } +} + +static int adreno_dump(struct kgsl_device *device) +{ + unsigned int r1, r2, r3, rbbm_status; + unsigned int cp_ib1_base, cp_ib1_bufsz, cp_stat; + unsigned int cp_ib2_base, cp_ib2_bufsz; + unsigned int pt_base; + unsigned int cp_rb_base, rb_count; + unsigned int cp_rb_wptr, cp_rb_rptr; + unsigned int i; + int result = 0; + uint32_t *rb_copy; + const uint32_t *rb_vaddr; + int num_item = 0; + int read_idx, write_idx; + unsigned int ts_processed, rb_memsize; + + static struct ib_list ib_list; + + struct adreno_device *adreno_dev = ADRENO_DEVICE(device); + + struct kgsl_pwrctrl *pwr = &device->pwrctrl; + + mb(); + + KGSL_LOG_DUMP(device, "POWER: FLAGS = %08lX | ACTIVE POWERLEVEL = %08X", + pwr->power_flags, pwr->active_pwrlevel); + + KGSL_LOG_DUMP(device, "POWER: INTERVAL TIMEOUT = %08X ", + pwr->interval_timeout); + + KGSL_LOG_DUMP(device, "GRP_CLK = %lu ", + kgsl_get_clkrate(pwr->grp_clks[0])); + + KGSL_LOG_DUMP(device, "BUS CLK = %lu ", + kgsl_get_clkrate(pwr->ebi1_clk)); + + + kgsl_regread(device, REG_RBBM_STATUS, &rbbm_status); + kgsl_regread(device, REG_RBBM_PM_OVERRIDE1, &r2); + kgsl_regread(device, REG_RBBM_PM_OVERRIDE2, &r3); + KGSL_LOG_DUMP(device, "RBBM: STATUS = %08X | PM_OVERRIDE1 = %08X | " + "PM_OVERRIDE2 = %08X\n", rbbm_status, r2, r3); + + kgsl_regread(device, REG_RBBM_INT_CNTL, &r1); + kgsl_regread(device, REG_RBBM_INT_STATUS, &r2); + kgsl_regread(device, REG_RBBM_READ_ERROR, &r3); + KGSL_LOG_DUMP(device, " INT_CNTL = %08X | INT_STATUS = %08X | " + "READ_ERROR = %08X\n", r1, r2, r3); + + { + char cmdFifo[16]; + struct log_field lines[] = { + {rbbm_status & 0x001F, cmdFifo}, + {rbbm_status & BIT(5), "TC busy "}, + {rbbm_status & BIT(8), "HIRQ pending"}, + {rbbm_status & BIT(9), "CPRQ pending"}, + {rbbm_status & BIT(10), "CFRQ pending"}, + {rbbm_status & BIT(11), "PFRQ pending"}, + {rbbm_status & BIT(12), "VGT 0DMA bsy"}, + {rbbm_status & BIT(14), "RBBM WU busy"}, + {rbbm_status & BIT(16), "CP NRT busy "}, + {rbbm_status & BIT(18), "MH busy "}, + {rbbm_status & BIT(19), "MH chncy bsy"}, + {rbbm_status & BIT(21), "SX busy "}, + {rbbm_status & BIT(22), "TPC busy "}, + {rbbm_status & BIT(24), "SC CNTX busy"}, + {rbbm_status & BIT(25), "PA busy "}, + {rbbm_status & BIT(26), "VGT busy "}, + {rbbm_status & BIT(27), "SQ cntx1 bsy"}, + {rbbm_status & BIT(28), "SQ cntx0 bsy"}, + {rbbm_status & BIT(30), "RB busy "}, + {rbbm_status & BIT(31), "Grphs pp bsy"}, + }; + snprintf(cmdFifo, sizeof(cmdFifo), "CMD FIFO=%01X ", + rbbm_status & 0xf); + adreno_dump_fields(device, " STATUS=", lines, + ARRAY_SIZE(lines)); + } + + kgsl_regread(device, REG_CP_RB_BASE, &cp_rb_base); + kgsl_regread(device, REG_CP_RB_CNTL, &r2); + rb_count = 2 << (r2 & (BIT(6)-1)); + kgsl_regread(device, REG_CP_RB_RPTR_ADDR, &r3); + KGSL_LOG_DUMP(device, + "CP_RB: BASE = %08X | CNTL = %08X | RPTR_ADDR = %08X" + "\n", cp_rb_base, r2, r3); + + kgsl_regread(device, REG_CP_RB_RPTR, &cp_rb_rptr); + kgsl_regread(device, REG_CP_RB_WPTR, &cp_rb_wptr); + kgsl_regread(device, REG_CP_RB_RPTR_WR, &r3); + KGSL_LOG_DUMP(device, + " RPTR = %08X | WPTR = %08X | RPTR_WR = %08X" + "\n", cp_rb_rptr, cp_rb_wptr, r3); + + kgsl_regread(device, REG_CP_IB1_BASE, &cp_ib1_base); + kgsl_regread(device, REG_CP_IB1_BUFSZ, &cp_ib1_bufsz); + KGSL_LOG_DUMP(device, + "CP_IB1: BASE = %08X | BUFSZ = %d\n", cp_ib1_base, + cp_ib1_bufsz); + + kgsl_regread(device, REG_CP_IB2_BASE, &cp_ib2_base); + kgsl_regread(device, REG_CP_IB2_BUFSZ, &cp_ib2_bufsz); + KGSL_LOG_DUMP(device, + "CP_IB2: BASE = %08X | BUFSZ = %d\n", cp_ib2_base, + cp_ib2_bufsz); + + kgsl_regread(device, REG_CP_INT_CNTL, &r1); + kgsl_regread(device, REG_CP_INT_STATUS, &r2); + KGSL_LOG_DUMP(device, "CP_INT: CNTL = %08X | STATUS = %08X\n", r1, r2); + + kgsl_regread(device, REG_CP_ME_CNTL, &r1); + kgsl_regread(device, REG_CP_ME_STATUS, &r2); + kgsl_regread(device, REG_MASTER_INT_SIGNAL, &r3); + KGSL_LOG_DUMP(device, + "CP_ME: CNTL = %08X | STATUS = %08X | MSTR_INT_SGNL = " + "%08X\n", r1, r2, r3); + + kgsl_regread(device, REG_CP_STAT, &cp_stat); + KGSL_LOG_DUMP(device, "CP_STAT = %08X\n", cp_stat); +#ifndef CONFIG_MSM_KGSL_PSTMRTMDMP_CP_STAT_NO_DETAIL + { + struct log_field lns[] = { + {cp_stat & BIT(0), "WR_BSY 0"}, + {cp_stat & BIT(1), "RD_RQ_BSY 1"}, + {cp_stat & BIT(2), "RD_RTN_BSY 2"}, + }; + adreno_dump_fields(device, " MIU=", lns, ARRAY_SIZE(lns)); + } + { + struct log_field lns[] = { + {cp_stat & BIT(5), "RING_BUSY 5"}, + {cp_stat & BIT(6), "NDRCTS_BSY 6"}, + {cp_stat & BIT(7), "NDRCT2_BSY 7"}, + {cp_stat & BIT(9), "ST_BUSY 9"}, + {cp_stat & BIT(10), "BUSY 10"}, + }; + adreno_dump_fields(device, " CSF=", lns, ARRAY_SIZE(lns)); + } + { + struct log_field lns[] = { + {cp_stat & BIT(11), "RNG_Q_BSY 11"}, + {cp_stat & BIT(12), "NDRCTS_Q_B12"}, + {cp_stat & BIT(13), "NDRCT2_Q_B13"}, + {cp_stat & BIT(16), "ST_QUEUE_B16"}, + {cp_stat & BIT(17), "PFP_BUSY 17"}, + }; + adreno_dump_fields(device, " RING=", lns, ARRAY_SIZE(lns)); + } + { + struct log_field lns[] = { + {cp_stat & BIT(3), "RBIU_BUSY 3"}, + {cp_stat & BIT(4), "RCIU_BUSY 4"}, + {cp_stat & BIT(18), "MQ_RG_BSY 18"}, + {cp_stat & BIT(19), "MQ_NDRS_BS19"}, + {cp_stat & BIT(20), "MQ_NDR2_BS20"}, + {cp_stat & BIT(21), "MIU_WC_STL21"}, + {cp_stat & BIT(22), "CP_NRT_BSY22"}, + {cp_stat & BIT(23), "3D_BUSY 23"}, + {cp_stat & BIT(26), "ME_BUSY 26"}, + {cp_stat & BIT(29), "ME_WC_BSY 29"}, + {cp_stat & BIT(30), "MIU_FF EM 30"}, + {cp_stat & BIT(31), "CP_BUSY 31"}, + }; + adreno_dump_fields(device, " CP_STT=", lns, ARRAY_SIZE(lns)); + } +#endif + + kgsl_regread(device, REG_SCRATCH_REG0, &r1); + KGSL_LOG_DUMP(device, "SCRATCH_REG0 = %08X\n", r1); + + kgsl_regread(device, REG_COHER_SIZE_PM4, &r1); + kgsl_regread(device, REG_COHER_BASE_PM4, &r2); + kgsl_regread(device, REG_COHER_STATUS_PM4, &r3); + KGSL_LOG_DUMP(device, + "COHER: SIZE_PM4 = %08X | BASE_PM4 = %08X | STATUS_PM4" + " = %08X\n", r1, r2, r3); + + kgsl_regread(device, REG_MH_AXI_ERROR, &r1); + KGSL_LOG_DUMP(device, "MH: AXI_ERROR = %08X\n", r1); + + kgsl_regread(device, REG_MH_MMU_PAGE_FAULT, &r1); + kgsl_regread(device, REG_MH_MMU_CONFIG, &r2); + kgsl_regread(device, REG_MH_MMU_MPU_BASE, &r3); + KGSL_LOG_DUMP(device, + "MH_MMU: PAGE_FAULT = %08X | CONFIG = %08X | MPU_BASE =" + " %08X\n", r1, r2, r3); + + kgsl_regread(device, REG_MH_MMU_MPU_END, &r1); + kgsl_regread(device, REG_MH_MMU_VA_RANGE, &r2); + kgsl_regread(device, REG_MH_MMU_PT_BASE, &pt_base); + KGSL_LOG_DUMP(device, + " MPU_END = %08X | VA_RANGE = %08X | PT_BASE =" + " %08X\n", r1, r2, pt_base); + + KGSL_LOG_DUMP(device, "PAGETABLE SIZE: %08X ", KGSL_PAGETABLE_SIZE); + + kgsl_regread(device, REG_MH_MMU_TRAN_ERROR, &r1); + KGSL_LOG_DUMP(device, " TRAN_ERROR = %08X\n", r1); + + kgsl_regread(device, REG_MH_INTERRUPT_MASK, &r1); + kgsl_regread(device, REG_MH_INTERRUPT_STATUS, &r2); + KGSL_LOG_DUMP(device, + "MH_INTERRUPT: MASK = %08X | STATUS = %08X\n", r1, r2); + + if (device->ftbl.device_readtimestamp != NULL) { + ts_processed = device->ftbl.device_readtimestamp( + device, KGSL_TIMESTAMP_RETIRED); + KGSL_LOG_DUMP(device, "TIMESTM RTRD: %08X\n", ts_processed); + } + + num_item = adreno_ringbuffer_count(&adreno_dev->ringbuffer, + cp_rb_rptr); + if (num_item <= 0) + KGSL_LOG_POSTMORTEM_WRITE(device, "Ringbuffer is Empty.\n"); + + rb_copy = vmalloc(rb_count<<2); + if (!rb_copy) { + KGSL_LOG_POSTMORTEM_WRITE(device, + "vmalloc(%d) failed\n", rb_count << 2); + result = -ENOMEM; + goto end; + } + + KGSL_LOG_DUMP(device, "RB: rd_addr:%8.8x rb_size:%d num_item:%d\n", + cp_rb_base, rb_count<<2, num_item); + rb_vaddr = (const uint32_t *)kgsl_sharedmem_convertaddr(device, pt_base, + cp_rb_base, &rb_memsize); + if (!rb_vaddr) { + KGSL_LOG_POSTMORTEM_WRITE(device, + "Can't fetch vaddr for CP_RB_BASE\n"); + goto error_vfree; + } + + read_idx = (int)cp_rb_rptr - 64; + if (read_idx < 0) + read_idx += rb_count; + write_idx = (int)cp_rb_wptr + 16; + if (write_idx > rb_count) + write_idx -= rb_count; + num_item += 64+16; + if (num_item > rb_count) + num_item = rb_count; + if (write_idx >= read_idx) + memcpy(rb_copy, rb_vaddr+read_idx, num_item<<2); + else { + int part1_c = rb_count-read_idx; + memcpy(rb_copy, rb_vaddr+read_idx, part1_c<<2); + memcpy(rb_copy+part1_c, rb_vaddr, (num_item-part1_c)<<2); + } + + /* extract the latest ib commands from the buffer */ + ib_list.count = 0; + i = 0; + for (read_idx = 0; read_idx < num_item; ) { + uint32_t this_cmd = rb_copy[read_idx++]; + if (this_cmd == pm4_type3_packet(PM4_INDIRECT_BUFFER_PFD, 2)) { + uint32_t ib_addr = rb_copy[read_idx++]; + uint32_t ib_size = rb_copy[read_idx++]; + dump_ib1(device, pt_base, (read_idx-3)<<2, ib_addr, + ib_size, &ib_list, 0); + for (; i < ib_list.count; ++i) + dump_ib(device, "IB2:", pt_base, + ib_list.offsets[i], + ib_list.bases[i], + ib_list.sizes[i], 0); + } + } + + read_idx = (int)cp_rb_rptr - 64; + if (read_idx < 0) + read_idx += rb_count; + KGSL_LOG_DUMP(device, + "RB: addr=%8.8x window:%4.4x-%4.4x, start:%4.4x\n", + cp_rb_base, cp_rb_rptr, cp_rb_wptr, read_idx); + adreno_dump_rb(device, rb_copy, num_item<<2, read_idx, rb_count); + + if (adreno_ib_dump_enabled()) { + for (read_idx = 64; read_idx >= 0; --read_idx) { + uint32_t this_cmd = rb_copy[read_idx]; + if (this_cmd == pm4_type3_packet( + PM4_INDIRECT_BUFFER_PFD, 2)) { + uint32_t ib_addr = rb_copy[read_idx+1]; + uint32_t ib_size = rb_copy[read_idx+2]; + if (cp_ib1_bufsz && cp_ib1_base == ib_addr) { + KGSL_LOG_DUMP(device, + "IB1: base:%8.8X " + "count:%d\n", ib_addr, ib_size); + dump_ib(device, "IB1: ", pt_base, + read_idx<<2, ib_addr, ib_size, + 1); + } + } + } + for (i = 0; i < ib_list.count; ++i) { + if (cp_ib2_bufsz && cp_ib2_base == ib_list.bases[i]) { + uint32_t ib_size = ib_list.sizes[i]; + uint32_t ib_offset = ib_list.offsets[i]; + KGSL_LOG_DUMP(device, + "IB2: base:%8.8X count:%d\n", + cp_ib2_base, ib_size); + dump_ib(device, "IB2: ", pt_base, ib_offset, + ib_list.bases[i], ib_size, 1); + } + } + } + + /* Dump the registers if the user asked for it */ + + if (adreno_is_a20x(adreno_dev)) + adreno_dump_regs(device, a200_registers, + ARRAY_SIZE(a200_registers) / 2); + else if (adreno_is_a22x(adreno_dev)) + adreno_dump_regs(device, a220_registers, + ARRAY_SIZE(a220_registers) / 2); + +error_vfree: + vfree(rb_copy); +end: + return result; +} + +/** + * adreno_postmortem_dump - Dump the current GPU state + * @device - A pointer to the KGSL device to dump + * @manual - A flag that indicates if this was a manually triggered + * dump (from debugfs). If zero, then this is assumed to be a + * dump automaticlaly triggered from a hang +*/ + +int adreno_postmortem_dump(struct kgsl_device *device, int manual) +{ + bool saved_nap; + + BUG_ON(device == NULL); + + kgsl_cffdump_hang(device->id); + + /* For a manual dump, make sure that the system is idle */ + + if (manual) { + if (device->active_cnt != 0) { + mutex_unlock(&device->mutex); + wait_for_completion(&device->suspend_gate); + mutex_lock(&device->mutex); + } + + if (device->state == KGSL_STATE_ACTIVE) + kgsl_idle(device, KGSL_TIMEOUT_DEFAULT); + + } + /* Disable the idle timer so we don't get interrupted */ + del_timer(&device->idle_timer); + + /* Turn off napping to make sure we have the clocks full + attention through the following process */ + saved_nap = device->pwrctrl.nap_allowed; + device->pwrctrl.nap_allowed = false; + + /* Force on the clocks */ + kgsl_pwrctrl_wake(device); + + /* Disable the irq */ + kgsl_pwrctrl_irq(device, KGSL_PWRFLAGS_OFF); + + /* If this is not a manual trigger, then set up the + state to try to recover */ + + if (!manual) { + device->state = KGSL_STATE_DUMP_AND_RECOVER; + KGSL_PWR_WARN(device, + "state -> DUMP_AND_RECOVER, device %d\n", + device->id); + } + + KGSL_DRV_ERR(device, + "wait for work in workqueue to complete\n"); + mutex_unlock(&device->mutex); + flush_workqueue(device->work_queue); + mutex_lock(&device->mutex); + adreno_dump(device); + + /* Restore nap mode */ + device->pwrctrl.nap_allowed = saved_nap; + + /* On a manual trigger, turn on the interrupts and put + the clocks to sleep. They will recover themselves + on the next event. For a hang, leave things as they + are until recovery kicks in. */ + + if (manual) { + kgsl_pwrctrl_irq(device, KGSL_PWRFLAGS_ON); + + /* try to go into a sleep mode until the next event */ + device->requested_state = KGSL_STATE_SLEEP; + kgsl_pwrctrl_sleep(device); + } + + KGSL_DRV_ERR(device, "Dump Finished\n"); + + return 0; +} diff --git a/drivers/gpu/msm/adreno_postmortem.h b/drivers/gpu/msm/adreno_postmortem.h new file mode 100644 index 00000000..1a432489 --- /dev/null +++ b/drivers/gpu/msm/adreno_postmortem.h @@ -0,0 +1,37 @@ +/* Copyright (c) 2010-2011, Code Aurora Forum. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are + * met: + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following + * disclaimer in the documentation and/or other materials provided + * with the distribution. + * * Neither the name of Code Aurora Forum, Inc. nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED "AS IS" AND ANY EXPRESS OR IMPLIED + * WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS + * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR + * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, + * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE + * OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN + * IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + */ + +#ifndef __ADRENO_POSTMORTEM_H +#define __ADRENO_POSTMORTEM_H + +struct kgsl_device; + +int adreno_postmortem_dump(struct kgsl_device *device, int manual); + +#endif /* __ADRENO_POSTMORTEM_H */ diff --git a/drivers/gpu/msm/adreno_ringbuffer.c b/drivers/gpu/msm/adreno_ringbuffer.c new file mode 100644 index 00000000..4aaa2c6c --- /dev/null +++ b/drivers/gpu/msm/adreno_ringbuffer.c @@ -0,0 +1,932 @@ +/* Copyright (c) 2002,2007-2011, Code Aurora Forum. All rights reserved. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 and + * only version 2 as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + */ +#include +#include +#include +#include + +#include "kgsl.h" + +#include "adreno.h" +#include "adreno_pm4types.h" +#include "adreno_ringbuffer.h" + +#include "a200_reg.h" + +#define VALID_STATUS_COUNT_MAX 10 +#define GSL_RB_NOP_SIZEDWORDS 2 +/* protected mode error checking below register address 0x800 +* note: if CP_INTERRUPT packet is used then checking needs +* to change to below register address 0x7C8 +*/ +#define GSL_RB_PROTECTED_MODE_CONTROL 0x200001F2 + +#define GSL_CP_INT_MASK \ + (CP_INT_CNTL__SW_INT_MASK | \ + CP_INT_CNTL__T0_PACKET_IN_IB_MASK | \ + CP_INT_CNTL__OPCODE_ERROR_MASK | \ + CP_INT_CNTL__PROTECTED_MODE_ERROR_MASK | \ + CP_INT_CNTL__RESERVED_BIT_ERROR_MASK | \ + CP_INT_CNTL__IB_ERROR_MASK | \ + CP_INT_CNTL__IB2_INT_MASK | \ + CP_INT_CNTL__IB1_INT_MASK | \ + CP_INT_CNTL__RB_INT_MASK) + +/* Firmware file names + * Legacy names must remain but replacing macro names to + * match current kgsl model. + * a200 is yamato + * a220 is leia + */ +#define A200_PFP_FW "yamato_pfp.fw" +#define A200_PM4_FW "yamato_pm4.fw" +#define A220_PFP_470_FW "leia_pfp_470.fw" +#define A220_PM4_470_FW "leia_pm4_470.fw" + +/* functions */ +void kgsl_cp_intrcallback(struct kgsl_device *device) +{ + unsigned int status = 0, num_reads = 0, master_status = 0; + struct adreno_device *adreno_dev = ADRENO_DEVICE(device); + struct adreno_ringbuffer *rb = &adreno_dev->ringbuffer; + + adreno_regread_isr(device, REG_MASTER_INT_SIGNAL, &master_status); + while (!status && (num_reads < VALID_STATUS_COUNT_MAX) && + (master_status & MASTER_INT_SIGNAL__CP_INT_STAT)) { + adreno_regread_isr(device, REG_CP_INT_STATUS, &status); + adreno_regread_isr(device, REG_MASTER_INT_SIGNAL, + &master_status); + num_reads++; + } + if (num_reads > 1) + KGSL_DRV_WARN(device, + "Looped %d times to read REG_CP_INT_STATUS\n", + num_reads); + if (!status) { + if (master_status & MASTER_INT_SIGNAL__CP_INT_STAT) { + /* This indicates that we could not read CP_INT_STAT. + * As a precaution just wake up processes so + * they can check their timestamps. Since, we + * did not ack any interrupts this interrupt will + * be generated again */ + KGSL_DRV_WARN(device, "Unable to read CP_INT_STATUS\n"); + wake_up_interruptible_all(&device->wait_queue); + } else + KGSL_DRV_WARN(device, "Spurious interrput detected\n"); + return; + } + + if (status & CP_INT_CNTL__RB_INT_MASK) { + /* signal intr completion event */ + unsigned int enableflag = 0; + kgsl_sharedmem_writel(&rb->device->memstore, + KGSL_DEVICE_MEMSTORE_OFFSET(ts_cmp_enable), + enableflag); + wmb(); + KGSL_CMD_WARN(rb->device, "ringbuffer rb interrupt\n"); + } + + if (status & CP_INT_CNTL__T0_PACKET_IN_IB_MASK) { + KGSL_CMD_CRIT(rb->device, + "ringbuffer TO packet in IB interrupt\n"); + adreno_regwrite_isr(rb->device, REG_CP_INT_CNTL, 0); + } + if (status & CP_INT_CNTL__OPCODE_ERROR_MASK) { + KGSL_CMD_CRIT(rb->device, + "ringbuffer opcode error interrupt\n"); + adreno_regwrite_isr(rb->device, REG_CP_INT_CNTL, 0); + } + if (status & CP_INT_CNTL__PROTECTED_MODE_ERROR_MASK) { + KGSL_CMD_CRIT(rb->device, + "ringbuffer protected mode error interrupt\n"); + adreno_regwrite_isr(rb->device, REG_CP_INT_CNTL, 0); + } + if (status & CP_INT_CNTL__RESERVED_BIT_ERROR_MASK) { + KGSL_CMD_CRIT(rb->device, + "ringbuffer reserved bit error interrupt\n"); + adreno_regwrite_isr(rb->device, REG_CP_INT_CNTL, 0); + } + if (status & CP_INT_CNTL__IB_ERROR_MASK) { + KGSL_CMD_CRIT(rb->device, + "ringbuffer IB error interrupt\n"); + adreno_regwrite_isr(rb->device, REG_CP_INT_CNTL, 0); + } + if (status & CP_INT_CNTL__SW_INT_MASK) + KGSL_CMD_INFO(rb->device, "ringbuffer software interrupt\n"); + + if (status & CP_INT_CNTL__IB2_INT_MASK) + KGSL_CMD_INFO(rb->device, "ringbuffer ib2 interrupt\n"); + + if (status & (~GSL_CP_INT_MASK)) + KGSL_CMD_WARN(rb->device, + "bad bits in REG_CP_INT_STATUS %08x\n", status); + + /* only ack bits we understand */ + status &= GSL_CP_INT_MASK; + adreno_regwrite_isr(device, REG_CP_INT_ACK, status); + + if (status & (CP_INT_CNTL__IB1_INT_MASK | CP_INT_CNTL__RB_INT_MASK)) { + KGSL_CMD_WARN(rb->device, "ringbuffer ib1/rb interrupt\n"); + wake_up_interruptible_all(&device->wait_queue); + atomic_notifier_call_chain(&(device->ts_notifier_list), + device->id, + NULL); + } +} + +static void adreno_ringbuffer_submit(struct adreno_ringbuffer *rb) +{ + BUG_ON(rb->wptr == 0); + + /*synchronize memory before informing the hardware of the + *new commands. + */ + mb(); + + adreno_regwrite(rb->device, REG_CP_RB_WPTR, rb->wptr); +} + +static int +adreno_ringbuffer_waitspace(struct adreno_ringbuffer *rb, unsigned int numcmds, + int wptr_ahead) +{ + int nopcount; + unsigned int freecmds; + unsigned int *cmds; + uint cmds_gpu; + + /* if wptr ahead, fill the remaining with NOPs */ + if (wptr_ahead) { + /* -1 for header */ + nopcount = rb->sizedwords - rb->wptr - 1; + + cmds = (unsigned int *)rb->buffer_desc.hostptr + rb->wptr; + cmds_gpu = rb->buffer_desc.gpuaddr + sizeof(uint)*rb->wptr; + + GSL_RB_WRITE(cmds, cmds_gpu, pm4_nop_packet(nopcount)); + + /* Make sure that rptr is not 0 before submitting + * commands at the end of ringbuffer. We do not + * want the rptr and wptr to become equal when + * the ringbuffer is not empty */ + do { + GSL_RB_GET_READPTR(rb, &rb->rptr); + } while (!rb->rptr); + + rb->wptr++; + + adreno_ringbuffer_submit(rb); + + rb->wptr = 0; + } + + /* wait for space in ringbuffer */ + do { + GSL_RB_GET_READPTR(rb, &rb->rptr); + + freecmds = rb->rptr - rb->wptr; + + } while ((freecmds != 0) && (freecmds <= numcmds)); + + return 0; +} + + +static unsigned int *adreno_ringbuffer_allocspace(struct adreno_ringbuffer *rb, + unsigned int numcmds) +{ + unsigned int *ptr = NULL; + int status = 0; + + BUG_ON(numcmds >= rb->sizedwords); + + GSL_RB_GET_READPTR(rb, &rb->rptr); + /* check for available space */ + if (rb->wptr >= rb->rptr) { + /* wptr ahead or equal to rptr */ + /* reserve dwords for nop packet */ + if ((rb->wptr + numcmds) > (rb->sizedwords - + GSL_RB_NOP_SIZEDWORDS)) + status = adreno_ringbuffer_waitspace(rb, numcmds, 1); + } else { + /* wptr behind rptr */ + if ((rb->wptr + numcmds) >= rb->rptr) + status = adreno_ringbuffer_waitspace(rb, numcmds, 0); + /* check for remaining space */ + /* reserve dwords for nop packet */ + if ((rb->wptr + numcmds) > (rb->sizedwords - + GSL_RB_NOP_SIZEDWORDS)) + status = adreno_ringbuffer_waitspace(rb, numcmds, 1); + } + + if (status == 0) { + ptr = (unsigned int *)rb->buffer_desc.hostptr + rb->wptr; + rb->wptr += numcmds; + } + + return ptr; +} + +static int _load_firmware(struct kgsl_device *device, const char *fwfile, + void **data, int *len) +{ + const struct firmware *fw = NULL; + int ret; + + ret = request_firmware(&fw, fwfile, device->dev); + + if (ret) { + KGSL_DRV_ERR(device, "request_firmware(%s) failed: %d\n", + fwfile, ret); + return ret; + } + + *data = kmalloc(fw->size, GFP_KERNEL); + + if (*data) { + memcpy(*data, fw->data, fw->size); + *len = fw->size; + } else + KGSL_MEM_ERR(device, "kmalloc(%d) failed\n", fw->size); + + release_firmware(fw); + return (*data != NULL) ? 0 : -ENOMEM; +} + +static int adreno_ringbuffer_load_pm4_ucode(struct kgsl_device *device) +{ + struct adreno_device *adreno_dev = ADRENO_DEVICE(device); + const char *fwfile; + int i, ret = 0; + + if (adreno_is_a220(adreno_dev)) + fwfile = A220_PM4_470_FW; + else + fwfile = A200_PM4_FW; + + if (adreno_dev->pm4_fw == NULL) { + int len; + unsigned int *ptr; + + ret = _load_firmware(device, fwfile, (void *) &ptr, &len); + if (ret) + goto err; + + /* PM4 size is 3 dword aligned plus 1 dword of version */ + if (len % ((sizeof(uint32_t) * 3)) != sizeof(uint32_t)) { + KGSL_DRV_ERR(device, "Bad firmware size: %d\n", len); + ret = -EINVAL; + kfree(ptr); + goto err; + } + + adreno_dev->pm4_fw_size = len / sizeof(uint32_t); + adreno_dev->pm4_fw = ptr; + } + + KGSL_DRV_INFO(device, "loading pm4 ucode version: %d\n", + adreno_dev->pm4_fw[0]); + + adreno_regwrite(device, REG_CP_DEBUG, 0x02000000); + adreno_regwrite(device, REG_CP_ME_RAM_WADDR, 0); + for (i = 1; i < adreno_dev->pm4_fw_size; i++) + adreno_regwrite(device, REG_CP_ME_RAM_DATA, + adreno_dev->pm4_fw[i]); +err: + return ret; +} + +static int adreno_ringbuffer_load_pfp_ucode(struct kgsl_device *device) +{ + struct adreno_device *adreno_dev = ADRENO_DEVICE(device); + const char *fwfile; + int i, ret = 0; + + if (adreno_is_a220(adreno_dev)) + fwfile = A220_PFP_470_FW; + else + fwfile = A200_PFP_FW; + + if (adreno_dev->pfp_fw == NULL) { + int len; + unsigned int *ptr; + + ret = _load_firmware(device, fwfile, (void *) &ptr, &len); + if (ret) + goto err; + + /* PFP size shold be dword aligned */ + if (len % sizeof(uint32_t) != 0) { + KGSL_DRV_ERR(device, "Bad firmware size: %d\n", len); + ret = -EINVAL; + kfree(ptr); + goto err; + } + + adreno_dev->pfp_fw_size = len / sizeof(uint32_t); + adreno_dev->pfp_fw = ptr; + } + + KGSL_DRV_INFO(device, "loading pfp ucode version: %d\n", + adreno_dev->pfp_fw[0]); + + adreno_regwrite(device, REG_CP_PFP_UCODE_ADDR, 0); + for (i = 1; i < adreno_dev->pfp_fw_size; i++) + adreno_regwrite(device, REG_CP_PFP_UCODE_DATA, + adreno_dev->pfp_fw[i]); +err: + return ret; +} + +int adreno_ringbuffer_start(struct adreno_ringbuffer *rb, unsigned int init_ram) +{ + int status; + /*cp_rb_cntl_u cp_rb_cntl; */ + union reg_cp_rb_cntl cp_rb_cntl; + unsigned int *cmds, rb_cntl; + struct kgsl_device *device = rb->device; + uint cmds_gpu; + + if (rb->flags & KGSL_FLAGS_STARTED) + return 0; + + if (init_ram) { + rb->timestamp = 0; + GSL_RB_INIT_TIMESTAMP(rb); + } + + kgsl_sharedmem_set(&rb->memptrs_desc, 0, 0, + sizeof(struct kgsl_rbmemptrs)); + + kgsl_sharedmem_set(&rb->buffer_desc, 0, 0xAA, + (rb->sizedwords << 2)); + + adreno_regwrite(device, REG_CP_RB_WPTR_BASE, + (rb->memptrs_desc.gpuaddr + + GSL_RB_MEMPTRS_WPTRPOLL_OFFSET)); + + /* setup WPTR delay */ + adreno_regwrite(device, REG_CP_RB_WPTR_DELAY, 0 /*0x70000010 */); + + /*setup REG_CP_RB_CNTL */ + adreno_regread(device, REG_CP_RB_CNTL, &rb_cntl); + cp_rb_cntl.val = rb_cntl; + + /* + * The size of the ringbuffer in the hardware is the log2 + * representation of the size in quadwords (sizedwords / 2) + */ + cp_rb_cntl.f.rb_bufsz = ilog2(rb->sizedwords >> 1); + + /* + * Specify the quadwords to read before updating mem RPTR. + * Like above, pass the log2 representation of the blocksize + * in quadwords. + */ + cp_rb_cntl.f.rb_blksz = ilog2(KGSL_RB_BLKSIZE >> 3); + + cp_rb_cntl.f.rb_poll_en = GSL_RB_CNTL_POLL_EN; /* WPTR polling */ + /* mem RPTR writebacks */ + cp_rb_cntl.f.rb_no_update = GSL_RB_CNTL_NO_UPDATE; + + adreno_regwrite(device, REG_CP_RB_CNTL, cp_rb_cntl.val); + + adreno_regwrite(device, REG_CP_RB_BASE, rb->buffer_desc.gpuaddr); + + adreno_regwrite(device, REG_CP_RB_RPTR_ADDR, + rb->memptrs_desc.gpuaddr + + GSL_RB_MEMPTRS_RPTR_OFFSET); + + /* explicitly clear all cp interrupts */ + adreno_regwrite(device, REG_CP_INT_ACK, 0xFFFFFFFF); + + /* setup scratch/timestamp */ + adreno_regwrite(device, REG_SCRATCH_ADDR, + device->memstore.gpuaddr + + KGSL_DEVICE_MEMSTORE_OFFSET(soptimestamp)); + + adreno_regwrite(device, REG_SCRATCH_UMSK, + GSL_RB_MEMPTRS_SCRATCH_MASK); + + /* load the CP ucode */ + + status = adreno_ringbuffer_load_pm4_ucode(device); + if (status != 0) + return status; + + /* load the prefetch parser ucode */ + status = adreno_ringbuffer_load_pfp_ucode(device); + if (status != 0) + return status; + + adreno_regwrite(device, REG_CP_QUEUE_THRESHOLDS, 0x000C0804); + + rb->rptr = 0; + rb->wptr = 0; + + /* clear ME_HALT to start micro engine */ + adreno_regwrite(device, REG_CP_ME_CNTL, 0); + + /* ME_INIT */ + cmds = adreno_ringbuffer_allocspace(rb, 19); + cmds_gpu = rb->buffer_desc.gpuaddr + sizeof(uint)*(rb->wptr-19); + + GSL_RB_WRITE(cmds, cmds_gpu, PM4_HDR_ME_INIT); + /* All fields present (bits 9:0) */ + GSL_RB_WRITE(cmds, cmds_gpu, 0x000003ff); + /* Disable/Enable Real-Time Stream processing (present but ignored) */ + GSL_RB_WRITE(cmds, cmds_gpu, 0x00000000); + /* Enable (2D <-> 3D) implicit synchronization (present but ignored) */ + GSL_RB_WRITE(cmds, cmds_gpu, 0x00000000); + + GSL_RB_WRITE(cmds, cmds_gpu, + GSL_HAL_SUBBLOCK_OFFSET(REG_RB_SURFACE_INFO)); + GSL_RB_WRITE(cmds, cmds_gpu, + GSL_HAL_SUBBLOCK_OFFSET(REG_PA_SC_WINDOW_OFFSET)); + GSL_RB_WRITE(cmds, cmds_gpu, + GSL_HAL_SUBBLOCK_OFFSET(REG_VGT_MAX_VTX_INDX)); + GSL_RB_WRITE(cmds, cmds_gpu, + GSL_HAL_SUBBLOCK_OFFSET(REG_SQ_PROGRAM_CNTL)); + GSL_RB_WRITE(cmds, cmds_gpu, + GSL_HAL_SUBBLOCK_OFFSET(REG_RB_DEPTHCONTROL)); + GSL_RB_WRITE(cmds, cmds_gpu, + GSL_HAL_SUBBLOCK_OFFSET(REG_PA_SU_POINT_SIZE)); + GSL_RB_WRITE(cmds, cmds_gpu, + GSL_HAL_SUBBLOCK_OFFSET(REG_PA_SC_LINE_CNTL)); + GSL_RB_WRITE(cmds, cmds_gpu, + GSL_HAL_SUBBLOCK_OFFSET(REG_PA_SU_POLY_OFFSET_FRONT_SCALE)); + + /* Vertex and Pixel Shader Start Addresses in instructions + * (3 DWORDS per instruction) */ + GSL_RB_WRITE(cmds, cmds_gpu, 0x80000180); + /* Maximum Contexts */ + GSL_RB_WRITE(cmds, cmds_gpu, 0x00000001); + /* Write Confirm Interval and The CP will wait the + * wait_interval * 16 clocks between polling */ + GSL_RB_WRITE(cmds, cmds_gpu, 0x00000000); + + /* NQ and External Memory Swap */ + GSL_RB_WRITE(cmds, cmds_gpu, 0x00000000); + /* Protected mode error checking */ + GSL_RB_WRITE(cmds, cmds_gpu, GSL_RB_PROTECTED_MODE_CONTROL); + /* Disable header dumping and Header dump address */ + GSL_RB_WRITE(cmds, cmds_gpu, 0x00000000); + /* Header dump size */ + GSL_RB_WRITE(cmds, cmds_gpu, 0x00000000); + + adreno_ringbuffer_submit(rb); + + /* idle device to validate ME INIT */ + status = adreno_idle(device, KGSL_TIMEOUT_DEFAULT); + + adreno_regwrite(rb->device, REG_CP_INT_CNTL, GSL_CP_INT_MASK); + if (status == 0) + rb->flags |= KGSL_FLAGS_STARTED; + + return status; +} + +int adreno_ringbuffer_stop(struct adreno_ringbuffer *rb) +{ + if (rb->flags & KGSL_FLAGS_STARTED) { + adreno_regwrite(rb->device, REG_CP_INT_CNTL, 0); + + /* ME_HALT */ + adreno_regwrite(rb->device, REG_CP_ME_CNTL, 0x10000000); + + rb->flags &= ~KGSL_FLAGS_STARTED; + } + + return 0; +} + +int adreno_ringbuffer_init(struct kgsl_device *device) +{ + int status; + struct adreno_device *adreno_dev = ADRENO_DEVICE(device); + struct adreno_ringbuffer *rb = &adreno_dev->ringbuffer; + + rb->device = device; + /* + * It is silly to convert this to words and then back to bytes + * immediately below, but most of the rest of the code deals + * in words, so we might as well only do the math once + */ + rb->sizedwords = KGSL_RB_SIZE >> 2; + + /* allocate memory for ringbuffer */ + status = kgsl_allocate_contig(&rb->buffer_desc, (rb->sizedwords << 2)); + + if (status != 0) { + adreno_ringbuffer_close(rb); + return status; + } + + /* allocate memory for polling and timestamps */ + /* This really can be at 4 byte alignment boundry but for using MMU + * we need to make it at page boundary */ + status = kgsl_allocate_contig(&rb->memptrs_desc, + sizeof(struct kgsl_rbmemptrs)); + + if (status != 0) { + adreno_ringbuffer_close(rb); + return status; + } + + /* overlay structure on memptrs memory */ + rb->memptrs = (struct kgsl_rbmemptrs *) rb->memptrs_desc.hostptr; + + return 0; +} + +int adreno_ringbuffer_close(struct adreno_ringbuffer *rb) +{ + struct adreno_device *adreno_dev = ADRENO_DEVICE(rb->device); + + kgsl_sharedmem_free(&rb->buffer_desc); + kgsl_sharedmem_free(&rb->memptrs_desc); + + kfree(adreno_dev->pfp_fw); + kfree(adreno_dev->pm4_fw); + + adreno_dev->pfp_fw = NULL; + adreno_dev->pm4_fw = NULL; + + memset(rb, 0, sizeof(struct adreno_ringbuffer)); + + return 0; +} + +static uint32_t +adreno_ringbuffer_addcmds(struct adreno_ringbuffer *rb, + unsigned int flags, unsigned int *cmds, + int sizedwords) +{ + unsigned int *ringcmds; + unsigned int timestamp; + unsigned int total_sizedwords = sizedwords + 6; + unsigned int i; + unsigned int rcmd_gpu; + + /* reserve space to temporarily turn off protected mode + * error checking if needed + */ + total_sizedwords += flags & KGSL_CMD_FLAGS_PMODE ? 4 : 0; + total_sizedwords += !(flags & KGSL_CMD_FLAGS_NO_TS_CMP) ? 7 : 0; + total_sizedwords += !(flags & KGSL_CMD_FLAGS_NOT_KERNEL_CMD) ? 2 : 0; + + ringcmds = adreno_ringbuffer_allocspace(rb, total_sizedwords); + rcmd_gpu = rb->buffer_desc.gpuaddr + + sizeof(uint)*(rb->wptr-total_sizedwords); + + if (!(flags & KGSL_CMD_FLAGS_NOT_KERNEL_CMD)) { + GSL_RB_WRITE(ringcmds, rcmd_gpu, pm4_nop_packet(1)); + GSL_RB_WRITE(ringcmds, rcmd_gpu, KGSL_CMD_IDENTIFIER); + } + if (flags & KGSL_CMD_FLAGS_PMODE) { + /* disable protected mode error checking */ + GSL_RB_WRITE(ringcmds, rcmd_gpu, + pm4_type3_packet(PM4_SET_PROTECTED_MODE, 1)); + GSL_RB_WRITE(ringcmds, rcmd_gpu, 0); + } + + for (i = 0; i < sizedwords; i++) { + GSL_RB_WRITE(ringcmds, rcmd_gpu, *cmds); + cmds++; + } + + if (flags & KGSL_CMD_FLAGS_PMODE) { + /* re-enable protected mode error checking */ + GSL_RB_WRITE(ringcmds, rcmd_gpu, + pm4_type3_packet(PM4_SET_PROTECTED_MODE, 1)); + GSL_RB_WRITE(ringcmds, rcmd_gpu, 1); + } + + rb->timestamp++; + timestamp = rb->timestamp; + + /* start-of-pipeline and end-of-pipeline timestamps */ + GSL_RB_WRITE(ringcmds, rcmd_gpu, pm4_type0_packet(REG_CP_TIMESTAMP, 1)); + GSL_RB_WRITE(ringcmds, rcmd_gpu, rb->timestamp); + GSL_RB_WRITE(ringcmds, rcmd_gpu, pm4_type3_packet(PM4_EVENT_WRITE, 3)); + GSL_RB_WRITE(ringcmds, rcmd_gpu, CACHE_FLUSH_TS); + GSL_RB_WRITE(ringcmds, rcmd_gpu, + (rb->device->memstore.gpuaddr + + KGSL_DEVICE_MEMSTORE_OFFSET(eoptimestamp))); + GSL_RB_WRITE(ringcmds, rcmd_gpu, rb->timestamp); + + if (!(flags & KGSL_CMD_FLAGS_NO_TS_CMP)) { + /* Conditional execution based on memory values */ + GSL_RB_WRITE(ringcmds, rcmd_gpu, + pm4_type3_packet(PM4_COND_EXEC, 4)); + GSL_RB_WRITE(ringcmds, rcmd_gpu, (rb->device->memstore.gpuaddr + + KGSL_DEVICE_MEMSTORE_OFFSET(ts_cmp_enable)) >> 2); + GSL_RB_WRITE(ringcmds, rcmd_gpu, (rb->device->memstore.gpuaddr + + KGSL_DEVICE_MEMSTORE_OFFSET(ref_wait_ts)) >> 2); + GSL_RB_WRITE(ringcmds, rcmd_gpu, rb->timestamp); + /* # of conditional command DWORDs */ + GSL_RB_WRITE(ringcmds, rcmd_gpu, 2); + GSL_RB_WRITE(ringcmds, rcmd_gpu, + pm4_type3_packet(PM4_INTERRUPT, 1)); + GSL_RB_WRITE(ringcmds, rcmd_gpu, CP_INT_CNTL__RB_INT_MASK); + } + + adreno_ringbuffer_submit(rb); + + /* return timestamp of issued coREG_ands */ + return timestamp; +} + +void +adreno_ringbuffer_issuecmds(struct kgsl_device *device, + unsigned int flags, + unsigned int *cmds, + int sizedwords) +{ + struct adreno_device *adreno_dev = ADRENO_DEVICE(device); + struct adreno_ringbuffer *rb = &adreno_dev->ringbuffer; + + if (device->state & KGSL_STATE_HUNG) + return; + adreno_ringbuffer_addcmds(rb, flags, cmds, sizedwords); +} + +int +adreno_ringbuffer_issueibcmds(struct kgsl_device_private *dev_priv, + struct kgsl_context *context, + struct kgsl_ibdesc *ibdesc, + unsigned int numibs, + uint32_t *timestamp, + unsigned int flags) +{ + struct kgsl_device *device = dev_priv->device; + struct adreno_device *adreno_dev = ADRENO_DEVICE(device); + unsigned int *link; + unsigned int *cmds; + unsigned int i; + struct adreno_context *drawctxt; + + if (device->state & KGSL_STATE_HUNG) + return -EBUSY; + if (!(adreno_dev->ringbuffer.flags & KGSL_FLAGS_STARTED) || + context == NULL || ibdesc == 0 || numibs == 0) + return -EINVAL; + + drawctxt = context->devctxt; + + if (drawctxt->flags & CTXT_FLAGS_GPU_HANG) { + KGSL_CTXT_WARN(device, "Context %p caused a gpu hang.." + " will not accept commands for this context\n", + drawctxt); + return -EDEADLK; + } + link = kzalloc(sizeof(unsigned int) * numibs * 3, GFP_KERNEL); + cmds = link; + if (!link) { + KGSL_MEM_ERR(device, "Failed to allocate memory for for command" + " submission, size %x\n", numibs * 3); + return -ENOMEM; + } + for (i = 0; i < numibs; i++) { + (void)kgsl_cffdump_parse_ibs(dev_priv, NULL, + ibdesc[i].gpuaddr, ibdesc[i].sizedwords, false); + + *cmds++ = PM4_HDR_INDIRECT_BUFFER_PFD; + *cmds++ = ibdesc[i].gpuaddr; + *cmds++ = ibdesc[i].sizedwords; + } + + kgsl_setstate(device, + kgsl_pt_get_flags(device->mmu.hwpagetable, + device->id)); + + adreno_drawctxt_switch(adreno_dev, drawctxt, flags); + + *timestamp = adreno_ringbuffer_addcmds(&adreno_dev->ringbuffer, + KGSL_CMD_FLAGS_NOT_KERNEL_CMD, + &link[0], (cmds - link)); + + KGSL_CMD_INFO(device, "ctxt %d g %08x numibs %d ts %d\n", + context->id, (unsigned int)ibdesc, numibs, *timestamp); + + kfree(link); + +#ifdef CONFIG_MSM_KGSL_CFF_DUMP + /* + * insert wait for idle after every IB1 + * this is conservative but works reliably and is ok + * even for performance simulations + */ + adreno_idle(device, KGSL_TIMEOUT_DEFAULT); +#endif + + return 0; +} + +int adreno_ringbuffer_extract(struct adreno_ringbuffer *rb, + unsigned int *temp_rb_buffer, + int *rb_size) +{ + struct kgsl_device *device = rb->device; + unsigned int rb_rptr; + unsigned int retired_timestamp; + unsigned int temp_idx = 0; + unsigned int value; + unsigned int val1; + unsigned int val2; + unsigned int val3; + unsigned int copy_rb_contents = 0; + unsigned int cur_context; + unsigned int j; + + GSL_RB_GET_READPTR(rb, &rb->rptr); + +/* drewis: still not sure where this struct was changed */ +#if 0 + retired_timestamp = device->ftbl->readtimestamp(device, + KGSL_TIMESTAMP_RETIRED); +#endif + retired_timestamp = device->ftbl.device_readtimestamp( + device, KGSL_TIMESTAMP_RETIRED); + KGSL_DRV_ERR(device, "GPU successfully executed till ts: %x\n", + retired_timestamp); + /* + * We need to go back in history by 4 dwords from the current location + * of read pointer as 4 dwords are read to match the end of a command. + * Also, take care of wrap around when moving back + */ + if (rb->rptr >= 4) + rb_rptr = (rb->rptr - 4) * sizeof(unsigned int); + else + rb_rptr = rb->buffer_desc.size - + ((4 - rb->rptr) * sizeof(unsigned int)); + /* Read the rb contents going backwards to locate end of last + * sucessfully executed command */ + while ((rb_rptr / sizeof(unsigned int)) != rb->wptr) { + kgsl_sharedmem_readl(&rb->buffer_desc, &value, rb_rptr); + if (value == retired_timestamp) { + rb_rptr = adreno_ringbuffer_inc_wrapped(rb_rptr, + rb->buffer_desc.size); + kgsl_sharedmem_readl(&rb->buffer_desc, &val1, rb_rptr); + rb_rptr = adreno_ringbuffer_inc_wrapped(rb_rptr, + rb->buffer_desc.size); + kgsl_sharedmem_readl(&rb->buffer_desc, &val2, rb_rptr); + rb_rptr = adreno_ringbuffer_inc_wrapped(rb_rptr, + rb->buffer_desc.size); + kgsl_sharedmem_readl(&rb->buffer_desc, &val3, rb_rptr); + /* match the pattern found at the end of a command */ + if ((val1 == 2 && + val2 == pm4_type3_packet(PM4_INTERRUPT, 1) + && val3 == CP_INT_CNTL__RB_INT_MASK) || + (val1 == pm4_type3_packet(PM4_EVENT_WRITE, 3) + && val2 == CACHE_FLUSH_TS && + val3 == (rb->device->memstore.gpuaddr + + KGSL_DEVICE_MEMSTORE_OFFSET(eoptimestamp)))) { + rb_rptr = adreno_ringbuffer_inc_wrapped(rb_rptr, + rb->buffer_desc.size); + KGSL_DRV_ERR(device, + "Found end of last executed " + "command at offset: %x\n", + rb_rptr / sizeof(unsigned int)); + break; + } else { + if (rb_rptr < (3 * sizeof(unsigned int))) + rb_rptr = rb->buffer_desc.size - + (3 * sizeof(unsigned int)) + + rb_rptr; + else + rb_rptr -= (3 * sizeof(unsigned int)); + } + } + + if (rb_rptr == 0) + rb_rptr = rb->buffer_desc.size - sizeof(unsigned int); + else + rb_rptr -= sizeof(unsigned int); + } + + if ((rb_rptr / sizeof(unsigned int)) == rb->wptr) { + KGSL_DRV_ERR(device, + "GPU recovery from hang not possible because last" + " successful timestamp is overwritten\n"); + return -EINVAL; + } + /* rb_rptr is now pointing to the first dword of the command following + * the last sucessfully executed command sequence. Assumption is that + * GPU is hung in the command sequence pointed by rb_rptr */ + /* make sure the GPU is not hung in a command submitted by kgsl + * itself */ + kgsl_sharedmem_readl(&rb->buffer_desc, &val1, rb_rptr); + kgsl_sharedmem_readl(&rb->buffer_desc, &val2, + adreno_ringbuffer_inc_wrapped(rb_rptr, + rb->buffer_desc.size)); + if (val1 == pm4_nop_packet(1) && val2 == KGSL_CMD_IDENTIFIER) { + KGSL_DRV_ERR(device, + "GPU recovery from hang not possible because " + "of hang in kgsl command\n"); + return -EINVAL; + } + + /* current_context is the context that is presently active in the + * GPU, i.e the context in which the hang is caused */ + kgsl_sharedmem_readl(&device->memstore, &cur_context, + KGSL_DEVICE_MEMSTORE_OFFSET(current_context)); + while ((rb_rptr / sizeof(unsigned int)) != rb->wptr) { + kgsl_sharedmem_readl(&rb->buffer_desc, &value, rb_rptr); + rb_rptr = adreno_ringbuffer_inc_wrapped(rb_rptr, + rb->buffer_desc.size); + /* check for context switch indicator */ + if (value == KGSL_CONTEXT_TO_MEM_IDENTIFIER) { + kgsl_sharedmem_readl(&rb->buffer_desc, &value, rb_rptr); + rb_rptr = adreno_ringbuffer_inc_wrapped(rb_rptr, + rb->buffer_desc.size); + BUG_ON(value != pm4_type3_packet(PM4_MEM_WRITE, 2)); + kgsl_sharedmem_readl(&rb->buffer_desc, &val1, rb_rptr); + rb_rptr = adreno_ringbuffer_inc_wrapped(rb_rptr, + rb->buffer_desc.size); + BUG_ON(val1 != (device->memstore.gpuaddr + + KGSL_DEVICE_MEMSTORE_OFFSET(current_context))); + kgsl_sharedmem_readl(&rb->buffer_desc, &value, rb_rptr); + rb_rptr = adreno_ringbuffer_inc_wrapped(rb_rptr, + rb->buffer_desc.size); + BUG_ON((copy_rb_contents == 0) && + (value == cur_context)); + /* + * If we were copying the commands and got to this point + * then we need to remove the 3 commands that appear + * before KGSL_CONTEXT_TO_MEM_IDENTIFIER + */ + if (temp_idx) + temp_idx -= 3; + /* if context switches to a context that did not cause + * hang then start saving the rb contents as those + * commands can be executed */ + if (value != cur_context) { + copy_rb_contents = 1; + temp_rb_buffer[temp_idx++] = pm4_nop_packet(1); + temp_rb_buffer[temp_idx++] = + KGSL_CMD_IDENTIFIER; + temp_rb_buffer[temp_idx++] = pm4_nop_packet(1); + temp_rb_buffer[temp_idx++] = + KGSL_CONTEXT_TO_MEM_IDENTIFIER; + temp_rb_buffer[temp_idx++] = + pm4_type3_packet(PM4_MEM_WRITE, 2); + temp_rb_buffer[temp_idx++] = val1; + temp_rb_buffer[temp_idx++] = value; + } else { + copy_rb_contents = 0; + } + } else if (copy_rb_contents) + temp_rb_buffer[temp_idx++] = value; + } + + *rb_size = temp_idx; + KGSL_DRV_ERR(device, "Extracted rb contents, size: %x\n", *rb_size); + for (temp_idx = 0; temp_idx < *rb_size;) { + char str[80]; + int idx = 0; + if ((temp_idx + 8) <= *rb_size) + j = 8; + else + j = *rb_size - temp_idx; + for (; j != 0; j--) + idx += scnprintf(str + idx, 80 - idx, + "%8.8X ", temp_rb_buffer[temp_idx++]); + printk(KERN_ALERT "%s", str); + } + return 0; +} + +void +adreno_ringbuffer_restore(struct adreno_ringbuffer *rb, unsigned int *rb_buff, + int num_rb_contents) +{ + int i; + unsigned int *ringcmds; + unsigned int rcmd_gpu; + + if (!num_rb_contents) + return; + + if (num_rb_contents > (rb->buffer_desc.size - rb->wptr)) { + adreno_regwrite(rb->device, REG_CP_RB_RPTR, 0); + rb->rptr = 0; + BUG_ON(num_rb_contents > rb->buffer_desc.size); + } + ringcmds = (unsigned int *)rb->buffer_desc.hostptr + rb->wptr; + rcmd_gpu = rb->buffer_desc.gpuaddr + sizeof(unsigned int) * rb->wptr; + for (i = 0; i < num_rb_contents; i++) + GSL_RB_WRITE(ringcmds, rcmd_gpu, rb_buff[i]); + rb->wptr += num_rb_contents; + adreno_ringbuffer_submit(rb); +} diff --git a/drivers/gpu/msm/adreno_ringbuffer.h b/drivers/gpu/msm/adreno_ringbuffer.h new file mode 100644 index 00000000..9162dea2 --- /dev/null +++ b/drivers/gpu/msm/adreno_ringbuffer.h @@ -0,0 +1,172 @@ +/* Copyright (c) 2002,2007-2011, Code Aurora Forum. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are + * met: + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following + * disclaimer in the documentation and/or other materials provided + * with the distribution. + * * Neither the name of Code Aurora Forum, Inc. nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED "AS IS" AND ANY EXPRESS OR IMPLIED + * WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS + * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR + * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, + * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE + * OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN + * IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + */ +#ifndef __ADRENO_RINGBUFFER_H +#define __ADRENO_RINGBUFFER_H + +#define GSL_RB_USE_MEM_RPTR +#define GSL_RB_USE_MEM_TIMESTAMP +#define GSL_DEVICE_SHADOW_MEMSTORE_TO_USER + +/* + * Adreno ringbuffer sizes in bytes - these are converted to + * the appropriate log2 values in the code + */ + +#define KGSL_RB_SIZE (32 * 1024) +#define KGSL_RB_BLKSIZE 16 + +/* CP timestamp register */ +#define REG_CP_TIMESTAMP REG_SCRATCH_REG0 + + +struct kgsl_device; +struct kgsl_device_private; + +#define GSL_RB_MEMPTRS_SCRATCH_COUNT 8 +struct kgsl_rbmemptrs { + int rptr; + int wptr_poll; +}; + +#define GSL_RB_MEMPTRS_RPTR_OFFSET \ + (offsetof(struct kgsl_rbmemptrs, rptr)) + +#define GSL_RB_MEMPTRS_WPTRPOLL_OFFSET \ + (offsetof(struct kgsl_rbmemptrs, wptr_poll)) + +struct adreno_ringbuffer { + struct kgsl_device *device; + uint32_t flags; + + struct kgsl_memdesc buffer_desc; + + struct kgsl_memdesc memptrs_desc; + struct kgsl_rbmemptrs *memptrs; + + /*ringbuffer size */ + unsigned int sizedwords; + + unsigned int wptr; /* write pointer offset in dwords from baseaddr */ + unsigned int rptr; /* read pointer offset in dwords from baseaddr */ + uint32_t timestamp; +}; + +/* dword base address of the GFX decode space */ +#define GSL_HAL_SUBBLOCK_OFFSET(reg) ((unsigned int)((reg) - (0x2000))) + +#define GSL_RB_WRITE(ring, gpuaddr, data) \ + do { \ + writel_relaxed(data, ring); \ + wmb(); \ + kgsl_cffdump_setmem(gpuaddr, data, 4); \ + ring++; \ + gpuaddr += sizeof(uint); \ + } while (0) + +/* timestamp */ +#ifdef GSL_DEVICE_SHADOW_MEMSTORE_TO_USER +#define GSL_RB_USE_MEM_TIMESTAMP +#endif /* GSL_DEVICE_SHADOW_MEMSTORE_TO_USER */ + +#ifdef GSL_RB_USE_MEM_TIMESTAMP +/* enable timestamp (...scratch0) memory shadowing */ +#define GSL_RB_MEMPTRS_SCRATCH_MASK 0x1 +#define GSL_RB_INIT_TIMESTAMP(rb) + +#else +#define GSL_RB_MEMPTRS_SCRATCH_MASK 0x0 +#define GSL_RB_INIT_TIMESTAMP(rb) \ + adreno_regwrite((rb)->device->id, REG_CP_TIMESTAMP, 0) + +#endif /* GSL_RB_USE_MEMTIMESTAMP */ + +/* mem rptr */ +#ifdef GSL_RB_USE_MEM_RPTR +#define GSL_RB_CNTL_NO_UPDATE 0x0 /* enable */ +#define GSL_RB_GET_READPTR(rb, data) \ + do { \ + *(data) = readl_relaxed(&(rb)->memptrs->rptr); \ + } while (0) +#else +#define GSL_RB_CNTL_NO_UPDATE 0x1 /* disable */ +#define GSL_RB_GET_READPTR(rb, data) \ + do { \ + adreno_regread((rb)->device->id, REG_CP_RB_RPTR, (data)); \ + } while (0) +#endif /* GSL_RB_USE_MEMRPTR */ + +#define GSL_RB_CNTL_POLL_EN 0x0 /* disable */ + +int adreno_ringbuffer_issueibcmds(struct kgsl_device_private *dev_priv, + struct kgsl_context *context, + struct kgsl_ibdesc *ibdesc, + unsigned int numibs, + uint32_t *timestamp, + unsigned int flags); + +int adreno_ringbuffer_init(struct kgsl_device *device); + +int adreno_ringbuffer_start(struct adreno_ringbuffer *rb, + unsigned int init_ram); + +int adreno_ringbuffer_stop(struct adreno_ringbuffer *rb); + +int adreno_ringbuffer_close(struct adreno_ringbuffer *rb); + +void adreno_ringbuffer_issuecmds(struct kgsl_device *device, + unsigned int flags, + unsigned int *cmdaddr, + int sizedwords); + +void kgsl_cp_intrcallback(struct kgsl_device *device); + +int adreno_ringbuffer_extract(struct adreno_ringbuffer *rb, + unsigned int *temp_rb_buffer, + int *rb_size); + +void +adreno_ringbuffer_restore(struct adreno_ringbuffer *rb, unsigned int *rb_buff, + int num_rb_contents); + +static inline int adreno_ringbuffer_count(struct adreno_ringbuffer *rb, + unsigned int rptr) +{ + if (rb->wptr >= rptr) + return rb->wptr - rptr; + return rb->wptr + rb->sizedwords - rptr; +} + +/* Increment a value by 4 bytes with wrap-around based on size */ +static inline unsigned int adreno_ringbuffer_inc_wrapped(unsigned int val, + unsigned int size) +{ + return (val + sizeof(unsigned int)) % size; +} + +#endif /* __ADRENO_RINGBUFFER_H */ diff --git a/drivers/gpu/msm/kgsl.c b/drivers/gpu/msm/kgsl.c new file mode 100644 index 00000000..2732ffeb --- /dev/null +++ b/drivers/gpu/msm/kgsl.c @@ -0,0 +1,2177 @@ +/* Copyright (c) 2008-2011, Code Aurora Forum. All rights reserved. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 and + * only version 2 as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + */ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include + +#include "kgsl.h" +#include "kgsl_debugfs.h" +#include "kgsl_cffdump.h" + +#undef MODULE_PARAM_PREFIX +#define MODULE_PARAM_PREFIX "kgsl." + +static int kgsl_pagetable_count = KGSL_PAGETABLE_COUNT; +module_param_named(ptcount, kgsl_pagetable_count, int, 0); +MODULE_PARM_DESC(kgsl_pagetable_count, +"Minimum number of pagetables for KGSL to allocate at initialization time"); + +static inline struct kgsl_mem_entry * +kgsl_mem_entry_create(void) +{ + struct kgsl_mem_entry *entry = kzalloc(sizeof(*entry), GFP_KERNEL); + + if (!entry) + KGSL_CORE_ERR("kzalloc(%d) failed\n", sizeof(*entry)); + else + kref_init(&entry->refcount); + + return entry; +} + +void +kgsl_mem_entry_destroy(struct kref *kref) +{ + struct kgsl_mem_entry *entry = container_of(kref, + struct kgsl_mem_entry, + refcount); + size_t size = entry->memdesc.size; + + kgsl_sharedmem_free(&entry->memdesc); + + if (entry->memtype == KGSL_USER_MEMORY) + entry->priv->stats.user -= size; + else if (entry->memtype == KGSL_MAPPED_MEMORY) { + if (entry->file_ptr) + fput(entry->file_ptr); + + kgsl_driver.stats.mapped -= size; + entry->priv->stats.mapped -= size; + } + + kfree(entry); +} +EXPORT_SYMBOL(kgsl_mem_entry_destroy); + +static +void kgsl_mem_entry_attach_process(struct kgsl_mem_entry *entry, + struct kgsl_process_private *process) +{ + spin_lock(&process->mem_lock); + list_add(&entry->list, &process->mem_list); + spin_unlock(&process->mem_lock); + + entry->priv = process; +} + +/* Allocate a new context id */ + +static struct kgsl_context * +kgsl_create_context(struct kgsl_device_private *dev_priv) +{ + struct kgsl_context *context; + int ret, id; + + context = kzalloc(sizeof(*context), GFP_KERNEL); + + if (context == NULL) + return NULL; + + while (1) { + if (idr_pre_get(&dev_priv->device->context_idr, + GFP_KERNEL) == 0) { + kfree(context); + return NULL; + } + + ret = idr_get_new(&dev_priv->device->context_idr, + context, &id); + + if (ret != -EAGAIN) + break; + } + + if (ret) { + kfree(context); + return NULL; + } + + context->id = id; + context->dev_priv = dev_priv; + + return context; +} + +static void +kgsl_destroy_context(struct kgsl_device_private *dev_priv, + struct kgsl_context *context) +{ + int id; + + if (context == NULL) + return; + + /* Fire a bug if the devctxt hasn't been freed */ + BUG_ON(context->devctxt); + + id = context->id; + kfree(context); + + idr_remove(&dev_priv->device->context_idr, id); +} + +/* to be called when a process is destroyed, this walks the memqueue and + * frees any entryies that belong to the dying process + */ +static void kgsl_memqueue_cleanup(struct kgsl_device *device, + struct kgsl_process_private *private) +{ + struct kgsl_mem_entry *entry, *entry_tmp; + + if (!private) + return; + + BUG_ON(!mutex_is_locked(&device->mutex)); + + list_for_each_entry_safe(entry, entry_tmp, &device->memqueue, list) { + if (entry->priv == private) { + list_del(&entry->list); + kgsl_mem_entry_put(entry); + } + } +} + +static void kgsl_memqueue_freememontimestamp(struct kgsl_device *device, + struct kgsl_mem_entry *entry, + uint32_t timestamp, + enum kgsl_timestamp_type type) +{ + BUG_ON(!mutex_is_locked(&device->mutex)); + + entry->free_timestamp = timestamp; + + list_add_tail(&entry->list, &device->memqueue); +} + +static void kgsl_memqueue_drain(struct kgsl_device *device) +{ + struct kgsl_mem_entry *entry, *entry_tmp; + uint32_t ts_processed; + + BUG_ON(!mutex_is_locked(&device->mutex)); + + /* get current EOP timestamp */ + ts_processed = device->ftbl.device_readtimestamp( + device, + KGSL_TIMESTAMP_RETIRED); + + list_for_each_entry_safe(entry, entry_tmp, &device->memqueue, list) { + KGSL_MEM_INFO(device, + "ts_processed %d ts_free %d gpuaddr %x)\n", + ts_processed, entry->free_timestamp, + entry->memdesc.gpuaddr); + if (!timestamp_cmp(ts_processed, entry->free_timestamp)) + break; + + list_del(&entry->list); + kgsl_mem_entry_put(entry); + } +} + +static void kgsl_memqueue_drain_unlocked(struct kgsl_device *device) +{ + mutex_lock(&device->mutex); + kgsl_check_suspended(device); + kgsl_memqueue_drain(device); + mutex_unlock(&device->mutex); +} + +static void kgsl_check_idle_locked(struct kgsl_device *device) +{ + if (device->pwrctrl.nap_allowed == true && + device->state == KGSL_STATE_ACTIVE && + device->requested_state == KGSL_STATE_NONE) { + device->requested_state = KGSL_STATE_NAP; + if (kgsl_pwrctrl_sleep(device) != 0) + mod_timer(&device->idle_timer, + jiffies + + device->pwrctrl.interval_timeout); + } +} + +static void kgsl_check_idle(struct kgsl_device *device) +{ + mutex_lock(&device->mutex); + kgsl_check_idle_locked(device); + mutex_unlock(&device->mutex); +} + +struct kgsl_device *kgsl_get_device(int dev_idx) +{ + int i; + struct kgsl_device *ret = NULL; + + mutex_lock(&kgsl_driver.devlock); + + for (i = 0; i < KGSL_DEVICE_MAX; i++) { + if (kgsl_driver.devp[i] && kgsl_driver.devp[i]->id == dev_idx) { + ret = kgsl_driver.devp[i]; + break; + } + } + + mutex_unlock(&kgsl_driver.devlock); + return ret; +} +EXPORT_SYMBOL(kgsl_get_device); + +static struct kgsl_device *kgsl_get_minor(int minor) +{ + struct kgsl_device *ret = NULL; + + if (minor < 0 || minor >= KGSL_DEVICE_MAX) + return NULL; + + mutex_lock(&kgsl_driver.devlock); + ret = kgsl_driver.devp[minor]; + mutex_unlock(&kgsl_driver.devlock); + + return ret; +} + +int kgsl_register_ts_notifier(struct kgsl_device *device, + struct notifier_block *nb) +{ + BUG_ON(device == NULL); + return atomic_notifier_chain_register(&device->ts_notifier_list, + nb); +} +EXPORT_SYMBOL(kgsl_register_ts_notifier); + +int kgsl_unregister_ts_notifier(struct kgsl_device *device, + struct notifier_block *nb) +{ + BUG_ON(device == NULL); + return atomic_notifier_chain_unregister(&device->ts_notifier_list, + nb); +} +EXPORT_SYMBOL(kgsl_unregister_ts_notifier); + +int kgsl_check_timestamp(struct kgsl_device *device, unsigned int timestamp) +{ + unsigned int ts_processed; + BUG_ON(device->ftbl.device_readtimestamp == NULL); + + ts_processed = device->ftbl.device_readtimestamp( + device, KGSL_TIMESTAMP_RETIRED); + + return timestamp_cmp(ts_processed, timestamp); +} +EXPORT_SYMBOL(kgsl_check_timestamp); + +int kgsl_setstate(struct kgsl_device *device, uint32_t flags) +{ + int status = -ENXIO; + + if (flags && device->ftbl.device_setstate) { + status = device->ftbl.device_setstate(device, flags); + } else + status = 0; + + return status; +} +EXPORT_SYMBOL(kgsl_setstate); + +int kgsl_idle(struct kgsl_device *device, unsigned int timeout) +{ + int status = -ENXIO; + + if (device->ftbl.device_idle) + status = device->ftbl.device_idle(device, timeout); + + return status; +} +EXPORT_SYMBOL(kgsl_idle); + +static int kgsl_suspend_device(struct kgsl_device *device, pm_message_t state) +{ + int status = -EINVAL; + unsigned int nap_allowed_saved; + + if (!device) + return -EINVAL; + + KGSL_PWR_WARN(device, "suspend start\n"); + + mutex_lock(&device->mutex); + nap_allowed_saved = device->pwrctrl.nap_allowed; + device->pwrctrl.nap_allowed = false; + device->requested_state = KGSL_STATE_SUSPEND; + /* Make sure no user process is waiting for a timestamp * + * before supending */ + if (device->active_cnt != 0) { + mutex_unlock(&device->mutex); + wait_for_completion(&device->suspend_gate); + mutex_lock(&device->mutex); + } + /* Don't let the timer wake us during suspended sleep. */ + del_timer(&device->idle_timer); + switch (device->state) { + case KGSL_STATE_INIT: + break; + case KGSL_STATE_ACTIVE: + /* Wait for the device to become idle */ + device->ftbl.device_idle(device, KGSL_TIMEOUT_DEFAULT); + case KGSL_STATE_NAP: + case KGSL_STATE_SLEEP: + /* Get the completion ready to be waited upon. */ + INIT_COMPLETION(device->hwaccess_gate); + device->ftbl.device_suspend_context(device); + device->ftbl.device_stop(device); + device->state = KGSL_STATE_SUSPEND; + KGSL_PWR_WARN(device, "state -> SUSPEND, device %d\n", + device->id); + break; + default: + KGSL_PWR_ERR(device, "suspend fail, device %d\n", + device->id); + goto end; + } + device->requested_state = KGSL_STATE_NONE; + device->pwrctrl.nap_allowed = nap_allowed_saved; + status = 0; + +end: + mutex_unlock(&device->mutex); + KGSL_PWR_WARN(device, "suspend end\n"); + return status; +} + +static int kgsl_resume_device(struct kgsl_device *device) +{ + int status = -EINVAL; + + if (!device) + return -EINVAL; + + KGSL_PWR_WARN(device, "resume start\n"); + mutex_lock(&device->mutex); + if (device->state == KGSL_STATE_SUSPEND) { + device->requested_state = KGSL_STATE_ACTIVE; + status = device->ftbl.device_start(device, 0); + if (status == 0) { + device->state = KGSL_STATE_ACTIVE; + KGSL_PWR_WARN(device, + "state -> ACTIVE, device %d\n", + device->id); + } else { + KGSL_PWR_ERR(device, + "resume failed, device %d\n", + device->id); + device->state = KGSL_STATE_INIT; + goto end; + } + status = device->ftbl.device_resume_context(device); + complete_all(&device->hwaccess_gate); + } + device->requested_state = KGSL_STATE_NONE; + +end: + mutex_unlock(&device->mutex); + KGSL_PWR_WARN(device, "resume end\n"); + return status; +} + +static int kgsl_suspend(struct device *dev) +{ + + pm_message_t arg = {0}; + struct kgsl_device *device = dev_get_drvdata(dev); + return kgsl_suspend_device(device, arg); +} + +static int kgsl_resume(struct device *dev) +{ + struct kgsl_device *device = dev_get_drvdata(dev); + return kgsl_resume_device(device); +} + +static int kgsl_runtime_suspend(struct device *dev) +{ + return 0; +} + +static int kgsl_runtime_resume(struct device *dev) +{ + return 0; +} + +const struct dev_pm_ops kgsl_pm_ops = { + .suspend = kgsl_suspend, + .resume = kgsl_resume, + .runtime_suspend = kgsl_runtime_suspend, + .runtime_resume = kgsl_runtime_resume, +}; +EXPORT_SYMBOL(kgsl_pm_ops); + +int kgsl_suspend_driver(struct platform_device *pdev, + pm_message_t state) +{ + struct kgsl_device *device = dev_get_drvdata(&pdev->dev); + return kgsl_suspend_device(device, state); +} +EXPORT_SYMBOL(kgsl_suspend_driver); + +int kgsl_resume_driver(struct platform_device *pdev) +{ + struct kgsl_device *device = dev_get_drvdata(&pdev->dev); + return kgsl_resume_device(device); +} +EXPORT_SYMBOL(kgsl_resume_driver); + +/* file operations */ +static struct kgsl_process_private * +kgsl_get_process_private(struct kgsl_device_private *cur_dev_priv) +{ + struct kgsl_process_private *private; + + mutex_lock(&kgsl_driver.process_mutex); + list_for_each_entry(private, &kgsl_driver.process_list, list) { + if (private->pid == task_tgid_nr(current)) { + private->refcnt++; + goto out; + } + } + + /* no existing process private found for this dev_priv, create one */ + private = kzalloc(sizeof(struct kgsl_process_private), GFP_KERNEL); + if (private == NULL) { + KGSL_DRV_ERR(cur_dev_priv->device, "kzalloc(%d) failed\n", + sizeof(struct kgsl_process_private)); + goto out; + } + + spin_lock_init(&private->mem_lock); + private->refcnt = 1; + private->pid = task_tgid_nr(current); + + INIT_LIST_HEAD(&private->mem_list); + +#ifdef CONFIG_MSM_KGSL_MMU + { + unsigned long pt_name; + +#ifdef CONFIG_KGSL_PER_PROCESS_PAGE_TABLE + pt_name = task_tgid_nr(current); +#else + pt_name = KGSL_MMU_GLOBAL_PT; +#endif + private->pagetable = kgsl_mmu_getpagetable(pt_name); + if (private->pagetable == NULL) { + kfree(private); + private = NULL; + goto out; + } + } +#endif + + list_add(&private->list, &kgsl_driver.process_list); + + kgsl_process_init_sysfs(private); + +out: + mutex_unlock(&kgsl_driver.process_mutex); + return private; +} + +static void +kgsl_put_process_private(struct kgsl_device *device, + struct kgsl_process_private *private) +{ + struct kgsl_mem_entry *entry = NULL; + struct kgsl_mem_entry *entry_tmp = NULL; + + if (!private) + return; + + mutex_lock(&kgsl_driver.process_mutex); + + if (--private->refcnt) + goto unlock; + + KGSL_MEM_INFO(device, + "Memory usage: user (%d/%d) mapped (%d/%d)\n", + private->stats.user, private->stats.user_max, + private->stats.mapped, private->stats.mapped_max); + + kgsl_process_uninit_sysfs(private); + + list_del(&private->list); + + list_for_each_entry_safe(entry, entry_tmp, &private->mem_list, list) { + list_del(&entry->list); + kgsl_mem_entry_put(entry); + } + + kgsl_mmu_putpagetable(private->pagetable); + kfree(private); +unlock: + mutex_unlock(&kgsl_driver.process_mutex); +} + +static int kgsl_release(struct inode *inodep, struct file *filep) +{ + int result = 0; + struct kgsl_device_private *dev_priv = filep->private_data; + struct kgsl_process_private *private = dev_priv->process_priv; + struct kgsl_device *device = dev_priv->device; + struct kgsl_context *context; + int next = 0; + + filep->private_data = NULL; + + mutex_lock(&device->mutex); + kgsl_check_suspended(device); + + while (1) { + context = idr_get_next(&device->context_idr, &next); + if (context == NULL) + break; + + if (context->dev_priv == dev_priv) { + device->ftbl.device_drawctxt_destroy(device, context); + kgsl_destroy_context(dev_priv, context); + } + + next = next + 1; + } + + device->open_count--; + if (device->open_count == 0) { + result = device->ftbl.device_stop(device); + device->state = KGSL_STATE_INIT; + KGSL_PWR_WARN(device, "state -> INIT, device %d\n", device->id); + } + /* clean up any to-be-freed entries that belong to this + * process and this device + */ + kgsl_memqueue_cleanup(device, private); + + mutex_unlock(&device->mutex); + kfree(dev_priv); + + kgsl_put_process_private(device, private); + + pm_runtime_put(device->parentdev); + return result; +} + +static int kgsl_open(struct inode *inodep, struct file *filep) +{ + int result; + struct kgsl_device_private *dev_priv; + struct kgsl_device *device; + unsigned int minor = iminor(inodep); + + device = kgsl_get_minor(minor); + BUG_ON(device == NULL); + + if (filep->f_flags & O_EXCL) { + KGSL_DRV_ERR(device, "O_EXCL not allowed\n"); + return -EBUSY; + } + + result = pm_runtime_get_sync(device->parentdev); + result = 0; + if (result < 0) { + KGSL_DRV_ERR(device, + "Runtime PM: Unable to wake up the device, rc = %d\n", + result); + return result; + } + result = 0; + + dev_priv = kzalloc(sizeof(struct kgsl_device_private), GFP_KERNEL); + if (dev_priv == NULL) { + KGSL_DRV_ERR(device, "kzalloc failed(%d)\n", + sizeof(struct kgsl_device_private)); + result = -ENOMEM; + goto err_pmruntime; + } + + dev_priv->device = device; + filep->private_data = dev_priv; + + /* Get file (per process) private struct */ + dev_priv->process_priv = kgsl_get_process_private(dev_priv); + if (dev_priv->process_priv == NULL) { + result = -ENOMEM; + goto err_freedevpriv; + } + + mutex_lock(&device->mutex); + kgsl_check_suspended(device); + + if (device->open_count == 0) { + result = device->ftbl.device_start(device, true); + + if (result) { + mutex_unlock(&device->mutex); + goto err_putprocess; + } + device->state = KGSL_STATE_ACTIVE; + KGSL_PWR_WARN(device, + "state -> ACTIVE, device %d\n", minor); + } + device->open_count++; + mutex_unlock(&device->mutex); + + KGSL_DRV_INFO(device, "Initialized %s: mmu=%s pagetable_count=%d\n", + device->name, kgsl_mmu_enabled() ? "on" : "off", + KGSL_PAGETABLE_COUNT); + + return result; + +err_putprocess: + kgsl_put_process_private(device, dev_priv->process_priv); +err_freedevpriv: + filep->private_data = NULL; + kfree(dev_priv); +err_pmruntime: + pm_runtime_put(device->parentdev); + return result; +} + + +/*call with private->mem_lock locked */ +static struct kgsl_mem_entry * +kgsl_sharedmem_find(struct kgsl_process_private *private, unsigned int gpuaddr) +{ + struct kgsl_mem_entry *entry = NULL, *result = NULL; + + BUG_ON(private == NULL); + + gpuaddr &= PAGE_MASK; + + list_for_each_entry(entry, &private->mem_list, list) { + if (entry->memdesc.gpuaddr == gpuaddr) { + result = entry; + break; + } + } + return result; +} + +/*call with private->mem_lock locked */ +struct kgsl_mem_entry * +kgsl_sharedmem_find_region(struct kgsl_process_private *private, + unsigned int gpuaddr, + size_t size) +{ + struct kgsl_mem_entry *entry = NULL, *result = NULL; + + BUG_ON(private == NULL); + + list_for_each_entry(entry, &private->mem_list, list) { + if (gpuaddr >= entry->memdesc.gpuaddr && + ((gpuaddr + size) <= + (entry->memdesc.gpuaddr + entry->memdesc.size))) { + result = entry; + break; + } + } + + return result; +} +EXPORT_SYMBOL(kgsl_sharedmem_find_region); + +uint8_t *kgsl_gpuaddr_to_vaddr(const struct kgsl_memdesc *memdesc, + unsigned int gpuaddr, unsigned int *size) +{ + BUG_ON(memdesc->hostptr == NULL); + + if (memdesc->gpuaddr == 0 || (gpuaddr < memdesc->gpuaddr || + gpuaddr >= memdesc->gpuaddr + memdesc->size)) + return NULL; + + *size = memdesc->size - (gpuaddr - memdesc->gpuaddr); + return memdesc->hostptr + (gpuaddr - memdesc->gpuaddr); +} +EXPORT_SYMBOL(kgsl_gpuaddr_to_vaddr); + +/*call all ioctl sub functions with driver locked*/ +static long kgsl_ioctl_device_getproperty(struct kgsl_device_private *dev_priv, + unsigned int cmd, void *data) +{ + int result = 0; + struct kgsl_device_getproperty *param = data; + + switch (param->type) { + case KGSL_PROP_VERSION: + { + struct kgsl_version version; + if (param->sizebytes != sizeof(version)) { + result = -EINVAL; + break; + } + + version.drv_major = KGSL_VERSION_MAJOR; + version.drv_minor = KGSL_VERSION_MINOR; + version.dev_major = dev_priv->device->ver_major; + version.dev_minor = dev_priv->device->ver_minor; + + if (copy_to_user(param->value, &version, sizeof(version))) + result = -EFAULT; + + break; + } + default: + result = dev_priv->device->ftbl.device_getproperty( + dev_priv->device, param->type, + param->value, param->sizebytes); + } + + + return result; +} + +static long kgsl_ioctl_device_waittimestamp(struct kgsl_device_private + *dev_priv, unsigned int cmd, + void *data) +{ + int result = 0; + struct kgsl_device_waittimestamp *param = data; + + /* Set the active count so that suspend doesn't do the + wrong thing */ + + dev_priv->device->active_cnt++; + + /* Don't wait forever, set a max value for now */ + if (param->timeout == -1) + param->timeout = 10 * MSEC_PER_SEC; + + result = dev_priv->device->ftbl.device_waittimestamp(dev_priv->device, + param->timestamp, + param->timeout); + + kgsl_memqueue_drain(dev_priv->device); + + /* Fire off any pending suspend operations that are in flight */ + + INIT_COMPLETION(dev_priv->device->suspend_gate); + dev_priv->device->active_cnt--; + complete(&dev_priv->device->suspend_gate); + + return result; +} +static bool check_ibdesc(struct kgsl_device_private *dev_priv, + struct kgsl_ibdesc *ibdesc, unsigned int numibs, + bool parse) +{ + bool result = true; + unsigned int i; + for (i = 0; i < numibs; i++) { + struct kgsl_mem_entry *entry; + spin_lock(&dev_priv->process_priv->mem_lock); + entry = kgsl_sharedmem_find_region(dev_priv->process_priv, + ibdesc[i].gpuaddr, ibdesc[i].sizedwords * sizeof(uint)); + spin_unlock(&dev_priv->process_priv->mem_lock); + if (entry == NULL) { + KGSL_DRV_ERR(dev_priv->device, + "invalid cmd buffer gpuaddr %08x " \ + "sizedwords %d\n", ibdesc[i].gpuaddr, + ibdesc[i].sizedwords); + result = false; + break; + } + + if (parse && !kgsl_cffdump_parse_ibs(dev_priv, &entry->memdesc, + ibdesc[i].gpuaddr, ibdesc[i].sizedwords, true)) { + KGSL_DRV_ERR(dev_priv->device, + "invalid cmd buffer gpuaddr %08x " \ + "sizedwords %d numibs %d/%d\n", + ibdesc[i].gpuaddr, + ibdesc[i].sizedwords, i+1, numibs); + result = false; + break; + } + } + return result; +} + +static long kgsl_ioctl_rb_issueibcmds(struct kgsl_device_private *dev_priv, + unsigned int cmd, void *data) +{ + int result = 0; + struct kgsl_ringbuffer_issueibcmds *param = data; + struct kgsl_ibdesc *ibdesc; + struct kgsl_context *context; + +#ifdef CONFIG_MSM_KGSL_DRM + kgsl_gpu_mem_flush(DRM_KGSL_GEM_CACHE_OP_TO_DEV); +#endif + + context = kgsl_find_context(dev_priv, param->drawctxt_id); + if (context == NULL) { + result = -EINVAL; + KGSL_DRV_ERR(dev_priv->device, + "invalid drawctxt drawctxt_id %d\n", + param->drawctxt_id); + goto done; + } + + if (param->flags & KGSL_CONTEXT_SUBMIT_IB_LIST) { + KGSL_DRV_INFO(dev_priv->device, + "Using IB list mode for ib submission, numibs: %d\n", + param->numibs); + if (!param->numibs) { + KGSL_DRV_ERR(dev_priv->device, + "Invalid numibs as parameter: %d\n", + param->numibs); + result = -EINVAL; + goto done; + } + + ibdesc = kzalloc(sizeof(struct kgsl_ibdesc) * param->numibs, + GFP_KERNEL); + if (!ibdesc) { + KGSL_MEM_ERR(dev_priv->device, + "kzalloc(%d) failed\n", + sizeof(struct kgsl_ibdesc) * param->numibs); + result = -ENOMEM; + goto done; + } + + if (copy_from_user(ibdesc, (void *)param->ibdesc_addr, + sizeof(struct kgsl_ibdesc) * param->numibs)) { + result = -EFAULT; + KGSL_DRV_ERR(dev_priv->device, + "copy_from_user failed\n"); + goto free_ibdesc; + } + } else { + KGSL_DRV_INFO(dev_priv->device, + "Using single IB submission mode for ib submission\n"); + /* If user space driver is still using the old mode of + * submitting single ib then we need to support that as well */ + ibdesc = kzalloc(sizeof(struct kgsl_ibdesc), GFP_KERNEL); + if (!ibdesc) { + KGSL_MEM_ERR(dev_priv->device, + "kzalloc(%d) failed\n", + sizeof(struct kgsl_ibdesc)); + result = -ENOMEM; + goto done; + } + ibdesc[0].gpuaddr = param->ibdesc_addr; + ibdesc[0].sizedwords = param->numibs; + param->numibs = 1; + } + + if (!check_ibdesc(dev_priv, ibdesc, param->numibs, true)) { + KGSL_DRV_ERR(dev_priv->device, "bad ibdesc"); + result = -EINVAL; + goto free_ibdesc; + } + + /* Let the pwrscale policy know that a new command buffer + is being issued */ + + kgsl_pwrscale_busy(dev_priv->device); + +/* drewis: don't know what changed this...diff from cherry-pick + f3c1074d1539be20cecbb82f37705bd16058418e */ +/* result = dev_priv->device->ftbl->issueibcmds(dev_priv,*/ + result = dev_priv->device->ftbl.device_issueibcmds(dev_priv, + context, + ibdesc, + param->numibs, + ¶m->timestamp, + param->flags); + + if (result != 0) + goto free_ibdesc; + + /* this is a check to try to detect if a command buffer was freed + * during issueibcmds(). + */ + if (!check_ibdesc(dev_priv, ibdesc, param->numibs, false)) { + KGSL_DRV_ERR(dev_priv->device, "bad ibdesc AFTER issue"); + result = -EINVAL; + goto free_ibdesc; + } + +free_ibdesc: + kfree(ibdesc); +done: + +#ifdef CONFIG_MSM_KGSL_DRM + kgsl_gpu_mem_flush(DRM_KGSL_GEM_CACHE_OP_FROM_DEV); +#endif + + return result; +} + +static long kgsl_ioctl_cmdstream_readtimestamp(struct kgsl_device_private + *dev_priv, unsigned int cmd, + void *data) +{ + struct kgsl_cmdstream_readtimestamp *param = data; + + param->timestamp = + dev_priv->device->ftbl.device_readtimestamp( + dev_priv->device, param->type); + + return 0; +} + +static long kgsl_ioctl_cmdstream_freememontimestamp(struct kgsl_device_private + *dev_priv, unsigned int cmd, + void *data) +{ + int result = 0; + struct kgsl_cmdstream_freememontimestamp *param = data; + struct kgsl_mem_entry *entry = NULL; + + spin_lock(&dev_priv->process_priv->mem_lock); + entry = kgsl_sharedmem_find(dev_priv->process_priv, param->gpuaddr); + if (entry) + list_del(&entry->list); + spin_unlock(&dev_priv->process_priv->mem_lock); + + if (entry) { + kgsl_memqueue_freememontimestamp(dev_priv->device, entry, + param->timestamp, param->type); + kgsl_memqueue_drain(dev_priv->device); + } else { + KGSL_DRV_ERR(dev_priv->device, + "invalid gpuaddr %08x\n", param->gpuaddr); + result = -EINVAL; + } + + return result; +} + +static long kgsl_ioctl_drawctxt_create(struct kgsl_device_private *dev_priv, + unsigned int cmd, void *data) +{ + int result = 0; + struct kgsl_drawctxt_create *param = data; + struct kgsl_context *context = NULL; + + context = kgsl_create_context(dev_priv); + + if (context == NULL) { + result = -ENOMEM; + goto done; + } + + if (dev_priv->device->ftbl.device_drawctxt_create != NULL) + result = dev_priv->device->ftbl.device_drawctxt_create(dev_priv, + param->flags, + context); + + param->drawctxt_id = context->id; + +done: + if (result && context) + kgsl_destroy_context(dev_priv, context); + + return result; +} + +static long kgsl_ioctl_drawctxt_destroy(struct kgsl_device_private *dev_priv, + unsigned int cmd, void *data) +{ + int result = 0; + struct kgsl_drawctxt_destroy *param = data; + struct kgsl_context *context; + + context = kgsl_find_context(dev_priv, param->drawctxt_id); + + if (context == NULL) { + result = -EINVAL; + goto done; + } + + result = dev_priv->device->ftbl.device_drawctxt_destroy( + dev_priv->device, + context); + + kgsl_destroy_context(dev_priv, context); + +done: + return result; +} + +static long kgsl_ioctl_sharedmem_free(struct kgsl_device_private *dev_priv, + unsigned int cmd, void *data) +{ + int result = 0; + struct kgsl_sharedmem_free *param = data; + struct kgsl_process_private *private = dev_priv->process_priv; + struct kgsl_mem_entry *entry = NULL; + + spin_lock(&private->mem_lock); + entry = kgsl_sharedmem_find(private, param->gpuaddr); + if (entry) + list_del(&entry->list); + spin_unlock(&private->mem_lock); + + if (entry) { + kgsl_mem_entry_put(entry); + } else { + KGSL_CORE_ERR("invalid gpuaddr %08x\n", param->gpuaddr); + result = -EINVAL; + } + + return result; +} + +static struct vm_area_struct *kgsl_get_vma_from_start_addr(unsigned int addr) +{ + struct vm_area_struct *vma; + + down_read(¤t->mm->mmap_sem); + vma = find_vma(current->mm, addr); + up_read(¤t->mm->mmap_sem); + if (!vma) + KGSL_CORE_ERR("find_vma(%x) failed\n", addr); + + return vma; +} + +static long +kgsl_ioctl_sharedmem_from_vmalloc(struct kgsl_device_private *dev_priv, + unsigned int cmd, void *data) +{ + int result = 0, len = 0; + struct kgsl_process_private *private = dev_priv->process_priv; + struct kgsl_sharedmem_from_vmalloc *param = data; + struct kgsl_mem_entry *entry = NULL; + struct vm_area_struct *vma; + + if (!kgsl_mmu_enabled()) + return -ENODEV; + + /* Make sure all pending freed memory is collected */ + kgsl_memqueue_drain_unlocked(dev_priv->device); + + if (!param->hostptr) { + KGSL_CORE_ERR("invalid hostptr %x\n", param->hostptr); + result = -EINVAL; + goto error; + } + + vma = kgsl_get_vma_from_start_addr(param->hostptr); + if (!vma) { + result = -EINVAL; + goto error; + } + + /* + * If the user specified a length, use it, otherwise try to + * infer the length if the vma region + */ + if (param->gpuaddr != 0) { + len = param->gpuaddr; + } else { + /* + * For this to work, we have to assume the VMA region is only + * for this single allocation. If it isn't, then bail out + */ + if (vma->vm_pgoff || (param->hostptr != vma->vm_start)) { + KGSL_CORE_ERR("VMA region does not match hostaddr\n"); + result = -EINVAL; + goto error; + } + + len = vma->vm_end - vma->vm_start; + } + + /* Make sure it fits */ + if (len == 0 || param->hostptr + len > vma->vm_end) { + KGSL_CORE_ERR("Invalid memory allocation length %d\n", len); + result = -EINVAL; + goto error; + } + + entry = kgsl_mem_entry_create(); + if (entry == NULL) { + result = -ENOMEM; + goto error; + } + + result = kgsl_sharedmem_vmalloc_user(&entry->memdesc, + private->pagetable, len, + param->flags); + if (result != 0) + goto error_free_entry; + + vma->vm_page_prot = pgprot_writecombine(vma->vm_page_prot); + + result = remap_vmalloc_range(vma, (void *) entry->memdesc.hostptr, 0); + if (result) { + KGSL_CORE_ERR("remap_vmalloc_range failed: %d\n", result); + goto error_free_vmalloc; + } + + param->gpuaddr = entry->memdesc.gpuaddr; + + entry->memtype = KGSL_USER_MEMORY; + + kgsl_mem_entry_attach_process(entry, private); + + /* Process specific statistics */ + KGSL_STATS_ADD(len, private->stats.user, + private->stats.user_max); + + kgsl_check_idle(dev_priv->device); + return 0; + +error_free_vmalloc: + kgsl_sharedmem_free(&entry->memdesc); + +error_free_entry: + kfree(entry); + +error: + kgsl_check_idle(dev_priv->device); + return result; +} + +static inline int _check_region(unsigned long start, unsigned long size, + uint64_t len) +{ + uint64_t end = ((uint64_t) start) + size; + return (end > len); +} + +static int kgsl_get_phys_file(int fd, unsigned long *start, unsigned long *len, + unsigned long *vstart, struct file **filep) +{ + struct file *fbfile; + int ret = 0; + dev_t rdev; + struct fb_info *info; + + *filep = NULL; +#ifdef CONFIG_ANDROID_PMEM + if (!get_pmem_file(fd, start, vstart, len, filep)) + return 0; +#endif + + fbfile = fget(fd); + if (fbfile == NULL) { + KGSL_CORE_ERR("fget_light failed\n"); + return -1; + } + + rdev = fbfile->f_dentry->d_inode->i_rdev; + info = MAJOR(rdev) == FB_MAJOR ? registered_fb[MINOR(rdev)] : NULL; + if (info) { + *start = info->fix.smem_start; + *len = info->fix.smem_len; + *vstart = (unsigned long)__va(info->fix.smem_start); + ret = 0; + } else { + KGSL_CORE_ERR("framebuffer minor %d not found\n", + MINOR(rdev)); + ret = -1; + } + + fput(fbfile); + + return ret; +} + +static int kgsl_setup_phys_file(struct kgsl_mem_entry *entry, + struct kgsl_pagetable *pagetable, + unsigned int fd, unsigned int offset, + size_t size) +{ + int ret; + unsigned long phys, virt, len; + struct file *filep; + + ret = kgsl_get_phys_file(fd, &phys, &len, &virt, &filep); + if (ret) + return ret; + + if (phys == 0) { + ret = -EINVAL; + goto err; + } + + if (offset >= len) { + ret = -EINVAL; + goto err; + } + + if (size == 0) + size = len; + + /* Adjust the size of the region to account for the offset */ + size += offset & ~PAGE_MASK; + + size = ALIGN(size, PAGE_SIZE); + + if (_check_region(offset & PAGE_MASK, size, len)) { + KGSL_CORE_ERR("Offset (%ld) + size (%d) is larger" + "than pmem region length %ld\n", + offset & PAGE_MASK, size, len); + ret = -EINVAL; + goto err; + + } + + entry->file_ptr = filep; + + entry->memdesc.pagetable = pagetable; + entry->memdesc.size = size; + entry->memdesc.physaddr = phys + (offset & PAGE_MASK); + entry->memdesc.hostptr = (void *) (virt + (offset & PAGE_MASK)); + entry->memdesc.ops = &kgsl_contig_ops; + + return 0; +err: +#ifdef CONFIG_ANDROID_PMEM + put_pmem_file(filep); +#endif + return ret; +} + +static int kgsl_setup_hostptr(struct kgsl_mem_entry *entry, + struct kgsl_pagetable *pagetable, + void *hostptr, unsigned int offset, + size_t size) +{ + struct vm_area_struct *vma; + unsigned int len; + + down_read(¤t->mm->mmap_sem); + vma = find_vma(current->mm, (unsigned int) hostptr); + up_read(¤t->mm->mmap_sem); + + if (!vma) { + KGSL_CORE_ERR("find_vma(%p) failed\n", hostptr); + return -EINVAL; + } + + /* We don't necessarily start at vma->vm_start */ + len = vma->vm_end - (unsigned long) hostptr; + + if (offset >= len) + return -EINVAL; + + if (!KGSL_IS_PAGE_ALIGNED((unsigned long) hostptr) || + !KGSL_IS_PAGE_ALIGNED(len)) { + KGSL_CORE_ERR("user address len(%u)" + "and start(%p) must be page" + "aligned\n", len, hostptr); + return -EINVAL; + } + + if (size == 0) + size = len; + + /* Adjust the size of the region to account for the offset */ + size += offset & ~PAGE_MASK; + + size = ALIGN(size, PAGE_SIZE); + + if (_check_region(offset & PAGE_MASK, size, len)) { + KGSL_CORE_ERR("Offset (%ld) + size (%d) is larger" + "than region length %d\n", + offset & PAGE_MASK, size, len); + return -EINVAL; + } + + entry->memdesc.pagetable = pagetable; + entry->memdesc.size = size; + entry->memdesc.hostptr = hostptr + (offset & PAGE_MASK); + entry->memdesc.ops = &kgsl_userptr_ops; + + return 0; +} + +#ifdef CONFIG_ASHMEM +static int kgsl_setup_ashmem(struct kgsl_mem_entry *entry, + struct kgsl_pagetable *pagetable, + int fd, void *hostptr, size_t size) +{ + int ret; + struct vm_area_struct *vma; + struct file *filep, *vmfile; + unsigned long len; + unsigned int hostaddr = (unsigned int) hostptr; + + vma = kgsl_get_vma_from_start_addr(hostaddr); + if (vma == NULL) + return -EINVAL; + + if (vma->vm_pgoff || vma->vm_start != hostaddr) { + KGSL_CORE_ERR("Invalid vma region\n"); + return -EINVAL; + } + + len = vma->vm_end - vma->vm_start; + + if (size == 0) + size = len; + + if (size != len) { + KGSL_CORE_ERR("Invalid size %d for vma region %p\n", + size, hostptr); + return -EINVAL; + } + + ret = get_ashmem_file(fd, &filep, &vmfile, &len); + + if (ret) { + KGSL_CORE_ERR("get_ashmem_file failed\n"); + return ret; + } + + if (vmfile != vma->vm_file) { + KGSL_CORE_ERR("ashmem shmem file does not match vma\n"); + ret = -EINVAL; + goto err; + } + + entry->file_ptr = filep; + + entry->memdesc.pagetable = pagetable; + entry->memdesc.size = ALIGN(size, PAGE_SIZE); + entry->memdesc.hostptr = hostptr; + entry->memdesc.ops = &kgsl_userptr_ops; + + return 0; + +err: + put_ashmem_file(filep); + return ret; +} +#else +static int kgsl_setup_ashmem(struct kgsl_mem_entry *entry, + struct kgsl_pagetable *pagetable, + int fd, void *hostptr, size_t size) +{ + return -EINVAL; +} +#endif + +static long kgsl_ioctl_map_user_mem(struct kgsl_device_private *dev_priv, + unsigned int cmd, void *data) +{ + int result = -EINVAL; + struct kgsl_map_user_mem *param = data; + struct kgsl_mem_entry *entry = NULL; + struct kgsl_process_private *private = dev_priv->process_priv; + enum kgsl_user_mem_type memtype; + + entry = kgsl_mem_entry_create(); + + if (entry == NULL) + return -ENOMEM; + + kgsl_memqueue_drain_unlocked(dev_priv->device); + + if (_IOC_SIZE(cmd) == sizeof(struct kgsl_sharedmem_from_pmem)) + memtype = KGSL_USER_MEM_TYPE_PMEM; + else + memtype = param->memtype; + + switch (memtype) { + case KGSL_USER_MEM_TYPE_PMEM: + if (param->fd == 0 || param->len == 0) + break; + + result = kgsl_setup_phys_file(entry, private->pagetable, + param->fd, param->offset, + param->len); + break; + + case KGSL_USER_MEM_TYPE_ADDR: + if (!kgsl_mmu_enabled()) { + KGSL_DRV_ERR(dev_priv->device, + "Cannot map paged memory with the " + "MMU disabled\n"); + break; + } + + if (param->hostptr == 0) + break; + + result = kgsl_setup_hostptr(entry, private->pagetable, + (void *) param->hostptr, + param->offset, param->len); + break; + + case KGSL_USER_MEM_TYPE_ASHMEM: + if (!kgsl_mmu_enabled()) { + KGSL_DRV_ERR(dev_priv->device, + "Cannot map paged memory with the " + "MMU disabled\n"); + break; + } + + if (param->hostptr == 0) + break; + + result = kgsl_setup_ashmem(entry, private->pagetable, + param->fd, (void *) param->hostptr, + param->len); + break; + default: + KGSL_CORE_ERR("Invalid memory type: %x\n", memtype); + break; + } + + if (result) + goto error; + + result = kgsl_mmu_map(private->pagetable, + &entry->memdesc, + GSL_PT_PAGE_RV | GSL_PT_PAGE_WV); + + if (result) + goto error_put_file_ptr; + + /* Adjust the returned value for a non 4k aligned offset */ + param->gpuaddr = entry->memdesc.gpuaddr + (param->offset & ~PAGE_MASK); + + entry->memtype = KGSL_MAPPED_MEMORY; + + KGSL_STATS_ADD(param->len, kgsl_driver.stats.mapped, + kgsl_driver.stats.mapped_max); + + /* Statistics */ + KGSL_STATS_ADD(param->len, private->stats.mapped, + private->stats.mapped_max); + + kgsl_mem_entry_attach_process(entry, private); + + kgsl_check_idle(dev_priv->device); + return result; + + error_put_file_ptr: + if (entry->file_ptr) + fput(entry->file_ptr); + +error: + kfree(entry); + kgsl_check_idle(dev_priv->device); + return result; +} + +/*This function flushes a graphics memory allocation from CPU cache + *when caching is enabled with MMU*/ +static long +kgsl_ioctl_sharedmem_flush_cache(struct kgsl_device_private *dev_priv, + unsigned int cmd, void *data) +{ + int result = 0; + struct kgsl_mem_entry *entry; + struct kgsl_sharedmem_free *param = data; + struct kgsl_process_private *private = dev_priv->process_priv; + + spin_lock(&private->mem_lock); + entry = kgsl_sharedmem_find(private, param->gpuaddr); + if (!entry) { + KGSL_CORE_ERR("invalid gpuaddr %08x\n", param->gpuaddr); + result = -EINVAL; + goto done; + } + if (!entry->memdesc.hostptr) + entry->memdesc.hostptr = + kgsl_gpuaddr_to_vaddr(&entry->memdesc, + param->gpuaddr, &entry->memdesc.size); + + if (!entry->memdesc.hostptr) { + KGSL_CORE_ERR("invalid hostptr with gpuaddr %08x\n", + param->gpuaddr); + goto done; + } + + kgsl_cache_range_op(&entry->memdesc, KGSL_CACHE_OP_CLEAN); + + /* Statistics - keep track of how many flushes each process does */ + private->stats.flushes++; +done: + spin_unlock(&private->mem_lock); + return result; +} + +static long +kgsl_ioctl_gpumem_alloc(struct kgsl_device_private *dev_priv, + unsigned int cmd, void *data) +{ + struct kgsl_process_private *private = dev_priv->process_priv; + struct kgsl_gpumem_alloc *param = data; + struct kgsl_mem_entry *entry; + int result; + + entry = kgsl_mem_entry_create(); + if (entry == NULL) + return -ENOMEM; + + /* Make sure all pending freed memory is collected */ + kgsl_memqueue_drain_unlocked(dev_priv->device); + + result = kgsl_allocate_user(&entry->memdesc, private->pagetable, + param->size, param->flags); + + if (result == 0) { + entry->memtype = KGSL_USER_MEMORY; + kgsl_mem_entry_attach_process(entry, private); + param->gpuaddr = entry->memdesc.gpuaddr; + + KGSL_STATS_ADD(entry->memdesc.size, private->stats.user, + private->stats.user_max); + } else + kfree(entry); + + kgsl_check_idle(dev_priv->device); + return result; +} +static long kgsl_ioctl_cff_syncmem(struct kgsl_device_private *dev_priv, + unsigned int cmd, void *data) +{ + int result = 0; + struct kgsl_cff_syncmem *param = data; + struct kgsl_process_private *private = dev_priv->process_priv; + struct kgsl_mem_entry *entry = NULL; + + spin_lock(&private->mem_lock); + entry = kgsl_sharedmem_find_region(private, param->gpuaddr, param->len); + if (entry) + kgsl_cffdump_syncmem(dev_priv, &entry->memdesc, param->gpuaddr, + param->len, true); + else + result = -EINVAL; + spin_unlock(&private->mem_lock); + return result; +} + +static long kgsl_ioctl_cff_user_event(struct kgsl_device_private *dev_priv, + unsigned int cmd, void *data) +{ + int result = 0; + struct kgsl_cff_user_event *param = data; + + kgsl_cffdump_user_event(param->cff_opcode, param->op1, param->op2, + param->op3, param->op4, param->op5); + + return result; +} + +typedef long (*kgsl_ioctl_func_t)(struct kgsl_device_private *, + unsigned int, void *); + +#define KGSL_IOCTL_FUNC(_cmd, _func, _lock) \ + [_IOC_NR(_cmd)] = { .cmd = _cmd, .func = _func, .lock = _lock } + +static const struct { + unsigned int cmd; + kgsl_ioctl_func_t func; + int lock; +} kgsl_ioctl_funcs[] = { + KGSL_IOCTL_FUNC(IOCTL_KGSL_DEVICE_GETPROPERTY, + kgsl_ioctl_device_getproperty, 1), + KGSL_IOCTL_FUNC(IOCTL_KGSL_DEVICE_WAITTIMESTAMP, + kgsl_ioctl_device_waittimestamp, 1), + KGSL_IOCTL_FUNC(IOCTL_KGSL_RINGBUFFER_ISSUEIBCMDS, + kgsl_ioctl_rb_issueibcmds, 1), + KGSL_IOCTL_FUNC(IOCTL_KGSL_CMDSTREAM_READTIMESTAMP, + kgsl_ioctl_cmdstream_readtimestamp, 1), + KGSL_IOCTL_FUNC(IOCTL_KGSL_CMDSTREAM_FREEMEMONTIMESTAMP, + kgsl_ioctl_cmdstream_freememontimestamp, 1), + KGSL_IOCTL_FUNC(IOCTL_KGSL_DRAWCTXT_CREATE, + kgsl_ioctl_drawctxt_create, 1), + KGSL_IOCTL_FUNC(IOCTL_KGSL_DRAWCTXT_DESTROY, + kgsl_ioctl_drawctxt_destroy, 1), + KGSL_IOCTL_FUNC(IOCTL_KGSL_MAP_USER_MEM, + kgsl_ioctl_map_user_mem, 0), + KGSL_IOCTL_FUNC(IOCTL_KGSL_SHAREDMEM_FROM_PMEM, + kgsl_ioctl_map_user_mem, 0), + KGSL_IOCTL_FUNC(IOCTL_KGSL_SHAREDMEM_FREE, + kgsl_ioctl_sharedmem_free, 0), + KGSL_IOCTL_FUNC(IOCTL_KGSL_SHAREDMEM_FROM_VMALLOC, + kgsl_ioctl_sharedmem_from_vmalloc, 0), + KGSL_IOCTL_FUNC(IOCTL_KGSL_SHAREDMEM_FLUSH_CACHE, + kgsl_ioctl_sharedmem_flush_cache, 0), + KGSL_IOCTL_FUNC(IOCTL_KGSL_GPUMEM_ALLOC, + kgsl_ioctl_gpumem_alloc, 0), + KGSL_IOCTL_FUNC(IOCTL_KGSL_CFF_SYNCMEM, + kgsl_ioctl_cff_syncmem, 0), + KGSL_IOCTL_FUNC(IOCTL_KGSL_CFF_USER_EVENT, + kgsl_ioctl_cff_user_event, 0), +}; + +static long kgsl_ioctl(struct file *filep, unsigned int cmd, unsigned long arg) +{ + struct kgsl_device_private *dev_priv = filep->private_data; + unsigned int nr = _IOC_NR(cmd); + kgsl_ioctl_func_t func; + int lock, ret; + char ustack[64]; + void *uptr = NULL; + + BUG_ON(dev_priv == NULL); + + /* Workaround for an previously incorrectly defined ioctl code. + This helps ensure binary compatability */ + + if (cmd == IOCTL_KGSL_CMDSTREAM_FREEMEMONTIMESTAMP_OLD) + cmd = IOCTL_KGSL_CMDSTREAM_FREEMEMONTIMESTAMP; + else if (cmd == IOCTL_KGSL_CMDSTREAM_READTIMESTAMP_OLD) + cmd = IOCTL_KGSL_CMDSTREAM_READTIMESTAMP; + + if (cmd & (IOC_IN | IOC_OUT)) { + if (_IOC_SIZE(cmd) < sizeof(ustack)) + uptr = ustack; + else { + uptr = kzalloc(_IOC_SIZE(cmd), GFP_KERNEL); + if (uptr == NULL) { + KGSL_MEM_ERR(dev_priv->device, + "kzalloc(%d) failed\n", _IOC_SIZE(cmd)); + ret = -ENOMEM; + goto done; + } + } + + if (cmd & IOC_IN) { + if (copy_from_user(uptr, (void __user *) arg, + _IOC_SIZE(cmd))) { + ret = -EFAULT; + goto done; + } + } else + memset(uptr, 0, _IOC_SIZE(cmd)); + } + + if (nr < ARRAY_SIZE(kgsl_ioctl_funcs) && + kgsl_ioctl_funcs[nr].func != NULL) { + func = kgsl_ioctl_funcs[nr].func; + lock = kgsl_ioctl_funcs[nr].lock; + } else { + func = dev_priv->device->ftbl.device_ioctl; + lock = 1; + } + + if (lock) { + mutex_lock(&dev_priv->device->mutex); + kgsl_check_suspended(dev_priv->device); + } + + ret = func(dev_priv, cmd, uptr); + + if (lock) { + kgsl_check_idle_locked(dev_priv->device); + mutex_unlock(&dev_priv->device->mutex); + } + + if (ret == 0 && (cmd & IOC_OUT)) { + if (copy_to_user((void __user *) arg, uptr, _IOC_SIZE(cmd))) + ret = -EFAULT; + } + +done: + if (_IOC_SIZE(cmd) >= sizeof(ustack)) + kfree(uptr); + + return ret; +} + +static int +kgsl_mmap_memstore(struct kgsl_device *device, struct vm_area_struct *vma) +{ + struct kgsl_memdesc *memdesc = &device->memstore; + int result; + unsigned int vma_size = vma->vm_end - vma->vm_start; + + /* The memstore can only be mapped as read only */ + + if (vma->vm_flags & VM_WRITE) + return -EPERM; + + if (memdesc->size != vma_size) { + KGSL_MEM_ERR(device, "memstore bad size: %d should be %d\n", + vma_size, memdesc->size); + return -EINVAL; + } + + vma->vm_page_prot = pgprot_noncached(vma->vm_page_prot); + + result = remap_pfn_range(vma, vma->vm_start, vma->vm_pgoff, + vma_size, vma->vm_page_prot); + if (result != 0) + KGSL_MEM_ERR(device, "remap_pfn_range failed: %d\n", + result); + + return result; +} + +/* + * kgsl_gpumem_vm_open is called whenever a vma region is copied or split. + * Increase the refcount to make sure that the accounting stays correct + */ + +static void kgsl_gpumem_vm_open(struct vm_area_struct *vma) +{ + struct kgsl_mem_entry *entry = vma->vm_private_data; + kgsl_mem_entry_get(entry); +} + +static int +kgsl_gpumem_vm_fault(struct vm_area_struct *vma, struct vm_fault *vmf) +{ + struct kgsl_mem_entry *entry = vma->vm_private_data; + + if (!entry->memdesc.ops->vmfault) + return VM_FAULT_SIGBUS; + + return entry->memdesc.ops->vmfault(&entry->memdesc, vma, vmf); +} + +static void +kgsl_gpumem_vm_close(struct vm_area_struct *vma) +{ + struct kgsl_mem_entry *entry = vma->vm_private_data; + kgsl_mem_entry_put(entry); +} + +static struct vm_operations_struct kgsl_gpumem_vm_ops = { + .open = kgsl_gpumem_vm_open, + .fault = kgsl_gpumem_vm_fault, + .close = kgsl_gpumem_vm_close, +}; + +static int kgsl_mmap(struct file *file, struct vm_area_struct *vma) +{ + unsigned long vma_offset = vma->vm_pgoff << PAGE_SHIFT; + struct kgsl_device_private *dev_priv = file->private_data; + struct kgsl_process_private *private = dev_priv->process_priv; + struct kgsl_mem_entry *entry; + struct kgsl_device *device = dev_priv->device; + + /* Handle leagacy behavior for memstore */ + + if (vma_offset == device->memstore.physaddr) + return kgsl_mmap_memstore(device, vma); + + /* Find a chunk of GPU memory */ + + spin_lock(&private->mem_lock); + list_for_each_entry(entry, &private->mem_list, list) { + if (vma_offset == entry->memdesc.gpuaddr) { + kgsl_mem_entry_get(entry); + break; + } + } + spin_unlock(&private->mem_lock); + + if (entry == NULL) + return -EINVAL; + + if (!entry->memdesc.ops->vmflags || !entry->memdesc.ops->vmfault) + return -EINVAL; + + vma->vm_flags |= entry->memdesc.ops->vmflags(&entry->memdesc); + + vma->vm_private_data = entry; + vma->vm_page_prot = pgprot_writecombine(vma->vm_page_prot); + vma->vm_ops = &kgsl_gpumem_vm_ops; + vma->vm_file = file; + + return 0; +} + +static const struct file_operations kgsl_fops = { + .owner = THIS_MODULE, + .release = kgsl_release, + .open = kgsl_open, + .mmap = kgsl_mmap, + .unlocked_ioctl = kgsl_ioctl, +}; + +struct kgsl_driver kgsl_driver = { + .process_mutex = __MUTEX_INITIALIZER(kgsl_driver.process_mutex), + .pt_mutex = __MUTEX_INITIALIZER(kgsl_driver.pt_mutex), + .devlock = __MUTEX_INITIALIZER(kgsl_driver.devlock), +}; +EXPORT_SYMBOL(kgsl_driver); + +void kgsl_unregister_device(struct kgsl_device *device) +{ + int minor; + + mutex_lock(&kgsl_driver.devlock); + for (minor = 0; minor < KGSL_DEVICE_MAX; minor++) { + if (device == kgsl_driver.devp[minor]) + break; + } + + mutex_unlock(&kgsl_driver.devlock); + + if (minor == KGSL_DEVICE_MAX) + return; + + kgsl_cffdump_close(device->id); + kgsl_pwrctrl_uninit_sysfs(device); + + wake_lock_destroy(&device->idle_wakelock); + idr_destroy(&device->context_idr); + + if (device->memstore.hostptr) + kgsl_sharedmem_free(&device->memstore); + + kgsl_mmu_close(device); + + if (device->work_queue) { + destroy_workqueue(device->work_queue); + device->work_queue = NULL; + } + + device_destroy(kgsl_driver.class, + MKDEV(MAJOR(kgsl_driver.major), minor)); + + mutex_lock(&kgsl_driver.devlock); + kgsl_driver.devp[minor] = NULL; + mutex_unlock(&kgsl_driver.devlock); + + atomic_dec(&kgsl_driver.device_count); +} +EXPORT_SYMBOL(kgsl_unregister_device); + +int +kgsl_register_device(struct kgsl_device *device) +{ + int minor, ret; + dev_t dev; + + /* Find a minor for the device */ + + mutex_lock(&kgsl_driver.devlock); + for (minor = 0; minor < KGSL_DEVICE_MAX; minor++) { + if (kgsl_driver.devp[minor] == NULL) { + kgsl_driver.devp[minor] = device; + break; + } + } + + mutex_unlock(&kgsl_driver.devlock); + + if (minor == KGSL_DEVICE_MAX) { + KGSL_CORE_ERR("minor devices exhausted\n"); + return -ENODEV; + } + + /* Create the device */ + dev = MKDEV(MAJOR(kgsl_driver.major), minor); + device->dev = device_create(kgsl_driver.class, + device->parentdev, + dev, device, + device->name); + + if (IS_ERR(device->dev)) { + ret = PTR_ERR(device->dev); + KGSL_CORE_ERR("device_create(%s): %d\n", device->name, ret); + goto err_devlist; + } + + dev_set_drvdata(device->parentdev, device); + + /* Generic device initialization */ + atomic_inc(&kgsl_driver.device_count); + + init_waitqueue_head(&device->wait_queue); + + kgsl_cffdump_open(device->id); + + init_completion(&device->hwaccess_gate); + init_completion(&device->suspend_gate); + + ATOMIC_INIT_NOTIFIER_HEAD(&device->ts_notifier_list); + + setup_timer(&device->idle_timer, kgsl_timer, (unsigned long) device); + ret = kgsl_create_device_workqueue(device); + if (ret) + goto err_devlist; + + INIT_WORK(&device->idle_check_ws, kgsl_idle_check); + + INIT_LIST_HEAD(&device->memqueue); + + ret = kgsl_mmu_init(device); + if (ret != 0) + goto err_dest_work_q; + + ret = kgsl_allocate_contig(&device->memstore, + sizeof(struct kgsl_devmemstore)); + + if (ret != 0) + goto err_close_mmu; + + kgsl_sharedmem_set(&device->memstore, 0, 0, device->memstore.size); + + wake_lock_init(&device->idle_wakelock, WAKE_LOCK_IDLE, device->name); + idr_init(&device->context_idr); + + /* sysfs and debugfs initalization - failure here is non fatal */ + + /* Initialize logging */ + kgsl_device_debugfs_init(device); + + /* Initialize common sysfs entries */ + kgsl_pwrctrl_init_sysfs(device); + + return 0; + +err_close_mmu: + kgsl_mmu_close(device); +err_dest_work_q: + destroy_workqueue(device->work_queue); + device->work_queue = NULL; +err_devlist: + mutex_lock(&kgsl_driver.devlock); + kgsl_driver.devp[minor] = NULL; + mutex_unlock(&kgsl_driver.devlock); + + return ret; +} +EXPORT_SYMBOL(kgsl_register_device); + +int kgsl_device_platform_probe(struct kgsl_device *device, + irqreturn_t (*dev_isr) (int, void*)) +{ + int status = -EINVAL; + struct kgsl_memregion *regspace = NULL; + struct resource *res; + struct platform_device *pdev = + container_of(device->parentdev, struct platform_device, dev); + + pm_runtime_enable(device->parentdev); + + status = kgsl_pwrctrl_init(device); + if (status) + goto error; + + res = platform_get_resource_byname(pdev, IORESOURCE_MEM, + device->iomemname); + if (res == NULL) { + KGSL_DRV_ERR(device, "platform_get_resource_byname failed\n"); + status = -EINVAL; + goto error_pwrctrl_close; + } + if (res->start == 0 || resource_size(res) == 0) { + KGSL_DRV_ERR(device, "dev %d invalid regspace\n", device->id); + status = -EINVAL; + goto error_pwrctrl_close; + } + + regspace = &device->regspace; + regspace->mmio_phys_base = res->start; + regspace->sizebytes = resource_size(res); + + if (!request_mem_region(regspace->mmio_phys_base, + regspace->sizebytes, device->name)) { + KGSL_DRV_ERR(device, "request_mem_region failed\n"); + status = -ENODEV; + goto error_pwrctrl_close; + } + + regspace->mmio_virt_base = ioremap(regspace->mmio_phys_base, + regspace->sizebytes); + + if (regspace->mmio_virt_base == NULL) { + KGSL_DRV_ERR(device, "ioremap failed\n"); + status = -ENODEV; + goto error_release_mem; + } + + status = request_irq(device->pwrctrl.interrupt_num, dev_isr, + IRQF_TRIGGER_HIGH, device->name, device); + if (status) { + KGSL_DRV_ERR(device, "request_irq(%d) failed: %d\n", + device->pwrctrl.interrupt_num, status); + goto error_iounmap; + } + device->pwrctrl.have_irq = 1; + disable_irq(device->pwrctrl.interrupt_num); + + KGSL_DRV_INFO(device, + "dev_id %d regs phys 0x%08x size 0x%08x virt %p\n", + device->id, regspace->mmio_phys_base, + regspace->sizebytes, regspace->mmio_virt_base); + + + status = kgsl_register_device(device); + if (!status) + return status; + + free_irq(device->pwrctrl.interrupt_num, NULL); + device->pwrctrl.have_irq = 0; +error_iounmap: + iounmap(regspace->mmio_virt_base); + regspace->mmio_virt_base = NULL; +error_release_mem: + release_mem_region(regspace->mmio_phys_base, regspace->sizebytes); +error_pwrctrl_close: + kgsl_pwrctrl_close(device); +error: + return status; +} +EXPORT_SYMBOL(kgsl_device_platform_probe); + +void kgsl_device_platform_remove(struct kgsl_device *device) +{ + struct kgsl_memregion *regspace = &device->regspace; + + kgsl_unregister_device(device); + + if (regspace->mmio_virt_base != NULL) { + iounmap(regspace->mmio_virt_base); + regspace->mmio_virt_base = NULL; + release_mem_region(regspace->mmio_phys_base, + regspace->sizebytes); + } + kgsl_pwrctrl_close(device); + + pm_runtime_disable(device->parentdev); +} +EXPORT_SYMBOL(kgsl_device_platform_remove); + +static int __devinit +kgsl_ptdata_init(void) +{ + INIT_LIST_HEAD(&kgsl_driver.pagetable_list); + + return kgsl_ptpool_init(&kgsl_driver.ptpool, KGSL_PAGETABLE_SIZE, + kgsl_pagetable_count); +} + +static void kgsl_core_exit(void) +{ + unregister_chrdev_region(kgsl_driver.major, KGSL_DEVICE_MAX); + + kgsl_ptpool_destroy(&kgsl_driver.ptpool); + + device_unregister(&kgsl_driver.virtdev); + + if (kgsl_driver.class) { + class_destroy(kgsl_driver.class); + kgsl_driver.class = NULL; + } + + kgsl_drm_exit(); + kgsl_cffdump_destroy(); + kgsl_core_debugfs_close(); + kgsl_sharedmem_uninit_sysfs(); +} + +static int __init kgsl_core_init(void) +{ + int result = 0; + + /* alloc major and minor device numbers */ + result = alloc_chrdev_region(&kgsl_driver.major, 0, KGSL_DEVICE_MAX, + KGSL_NAME); + if (result < 0) { + KGSL_CORE_ERR("alloc_chrdev_region failed err = %d\n", result); + goto err; + } + + cdev_init(&kgsl_driver.cdev, &kgsl_fops); + kgsl_driver.cdev.owner = THIS_MODULE; + kgsl_driver.cdev.ops = &kgsl_fops; + result = cdev_add(&kgsl_driver.cdev, MKDEV(MAJOR(kgsl_driver.major), 0), + KGSL_DEVICE_MAX); + + if (result) { + KGSL_CORE_ERR("kgsl: cdev_add() failed, dev_num= %d," + " result= %d\n", kgsl_driver.major, result); + goto err; + } + + kgsl_driver.class = class_create(THIS_MODULE, KGSL_NAME); + + if (IS_ERR(kgsl_driver.class)) { + result = PTR_ERR(kgsl_driver.class); + KGSL_CORE_ERR("failed to create class %s", KGSL_NAME); + goto err; + } + + /* Make a virtual device for managing core related things + in sysfs */ + kgsl_driver.virtdev.class = kgsl_driver.class; + dev_set_name(&kgsl_driver.virtdev, "kgsl"); + result = device_register(&kgsl_driver.virtdev); + if (result) { + KGSL_CORE_ERR("driver_register failed\n"); + goto err; + } + + /* Make kobjects in the virtual device for storing statistics */ + + kgsl_driver.ptkobj = + kobject_create_and_add("pagetables", + &kgsl_driver.virtdev.kobj); + + kgsl_driver.prockobj = + kobject_create_and_add("proc", + &kgsl_driver.virtdev.kobj); + + kgsl_core_debugfs_init(); + + kgsl_sharedmem_init_sysfs(); + kgsl_cffdump_init(); + + /* Generic device initialization */ + atomic_set(&kgsl_driver.device_count, -1); + + INIT_LIST_HEAD(&kgsl_driver.process_list); + + result = kgsl_ptdata_init(); + if (result) + goto err; + + result = kgsl_drm_init(NULL); + + if (result) + goto err; + + return 0; + +err: + kgsl_core_exit(); + return result; +} + +module_init(kgsl_core_init); +module_exit(kgsl_core_exit); + +MODULE_AUTHOR("Qualcomm Innovation Center, Inc."); +MODULE_DESCRIPTION("MSM GPU driver"); +MODULE_LICENSE("GPL"); diff --git a/drivers/gpu/msm/kgsl.h b/drivers/gpu/msm/kgsl.h new file mode 100644 index 00000000..d107a0b0 --- /dev/null +++ b/drivers/gpu/msm/kgsl.h @@ -0,0 +1,290 @@ +/* Copyright (c) 2008-2011, Code Aurora Forum. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are + * met: + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following + * disclaimer in the documentation and/or other materials provided + * with the distribution. + * * Neither the name of Code Aurora Forum, Inc. nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED "AS IS" AND ANY EXPRESS OR IMPLIED + * WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS + * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR + * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, + * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE + * OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN + * IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + */ +#ifndef __KGSL_H +#define __KGSL_H + +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +#include "kgsl_device.h" +#include "kgsl_pwrctrl.h" +#include "kgsl_sharedmem.h" +#include "kgsl_log.h" +#include "kgsl_cffdump.h" + +#define KGSL_NAME "kgsl" + +#define CHIP_REV_251 0x020501 + +/* Flags to control whether to flush or invalidate a cached memory range */ +#define KGSL_CACHE_INV 0x00000000 +#define KGSL_CACHE_CLEAN 0x00000001 +#define KGSL_CACHE_FLUSH 0x00000002 + +#define KGSL_CACHE_USER_ADDR 0x00000010 +#define KGSL_CACHE_VMALLOC_ADDR 0x00000020 + +/*cache coherency ops */ +#define DRM_KGSL_GEM_CACHE_OP_TO_DEV 0x0001 +#define DRM_KGSL_GEM_CACHE_OP_FROM_DEV 0x0002 + +/* The size of each entry in a page table */ +#define KGSL_PAGETABLE_ENTRY_SIZE 4 + +/* Pagetable Virtual Address base */ +#define KGSL_PAGETABLE_BASE 0x66000000 + +/* Extra accounting entries needed in the pagetable */ +#define KGSL_PT_EXTRA_ENTRIES 16 + +#define KGSL_PAGETABLE_ENTRIES(_sz) (((_sz) >> PAGE_SHIFT) + \ + KGSL_PT_EXTRA_ENTRIES) + +#ifdef CONFIG_MSM_KGSL_MMU +#define KGSL_PAGETABLE_SIZE \ +ALIGN(KGSL_PAGETABLE_ENTRIES(CONFIG_MSM_KGSL_PAGE_TABLE_SIZE) * \ +KGSL_PAGETABLE_ENTRY_SIZE, PAGE_SIZE) +#else +#define KGSL_PAGETABLE_SIZE 0 +#endif + +#ifdef CONFIG_KGSL_PER_PROCESS_PAGE_TABLE +#define KGSL_PAGETABLE_COUNT (CONFIG_MSM_KGSL_PAGE_TABLE_COUNT) +#else +#define KGSL_PAGETABLE_COUNT 1 +#endif + +/* Casting using container_of() for structures that kgsl owns. */ +#define KGSL_CONTAINER_OF(ptr, type, member) \ + container_of(ptr, type, member) + +/* A macro for memory statistics - add the new size to the stat and if + the statisic is greater then _max, set _max +*/ + +#define KGSL_STATS_ADD(_size, _stat, _max) \ + do { _stat += (_size); if (_stat > _max) _max = _stat; } while (0) + +struct kgsl_driver { + struct cdev cdev; + dev_t major; + struct class *class; + /* Virtual device for managing the core */ + struct device virtdev; + /* Kobjects for storing pagetable and process statistics */ + struct kobject *ptkobj; + struct kobject *prockobj; + atomic_t device_count; + struct kgsl_device *devp[KGSL_DEVICE_MAX]; + + uint32_t flags_debug; + + /* Global lilst of open processes */ + struct list_head process_list; + /* Global list of pagetables */ + struct list_head pagetable_list; + /* Mutex for accessing the pagetable list */ + struct mutex pt_mutex; + /* Mutex for accessing the process list */ + struct mutex process_mutex; + + /* Mutex for protecting the device list */ + struct mutex devlock; + + struct kgsl_ptpool ptpool; + + struct { + unsigned int vmalloc; + unsigned int vmalloc_max; + unsigned int coherent; + unsigned int coherent_max; + unsigned int mapped; + unsigned int mapped_max; + unsigned int histogram[16]; + } stats; +}; + +extern struct kgsl_driver kgsl_driver; + +#define KGSL_USER_MEMORY 1 +#define KGSL_MAPPED_MEMORY 2 + +struct kgsl_mem_entry { + struct kref refcount; + struct kgsl_memdesc memdesc; + int memtype; + struct file *file_ptr; + struct list_head list; + uint32_t free_timestamp; + /* back pointer to private structure under whose context this + * allocation is made */ + struct kgsl_process_private *priv; +}; + +#ifdef CONFIG_MSM_KGSL_MMU_PAGE_FAULT +#define MMU_CONFIG 2 +#else +#define MMU_CONFIG 1 +#endif + +void kgsl_mem_entry_destroy(struct kref *kref); +uint8_t *kgsl_gpuaddr_to_vaddr(const struct kgsl_memdesc *memdesc, + unsigned int gpuaddr, unsigned int *size); +struct kgsl_mem_entry *kgsl_sharedmem_find_region( + struct kgsl_process_private *private, unsigned int gpuaddr, + size_t size); +int kgsl_idle(struct kgsl_device *device, unsigned int timeout); +int kgsl_setstate(struct kgsl_device *device, uint32_t flags); + +static inline void kgsl_regread(struct kgsl_device *device, + unsigned int offsetwords, + unsigned int *value) +{ + device->ftbl.device_regread(device, offsetwords, value); +} + +static inline void kgsl_regwrite(struct kgsl_device *device, + unsigned int offsetwords, + unsigned int value) +{ + device->ftbl.device_regwrite(device, offsetwords, value); +} + +static inline void kgsl_regread_isr(struct kgsl_device *device, + unsigned int offsetwords, + unsigned int *value) +{ + device->ftbl.device_regread_isr(device, offsetwords, value); +} + +static inline void kgsl_regwrite_isr(struct kgsl_device *device, + unsigned int offsetwords, + unsigned int value) +{ + device->ftbl.device_regwrite_isr(device, offsetwords, value); +} + +int kgsl_check_timestamp(struct kgsl_device *device, unsigned int timestamp); + +int kgsl_register_ts_notifier(struct kgsl_device *device, + struct notifier_block *nb); + +int kgsl_unregister_ts_notifier(struct kgsl_device *device, + struct notifier_block *nb); + +int kgsl_device_platform_probe(struct kgsl_device *device, + irqreturn_t (*dev_isr) (int, void*)); +void kgsl_device_platform_remove(struct kgsl_device *device); + +extern const struct dev_pm_ops kgsl_pm_ops; + +int kgsl_suspend_driver(struct platform_device *pdev, pm_message_t state); +int kgsl_resume_driver(struct platform_device *pdev); + +#ifdef CONFIG_MSM_KGSL_DRM +extern int kgsl_drm_init(struct platform_device *dev); +extern void kgsl_drm_exit(void); +extern void kgsl_gpu_mem_flush(int op); +#else +static inline int kgsl_drm_init(struct platform_device *dev) +{ + return 0; +} + +static inline void kgsl_drm_exit(void) +{ +} +#endif + +static inline int kgsl_gpuaddr_in_memdesc(const struct kgsl_memdesc *memdesc, + unsigned int gpuaddr) +{ + if (gpuaddr >= memdesc->gpuaddr && (gpuaddr + sizeof(unsigned int)) <= + (memdesc->gpuaddr + memdesc->size)) { + return 1; + } + return 0; +} + +static inline struct kgsl_device *kgsl_device_from_dev(struct device *dev) +{ + int i; + + for (i = 0; i < KGSL_DEVICE_MAX; i++) { + if (kgsl_driver.devp[i] && kgsl_driver.devp[i]->dev == dev) + return kgsl_driver.devp[i]; + } + + return NULL; +} + +static inline bool timestamp_cmp(unsigned int new, unsigned int old) +{ + int ts_diff = new - old; + return (ts_diff >= 0) || (ts_diff < -20000); +} + +static inline void +kgsl_mem_entry_get(struct kgsl_mem_entry *entry) +{ + kref_get(&entry->refcount); +} + +static inline void +kgsl_mem_entry_put(struct kgsl_mem_entry *entry) +{ + kref_put(&entry->refcount, kgsl_mem_entry_destroy); +} + +static inline int kgsl_create_device_sysfs_files(struct device *root, + struct device_attribute **list) +{ + int ret = 0, i; + for (i = 0; list[i] != NULL; i++) + ret |= device_create_file(root, list[i]); + return ret; +} + +static inline void kgsl_remove_device_sysfs_files(struct device *root, + struct device_attribute **list) +{ + int i; + for (i = 0; list[i] != NULL; i++) + device_remove_file(root, list[i]); +} + +#endif /* __KGSL_H */ diff --git a/drivers/gpu/msm/kgsl_cffdump.c b/drivers/gpu/msm/kgsl_cffdump.c new file mode 100644 index 00000000..702e1ac7 --- /dev/null +++ b/drivers/gpu/msm/kgsl_cffdump.c @@ -0,0 +1,798 @@ +/* Copyright (c) 2010-2011, Code Aurora Forum. All rights reserved. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 and + * only version 2 as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + */ + +/* #define DEBUG */ +#define ALIGN_CPU + +#include +#include +#include +#include +#include +#include + +#include "kgsl.h" +#include "kgsl_cffdump.h" +#include "kgsl_debugfs.h" +#include "kgsl_log.h" +#include "kgsl_sharedmem.h" +#include "adreno_pm4types.h" + +static struct rchan *chan; +static struct dentry *dir; +static int suspended; +static size_t dropped; +static size_t subbuf_size = 256*1024; +static size_t n_subbufs = 64; + +/* forward declarations */ +static void destroy_channel(void); +static struct rchan *create_channel(unsigned subbuf_size, unsigned n_subbufs); + +static spinlock_t cffdump_lock; +static ulong serial_nr; +static ulong total_bytes; +static ulong total_syncmem; +static long last_sec; + +#define MEMBUF_SIZE 64 + +#define CFF_OP_WRITE_REG 0x00000002 +struct cff_op_write_reg { + unsigned char op; + uint addr; + uint value; +} __packed; + +#define CFF_OP_POLL_REG 0x00000004 +struct cff_op_poll_reg { + unsigned char op; + uint addr; + uint value; + uint mask; +} __packed; + +#define CFF_OP_WAIT_IRQ 0x00000005 +struct cff_op_wait_irq { + unsigned char op; +} __packed; + +#define CFF_OP_RMW 0x0000000a + +#define CFF_OP_WRITE_MEM 0x0000000b +struct cff_op_write_mem { + unsigned char op; + uint addr; + uint value; +} __packed; + +#define CFF_OP_WRITE_MEMBUF 0x0000000c +struct cff_op_write_membuf { + unsigned char op; + uint addr; + ushort count; + uint buffer[MEMBUF_SIZE]; +} __packed; + +#define CFF_OP_MEMORY_BASE 0x0000000d +struct cff_op_memory_base { + unsigned char op; + uint base; + uint size; + uint gmemsize; +} __packed; + +#define CFF_OP_HANG 0x0000000e +struct cff_op_hang { + unsigned char op; +} __packed; + +#define CFF_OP_EOF 0xffffffff +struct cff_op_eof { + unsigned char op; +} __packed; + +#define CFF_OP_VERIFY_MEM_FILE 0x00000007 +#define CFF_OP_WRITE_SURFACE_PARAMS 0x00000011 +struct cff_op_user_event { + unsigned char op; + unsigned int op1; + unsigned int op2; + unsigned int op3; + unsigned int op4; + unsigned int op5; +} __packed; + + +static void b64_encodeblock(unsigned char in[3], unsigned char out[4], int len) +{ + static const char tob64[] = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmno" + "pqrstuvwxyz0123456789+/"; + + out[0] = tob64[in[0] >> 2]; + out[1] = tob64[((in[0] & 0x03) << 4) | ((in[1] & 0xf0) >> 4)]; + out[2] = (unsigned char) (len > 1 ? tob64[((in[1] & 0x0f) << 2) + | ((in[2] & 0xc0) >> 6)] : '='); + out[3] = (unsigned char) (len > 2 ? tob64[in[2] & 0x3f] : '='); +} + +static void b64_encode(const unsigned char *in_buf, int in_size, + unsigned char *out_buf, int out_bufsize, int *out_size) +{ + unsigned char in[3], out[4]; + int i, len; + + *out_size = 0; + while (in_size > 0) { + len = 0; + for (i = 0; i < 3; ++i) { + if (in_size-- > 0) { + in[i] = *in_buf++; + ++len; + } else + in[i] = 0; + } + if (len) { + b64_encodeblock(in, out, len); + if (out_bufsize < 4) { + pr_warn("kgsl: cffdump: %s: out of buffer\n", + __func__); + return; + } + for (i = 0; i < 4; ++i) + *out_buf++ = out[i]; + *out_size += 4; + out_bufsize -= 4; + } + } +} + +#define KLOG_TMPBUF_SIZE (1024) +static void klog_printk(const char *fmt, ...) +{ + /* per-cpu klog formatting temporary buffer */ + static char klog_buf[NR_CPUS][KLOG_TMPBUF_SIZE]; + + va_list args; + int len; + char *cbuf; + unsigned long flags; + + local_irq_save(flags); + cbuf = klog_buf[smp_processor_id()]; + va_start(args, fmt); + len = vsnprintf(cbuf, KLOG_TMPBUF_SIZE, fmt, args); + total_bytes += len; + va_end(args); + relay_write(chan, cbuf, len); + local_irq_restore(flags); +} + +static struct cff_op_write_membuf cff_op_write_membuf; +static void cffdump_membuf(int id, unsigned char *out_buf, int out_bufsize) +{ + void *data; + int len, out_size; + struct cff_op_write_mem cff_op_write_mem; + + uint addr = cff_op_write_membuf.addr + - sizeof(uint)*cff_op_write_membuf.count; + + if (!cff_op_write_membuf.count) { + pr_warn("kgsl: cffdump: membuf: count == 0, skipping"); + return; + } + + if (cff_op_write_membuf.count != 1) { + cff_op_write_membuf.op = CFF_OP_WRITE_MEMBUF; + cff_op_write_membuf.addr = addr; + len = sizeof(cff_op_write_membuf) - + sizeof(uint)*(MEMBUF_SIZE - cff_op_write_membuf.count); + data = &cff_op_write_membuf; + } else { + cff_op_write_mem.op = CFF_OP_WRITE_MEM; + cff_op_write_mem.addr = addr; + cff_op_write_mem.value = cff_op_write_membuf.buffer[0]; + data = &cff_op_write_mem; + len = sizeof(cff_op_write_mem); + } + b64_encode(data, len, out_buf, out_bufsize, &out_size); + out_buf[out_size] = 0; + klog_printk("%ld:%d;%s\n", ++serial_nr, id, out_buf); + cff_op_write_membuf.count = 0; + cff_op_write_membuf.addr = 0; +} + +static void cffdump_printline(int id, uint opcode, uint op1, uint op2, + uint op3, uint op4, uint op5) +{ + struct cff_op_write_reg cff_op_write_reg; + struct cff_op_poll_reg cff_op_poll_reg; + struct cff_op_wait_irq cff_op_wait_irq; + struct cff_op_memory_base cff_op_memory_base; + struct cff_op_hang cff_op_hang; + struct cff_op_eof cff_op_eof; + struct cff_op_user_event cff_op_user_event; + unsigned char out_buf[sizeof(cff_op_write_membuf)/3*4 + 16]; + void *data; + int len = 0, out_size; + long cur_secs; + + spin_lock(&cffdump_lock); + if (opcode == CFF_OP_WRITE_MEM) { + if (op1 < 0x40000000 || op1 >= 0x60000000) + KGSL_CORE_ERR("addr out-of-range: op1=%08x", op1); + if ((cff_op_write_membuf.addr != op1 && + cff_op_write_membuf.count) + || (cff_op_write_membuf.count == MEMBUF_SIZE)) + cffdump_membuf(id, out_buf, sizeof(out_buf)); + + cff_op_write_membuf.buffer[cff_op_write_membuf.count++] = op2; + cff_op_write_membuf.addr = op1 + sizeof(uint); + spin_unlock(&cffdump_lock); + return; + } else if (cff_op_write_membuf.count) + cffdump_membuf(id, out_buf, sizeof(out_buf)); + spin_unlock(&cffdump_lock); + + switch (opcode) { + case CFF_OP_WRITE_REG: + cff_op_write_reg.op = opcode; + cff_op_write_reg.addr = op1; + cff_op_write_reg.value = op2; + data = &cff_op_write_reg; + len = sizeof(cff_op_write_reg); + break; + + case CFF_OP_POLL_REG: + cff_op_poll_reg.op = opcode; + cff_op_poll_reg.addr = op1; + cff_op_poll_reg.value = op2; + cff_op_poll_reg.mask = op3; + data = &cff_op_poll_reg; + len = sizeof(cff_op_poll_reg); + break; + + case CFF_OP_WAIT_IRQ: + cff_op_wait_irq.op = opcode; + data = &cff_op_wait_irq; + len = sizeof(cff_op_wait_irq); + break; + + case CFF_OP_MEMORY_BASE: + cff_op_memory_base.op = opcode; + cff_op_memory_base.base = op1; + cff_op_memory_base.size = op2; + cff_op_memory_base.gmemsize = op3; + data = &cff_op_memory_base; + len = sizeof(cff_op_memory_base); + break; + + case CFF_OP_HANG: + cff_op_hang.op = opcode; + data = &cff_op_hang; + len = sizeof(cff_op_hang); + break; + + case CFF_OP_EOF: + cff_op_eof.op = opcode; + data = &cff_op_eof; + len = sizeof(cff_op_eof); + break; + + case CFF_OP_WRITE_SURFACE_PARAMS: + case CFF_OP_VERIFY_MEM_FILE: + cff_op_user_event.op = opcode; + cff_op_user_event.op1 = op1; + cff_op_user_event.op2 = op2; + cff_op_user_event.op3 = op3; + cff_op_user_event.op4 = op4; + cff_op_user_event.op5 = op5; + data = &cff_op_user_event; + len = sizeof(cff_op_user_event); + break; + } + + if (len) { + b64_encode(data, len, out_buf, sizeof(out_buf), &out_size); + out_buf[out_size] = 0; + klog_printk("%ld:%d;%s\n", ++serial_nr, id, out_buf); + } else + pr_warn("kgsl: cffdump: unhandled opcode: %d\n", opcode); + + cur_secs = get_seconds(); + if ((cur_secs - last_sec) > 10 || (last_sec - cur_secs) > 10) { + pr_info("kgsl: cffdump: total [bytes:%lu kB, syncmem:%lu kB], " + "seq#: %lu\n", total_bytes/1024, total_syncmem/1024, + serial_nr); + last_sec = cur_secs; + } +} + +void kgsl_cffdump_init() +{ + struct dentry *debugfs_dir = kgsl_get_debugfs_dir(); + +#ifdef ALIGN_CPU + cpumask_t mask; + + cpumask_clear(&mask); + cpumask_set_cpu(0, &mask); + sched_setaffinity(0, &mask); +#endif + if (!debugfs_dir || IS_ERR(debugfs_dir)) { + KGSL_CORE_ERR("Debugfs directory is bad\n"); + return; + } + + kgsl_cff_dump_enable = 1; + + spin_lock_init(&cffdump_lock); + + dir = debugfs_create_dir("cff", debugfs_dir); + if (!dir) { + KGSL_CORE_ERR("debugfs_create_dir failed\n"); + return; + } + + chan = create_channel(subbuf_size, n_subbufs); +} + +void kgsl_cffdump_destroy() +{ + if (chan) + relay_flush(chan); + destroy_channel(); + if (dir) + debugfs_remove(dir); +} + +void kgsl_cffdump_open(enum kgsl_deviceid device_id) +{ + /*TODO: move this to where we can report correct gmemsize*/ + unsigned int va_base; + + /* XXX: drewis edit: only for 8x50 */ + va_base = 0x20000000; + + kgsl_cffdump_memory_base(device_id, va_base, + CONFIG_MSM_KGSL_PAGE_TABLE_SIZE, SZ_256K); +} + +void kgsl_cffdump_memory_base(enum kgsl_deviceid device_id, unsigned int base, + unsigned int range, unsigned gmemsize) +{ + cffdump_printline(device_id, CFF_OP_MEMORY_BASE, base, + range, gmemsize, 0, 0); +} + +void kgsl_cffdump_hang(enum kgsl_deviceid device_id) +{ + cffdump_printline(device_id, CFF_OP_HANG, 0, 0, 0, 0, 0); +} + +void kgsl_cffdump_close(enum kgsl_deviceid device_id) +{ + cffdump_printline(device_id, CFF_OP_EOF, 0, 0, 0, 0, 0); +} + +void kgsl_cffdump_user_event(unsigned int cff_opcode, unsigned int op1, + unsigned int op2, unsigned int op3, + unsigned int op4, unsigned int op5) +{ + cffdump_printline(-1, cff_opcode, op1, op2, op3, op4, op5); +} + +void kgsl_cffdump_syncmem(struct kgsl_device_private *dev_priv, + const struct kgsl_memdesc *memdesc, uint gpuaddr, uint sizebytes, + bool clean_cache) +{ + const void *src; + uint host_size; + uint physaddr; + + if (!kgsl_cff_dump_enable) + return; + + total_syncmem += sizebytes; + + if (memdesc == NULL) { + struct kgsl_mem_entry *entry; + spin_lock(&dev_priv->process_priv->mem_lock); + entry = kgsl_sharedmem_find_region(dev_priv->process_priv, + gpuaddr, sizebytes); + spin_unlock(&dev_priv->process_priv->mem_lock); + if (entry == NULL) { + KGSL_CORE_ERR("did not find mapping " + "for gpuaddr: 0x%08x\n", gpuaddr); + return; + } + memdesc = &entry->memdesc; + } + BUG_ON(memdesc->gpuaddr == 0); + BUG_ON(gpuaddr == 0); + physaddr = kgsl_get_realaddr(memdesc) + (gpuaddr - memdesc->gpuaddr); + + src = kgsl_gpuaddr_to_vaddr(memdesc, gpuaddr, &host_size); + if (src == NULL || host_size < sizebytes) { + KGSL_CORE_ERR("did not find mapping for " + "gpuaddr: 0x%08x, m->host: 0x%p, phys: 0x%08x\n", + gpuaddr, memdesc->hostptr, memdesc->physaddr); + return; + } + + if (clean_cache) { + /* Ensure that this memory region is not read from the + * cache but fetched fresh */ + + mb(); + + kgsl_cache_range_op((struct kgsl_memdesc *)memdesc, + KGSL_CACHE_OP_INV); + } + + BUG_ON(physaddr > 0x66000000 && physaddr < 0x66ffffff); + while (sizebytes > 3) { + cffdump_printline(-1, CFF_OP_WRITE_MEM, gpuaddr, *(uint *)src, + 0, 0, 0); + gpuaddr += 4; + src += 4; + sizebytes -= 4; + } + if (sizebytes > 0) + cffdump_printline(-1, CFF_OP_WRITE_MEM, gpuaddr, *(uint *)src, + 0, 0, 0); +} + +void kgsl_cffdump_setmem(uint addr, uint value, uint sizebytes) +{ + if (!kgsl_cff_dump_enable) + return; + + BUG_ON(addr > 0x66000000 && addr < 0x66ffffff); + while (sizebytes > 3) { + /* Use 32bit memory writes as long as there's at least + * 4 bytes left */ + cffdump_printline(-1, CFF_OP_WRITE_MEM, addr, value, + 0, 0, 0); + addr += 4; + sizebytes -= 4; + } + if (sizebytes > 0) + cffdump_printline(-1, CFF_OP_WRITE_MEM, addr, value, + 0, 0, 0); +} + +void kgsl_cffdump_regwrite(enum kgsl_deviceid device_id, uint addr, + uint value) +{ + if (!kgsl_cff_dump_enable) + return; + + cffdump_printline(device_id, CFF_OP_WRITE_REG, addr, value, + 0, 0, 0); +} + +void kgsl_cffdump_regpoll(enum kgsl_deviceid device_id, uint addr, + uint value, uint mask) +{ + if (!kgsl_cff_dump_enable) + return; + + cffdump_printline(device_id, CFF_OP_POLL_REG, addr, value, + mask, 0, 0); +} + +void kgsl_cffdump_slavewrite(uint addr, uint value) +{ + if (!kgsl_cff_dump_enable) + return; + + cffdump_printline(-1, CFF_OP_WRITE_REG, addr, value, 0, 0, 0); +} + +int kgsl_cffdump_waitirq(void) +{ + if (!kgsl_cff_dump_enable) + return 0; + + cffdump_printline(-1, CFF_OP_WAIT_IRQ, 0, 0, 0, 0, 0); + + return 1; +} +EXPORT_SYMBOL(kgsl_cffdump_waitirq); + +#define ADDRESS_STACK_SIZE 256 +#define GET_PM4_TYPE3_OPCODE(x) ((*(x) >> 8) & 0xFF) +static unsigned int kgsl_cffdump_addr_count; + +static bool kgsl_cffdump_handle_type3(struct kgsl_device_private *dev_priv, + uint *hostaddr, bool check_only) +{ + static uint addr_stack[ADDRESS_STACK_SIZE]; + static uint size_stack[ADDRESS_STACK_SIZE]; + + switch (GET_PM4_TYPE3_OPCODE(hostaddr)) { + case PM4_INDIRECT_BUFFER_PFD: + case PM4_INDIRECT_BUFFER: + { + /* traverse indirect buffers */ + int i; + uint ibaddr = hostaddr[1]; + uint ibsize = hostaddr[2]; + + /* is this address already in encountered? */ + for (i = 0; + i < kgsl_cffdump_addr_count && addr_stack[i] != ibaddr; + ++i) + ; + + if (kgsl_cffdump_addr_count == i) { + addr_stack[kgsl_cffdump_addr_count] = ibaddr; + size_stack[kgsl_cffdump_addr_count++] = ibsize; + + if (kgsl_cffdump_addr_count >= ADDRESS_STACK_SIZE) { + KGSL_CORE_ERR("stack overflow\n"); + return false; + } + + return kgsl_cffdump_parse_ibs(dev_priv, NULL, + ibaddr, ibsize, check_only); + } else if (size_stack[i] != ibsize) { + KGSL_CORE_ERR("gpuaddr: 0x%08x, " + "wc: %u, with size wc: %u already on the " + "stack\n", ibaddr, ibsize, size_stack[i]); + return false; + } + } + break; + } + + return true; +} + +/* + * Traverse IBs and dump them to test vector. Detect swap by inspecting + * register writes, keeping note of the current state, and dump + * framebuffer config to test vector + */ +bool kgsl_cffdump_parse_ibs(struct kgsl_device_private *dev_priv, + const struct kgsl_memdesc *memdesc, uint gpuaddr, int sizedwords, + bool check_only) +{ + static uint level; /* recursion level */ + bool ret = true; + uint host_size; + uint *hostaddr, *hoststart; + int dwords_left = sizedwords; /* dwords left in the current command + buffer */ + + if (level == 0) + kgsl_cffdump_addr_count = 0; + + if (memdesc == NULL) { + struct kgsl_mem_entry *entry; + spin_lock(&dev_priv->process_priv->mem_lock); + entry = kgsl_sharedmem_find_region(dev_priv->process_priv, + gpuaddr, sizedwords * sizeof(uint)); + spin_unlock(&dev_priv->process_priv->mem_lock); + if (entry == NULL) { + KGSL_CORE_ERR("did not find mapping " + "for gpuaddr: 0x%08x\n", gpuaddr); + return true; + } + memdesc = &entry->memdesc; + } + + hostaddr = (uint *)kgsl_gpuaddr_to_vaddr(memdesc, gpuaddr, &host_size); + if (hostaddr == NULL) { + KGSL_CORE_ERR("did not find mapping for " + "gpuaddr: 0x%08x\n", gpuaddr); + return true; + } + + hoststart = hostaddr; + + level++; + + if (!memdesc->physaddr) { + KGSL_CORE_ERR("no physaddr"); + return true; + } else { + mb(); + kgsl_cache_range_op((struct kgsl_memdesc *)memdesc, + KGSL_CACHE_OP_INV); + } + +#ifdef DEBUG + pr_info("kgsl: cffdump: ib: gpuaddr:0x%08x, wc:%d, hptr:%p\n", + gpuaddr, sizedwords, hostaddr); +#endif + + while (dwords_left > 0) { + int count = 0; /* dword count including packet header */ + bool cur_ret = true; + + switch (*hostaddr >> 30) { + case 0x0: /* type-0 */ + count = (*hostaddr >> 16)+2; + break; + case 0x1: /* type-1 */ + count = 2; + break; + case 0x3: /* type-3 */ + count = ((*hostaddr >> 16) & 0x3fff) + 2; + cur_ret = kgsl_cffdump_handle_type3(dev_priv, + hostaddr, check_only); + break; + default: + pr_warn("kgsl: cffdump: parse-ib: unexpected type: " + "type:%d, word:0x%08x @ 0x%p, gpu:0x%08x\n", + *hostaddr >> 30, *hostaddr, hostaddr, + gpuaddr+4*(sizedwords-dwords_left)); + cur_ret = false; + count = dwords_left; + break; + } + +#ifdef DEBUG + if (!cur_ret) { + pr_info("kgsl: cffdump: bad sub-type: #:%d/%d, v:0x%08x" + " @ 0x%p[gb:0x%08x], level:%d\n", + sizedwords-dwords_left, sizedwords, *hostaddr, + hostaddr, gpuaddr+4*(sizedwords-dwords_left), + level); + + print_hex_dump(KERN_ERR, level == 1 ? "IB1:" : "IB2:", + DUMP_PREFIX_OFFSET, 32, 4, hoststart, + sizedwords*4, 0); + } +#endif + ret = ret && cur_ret; + + /* jump to next packet */ + dwords_left -= count; + hostaddr += count; + cur_ret = dwords_left >= 0; + +#ifdef DEBUG + if (!cur_ret) { + pr_info("kgsl: cffdump: bad count: c:%d, #:%d/%d, " + "v:0x%08x @ 0x%p[gb:0x%08x], level:%d\n", + count, sizedwords-(dwords_left+count), + sizedwords, *(hostaddr-count), hostaddr-count, + gpuaddr+4*(sizedwords-(dwords_left+count)), + level); + + print_hex_dump(KERN_ERR, level == 1 ? "IB1:" : "IB2:", + DUMP_PREFIX_OFFSET, 32, 4, hoststart, + sizedwords*4, 0); + } +#endif + + ret = ret && cur_ret; + } + + if (!ret) + pr_info("kgsl: cffdump: parsing failed: gpuaddr:0x%08x, " + "host:0x%p, wc:%d\n", gpuaddr, hoststart, sizedwords); + + if (!check_only) { +#ifdef DEBUG + uint offset = gpuaddr - memdesc->gpuaddr; + pr_info("kgsl: cffdump: ib-dump: hostptr:%p, gpuaddr:%08x, " + "physaddr:%08x, offset:%d, size:%d", hoststart, + gpuaddr, memdesc->physaddr + offset, offset, + sizedwords*4); +#endif + kgsl_cffdump_syncmem(dev_priv, memdesc, gpuaddr, sizedwords*4, + false); + } + + level--; + + return ret; +} + +static int subbuf_start_handler(struct rchan_buf *buf, + void *subbuf, void *prev_subbuf, uint prev_padding) +{ + pr_debug("kgsl: cffdump: subbuf_start_handler(subbuf=%p, prev_subbuf" + "=%p, prev_padding=%08x)\n", subbuf, prev_subbuf, prev_padding); + + if (relay_buf_full(buf)) { + if (!suspended) { + suspended = 1; + pr_warn("kgsl: cffdump: relay: cpu %d buffer full!!!\n", + smp_processor_id()); + } + dropped++; + return 0; + } else if (suspended) { + suspended = 0; + pr_warn("kgsl: cffdump: relay: cpu %d buffer no longer full.\n", + smp_processor_id()); + } + + subbuf_start_reserve(buf, 0); + return 1; +} + +static struct dentry *create_buf_file_handler(const char *filename, + struct dentry *parent, int mode, struct rchan_buf *buf, + int *is_global) +{ + return debugfs_create_file(filename, mode, parent, buf, + &relay_file_operations); +} + +/* + * file_remove() default callback. Removes relay file in debugfs. + */ +static int remove_buf_file_handler(struct dentry *dentry) +{ + pr_info("kgsl: cffdump: %s()\n", __func__); + debugfs_remove(dentry); + return 0; +} + +/* + * relay callbacks + */ +static struct rchan_callbacks relay_callbacks = { + .subbuf_start = subbuf_start_handler, + .create_buf_file = create_buf_file_handler, + .remove_buf_file = remove_buf_file_handler, +}; + +/** + * create_channel - creates channel /debug/klog/cpuXXX + * + * Creates channel along with associated produced/consumed control files + * + * Returns channel on success, NULL otherwise + */ +static struct rchan *create_channel(unsigned subbuf_size, unsigned n_subbufs) +{ + struct rchan *chan; + + pr_info("kgsl: cffdump: relay: create_channel: subbuf_size %u, " + "n_subbufs %u, dir 0x%p\n", subbuf_size, n_subbufs, dir); + + chan = relay_open("cpu", dir, subbuf_size, + n_subbufs, &relay_callbacks, NULL); + if (!chan) { + KGSL_CORE_ERR("relay_open failed\n"); + return NULL; + } + + suspended = 0; + dropped = 0; + + return chan; +} + +/** + * destroy_channel - destroys channel /debug/kgsl/cff/cpuXXX + * + * Destroys channel along with associated produced/consumed control files + */ +static void destroy_channel(void) +{ + pr_info("kgsl: cffdump: relay: destroy_channel\n"); + if (chan) { + relay_close(chan); + chan = NULL; + } +} + diff --git a/drivers/gpu/msm/kgsl_cffdump.h b/drivers/gpu/msm/kgsl_cffdump.h new file mode 100644 index 00000000..d2f9d172 --- /dev/null +++ b/drivers/gpu/msm/kgsl_cffdump.h @@ -0,0 +1,85 @@ +/* Copyright (c) 2010-2011, Code Aurora Forum. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are + * met: + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following + * disclaimer in the documentation and/or other materials provided + * with the distribution. + * * Neither the name of Code Aurora Forum, Inc. nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED "AS IS" AND ANY EXPRESS OR IMPLIED + * WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS + * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR + * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, + * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE + * OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN + * IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + */ + +#ifndef __KGSL_CFFDUMP_H +#define __KGSL_CFFDUMP_H + +#ifdef CONFIG_MSM_KGSL_CFF_DUMP + +#include + +#include "kgsl_device.h" + +void kgsl_cffdump_init(void); +void kgsl_cffdump_destroy(void); +void kgsl_cffdump_open(enum kgsl_deviceid device_id); +void kgsl_cffdump_close(enum kgsl_deviceid device_id); +void kgsl_cffdump_syncmem(struct kgsl_device_private *dev_priv, + const struct kgsl_memdesc *memdesc, uint physaddr, uint sizebytes, + bool clean_cache); +void kgsl_cffdump_setmem(uint addr, uint value, uint sizebytes); +void kgsl_cffdump_regwrite(enum kgsl_deviceid device_id, uint addr, + uint value); +void kgsl_cffdump_regpoll(enum kgsl_deviceid device_id, uint addr, + uint value, uint mask); +bool kgsl_cffdump_parse_ibs(struct kgsl_device_private *dev_priv, + const struct kgsl_memdesc *memdesc, uint gpuaddr, int sizedwords, + bool check_only); +void kgsl_cffdump_user_event(unsigned int cff_opcode, unsigned int op1, + unsigned int op2, unsigned int op3, + unsigned int op4, unsigned int op5); +static inline bool kgsl_cffdump_flags_no_memzero(void) { return true; } + +void kgsl_cffdump_memory_base(enum kgsl_deviceid device_id, unsigned int base, + unsigned int range, unsigned int gmemsize); + +void kgsl_cffdump_hang(enum kgsl_deviceid device_id); + +#else + +#define kgsl_cffdump_init() (void)0 +#define kgsl_cffdump_destroy() (void)0 +#define kgsl_cffdump_open(device_id) (void)0 +#define kgsl_cffdump_close(device_id) (void)0 +#define kgsl_cffdump_syncmem(dev_priv, memdesc, addr, sizebytes, clean_cache) \ + (void) 0 +#define kgsl_cffdump_setmem(addr, value, sizebytes) (void)0 +#define kgsl_cffdump_regwrite(device_id, addr, value) (void)0 +#define kgsl_cffdump_regpoll(device_id, addr, value, mask) (void)0 +#define kgsl_cffdump_parse_ibs(dev_priv, memdesc, gpuaddr, \ + sizedwords, check_only) true +#define kgsl_cffdump_flags_no_memzero() true +#define kgsl_cffdump_memory_base(base, range, gmemsize) (void)0 +#define kgsl_cffdump_hang(device_id) (void)0 +#define kgsl_cffdump_user_event(cff_opcode, op1, op2, op3, op4, op5) \ + (void)param + +#endif /* CONFIG_MSM_KGSL_CFF_DUMP */ + +#endif /* __KGSL_CFFDUMP_H */ diff --git a/drivers/gpu/msm/kgsl_debugfs.c b/drivers/gpu/msm/kgsl_debugfs.c new file mode 100644 index 00000000..d98586a9 --- /dev/null +++ b/drivers/gpu/msm/kgsl_debugfs.c @@ -0,0 +1,86 @@ +/* Copyright (c) 2002,2008-2011, Code Aurora Forum. All rights reserved. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 and + * only version 2 as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + */ + +#include + +#include "kgsl.h" + +/*default log levels is error for everything*/ +#define KGSL_LOG_LEVEL_DEFAULT 3 +#define KGSL_LOG_LEVEL_MAX 7 + +struct dentry *kgsl_debugfs_dir; + +static inline int kgsl_log_set(unsigned int *log_val, void *data, u64 val) +{ + *log_val = min((unsigned int)val, (unsigned int)KGSL_LOG_LEVEL_MAX); + return 0; +} + +#define KGSL_DEBUGFS_LOG(__log) \ +static int __log ## _set(void *data, u64 val) \ +{ \ + struct kgsl_device *device = data; \ + return kgsl_log_set(&device->__log, data, val); \ +} \ +static int __log ## _get(void *data, u64 *val) \ +{ \ + struct kgsl_device *device = data; \ + *val = device->__log; \ + return 0; \ +} \ +DEFINE_SIMPLE_ATTRIBUTE(__log ## _fops, \ +__log ## _get, __log ## _set, "%llu\n"); \ + +KGSL_DEBUGFS_LOG(drv_log); +KGSL_DEBUGFS_LOG(cmd_log); +KGSL_DEBUGFS_LOG(ctxt_log); +KGSL_DEBUGFS_LOG(mem_log); +KGSL_DEBUGFS_LOG(pwr_log); + +void kgsl_device_debugfs_init(struct kgsl_device *device) +{ + if (kgsl_debugfs_dir && !IS_ERR(kgsl_debugfs_dir)) + device->d_debugfs = debugfs_create_dir(device->name, + kgsl_debugfs_dir); + + if (!device->d_debugfs || IS_ERR(device->d_debugfs)) + return; + + device->cmd_log = KGSL_LOG_LEVEL_DEFAULT; + device->ctxt_log = KGSL_LOG_LEVEL_DEFAULT; + device->drv_log = KGSL_LOG_LEVEL_DEFAULT; + device->mem_log = KGSL_LOG_LEVEL_DEFAULT; + device->pwr_log = KGSL_LOG_LEVEL_DEFAULT; + + debugfs_create_file("log_level_cmd", 0644, device->d_debugfs, device, + &cmd_log_fops); + debugfs_create_file("log_level_ctxt", 0644, device->d_debugfs, device, + &ctxt_log_fops); + debugfs_create_file("log_level_drv", 0644, device->d_debugfs, device, + &drv_log_fops); + debugfs_create_file("log_level_mem", 0644, device->d_debugfs, device, + &mem_log_fops); + debugfs_create_file("log_level_pwr", 0644, device->d_debugfs, device, + &pwr_log_fops); +} + +void kgsl_core_debugfs_init(void) +{ + kgsl_debugfs_dir = debugfs_create_dir("kgsl", 0); +} + +void kgsl_core_debugfs_close(void) +{ + debugfs_remove_recursive(kgsl_debugfs_dir); +} diff --git a/drivers/gpu/msm/kgsl_debugfs.h b/drivers/gpu/msm/kgsl_debugfs.h new file mode 100644 index 00000000..5e109880 --- /dev/null +++ b/drivers/gpu/msm/kgsl_debugfs.h @@ -0,0 +1,39 @@ +/* Copyright (c) 2002,2008-2011, Code Aurora Forum. All rights reserved. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 and + * only version 2 as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + */ + +#ifndef _KGSL_DEBUGFS_H +#define _KGSL_DEBUGFS_H + +struct kgsl_device; + +#ifdef CONFIG_DEBUG_FS +void kgsl_core_debugfs_init(void); +void kgsl_core_debugfs_close(void); + +void kgsl_device_debugfs_init(struct kgsl_device *device); + +extern struct dentry *kgsl_debugfs_dir; +static inline struct dentry *kgsl_get_debugfs_dir(void) +{ + return kgsl_debugfs_dir; +} + +#else +static inline void kgsl_core_debugfs_init(void) { } +static inline void kgsl_device_debugfs_init(struct kgsl_device *device) { } +static inline void kgsl_core_debugfs_close(void) { } +static inline struct dentry *kgsl_get_debugfs_dir(void) { return NULL; } + +#endif + +#endif diff --git a/drivers/gpu/msm/kgsl_device.h b/drivers/gpu/msm/kgsl_device.h new file mode 100644 index 00000000..bb0f2b35 --- /dev/null +++ b/drivers/gpu/msm/kgsl_device.h @@ -0,0 +1,247 @@ +/* Copyright (c) 2002,2007-2011, Code Aurora Forum. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are + * met: + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following + * disclaimer in the documentation and/or other materials provided + * with the distribution. + * * Neither the name of Code Aurora Forum, Inc. nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED "AS IS" AND ANY EXPRESS OR IMPLIED + * WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS + * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR + * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, + * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE + * OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN + * IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + */ +#ifndef __KGSL_DEVICE_H +#define __KGSL_DEVICE_H + +#include +#include + +#include "kgsl_mmu.h" +#include "kgsl_pwrctrl.h" +#include "kgsl_log.h" +#include "kgsl_pwrscale.h" + +#define KGSL_TIMEOUT_NONE 0 +#define KGSL_TIMEOUT_DEFAULT 0xFFFFFFFF + +#define FIRST_TIMEOUT (HZ / 2) + + +/* KGSL device state is initialized to INIT when platform_probe * + * sucessfully initialized the device. Once a device has been opened * + * (started) it becomes active. NAP implies that only low latency * + * resources (for now clocks on some platforms) are off. SLEEP implies * + * that the KGSL module believes a device is idle (has been inactive * + * past its timer) and all system resources are released. SUSPEND is * + * requested by the kernel and will be enforced upon all open devices. */ + +#define KGSL_STATE_NONE 0x00000000 +#define KGSL_STATE_INIT 0x00000001 +#define KGSL_STATE_ACTIVE 0x00000002 +#define KGSL_STATE_NAP 0x00000004 +#define KGSL_STATE_SLEEP 0x00000008 +#define KGSL_STATE_SUSPEND 0x00000010 +#define KGSL_STATE_HUNG 0x00000020 +#define KGSL_STATE_DUMP_AND_RECOVER 0x00000040 + +#define KGSL_GRAPHICS_MEMORY_LOW_WATERMARK 0x1000000 + +#define KGSL_IS_PAGE_ALIGNED(addr) (!((addr) & (~PAGE_MASK))) + +struct kgsl_device; +struct platform_device; +struct kgsl_device_private; +struct kgsl_context; +struct kgsl_power_stats; + +struct kgsl_functable { + void (*device_regread) (struct kgsl_device *device, + unsigned int offsetwords, + unsigned int *value); + void (*device_regwrite) (struct kgsl_device *device, + unsigned int offsetwords, + unsigned int value); + void (*device_regread_isr) (struct kgsl_device *device, + unsigned int offsetwords, + unsigned int *value); + void (*device_regwrite_isr) (struct kgsl_device *device, + unsigned int offsetwords, + unsigned int value); + int (*device_setstate) (struct kgsl_device *device, uint32_t flags); + int (*device_idle) (struct kgsl_device *device, unsigned int timeout); + unsigned int (*device_isidle) (struct kgsl_device *device); + int (*device_suspend_context) (struct kgsl_device *device); + int (*device_resume_context) (struct kgsl_device *device); + int (*device_start) (struct kgsl_device *device, unsigned int init_ram); + int (*device_stop) (struct kgsl_device *device); + int (*device_getproperty) (struct kgsl_device *device, + enum kgsl_property_type type, + void *value, + unsigned int sizebytes); + int (*device_waittimestamp) (struct kgsl_device *device, + unsigned int timestamp, + unsigned int msecs); + unsigned int (*device_readtimestamp) ( + struct kgsl_device *device, + enum kgsl_timestamp_type type); + int (*device_issueibcmds) (struct kgsl_device_private *dev_priv, + struct kgsl_context *context, + struct kgsl_ibdesc *ibdesc, + unsigned int sizedwords, + uint32_t *timestamp, + unsigned int flags); + int (*device_drawctxt_create) (struct kgsl_device_private *dev_priv, + uint32_t flags, + struct kgsl_context *context); + int (*device_drawctxt_destroy) (struct kgsl_device *device, + struct kgsl_context *context); + long (*device_ioctl) (struct kgsl_device_private *dev_priv, + unsigned int cmd, void *data); + int (*device_setup_pt)(struct kgsl_device *device, + struct kgsl_pagetable *pagetable); + + int (*device_cleanup_pt)(struct kgsl_device *device, + struct kgsl_pagetable *pagetable); + void (*device_power_stats)(struct kgsl_device *device, + struct kgsl_power_stats *stats); +}; + +struct kgsl_memregion { + unsigned char *mmio_virt_base; + unsigned int mmio_phys_base; + uint32_t gpu_base; + unsigned int sizebytes; +}; + +struct kgsl_device { + struct device *dev; + const char *name; + unsigned int ver_major; + unsigned int ver_minor; + uint32_t flags; + enum kgsl_deviceid id; + struct kgsl_memregion regspace; + struct kgsl_memdesc memstore; + const char *iomemname; + + struct kgsl_mmu mmu; + struct completion hwaccess_gate; + struct kgsl_functable ftbl; + struct work_struct idle_check_ws; + struct timer_list idle_timer; + struct kgsl_pwrctrl pwrctrl; + int open_count; + + struct atomic_notifier_head ts_notifier_list; + struct mutex mutex; + uint32_t state; + uint32_t requested_state; + + struct list_head memqueue; + unsigned int active_cnt; + struct completion suspend_gate; + + wait_queue_head_t wait_queue; + struct workqueue_struct *work_queue; + struct device *parentdev; + struct completion recovery_gate; + struct dentry *d_debugfs; + struct idr context_idr; + + /* Logging levels */ + int cmd_log; + int ctxt_log; + int drv_log; + int mem_log; + int pwr_log; + struct wake_lock idle_wakelock; + struct kgsl_pwrscale pwrscale; + struct kobject pwrscale_kobj; +}; + +struct kgsl_context { + uint32_t id; + + /* Pointer to the owning device instance */ + struct kgsl_device_private *dev_priv; + + /* Pointer to the device specific context information */ + void *devctxt; +}; + +struct kgsl_process_private { + unsigned int refcnt; + pid_t pid; + spinlock_t mem_lock; + struct list_head mem_list; + struct kgsl_pagetable *pagetable; + struct list_head list; + struct kobject *kobj; + + struct { + unsigned int user; + unsigned int user_max; + unsigned int mapped; + unsigned int mapped_max; + unsigned int flushes; + } stats; +}; + +struct kgsl_device_private { + struct kgsl_device *device; + struct kgsl_process_private *process_priv; +}; + +struct kgsl_power_stats { + s64 total_time; + s64 busy_time; +}; + +struct kgsl_device *kgsl_get_device(int dev_idx); + +static inline struct kgsl_mmu * +kgsl_get_mmu(struct kgsl_device *device) +{ + return (struct kgsl_mmu *) (device ? &device->mmu : NULL); +} + +static inline int kgsl_create_device_workqueue(struct kgsl_device *device) +{ + device->work_queue = create_workqueue(device->name); + if (!device->work_queue) { + KGSL_DRV_ERR(device, "create_workqueue(%s) failed\n", + device->name); + return -EINVAL; + } + return 0; +} + +static inline struct kgsl_context * +kgsl_find_context(struct kgsl_device_private *dev_priv, uint32_t id) +{ + struct kgsl_context *ctxt = + idr_find(&dev_priv->device->context_idr, id); + + /* Make sure that the context belongs to the current instance so + that other processes can't guess context IDs and mess things up */ + + return (ctxt && ctxt->dev_priv == dev_priv) ? ctxt : NULL; +} + +#endif /* __KGSL_DEVICE_H */ diff --git a/drivers/gpu/msm/kgsl_drm.c b/drivers/gpu/msm/kgsl_drm.c new file mode 100644 index 00000000..1e878e15 --- /dev/null +++ b/drivers/gpu/msm/kgsl_drm.c @@ -0,0 +1,1690 @@ +/* Copyright (c) 2009-2011, Code Aurora Forum. All rights reserved. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 and + * only version 2 as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + */ + +/* Implements an interface between KGSL and the DRM subsystem. For now this + * is pretty simple, but it will take on more of the workload as time goes + * on + */ +#include "drmP.h" +#include "drm.h" +#include +#include + +#include "kgsl.h" +#include "kgsl_device.h" +#include "kgsl_drm.h" +#include "kgsl_mmu.h" +#include "kgsl_sharedmem.h" + +#define DRIVER_AUTHOR "Qualcomm" +#define DRIVER_NAME "kgsl" +#define DRIVER_DESC "KGSL DRM" +#define DRIVER_DATE "20100127" + +#define DRIVER_MAJOR 2 +#define DRIVER_MINOR 1 +#define DRIVER_PATCHLEVEL 1 + +#define DRM_KGSL_GEM_FLAG_MAPPED (1 << 0) + +#define ENTRY_EMPTY -1 +#define ENTRY_NEEDS_CLEANUP -2 + +#define DRM_KGSL_NUM_FENCE_ENTRIES (DRM_KGSL_HANDLE_WAIT_ENTRIES << 2) +#define DRM_KGSL_HANDLE_WAIT_ENTRIES 5 + +/* Returns true if the memory type is in PMEM */ + +#ifdef CONFIG_KERNEL_PMEM_SMI_REGION +#define TYPE_IS_PMEM(_t) \ + (((_t & DRM_KGSL_GEM_TYPE_MEM_MASK) == DRM_KGSL_GEM_TYPE_EBI) || \ + ((_t & DRM_KGSL_GEM_TYPE_MEM_MASK) == DRM_KGSL_GEM_TYPE_SMI) || \ + ((_t) & DRM_KGSL_GEM_TYPE_PMEM)) +#else +#define TYPE_IS_PMEM(_t) \ + (((_t & DRM_KGSL_GEM_TYPE_MEM_MASK) == DRM_KGSL_GEM_TYPE_EBI) || \ + ((_t) & (DRM_KGSL_GEM_TYPE_PMEM | DRM_KGSL_GEM_PMEM_EBI))) +#endif + +/* Returns true if the memory type is regular */ + +#define TYPE_IS_MEM(_t) \ + (((_t & DRM_KGSL_GEM_TYPE_MEM_MASK) == DRM_KGSL_GEM_TYPE_KMEM) || \ + ((_t & DRM_KGSL_GEM_TYPE_MEM_MASK) == DRM_KGSL_GEM_TYPE_KMEM_NOCACHE) || \ + ((_t) & DRM_KGSL_GEM_TYPE_MEM)) + +#define TYPE_IS_FD(_t) ((_t) & DRM_KGSL_GEM_TYPE_FD_MASK) + +/* Returns true if KMEM region is uncached */ + +#define IS_MEM_UNCACHED(_t) \ + ((_t == DRM_KGSL_GEM_TYPE_KMEM_NOCACHE) || \ + (_t == DRM_KGSL_GEM_TYPE_KMEM) || \ + (TYPE_IS_MEM(_t) && (_t & DRM_KGSL_GEM_CACHE_WCOMBINE))) + +struct drm_kgsl_gem_object_wait_list_entry { + struct list_head list; + int pid; + int in_use; + wait_queue_head_t process_wait_q; +}; + +struct drm_kgsl_gem_object_fence { + int32_t fence_id; + unsigned int num_buffers; + int ts_valid; + unsigned int timestamp; + int ts_device; + int lockpid; + struct list_head buffers_in_fence; +}; + +struct drm_kgsl_gem_object_fence_list_entry { + struct list_head list; + int in_use; + struct drm_gem_object *gem_obj; +}; + +static int32_t fence_id = 0x1; + +static struct drm_kgsl_gem_object_fence + gem_buf_fence[DRM_KGSL_NUM_FENCE_ENTRIES]; + +struct drm_kgsl_gem_object { + struct drm_gem_object *obj; + uint32_t type; + struct kgsl_memdesc memdesc; + struct kgsl_pagetable *pagetable; + uint64_t mmap_offset; + int bufcount; + int flags; + struct list_head list; + int active; + + struct { + uint32_t offset; + uint32_t gpuaddr; + } bufs[DRM_KGSL_GEM_MAX_BUFFERS]; + + int bound; + int lockpid; + /* Put these here to avoid allocing all the time */ + struct drm_kgsl_gem_object_wait_list_entry + wait_entries[DRM_KGSL_HANDLE_WAIT_ENTRIES]; + /* Each object can only appear in a single fence */ + struct drm_kgsl_gem_object_fence_list_entry + fence_entries[DRM_KGSL_NUM_FENCE_ENTRIES]; + + struct list_head wait_list; +}; + +/* This is a global list of all the memory currently mapped in the MMU */ +static struct list_head kgsl_mem_list; + +static void kgsl_gem_mem_flush(struct kgsl_memdesc *memdesc, int type, int op) +{ + int cacheop = 0; + + switch (op) { + case DRM_KGSL_GEM_CACHE_OP_TO_DEV: + if (type & (DRM_KGSL_GEM_CACHE_WBACK | + DRM_KGSL_GEM_CACHE_WBACKWA)) + cacheop = KGSL_CACHE_OP_CLEAN; + + break; + + case DRM_KGSL_GEM_CACHE_OP_FROM_DEV: + if (type & (DRM_KGSL_GEM_CACHE_WBACK | + DRM_KGSL_GEM_CACHE_WBACKWA | + DRM_KGSL_GEM_CACHE_WTHROUGH)) + cacheop = KGSL_CACHE_OP_INV; + } + + kgsl_cache_range_op(memdesc, cacheop); +} + +/* Flush all the memory mapped in the MMU */ + +void kgsl_gpu_mem_flush(int op) +{ + struct drm_kgsl_gem_object *entry; + + list_for_each_entry(entry, &kgsl_mem_list, list) { + kgsl_gem_mem_flush(&entry->memdesc, entry->type, op); + } + + /* Takes care of WT/WC case. + * More useful when we go barrierless + */ + dmb(); +} + +/* TODO: + * Add vsync wait */ + +static int kgsl_drm_load(struct drm_device *dev, unsigned long flags) +{ + return 0; +} + +static int kgsl_drm_unload(struct drm_device *dev) +{ + return 0; +} + +struct kgsl_drm_device_priv { + struct kgsl_device *device[KGSL_DEVICE_MAX]; + struct kgsl_device_private *devpriv[KGSL_DEVICE_MAX]; +}; + +static int kgsl_ts_notifier_cb(struct notifier_block *blk, + unsigned long code, void *_param); + +static struct notifier_block kgsl_ts_nb[KGSL_DEVICE_MAX]; + +static int kgsl_drm_firstopen(struct drm_device *dev) +{ + int i; + + for (i = 0; i < KGSL_DEVICE_MAX; i++) { + struct kgsl_device *device = kgsl_get_device(i); + + if (device == NULL) + continue; + + kgsl_ts_nb[i].notifier_call = kgsl_ts_notifier_cb; + kgsl_register_ts_notifier(device, &kgsl_ts_nb[i]); + } + + return 0; +} + +void kgsl_drm_lastclose(struct drm_device *dev) +{ + int i; + + for (i = 0; i < KGSL_DEVICE_MAX; i++) { + struct kgsl_device *device = kgsl_get_device(i); + if (device == NULL) + continue; + + kgsl_unregister_ts_notifier(device, &kgsl_ts_nb[i]); + } +} + +void kgsl_drm_preclose(struct drm_device *dev, struct drm_file *file_priv) +{ +} + +static int kgsl_drm_suspend(struct drm_device *dev, pm_message_t state) +{ + return 0; +} + +static int kgsl_drm_resume(struct drm_device *dev) +{ + return 0; +} + +static void +kgsl_gem_free_mmap_offset(struct drm_gem_object *obj) +{ + struct drm_device *dev = obj->dev; + struct drm_gem_mm *mm = dev->mm_private; + struct drm_kgsl_gem_object *priv = obj->driver_private; + struct drm_map_list *list; + + list = &obj->map_list; + drm_ht_remove_item(&mm->offset_hash, &list->hash); + if (list->file_offset_node) { + drm_mm_put_block(list->file_offset_node); + list->file_offset_node = NULL; + } + + kfree(list->map); + list->map = NULL; + + priv->mmap_offset = 0; +} + +static int +kgsl_gem_memory_allocated(struct drm_gem_object *obj) +{ + struct drm_kgsl_gem_object *priv = obj->driver_private; + return priv->memdesc.size ? 1 : 0; +} + +static int +kgsl_gem_alloc_memory(struct drm_gem_object *obj) +{ + struct drm_kgsl_gem_object *priv = obj->driver_private; + int index; + + /* Return if the memory is already allocated */ + + if (kgsl_gem_memory_allocated(obj) || TYPE_IS_FD(priv->type)) + return 0; + + if (TYPE_IS_PMEM(priv->type)) { + int type; + + if (priv->type == DRM_KGSL_GEM_TYPE_EBI || + priv->type & DRM_KGSL_GEM_PMEM_EBI) + type = PMEM_MEMTYPE_EBI1; + else + type = PMEM_MEMTYPE_SMI; + + priv->memdesc.physaddr = + pmem_kalloc(obj->size * priv->bufcount, + type | PMEM_ALIGNMENT_4K); + + if (IS_ERR((void *) priv->memdesc.physaddr)) { + DRM_ERROR("Unable to allocate PMEM memory\n"); + return -ENOMEM; + } + + priv->memdesc.size = obj->size * priv->bufcount; + priv->memdesc.ops = &kgsl_contig_ops; + + } else if (TYPE_IS_MEM(priv->type)) { + priv->memdesc.hostptr = + vmalloc_user(obj->size * priv->bufcount); + + if (priv->memdesc.hostptr == NULL) { + DRM_ERROR("Unable to allocate vmalloc memory\n"); + return -ENOMEM; + } + + priv->memdesc.size = obj->size * priv->bufcount; + priv->memdesc.ops = &kgsl_vmalloc_ops; + } else + return -EINVAL; + + for (index = 0; index < priv->bufcount; index++) + priv->bufs[index].offset = index * obj->size; + + + return 0; +} + +#ifdef CONFIG_MSM_KGSL_MMU +static void +kgsl_gem_unmap(struct drm_gem_object *obj) +{ + struct drm_kgsl_gem_object *priv = obj->driver_private; + + if (!priv->flags & DRM_KGSL_GEM_FLAG_MAPPED) + return; + + kgsl_mmu_unmap(priv->pagetable, &priv->memdesc); + + kgsl_mmu_putpagetable(priv->pagetable); + priv->pagetable = NULL; + + if ((priv->type == DRM_KGSL_GEM_TYPE_KMEM) || + (priv->type & DRM_KGSL_GEM_CACHE_MASK)) + list_del(&priv->list); + + priv->flags &= ~DRM_KGSL_GEM_FLAG_MAPPED; +} +#else +static void +kgsl_gem_unmap(struct drm_gem_object *obj) +{ +} +#endif + +static void +kgsl_gem_free_memory(struct drm_gem_object *obj) +{ + struct drm_kgsl_gem_object *priv = obj->driver_private; + + if (!kgsl_gem_memory_allocated(obj) || TYPE_IS_FD(priv->type)) + return; + + kgsl_gem_mem_flush(&priv->memdesc, priv->type, + DRM_KGSL_GEM_CACHE_OP_FROM_DEV); + + kgsl_gem_unmap(obj); + + if (TYPE_IS_PMEM(priv->type)) + pmem_kfree(priv->memdesc.physaddr); + + kgsl_sharedmem_free(&priv->memdesc); +} + +int +kgsl_gem_init_object(struct drm_gem_object *obj) +{ + struct drm_kgsl_gem_object *priv; + priv = kzalloc(sizeof(*priv), GFP_KERNEL); + if (priv == NULL) { + DRM_ERROR("Unable to create GEM object\n"); + return -ENOMEM; + } + + obj->driver_private = priv; + priv->obj = obj; + + return 0; +} + +void +kgsl_gem_free_object(struct drm_gem_object *obj) +{ + kgsl_gem_free_memory(obj); + kgsl_gem_free_mmap_offset(obj); + drm_gem_object_release(obj); + kfree(obj->driver_private); +} + +static int +kgsl_gem_create_mmap_offset(struct drm_gem_object *obj) +{ + struct drm_device *dev = obj->dev; + struct drm_gem_mm *mm = dev->mm_private; + struct drm_kgsl_gem_object *priv = obj->driver_private; + struct drm_map_list *list; + int msize; + + list = &obj->map_list; + list->map = kzalloc(sizeof(struct drm_map_list), GFP_KERNEL); + if (list->map == NULL) { + DRM_ERROR("Unable to allocate drm_map_list\n"); + return -ENOMEM; + } + + msize = obj->size * priv->bufcount; + + list->map->type = _DRM_GEM; + list->map->size = msize; + list->map->handle = obj; + + /* Allocate a mmap offset */ + list->file_offset_node = drm_mm_search_free(&mm->offset_manager, + msize / PAGE_SIZE, + 0, 0); + + if (!list->file_offset_node) { + DRM_ERROR("Failed to allocate offset for %d\n", obj->name); + kfree(list->map); + return -ENOMEM; + } + + list->file_offset_node = drm_mm_get_block(list->file_offset_node, + msize / PAGE_SIZE, 0); + + if (!list->file_offset_node) { + DRM_ERROR("Unable to create the file_offset_node\n"); + kfree(list->map); + return -ENOMEM; + } + + list->hash.key = list->file_offset_node->start; + if (drm_ht_insert_item(&mm->offset_hash, &list->hash)) { + DRM_ERROR("Failed to add to map hash\n"); + drm_mm_put_block(list->file_offset_node); + kfree(list->map); + return -ENOMEM; + } + + priv->mmap_offset = ((uint64_t) list->hash.key) << PAGE_SHIFT; + + return 0; +} + +int +kgsl_gem_obj_addr(int drm_fd, int handle, unsigned long *start, + unsigned long *len) +{ + struct file *filp; + struct drm_device *dev; + struct drm_file *file_priv; + struct drm_gem_object *obj; + struct drm_kgsl_gem_object *priv; + int ret = 0; + + filp = fget(drm_fd); + if (unlikely(filp == NULL)) { + DRM_ERROR("Unable to ghet the DRM file descriptor\n"); + return -EINVAL; + } + file_priv = filp->private_data; + if (unlikely(file_priv == NULL)) { + DRM_ERROR("Unable to get the file private data\n"); + fput(filp); + return -EINVAL; + } + dev = file_priv->minor->dev; + if (unlikely(dev == NULL)) { + DRM_ERROR("Unable to get the minor device\n"); + fput(filp); + return -EINVAL; + } + + obj = drm_gem_object_lookup(dev, file_priv, handle); + if (unlikely(obj == NULL)) { + DRM_ERROR("Invalid GEM handle %x\n", handle); + fput(filp); + return -EBADF; + } + + mutex_lock(&dev->struct_mutex); + priv = obj->driver_private; + + /* We can only use the MDP for PMEM regions */ + + if (TYPE_IS_PMEM(priv->type)) { + *start = priv->memdesc.physaddr + + priv->bufs[priv->active].offset; + + *len = priv->memdesc.size; + + kgsl_gem_mem_flush(&priv->memdesc, + priv->type, DRM_KGSL_GEM_CACHE_OP_TO_DEV); + } else { + *start = 0; + *len = 0; + ret = -EINVAL; + } + + drm_gem_object_unreference(obj); + mutex_unlock(&dev->struct_mutex); + + fput(filp); + return ret; +} + +static int +kgsl_gem_init_obj(struct drm_device *dev, + struct drm_file *file_priv, + struct drm_gem_object *obj, + int *handle) +{ + struct drm_kgsl_gem_object *priv; + int ret, i; + + mutex_lock(&dev->struct_mutex); + priv = obj->driver_private; + + memset(&priv->memdesc, 0, sizeof(priv->memdesc)); + priv->bufcount = 1; + priv->active = 0; + priv->bound = 0; + + /* To preserve backwards compatability, the default memory source + is EBI */ + + priv->type = DRM_KGSL_GEM_TYPE_PMEM | DRM_KGSL_GEM_PMEM_EBI; + + ret = drm_gem_handle_create(file_priv, obj, handle); + + drm_gem_object_handle_unreference(obj); + INIT_LIST_HEAD(&priv->wait_list); + + for (i = 0; i < DRM_KGSL_HANDLE_WAIT_ENTRIES; i++) { + INIT_LIST_HEAD((struct list_head *) &priv->wait_entries[i]); + priv->wait_entries[i].pid = 0; + init_waitqueue_head(&priv->wait_entries[i].process_wait_q); + } + + for (i = 0; i < DRM_KGSL_NUM_FENCE_ENTRIES; i++) { + INIT_LIST_HEAD((struct list_head *) &priv->fence_entries[i]); + priv->fence_entries[i].in_use = 0; + priv->fence_entries[i].gem_obj = obj; + } + + mutex_unlock(&dev->struct_mutex); + return ret; +} + +int +kgsl_gem_create_ioctl(struct drm_device *dev, void *data, + struct drm_file *file_priv) +{ + struct drm_kgsl_gem_create *create = data; + struct drm_gem_object *obj; + int ret, handle; + + /* Page align the size so we can allocate multiple buffers */ + create->size = ALIGN(create->size, 4096); + + obj = drm_gem_object_alloc(dev, create->size); + + if (obj == NULL) { + DRM_ERROR("Unable to allocate the GEM object\n"); + return -ENOMEM; + } + + ret = kgsl_gem_init_obj(dev, file_priv, obj, &handle); + if (ret) + return ret; + + create->handle = handle; + return 0; +} + +int +kgsl_gem_create_fd_ioctl(struct drm_device *dev, void *data, + struct drm_file *file_priv) +{ + struct drm_kgsl_gem_create_fd *args = data; + struct file *file; + dev_t rdev; + struct fb_info *info; + struct drm_gem_object *obj; + struct drm_kgsl_gem_object *priv; + int ret, put_needed, handle; + + file = fget_light(args->fd, &put_needed); + + if (file == NULL) { + DRM_ERROR("Unable to get the file object\n"); + return -EBADF; + } + + rdev = file->f_dentry->d_inode->i_rdev; + + /* Only framebuffer objects are supported ATM */ + + if (MAJOR(rdev) != FB_MAJOR) { + DRM_ERROR("File descriptor is not a framebuffer\n"); + ret = -EBADF; + goto error_fput; + } + + info = registered_fb[MINOR(rdev)]; + + if (info == NULL) { + DRM_ERROR("Framebuffer minor %d is not registered\n", + MINOR(rdev)); + ret = -EBADF; + goto error_fput; + } + + obj = drm_gem_object_alloc(dev, info->fix.smem_len); + + if (obj == NULL) { + DRM_ERROR("Unable to allocate GEM object\n"); + ret = -ENOMEM; + goto error_fput; + } + + ret = kgsl_gem_init_obj(dev, file_priv, obj, &handle); + + if (ret) + goto error_fput; + + mutex_lock(&dev->struct_mutex); + + priv = obj->driver_private; + priv->memdesc.physaddr = info->fix.smem_start; + priv->type = DRM_KGSL_GEM_TYPE_FD_FBMEM; + + mutex_unlock(&dev->struct_mutex); + args->handle = handle; + +error_fput: + fput_light(file, put_needed); + + return ret; +} + +int +kgsl_gem_setmemtype_ioctl(struct drm_device *dev, void *data, + struct drm_file *file_priv) +{ + struct drm_kgsl_gem_memtype *args = data; + struct drm_gem_object *obj; + struct drm_kgsl_gem_object *priv; + int ret = 0; + + obj = drm_gem_object_lookup(dev, file_priv, args->handle); + + if (obj == NULL) { + DRM_ERROR("Invalid GEM handle %x\n", args->handle); + return -EBADF; + } + + mutex_lock(&dev->struct_mutex); + priv = obj->driver_private; + + if (TYPE_IS_FD(priv->type)) + ret = -EINVAL; + else { + if (TYPE_IS_PMEM(args->type) || TYPE_IS_MEM(args->type)) + priv->type = args->type; + else + ret = -EINVAL; + } + + drm_gem_object_unreference(obj); + mutex_unlock(&dev->struct_mutex); + + return ret; +} + +int +kgsl_gem_getmemtype_ioctl(struct drm_device *dev, void *data, + struct drm_file *file_priv) +{ + struct drm_kgsl_gem_memtype *args = data; + struct drm_gem_object *obj; + struct drm_kgsl_gem_object *priv; + + obj = drm_gem_object_lookup(dev, file_priv, args->handle); + + if (obj == NULL) { + DRM_ERROR("Invalid GEM handle %x\n", args->handle); + return -EBADF; + } + + mutex_lock(&dev->struct_mutex); + priv = obj->driver_private; + + args->type = priv->type; + + drm_gem_object_unreference(obj); + mutex_unlock(&dev->struct_mutex); + + return 0; +} + +int +kgsl_gem_unbind_gpu_ioctl(struct drm_device *dev, void *data, + struct drm_file *file_priv) +{ + struct drm_kgsl_gem_bind_gpu *args = data; + struct drm_gem_object *obj; + struct drm_kgsl_gem_object *priv; + + obj = drm_gem_object_lookup(dev, file_priv, args->handle); + + if (obj == NULL) { + DRM_ERROR("Invalid GEM handle %x\n", args->handle); + return -EBADF; + } + + mutex_lock(&dev->struct_mutex); + priv = obj->driver_private; + + if (--priv->bound == 0) + kgsl_gem_unmap(obj); + + drm_gem_object_unreference(obj); + mutex_unlock(&dev->struct_mutex); + return 0; +} + +#ifdef CONFIG_MSM_KGSL_MMU +static int +kgsl_gem_map(struct drm_gem_object *obj) +{ + struct drm_kgsl_gem_object *priv = obj->driver_private; + int index; + int ret = -EINVAL; + + if (priv->flags & DRM_KGSL_GEM_FLAG_MAPPED) + return 0; + + /* Get the global page table */ + + if (priv->pagetable == NULL) { + priv->pagetable = kgsl_mmu_getpagetable(KGSL_MMU_GLOBAL_PT); + + if (priv->pagetable == NULL) { + DRM_ERROR("Unable to get the GPU MMU pagetable\n"); + return -EINVAL; + } + } + + priv->memdesc.pagetable = priv->pagetable; + + ret = kgsl_mmu_map(priv->pagetable, &priv->memdesc, + GSL_PT_PAGE_RV | GSL_PT_PAGE_WV); + + if (!ret) { + for (index = 0; index < priv->bufcount; index++) { + priv->bufs[index].gpuaddr = + priv->memdesc.gpuaddr + + priv->bufs[index].offset; + } + } + + /* Add cached memory to the list to be cached */ + + if (priv->type == DRM_KGSL_GEM_TYPE_KMEM || + priv->type & DRM_KGSL_GEM_CACHE_MASK) + list_add(&priv->list, &kgsl_mem_list); + + priv->flags |= DRM_KGSL_GEM_FLAG_MAPPED; + + return ret; +} +#else +static int +kgsl_gem_map(struct drm_gem_object *obj) +{ + struct drm_kgsl_gem_object *priv = obj->driver_private; + int index; + + if (TYPE_IS_PMEM(priv->type)) { + for (index = 0; index < priv->bufcount; index++) + priv->bufs[index].gpuaddr = + priv->memdesc.physaddr + priv->bufs[index].offset; + + return 0; + } + + return -EINVAL; +} +#endif + +int +kgsl_gem_bind_gpu_ioctl(struct drm_device *dev, void *data, + struct drm_file *file_priv) +{ + struct drm_kgsl_gem_bind_gpu *args = data; + struct drm_gem_object *obj; + struct drm_kgsl_gem_object *priv; + int ret = 0; + + obj = drm_gem_object_lookup(dev, file_priv, args->handle); + + if (obj == NULL) { + DRM_ERROR("Invalid GEM handle %x\n", args->handle); + return -EBADF; + } + + mutex_lock(&dev->struct_mutex); + priv = obj->driver_private; + + if (priv->bound++ == 0) { + + if (!kgsl_gem_memory_allocated(obj)) { + DRM_ERROR("Memory not allocated for this object\n"); + ret = -ENOMEM; + goto out; + } + + ret = kgsl_gem_map(obj); + + /* This is legacy behavior - use GET_BUFFERINFO instead */ + args->gpuptr = priv->bufs[0].gpuaddr; + } +out: + drm_gem_object_unreference(obj); + mutex_unlock(&dev->struct_mutex); + return ret; +} + +/* Allocate the memory and prepare it for CPU mapping */ + +int +kgsl_gem_alloc_ioctl(struct drm_device *dev, void *data, + struct drm_file *file_priv) +{ + struct drm_kgsl_gem_alloc *args = data; + struct drm_gem_object *obj; + struct drm_kgsl_gem_object *priv; + int ret; + + obj = drm_gem_object_lookup(dev, file_priv, args->handle); + + if (obj == NULL) { + DRM_ERROR("Invalid GEM handle %x\n", args->handle); + return -EBADF; + } + + mutex_lock(&dev->struct_mutex); + priv = obj->driver_private; + + ret = kgsl_gem_alloc_memory(obj); + + if (ret) { + DRM_ERROR("Unable to allocate object memory\n"); + } else if (!priv->mmap_offset) { + ret = kgsl_gem_create_mmap_offset(obj); + if (ret) + DRM_ERROR("Unable to create a mmap offset\n"); + } + + args->offset = priv->mmap_offset; + + drm_gem_object_unreference(obj); + mutex_unlock(&dev->struct_mutex); + + return ret; +} + +int +kgsl_gem_mmap_ioctl(struct drm_device *dev, void *data, + struct drm_file *file_priv) +{ + struct drm_kgsl_gem_mmap *args = data; + struct drm_gem_object *obj; + unsigned long addr; + + obj = drm_gem_object_lookup(dev, file_priv, args->handle); + + if (obj == NULL) { + DRM_ERROR("Invalid GEM handle %x\n", args->handle); + return -EBADF; + } + + down_write(¤t->mm->mmap_sem); + + addr = do_mmap(obj->filp, 0, args->size, + PROT_READ | PROT_WRITE, MAP_SHARED, + args->offset); + + up_write(¤t->mm->mmap_sem); + + mutex_lock(&dev->struct_mutex); + drm_gem_object_unreference(obj); + mutex_unlock(&dev->struct_mutex); + + if (IS_ERR((void *) addr)) + return addr; + + args->hostptr = (uint32_t) addr; + return 0; +} + +/* This function is deprecated */ + +int +kgsl_gem_prep_ioctl(struct drm_device *dev, void *data, + struct drm_file *file_priv) +{ + struct drm_kgsl_gem_prep *args = data; + struct drm_gem_object *obj; + struct drm_kgsl_gem_object *priv; + int ret; + + obj = drm_gem_object_lookup(dev, file_priv, args->handle); + + if (obj == NULL) { + DRM_ERROR("Invalid GEM handle %x\n", args->handle); + return -EBADF; + } + + mutex_lock(&dev->struct_mutex); + priv = obj->driver_private; + + ret = kgsl_gem_alloc_memory(obj); + if (ret) { + DRM_ERROR("Unable to allocate object memory\n"); + drm_gem_object_unreference(obj); + mutex_unlock(&dev->struct_mutex); + return ret; + } + + if (priv->mmap_offset == 0) { + ret = kgsl_gem_create_mmap_offset(obj); + if (ret) { + drm_gem_object_unreference(obj); + mutex_unlock(&dev->struct_mutex); + return ret; + } + } + + args->offset = priv->mmap_offset; + args->phys = priv->memdesc.physaddr; + + drm_gem_object_unreference(obj); + mutex_unlock(&dev->struct_mutex); + + return 0; +} + +int +kgsl_gem_get_bufinfo_ioctl(struct drm_device *dev, void *data, + struct drm_file *file_priv) +{ + struct drm_kgsl_gem_bufinfo *args = data; + struct drm_gem_object *obj; + struct drm_kgsl_gem_object *priv; + int ret = -EINVAL; + int index; + + obj = drm_gem_object_lookup(dev, file_priv, args->handle); + + if (obj == NULL) { + DRM_ERROR("Invalid GEM handle %x\n", args->handle); + return -EBADF; + } + + mutex_lock(&dev->struct_mutex); + priv = obj->driver_private; + + if (!kgsl_gem_memory_allocated(obj)) { + DRM_ERROR("Memory not allocated for this object\n"); + goto out; + } + + for (index = 0; index < priv->bufcount; index++) { + args->offset[index] = priv->bufs[index].offset; + args->gpuaddr[index] = priv->bufs[index].gpuaddr; + } + + args->count = priv->bufcount; + args->active = priv->active; + + ret = 0; + +out: + drm_gem_object_unreference(obj); + mutex_unlock(&dev->struct_mutex); + + return ret; +} + +int +kgsl_gem_set_bufcount_ioctl(struct drm_device *dev, void *data, + struct drm_file *file_priv) +{ + struct drm_kgsl_gem_bufcount *args = data; + struct drm_gem_object *obj; + struct drm_kgsl_gem_object *priv; + int ret = -EINVAL; + + if (args->bufcount < 1 || args->bufcount > DRM_KGSL_GEM_MAX_BUFFERS) + return -EINVAL; + + obj = drm_gem_object_lookup(dev, file_priv, args->handle); + + if (obj == NULL) { + DRM_ERROR("Invalid GEM handle %x\n", args->handle); + return -EBADF; + } + + mutex_lock(&dev->struct_mutex); + priv = obj->driver_private; + + /* It is too much math to worry about what happens if we are already + allocated, so just bail if we are */ + + if (kgsl_gem_memory_allocated(obj)) { + DRM_ERROR("Memory already allocated - cannot change" + "number of buffers\n"); + goto out; + } + + priv->bufcount = args->bufcount; + ret = 0; + +out: + drm_gem_object_unreference(obj); + mutex_unlock(&dev->struct_mutex); + + return ret; +} + +int +kgsl_gem_set_active_ioctl(struct drm_device *dev, void *data, + struct drm_file *file_priv) +{ + struct drm_kgsl_gem_active *args = data; + struct drm_gem_object *obj; + struct drm_kgsl_gem_object *priv; + int ret = -EINVAL; + + obj = drm_gem_object_lookup(dev, file_priv, args->handle); + + if (obj == NULL) { + DRM_ERROR("Invalid GEM handle %x\n", args->handle); + return -EBADF; + } + + mutex_lock(&dev->struct_mutex); + priv = obj->driver_private; + + if (args->active < 0 || args->active >= priv->bufcount) { + DRM_ERROR("Invalid active buffer %d\n", args->active); + goto out; + } + + priv->active = args->active; + ret = 0; + +out: + drm_gem_object_unreference(obj); + mutex_unlock(&dev->struct_mutex); + + return ret; +} + +int kgsl_gem_kmem_fault(struct vm_area_struct *vma, struct vm_fault *vmf) +{ + struct drm_gem_object *obj = vma->vm_private_data; + struct drm_device *dev = obj->dev; + struct drm_kgsl_gem_object *priv; + unsigned long offset, pg; + struct page *page; + + mutex_lock(&dev->struct_mutex); + + priv = obj->driver_private; + + offset = (unsigned long) vmf->virtual_address - vma->vm_start; + pg = (unsigned long) priv->memdesc.hostptr + offset; + + page = vmalloc_to_page((void *) pg); + if (!page) { + mutex_unlock(&dev->struct_mutex); + return VM_FAULT_SIGBUS; + } + + get_page(page); + vmf->page = page; + + mutex_unlock(&dev->struct_mutex); + return 0; +} + +int kgsl_gem_phys_fault(struct vm_area_struct *vma, struct vm_fault *vmf) +{ + struct drm_gem_object *obj = vma->vm_private_data; + struct drm_device *dev = obj->dev; + struct drm_kgsl_gem_object *priv; + unsigned long offset, pfn; + int ret = 0; + + offset = ((unsigned long) vmf->virtual_address - vma->vm_start) >> + PAGE_SHIFT; + + mutex_lock(&dev->struct_mutex); + + priv = obj->driver_private; + + pfn = (priv->memdesc.physaddr >> PAGE_SHIFT) + offset; + ret = vm_insert_pfn(vma, + (unsigned long) vmf->virtual_address, pfn); + mutex_unlock(&dev->struct_mutex); + + switch (ret) { + case -ENOMEM: + case -EAGAIN: + return VM_FAULT_OOM; + case -EFAULT: + return VM_FAULT_SIGBUS; + default: + return VM_FAULT_NOPAGE; + } +} + +static struct vm_operations_struct kgsl_gem_kmem_vm_ops = { + .fault = kgsl_gem_kmem_fault, + .open = drm_gem_vm_open, + .close = drm_gem_vm_close, +}; + +static struct vm_operations_struct kgsl_gem_phys_vm_ops = { + .fault = kgsl_gem_phys_fault, + .open = drm_gem_vm_open, + .close = drm_gem_vm_close, +}; + +/* This is a clone of the standard drm_gem_mmap function modified to allow + us to properly map KMEM regions as well as the PMEM regions */ + +int msm_drm_gem_mmap(struct file *filp, struct vm_area_struct *vma) +{ + struct drm_file *priv = filp->private_data; + struct drm_device *dev = priv->minor->dev; + struct drm_gem_mm *mm = dev->mm_private; + struct drm_local_map *map = NULL; + struct drm_gem_object *obj; + struct drm_hash_item *hash; + struct drm_kgsl_gem_object *gpriv; + int ret = 0; + + mutex_lock(&dev->struct_mutex); + + if (drm_ht_find_item(&mm->offset_hash, vma->vm_pgoff, &hash)) { + mutex_unlock(&dev->struct_mutex); + return drm_mmap(filp, vma); + } + + map = drm_hash_entry(hash, struct drm_map_list, hash)->map; + if (!map || + ((map->flags & _DRM_RESTRICTED) && !capable(CAP_SYS_ADMIN))) { + ret = -EPERM; + goto out_unlock; + } + + /* Check for valid size. */ + if (map->size < vma->vm_end - vma->vm_start) { + ret = -EINVAL; + goto out_unlock; + } + + obj = map->handle; + + gpriv = obj->driver_private; + + /* VM_PFNMAP is only for memory that doesn't use struct page + * in other words, not "normal" memory. If you try to use it + * with "normal" memory then the mappings don't get flushed. */ + + if (TYPE_IS_MEM(gpriv->type)) { + vma->vm_flags |= VM_RESERVED | VM_DONTEXPAND; + vma->vm_ops = &kgsl_gem_kmem_vm_ops; + } else { + vma->vm_flags |= VM_RESERVED | VM_IO | VM_PFNMAP | + VM_DONTEXPAND; + vma->vm_ops = &kgsl_gem_phys_vm_ops; + } + + vma->vm_private_data = map->handle; + + + /* Take care of requested caching policy */ + if (gpriv->type == DRM_KGSL_GEM_TYPE_KMEM || + gpriv->type & DRM_KGSL_GEM_CACHE_MASK) { + if (gpriv->type & DRM_KGSL_GEM_CACHE_WBACKWA) + vma->vm_page_prot = + pgprot_writebackwacache(vma->vm_page_prot); + else if (gpriv->type & DRM_KGSL_GEM_CACHE_WBACK) + vma->vm_page_prot = + pgprot_writebackcache(vma->vm_page_prot); + else if (gpriv->type & DRM_KGSL_GEM_CACHE_WTHROUGH) + vma->vm_page_prot = + pgprot_writethroughcache(vma->vm_page_prot); + else + vma->vm_page_prot = + pgprot_writecombine(vma->vm_page_prot); + } else { + if (gpriv->type == DRM_KGSL_GEM_TYPE_KMEM_NOCACHE) + vma->vm_page_prot = + pgprot_noncached(vma->vm_page_prot); + else + /* default pmem is WC */ + vma->vm_page_prot = + pgprot_writecombine(vma->vm_page_prot); + } + + /* flush out existing KMEM cached mappings if new ones are + * of uncached type */ + if (IS_MEM_UNCACHED(gpriv->type)) + kgsl_cache_range_op(&gpriv->memdesc, + KGSL_CACHE_OP_FLUSH); + + /* Add the other memory types here */ + + /* Take a ref for this mapping of the object, so that the fault + * handler can dereference the mmap offset's pointer to the object. + * This reference is cleaned up by the corresponding vm_close + * (which should happen whether the vma was created by this call, or + * by a vm_open due to mremap or partial unmap or whatever). + */ + drm_gem_object_reference(obj); + + vma->vm_file = filp; /* Needed for drm_vm_open() */ + drm_vm_open_locked(vma); + +out_unlock: + mutex_unlock(&dev->struct_mutex); + + return ret; +} + +void +cleanup_fence(struct drm_kgsl_gem_object_fence *fence, int check_waiting) +{ + int j; + struct drm_kgsl_gem_object_fence_list_entry *this_fence_entry = NULL; + struct drm_kgsl_gem_object *unlock_obj; + struct drm_gem_object *obj; + struct drm_kgsl_gem_object_wait_list_entry *lock_next; + + fence->ts_valid = 0; + fence->timestamp = -1; + fence->ts_device = -1; + + /* Walk the list of buffers in this fence and clean up the */ + /* references. Note that this can cause memory allocations */ + /* to be freed */ + for (j = fence->num_buffers; j > 0; j--) { + this_fence_entry = + (struct drm_kgsl_gem_object_fence_list_entry *) + fence->buffers_in_fence.prev; + + this_fence_entry->in_use = 0; + obj = this_fence_entry->gem_obj; + unlock_obj = obj->driver_private; + + /* Delete it from the list */ + + list_del(&this_fence_entry->list); + + /* we are unlocking - see if there are other pids waiting */ + if (check_waiting) { + if (!list_empty(&unlock_obj->wait_list)) { + lock_next = + (struct drm_kgsl_gem_object_wait_list_entry *) + unlock_obj->wait_list.prev; + + list_del((struct list_head *)&lock_next->list); + + unlock_obj->lockpid = 0; + wake_up_interruptible( + &lock_next->process_wait_q); + lock_next->pid = 0; + + } else { + /* List is empty so set pid to 0 */ + unlock_obj->lockpid = 0; + } + } + + drm_gem_object_unreference(obj); + } + /* here all the buffers in the fence are released */ + /* clear the fence entry */ + fence->fence_id = ENTRY_EMPTY; +} + +int +find_empty_fence(void) +{ + int i; + + for (i = 0; i < DRM_KGSL_NUM_FENCE_ENTRIES; i++) { + if (gem_buf_fence[i].fence_id == ENTRY_EMPTY) { + gem_buf_fence[i].fence_id = fence_id++; + gem_buf_fence[i].ts_valid = 0; + INIT_LIST_HEAD(&(gem_buf_fence[i].buffers_in_fence)); + if (fence_id == 0xFFFFFFF0) + fence_id = 1; + return i; + } else { + + /* Look for entries to be cleaned up */ + if (gem_buf_fence[i].fence_id == ENTRY_NEEDS_CLEANUP) + cleanup_fence(&gem_buf_fence[i], 0); + } + } + + return ENTRY_EMPTY; +} + +int +find_fence(int index) +{ + int i; + + for (i = 0; i < DRM_KGSL_NUM_FENCE_ENTRIES; i++) { + if (gem_buf_fence[i].fence_id == index) + return i; + } + + return ENTRY_EMPTY; +} + +void +wakeup_fence_entries(struct drm_kgsl_gem_object_fence *fence) +{ + struct drm_kgsl_gem_object_fence_list_entry *this_fence_entry = NULL; + struct drm_kgsl_gem_object_wait_list_entry *lock_next; + struct drm_kgsl_gem_object *unlock_obj; + struct drm_gem_object *obj; + + /* TS has expired when we get here */ + fence->ts_valid = 0; + fence->timestamp = -1; + fence->ts_device = -1; + + list_for_each_entry(this_fence_entry, &fence->buffers_in_fence, list) { + obj = this_fence_entry->gem_obj; + unlock_obj = obj->driver_private; + + if (!list_empty(&unlock_obj->wait_list)) { + lock_next = + (struct drm_kgsl_gem_object_wait_list_entry *) + unlock_obj->wait_list.prev; + + /* Unblock the pid */ + lock_next->pid = 0; + + /* Delete it from the list */ + list_del((struct list_head *)&lock_next->list); + + unlock_obj->lockpid = 0; + wake_up_interruptible(&lock_next->process_wait_q); + + } else { + /* List is empty so set pid to 0 */ + unlock_obj->lockpid = 0; + } + } + fence->fence_id = ENTRY_NEEDS_CLEANUP; /* Mark it as needing cleanup */ +} + +static int kgsl_ts_notifier_cb(struct notifier_block *blk, + unsigned long code, void *_param) +{ + struct drm_kgsl_gem_object_fence *fence; + struct kgsl_device *device = kgsl_get_device(code); + int i; + + /* loop through the fences to see what things can be processed */ + + for (i = 0; i < DRM_KGSL_NUM_FENCE_ENTRIES; i++) { + fence = &gem_buf_fence[i]; + if (!fence->ts_valid || fence->ts_device != code) + continue; + + if (kgsl_check_timestamp(device, fence->timestamp)) + wakeup_fence_entries(fence); + } + + return 0; +} + +int +kgsl_gem_lock_handle_ioctl(struct drm_device *dev, void *data, + struct drm_file *file_priv) +{ + /* The purpose of this function is to lock a given set of handles. */ + /* The driver will maintain a list of locked handles. */ + /* If a request comes in for a handle that's locked the thread will */ + /* block until it's no longer in use. */ + + struct drm_kgsl_gem_lock_handles *args = data; + struct drm_gem_object *obj; + struct drm_kgsl_gem_object *priv; + struct drm_kgsl_gem_object_fence_list_entry *this_fence_entry = NULL; + struct drm_kgsl_gem_object_fence *fence; + struct drm_kgsl_gem_object_wait_list_entry *lock_item; + int i, j; + int result = 0; + uint32_t *lock_list; + uint32_t *work_list = NULL; + int32_t fence_index; + + /* copy in the data from user space */ + lock_list = kzalloc(sizeof(uint32_t) * args->num_handles, GFP_KERNEL); + if (!lock_list) { + DRM_ERROR("Unable allocate memory for lock list\n"); + result = -ENOMEM; + goto error; + } + + if (copy_from_user(lock_list, args->handle_list, + sizeof(uint32_t) * args->num_handles)) { + DRM_ERROR("Unable to copy the lock list from the user\n"); + result = -EFAULT; + goto free_handle_list; + } + + + work_list = lock_list; + mutex_lock(&dev->struct_mutex); + + /* build the fence for this group of handles */ + fence_index = find_empty_fence(); + if (fence_index == ENTRY_EMPTY) { + DRM_ERROR("Unable to find a empty fence\n"); + args->lock_id = 0xDEADBEEF; + result = -EFAULT; + goto out_unlock; + } + + fence = &gem_buf_fence[fence_index]; + gem_buf_fence[fence_index].num_buffers = args->num_handles; + args->lock_id = gem_buf_fence[fence_index].fence_id; + + for (j = args->num_handles; j > 0; j--, lock_list++) { + obj = drm_gem_object_lookup(dev, file_priv, *lock_list); + + if (obj == NULL) { + DRM_ERROR("Invalid GEM handle %x\n", *lock_list); + result = -EBADF; + goto out_unlock; + } + + priv = obj->driver_private; + this_fence_entry = NULL; + + /* get a fence entry to hook into the fence */ + for (i = 0; i < DRM_KGSL_NUM_FENCE_ENTRIES; i++) { + if (!priv->fence_entries[i].in_use) { + this_fence_entry = &priv->fence_entries[i]; + this_fence_entry->in_use = 1; + break; + } + } + + if (this_fence_entry == NULL) { + fence->num_buffers = 0; + fence->fence_id = ENTRY_EMPTY; + args->lock_id = 0xDEADBEAD; + result = -EFAULT; + drm_gem_object_unreference(obj); + goto out_unlock; + } + + /* We're trying to lock - add to a fence */ + list_add((struct list_head *)this_fence_entry, + &gem_buf_fence[fence_index].buffers_in_fence); + if (priv->lockpid) { + + if (priv->lockpid == args->pid) { + /* now that things are running async this */ + /* happens when an op isn't done */ + /* so it's already locked by the calling pid */ + continue; + } + + + /* if a pid already had it locked */ + /* create and add to wait list */ + for (i = 0; i < DRM_KGSL_HANDLE_WAIT_ENTRIES; i++) { + if (priv->wait_entries[i].in_use == 0) { + /* this one is empty */ + lock_item = &priv->wait_entries[i]; + lock_item->in_use = 1; + lock_item->pid = args->pid; + INIT_LIST_HEAD((struct list_head *) + &priv->wait_entries[i]); + break; + } + } + + if (i == DRM_KGSL_HANDLE_WAIT_ENTRIES) { + + result = -EFAULT; + drm_gem_object_unreference(obj); + goto out_unlock; + } + + list_add_tail((struct list_head *)&lock_item->list, + &priv->wait_list); + mutex_unlock(&dev->struct_mutex); + /* here we need to block */ + wait_event_interruptible_timeout( + priv->wait_entries[i].process_wait_q, + (priv->lockpid == 0), + msecs_to_jiffies(64)); + mutex_lock(&dev->struct_mutex); + lock_item->in_use = 0; + } + + /* Getting here means no one currently holds the lock */ + priv->lockpid = args->pid; + + args->lock_id = gem_buf_fence[fence_index].fence_id; + } + fence->lockpid = args->pid; + +out_unlock: + mutex_unlock(&dev->struct_mutex); + +free_handle_list: + kfree(work_list); + +error: + return result; +} + +int +kgsl_gem_unlock_handle_ioctl(struct drm_device *dev, void *data, + struct drm_file *file_priv) +{ + struct drm_kgsl_gem_unlock_handles *args = data; + int result = 0; + int32_t fence_index; + + mutex_lock(&dev->struct_mutex); + fence_index = find_fence(args->lock_id); + if (fence_index == ENTRY_EMPTY) { + DRM_ERROR("Invalid lock ID: %x\n", args->lock_id); + result = -EFAULT; + goto out_unlock; + } + + cleanup_fence(&gem_buf_fence[fence_index], 1); + +out_unlock: + mutex_unlock(&dev->struct_mutex); + + return result; +} + + +int +kgsl_gem_unlock_on_ts_ioctl(struct drm_device *dev, void *data, + struct drm_file *file_priv) +{ + struct drm_kgsl_gem_unlock_on_ts *args = data; + int result = 0; + int ts_done = 0; + int32_t fence_index, ts_device; + struct drm_kgsl_gem_object_fence *fence; + struct kgsl_device *device; + + if (args->type == DRM_KGSL_GEM_TS_3D) + ts_device = KGSL_DEVICE_3D0; + else if (args->type == DRM_KGSL_GEM_TS_2D) + ts_device = KGSL_DEVICE_2D0; + else { + result = -EINVAL; + goto error; + } + + device = kgsl_get_device(ts_device); + ts_done = kgsl_check_timestamp(device, args->timestamp); + + mutex_lock(&dev->struct_mutex); + + fence_index = find_fence(args->lock_id); + if (fence_index == ENTRY_EMPTY) { + DRM_ERROR("Invalid lock ID: %x\n", args->lock_id); + result = -EFAULT; + goto out_unlock; + } + + fence = &gem_buf_fence[fence_index]; + fence->ts_device = ts_device; + + if (!ts_done) + fence->ts_valid = 1; + else + cleanup_fence(fence, 1); + + +out_unlock: + mutex_unlock(&dev->struct_mutex); + +error: + return result; +} + +struct drm_ioctl_desc kgsl_drm_ioctls[] = { + DRM_IOCTL_DEF_DRV(KGSL_GEM_CREATE, kgsl_gem_create_ioctl, 0), + DRM_IOCTL_DEF_DRV(KGSL_GEM_PREP, kgsl_gem_prep_ioctl, 0), + DRM_IOCTL_DEF_DRV(KGSL_GEM_SETMEMTYPE, kgsl_gem_setmemtype_ioctl, 0), + DRM_IOCTL_DEF_DRV(KGSL_GEM_GETMEMTYPE, kgsl_gem_getmemtype_ioctl, 0), + DRM_IOCTL_DEF_DRV(KGSL_GEM_BIND_GPU, kgsl_gem_bind_gpu_ioctl, 0), + DRM_IOCTL_DEF_DRV(KGSL_GEM_UNBIND_GPU, kgsl_gem_unbind_gpu_ioctl, 0), + DRM_IOCTL_DEF_DRV(KGSL_GEM_ALLOC, kgsl_gem_alloc_ioctl, 0), + DRM_IOCTL_DEF_DRV(KGSL_GEM_MMAP, kgsl_gem_mmap_ioctl, 0), + DRM_IOCTL_DEF_DRV(KGSL_GEM_GET_BUFINFO, kgsl_gem_get_bufinfo_ioctl, 0), + DRM_IOCTL_DEF_DRV(KGSL_GEM_SET_BUFCOUNT, + kgsl_gem_set_bufcount_ioctl, 0), + DRM_IOCTL_DEF_DRV(KGSL_GEM_SET_ACTIVE, kgsl_gem_set_active_ioctl, 0), + DRM_IOCTL_DEF_DRV(KGSL_GEM_LOCK_HANDLE, + kgsl_gem_lock_handle_ioctl, 0), + DRM_IOCTL_DEF_DRV(KGSL_GEM_UNLOCK_HANDLE, + kgsl_gem_unlock_handle_ioctl, 0), + DRM_IOCTL_DEF_DRV(KGSL_GEM_UNLOCK_ON_TS, + kgsl_gem_unlock_on_ts_ioctl, 0), + DRM_IOCTL_DEF_DRV(KGSL_GEM_CREATE_FD, kgsl_gem_create_fd_ioctl, + DRM_MASTER), +}; + +static struct drm_driver driver = { + .driver_features = DRIVER_USE_PLATFORM_DEVICE | DRIVER_GEM, + .load = kgsl_drm_load, + .unload = kgsl_drm_unload, + .firstopen = kgsl_drm_firstopen, + .lastclose = kgsl_drm_lastclose, + .preclose = kgsl_drm_preclose, + .suspend = kgsl_drm_suspend, + .resume = kgsl_drm_resume, + .reclaim_buffers = drm_core_reclaim_buffers, + .gem_init_object = kgsl_gem_init_object, + .gem_free_object = kgsl_gem_free_object, + .ioctls = kgsl_drm_ioctls, + + .fops = { + .owner = THIS_MODULE, + .open = drm_open, + .release = drm_release, + .unlocked_ioctl = drm_ioctl, + .mmap = msm_drm_gem_mmap, + .poll = drm_poll, + .fasync = drm_fasync, + }, + + .name = DRIVER_NAME, + .desc = DRIVER_DESC, + .date = DRIVER_DATE, + .major = DRIVER_MAJOR, + .minor = DRIVER_MINOR, + .patchlevel = DRIVER_PATCHLEVEL, +}; + +int kgsl_drm_init(struct platform_device *dev) +{ + int i; + + driver.num_ioctls = DRM_ARRAY_SIZE(kgsl_drm_ioctls); + driver.platform_device = dev; + + INIT_LIST_HEAD(&kgsl_mem_list); + + for (i = 0; i < DRM_KGSL_NUM_FENCE_ENTRIES; i++) { + gem_buf_fence[i].num_buffers = 0; + gem_buf_fence[i].ts_valid = 0; + gem_buf_fence[i].fence_id = ENTRY_EMPTY; + } + + return drm_init(&driver); +} + +void kgsl_drm_exit(void) +{ + drm_exit(&driver); +} diff --git a/drivers/gpu/msm/kgsl_log.h b/drivers/gpu/msm/kgsl_log.h new file mode 100644 index 00000000..e816e568 --- /dev/null +++ b/drivers/gpu/msm/kgsl_log.h @@ -0,0 +1,118 @@ +/* Copyright (c) 2002,2008-2011, Code Aurora Forum. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are + * met: + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following + * disclaimer in the documentation and/or other materials provided + * with the distribution. + * * Neither the name of Code Aurora Forum, Inc. nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED "AS IS" AND ANY EXPRESS OR IMPLIED + * WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS + * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR + * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, + * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE + * OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN + * IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + */ +#ifndef __KGSL_LOG_H +#define __KGSL_LOG_H + +extern unsigned int kgsl_cff_dump_enable; + +#define KGSL_LOG_INFO(dev, lvl, fmt, args...) \ + do { \ + if ((lvl) >= 6) \ + dev_info(dev, "|%s| " fmt, \ + __func__, ##args);\ + } while (0) + +#define KGSL_LOG_WARN(dev, lvl, fmt, args...) \ + do { \ + if ((lvl) >= 4) \ + dev_warn(dev, "|%s| " fmt, \ + __func__, ##args);\ + } while (0) + +#define KGSL_LOG_ERR(dev, lvl, fmt, args...) \ + do { \ + if ((lvl) >= 3) \ + dev_err(dev, "|%s| " fmt, \ + __func__, ##args);\ + } while (0) + +#define KGSL_LOG_CRIT(dev, lvl, fmt, args...) \ + do { \ + if ((lvl) >= 2) \ + dev_crit(dev, "|%s| " fmt, \ + __func__, ##args);\ + } while (0) + +#define KGSL_LOG_POSTMORTEM_WRITE(_dev, fmt, args...) \ + do { dev_crit(_dev->dev, fmt, ##args); } while (0) + +#define KGSL_LOG_DUMP(_dev, fmt, args...) dev_err(_dev->dev, fmt, ##args) + +#define KGSL_DRV_INFO(_dev, fmt, args...) \ +KGSL_LOG_INFO(_dev->dev, _dev->drv_log, fmt, ##args) +#define KGSL_DRV_WARN(_dev, fmt, args...) \ +KGSL_LOG_WARN(_dev->dev, _dev->drv_log, fmt, ##args) +#define KGSL_DRV_ERR(_dev, fmt, args...) \ +KGSL_LOG_ERR(_dev->dev, _dev->drv_log, fmt, ##args) +#define KGSL_DRV_CRIT(_dev, fmt, args...) \ +KGSL_LOG_CRIT(_dev->dev, _dev->drv_log, fmt, ##args) + +#define KGSL_CMD_INFO(_dev, fmt, args...) \ +KGSL_LOG_INFO(_dev->dev, _dev->cmd_log, fmt, ##args) +#define KGSL_CMD_WARN(_dev, fmt, args...) \ +KGSL_LOG_WARN(_dev->dev, _dev->cmd_log, fmt, ##args) +#define KGSL_CMD_ERR(_dev, fmt, args...) \ +KGSL_LOG_ERR(_dev->dev, _dev->cmd_log, fmt, ##args) +#define KGSL_CMD_CRIT(_dev, fmt, args...) \ +KGSL_LOG_CRIT(_dev->dev, _dev->cmd_log, fmt, ##args) + +#define KGSL_CTXT_INFO(_dev, fmt, args...) \ +KGSL_LOG_INFO(_dev->dev, _dev->ctxt_log, fmt, ##args) +#define KGSL_CTXT_WARN(_dev, fmt, args...) \ +KGSL_LOG_WARN(_dev->dev, _dev->ctxt_log, fmt, ##args) +#define KGSL_CTXT_ERR(_dev, fmt, args...) \ +KGSL_LOG_ERR(_dev->dev, _dev->ctxt_log, fmt, ##args) +#define KGSL_CTXT_CRIT(_dev, fmt, args...) \ +KGSL_LOG_CRIT(_dev->dev, _dev->ctxt_log, fmt, ##args) + +#define KGSL_MEM_INFO(_dev, fmt, args...) \ +KGSL_LOG_INFO(_dev->dev, _dev->mem_log, fmt, ##args) +#define KGSL_MEM_WARN(_dev, fmt, args...) \ +KGSL_LOG_WARN(_dev->dev, _dev->mem_log, fmt, ##args) +#define KGSL_MEM_ERR(_dev, fmt, args...) \ +KGSL_LOG_ERR(_dev->dev, _dev->mem_log, fmt, ##args) +#define KGSL_MEM_CRIT(_dev, fmt, args...) \ +KGSL_LOG_CRIT(_dev->dev, _dev->mem_log, fmt, ##args) + +#define KGSL_PWR_INFO(_dev, fmt, args...) \ +KGSL_LOG_INFO(_dev->dev, _dev->pwr_log, fmt, ##args) +#define KGSL_PWR_WARN(_dev, fmt, args...) \ +KGSL_LOG_WARN(_dev->dev, _dev->pwr_log, fmt, ##args) +#define KGSL_PWR_ERR(_dev, fmt, args...) \ +KGSL_LOG_ERR(_dev->dev, _dev->pwr_log, fmt, ##args) +#define KGSL_PWR_CRIT(_dev, fmt, args...) \ +KGSL_LOG_CRIT(_dev->dev, _dev->pwr_log, fmt, ##args) + +/* Core error messages - these are for core KGSL functions that have + no device associated with them (such as memory) */ + +#define KGSL_CORE_ERR(fmt, args...) \ +pr_err("kgsl: %s: " fmt, __func__, ##args) + +#endif /* __KGSL_LOG_H */ diff --git a/drivers/gpu/msm/kgsl_mmu.c b/drivers/gpu/msm/kgsl_mmu.c new file mode 100644 index 00000000..bc35c411 --- /dev/null +++ b/drivers/gpu/msm/kgsl_mmu.c @@ -0,0 +1,1141 @@ +/* Copyright (c) 2002,2007-2011, Code Aurora Forum. All rights reserved. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 and + * only version 2 as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + */ +#include +#include +#include +#include +#include +#include + +#include "kgsl.h" +#include "kgsl_mmu.h" + +#define KGSL_MMU_ALIGN_SHIFT 13 +#define KGSL_MMU_ALIGN_MASK (~((1 << KGSL_MMU_ALIGN_SHIFT) - 1)) + +#define GSL_PT_PAGE_BITS_MASK 0x00000007 +#define GSL_PT_PAGE_ADDR_MASK PAGE_MASK + +#define GSL_MMU_INT_MASK \ + (MH_INTERRUPT_MASK__AXI_READ_ERROR | \ + MH_INTERRUPT_MASK__AXI_WRITE_ERROR) + +static ssize_t +sysfs_show_ptpool_entries(struct kobject *kobj, + struct kobj_attribute *attr, + char *buf) +{ + return snprintf(buf, PAGE_SIZE, "%d\n", kgsl_driver.ptpool.entries); +} + +static ssize_t +sysfs_show_ptpool_min(struct kobject *kobj, + struct kobj_attribute *attr, + char *buf) +{ + return snprintf(buf, PAGE_SIZE, "%d\n", + kgsl_driver.ptpool.static_entries); +} + +static ssize_t +sysfs_show_ptpool_chunks(struct kobject *kobj, + struct kobj_attribute *attr, + char *buf) +{ + return snprintf(buf, PAGE_SIZE, "%d\n", kgsl_driver.ptpool.chunks); +} + +static ssize_t +sysfs_show_ptpool_ptsize(struct kobject *kobj, + struct kobj_attribute *attr, + char *buf) +{ + return snprintf(buf, PAGE_SIZE, "%d\n", kgsl_driver.ptpool.ptsize); +} + +static struct kobj_attribute attr_ptpool_entries = { + .attr = { .name = "ptpool_entries", .mode = 0444 }, + .show = sysfs_show_ptpool_entries, + .store = NULL, +}; + +static struct kobj_attribute attr_ptpool_min = { + .attr = { .name = "ptpool_min", .mode = 0444 }, + .show = sysfs_show_ptpool_min, + .store = NULL, +}; + +static struct kobj_attribute attr_ptpool_chunks = { + .attr = { .name = "ptpool_chunks", .mode = 0444 }, + .show = sysfs_show_ptpool_chunks, + .store = NULL, +}; + +static struct kobj_attribute attr_ptpool_ptsize = { + .attr = { .name = "ptpool_ptsize", .mode = 0444 }, + .show = sysfs_show_ptpool_ptsize, + .store = NULL, +}; + +static struct attribute *ptpool_attrs[] = { + &attr_ptpool_entries.attr, + &attr_ptpool_min.attr, + &attr_ptpool_chunks.attr, + &attr_ptpool_ptsize.attr, + NULL, +}; + +static struct attribute_group ptpool_attr_group = { + .attrs = ptpool_attrs, +}; + +static int +_kgsl_ptpool_add_entries(struct kgsl_ptpool *pool, int count, int dynamic) +{ + struct kgsl_ptpool_chunk *chunk; + size_t size = ALIGN(count * pool->ptsize, PAGE_SIZE); + + BUG_ON(count == 0); + + if (get_order(size) >= MAX_ORDER) { + KGSL_CORE_ERR("ptpool allocation is too big: %d\n", size); + return -EINVAL; + } + + chunk = kzalloc(sizeof(*chunk), GFP_KERNEL); + if (chunk == NULL) { + KGSL_CORE_ERR("kzalloc(%d) failed\n", sizeof(*chunk)); + return -ENOMEM; + } + + chunk->size = size; + chunk->count = count; + chunk->dynamic = dynamic; + + chunk->data = dma_alloc_coherent(NULL, size, + &chunk->phys, GFP_KERNEL); + + if (chunk->data == NULL) { + KGSL_CORE_ERR("dma_alloc_coherent(%d) failed\n", size); + goto err; + } + + chunk->bitmap = kzalloc(BITS_TO_LONGS(count) * 4, GFP_KERNEL); + + if (chunk->bitmap == NULL) { + KGSL_CORE_ERR("kzalloc(%d) failed\n", + BITS_TO_LONGS(count) * 4); + goto err_dma; + } + + list_add_tail(&chunk->list, &pool->list); + + pool->chunks++; + pool->entries += count; + + if (!dynamic) + pool->static_entries += count; + + return 0; + +err_dma: + dma_free_coherent(NULL, chunk->size, chunk->data, chunk->phys); +err: + kfree(chunk); + return -ENOMEM; +} + +static void * +_kgsl_ptpool_get_entry(struct kgsl_ptpool *pool, unsigned int *physaddr) +{ + struct kgsl_ptpool_chunk *chunk; + + list_for_each_entry(chunk, &pool->list, list) { + int bit = find_first_zero_bit(chunk->bitmap, chunk->count); + + if (bit >= chunk->count) + continue; + + set_bit(bit, chunk->bitmap); + *physaddr = chunk->phys + (bit * pool->ptsize); + + return chunk->data + (bit * pool->ptsize); + } + + return NULL; +} + +/** + * kgsl_ptpool_add + * @pool: A pointer to a ptpool structure + * @entries: Number of entries to add + * + * Add static entries to the pagetable pool. + */ + +int +kgsl_ptpool_add(struct kgsl_ptpool *pool, int count) +{ + int ret = 0; + BUG_ON(count == 0); + + mutex_lock(&pool->lock); + + /* Only 4MB can be allocated in one chunk, so larger allocations + need to be split into multiple sections */ + + while (count) { + int entries = ((count * pool->ptsize) > SZ_4M) ? + SZ_4M / pool->ptsize : count; + + /* Add the entries as static, i.e. they don't ever stand + a chance of being removed */ + + ret = _kgsl_ptpool_add_entries(pool, entries, 0); + if (ret) + break; + + count -= entries; + } + + mutex_unlock(&pool->lock); + return ret; +} + +/** + * kgsl_ptpool_alloc + * @pool: A pointer to a ptpool structure + * @addr: A pointer to store the physical address of the chunk + * + * Allocate a pagetable from the pool. Returns the virtual address + * of the pagetable, the physical address is returned in physaddr + */ + +void *kgsl_ptpool_alloc(struct kgsl_ptpool *pool, unsigned int *physaddr) +{ + void *addr = NULL; + int ret; + + mutex_lock(&pool->lock); + addr = _kgsl_ptpool_get_entry(pool, physaddr); + if (addr) + goto done; + + /* Add a chunk for 1 more pagetable and mark it as dynamic */ + ret = _kgsl_ptpool_add_entries(pool, 1, 1); + + if (ret) + goto done; + + addr = _kgsl_ptpool_get_entry(pool, physaddr); +done: + mutex_unlock(&pool->lock); + return addr; +} + +static inline void _kgsl_ptpool_rm_chunk(struct kgsl_ptpool_chunk *chunk) +{ + list_del(&chunk->list); + + if (chunk->data) + dma_free_coherent(NULL, chunk->size, chunk->data, + chunk->phys); + kfree(chunk->bitmap); + kfree(chunk); +} + +/** + * kgsl_ptpool_free + * @pool: A pointer to a ptpool structure + * @addr: A pointer to the virtual address to free + * + * Free a pagetable allocated from the pool + */ + +void kgsl_ptpool_free(struct kgsl_ptpool *pool, void *addr) +{ + struct kgsl_ptpool_chunk *chunk, *tmp; + + if (pool == NULL || addr == NULL) + return; + + mutex_lock(&pool->lock); + list_for_each_entry_safe(chunk, tmp, &pool->list, list) { + if (addr >= chunk->data && + addr < chunk->data + chunk->size) { + int bit = ((unsigned long) (addr - chunk->data)) / + pool->ptsize; + + clear_bit(bit, chunk->bitmap); + memset(addr, 0, pool->ptsize); + + if (chunk->dynamic && + bitmap_empty(chunk->bitmap, chunk->count)) + _kgsl_ptpool_rm_chunk(chunk); + + break; + } + } + + mutex_unlock(&pool->lock); +} + +void kgsl_ptpool_destroy(struct kgsl_ptpool *pool) +{ + struct kgsl_ptpool_chunk *chunk, *tmp; + + if (pool == NULL) + return; + + mutex_lock(&pool->lock); + list_for_each_entry_safe(chunk, tmp, &pool->list, list) + _kgsl_ptpool_rm_chunk(chunk); + mutex_unlock(&pool->lock); + + memset(pool, 0, sizeof(*pool)); +} + +/** + * kgsl_ptpool_init + * @pool: A pointer to a ptpool structure to initialize + * @ptsize: The size of each pagetable entry + * @entries: The number of inital entries to add to the pool + * + * Initalize a pool and allocate an initial chunk of entries. + */ + +int kgsl_ptpool_init(struct kgsl_ptpool *pool, int ptsize, int entries) +{ + int ret = 0; + BUG_ON(ptsize == 0); + + pool->ptsize = ptsize; + mutex_init(&pool->lock); + INIT_LIST_HEAD(&pool->list); + + if (entries) { + ret = kgsl_ptpool_add(pool, entries); + if (ret) + return ret; + } + + return sysfs_create_group(kgsl_driver.ptkobj, &ptpool_attr_group); +} + +/* pt_mutex needs to be held in this function */ + +static struct kgsl_pagetable * +kgsl_get_pagetable(unsigned long name) +{ + struct kgsl_pagetable *pt; + + list_for_each_entry(pt, &kgsl_driver.pagetable_list, list) { + if (pt->name == name) + return pt; + } + + return NULL; +} + +static struct kgsl_pagetable * +_get_pt_from_kobj(struct kobject *kobj) +{ + unsigned long ptname; + + if (!kobj) + return NULL; + + if (sscanf(kobj->name, "%ld", &ptname) != 1) + return NULL; + + return kgsl_get_pagetable(ptname); +} + +static ssize_t +sysfs_show_entries(struct kobject *kobj, + struct kobj_attribute *attr, + char *buf) +{ + struct kgsl_pagetable *pt; + int ret = 0; + + mutex_lock(&kgsl_driver.pt_mutex); + pt = _get_pt_from_kobj(kobj); + + if (pt) + ret += snprintf(buf, PAGE_SIZE, "%d\n", pt->stats.entries); + + mutex_unlock(&kgsl_driver.pt_mutex); + return ret; +} + +static ssize_t +sysfs_show_mapped(struct kobject *kobj, + struct kobj_attribute *attr, + char *buf) +{ + struct kgsl_pagetable *pt; + int ret = 0; + + mutex_lock(&kgsl_driver.pt_mutex); + pt = _get_pt_from_kobj(kobj); + + if (pt) + ret += snprintf(buf, PAGE_SIZE, "%d\n", pt->stats.mapped); + + mutex_unlock(&kgsl_driver.pt_mutex); + return ret; +} + +static ssize_t +sysfs_show_va_range(struct kobject *kobj, + struct kobj_attribute *attr, + char *buf) +{ + struct kgsl_pagetable *pt; + int ret = 0; + + mutex_lock(&kgsl_driver.pt_mutex); + pt = _get_pt_from_kobj(kobj); + + if (pt) + ret += snprintf(buf, PAGE_SIZE, "0x%x\n", pt->va_range); + + mutex_unlock(&kgsl_driver.pt_mutex); + return ret; +} + +static ssize_t +sysfs_show_max_mapped(struct kobject *kobj, + struct kobj_attribute *attr, + char *buf) +{ + struct kgsl_pagetable *pt; + int ret = 0; + + mutex_lock(&kgsl_driver.pt_mutex); + pt = _get_pt_from_kobj(kobj); + + if (pt) + ret += snprintf(buf, PAGE_SIZE, "%d\n", pt->stats.max_mapped); + + mutex_unlock(&kgsl_driver.pt_mutex); + return ret; +} + +static ssize_t +sysfs_show_max_entries(struct kobject *kobj, + struct kobj_attribute *attr, + char *buf) +{ + struct kgsl_pagetable *pt; + int ret = 0; + + mutex_lock(&kgsl_driver.pt_mutex); + pt = _get_pt_from_kobj(kobj); + + if (pt) + ret += snprintf(buf, PAGE_SIZE, "%d\n", pt->stats.max_entries); + + mutex_unlock(&kgsl_driver.pt_mutex); + return ret; +} + +static struct kobj_attribute attr_entries = { + .attr = { .name = "entries", .mode = 0444 }, + .show = sysfs_show_entries, + .store = NULL, +}; + +static struct kobj_attribute attr_mapped = { + .attr = { .name = "mapped", .mode = 0444 }, + .show = sysfs_show_mapped, + .store = NULL, +}; + +static struct kobj_attribute attr_va_range = { + .attr = { .name = "va_range", .mode = 0444 }, + .show = sysfs_show_va_range, + .store = NULL, +}; + +static struct kobj_attribute attr_max_mapped = { + .attr = { .name = "max_mapped", .mode = 0444 }, + .show = sysfs_show_max_mapped, + .store = NULL, +}; + +static struct kobj_attribute attr_max_entries = { + .attr = { .name = "max_entries", .mode = 0444 }, + .show = sysfs_show_max_entries, + .store = NULL, +}; + +static struct attribute *pagetable_attrs[] = { + &attr_entries.attr, + &attr_mapped.attr, + &attr_va_range.attr, + &attr_max_mapped.attr, + &attr_max_entries.attr, + NULL, +}; + +static struct attribute_group pagetable_attr_group = { + .attrs = pagetable_attrs, +}; + +static void +pagetable_remove_sysfs_objects(struct kgsl_pagetable *pagetable) +{ + if (pagetable->kobj) + sysfs_remove_group(pagetable->kobj, + &pagetable_attr_group); + + kobject_put(pagetable->kobj); +} + +static int +pagetable_add_sysfs_objects(struct kgsl_pagetable *pagetable) +{ + char ptname[16]; + int ret = -ENOMEM; + + snprintf(ptname, sizeof(ptname), "%d", pagetable->name); + pagetable->kobj = kobject_create_and_add(ptname, + kgsl_driver.ptkobj); + if (pagetable->kobj == NULL) + goto err; + + ret = sysfs_create_group(pagetable->kobj, &pagetable_attr_group); + +err: + if (ret) { + if (pagetable->kobj) + kobject_put(pagetable->kobj); + + pagetable->kobj = NULL; + } + + return ret; +} + +static inline uint32_t +kgsl_pt_entry_get(struct kgsl_pagetable *pt, uint32_t va) +{ + return (va - pt->va_base) >> PAGE_SHIFT; +} + +static inline void +kgsl_pt_map_set(struct kgsl_pagetable *pt, uint32_t pte, uint32_t val) +{ + uint32_t *baseptr = (uint32_t *)pt->base.hostptr; + + writel_relaxed(val, &baseptr[pte]); +} + +static inline uint32_t +kgsl_pt_map_getaddr(struct kgsl_pagetable *pt, uint32_t pte) +{ + uint32_t *baseptr = (uint32_t *)pt->base.hostptr; + uint32_t ret = readl_relaxed(&baseptr[pte]) & GSL_PT_PAGE_ADDR_MASK; + return ret; +} + +void kgsl_mh_intrcallback(struct kgsl_device *device) +{ + unsigned int status = 0; + unsigned int reg; + + kgsl_regread_isr(device, device->mmu.reg.interrupt_status, &status); + + if (status & MH_INTERRUPT_MASK__AXI_READ_ERROR) { + kgsl_regread_isr(device, device->mmu.reg.axi_error, ®); + KGSL_MEM_CRIT(device, "axi read error interrupt: %08x\n", reg); + } else if (status & MH_INTERRUPT_MASK__AXI_WRITE_ERROR) { + kgsl_regread_isr(device, device->mmu.reg.axi_error, ®); + KGSL_MEM_CRIT(device, "axi write error interrupt: %08x\n", reg); + } else if (status & MH_INTERRUPT_MASK__MMU_PAGE_FAULT) { + kgsl_regread_isr(device, device->mmu.reg.page_fault, ®); + KGSL_MEM_CRIT(device, "mmu page fault interrupt: %08x\n", reg); + } else { + KGSL_MEM_WARN(device, + "bad bits in REG_MH_INTERRUPT_STATUS %08x\n", status); + } + + kgsl_regwrite_isr(device, device->mmu.reg.interrupt_clear, status); + + /*TODO: figure out how to handle errror interupts. + * specifically, page faults should probably nuke the client that + * caused them, but we don't have enough info to figure that out yet. + */ +} +EXPORT_SYMBOL(kgsl_mh_intrcallback); + +static int kgsl_setup_pt(struct kgsl_pagetable *pt) +{ + int i = 0; + int status = 0; + + for (i = 0; i < KGSL_DEVICE_MAX; i++) { + struct kgsl_device *device = kgsl_driver.devp[i]; + if (device) { + status = device->ftbl.device_setup_pt(device, pt); + if (status) + goto error_pt; + } + } + return status; +error_pt: + while (i >= 0) { + struct kgsl_device *device = kgsl_driver.devp[i]; + if (device) + device->ftbl.device_cleanup_pt(device, pt); + i--; + } + return status; +} + +static int kgsl_cleanup_pt(struct kgsl_pagetable *pt) +{ + int i; + for (i = 0; i < KGSL_DEVICE_MAX; i++) { + struct kgsl_device *device = kgsl_driver.devp[i]; + if (device) + device->ftbl.device_cleanup_pt(device, pt); + } + return 0; +} + +static struct kgsl_pagetable *kgsl_mmu_createpagetableobject( + unsigned int name) +{ + int status = 0; + struct kgsl_pagetable *pagetable = NULL; + + pagetable = kzalloc(sizeof(struct kgsl_pagetable), GFP_KERNEL); + if (pagetable == NULL) { + KGSL_CORE_ERR("kzalloc(%d) failed\n", + sizeof(struct kgsl_pagetable)); + return NULL; + } + + pagetable->refcnt = 1; + + spin_lock_init(&pagetable->lock); + pagetable->tlb_flags = 0; + pagetable->name = name; + pagetable->va_base = KGSL_PAGETABLE_BASE; + pagetable->va_range = CONFIG_MSM_KGSL_PAGE_TABLE_SIZE; + pagetable->last_superpte = 0; + pagetable->max_entries = KGSL_PAGETABLE_ENTRIES(pagetable->va_range); + + pagetable->tlbflushfilter.size = (pagetable->va_range / + (PAGE_SIZE * GSL_PT_SUPER_PTE * 8)) + 1; + pagetable->tlbflushfilter.base = (unsigned int *) + kzalloc(pagetable->tlbflushfilter.size, GFP_KERNEL); + if (!pagetable->tlbflushfilter.base) { + KGSL_CORE_ERR("kzalloc(%d) failed\n", + pagetable->tlbflushfilter.size); + goto err_alloc; + } + GSL_TLBFLUSH_FILTER_RESET(); + + pagetable->pool = gen_pool_create(PAGE_SHIFT, -1); + if (pagetable->pool == NULL) { + KGSL_CORE_ERR("gen_pool_create(%d) failed\n", PAGE_SHIFT); + goto err_flushfilter; + } + + if (gen_pool_add(pagetable->pool, pagetable->va_base, + pagetable->va_range, -1)) { + KGSL_CORE_ERR("gen_pool_add failed\n"); + goto err_pool; + } + + pagetable->base.hostptr = kgsl_ptpool_alloc(&kgsl_driver.ptpool, + &pagetable->base.physaddr); + + if (pagetable->base.hostptr == NULL) + goto err_pool; + + /* ptpool allocations are from coherent memory, so update the + device statistics acordingly */ + + KGSL_STATS_ADD(KGSL_PAGETABLE_SIZE, kgsl_driver.stats.coherent, + kgsl_driver.stats.coherent_max); + + pagetable->base.gpuaddr = pagetable->base.physaddr; + pagetable->base.size = KGSL_PAGETABLE_SIZE; + + status = kgsl_setup_pt(pagetable); + if (status) + goto err_free_sharedmem; + + list_add(&pagetable->list, &kgsl_driver.pagetable_list); + + /* Create the sysfs entries */ + pagetable_add_sysfs_objects(pagetable); + + return pagetable; + +err_free_sharedmem: + kgsl_ptpool_free(&kgsl_driver.ptpool, &pagetable->base.hostptr); +err_pool: + gen_pool_destroy(pagetable->pool); +err_flushfilter: + kfree(pagetable->tlbflushfilter.base); +err_alloc: + kfree(pagetable); + + return NULL; +} + +static void kgsl_mmu_destroypagetable(struct kgsl_pagetable *pagetable) +{ + list_del(&pagetable->list); + + pagetable_remove_sysfs_objects(pagetable); + + kgsl_cleanup_pt(pagetable); + + kgsl_ptpool_free(&kgsl_driver.ptpool, pagetable->base.hostptr); + + kgsl_driver.stats.coherent -= KGSL_PAGETABLE_SIZE; + + if (pagetable->pool) { + gen_pool_destroy(pagetable->pool); + pagetable->pool = NULL; + } + + if (pagetable->tlbflushfilter.base) { + pagetable->tlbflushfilter.size = 0; + kfree(pagetable->tlbflushfilter.base); + pagetable->tlbflushfilter.base = NULL; + } + + kfree(pagetable); +} + +struct kgsl_pagetable *kgsl_mmu_getpagetable(unsigned long name) +{ + struct kgsl_pagetable *pt; + + mutex_lock(&kgsl_driver.pt_mutex); + + pt = kgsl_get_pagetable(name); + + if (pt) { + spin_lock(&pt->lock); + pt->refcnt++; + spin_unlock(&pt->lock); + goto done; + } + + pt = kgsl_mmu_createpagetableobject(name); + +done: + mutex_unlock(&kgsl_driver.pt_mutex); + return pt; +} + +void kgsl_mmu_putpagetable(struct kgsl_pagetable *pagetable) +{ + bool dead; + if (pagetable == NULL) + return; + + mutex_lock(&kgsl_driver.pt_mutex); + + spin_lock(&pagetable->lock); + dead = (--pagetable->refcnt) == 0; + spin_unlock(&pagetable->lock); + + if (dead) + kgsl_mmu_destroypagetable(pagetable); + + mutex_unlock(&kgsl_driver.pt_mutex); +} + +int kgsl_mmu_setstate(struct kgsl_device *device, + struct kgsl_pagetable *pagetable) +{ + int status = 0; + struct kgsl_mmu *mmu = &device->mmu; + + if (mmu->flags & KGSL_FLAGS_STARTED) { + /* page table not current, then setup mmu to use new + * specified page table + */ + if (mmu->hwpagetable != pagetable) { + mmu->hwpagetable = pagetable; + spin_lock(&mmu->hwpagetable->lock); + mmu->hwpagetable->tlb_flags &= ~(1<id); + spin_unlock(&mmu->hwpagetable->lock); + + /* call device specific set page table */ + status = kgsl_setstate(mmu->device, + KGSL_MMUFLAGS_TLBFLUSH | + KGSL_MMUFLAGS_PTUPDATE); + + } + } + + return status; +} +EXPORT_SYMBOL(kgsl_mmu_setstate); + +int kgsl_mmu_init(struct kgsl_device *device) +{ + /* + * intialize device mmu + * + * call this with the global lock held + */ + int status = 0; + struct kgsl_mmu *mmu = &device->mmu; + + mmu->device = device; + + /* make sure aligned to pagesize */ + BUG_ON(mmu->mpu_base & (PAGE_SIZE - 1)); + BUG_ON((mmu->mpu_base + mmu->mpu_range) & (PAGE_SIZE - 1)); + + /* sub-client MMU lookups require address translation */ + if ((mmu->config & ~0x1) > 0) { + /*make sure virtual address range is a multiple of 64Kb */ + BUG_ON(CONFIG_MSM_KGSL_PAGE_TABLE_SIZE & ((1 << 16) - 1)); + + /* allocate memory used for completing r/w operations that + * cannot be mapped by the MMU + */ + status = kgsl_allocate_contig(&mmu->dummyspace, 64); + if (!status) + kgsl_sharedmem_set(&mmu->dummyspace, 0, 0, + mmu->dummyspace.size); + } + + return status; +} + +int kgsl_mmu_start(struct kgsl_device *device) +{ + /* + * intialize device mmu + * + * call this with the global lock held + */ + int status; + struct kgsl_mmu *mmu = &device->mmu; + + if (mmu->flags & KGSL_FLAGS_STARTED) + return 0; + + /* MMU not enabled */ + if ((mmu->config & 0x1) == 0) + return 0; + + mmu->flags |= KGSL_FLAGS_STARTED; + + /* setup MMU and sub-client behavior */ + kgsl_regwrite(device, device->mmu.reg.config, mmu->config); + + /* enable axi interrupts */ + kgsl_regwrite(device, device->mmu.reg.interrupt_mask, + GSL_MMU_INT_MASK); + + /* idle device */ + kgsl_idle(device, KGSL_TIMEOUT_DEFAULT); + + /* define physical memory range accessible by the core */ + kgsl_regwrite(device, device->mmu.reg.mpu_base, mmu->mpu_base); + kgsl_regwrite(device, device->mmu.reg.mpu_end, + mmu->mpu_base + mmu->mpu_range); + + /* enable axi interrupts */ + kgsl_regwrite(device, device->mmu.reg.interrupt_mask, + GSL_MMU_INT_MASK | MH_INTERRUPT_MASK__MMU_PAGE_FAULT); + + /* sub-client MMU lookups require address translation */ + if ((mmu->config & ~0x1) > 0) { + + kgsl_sharedmem_set(&mmu->dummyspace, 0, 0, + mmu->dummyspace.size); + + /* TRAN_ERROR needs a 32 byte (32 byte aligned) chunk of memory + * to complete transactions in case of an MMU fault. Note that + * we'll leave the bottom 32 bytes of the dummyspace for other + * purposes (e.g. use it when dummy read cycles are needed + * for other blocks */ + kgsl_regwrite(device, device->mmu.reg.tran_error, + mmu->dummyspace.physaddr + 32); + + if (mmu->defaultpagetable == NULL) + mmu->defaultpagetable = + kgsl_mmu_getpagetable(KGSL_MMU_GLOBAL_PT); + mmu->hwpagetable = mmu->defaultpagetable; + + kgsl_regwrite(device, device->mmu.reg.pt_page, + mmu->hwpagetable->base.gpuaddr); + kgsl_regwrite(device, device->mmu.reg.va_range, + (mmu->hwpagetable->va_base | + (mmu->hwpagetable->va_range >> 16))); + status = kgsl_setstate(device, KGSL_MMUFLAGS_TLBFLUSH); + if (status) { + KGSL_MEM_ERR(device, "Failed to setstate TLBFLUSH\n"); + goto error; + } + } + + return 0; +error: + /* disable MMU */ + kgsl_regwrite(device, device->mmu.reg.interrupt_mask, 0); + kgsl_regwrite(device, device->mmu.reg.config, 0x00000000); + return status; +} +EXPORT_SYMBOL(kgsl_mmu_start); + +unsigned int kgsl_virtaddr_to_physaddr(void *virtaddr) +{ + unsigned int physaddr = 0; + pgd_t *pgd_ptr = NULL; + pmd_t *pmd_ptr = NULL; + pte_t *pte_ptr = NULL, pte; + + pgd_ptr = pgd_offset(current->mm, (unsigned long) virtaddr); + if (pgd_none(*pgd) || pgd_bad(*pgd)) { + KGSL_CORE_ERR("Invalid pgd entry\n"); + return 0; + } + + pmd_ptr = pmd_offset(pgd_ptr, (unsigned long) virtaddr); + if (pmd_none(*pmd_ptr) || pmd_bad(*pmd_ptr)) { + KGSL_CORE_ERR("Invalid pmd entry\n"); + return 0; + } + + pte_ptr = pte_offset_map(pmd_ptr, (unsigned long) virtaddr); + if (!pte_ptr) { + KGSL_CORE_ERR("pt_offset_map failed\n"); + return 0; + } + pte = *pte_ptr; + physaddr = pte_pfn(pte); + pte_unmap(pte_ptr); + physaddr <<= PAGE_SHIFT; + return physaddr; +} + +int +kgsl_mmu_map(struct kgsl_pagetable *pagetable, + struct kgsl_memdesc *memdesc, + unsigned int protflags) +{ + int numpages; + unsigned int pte, ptefirst, ptelast, physaddr; + int flushtlb; + unsigned int offset = 0; + + BUG_ON(protflags & ~(GSL_PT_PAGE_RV | GSL_PT_PAGE_WV)); + BUG_ON(protflags == 0); + + memdesc->gpuaddr = gen_pool_alloc_aligned(pagetable->pool, + memdesc->size, KGSL_MMU_ALIGN_SHIFT); + + if (memdesc->gpuaddr == 0) { + KGSL_CORE_ERR("gen_pool_alloc(%d) failed\n", memdesc->size); + KGSL_CORE_ERR(" [%d] allocated=%d, entries=%d\n", + pagetable->name, pagetable->stats.mapped, + pagetable->stats.entries); + return -ENOMEM; + } + + numpages = (memdesc->size >> PAGE_SHIFT); + + ptefirst = kgsl_pt_entry_get(pagetable, memdesc->gpuaddr); + ptelast = ptefirst + numpages; + + pte = ptefirst; + flushtlb = 0; + + /* tlb needs to be flushed when the first and last pte are not at + * superpte boundaries */ + if ((ptefirst & (GSL_PT_SUPER_PTE - 1)) != 0 || + ((ptelast + 1) & (GSL_PT_SUPER_PTE-1)) != 0) + flushtlb = 1; + + spin_lock(&pagetable->lock); + for (pte = ptefirst; pte < ptelast; pte++, offset += PAGE_SIZE) { +#ifdef VERBOSE_DEBUG + /* check if PTE exists */ + uint32_t val = kgsl_pt_map_getaddr(pagetable, pte); + BUG_ON(val != 0 && val != GSL_PT_PAGE_DIRTY); +#endif + if ((pte & (GSL_PT_SUPER_PTE-1)) == 0) + if (GSL_TLBFLUSH_FILTER_ISDIRTY(pte / GSL_PT_SUPER_PTE)) + flushtlb = 1; + /* mark pte as in use */ + + physaddr = memdesc->ops->physaddr(memdesc, offset); + BUG_ON(physaddr == 0); + kgsl_pt_map_set(pagetable, pte, physaddr | protflags); + } + + /* Keep track of the statistics for the sysfs files */ + + KGSL_STATS_ADD(1, pagetable->stats.entries, + pagetable->stats.max_entries); + + KGSL_STATS_ADD(memdesc->size, pagetable->stats.mapped, + pagetable->stats.max_mapped); + + /* Post all writes to the pagetable */ + wmb(); + + /* Invalidate tlb only if current page table used by GPU is the + * pagetable that we used to allocate */ + if (flushtlb) { + /*set all devices as needing flushing*/ + pagetable->tlb_flags = UINT_MAX; + GSL_TLBFLUSH_FILTER_RESET(); + } + spin_unlock(&pagetable->lock); + + return 0; +} + +int +kgsl_mmu_unmap(struct kgsl_pagetable *pagetable, + struct kgsl_memdesc *memdesc) +{ + unsigned int numpages; + unsigned int pte, ptefirst, ptelast, superpte; + unsigned int range = memdesc->size; + + /* All GPU addresses as assigned are page aligned, but some + functions purturb the gpuaddr with an offset, so apply the + mask here to make sure we have the right address */ + + unsigned int gpuaddr = memdesc->gpuaddr & KGSL_MMU_ALIGN_MASK; + + if (range == 0 || gpuaddr == 0) + return 0; + + numpages = (range >> PAGE_SHIFT); + if (range & (PAGE_SIZE - 1)) + numpages++; + + ptefirst = kgsl_pt_entry_get(pagetable, gpuaddr); + ptelast = ptefirst + numpages; + + spin_lock(&pagetable->lock); + superpte = ptefirst - (ptefirst & (GSL_PT_SUPER_PTE-1)); + GSL_TLBFLUSH_FILTER_SETDIRTY(superpte / GSL_PT_SUPER_PTE); + for (pte = ptefirst; pte < ptelast; pte++) { +#ifdef VERBOSE_DEBUG + /* check if PTE exists */ + BUG_ON(!kgsl_pt_map_getaddr(pagetable, pte)); +#endif + kgsl_pt_map_set(pagetable, pte, GSL_PT_PAGE_DIRTY); + superpte = pte - (pte & (GSL_PT_SUPER_PTE - 1)); + if (pte == superpte) + GSL_TLBFLUSH_FILTER_SETDIRTY(superpte / + GSL_PT_SUPER_PTE); + } + + /* Remove the statistics */ + pagetable->stats.entries--; + pagetable->stats.mapped -= range; + + /* Post all writes to the pagetable */ + wmb(); + + spin_unlock(&pagetable->lock); + + gen_pool_free(pagetable->pool, gpuaddr, range); + + return 0; +} +EXPORT_SYMBOL(kgsl_mmu_unmap); + +int kgsl_mmu_map_global(struct kgsl_pagetable *pagetable, + struct kgsl_memdesc *memdesc, unsigned int protflags) +{ + int result = -EINVAL; + unsigned int gpuaddr = 0; + + if (memdesc == NULL) { + KGSL_CORE_ERR("invalid memdesc\n"); + goto error; + } + + gpuaddr = memdesc->gpuaddr; + + result = kgsl_mmu_map(pagetable, memdesc, protflags); + if (result) + goto error; + + /*global mappings must have the same gpu address in all pagetables*/ + if (gpuaddr && gpuaddr != memdesc->gpuaddr) { + KGSL_CORE_ERR("pt %p addr mismatch phys 0x%08x" + "gpu 0x%0x 0x%08x", pagetable, memdesc->physaddr, + gpuaddr, memdesc->gpuaddr); + goto error_unmap; + } + return result; +error_unmap: + kgsl_mmu_unmap(pagetable, memdesc); +error: + return result; +} +EXPORT_SYMBOL(kgsl_mmu_map_global); + +int kgsl_mmu_stop(struct kgsl_device *device) +{ + /* + * stop device mmu + * + * call this with the global lock held + */ + struct kgsl_mmu *mmu = &device->mmu; + + if (mmu->flags & KGSL_FLAGS_STARTED) { + /* disable mh interrupts */ + /* disable MMU */ + kgsl_regwrite(device, device->mmu.reg.interrupt_mask, 0); + kgsl_regwrite(device, device->mmu.reg.config, 0x00000000); + + mmu->flags &= ~KGSL_FLAGS_STARTED; + } + + return 0; +} +EXPORT_SYMBOL(kgsl_mmu_stop); + +int kgsl_mmu_close(struct kgsl_device *device) +{ + /* + * close device mmu + * + * call this with the global lock held + */ + struct kgsl_mmu *mmu = &device->mmu; + + if (mmu->dummyspace.gpuaddr) + kgsl_sharedmem_free(&mmu->dummyspace); + + if (mmu->defaultpagetable) + kgsl_mmu_putpagetable(mmu->defaultpagetable); + + return 0; +} diff --git a/drivers/gpu/msm/kgsl_mmu.h b/drivers/gpu/msm/kgsl_mmu.h new file mode 100644 index 00000000..4a67ea67 --- /dev/null +++ b/drivers/gpu/msm/kgsl_mmu.h @@ -0,0 +1,267 @@ +/* Copyright (c) 2002,2007-2011, Code Aurora Forum. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are + * met: + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following + * disclaimer in the documentation and/or other materials provided + * with the distribution. + * * Neither the name of Code Aurora Forum, Inc. nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED "AS IS" AND ANY EXPRESS OR IMPLIED + * WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS + * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR + * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, + * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE + * OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN + * IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + */ +#ifndef __KGSL_MMU_H +#define __KGSL_MMU_H +#include "kgsl_sharedmem.h" + +/* Identifier for the global page table */ +/* Per process page tables will probably pass in the thread group + as an identifier */ + +#define KGSL_MMU_GLOBAL_PT 0 + +#define GSL_PT_SUPER_PTE 8 +#define GSL_PT_PAGE_WV 0x00000001 +#define GSL_PT_PAGE_RV 0x00000002 +#define GSL_PT_PAGE_DIRTY 0x00000004 +/* MMU Flags */ +#define KGSL_MMUFLAGS_TLBFLUSH 0x10000000 +#define KGSL_MMUFLAGS_PTUPDATE 0x20000000 + +#define MH_INTERRUPT_MASK__AXI_READ_ERROR 0x00000001L +#define MH_INTERRUPT_MASK__AXI_WRITE_ERROR 0x00000002L +#define MH_INTERRUPT_MASK__MMU_PAGE_FAULT 0x00000004L + +/* Macros to manage TLB flushing */ +#define GSL_TLBFLUSH_FILTER_ENTRY_NUMBITS (sizeof(unsigned char) * 8) +#define GSL_TLBFLUSH_FILTER_GET(superpte) \ + (*((unsigned char *) \ + (((unsigned int)pagetable->tlbflushfilter.base) \ + + (superpte / GSL_TLBFLUSH_FILTER_ENTRY_NUMBITS)))) +#define GSL_TLBFLUSH_FILTER_SETDIRTY(superpte) \ + (GSL_TLBFLUSH_FILTER_GET((superpte)) |= 1 << \ + (superpte % GSL_TLBFLUSH_FILTER_ENTRY_NUMBITS)) +#define GSL_TLBFLUSH_FILTER_ISDIRTY(superpte) \ + (GSL_TLBFLUSH_FILTER_GET((superpte)) & \ + (1 << (superpte % GSL_TLBFLUSH_FILTER_ENTRY_NUMBITS))) +#define GSL_TLBFLUSH_FILTER_RESET() memset(pagetable->tlbflushfilter.base,\ + 0, pagetable->tlbflushfilter.size) + + +struct kgsl_device; + +struct kgsl_tlbflushfilter { + unsigned int *base; + unsigned int size; +}; + +struct kgsl_pagetable { + spinlock_t lock; + unsigned int refcnt; + struct kgsl_memdesc base; + uint32_t va_base; + unsigned int va_range; + unsigned int last_superpte; + unsigned int max_entries; + struct gen_pool *pool; + struct list_head list; + unsigned int name; + /* Maintain filter to manage tlb flushing */ + struct kgsl_tlbflushfilter tlbflushfilter; + unsigned int tlb_flags; + struct kobject *kobj; + + struct { + unsigned int entries; + unsigned int mapped; + unsigned int max_mapped; + unsigned int max_entries; + } stats; +}; + +struct kgsl_mmu_reg { + + uint32_t config; + uint32_t mpu_base; + uint32_t mpu_end; + uint32_t va_range; + uint32_t pt_page; + uint32_t page_fault; + uint32_t tran_error; + uint32_t invalidate; + uint32_t interrupt_mask; + uint32_t interrupt_status; + uint32_t interrupt_clear; + uint32_t axi_error; +}; + +struct kgsl_mmu { + unsigned int refcnt; + uint32_t flags; + struct kgsl_device *device; + unsigned int config; + uint32_t mpu_base; + int mpu_range; + struct kgsl_memdesc dummyspace; + struct kgsl_mmu_reg reg; + /* current page table object being used by device mmu */ + struct kgsl_pagetable *defaultpagetable; + struct kgsl_pagetable *hwpagetable; +}; + +struct kgsl_ptpool_chunk { + size_t size; + unsigned int count; + int dynamic; + + void *data; + unsigned int phys; + + unsigned long *bitmap; + struct list_head list; +}; + +struct kgsl_ptpool { + size_t ptsize; + struct mutex lock; + struct list_head list; + int entries; + int static_entries; + int chunks; +}; + +struct kgsl_pagetable *kgsl_mmu_getpagetable(unsigned long name); + +#ifdef CONFIG_MSM_KGSL_MMU + +int kgsl_mmu_init(struct kgsl_device *device); +int kgsl_mmu_start(struct kgsl_device *device); +int kgsl_mmu_stop(struct kgsl_device *device); +int kgsl_mmu_close(struct kgsl_device *device); +int kgsl_mmu_setstate(struct kgsl_device *device, + struct kgsl_pagetable *pagetable); +int kgsl_mmu_map(struct kgsl_pagetable *pagetable, + struct kgsl_memdesc *memdesc, + unsigned int protflags); +int kgsl_mmu_map_global(struct kgsl_pagetable *pagetable, + struct kgsl_memdesc *memdesc, unsigned int protflags); +int kgsl_mmu_unmap(struct kgsl_pagetable *pagetable, + struct kgsl_memdesc *memdesc); +void kgsl_ptpool_destroy(struct kgsl_ptpool *pool); +int kgsl_ptpool_init(struct kgsl_ptpool *pool, int ptsize, int entries); +void kgsl_mh_intrcallback(struct kgsl_device *device); +void kgsl_mmu_putpagetable(struct kgsl_pagetable *pagetable); +unsigned int kgsl_virtaddr_to_physaddr(void *virtaddr); + +static inline int kgsl_mmu_enabled(void) +{ + return 1; +} + +#else + +static inline int kgsl_mmu_enabled(void) +{ + return 0; +} + +static inline int kgsl_mmu_init(struct kgsl_device *device) +{ + return 0; +} + +static inline int kgsl_mmu_start(struct kgsl_device *device) +{ + return 0; +} + +static inline int kgsl_mmu_stop(struct kgsl_device *device) +{ + return 0; +} + +static inline int kgsl_mmu_close(struct kgsl_device *device) +{ + return 0; +} + +static inline int kgsl_mmu_setstate(struct kgsl_device *device, + struct kgsl_pagetable *pagetable) +{ + return 0; +} + +static inline int kgsl_mmu_map(struct kgsl_pagetable *pagetable, + struct kgsl_memdesc *memdesc, + unsigned int protflags) +{ + memdesc->gpuaddr = memdesc->physaddr; + return 0; +} + +static inline int kgsl_mmu_unmap(struct kgsl_pagetable *pagetable, + struct kgsl_memdesc *memdesc) +{ + return 0; +} + +static inline int kgsl_ptpool_init(struct kgsl_ptpool *pool, int ptsize, + int entries) +{ + return 0; +} + +static inline int kgsl_mmu_map_global(struct kgsl_pagetable *pagetable, + struct kgsl_memdesc *memdesc, unsigned int protflags) +{ + /* gpuaddr is the same that gets passed in */ + return 0; +} + +static inline void kgsl_ptpool_destroy(struct kgsl_ptpool *pool) { } + +static inline void kgsl_mh_intrcallback(struct kgsl_device *device) { } + +static inline void kgsl_mmu_putpagetable(struct kgsl_pagetable *pagetable) { } + +static inline unsigned int kgsl_virtaddr_to_physaddr(void *virtaddr) +{ + return 0; +} + +#endif + +static inline unsigned int kgsl_pt_get_flags(struct kgsl_pagetable *pt, + enum kgsl_deviceid id) +{ + unsigned int result = 0; + + if (pt == NULL) + return 0; + + spin_lock(&pt->lock); + if (pt->tlb_flags && (1<tlb_flags &= ~(1<lock); + return result; +} + +#endif /* __KGSL_MMU_H */ diff --git a/drivers/gpu/msm/kgsl_pwrctrl.c b/drivers/gpu/msm/kgsl_pwrctrl.c new file mode 100644 index 00000000..dcac6aba --- /dev/null +++ b/drivers/gpu/msm/kgsl_pwrctrl.c @@ -0,0 +1,771 @@ +/* Copyright (c) 2010-2011, Code Aurora Forum. All rights reserved. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 and + * only version 2 as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + */ +#include +#include +#include + +#include "kgsl.h" + +#define SWITCH_OFF 200 +#define TZ_UPDATE_ID 0x01404000 +#define TZ_RESET_ID 0x01403000 + +#ifdef CONFIG_MSM_SECURE_IO +/* Trap into the TrustZone, and call funcs there. */ +static int __secure_tz_entry(u32 cmd, u32 val) +{ + register u32 r0 asm("r0") = cmd; + register u32 r1 asm("r1") = 0x0; + register u32 r2 asm("r2") = val; + + __iowmb(); + asm( + __asmeq("%0", "r0") + __asmeq("%1", "r0") + __asmeq("%2", "r1") + __asmeq("%3", "r2") + "smc #0 @ switch to secure world\n" + : "=r" (r0) + : "r" (r0), "r" (r1), "r" (r2) + ); + return r0; +} +#else +static int __secure_tz_entry(u32 cmd, u32 val) +{ + return 0; +} +#endif /* CONFIG_MSM_SECURE_IO */ + +/* Returns the requested update to our power level. * + * Either up/down (-1/1) a level, or stay the same (0). */ +static inline int kgsl_pwrctrl_tz_update(u32 idle) +{ + return __secure_tz_entry(TZ_UPDATE_ID, idle); +} + +static inline void kgsl_pwrctrl_tz_reset(void) +{ + __secure_tz_entry(TZ_RESET_ID, 0); +} + +void kgsl_pwrctrl_pwrlevel_change(struct kgsl_device *device, + unsigned int new_level) +{ + struct kgsl_pwrctrl *pwr = &device->pwrctrl; + if (new_level < (pwr->num_pwrlevels - 1) && + new_level >= pwr->thermal_pwrlevel && + new_level != pwr->active_pwrlevel) { + pwr->active_pwrlevel = new_level; + if (test_bit(KGSL_PWRFLAGS_CLK_ON, &pwr->power_flags)) + clk_set_rate(pwr->grp_clks[0], + pwr->pwrlevels[pwr->active_pwrlevel]. + gpu_freq); + if (test_bit(KGSL_PWRFLAGS_AXI_ON, &pwr->power_flags)) + if (pwr->pcl) + msm_bus_scale_client_update_request(pwr->pcl, + pwr->pwrlevels[pwr->active_pwrlevel]. + bus_freq); + KGSL_PWR_WARN(device, "pwr level changed to %d\n", + pwr->active_pwrlevel); + } +} +EXPORT_SYMBOL(kgsl_pwrctrl_pwrlevel_change); + +static int __gpuclk_store(int max, struct device *dev, + struct device_attribute *attr, + const char *buf, size_t count) +{ int ret, i, delta = 5000000; + unsigned long val; + struct kgsl_device *device = kgsl_device_from_dev(dev); + struct kgsl_pwrctrl *pwr; + + if (device == NULL) + return 0; + pwr = &device->pwrctrl; + + ret = sscanf(buf, "%ld", &val); + if (ret != 1) + return count; + + mutex_lock(&device->mutex); + for (i = 0; i < pwr->num_pwrlevels; i++) { + if (abs(pwr->pwrlevels[i].gpu_freq - val) < delta) { + if (max) + pwr->thermal_pwrlevel = i; + break; + } + } + + if (i == pwr->num_pwrlevels) + goto done; + + /* + * If the current or requested clock speed is greater than the + * thermal limit, bump down immediately. + */ + + if (pwr->pwrlevels[pwr->active_pwrlevel].gpu_freq > + pwr->pwrlevels[pwr->thermal_pwrlevel].gpu_freq) + kgsl_pwrctrl_pwrlevel_change(device, pwr->thermal_pwrlevel); + else if (!max) + kgsl_pwrctrl_pwrlevel_change(device, i); + +done: + mutex_unlock(&device->mutex); + return count; +} + +static int kgsl_pwrctrl_max_gpuclk_store(struct device *dev, + struct device_attribute *attr, + const char *buf, size_t count) +{ + return __gpuclk_store(1, dev, attr, buf, count); +} + +static int kgsl_pwrctrl_max_gpuclk_show(struct device *dev, + struct device_attribute *attr, + char *buf) +{ + struct kgsl_device *device = kgsl_device_from_dev(dev); + struct kgsl_pwrctrl *pwr; + if (device == NULL) + return 0; + pwr = &device->pwrctrl; + return snprintf(buf, PAGE_SIZE, "%d\n", + pwr->pwrlevels[pwr->thermal_pwrlevel].gpu_freq); +} + +static int kgsl_pwrctrl_gpuclk_store(struct device *dev, + struct device_attribute *attr, + const char *buf, size_t count) +{ + return __gpuclk_store(0, dev, attr, buf, count); +} + +static int kgsl_pwrctrl_gpuclk_show(struct device *dev, + struct device_attribute *attr, + char *buf) +{ + struct kgsl_device *device = kgsl_device_from_dev(dev); + struct kgsl_pwrctrl *pwr; + if (device == NULL) + return 0; + pwr = &device->pwrctrl; + return snprintf(buf, PAGE_SIZE, "%d\n", + pwr->pwrlevels[pwr->active_pwrlevel].gpu_freq); +} + +static int kgsl_pwrctrl_pwrnap_store(struct device *dev, + struct device_attribute *attr, + const char *buf, size_t count) +{ + char temp[20]; + unsigned long val; + struct kgsl_device *device = kgsl_device_from_dev(dev); + struct kgsl_pwrctrl *pwr; + int rc; + + if (device == NULL) + return 0; + pwr = &device->pwrctrl; + + snprintf(temp, sizeof(temp), "%.*s", + (int)min(count, sizeof(temp) - 1), buf); + rc = strict_strtoul(temp, 0, &val); + if (rc) + return rc; + + mutex_lock(&device->mutex); + + if (val == 1) + pwr->nap_allowed = true; + else if (val == 0) + pwr->nap_allowed = false; + + mutex_unlock(&device->mutex); + + return count; +} + +static int kgsl_pwrctrl_pwrnap_show(struct device *dev, + struct device_attribute *attr, + char *buf) +{ + struct kgsl_device *device = kgsl_device_from_dev(dev); + if (device == NULL) + return 0; + return snprintf(buf, PAGE_SIZE, "%d\n", device->pwrctrl.nap_allowed); +} + + +static int kgsl_pwrctrl_idle_timer_store(struct device *dev, + struct device_attribute *attr, + const char *buf, size_t count) +{ + char temp[20]; + unsigned long val; + struct kgsl_device *device = kgsl_device_from_dev(dev); + struct kgsl_pwrctrl *pwr; + const long div = 1000/HZ; + static unsigned int org_interval_timeout = 1; + int rc; + + if (device == NULL) + return 0; + pwr = &device->pwrctrl; + + snprintf(temp, sizeof(temp), "%.*s", + (int)min(count, sizeof(temp) - 1), buf); + rc = strict_strtoul(temp, 0, &val); + if (rc) + return rc; + + if (org_interval_timeout == 1) + org_interval_timeout = pwr->interval_timeout; + + mutex_lock(&device->mutex); + + /* Let the timeout be requested in ms, but convert to jiffies. */ + val /= div; + if (val >= org_interval_timeout) + pwr->interval_timeout = val; + + mutex_unlock(&device->mutex); + + return count; +} + +static int kgsl_pwrctrl_idle_timer_show(struct device *dev, + struct device_attribute *attr, + char *buf) +{ + struct kgsl_device *device = kgsl_device_from_dev(dev); + if (device == NULL) + return 0; + return snprintf(buf, PAGE_SIZE, "%d\n", + device->pwrctrl.interval_timeout); +} + +static int kgsl_pwrctrl_scaling_governor_store(struct device *dev, + struct device_attribute *attr, + const char *buf, size_t count) +{ + char temp[20]; + struct kgsl_device *device = kgsl_device_from_dev(dev); + struct kgsl_pwrctrl *pwr = &device->pwrctrl; + unsigned int reset = pwr->idle_pass; + + snprintf(temp, sizeof(temp), "%.*s", + (int)min(count, sizeof(temp) - 1), buf); + if (strncmp(temp, "ondemand", 8) == 0) + reset = 1; + else if (strncmp(temp, "performance", 11) == 0) + reset = 0; + + mutex_lock(&device->mutex); + pwr->idle_pass = reset; + if (pwr->idle_pass == 0) + kgsl_pwrctrl_pwrlevel_change(device, pwr->thermal_pwrlevel); + mutex_unlock(&device->mutex); + + return count; +} + +static int kgsl_pwrctrl_scaling_governor_show(struct device *dev, + struct device_attribute *attr, + char *buf) +{ + struct kgsl_device *device = kgsl_device_from_dev(dev); + struct kgsl_pwrctrl *pwr = &device->pwrctrl; + if (pwr->idle_pass) + return snprintf(buf, 10, "ondemand\n"); + else + return snprintf(buf, 13, "performance\n"); +} + +DEVICE_ATTR(gpuclk, 0644, kgsl_pwrctrl_gpuclk_show, kgsl_pwrctrl_gpuclk_store); +DEVICE_ATTR(max_gpuclk, 0644, kgsl_pwrctrl_max_gpuclk_show, + kgsl_pwrctrl_max_gpuclk_store); +DEVICE_ATTR(pwrnap, 0644, kgsl_pwrctrl_pwrnap_show, kgsl_pwrctrl_pwrnap_store); +DEVICE_ATTR(idle_timer, 0644, kgsl_pwrctrl_idle_timer_show, + kgsl_pwrctrl_idle_timer_store); +DEVICE_ATTR(scaling_governor, 0644, kgsl_pwrctrl_scaling_governor_show, + kgsl_pwrctrl_scaling_governor_store); + +static const struct device_attribute *pwrctrl_attr_list[] = { + &dev_attr_gpuclk, + &dev_attr_max_gpuclk, + &dev_attr_pwrnap, + &dev_attr_idle_timer, + &dev_attr_scaling_governor, + NULL +}; + +int kgsl_pwrctrl_init_sysfs(struct kgsl_device *device) +{ + return kgsl_create_device_sysfs_files(device->dev, pwrctrl_attr_list); +} + +void kgsl_pwrctrl_uninit_sysfs(struct kgsl_device *device) +{ + kgsl_remove_device_sysfs_files(device->dev, pwrctrl_attr_list); +} + +static void kgsl_pwrctrl_idle_calc(struct kgsl_device *device) +{ + int val; + struct kgsl_pwrctrl *pwr = &device->pwrctrl; + struct kgsl_power_stats stats; + + device->ftbl.device_power_stats(device, &stats); + + if (stats.total_time == 0) + return; + + /* If the GPU has stayed in turbo mode for a while, * + * stop writing out values. */ + if (pwr->active_pwrlevel) + pwr->no_switch_cnt = 0; + else if (pwr->no_switch_cnt > SWITCH_OFF) + return; + pwr->no_switch_cnt++; + val = kgsl_pwrctrl_tz_update(stats.total_time - stats.busy_time); + if (val) + kgsl_pwrctrl_pwrlevel_change(device, + pwr->active_pwrlevel + val); +} + +void kgsl_pwrctrl_clk(struct kgsl_device *device, int state) +{ + struct kgsl_pwrctrl *pwr = &device->pwrctrl; + int i = 0; + if (state == KGSL_PWRFLAGS_OFF) { + if (test_and_clear_bit(KGSL_PWRFLAGS_CLK_ON, + &pwr->power_flags)) { + KGSL_PWR_INFO(device, + "clocks off, device %d\n", device->id); + for (i = KGSL_MAX_CLKS - 1; i > 0; i--) + if (pwr->grp_clks[i]) + clk_disable(pwr->grp_clks[i]); + if ((pwr->pwrlevels[0].gpu_freq > 0) && + (device->requested_state != KGSL_STATE_NAP)) + clk_set_rate(pwr->grp_clks[0], + pwr->pwrlevels[pwr->num_pwrlevels - 1]. + gpu_freq); + } + } else if (state == KGSL_PWRFLAGS_ON) { + if (!test_and_set_bit(KGSL_PWRFLAGS_CLK_ON, + &pwr->power_flags)) { + KGSL_PWR_INFO(device, + "clocks on, device %d\n", device->id); + + if ((pwr->pwrlevels[0].gpu_freq > 0) && + (device->state != KGSL_STATE_NAP)) + clk_set_rate(pwr->grp_clks[0], + pwr->pwrlevels[pwr->active_pwrlevel]. + gpu_freq); + + /* as last step, enable grp_clk + this is to let GPU interrupt to come */ + for (i = KGSL_MAX_CLKS - 1; i > 0; i--) + if (pwr->grp_clks[i]) + clk_enable(pwr->grp_clks[i]); + } + } +} +EXPORT_SYMBOL(kgsl_pwrctrl_clk); + +void kgsl_pwrctrl_axi(struct kgsl_device *device, int state) +{ + struct kgsl_pwrctrl *pwr = &device->pwrctrl; + + if (state == KGSL_PWRFLAGS_OFF) { + if (test_and_clear_bit(KGSL_PWRFLAGS_AXI_ON, + &pwr->power_flags)) { + KGSL_PWR_INFO(device, + "axi off, device %d\n", device->id); + if (pwr->ebi1_clk) { + clk_set_rate(pwr->ebi1_clk, 0); + clk_disable(pwr->ebi1_clk); + } + if (pwr->pcl) + msm_bus_scale_client_update_request(pwr->pcl, + 0); + } + } else if (state == KGSL_PWRFLAGS_ON) { + if (!test_and_set_bit(KGSL_PWRFLAGS_AXI_ON, + &pwr->power_flags)) { + KGSL_PWR_INFO(device, + "axi on, device %d\n", device->id); + if (pwr->ebi1_clk) { + clk_enable(pwr->ebi1_clk); + clk_set_rate(pwr->ebi1_clk, + pwr->pwrlevels[pwr->active_pwrlevel]. + bus_freq); + } + if (pwr->pcl) + msm_bus_scale_client_update_request(pwr->pcl, + pwr->pwrlevels[pwr->active_pwrlevel]. + bus_freq); + } + } +} +EXPORT_SYMBOL(kgsl_pwrctrl_axi); + + +void kgsl_pwrctrl_pwrrail(struct kgsl_device *device, int state) +{ + struct kgsl_pwrctrl *pwr = &device->pwrctrl; + + if (state == KGSL_PWRFLAGS_OFF) { + if (test_and_clear_bit(KGSL_PWRFLAGS_POWER_ON, + &pwr->power_flags)) { + KGSL_PWR_INFO(device, + "power off, device %d\n", device->id); + if (pwr->gpu_reg) + regulator_disable(pwr->gpu_reg); + } + } else if (state == KGSL_PWRFLAGS_ON) { + if (!test_and_set_bit(KGSL_PWRFLAGS_POWER_ON, + &pwr->power_flags)) { + KGSL_PWR_INFO(device, + "power on, device %d\n", device->id); + if (pwr->gpu_reg) + regulator_enable(pwr->gpu_reg); + } + } +} +EXPORT_SYMBOL(kgsl_pwrctrl_pwrrail); + +void kgsl_pwrctrl_irq(struct kgsl_device *device, int state) +{ + struct kgsl_pwrctrl *pwr = &device->pwrctrl; + + if (state == KGSL_PWRFLAGS_ON) { + if (!test_and_set_bit(KGSL_PWRFLAGS_IRQ_ON, + &pwr->power_flags)) { + KGSL_PWR_INFO(device, + "irq on, device %d\n", device->id); + enable_irq(pwr->interrupt_num); + } + } else if (state == KGSL_PWRFLAGS_OFF) { + if (test_and_clear_bit(KGSL_PWRFLAGS_IRQ_ON, + &pwr->power_flags)) { + KGSL_PWR_INFO(device, + "irq off, device %d\n", device->id); + disable_irq(pwr->interrupt_num); + } + } +} +EXPORT_SYMBOL(kgsl_pwrctrl_irq); + +int kgsl_pwrctrl_init(struct kgsl_device *device) +{ + int i, result = 0; + struct clk *clk; + struct platform_device *pdev = + container_of(device->parentdev, struct platform_device, dev); + struct kgsl_pwrctrl *pwr = &device->pwrctrl; + struct kgsl_device_platform_data *pdata_dev = pdev->dev.platform_data; + struct kgsl_device_pwr_data *pdata_pwr = &pdata_dev->pwr_data; + const char *clk_names[KGSL_MAX_CLKS] = {pwr->src_clk_name, + pdata_dev->clk.name.clk, + pdata_dev->clk.name.pclk, + pdata_dev->imem_clk_name.clk, + pdata_dev->imem_clk_name.pclk}; + + /*acquire clocks */ + for (i = 1; i < KGSL_MAX_CLKS; i++) { + if (clk_names[i]) { + clk = clk_get(&pdev->dev, clk_names[i]); + if (IS_ERR(clk)) + goto clk_err; + pwr->grp_clks[i] = clk; + } + } + /* Make sure we have a source clk for freq setting */ + clk = clk_get(&pdev->dev, clk_names[0]); + pwr->grp_clks[0] = (IS_ERR(clk)) ? pwr->grp_clks[1] : clk; + + /* put the AXI bus into asynchronous mode with the graphics cores */ + if (pdata_pwr->set_grp_async != NULL) + pdata_pwr->set_grp_async(); + + if (pdata_pwr->num_levels > KGSL_MAX_PWRLEVELS) { + KGSL_PWR_ERR(device, "invalid power level count: %d\n", + pdata_pwr->num_levels); + result = -EINVAL; + goto done; + } + pwr->num_pwrlevels = pdata_pwr->num_levels; + pwr->active_pwrlevel = pdata_pwr->init_level; + for (i = 0; i < pdata_pwr->num_levels; i++) { +// pwr->pwrlevels[i].gpu_freq = +// (pdata_pwr->pwrlevel[i].gpu_freq > 0) ? +// clk_round_rate(pwr->grp_clks[0], +// pdata_pwr->pwrlevel[i]. +// gpu_freq) : 0; + pwr->pwrlevels[i].gpu_freq =(pdata_pwr->pwrlevel[i].gpu_freq > 0)? + pdata_pwr->pwrlevel[i].gpu_freq:0; + pwr->pwrlevels[i].bus_freq = + pdata_pwr->pwrlevel[i].bus_freq; + } + /* Do not set_rate for targets in sync with AXI */ + if (pwr->pwrlevels[0].gpu_freq > 0) + clk_set_rate(pwr->grp_clks[0], pwr-> + pwrlevels[pwr->num_pwrlevels - 1].gpu_freq); + + pwr->gpu_reg = regulator_get(NULL, pwr->regulator_name); + if (IS_ERR(pwr->gpu_reg)) + pwr->gpu_reg = NULL; + + pwr->power_flags = 0; + + pwr->nap_allowed = pdata_pwr->nap_allowed; +/* drewis: below was removed at some point before i cherry-picked the below commit */ + pwr->idle_pass = pdata_pwr->idle_pass; +/*dc14311... msm: kgsl: Replace internal_power_rail API calls with regulator APIs*/ + pwr->interval_timeout = pdata_pwr->idle_timeout; + pwr->ebi1_clk = clk_get(NULL, "ebi1_kgsl_clk"); + if (IS_ERR(pwr->ebi1_clk)) + pwr->ebi1_clk = NULL; + else + clk_set_rate(pwr->ebi1_clk, + pwr->pwrlevels[pwr->active_pwrlevel]. + bus_freq); + if (pdata_dev->clk.bus_scale_table != NULL) { + pwr->pcl = + msm_bus_scale_register_client(pdata_dev->clk. + bus_scale_table); + if (!pwr->pcl) { + KGSL_PWR_ERR(device, + "msm_bus_scale_register_client failed: " + "id %d table %p", device->id, + pdata_dev->clk.bus_scale_table); + result = -EINVAL; + goto done; + } + } + + /*acquire interrupt */ + pwr->interrupt_num = + platform_get_irq_byname(pdev, pwr->irq_name); + + if (pwr->interrupt_num <= 0) { + KGSL_PWR_ERR(device, "platform_get_irq_byname failed: %d\n", + pwr->interrupt_num); + result = -EINVAL; + goto done; + } + return result; + +clk_err: + result = PTR_ERR(clk); + KGSL_PWR_ERR(device, "clk_get(%s) failed: %d\n", + clk_names[i], result); + +done: + return result; +} + +void kgsl_pwrctrl_close(struct kgsl_device *device) +{ + struct kgsl_pwrctrl *pwr = &device->pwrctrl; + int i; + + KGSL_PWR_INFO(device, "close device %d\n", device->id); + + if (pwr->interrupt_num > 0) { + if (pwr->have_irq) { + free_irq(pwr->interrupt_num, NULL); + pwr->have_irq = 0; + } + pwr->interrupt_num = 0; + } + + clk_put(pwr->ebi1_clk); + + if (pwr->pcl) + msm_bus_scale_unregister_client(pwr->pcl); + + pwr->pcl = 0; + + if (pwr->gpu_reg) { + regulator_put(pwr->gpu_reg); + pwr->gpu_reg = NULL; + } + + for (i = 1; i < KGSL_MAX_CLKS; i++) + if (pwr->grp_clks[i]) { + clk_put(pwr->grp_clks[i]); + pwr->grp_clks[i] = NULL; + } + + pwr->grp_clks[0] = NULL; + pwr->power_flags = 0; +} + +void kgsl_idle_check(struct work_struct *work) +{ + struct kgsl_device *device = container_of(work, struct kgsl_device, + idle_check_ws); + + mutex_lock(&device->mutex); + if ((device->pwrctrl.idle_pass) && + (device->requested_state != KGSL_STATE_SLEEP)) + kgsl_pwrctrl_idle_calc(device); + + if (device->state & (KGSL_STATE_ACTIVE | KGSL_STATE_NAP)) { + if (kgsl_pwrctrl_sleep(device) != 0) + mod_timer(&device->idle_timer, + jiffies + + device->pwrctrl.interval_timeout); + } else if (device->state & (KGSL_STATE_HUNG | + KGSL_STATE_DUMP_AND_RECOVER)) { + device->requested_state = KGSL_STATE_NONE; + } + + mutex_unlock(&device->mutex); +} + +void kgsl_timer(unsigned long data) +{ + struct kgsl_device *device = (struct kgsl_device *) data; + + KGSL_PWR_INFO(device, "idle timer expired device %d\n", device->id); + if (device->requested_state != KGSL_STATE_SUSPEND) { + device->requested_state = KGSL_STATE_SLEEP; + /* Have work run in a non-interrupt context. */ + queue_work(device->work_queue, &device->idle_check_ws); + } +} + +void kgsl_pre_hwaccess(struct kgsl_device *device) +{ + BUG_ON(!mutex_is_locked(&device->mutex)); + if (device->state & (KGSL_STATE_SLEEP | KGSL_STATE_NAP)) + kgsl_pwrctrl_wake(device); +} +EXPORT_SYMBOL(kgsl_pre_hwaccess); + +void kgsl_check_suspended(struct kgsl_device *device) +{ + if (device->requested_state == KGSL_STATE_SUSPEND || + device->state == KGSL_STATE_SUSPEND) { + mutex_unlock(&device->mutex); + wait_for_completion(&device->hwaccess_gate); + mutex_lock(&device->mutex); + } + if (device->state == KGSL_STATE_DUMP_AND_RECOVER) { + mutex_unlock(&device->mutex); + wait_for_completion(&device->recovery_gate); + mutex_lock(&device->mutex); + } + } + + +/******************************************************************/ +/* Caller must hold the device mutex. */ +int kgsl_pwrctrl_sleep(struct kgsl_device *device) +{ + struct kgsl_pwrctrl *pwr = &device->pwrctrl; + KGSL_PWR_INFO(device, "sleep device %d\n", device->id); + + /* Work through the legal state transitions */ + if (device->requested_state == KGSL_STATE_NAP) { + if (device->ftbl.device_isidle(device)) + goto nap; + } else if (device->requested_state == KGSL_STATE_SLEEP) { + if (device->state == KGSL_STATE_NAP || + device->ftbl.device_isidle(device)) + goto sleep; + } + + device->requested_state = KGSL_STATE_NONE; + return -EBUSY; + +sleep: + kgsl_pwrctrl_irq(device, KGSL_PWRFLAGS_OFF); + kgsl_pwrctrl_axi(device, KGSL_PWRFLAGS_OFF); + if (pwr->pwrlevels[0].gpu_freq > 0) + clk_set_rate(pwr->grp_clks[0], + pwr->pwrlevels[pwr->num_pwrlevels - 1]. + gpu_freq); + device->pwrctrl.no_switch_cnt = 0; + device->pwrctrl.time = 0; + kgsl_pwrctrl_tz_reset(); + goto clk_off; + +nap: + kgsl_pwrctrl_irq(device, KGSL_PWRFLAGS_OFF); +clk_off: + kgsl_pwrctrl_clk(device, KGSL_PWRFLAGS_OFF); + + device->state = device->requested_state; + device->requested_state = KGSL_STATE_NONE; + wake_unlock(&device->idle_wakelock); + KGSL_PWR_WARN(device, "state -> NAP/SLEEP(%d), device %d\n", + device->state, device->id); + + return 0; +} +EXPORT_SYMBOL(kgsl_pwrctrl_sleep); + +/******************************************************************/ +/* Caller must hold the device mutex. */ +void kgsl_pwrctrl_wake(struct kgsl_device *device) +{ + if (device->state == KGSL_STATE_SUSPEND) + return; + + if (device->state != KGSL_STATE_NAP) { + if (device->pwrctrl.idle_pass) + kgsl_pwrctrl_pwrlevel_change(device, + device->pwrctrl.thermal_pwrlevel); + kgsl_pwrctrl_axi(device, KGSL_PWRFLAGS_ON); + } + /* Turn on the core clocks */ + kgsl_pwrctrl_clk(device, KGSL_PWRFLAGS_ON); + + /* Enable state before turning on irq */ + device->state = KGSL_STATE_ACTIVE; + KGSL_PWR_WARN(device, "state -> ACTIVE, device %d\n", device->id); + kgsl_pwrctrl_irq(device, KGSL_PWRFLAGS_ON); + + /* Re-enable HW access */ + mod_timer(&device->idle_timer, + jiffies + device->pwrctrl.interval_timeout); + + wake_lock(&device->idle_wakelock); + KGSL_PWR_INFO(device, "wake return for device %d\n", device->id); +} +EXPORT_SYMBOL(kgsl_pwrctrl_wake); + +void kgsl_pwrctrl_enable(struct kgsl_device *device) +{ + /* Order pwrrail/clk sequence based upon platform */ + kgsl_pwrctrl_pwrrail(device, KGSL_PWRFLAGS_ON); + kgsl_pwrctrl_clk(device, KGSL_PWRFLAGS_ON); + kgsl_pwrctrl_axi(device, KGSL_PWRFLAGS_ON); +} +EXPORT_SYMBOL(kgsl_pwrctrl_enable); + +void kgsl_pwrctrl_disable(struct kgsl_device *device) +{ + /* Order pwrrail/clk sequence based upon platform */ + kgsl_pwrctrl_axi(device, KGSL_PWRFLAGS_OFF); + kgsl_pwrctrl_clk(device, KGSL_PWRFLAGS_OFF); + kgsl_pwrctrl_pwrrail(device, KGSL_PWRFLAGS_OFF); +} +EXPORT_SYMBOL(kgsl_pwrctrl_disable); diff --git a/drivers/gpu/msm/kgsl_pwrctrl.h b/drivers/gpu/msm/kgsl_pwrctrl.h new file mode 100644 index 00000000..108cc309 --- /dev/null +++ b/drivers/gpu/msm/kgsl_pwrctrl.h @@ -0,0 +1,94 @@ +/* Copyright (c) 2010-2011, Code Aurora Forum. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are + * met: + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following + * disclaimer in the documentation and/or other materials provided + * with the distribution. + * * Neither the name of Code Aurora Forum, Inc. nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED "AS IS" AND ANY EXPRESS OR IMPLIED + * WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS + * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR + * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, + * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE + * OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN + * IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + */ +#ifndef __KGSL_PWRCTRL_H +#define __KGSL_PWRCTRL_H + +/***************************************************************************** +** power flags +*****************************************************************************/ +#define KGSL_PWRFLAGS_POWER_ON 0 +#define KGSL_PWRFLAGS_CLK_ON 1 +#define KGSL_PWRFLAGS_AXI_ON 2 +#define KGSL_PWRFLAGS_IRQ_ON 3 + +#define KGSL_PWRFLAGS_ON 1 +#define KGSL_PWRFLAGS_OFF 0 + +#define KGSL_DEFAULT_PWRLEVEL 1 +#define KGSL_MAX_CLKS 5 + +struct platform_device; + +struct kgsl_pwrctrl { + int interrupt_num; + int have_irq; + struct clk *ebi1_clk; + struct clk *grp_clks[KGSL_MAX_CLKS]; + unsigned long power_flags; + struct kgsl_pwrlevel pwrlevels[KGSL_MAX_PWRLEVELS]; + unsigned int active_pwrlevel; + int thermal_pwrlevel; + unsigned int num_pwrlevels; + unsigned int interval_timeout; + struct regulator *gpu_reg; + uint32_t pcl; + unsigned int nap_allowed; + struct adreno_context *suspended_ctxt; + const char *regulator_name; + const char *irq_name; + const char *src_clk_name; + s64 time; + unsigned int no_switch_cnt; + unsigned int idle_pass; +}; + +void kgsl_pwrctrl_clk(struct kgsl_device *device, int state); +void kgsl_pwrctrl_axi(struct kgsl_device *device, int state); +void kgsl_pwrctrl_pwrrail(struct kgsl_device *device, int state); +void kgsl_pwrctrl_irq(struct kgsl_device *device, int state); +int kgsl_pwrctrl_init(struct kgsl_device *device); +void kgsl_pwrctrl_close(struct kgsl_device *device); +void kgsl_timer(unsigned long data); +void kgsl_idle_check(struct work_struct *work); +void kgsl_pre_hwaccess(struct kgsl_device *device); +void kgsl_check_suspended(struct kgsl_device *device); +int kgsl_pwrctrl_sleep(struct kgsl_device *device); +void kgsl_pwrctrl_wake(struct kgsl_device *device); +void kgsl_pwrctrl_pwrlevel_change(struct kgsl_device *device, + unsigned int level); +int kgsl_pwrctrl_init_sysfs(struct kgsl_device *device); +void kgsl_pwrctrl_uninit_sysfs(struct kgsl_device *device); +void kgsl_pwrctrl_enable(struct kgsl_device *device); +void kgsl_pwrctrl_disable(struct kgsl_device *device); +static inline unsigned long kgsl_get_clkrate(struct clk *clk) +{ + return (clk != NULL) ? clk_get_rate(clk) : 0; +} + +#endif /* __KGSL_PWRCTRL_H */ diff --git a/drivers/gpu/msm/kgsl_pwrscale.c b/drivers/gpu/msm/kgsl_pwrscale.c new file mode 100644 index 00000000..f2250283 --- /dev/null +++ b/drivers/gpu/msm/kgsl_pwrscale.c @@ -0,0 +1,308 @@ +/* Copyright (c) 2010-2011, Code Aurora Forum. All rights reserved. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 and + * only version 2 as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + */ + +#include + +#include "kgsl.h" +#include "kgsl_pwrscale.h" + +struct kgsl_pwrscale_attribute { + struct attribute attr; + ssize_t (*show)(struct kgsl_device *device, char *buf); + ssize_t (*store)(struct kgsl_device *device, const char *buf, + size_t count); +}; + +#define to_pwrscale(k) container_of(k, struct kgsl_pwrscale, kobj) +#define pwrscale_to_device(p) container_of(p, struct kgsl_device, pwrscale) +#define to_device(k) container_of(k, struct kgsl_device, pwrscale_kobj) +#define to_pwrscale_attr(a) \ +container_of(a, struct kgsl_pwrscale_attribute, attr) +#define to_policy_attr(a) \ +container_of(a, struct kgsl_pwrscale_policy_attribute, attr) + +#define PWRSCALE_ATTR(_name, _mode, _show, _store) \ +struct kgsl_pwrscale_attribute pwrscale_attr_##_name = \ +__ATTR(_name, _mode, _show, _store) + +/* Master list of available policies */ + +static struct kgsl_pwrscale_policy *kgsl_pwrscale_policies[] = { + NULL +}; + +static ssize_t pwrscale_policy_store(struct kgsl_device *device, + const char *buf, size_t count) +{ + int i; + struct kgsl_pwrscale_policy *policy = NULL; + + /* The special keyword none allows the user to detach all + policies */ + if (!strncmp("none", buf, 4)) { + kgsl_pwrscale_detach_policy(device); + return count; + } + + for (i = 0; kgsl_pwrscale_policies[i]; i++) { + if (!strncmp(kgsl_pwrscale_policies[i]->name, buf, + strnlen(kgsl_pwrscale_policies[i]->name, + PAGE_SIZE))) { + policy = kgsl_pwrscale_policies[i]; + break; + } + } + + if (policy) + if (kgsl_pwrscale_attach_policy(device, policy)) + return -EIO; + + return count; +} + +static ssize_t pwrscale_policy_show(struct kgsl_device *device, char *buf) +{ + int ret; + + if (device->pwrscale.policy) + ret = snprintf(buf, PAGE_SIZE, "%s\n", + device->pwrscale.policy->name); + else + ret = snprintf(buf, PAGE_SIZE, "none\n"); + + return ret; +} + +PWRSCALE_ATTR(policy, 0644, pwrscale_policy_show, pwrscale_policy_store); + +static ssize_t pwrscale_avail_policies_show(struct kgsl_device *device, + char *buf) +{ + int i, ret = 0; + + for (i = 0; kgsl_pwrscale_policies[i]; i++) { + ret += snprintf(buf + ret, PAGE_SIZE - ret, "%s ", + kgsl_pwrscale_policies[i]->name); + } + + ret += snprintf(buf + ret, PAGE_SIZE - ret, "none\n"); + return ret; +} +PWRSCALE_ATTR(avail_policies, 0444, pwrscale_avail_policies_show, NULL); + +static struct attribute *pwrscale_attrs[] = { + &pwrscale_attr_policy.attr, + &pwrscale_attr_avail_policies.attr, + NULL +}; + +static ssize_t policy_sysfs_show(struct kobject *kobj, + struct attribute *attr, char *buf) +{ + struct kgsl_pwrscale *pwrscale = to_pwrscale(kobj); + struct kgsl_device *device = pwrscale_to_device(pwrscale); + struct kgsl_pwrscale_policy_attribute *pattr = to_policy_attr(attr); + ssize_t ret; + + if (pattr->show) + ret = pattr->show(device, pwrscale, buf); + else + ret = -EIO; + + return ret; +} + +static ssize_t policy_sysfs_store(struct kobject *kobj, + struct attribute *attr, + const char *buf, size_t count) +{ + struct kgsl_pwrscale *pwrscale = to_pwrscale(kobj); + struct kgsl_device *device = pwrscale_to_device(pwrscale); + struct kgsl_pwrscale_policy_attribute *pattr = to_policy_attr(attr); + ssize_t ret; + + if (pattr->store) + ret = pattr->store(device, pwrscale, buf, count); + else + ret = -EIO; + + return ret; +} + +static void policy_sysfs_release(struct kobject *kobj) +{ + struct kgsl_pwrscale *pwrscale = to_pwrscale(kobj); + + complete(&pwrscale->kobj_unregister); +} + +static ssize_t pwrscale_sysfs_show(struct kobject *kobj, + struct attribute *attr, char *buf) +{ + struct kgsl_device *device = to_device(kobj); + struct kgsl_pwrscale_attribute *pattr = to_pwrscale_attr(attr); + ssize_t ret; + + if (pattr->show) + ret = pattr->show(device, buf); + else + ret = -EIO; + + return ret; +} + +static ssize_t pwrscale_sysfs_store(struct kobject *kobj, + struct attribute *attr, + const char *buf, size_t count) +{ + struct kgsl_device *device = to_device(kobj); + struct kgsl_pwrscale_attribute *pattr = to_pwrscale_attr(attr); + ssize_t ret; + + if (pattr->store) + ret = pattr->store(device, buf, count); + else + ret = -EIO; + + return ret; +} + +static void pwrscale_sysfs_release(struct kobject *kobj) +{ +} + +static const struct sysfs_ops policy_sysfs_ops = { + .show = policy_sysfs_show, + .store = policy_sysfs_store +}; + +static const struct sysfs_ops pwrscale_sysfs_ops = { + .show = pwrscale_sysfs_show, + .store = pwrscale_sysfs_store +}; + +static struct kobj_type ktype_pwrscale_policy = { + .sysfs_ops = &policy_sysfs_ops, + .default_attrs = NULL, + .release = policy_sysfs_release +}; + +static struct kobj_type ktype_pwrscale = { + .sysfs_ops = &pwrscale_sysfs_ops, + .default_attrs = pwrscale_attrs, + .release = pwrscale_sysfs_release +}; + +void kgsl_pwrscale_sleep(struct kgsl_device *device) +{ + if (device->pwrscale.policy && device->pwrscale.policy->sleep) + device->pwrscale.policy->sleep(device, &device->pwrscale); +} +EXPORT_SYMBOL(kgsl_pwrscale_sleep); + +void kgsl_pwrscale_wake(struct kgsl_device *device) +{ + if (device->pwrscale.policy && device->pwrscale.policy->wake) + device->pwrscale.policy->wake(device, &device->pwrscale); +} +EXPORT_SYMBOL(kgsl_pwrscale_wake); + +void kgsl_pwrscale_busy(struct kgsl_device *device) +{ + if (device->pwrscale.policy && device->pwrscale.policy->busy) + device->pwrscale.policy->busy(device, &device->pwrscale); +} + +void kgsl_pwrscale_idle(struct kgsl_device *device) +{ + if (device->pwrscale.policy && device->pwrscale.policy->idle) + device->pwrscale.policy->idle(device, &device->pwrscale); +} +EXPORT_SYMBOL(kgsl_pwrscale_idle); + +int kgsl_pwrscale_policy_add_files(struct kgsl_device *device, + struct kgsl_pwrscale *pwrscale, + struct attribute_group *attr_group) +{ + int ret; + + init_completion(&pwrscale->kobj_unregister); + + ret = kobject_init_and_add(&pwrscale->kobj, + &ktype_pwrscale_policy, + &device->pwrscale_kobj, + "%s", pwrscale->policy->name); + + if (ret) + return ret; + + ret = sysfs_create_group(&pwrscale->kobj, attr_group); + + if (ret) { + kobject_put(&pwrscale->kobj); + wait_for_completion(&pwrscale->kobj_unregister); + } + + return ret; +} + +void kgsl_pwrscale_policy_remove_files(struct kgsl_device *device, + struct kgsl_pwrscale *pwrscale, + struct attribute_group *attr_group) +{ + sysfs_remove_group(&pwrscale->kobj, attr_group); + kobject_put(&pwrscale->kobj); + wait_for_completion(&pwrscale->kobj_unregister); +} + +void kgsl_pwrscale_detach_policy(struct kgsl_device *device) +{ + mutex_lock(&device->mutex); + if (device->pwrscale.policy != NULL) + device->pwrscale.policy->close(device, &device->pwrscale); + device->pwrscale.policy = NULL; + mutex_unlock(&device->mutex); +} +EXPORT_SYMBOL(kgsl_pwrscale_detach_policy); + +int kgsl_pwrscale_attach_policy(struct kgsl_device *device, + struct kgsl_pwrscale_policy *policy) +{ + int ret; + + if (device->pwrscale.policy != NULL) + kgsl_pwrscale_detach_policy(device); + + mutex_lock(&device->mutex); + device->pwrscale.policy = policy; + ret = device->pwrscale.policy->init(device, &device->pwrscale); + if (ret) + device->pwrscale.policy = NULL; + mutex_unlock(&device->mutex); + + return ret; +} +EXPORT_SYMBOL(kgsl_pwrscale_attach_policy); + +int kgsl_pwrscale_init(struct kgsl_device *device) +{ + return kobject_init_and_add(&device->pwrscale_kobj, &ktype_pwrscale, + &device->dev->kobj, "pwrscale"); +} +EXPORT_SYMBOL(kgsl_pwrscale_init); + +void kgsl_pwrscale_close(struct kgsl_device *device) +{ + kobject_put(&device->pwrscale_kobj); +} +EXPORT_SYMBOL(kgsl_pwrscale_close); diff --git a/drivers/gpu/msm/kgsl_pwrscale.h b/drivers/gpu/msm/kgsl_pwrscale.h new file mode 100644 index 00000000..f05adbd5 --- /dev/null +++ b/drivers/gpu/msm/kgsl_pwrscale.h @@ -0,0 +1,89 @@ +/* Copyright (c) 2010-2011, Code Aurora Forum. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are + * met: + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following + * disclaimer in the documentation and/or other materials provided + * with the distribution. + * * Neither the name of Code Aurora Forum, Inc. nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED "AS IS" AND ANY EXPRESS OR IMPLIED + * WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS + * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR + * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, + * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE + * OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN + * IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + */ + +#ifndef __KGSL_PWRSCALE_H +#define __KGSL_PWRSCALE_H + +struct kgsl_pwrscale; + +struct kgsl_pwrscale_policy { + const char *name; + int (*init)(struct kgsl_device *device, + struct kgsl_pwrscale *pwrscale); + void (*close)(struct kgsl_device *device, + struct kgsl_pwrscale *pwrscale); + void (*idle)(struct kgsl_device *device, + struct kgsl_pwrscale *pwrscale); + void (*busy)(struct kgsl_device *device, + struct kgsl_pwrscale *pwrscale); + void (*sleep)(struct kgsl_device *device, + struct kgsl_pwrscale *pwrscale); + void (*wake)(struct kgsl_device *device, + struct kgsl_pwrscale *pwrscale); +}; + +struct kgsl_pwrscale { + struct kgsl_pwrscale_policy *policy; + struct kobject kobj; + struct completion kobj_unregister; + void *priv; +}; + +struct kgsl_pwrscale_policy_attribute { + struct attribute attr; + ssize_t (*show)(struct kgsl_device *device, + struct kgsl_pwrscale *pwrscale, char *buf); + ssize_t (*store)(struct kgsl_device *device, + struct kgsl_pwrscale *pwrscale, const char *buf, + size_t count); +}; + +#define PWRSCALE_POLICY_ATTR(_name, _mode, _show, _store) \ + struct kgsl_pwrscale_policy_attribute policy_attr_##_name = \ + __ATTR(_name, _mode, _show, _store) + +int kgsl_pwrscale_init(struct kgsl_device *device); +void kgsl_pwrscale_close(struct kgsl_device *device); +int kgsl_pwrscale_attach_policy(struct kgsl_device *device, + struct kgsl_pwrscale_policy *policy); +void kgsl_pwrscale_detach_policy(struct kgsl_device *device); + +void kgsl_pwrscale_idle(struct kgsl_device *device); +void kgsl_pwrscale_busy(struct kgsl_device *device); +void kgsl_pwrscale_sleep(struct kgsl_device *device); +void kgsl_pwrscale_wake(struct kgsl_device *device); + +int kgsl_pwrscale_policy_add_files(struct kgsl_device *device, + struct kgsl_pwrscale *pwrscale, + struct attribute_group *attr_group); + +void kgsl_pwrscale_policy_remove_files(struct kgsl_device *device, + struct kgsl_pwrscale *pwrscale, + struct attribute_group *attr_group); +#endif diff --git a/drivers/gpu/msm/kgsl_sharedmem.c b/drivers/gpu/msm/kgsl_sharedmem.c new file mode 100644 index 00000000..36ff19c7 --- /dev/null +++ b/drivers/gpu/msm/kgsl_sharedmem.c @@ -0,0 +1,528 @@ +/* Copyright (c) 2002,2007-2011, Code Aurora Forum. All rights reserved. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 and + * only version 2 as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + */ +#include +#include + +#include "kgsl.h" +#include "kgsl_sharedmem.h" +#include "kgsl_cffdump.h" + +static struct kgsl_process_private * +_get_priv_from_kobj(struct kobject *kobj) +{ + struct kgsl_process_private *private; + unsigned long name; + + if (!kobj) + return NULL; + + if (sscanf(kobj->name, "%ld", &name) != 1) + return NULL; + + list_for_each_entry(private, &kgsl_driver.process_list, list) { + if (private->pid == name) + return private; + } + + return NULL; +} + +/* sharedmem / memory sysfs files */ + +static ssize_t +process_show(struct kobject *kobj, + struct kobj_attribute *attr, + char *buf) +{ + struct kgsl_process_private *priv; + unsigned int val = 0; + + mutex_lock(&kgsl_driver.process_mutex); + priv = _get_priv_from_kobj(kobj); + + if (priv == NULL) { + mutex_unlock(&kgsl_driver.process_mutex); + return 0; + } + + if (!strncmp(attr->attr.name, "user", 4)) + val = priv->stats.user; + if (!strncmp(attr->attr.name, "user_max", 8)) + val = priv->stats.user_max; + if (!strncmp(attr->attr.name, "mapped", 6)) + val = priv->stats.mapped; + if (!strncmp(attr->attr.name, "mapped_max", 10)) + val = priv->stats.mapped_max; + if (!strncmp(attr->attr.name, "flushes", 7)) + val = priv->stats.flushes; + + mutex_unlock(&kgsl_driver.process_mutex); + return snprintf(buf, PAGE_SIZE, "%u\n", val); +} + +#define KGSL_MEMSTAT_ATTR(_name, _show) \ + static struct kobj_attribute attr_##_name = \ + __ATTR(_name, 0444, _show, NULL) + +KGSL_MEMSTAT_ATTR(user, process_show); +KGSL_MEMSTAT_ATTR(user_max, process_show); +KGSL_MEMSTAT_ATTR(mapped, process_show); +KGSL_MEMSTAT_ATTR(mapped_max, process_show); +KGSL_MEMSTAT_ATTR(flushes, process_show); + +static struct attribute *process_attrs[] = { + &attr_user.attr, + &attr_user_max.attr, + &attr_mapped.attr, + &attr_mapped_max.attr, + &attr_flushes.attr, + NULL +}; + +static struct attribute_group process_attr_group = { + .attrs = process_attrs, +}; + +void +kgsl_process_uninit_sysfs(struct kgsl_process_private *private) +{ + /* Remove the sysfs entry */ + if (private->kobj) { + sysfs_remove_group(private->kobj, &process_attr_group); + kobject_put(private->kobj); + } +} + +void +kgsl_process_init_sysfs(struct kgsl_process_private *private) +{ + unsigned char name[16]; + + /* Add a entry to the sysfs device */ + snprintf(name, sizeof(name), "%d", private->pid); + private->kobj = kobject_create_and_add(name, kgsl_driver.prockobj); + + /* sysfs failure isn't fatal, just annoying */ + if (private->kobj != NULL) { + if (sysfs_create_group(private->kobj, &process_attr_group)) { + kobject_put(private->kobj); + private->kobj = NULL; + } + } +} + +static int kgsl_drv_memstat_show(struct device *dev, + struct device_attribute *attr, + char *buf) +{ + unsigned int val = 0; + + if (!strncmp(attr->attr.name, "vmalloc", 7)) + val = kgsl_driver.stats.vmalloc; + else if (!strncmp(attr->attr.name, "vmalloc_max", 11)) + val = kgsl_driver.stats.vmalloc_max; + else if (!strncmp(attr->attr.name, "coherent", 8)) + val = kgsl_driver.stats.coherent; + else if (!strncmp(attr->attr.name, "coherent_max", 12)) + val = kgsl_driver.stats.coherent_max; + else if (!strncmp(attr->attr.name, "mapped", 6)) + val = kgsl_driver.stats.mapped; + else if (!strncmp(attr->attr.name, "mapped_max", 10)) + val = kgsl_driver.stats.mapped_max; + + return snprintf(buf, PAGE_SIZE, "%u\n", val); +} + +static int kgsl_drv_histogram_show(struct device *dev, + struct device_attribute *attr, + char *buf) +{ + int len = 0; + int i; + + for (i = 0; i < 16; i++) + len += snprintf(buf + len, PAGE_SIZE - len, "%d ", + kgsl_driver.stats.histogram[i]); + + len += snprintf(buf + len, PAGE_SIZE - len, "\n"); + return len; +} + +DEVICE_ATTR(vmalloc, 0444, kgsl_drv_memstat_show, NULL); +DEVICE_ATTR(vmalloc_max, 0444, kgsl_drv_memstat_show, NULL); +DEVICE_ATTR(coherent, 0444, kgsl_drv_memstat_show, NULL); +DEVICE_ATTR(coherent_max, 0444, kgsl_drv_memstat_show, NULL); +DEVICE_ATTR(mapped, 0444, kgsl_drv_memstat_show, NULL); +DEVICE_ATTR(mapped_max, 0444, kgsl_drv_memstat_show, NULL); +DEVICE_ATTR(histogram, 0444, kgsl_drv_histogram_show, NULL); + +static const struct device_attribute *drv_attr_list[] = { + &dev_attr_vmalloc, + &dev_attr_vmalloc_max, + &dev_attr_coherent, + &dev_attr_coherent_max, + &dev_attr_mapped, + &dev_attr_mapped_max, + &dev_attr_histogram, + NULL +}; + +void +kgsl_sharedmem_uninit_sysfs(void) +{ + kgsl_remove_device_sysfs_files(&kgsl_driver.virtdev, drv_attr_list); +} + +int +kgsl_sharedmem_init_sysfs(void) +{ + return kgsl_create_device_sysfs_files(&kgsl_driver.virtdev, + drv_attr_list); +} + +#ifdef CONFIG_OUTER_CACHE +static void _outer_cache_range_op(int op, unsigned long addr, size_t size) +{ + switch (op) { + case KGSL_CACHE_OP_FLUSH: + outer_flush_range(addr, addr + size); + break; + case KGSL_CACHE_OP_CLEAN: + outer_clean_range(addr, addr + size); + break; + case KGSL_CACHE_OP_INV: + outer_inv_range(addr, addr + size); + break; + } +} +#endif + +static unsigned long kgsl_vmalloc_physaddr(struct kgsl_memdesc *memdesc, + unsigned int offset) +{ + unsigned int addr; + + if (offset > memdesc->size) + return 0; + + addr = vmalloc_to_pfn(memdesc->hostptr + offset); + return addr << PAGE_SHIFT; +} + +#ifdef CONFIG_OUTER_CACHE +static void kgsl_vmalloc_outer_cache(struct kgsl_memdesc *memdesc, int op) +{ + void *vaddr = memdesc->hostptr; + for (; vaddr < (memdesc->hostptr + memdesc->size); vaddr += PAGE_SIZE) { + unsigned long paddr = page_to_phys(vmalloc_to_page(vaddr)); + _outer_cache_range_op(op, paddr, PAGE_SIZE); + } +} +#endif + +static int kgsl_vmalloc_vmfault(struct kgsl_memdesc *memdesc, + struct vm_area_struct *vma, + struct vm_fault *vmf) +{ + unsigned long offset, pg; + struct page *page; + + offset = (unsigned long) vmf->virtual_address - vma->vm_start; + pg = (unsigned long) memdesc->hostptr + offset; + + page = vmalloc_to_page((void *) pg); + if (page == NULL) + return VM_FAULT_SIGBUS; + + get_page(page); + + vmf->page = page; + return 0; +} + +static int kgsl_vmalloc_vmflags(struct kgsl_memdesc *memdesc) +{ + return VM_RESERVED | VM_DONTEXPAND; +} + +static void kgsl_vmalloc_free(struct kgsl_memdesc *memdesc) +{ + kgsl_driver.stats.vmalloc -= memdesc->size; + vfree(memdesc->hostptr); +} + +static void kgsl_coherent_free(struct kgsl_memdesc *memdesc) +{ + kgsl_driver.stats.coherent -= memdesc->size; + dma_free_coherent(NULL, memdesc->size, + memdesc->hostptr, memdesc->physaddr); +} + +static unsigned long kgsl_contig_physaddr(struct kgsl_memdesc *memdesc, + unsigned int offset) +{ + if (offset > memdesc->size) + return 0; + + return memdesc->physaddr + offset; +} + +#ifdef CONFIG_OUTER_CACHE +static void kgsl_contig_outer_cache(struct kgsl_memdesc *memdesc, int op) +{ + _outer_cache_range_op(op, memdesc->physaddr, memdesc->size); +} +#endif + +#ifdef CONFIG_OUTER_CACHE +static void kgsl_userptr_outer_cache(struct kgsl_memdesc *memdesc, int op) +{ + void *vaddr = memdesc->hostptr; + for (; vaddr < (memdesc->hostptr + memdesc->size); vaddr += PAGE_SIZE) { + unsigned long paddr = kgsl_virtaddr_to_physaddr(vaddr); + if (paddr) + _outer_cache_range_op(op, paddr, PAGE_SIZE); + } +} +#endif + +static unsigned long kgsl_userptr_physaddr(struct kgsl_memdesc *memdesc, + unsigned int offset) +{ + return kgsl_virtaddr_to_physaddr(memdesc->hostptr + offset); +} + +/* Global - also used by kgsl_drm.c */ +struct kgsl_memdesc_ops kgsl_vmalloc_ops = { + .physaddr = kgsl_vmalloc_physaddr, + .free = kgsl_vmalloc_free, + .vmflags = kgsl_vmalloc_vmflags, + .vmfault = kgsl_vmalloc_vmfault, +#ifdef CONFIG_OUTER_CACHE + .outer_cache = kgsl_vmalloc_outer_cache, +#endif +}; +EXPORT_SYMBOL(kgsl_vmalloc_ops); + +static struct kgsl_memdesc_ops kgsl_coherent_ops = { + .physaddr = kgsl_contig_physaddr, + .free = kgsl_coherent_free, +#ifdef CONFIG_OUTER_CACHE + .outer_cache = kgsl_contig_outer_cache, +#endif +}; + +/* Global - also used by kgsl.c and kgsl_drm.c */ +struct kgsl_memdesc_ops kgsl_contig_ops = { + .physaddr = kgsl_contig_physaddr, +#ifdef CONFIG_OUTER_CACHE + .outer_cache = kgsl_contig_outer_cache +#endif +}; +EXPORT_SYMBOL(kgsl_contig_ops); + +/* Global - also used by kgsl.c */ +struct kgsl_memdesc_ops kgsl_userptr_ops = { + .physaddr = kgsl_userptr_physaddr, +#ifdef CONFIG_OUTER_CACHE + .outer_cache = kgsl_userptr_outer_cache, +#endif +}; +EXPORT_SYMBOL(kgsl_userptr_ops); + +void kgsl_cache_range_op(struct kgsl_memdesc *memdesc, int op) +{ + void *addr = memdesc->hostptr; + int size = memdesc->size; + + switch (op) { + case KGSL_CACHE_OP_FLUSH: + dmac_flush_range(addr, addr + size); + break; + case KGSL_CACHE_OP_CLEAN: + dmac_clean_range(addr, addr + size); + break; + case KGSL_CACHE_OP_INV: + dmac_inv_range(addr, addr + size); + break; + } + + if (memdesc->ops->outer_cache) + memdesc->ops->outer_cache(memdesc, op); +} +EXPORT_SYMBOL(kgsl_cache_range_op); + +static int +_kgsl_sharedmem_vmalloc(struct kgsl_memdesc *memdesc, + struct kgsl_pagetable *pagetable, + void *ptr, size_t size, unsigned int protflags) +{ + int result; + + memdesc->size = size; + memdesc->pagetable = pagetable; + memdesc->priv = KGSL_MEMFLAGS_CACHED; + memdesc->ops = &kgsl_vmalloc_ops; + memdesc->hostptr = (void *) ptr; + + kgsl_cache_range_op(memdesc, KGSL_CACHE_OP_INV); + + result = kgsl_mmu_map(pagetable, memdesc, protflags); + + if (result) { + kgsl_sharedmem_free(memdesc); + } else { + int order; + + KGSL_STATS_ADD(size, kgsl_driver.stats.vmalloc, + kgsl_driver.stats.vmalloc_max); + + order = get_order(size); + + if (order < 16) + kgsl_driver.stats.histogram[order]++; + } + + return result; +} + +int +kgsl_sharedmem_vmalloc(struct kgsl_memdesc *memdesc, + struct kgsl_pagetable *pagetable, size_t size) +{ + void *ptr; + + BUG_ON(size == 0); + + size = ALIGN(size, PAGE_SIZE * 2); + ptr = vmalloc(size); + + if (ptr == NULL) { + KGSL_CORE_ERR("vmalloc(%d) failed\n", size); + return -ENOMEM; + } + + return _kgsl_sharedmem_vmalloc(memdesc, pagetable, ptr, size, + GSL_PT_PAGE_RV | GSL_PT_PAGE_WV); +} +EXPORT_SYMBOL(kgsl_sharedmem_vmalloc); + +int +kgsl_sharedmem_vmalloc_user(struct kgsl_memdesc *memdesc, + struct kgsl_pagetable *pagetable, + size_t size, int flags) +{ + void *ptr; + unsigned int protflags; + + BUG_ON(size == 0); + ptr = vmalloc_user(size); + + if (ptr == NULL) { + KGSL_CORE_ERR("vmalloc_user(%d) failed: allocated=%d\n", + size, kgsl_driver.stats.vmalloc); + return -ENOMEM; + } + + protflags = GSL_PT_PAGE_RV; + if (!(flags & KGSL_MEMFLAGS_GPUREADONLY)) + protflags |= GSL_PT_PAGE_WV; + + return _kgsl_sharedmem_vmalloc(memdesc, pagetable, ptr, size, + protflags); +} +EXPORT_SYMBOL(kgsl_sharedmem_vmalloc_user); + +int +kgsl_sharedmem_alloc_coherent(struct kgsl_memdesc *memdesc, size_t size) +{ + size = ALIGN(size, PAGE_SIZE); + + memdesc->hostptr = dma_alloc_coherent(NULL, size, &memdesc->physaddr, + GFP_KERNEL); + if (memdesc->hostptr == NULL) { + KGSL_CORE_ERR("dma_alloc_coherent(%d) failed\n", size); + return -ENOMEM; + } + + memdesc->size = size; + memdesc->ops = &kgsl_coherent_ops; + + /* Record statistics */ + + KGSL_STATS_ADD(size, kgsl_driver.stats.coherent, + kgsl_driver.stats.coherent_max); + + return 0; +} +EXPORT_SYMBOL(kgsl_sharedmem_alloc_coherent); + +void kgsl_sharedmem_free(struct kgsl_memdesc *memdesc) +{ + if (memdesc == NULL || memdesc->size == 0) + return; + + if (memdesc->gpuaddr) + kgsl_mmu_unmap(memdesc->pagetable, memdesc); + + if (memdesc->ops->free) + memdesc->ops->free(memdesc); + + memset(memdesc, 0, sizeof(*memdesc)); +} +EXPORT_SYMBOL(kgsl_sharedmem_free); + +int +kgsl_sharedmem_readl(const struct kgsl_memdesc *memdesc, + uint32_t *dst, + unsigned int offsetbytes) +{ + BUG_ON(memdesc == NULL || memdesc->hostptr == NULL || dst == NULL); + WARN_ON(offsetbytes + sizeof(unsigned int) > memdesc->size); + + if (offsetbytes + sizeof(unsigned int) > memdesc->size) + return -ERANGE; + + *dst = readl_relaxed(memdesc->hostptr + offsetbytes); + return 0; +} +EXPORT_SYMBOL(kgsl_sharedmem_readl); + +int +kgsl_sharedmem_writel(const struct kgsl_memdesc *memdesc, + unsigned int offsetbytes, + uint32_t src) +{ + BUG_ON(memdesc == NULL || memdesc->hostptr == NULL); + BUG_ON(offsetbytes + sizeof(unsigned int) > memdesc->size); + + kgsl_cffdump_setmem(memdesc->physaddr + offsetbytes, + src, sizeof(uint)); + writel_relaxed(src, memdesc->hostptr + offsetbytes); + return 0; +} +EXPORT_SYMBOL(kgsl_sharedmem_writel); + +int +kgsl_sharedmem_set(const struct kgsl_memdesc *memdesc, unsigned int offsetbytes, + unsigned int value, unsigned int sizebytes) +{ + BUG_ON(memdesc == NULL || memdesc->hostptr == NULL); + BUG_ON(offsetbytes + sizebytes > memdesc->size); + + kgsl_cffdump_setmem(memdesc->physaddr + offsetbytes, value, + sizebytes); + memset(memdesc->hostptr + offsetbytes, value, sizebytes); + return 0; +} +EXPORT_SYMBOL(kgsl_sharedmem_set); diff --git a/drivers/gpu/msm/kgsl_sharedmem.h b/drivers/gpu/msm/kgsl_sharedmem.h new file mode 100644 index 00000000..d0070584 --- /dev/null +++ b/drivers/gpu/msm/kgsl_sharedmem.h @@ -0,0 +1,116 @@ +/* Copyright (c) 2002,2007-2011, Code Aurora Forum. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are + * met: + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following + * disclaimer in the documentation and/or other materials provided + * with the distribution. + * * Neither the name of Code Aurora Forum, Inc. nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED "AS IS" AND ANY EXPRESS OR IMPLIED + * WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS + * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR + * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, + * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE + * OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN + * IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + */ +#ifndef __KGSL_SHAREDMEM_H +#define __KGSL_SHAREDMEM_H + +#include + +struct kgsl_pagetable; +struct kgsl_device; +struct kgsl_process_private; + +#define KGSL_CACHE_OP_INV 0x01 +#define KGSL_CACHE_OP_FLUSH 0x02 +#define KGSL_CACHE_OP_CLEAN 0x03 + +/** Set if the memdesc describes cached memory */ +#define KGSL_MEMFLAGS_CACHED 0x00000001 + +struct kgsl_memdesc; + +struct kgsl_memdesc_ops { + unsigned long (*physaddr)(struct kgsl_memdesc *, unsigned int); + void (*outer_cache)(struct kgsl_memdesc *, int); + int (*vmflags)(struct kgsl_memdesc *); + int (*vmfault)(struct kgsl_memdesc *, struct vm_area_struct *, + struct vm_fault *); + void (*free)(struct kgsl_memdesc *memdesc); +}; + +/* shared memory allocation */ +struct kgsl_memdesc { + struct kgsl_pagetable *pagetable; + void *hostptr; + unsigned int gpuaddr; + unsigned int physaddr; + unsigned int size; + unsigned int priv; + struct kgsl_memdesc_ops *ops; +}; + +extern struct kgsl_memdesc_ops kgsl_vmalloc_ops; +extern struct kgsl_memdesc_ops kgsl_contig_ops; +extern struct kgsl_memdesc_ops kgsl_userptr_ops; + +int kgsl_sharedmem_vmalloc(struct kgsl_memdesc *memdesc, + struct kgsl_pagetable *pagetable, size_t size); + +int kgsl_sharedmem_vmalloc_user(struct kgsl_memdesc *memdesc, + struct kgsl_pagetable *pagetable, + size_t size, int flags); + +int kgsl_sharedmem_alloc_coherent(struct kgsl_memdesc *memdesc, size_t size); + +void kgsl_sharedmem_free(struct kgsl_memdesc *memdesc); + +int kgsl_sharedmem_readl(const struct kgsl_memdesc *memdesc, + uint32_t *dst, + unsigned int offsetbytes); + +int kgsl_sharedmem_writel(const struct kgsl_memdesc *memdesc, + unsigned int offsetbytes, + uint32_t src); + +int kgsl_sharedmem_set(const struct kgsl_memdesc *memdesc, + unsigned int offsetbytes, unsigned int value, + unsigned int sizebytes); + +void kgsl_cache_range_op(struct kgsl_memdesc *memdesc, int op); + +void kgsl_process_init_sysfs(struct kgsl_process_private *private); +void kgsl_process_uninit_sysfs(struct kgsl_process_private *private); + +int kgsl_sharedmem_init_sysfs(void); +void kgsl_sharedmem_uninit_sysfs(void); + +static inline int +kgsl_allocate_user(struct kgsl_memdesc *memdesc, + struct kgsl_pagetable *pagetable, + size_t size, unsigned int flags) +{ + return kgsl_sharedmem_vmalloc_user(memdesc, pagetable, size, flags); +} + +static inline int +kgsl_allocate_contig(struct kgsl_memdesc *memdesc, size_t size) +{ + return kgsl_sharedmem_alloc_coherent(memdesc, size); +} + +#endif /* __KGSL_SHAREDMEM_H */ diff --git a/drivers/gpu/msm/z180.c b/drivers/gpu/msm/z180.c new file mode 100644 index 00000000..27da432e --- /dev/null +++ b/drivers/gpu/msm/z180.c @@ -0,0 +1,1067 @@ +/* Copyright (c) 2002,2007-2011, Code Aurora Forum. All rights reserved. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 and + * only version 2 as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + */ +#include + +#include "kgsl.h" +#include "kgsl_cffdump.h" + +#include "z180.h" +#include "z180_reg.h" + +#define DRIVER_VERSION_MAJOR 3 +#define DRIVER_VERSION_MINOR 1 + +#define Z180_DEVICE(device) \ + KGSL_CONTAINER_OF(device, struct z180_device, dev) + +#define GSL_VGC_INT_MASK \ + (REG_VGC_IRQSTATUS__MH_MASK | \ + REG_VGC_IRQSTATUS__G2D_MASK | \ + REG_VGC_IRQSTATUS__FIFO_MASK) + +#define VGV3_NEXTCMD_JUMP 0x01 + +#define VGV3_NEXTCMD_NEXTCMD_FSHIFT 12 +#define VGV3_NEXTCMD_NEXTCMD_FMASK 0x7 + +#define VGV3_CONTROL_MARKADD_FSHIFT 0 +#define VGV3_CONTROL_MARKADD_FMASK 0xfff + +#define Z180_PACKET_SIZE 15 +#define Z180_MARKER_SIZE 10 +#define Z180_CALL_CMD 0x1000 +#define Z180_MARKER_CMD 0x8000 +#define Z180_STREAM_END_CMD 0x9000 +#define Z180_STREAM_PACKET 0x7C000176 +#define Z180_STREAM_PACKET_CALL 0x7C000275 +#define Z180_PACKET_COUNT 8 +#define Z180_RB_SIZE (Z180_PACKET_SIZE*Z180_PACKET_COUNT \ + *sizeof(uint32_t)) + +#define NUMTEXUNITS 4 +#define TEXUNITREGCOUNT 25 +#define VG_REGCOUNT 0x39 + +#define PACKETSIZE_BEGIN 3 +#define PACKETSIZE_G2DCOLOR 2 +#define PACKETSIZE_TEXUNIT (TEXUNITREGCOUNT * 2) +#define PACKETSIZE_REG (VG_REGCOUNT * 2) +#define PACKETSIZE_STATE (PACKETSIZE_TEXUNIT * NUMTEXUNITS + \ + PACKETSIZE_REG + PACKETSIZE_BEGIN + \ + PACKETSIZE_G2DCOLOR) +#define PACKETSIZE_STATESTREAM (ALIGN((PACKETSIZE_STATE * \ + sizeof(unsigned int)), 32) / \ + sizeof(unsigned int)) + +#define Z180_INVALID_CONTEXT UINT_MAX + +/* z180 MH arbiter config*/ +#define Z180_CFG_MHARB \ + (0x10 \ + | (0 << MH_ARBITER_CONFIG__SAME_PAGE_GRANULARITY__SHIFT) \ + | (1 << MH_ARBITER_CONFIG__L1_ARB_ENABLE__SHIFT) \ + | (1 << MH_ARBITER_CONFIG__L1_ARB_HOLD_ENABLE__SHIFT) \ + | (0 << MH_ARBITER_CONFIG__L2_ARB_CONTROL__SHIFT) \ + | (1 << MH_ARBITER_CONFIG__PAGE_SIZE__SHIFT) \ + | (1 << MH_ARBITER_CONFIG__TC_REORDER_ENABLE__SHIFT) \ + | (1 << MH_ARBITER_CONFIG__TC_ARB_HOLD_ENABLE__SHIFT) \ + | (0 << MH_ARBITER_CONFIG__IN_FLIGHT_LIMIT_ENABLE__SHIFT) \ + | (0x8 << MH_ARBITER_CONFIG__IN_FLIGHT_LIMIT__SHIFT) \ + | (1 << MH_ARBITER_CONFIG__CP_CLNT_ENABLE__SHIFT) \ + | (1 << MH_ARBITER_CONFIG__VGT_CLNT_ENABLE__SHIFT) \ + | (1 << MH_ARBITER_CONFIG__TC_CLNT_ENABLE__SHIFT) \ + | (1 << MH_ARBITER_CONFIG__RB_CLNT_ENABLE__SHIFT) \ + | (1 << MH_ARBITER_CONFIG__PA_CLNT_ENABLE__SHIFT)) + +#define Z180_TIMESTAMP_EPSILON 20000 +#define Z180_IDLE_COUNT_MAX 1000000 + +#define Z180_CMDWINDOW_TARGET_MASK 0x000000FF +#define Z180_CMDWINDOW_ADDR_MASK 0x00FFFF00 +#define Z180_CMDWINDOW_TARGET_SHIFT 0 +#define Z180_CMDWINDOW_ADDR_SHIFT 8 + +static int z180_start(struct kgsl_device *device, unsigned int init_ram); +static int z180_stop(struct kgsl_device *device); +static int z180_wait(struct kgsl_device *device, + unsigned int timestamp, + unsigned int msecs); +static void z180_regread(struct kgsl_device *device, + unsigned int offsetwords, + unsigned int *value); +static void z180_regwrite(struct kgsl_device *device, + unsigned int offsetwords, + unsigned int value); +static int z180_cmdwindow_write(struct kgsl_device *device, + enum kgsl_cmdwindow_type target, + unsigned int addr, + unsigned int data); +static void z180_regread_isr(struct kgsl_device *device, + unsigned int offsetwords, + unsigned int *value); +static void z180_regwrite_isr(struct kgsl_device *device, + unsigned int offsetwords, + unsigned int value); +static void __devinit z180_getfunctable(struct kgsl_functable *ftbl); + +#define Z180_MMU_CONFIG \ + (0x01 \ + | (MMU_CONFIG << MH_MMU_CONFIG__RB_W_CLNT_BEHAVIOR__SHIFT) \ + | (MMU_CONFIG << MH_MMU_CONFIG__CP_W_CLNT_BEHAVIOR__SHIFT) \ + | (MMU_CONFIG << MH_MMU_CONFIG__CP_R0_CLNT_BEHAVIOR__SHIFT) \ + | (MMU_CONFIG << MH_MMU_CONFIG__CP_R1_CLNT_BEHAVIOR__SHIFT) \ + | (MMU_CONFIG << MH_MMU_CONFIG__CP_R2_CLNT_BEHAVIOR__SHIFT) \ + | (MMU_CONFIG << MH_MMU_CONFIG__CP_R3_CLNT_BEHAVIOR__SHIFT) \ + | (MMU_CONFIG << MH_MMU_CONFIG__CP_R4_CLNT_BEHAVIOR__SHIFT) \ + | (MMU_CONFIG << MH_MMU_CONFIG__VGT_R0_CLNT_BEHAVIOR__SHIFT) \ + | (MMU_CONFIG << MH_MMU_CONFIG__VGT_R1_CLNT_BEHAVIOR__SHIFT) \ + | (MMU_CONFIG << MH_MMU_CONFIG__TC_R_CLNT_BEHAVIOR__SHIFT) \ + | (MMU_CONFIG << MH_MMU_CONFIG__PA_W_CLNT_BEHAVIOR__SHIFT)) + +static struct z180_device device_2d0 = { + .dev = { + .name = DEVICE_2D0_NAME, + .id = KGSL_DEVICE_2D0, + .ver_major = DRIVER_VERSION_MAJOR, + .ver_minor = DRIVER_VERSION_MINOR, + .mmu = { + .config = Z180_MMU_CONFIG, + /* turn off memory protection unit by setting + acceptable physical address range to include + all pages. */ + .mpu_base = 0x00000000, + .mpu_range = 0xFFFFF000, + .reg = { + .config = ADDR_MH_MMU_CONFIG, + .mpu_base = ADDR_MH_MMU_MPU_BASE, + .mpu_end = ADDR_MH_MMU_MPU_END, + .va_range = ADDR_MH_MMU_VA_RANGE, + .pt_page = ADDR_MH_MMU_PT_BASE, + .page_fault = ADDR_MH_MMU_PAGE_FAULT, + .tran_error = ADDR_MH_MMU_TRAN_ERROR, + .invalidate = ADDR_MH_MMU_INVALIDATE, + .interrupt_mask = ADDR_MH_INTERRUPT_MASK, + .interrupt_status = ADDR_MH_INTERRUPT_STATUS, + .interrupt_clear = ADDR_MH_INTERRUPT_CLEAR, + .axi_error = ADDR_MH_AXI_ERROR, + }, + }, + .pwrctrl = { + .regulator_name = "fs_gfx2d0", + .irq_name = KGSL_2D0_IRQ, + }, + .mutex = __MUTEX_INITIALIZER(device_2d0.dev.mutex), + .state = KGSL_STATE_INIT, + .active_cnt = 0, + .iomemname = KGSL_2D0_REG_MEMORY, + }, +}; + +static struct z180_device device_2d1 = { + .dev = { + .name = DEVICE_2D1_NAME, + .id = KGSL_DEVICE_2D1, + .ver_major = DRIVER_VERSION_MAJOR, + .ver_minor = DRIVER_VERSION_MINOR, + .mmu = { + .config = Z180_MMU_CONFIG, + /* turn off memory protection unit by setting + acceptable physical address range to include + all pages. */ + .mpu_base = 0x00000000, + .mpu_range = 0xFFFFF000, + .reg = { + .config = ADDR_MH_MMU_CONFIG, + .mpu_base = ADDR_MH_MMU_MPU_BASE, + .mpu_end = ADDR_MH_MMU_MPU_END, + .va_range = ADDR_MH_MMU_VA_RANGE, + .pt_page = ADDR_MH_MMU_PT_BASE, + .page_fault = ADDR_MH_MMU_PAGE_FAULT, + .tran_error = ADDR_MH_MMU_TRAN_ERROR, + .invalidate = ADDR_MH_MMU_INVALIDATE, + .interrupt_mask = ADDR_MH_INTERRUPT_MASK, + .interrupt_status = ADDR_MH_INTERRUPT_STATUS, + .interrupt_clear = ADDR_MH_INTERRUPT_CLEAR, + .axi_error = ADDR_MH_AXI_ERROR, + }, + }, + .pwrctrl = { + .regulator_name = "fs_gfx2d1", + .irq_name = KGSL_2D1_IRQ, + }, + .mutex = __MUTEX_INITIALIZER(device_2d1.dev.mutex), + .state = KGSL_STATE_INIT, + .active_cnt = 0, + .iomemname = KGSL_2D1_REG_MEMORY, + }, +}; + +static irqreturn_t z180_isr(int irq, void *data) +{ + irqreturn_t result = IRQ_NONE; + unsigned int status; + struct kgsl_device *device = (struct kgsl_device *) data; + struct z180_device *z180_dev = Z180_DEVICE(device); + + z180_regread_isr(device, ADDR_VGC_IRQSTATUS >> 2, &status); + + if (status & GSL_VGC_INT_MASK) { + z180_regwrite_isr(device, + ADDR_VGC_IRQSTATUS >> 2, status & GSL_VGC_INT_MASK); + + result = IRQ_HANDLED; + + if (status & REG_VGC_IRQSTATUS__FIFO_MASK) + KGSL_DRV_ERR(device, "z180 fifo interrupt\n"); + if (status & REG_VGC_IRQSTATUS__MH_MASK) + kgsl_mh_intrcallback(device); + if (status & REG_VGC_IRQSTATUS__G2D_MASK) { + int count; + + z180_regread_isr(device, + ADDR_VGC_IRQ_ACTIVE_CNT >> 2, + &count); + + count >>= 8; + count &= 255; + z180_dev->timestamp += count; + + wake_up_interruptible(&device->wait_queue); + + atomic_notifier_call_chain( + &(device->ts_notifier_list), + device->id, NULL); + } + } + + if ((device->pwrctrl.nap_allowed == true) && + (device->requested_state == KGSL_STATE_NONE)) { + device->requested_state = KGSL_STATE_NAP; + queue_work(device->work_queue, &device->idle_check_ws); + } + mod_timer(&device->idle_timer, + jiffies + device->pwrctrl.interval_timeout); + + return result; +} + +static int z180_cleanup_pt(struct kgsl_device *device, + struct kgsl_pagetable *pagetable) +{ + struct z180_device *z180_dev = Z180_DEVICE(device); + + kgsl_mmu_unmap(pagetable, &device->mmu.dummyspace); + + kgsl_mmu_unmap(pagetable, &device->memstore); + + kgsl_mmu_unmap(pagetable, &z180_dev->ringbuffer.cmdbufdesc); + + return 0; +} + +static int z180_setup_pt(struct kgsl_device *device, + struct kgsl_pagetable *pagetable) +{ + int result = 0; + struct z180_device *z180_dev = Z180_DEVICE(device); + + result = kgsl_mmu_map_global(pagetable, &device->mmu.dummyspace, + GSL_PT_PAGE_RV | GSL_PT_PAGE_WV); + + if (result) + goto error; + + result = kgsl_mmu_map_global(pagetable, &device->memstore, + GSL_PT_PAGE_RV | GSL_PT_PAGE_WV); + if (result) + goto error_unmap_dummy; + + result = kgsl_mmu_map_global(pagetable, + &z180_dev->ringbuffer.cmdbufdesc, + GSL_PT_PAGE_RV); + if (result) + goto error_unmap_memstore; + return result; + +error_unmap_dummy: + kgsl_mmu_unmap(pagetable, &device->mmu.dummyspace); + +error_unmap_memstore: + kgsl_mmu_unmap(pagetable, &device->memstore); + +error: + return result; +} + +static inline unsigned int rb_offset(unsigned int index) +{ + return index*sizeof(unsigned int)*(Z180_PACKET_SIZE); +} + +static void addmarker(struct z180_ringbuffer *rb, unsigned int index) +{ + char *ptr = (char *)(rb->cmdbufdesc.hostptr); + unsigned int *p = (unsigned int *)(ptr + rb_offset(index)); + + *p++ = Z180_STREAM_PACKET; + *p++ = (Z180_MARKER_CMD | 5); + *p++ = ADDR_VGV3_LAST << 24; + *p++ = ADDR_VGV3_LAST << 24; + *p++ = ADDR_VGV3_LAST << 24; + *p++ = Z180_STREAM_PACKET; + *p++ = 5; + *p++ = ADDR_VGV3_LAST << 24; + *p++ = ADDR_VGV3_LAST << 24; + *p++ = ADDR_VGV3_LAST << 24; +} + +static void addcmd(struct z180_ringbuffer *rb, unsigned int index, + unsigned int cmd, unsigned int nextcnt) +{ + char * ptr = (char *)(rb->cmdbufdesc.hostptr); + unsigned int *p = (unsigned int *)(ptr + (rb_offset(index) + + (Z180_MARKER_SIZE * sizeof(unsigned int)))); + + *p++ = Z180_STREAM_PACKET_CALL; + *p++ = cmd; + *p++ = Z180_CALL_CMD | nextcnt; + *p++ = ADDR_VGV3_LAST << 24; + *p++ = ADDR_VGV3_LAST << 24; +} + +static int z180_cmdstream_start(struct kgsl_device *device) +{ + struct z180_device *z180_dev = Z180_DEVICE(device); + int result; + unsigned int cmd = VGV3_NEXTCMD_JUMP << VGV3_NEXTCMD_NEXTCMD_FSHIFT; + + z180_dev->timestamp = 0; + z180_dev->current_timestamp = 0; + + addmarker(&z180_dev->ringbuffer, 0); + + result = z180_cmdwindow_write(device, KGSL_CMDWINDOW_2D, + ADDR_VGV3_MODE, 4); + if (result != 0) + return result; + + result = z180_cmdwindow_write(device, KGSL_CMDWINDOW_2D, + ADDR_VGV3_NEXTADDR, + z180_dev->ringbuffer.cmdbufdesc.gpuaddr); + if (result != 0) + return result; + + result = z180_cmdwindow_write(device, KGSL_CMDWINDOW_2D, + ADDR_VGV3_NEXTCMD, cmd | 5); + if (result != 0) + return result; + + result = z180_cmdwindow_write(device, KGSL_CMDWINDOW_2D, + ADDR_VGV3_WRITEADDR, device->memstore.gpuaddr); + + if (result != 0) + return result; + + cmd = (int)(((1) & VGV3_CONTROL_MARKADD_FMASK) + << VGV3_CONTROL_MARKADD_FSHIFT); + + result = z180_cmdwindow_write(device, KGSL_CMDWINDOW_2D, + ADDR_VGV3_CONTROL, cmd); + + if (result != 0) + return result; + + result = z180_cmdwindow_write(device, KGSL_CMDWINDOW_2D, + ADDR_VGV3_CONTROL, 0); + if (result != 0) + return result; + + return result; +} + +static int room_in_rb(struct z180_device *device) +{ + int ts_diff; + + ts_diff = device->current_timestamp - device->timestamp; + + return ts_diff < Z180_PACKET_COUNT; +} + +static int z180_idle(struct kgsl_device *device, unsigned int timeout) +{ + int status = 0; + struct z180_device *z180_dev = Z180_DEVICE(device); + + if (z180_dev->current_timestamp > z180_dev->timestamp) + status = z180_wait(device, z180_dev->current_timestamp, + timeout); + + if (status) + KGSL_DRV_ERR(device, "z180_waittimestamp() timed out\n"); + + return status; +} + +static int z180_setstate(struct kgsl_device *device, uint32_t flags) +{ +#ifdef CONFIG_MSM_KGSL_MMU + unsigned int mh_mmu_invalidate = 0x00000003; /*invalidate all and tc */ + + if (flags & KGSL_MMUFLAGS_PTUPDATE) { + z180_idle(device, KGSL_TIMEOUT_DEFAULT); + z180_regwrite(device, ADDR_MH_MMU_PT_BASE, + device->mmu.hwpagetable->base.gpuaddr); + z180_regwrite(device, ADDR_MH_MMU_VA_RANGE, + (device->mmu.hwpagetable-> + va_base | (device->mmu.hwpagetable-> + va_range >> 16))); + z180_regwrite(device, ADDR_MH_MMU_INVALIDATE, + mh_mmu_invalidate); + } + + if (flags & KGSL_MMUFLAGS_TLBFLUSH) + z180_regwrite(device, ADDR_MH_MMU_INVALIDATE, + mh_mmu_invalidate); +#endif + return 0; +} + +int +z180_cmdstream_issueibcmds(struct kgsl_device_private *dev_priv, + struct kgsl_context *context, + struct kgsl_ibdesc *ibdesc, + unsigned int numibs, + uint32_t *timestamp, + unsigned int ctrl) +{ + unsigned int result = 0; + unsigned int ofs = PACKETSIZE_STATESTREAM * sizeof(unsigned int); + unsigned int cnt = 5; + unsigned int nextaddr = 0; + unsigned int index = 0; + unsigned int nextindex; + unsigned int nextcnt = Z180_STREAM_END_CMD | 5; + struct kgsl_memdesc tmp = {0}; + unsigned int cmd; + struct kgsl_device *device = dev_priv->device; + struct kgsl_pagetable *pagetable = dev_priv->process_priv->pagetable; + struct z180_device *z180_dev = Z180_DEVICE(device); + unsigned int sizedwords; + + if (device->state & KGSL_STATE_HUNG) { + return -EINVAL; + goto error; + } + if (numibs != 1) { + KGSL_DRV_ERR(device, "Invalid number of ibs: %d\n", numibs); + result = -EINVAL; + goto error; + } + cmd = ibdesc[0].gpuaddr; + sizedwords = ibdesc[0].sizedwords; + + tmp.hostptr = (void *)*timestamp; + + KGSL_CMD_INFO(device, "ctxt %d ibaddr 0x%08x sizedwords %d\n", + context->id, cmd, sizedwords); + /* context switch */ + if ((context->id != (int)z180_dev->ringbuffer.prevctx) || + (ctrl & KGSL_CONTEXT_CTX_SWITCH)) { + KGSL_CMD_INFO(device, "context switch %d -> %d\n", + context->id, z180_dev->ringbuffer.prevctx); + kgsl_mmu_setstate(device, pagetable); + cnt = PACKETSIZE_STATESTREAM; + ofs = 0; + } + z180_setstate(device, kgsl_pt_get_flags(device->mmu.hwpagetable, + device->id)); + + result = wait_event_interruptible_timeout(device->wait_queue, + room_in_rb(z180_dev), + msecs_to_jiffies(KGSL_TIMEOUT_DEFAULT)); + if (result < 0) { + KGSL_CMD_ERR(device, "wait_event_interruptible_timeout " + "failed: %d\n", result); + goto error; + } + result = 0; + + index = z180_dev->current_timestamp % Z180_PACKET_COUNT; + z180_dev->current_timestamp++; + nextindex = z180_dev->current_timestamp % Z180_PACKET_COUNT; + *timestamp = z180_dev->current_timestamp; + + z180_dev->ringbuffer.prevctx = context->id; + + addcmd(&z180_dev->ringbuffer, index, cmd + ofs, cnt); + + /* Make sure the next ringbuffer entry has a marker */ + addmarker(&z180_dev->ringbuffer, nextindex); + + nextaddr = z180_dev->ringbuffer.cmdbufdesc.gpuaddr + + rb_offset(nextindex); + + tmp.hostptr = (void *)(tmp.hostptr + + (sizedwords * sizeof(unsigned int))); + tmp.size = 12; + + kgsl_sharedmem_writel(&tmp, 4, nextaddr); + kgsl_sharedmem_writel(&tmp, 8, nextcnt); + + /* sync memory before activating the hardware for the new command*/ + mb(); + + cmd = (int)(((2) & VGV3_CONTROL_MARKADD_FMASK) + << VGV3_CONTROL_MARKADD_FSHIFT); + + z180_cmdwindow_write(device, + KGSL_CMDWINDOW_2D, ADDR_VGV3_CONTROL, cmd); + z180_cmdwindow_write(device, + KGSL_CMDWINDOW_2D, ADDR_VGV3_CONTROL, 0); +error: + return result; +} + +static int z180_ringbuffer_init(struct kgsl_device *device) +{ + struct z180_device *z180_dev = Z180_DEVICE(device); + memset(&z180_dev->ringbuffer, 0, sizeof(struct z180_ringbuffer)); + z180_dev->ringbuffer.prevctx = Z180_INVALID_CONTEXT; + return kgsl_sharedmem_alloc_coherent( + &z180_dev->ringbuffer.cmdbufdesc, + Z180_RB_SIZE); +} + +static void z180_ringbuffer_close(struct kgsl_device *device) +{ + struct z180_device *z180_dev = Z180_DEVICE(device); + kgsl_sharedmem_free(&z180_dev->ringbuffer.cmdbufdesc); + memset(&z180_dev->ringbuffer, 0, sizeof(struct z180_ringbuffer)); +} + +static int __devinit z180_probe(struct platform_device *pdev) +{ + int status = -EINVAL; + struct kgsl_device *device = NULL; + struct z180_device *z180_dev; + + device = (struct kgsl_device *)pdev->id_entry->driver_data; + device->parentdev = &pdev->dev; + + z180_getfunctable(&device->ftbl); + + z180_dev = Z180_DEVICE(device); + spin_lock_init(&z180_dev->cmdwin_lock); + + status = z180_ringbuffer_init(device); + if (status != 0) + goto error; + + status = kgsl_device_platform_probe(device, z180_isr); + if (status) + goto error_close_ringbuffer; + + return status; + +error_close_ringbuffer: + z180_ringbuffer_close(device); +error: + device->parentdev = NULL; + return status; +} + +static int __devexit z180_remove(struct platform_device *pdev) +{ + struct kgsl_device *device = NULL; + + device = (struct kgsl_device *)pdev->id_entry->driver_data; + + kgsl_device_platform_remove(device); + + z180_ringbuffer_close(device); + + return 0; +} + +static int z180_start(struct kgsl_device *device, unsigned int init_ram) +{ + int status = 0; + + device->state = KGSL_STATE_INIT; + device->requested_state = KGSL_STATE_NONE; + KGSL_PWR_WARN(device, "state -> INIT, device %d\n", device->id); + + kgsl_pwrctrl_enable(device); + + /* Set up MH arbiter. MH offsets are considered to be dword + * based, therefore no down shift. */ + z180_regwrite(device, ADDR_MH_ARBITER_CONFIG, Z180_CFG_MHARB); + + z180_regwrite(device, ADDR_MH_CLNT_INTF_CTRL_CONFIG1, 0x00030F27); + z180_regwrite(device, ADDR_MH_CLNT_INTF_CTRL_CONFIG2, 0x004B274F); + + z180_regwrite(device, (ADDR_VGC_IRQENABLE >> 2), 0x3); + + status = kgsl_mmu_start(device); + if (status) + goto error_clk_off; + + status = z180_cmdstream_start(device); + if (status) + goto error_mmu_stop; + + mod_timer(&device->idle_timer, jiffies + FIRST_TIMEOUT); + kgsl_pwrctrl_irq(device, KGSL_PWRFLAGS_IRQ_ON); + return 0; +error_clk_off: + z180_regwrite(device, (ADDR_VGC_IRQENABLE >> 2), 0); + kgsl_pwrctrl_disable(device); +error_mmu_stop: + kgsl_mmu_stop(device); + return status; +} + +static int z180_stop(struct kgsl_device *device) +{ + z180_idle(device, KGSL_TIMEOUT_DEFAULT); + + del_timer(&device->idle_timer); + + kgsl_mmu_stop(device); + + /* Disable the clocks before the power rail. */ + kgsl_pwrctrl_irq(device, KGSL_PWRFLAGS_OFF); + + kgsl_pwrctrl_disable(device); + + return 0; +} + +static int z180_getproperty(struct kgsl_device *device, + enum kgsl_property_type type, + void *value, + unsigned int sizebytes) +{ + int status = -EINVAL; + + switch (type) { + case KGSL_PROP_DEVICE_INFO: + { + struct kgsl_devinfo devinfo; + + if (sizebytes != sizeof(devinfo)) { + status = -EINVAL; + break; + } + + memset(&devinfo, 0, sizeof(devinfo)); + devinfo.device_id = device->id+1; + devinfo.chip_id = 0; + devinfo.mmu_enabled = kgsl_mmu_enabled(); + + if (copy_to_user(value, &devinfo, sizeof(devinfo)) != + 0) { + status = -EFAULT; + break; + } + status = 0; + } + break; + case KGSL_PROP_MMU_ENABLE: + { +#ifdef CONFIG_MSM_KGSL_MMU + int mmuProp = 1; +#else + int mmuProp = 0; +#endif + if (sizebytes != sizeof(int)) { + status = -EINVAL; + break; + } + if (copy_to_user(value, &mmuProp, sizeof(mmuProp))) { + status = -EFAULT; + break; + } + status = 0; + } + break; + + default: + KGSL_DRV_ERR(device, "invalid property: %d\n", type); + status = -EINVAL; + } + return status; +} + +static unsigned int z180_isidle(struct kgsl_device *device) +{ + int status = false; + struct z180_device *z180_dev = Z180_DEVICE(device); + + int timestamp = z180_dev->timestamp; + + if (timestamp == z180_dev->current_timestamp) + status = true; + + return status; +} + +static int z180_resume_context(struct kgsl_device *device) +{ + /* Context is in the pre-amble, automatically restored. */ + + return 0; +} + +static int z180_suspend_context(struct kgsl_device *device) +{ + struct z180_device *z180_dev = Z180_DEVICE(device); + + z180_dev->ringbuffer.prevctx = Z180_INVALID_CONTEXT; + + return 0; +} + +/* Not all Z180 registers are directly accessible. + * The _z180_(read|write)_simple functions below handle the ones that are. + */ +static void _z180_regread_simple(struct kgsl_device *device, + unsigned int offsetwords, + unsigned int *value) +{ + unsigned int *reg; + + BUG_ON(offsetwords * sizeof(uint32_t) >= device->regspace.sizebytes); + + reg = (unsigned int *)(device->regspace.mmio_virt_base + + (offsetwords << 2)); + + /*ensure this read finishes before the next one. + * i.e. act like normal readl() */ + *value = __raw_readl(reg); + rmb(); + +} + +static void _z180_regwrite_simple(struct kgsl_device *device, + unsigned int offsetwords, + unsigned int value) +{ + unsigned int *reg; + + BUG_ON(offsetwords*sizeof(uint32_t) >= device->regspace.sizebytes); + + reg = (unsigned int *)(device->regspace.mmio_virt_base + + (offsetwords << 2)); + kgsl_cffdump_regwrite(device->id, offsetwords << 2, value); + /*ensure previous writes post before this one, + * i.e. act like normal writel() */ + wmb(); + __raw_writel(value, reg); +} + + +/* The MH registers must be accessed through via a 2 step write, (read|write) + * process. These registers may be accessed from interrupt context during + * the handling of MH or MMU error interrupts. Therefore a spin lock is used + * to ensure that the 2 step sequence is not interrupted. + */ +static void _z180_regread_mmu(struct kgsl_device *device, + unsigned int offsetwords, + unsigned int *value) +{ + struct z180_device *z180_dev = Z180_DEVICE(device); + unsigned long flags; + + spin_lock_irqsave(&z180_dev->cmdwin_lock, flags); + _z180_regwrite_simple(device, (ADDR_VGC_MH_READ_ADDR >> 2), + offsetwords); + _z180_regread_simple(device, (ADDR_VGC_MH_DATA_ADDR >> 2), value); + spin_unlock_irqrestore(&z180_dev->cmdwin_lock, flags); +} + + +static void _z180_regwrite_mmu(struct kgsl_device *device, + unsigned int offsetwords, + unsigned int value) +{ + struct z180_device *z180_dev = Z180_DEVICE(device); + unsigned int cmdwinaddr; + unsigned long flags; + + cmdwinaddr = ((KGSL_CMDWINDOW_MMU << Z180_CMDWINDOW_TARGET_SHIFT) & + Z180_CMDWINDOW_TARGET_MASK); + cmdwinaddr |= ((offsetwords << Z180_CMDWINDOW_ADDR_SHIFT) & + Z180_CMDWINDOW_ADDR_MASK); + + spin_lock_irqsave(&z180_dev->cmdwin_lock, flags); + _z180_regwrite_simple(device, ADDR_VGC_MMUCOMMANDSTREAM >> 2, + cmdwinaddr); + _z180_regwrite_simple(device, ADDR_VGC_MMUCOMMANDSTREAM >> 2, value); + spin_unlock_irqrestore(&z180_dev->cmdwin_lock, flags); +} + +/* the rest of the code doesn't want to think about if it is writing mmu + * registers or normal registers so handle it here + */ +static void _z180_regread(struct kgsl_device *device, unsigned int offsetwords, + unsigned int *value) +{ + if ((offsetwords >= ADDR_MH_ARBITER_CONFIG && + offsetwords <= ADDR_MH_AXI_HALT_CONTROL) || + (offsetwords >= ADDR_MH_MMU_CONFIG && + offsetwords <= ADDR_MH_MMU_MPU_END)) { + _z180_regread_mmu(device, offsetwords, value); + } else { + _z180_regread_simple(device, offsetwords, value); + } +} + +static void _z180_regwrite(struct kgsl_device *device, unsigned int offsetwords, + unsigned int value) +{ + if ((offsetwords >= ADDR_MH_ARBITER_CONFIG && + offsetwords <= ADDR_MH_CLNT_INTF_CTRL_CONFIG2) || + (offsetwords >= ADDR_MH_MMU_CONFIG && + offsetwords <= ADDR_MH_MMU_MPU_END)) { + _z180_regwrite_mmu(device, offsetwords, value); + + } else { + _z180_regwrite_simple(device, offsetwords, value); + } +} + + +static void z180_regread(struct kgsl_device *device, + unsigned int offsetwords, + unsigned int *value) +{ + kgsl_pre_hwaccess(device); + _z180_regread(device, offsetwords, value); +} + +static void z180_regread_isr(struct kgsl_device *device, + unsigned int offsetwords, + unsigned int *value) +{ + _z180_regread(device, offsetwords, value); +} + +static void z180_regwrite(struct kgsl_device *device, + unsigned int offsetwords, + unsigned int value) +{ + kgsl_pre_hwaccess(device); + _z180_regwrite(device, offsetwords, value); +} + +static void z180_regwrite_isr(struct kgsl_device *device, + unsigned int offsetwords, + unsigned int value) +{ + _z180_regwrite(device, offsetwords, value); +} + +static int z180_cmdwindow_write(struct kgsl_device *device, + enum kgsl_cmdwindow_type target, unsigned int addr, + unsigned int data) +{ + unsigned int cmdwinaddr; + unsigned int cmdstream; + + if (target < KGSL_CMDWINDOW_MIN || + target > KGSL_CMDWINDOW_MAX) { + KGSL_DRV_ERR(device, "invalid target\n"); + return -EINVAL; + } + + if (target == KGSL_CMDWINDOW_MMU) + cmdstream = ADDR_VGC_MMUCOMMANDSTREAM; + else + cmdstream = ADDR_VGC_COMMANDSTREAM; + + cmdwinaddr = ((target << Z180_CMDWINDOW_TARGET_SHIFT) & + Z180_CMDWINDOW_TARGET_MASK); + cmdwinaddr |= ((addr << Z180_CMDWINDOW_ADDR_SHIFT) & + Z180_CMDWINDOW_ADDR_MASK); + + z180_regwrite(device, cmdstream >> 2, cmdwinaddr); + z180_regwrite(device, cmdstream >> 2, data); + + return 0; +} + +static unsigned int z180_readtimestamp(struct kgsl_device *device, + enum kgsl_timestamp_type type) +{ + struct z180_device *z180_dev = Z180_DEVICE(device); + /* get current EOP timestamp */ + return z180_dev->timestamp; +} + +static int z180_waittimestamp(struct kgsl_device *device, + unsigned int timestamp, + unsigned int msecs) +{ + int status = -EINVAL; + mutex_unlock(&device->mutex); + status = z180_wait(device, timestamp, msecs); + mutex_lock(&device->mutex); + + return status; +} + +static int z180_wait(struct kgsl_device *device, + unsigned int timestamp, + unsigned int msecs) +{ + int status = -EINVAL; + long timeout = 0; + + timeout = wait_io_event_interruptible_timeout( + device->wait_queue, + kgsl_check_timestamp(device, timestamp), + msecs_to_jiffies(msecs)); + + if (timeout > 0) + status = 0; + else if (timeout == 0) { + status = -ETIMEDOUT; + device->state = KGSL_STATE_HUNG; + KGSL_PWR_WARN(device, "state -> HUNG, device %d\n", device->id); + } else + status = timeout; + + return status; +} + +static long +z180_ioctl_cmdwindow_write(struct kgsl_device_private *dev_priv, + void *data) +{ + struct kgsl_cmdwindow_write *param = data; + + return z180_cmdwindow_write(dev_priv->device, + param->target, + param->addr, + param->data); +} + +static int +z180_drawctxt_destroy(struct kgsl_device *device, + struct kgsl_context *context) +{ + struct z180_device *z180_dev = Z180_DEVICE(device); + + z180_idle(device, KGSL_TIMEOUT_DEFAULT); + + if (z180_dev->ringbuffer.prevctx == context->id) { + z180_dev->ringbuffer.prevctx = Z180_INVALID_CONTEXT; + device->mmu.hwpagetable = device->mmu.defaultpagetable; + kgsl_setstate(device, KGSL_MMUFLAGS_PTUPDATE); + } + + return 0; +} + +static long z180_ioctl(struct kgsl_device_private *dev_priv, + unsigned int cmd, void *data) +{ + int result = 0; + + switch (cmd) { + case IOCTL_KGSL_CMDWINDOW_WRITE: + result = z180_ioctl_cmdwindow_write(dev_priv, data); + break; + default: + KGSL_DRV_INFO(dev_priv->device, + "invalid ioctl code %08x\n", cmd); + result = -EINVAL; + break; + } + return result; + +} + +static void z180_power_stats(struct kgsl_device *device, + struct kgsl_power_stats *stats) +{ + stats->total_time = 0; + stats->busy_time = 0; +} + +static void __devinit z180_getfunctable(struct kgsl_functable *ftbl) +{ + if (ftbl == NULL) + return; + ftbl->device_regread = z180_regread; + ftbl->device_regwrite = z180_regwrite; + ftbl->device_regread_isr = z180_regread_isr; + ftbl->device_regwrite_isr = z180_regwrite_isr; + ftbl->device_setstate = z180_setstate; + ftbl->device_idle = z180_idle; + ftbl->device_isidle = z180_isidle; + ftbl->device_suspend_context = z180_suspend_context; + ftbl->device_resume_context = z180_resume_context; + ftbl->device_start = z180_start; + ftbl->device_stop = z180_stop; + ftbl->device_getproperty = z180_getproperty; + ftbl->device_waittimestamp = z180_waittimestamp; + ftbl->device_readtimestamp = z180_readtimestamp; + ftbl->device_issueibcmds = z180_cmdstream_issueibcmds; + ftbl->device_drawctxt_create = NULL; + ftbl->device_drawctxt_destroy = z180_drawctxt_destroy; + ftbl->device_ioctl = z180_ioctl; + ftbl->device_setup_pt = z180_setup_pt; + ftbl->device_cleanup_pt = z180_cleanup_pt; + ftbl->device_power_stats = z180_power_stats, +} + +static struct platform_device_id z180_id_table[] = { + { DEVICE_2D0_NAME, (kernel_ulong_t)&device_2d0.dev, }, + { DEVICE_2D1_NAME, (kernel_ulong_t)&device_2d1.dev, }, + { }, +}; +MODULE_DEVICE_TABLE(platform, z180_id_table); + +static struct platform_driver z180_platform_driver = { + .probe = z180_probe, + .remove = __devexit_p(z180_remove), + .suspend = kgsl_suspend_driver, + .resume = kgsl_resume_driver, + .id_table = z180_id_table, + .driver = { + .owner = THIS_MODULE, + .name = DEVICE_2D_NAME, + .pm = &kgsl_pm_ops, + } +}; + +static int __init kgsl_2d_init(void) +{ + return platform_driver_register(&z180_platform_driver); +} + +static void __exit kgsl_2d_exit(void) +{ + platform_driver_unregister(&z180_platform_driver); +} + +module_init(kgsl_2d_init); +module_exit(kgsl_2d_exit); + +MODULE_DESCRIPTION("2D Graphics driver"); +MODULE_VERSION("1.2"); +MODULE_LICENSE("GPL v2"); +MODULE_ALIAS("platform:kgsl_2d"); diff --git a/drivers/gpu/msm/z180.h b/drivers/gpu/msm/z180.h new file mode 100644 index 00000000..c62398a5 --- /dev/null +++ b/drivers/gpu/msm/z180.h @@ -0,0 +1,49 @@ +/* Copyright (c) 2008-2011, Code Aurora Forum. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are + * met: + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following + * disclaimer in the documentation and/or other materials provided + * with the distribution. + * * Neither the name of Code Aurora Forum, Inc. nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED "AS IS" AND ANY EXPRESS OR IMPLIED + * WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS + * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR + * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, + * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE + * OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN + * IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + */ +#ifndef __Z180_H +#define __Z180_H + +#define DEVICE_2D_NAME "kgsl-2d" +#define DEVICE_2D0_NAME "kgsl-2d0" +#define DEVICE_2D1_NAME "kgsl-2d1" + +struct z180_ringbuffer { + unsigned int prevctx; + struct kgsl_memdesc cmdbufdesc; +}; + +struct z180_device { + struct kgsl_device dev; /* Must be first field in this struct */ + int current_timestamp; + int timestamp; + struct z180_ringbuffer ringbuffer; + spinlock_t cmdwin_lock; +}; + +#endif /* __Z180_H */ diff --git a/drivers/gpu/msm/z180_reg.h b/drivers/gpu/msm/z180_reg.h new file mode 100644 index 00000000..f5625535 --- /dev/null +++ b/drivers/gpu/msm/z180_reg.h @@ -0,0 +1,93 @@ +/* Copyright (c) 2002,2007-2011, Code Aurora Forum. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are + * met: + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following + * disclaimer in the documentation and/or other materials provided + * with the distribution. + * * Neither the name of Code Aurora Forum, Inc. nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED "AS IS" AND ANY EXPRESS OR IMPLIED + * WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS + * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR + * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, + * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE + * OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN + * IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + */ +#ifndef __Z80_REG_H +#define __Z80_REG_H + +#define REG_VGC_IRQSTATUS__MH_MASK 0x00000001L +#define REG_VGC_IRQSTATUS__G2D_MASK 0x00000002L +#define REG_VGC_IRQSTATUS__FIFO_MASK 0x00000004L + +#define MH_ARBITER_CONFIG__SAME_PAGE_GRANULARITY__SHIFT 0x00000006 +#define MH_ARBITER_CONFIG__L1_ARB_ENABLE__SHIFT 0x00000007 +#define MH_ARBITER_CONFIG__L1_ARB_HOLD_ENABLE__SHIFT 0x00000008 +#define MH_ARBITER_CONFIG__L2_ARB_CONTROL__SHIFT 0x00000009 +#define MH_ARBITER_CONFIG__PAGE_SIZE__SHIFT 0x0000000a +#define MH_ARBITER_CONFIG__TC_REORDER_ENABLE__SHIFT 0x0000000d +#define MH_ARBITER_CONFIG__TC_ARB_HOLD_ENABLE__SHIFT 0x0000000e +#define MH_ARBITER_CONFIG__IN_FLIGHT_LIMIT_ENABLE__SHIFT 0x0000000f +#define MH_ARBITER_CONFIG__IN_FLIGHT_LIMIT__SHIFT 0x00000010 +#define MH_ARBITER_CONFIG__CP_CLNT_ENABLE__SHIFT 0x00000016 +#define MH_ARBITER_CONFIG__VGT_CLNT_ENABLE__SHIFT 0x00000017 +#define MH_ARBITER_CONFIG__TC_CLNT_ENABLE__SHIFT 0x00000018 +#define MH_ARBITER_CONFIG__RB_CLNT_ENABLE__SHIFT 0x00000019 +#define MH_ARBITER_CONFIG__PA_CLNT_ENABLE__SHIFT 0x0000001a + +#define MH_MMU_CONFIG__RB_W_CLNT_BEHAVIOR__SHIFT 0x00000004 +#define MH_MMU_CONFIG__CP_W_CLNT_BEHAVIOR__SHIFT 0x00000006 +#define MH_MMU_CONFIG__CP_R0_CLNT_BEHAVIOR__SHIFT 0x00000008 +#define MH_MMU_CONFIG__CP_R1_CLNT_BEHAVIOR__SHIFT 0x0000000a +#define MH_MMU_CONFIG__CP_R2_CLNT_BEHAVIOR__SHIFT 0x0000000c +#define MH_MMU_CONFIG__CP_R3_CLNT_BEHAVIOR__SHIFT 0x0000000e +#define MH_MMU_CONFIG__CP_R4_CLNT_BEHAVIOR__SHIFT 0x00000010 +#define MH_MMU_CONFIG__VGT_R0_CLNT_BEHAVIOR__SHIFT 0x00000012 +#define MH_MMU_CONFIG__VGT_R1_CLNT_BEHAVIOR__SHIFT 0x00000014 +#define MH_MMU_CONFIG__TC_R_CLNT_BEHAVIOR__SHIFT 0x00000016 +#define MH_MMU_CONFIG__PA_W_CLNT_BEHAVIOR__SHIFT 0x00000018 + +#define ADDR_MH_ARBITER_CONFIG 0x0A40 +#define ADDR_MH_INTERRUPT_CLEAR 0x0A44 +#define ADDR_MH_INTERRUPT_MASK 0x0A42 +#define ADDR_MH_INTERRUPT_STATUS 0x0A43 +#define ADDR_MH_AXI_ERROR 0x0A45 +#define ADDR_MH_AXI_HALT_CONTROL 0x0A50 +#define ADDR_MH_CLNT_INTF_CTRL_CONFIG1 0x0A54 +#define ADDR_MH_CLNT_INTF_CTRL_CONFIG2 0x0A55 +#define ADDR_MH_MMU_CONFIG 0x0040 +#define ADDR_MH_MMU_INVALIDATE 0x0045 +#define ADDR_MH_MMU_MPU_BASE 0x0046 +#define ADDR_MH_MMU_MPU_END 0x0047 +#define ADDR_MH_MMU_PT_BASE 0x0042 +#define ADDR_MH_MMU_TRAN_ERROR 0x0044 +#define ADDR_MH_MMU_VA_RANGE 0x0041 +#define ADDR_VGC_MH_READ_ADDR 0x0510 +#define ADDR_VGC_MH_DATA_ADDR 0x0518 +#define ADDR_MH_MMU_PAGE_FAULT 0x0043 +#define ADDR_VGC_COMMANDSTREAM 0x0000 +#define ADDR_VGC_IRQENABLE 0x0438 +#define ADDR_VGC_IRQSTATUS 0x0418 +#define ADDR_VGC_IRQ_ACTIVE_CNT 0x04E0 +#define ADDR_VGC_MMUCOMMANDSTREAM 0x03FC +#define ADDR_VGV3_CONTROL 0x0070 +#define ADDR_VGV3_LAST 0x007F +#define ADDR_VGV3_MODE 0x0071 +#define ADDR_VGV3_NEXTADDR 0x0075 +#define ADDR_VGV3_NEXTCMD 0x0076 +#define ADDR_VGV3_WRITEADDR 0x0072 + +#endif /* __Z180_REG_H */ diff --git a/drivers/misc/pmem.c b/drivers/misc/pmem.c index 7f3b5321..6c09d7c6 100644 --- a/drivers/misc/pmem.c +++ b/drivers/misc/pmem.c @@ -1,3 +1,3045 @@ +#ifdef CONFIG_MSM_KGSL +/* drivers/android/pmem.c + * + * Copyright (C) 2007 Google, Inc. + * Copyright (c) 2009-2010, Code Aurora Forum. All rights reserved. +* + * This software is licensed under the terms of the GNU General Public + * License version 2, as published by the Free Software Foundation, and + * may be copied, distributed, and modified under those terms. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + */ + + + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#ifdef CONFIG_MEMORY_HOTPLUG +#include +#include +#endif +#include +#include +#include +#include +#include +#include + +#define PMEM_MAX_USER_SPACE_DEVICES (10) +#define PMEM_MAX_KERNEL_SPACE_DEVICES (2) +#define PMEM_MAX_DEVICES \ + (PMEM_MAX_USER_SPACE_DEVICES + PMEM_MAX_KERNEL_SPACE_DEVICES) + +#define PMEM_MAX_ORDER (128) +#define PMEM_MIN_ALLOC PAGE_SIZE + +#define PMEM_INITIAL_NUM_BITMAP_ALLOCATIONS (64) + +#define PMEM_32BIT_WORD_ORDER (5) +#define PMEM_BITS_PER_WORD_MASK (BITS_PER_LONG - 1) + +#ifdef CONFIG_ANDROID_PMEM_DEBUG +#define PMEM_DEBUG 1 +#else +#define PMEM_DEBUG 0 +#endif + +/* indicates that a refernce to this file has been taken via get_pmem_file, + * the file should not be released until put_pmem_file is called */ +#define PMEM_FLAGS_BUSY 0x1 +/* indicates that this is a suballocation of a larger master range */ +#define PMEM_FLAGS_CONNECTED 0x1 << 1 +/* indicates this is a master and not a sub allocation and that it is mmaped */ +#define PMEM_FLAGS_MASTERMAP 0x1 << 2 +/* submap and unsubmap flags indicate: + * 00: subregion has never been mmaped + * 10: subregion has been mmaped, reference to the mm was taken + * 11: subretion has ben released, refernece to the mm still held + * 01: subretion has been released, reference to the mm has been released + */ +#define PMEM_FLAGS_SUBMAP 0x1 << 3 +#define PMEM_FLAGS_UNSUBMAP 0x1 << 4 + +struct pmem_data { + /* in alloc mode: an index into the bitmap + * in no_alloc mode: the size of the allocation */ + int index; + /* see flags above for descriptions */ + unsigned int flags; + /* protects this data field, if the mm_mmap sem will be held at the + * same time as this sem, the mm sem must be taken first (as this is + * the order for vma_open and vma_close ops */ + struct rw_semaphore sem; + /* info about the mmaping process */ + struct vm_area_struct *vma; + /* task struct of the mapping process */ + struct task_struct *task; + /* process id of teh mapping process */ + pid_t pid; + /* file descriptor of the master */ + int master_fd; + /* file struct of the master */ + struct file *master_file; + /* a list of currently available regions if this is a suballocation */ + struct list_head region_list; + /* a linked list of data so we can access them for debugging */ + struct list_head list; +#if PMEM_DEBUG + int ref; +#endif +}; + +struct pmem_bits { + unsigned allocated:1; /* 1 if allocated, 0 if free */ + unsigned order:7; /* size of the region in pmem space */ +}; + +struct pmem_region_node { + struct pmem_region region; + struct list_head list; +}; + +#define PMEM_DEBUG_MSGS 0 +#if PMEM_DEBUG_MSGS +#define DLOG(fmt,args...) \ + do { pr_debug("[%s:%s:%d] "fmt, __FILE__, __func__, __LINE__, \ + ##args); } \ + while (0) +#else +#define DLOG(x...) do {} while (0) +#endif + +enum pmem_align { + PMEM_ALIGN_4K, + PMEM_ALIGN_1M, +}; + +#define PMEM_NAME_SIZE 16 + +#define MEMORY_STABLE 0 +#define MEMORY_UNSTABLE_NO_MEMORY_ALLOCATED 1 +#define MEMORY_UNSTABLE_MEMORY_ALLOCATED 2 + +#define NO_UNSTABLE_MEMORY 0 +#define UNSTABLE_UNINITIALIZED 1 +#define UNSTABLE_INITIALIZED 2 + +int unstable_pmem_present; +/* start of unstable PMEM physical memory */ +unsigned long unstable_pmem_start; +/* size of unstable PMEM physical memory */ +unsigned long unstable_pmem_size; + +struct pmem_info { + struct miscdevice dev; + /* physical start address of the remaped pmem space */ + unsigned long base; + /* vitual start address of the remaped pmem space */ + unsigned char __iomem *vbase; + /* total size of the pmem space */ + unsigned long size; + /* number of entries in the pmem space */ + unsigned long num_entries; + /* pfn of the garbage page in memory */ + unsigned long garbage_pfn; + /* memory state (stable/unstable with or without memory */ + int memory_state; + + char name[PMEM_NAME_SIZE]; + + /* index of the garbage page in the pmem space */ + int garbage_index; + + enum pmem_allocator_type allocator_type; + + int (*allocate)(const int, + const unsigned long, + const unsigned int); + int (*free)(int, int); + int (*free_space)(int, struct pmem_freespace *); + unsigned long (*len)(int, struct pmem_data *); + unsigned long (*start_addr)(int, struct pmem_data *); + int (*kapi_free_index)(const int32_t, int); + + /* actual size of memory element, e.g.: (4 << 10) is 4K */ + unsigned int quantum; + + /* indicates maps of this region should be cached, if a mix of + * cached and uncached is desired, set this and open the device with + * O_SYNC to get an uncached region */ + unsigned cached; + unsigned buffered; + union { + struct { + /* in all_or_nothing allocator mode the first mapper + * gets the whole space and sets this flag */ + unsigned allocated; + } all_or_nothing; + + struct { + /* the buddy allocator bitmap for the region + * indicating which entries are allocated and which + * are free. + */ + + struct pmem_bits *buddy_bitmap; + } buddy_bestfit; + + struct { + unsigned int bitmap_free; /* # of zero bits/quanta */ + uint32_t *bitmap; + int32_t bitmap_allocs; + struct { + short bit; + unsigned short quanta; + } *bitm_alloc; + } bitmap; + } allocator; + + int id; + struct kobject kobj; + + /* for debugging, creates a list of pmem file structs, the + * data_list_mutex should be taken before pmem_data->sem if both are + * needed */ + struct mutex data_list_mutex; + struct list_head data_list; + /* arena_mutex protects the global allocation arena + * + * IF YOU TAKE BOTH LOCKS TAKE THEM IN THIS ORDER: + * down(pmem_data->sem) => mutex_lock(arena_mutex) + */ + struct mutex arena_mutex; + + long (*ioctl)(struct file *, unsigned int, unsigned long); + int (*release)(struct inode *, struct file *); +}; +#define to_pmem_info_id(a) (container_of(a, struct pmem_info, kobj)->id) + +static struct pmem_info pmem[PMEM_MAX_DEVICES]; +static int id_count; +static struct { + const char * const name; + const int memtype; + const int fallback_memtype; + int info_id; +} kapi_memtypes[] = { +#ifdef CONFIG_KERNEL_PMEM_SMI_REGION + { PMEM_KERNEL_SMI_DATA_NAME, + PMEM_MEMTYPE_SMI, + PMEM_MEMTYPE_EBI1, /* Fall back to EBI1 automatically */ + -1 }, +#endif + { PMEM_KERNEL_EBI1_DATA_NAME, + PMEM_MEMTYPE_EBI1, + PMEM_INVALID_MEMTYPE, /* MUST be set invalid if no fallback */ + -1 }, +}; + +#define PMEM_SYSFS_DIR_NAME "pmem_regions" /* under /sys/kernel/ */ +static struct kset *pmem_kset; + +#define PMEM_IS_FREE_BUDDY(id, index) \ + (!(pmem[id].allocator.buddy_bestfit.buddy_bitmap[index].allocated)) +#define PMEM_BUDDY_ORDER(id, index) \ + (pmem[id].allocator.buddy_bestfit.buddy_bitmap[index].order) +#define PMEM_BUDDY_INDEX(id, index) \ + (index ^ (1 << PMEM_BUDDY_ORDER(id, index))) +#define PMEM_BUDDY_NEXT_INDEX(id, index) \ + (index + (1 << PMEM_BUDDY_ORDER(id, index))) +#define PMEM_OFFSET(index) (index * pmem[id].quantum) +#define PMEM_START_ADDR(id, index) \ + (PMEM_OFFSET(index) + pmem[id].base) +#define PMEM_BUDDY_LEN(id, index) \ + ((1 << PMEM_BUDDY_ORDER(id, index)) * pmem[id].quantum) +#define PMEM_END_ADDR(id, index) \ + (PMEM_START_ADDR(id, index) + PMEM_LEN(id, index)) +#define PMEM_START_VADDR(id, index) \ + (PMEM_OFFSET(id, index) + pmem[id].vbase) +#define PMEM_END_VADDR(id, index) \ + (PMEM_START_VADDR(id, index) + PMEM_LEN(id, index)) +#define PMEM_REVOKED(data) (data->flags & PMEM_FLAGS_REVOKED) +#define PMEM_IS_PAGE_ALIGNED(addr) (!((addr) & (~PAGE_MASK))) +#define PMEM_IS_SUBMAP(data) \ + ((data->flags & PMEM_FLAGS_SUBMAP) && \ + (!(data->flags & PMEM_FLAGS_UNSUBMAP))) + +static int pmem_release(struct inode *, struct file *); +static int pmem_mmap(struct file *, struct vm_area_struct *); +static int pmem_open(struct inode *, struct file *); +static long pmem_ioctl(struct file *, unsigned int, unsigned long); + +struct file_operations pmem_fops = { + .release = pmem_release, + .mmap = pmem_mmap, + .open = pmem_open, + .unlocked_ioctl = pmem_ioctl, +}; + +#define PMEM_ATTR(_name, _mode, _show, _store) { \ + .attr = {.name = __stringify(_name), .mode = _mode }, \ + .show = _show, \ + .store = _store, \ +} + +struct pmem_attr { + struct attribute attr; + ssize_t(*show) (const int id, char * const); + ssize_t(*store) (const int id, const char * const, const size_t count); +}; +#define to_pmem_attr(a) container_of(a, struct pmem_attr, attr) + +#define RW_PMEM_ATTR(name) \ +static struct pmem_attr pmem_attr_## name = \ + PMEM_ATTR(name, S_IRUGO | S_IWUSR, show_pmem_## name, store_pmem_## name) + +#define RO_PMEM_ATTR(name) \ +static struct pmem_attr pmem_attr_## name = \ + PMEM_ATTR(name, S_IRUGO, show_pmem_## name, NULL) + +#define WO_PMEM_ATTR(name) \ +static struct pmem_attr pmem_attr_## name = \ + PMEM_ATTR(name, S_IWUSR, NULL, store_pmem_## name) +/*HTC_START*/ +static struct dentry *root = NULL; +u32 misc_msg_pmem_qcom = 0; + +static struct dentry *vidc_debugfs_root; + +static struct dentry *vidc_get_debugfs_root(void) +{ + if (vidc_debugfs_root == NULL) + vidc_debugfs_root = debugfs_create_dir("misc", NULL); + return vidc_debugfs_root; +} + +static void vidc_debugfs_file_create(struct dentry *root, const char *name, + u32 *var) +{ + struct dentry *vidc_debugfs_file = + debugfs_create_u32(name, S_IRUGO | S_IWUSR, root, var); + if (!vidc_debugfs_file) + pr_info("%s(): Error creating/opening file %s\n", __func__, name); +} +/*HTC_END*/ +static ssize_t show_pmem(struct kobject *kobj, + struct attribute *attr, + char *buf) +{ + struct pmem_attr *a = to_pmem_attr(attr); + return a->show ? a->show(to_pmem_info_id(kobj), buf) : -EIO; +} + +static ssize_t store_pmem(struct kobject *kobj, struct attribute *attr, + const char *buf, size_t count) +{ + struct pmem_attr *a = to_pmem_attr(attr); + return a->store ? a->store(to_pmem_info_id(kobj), buf, count) : -EIO; +} + +static struct sysfs_ops pmem_ops = { + .show = show_pmem, + .store = store_pmem, +}; + +static ssize_t show_pmem_base(int id, char *buf) +{ + return scnprintf(buf, PAGE_SIZE, "%lu(%#lx)\n", + pmem[id].base, pmem[id].base); +} +RO_PMEM_ATTR(base); + +static ssize_t show_pmem_size(int id, char *buf) +{ + return scnprintf(buf, PAGE_SIZE, "%lu(%#lx)\n", + pmem[id].size, pmem[id].size); +} +RO_PMEM_ATTR(size); + +static ssize_t show_pmem_allocator_type(int id, char *buf) +{ + switch (pmem[id].allocator_type) { + case PMEM_ALLOCATORTYPE_ALLORNOTHING: + return scnprintf(buf, PAGE_SIZE, "%s\n", "All or Nothing"); + case PMEM_ALLOCATORTYPE_BUDDYBESTFIT: + return scnprintf(buf, PAGE_SIZE, "%s\n", "Buddy Bestfit"); + case PMEM_ALLOCATORTYPE_BITMAP: + return scnprintf(buf, PAGE_SIZE, "%s\n", "Bitmap"); + default: + return scnprintf(buf, PAGE_SIZE, + "??? Invalid allocator type (%d) for this region! " + "Something isn't right.\n", + pmem[id].allocator_type); + } +} +RO_PMEM_ATTR(allocator_type); + +static ssize_t show_pmem_mapped_regions(int id, char *buf) +{ + struct list_head *elt; + int ret; + + ret = scnprintf(buf, PAGE_SIZE, + "pid #: mapped regions (offset, len) (offset,len)...\n"); + + mutex_lock(&pmem[id].data_list_mutex); + list_for_each(elt, &pmem[id].data_list) { + struct pmem_data *data = + list_entry(elt, struct pmem_data, list); + struct list_head *elt2; + + down_read(&data->sem); + ret += scnprintf(buf + ret, PAGE_SIZE - ret, "pid %u:", + data->pid); + list_for_each(elt2, &data->region_list) { + struct pmem_region_node *region_node = list_entry(elt2, + struct pmem_region_node, + list); + ret += scnprintf(buf + ret, PAGE_SIZE - ret, + "(%lx,%lx) ", + region_node->region.offset, + region_node->region.len); + } + up_read(&data->sem); + ret += scnprintf(buf + ret, PAGE_SIZE - ret, "\n"); + } + mutex_unlock(&pmem[id].data_list_mutex); + return ret; +} +RO_PMEM_ATTR(mapped_regions); + +#define PMEM_COMMON_SYSFS_ATTRS \ + &pmem_attr_base.attr, \ + &pmem_attr_size.attr, \ + &pmem_attr_allocator_type.attr, \ + &pmem_attr_mapped_regions.attr + + +static ssize_t show_pmem_allocated(int id, char *buf) +{ + ssize_t ret; + + mutex_lock(&pmem[id].arena_mutex); + ret = scnprintf(buf, PAGE_SIZE, "%s\n", + pmem[id].allocator.all_or_nothing.allocated ? + "is allocated" : "is NOT allocated"); + mutex_unlock(&pmem[id].arena_mutex); + return ret; +} +RO_PMEM_ATTR(allocated); + +static struct attribute *pmem_allornothing_attrs[] = { + PMEM_COMMON_SYSFS_ATTRS, + + &pmem_attr_allocated.attr, + + NULL +}; + +static struct kobj_type pmem_allornothing_ktype = { + .sysfs_ops = &pmem_ops, + .default_attrs = pmem_allornothing_attrs, +}; + +static ssize_t show_pmem_total_entries(int id, char *buf) +{ + return scnprintf(buf, PAGE_SIZE, "%lu\n", pmem[id].num_entries); +} +RO_PMEM_ATTR(total_entries); + +static ssize_t show_pmem_quantum_size(int id, char *buf) +{ + return scnprintf(buf, PAGE_SIZE, "%u (%#x)\n", + pmem[id].quantum, pmem[id].quantum); +} +RO_PMEM_ATTR(quantum_size); + +static ssize_t show_pmem_buddy_bitmap_dump(int id, char *buf) +{ + int ret, i; + + mutex_lock(&pmem[id].data_list_mutex); + ret = scnprintf(buf, PAGE_SIZE, "index\torder\tlength\tallocated\n"); + + for (i = 0; i < pmem[id].num_entries && (PAGE_SIZE - ret); + i = PMEM_BUDDY_NEXT_INDEX(id, i)) + ret += scnprintf(buf + ret, PAGE_SIZE - ret, "%d\t%d\t%d\t%d\n", + i, PMEM_BUDDY_ORDER(id, i), + PMEM_BUDDY_LEN(id, i), + !PMEM_IS_FREE_BUDDY(id, i)); + + mutex_unlock(&pmem[id].data_list_mutex); + return ret; +} +RO_PMEM_ATTR(buddy_bitmap_dump); + +#define PMEM_BITMAP_BUDDY_BESTFIT_COMMON_SYSFS_ATTRS \ + &pmem_attr_quantum_size.attr, \ + &pmem_attr_total_entries.attr + +static struct attribute *pmem_buddy_bestfit_attrs[] = { + PMEM_COMMON_SYSFS_ATTRS, + + PMEM_BITMAP_BUDDY_BESTFIT_COMMON_SYSFS_ATTRS, + + &pmem_attr_buddy_bitmap_dump.attr, + + NULL +}; + +static struct kobj_type pmem_buddy_bestfit_ktype = { + .sysfs_ops = &pmem_ops, + .default_attrs = pmem_buddy_bestfit_attrs, +}; + +static ssize_t show_pmem_free_quanta(int id, char *buf) +{ + ssize_t ret; + + mutex_lock(&pmem[id].arena_mutex); + ret = scnprintf(buf, PAGE_SIZE, "%u\n", + pmem[id].allocator.bitmap.bitmap_free); + mutex_unlock(&pmem[id].arena_mutex); + return ret; +} +RO_PMEM_ATTR(free_quanta); + +static ssize_t show_pmem_bits_allocated(int id, char *buf) +{ + ssize_t ret; + unsigned int i; + + mutex_lock(&pmem[id].arena_mutex); + + ret = scnprintf(buf, PAGE_SIZE, + "id: %d\nbitnum\tindex\tquanta allocated\n", id); + + for (i = 0; i < pmem[id].allocator.bitmap.bitmap_allocs; i++) + if (pmem[id].allocator.bitmap.bitm_alloc[i].bit != -1) + ret += scnprintf(buf + ret, PAGE_SIZE - ret, + "%u\t%u\t%u\n", + i, + pmem[id].allocator.bitmap.bitm_alloc[i].bit, + pmem[id].allocator.bitmap.bitm_alloc[i].quanta + ); + + mutex_unlock(&pmem[id].arena_mutex); + return ret; +} +RO_PMEM_ATTR(bits_allocated); + +static struct attribute *pmem_bitmap_attrs[] = { + PMEM_COMMON_SYSFS_ATTRS, + + PMEM_BITMAP_BUDDY_BESTFIT_COMMON_SYSFS_ATTRS, + + &pmem_attr_free_quanta.attr, + &pmem_attr_bits_allocated.attr, + + NULL +}; + +static struct kobj_type pmem_bitmap_ktype = { + .sysfs_ops = &pmem_ops, + .default_attrs = pmem_bitmap_attrs, +}; + +static int get_id(struct file *file) +{ + return MINOR(file->f_dentry->d_inode->i_rdev); +} + +static char *get_name(struct file *file) +{ + int id = get_id(file); + return pmem[id].name; +} + +int is_pmem_file(struct file *file) +{ + int id; + + if (unlikely(!file || !file->f_dentry || !file->f_dentry->d_inode)) + return 0; + + id = get_id(file); + return (unlikely(id >= PMEM_MAX_DEVICES || + file->f_dentry->d_inode->i_rdev != + MKDEV(MISC_MAJOR, pmem[id].dev.minor))) ? 0 : 1; +} + +static int has_allocation(struct file *file) +{ + /* must be called with at least read lock held on + * ((struct pmem_data *)(file->private_data))->sem which + * means that file is guaranteed not to be NULL upon entry!! + * check is_pmem_file first if not accessed via pmem_file_ops */ + struct pmem_data *pdata = file->private_data; + return pdata && pdata->index >= 0; +} + +static int is_master_owner(struct file *file) +{ + struct file *master_file; + struct pmem_data *data = file->private_data; + int put_needed, ret = 0; + + if (!has_allocation(file)) + return 0; + if (PMEM_FLAGS_MASTERMAP & data->flags) + return 1; + master_file = fget_light(data->master_fd, &put_needed); + if (master_file && data->master_file == master_file) + ret = 1; + fput_light(master_file, put_needed); + return ret; +} + +static int pmem_free_all_or_nothing(int id, int index) +{ + /* caller should hold the lock on arena_mutex! */ + DLOG("index %d\n", index); + + pmem[id].allocator.all_or_nothing.allocated = 0; + return 0; +} + +static int pmem_free_space_all_or_nothing(int id, + struct pmem_freespace *fs) +{ + /* caller should hold the lock on arena_mutex! */ + fs->total = (unsigned long) + pmem[id].allocator.all_or_nothing.allocated == 0 ? + pmem[id].size : 0; + + fs->largest = fs->total; + return 0; +} + + +static int pmem_free_buddy_bestfit(int id, int index) +{ + /* caller should hold the lock on arena_mutex! */ + int curr = index; + DLOG("index %d\n", index); + + + /* clean up the bitmap, merging any buddies */ + pmem[id].allocator.buddy_bestfit.buddy_bitmap[curr].allocated = 0; + /* find a slots buddy Buddy# = Slot# ^ (1 << order) + * if the buddy is also free merge them + * repeat until the buddy is not free or end of the bitmap is reached + */ + do { + int buddy = PMEM_BUDDY_INDEX(id, curr); + if (buddy < pmem[id].num_entries && + PMEM_IS_FREE_BUDDY(id, buddy) && + PMEM_BUDDY_ORDER(id, buddy) == + PMEM_BUDDY_ORDER(id, curr)) { + PMEM_BUDDY_ORDER(id, buddy)++; + PMEM_BUDDY_ORDER(id, curr)++; + curr = min(buddy, curr); + } else { + break; + } + } while (curr < pmem[id].num_entries); + + return 0; +} + + +static int pmem_free_space_buddy_bestfit(int id, + struct pmem_freespace *fs) +{ + /* caller should hold the lock on arena_mutex! */ + int curr; + unsigned long size; + fs->total = 0; + fs->largest = 0; + + for (curr = 0; curr < pmem[id].num_entries; + curr = PMEM_BUDDY_NEXT_INDEX(id, curr)) { + if (PMEM_IS_FREE_BUDDY(id, curr)) { + size = PMEM_BUDDY_LEN(id, curr); + if (size > fs->largest) + fs->largest = size; + fs->total += size; + } + } + return 0; +} + + +static inline uint32_t start_mask(int bit_start) +{ + return (uint32_t)(~0) << (bit_start & PMEM_BITS_PER_WORD_MASK); +} + +static inline uint32_t end_mask(int bit_end) +{ + return (uint32_t)(~0) >> + ((BITS_PER_LONG - bit_end) & PMEM_BITS_PER_WORD_MASK); +} + +static inline int compute_total_words(int bit_end, int word_index) +{ + return ((bit_end + BITS_PER_LONG - 1) >> + PMEM_32BIT_WORD_ORDER) - word_index; +} + +static void bitmap_bits_clear_all(uint32_t *bitp, int bit_start, int bit_end) +{ + int word_index = bit_start >> PMEM_32BIT_WORD_ORDER, total_words; + + total_words = compute_total_words(bit_end, word_index); + if (total_words > 0) { + if (total_words == 1) { + bitp[word_index] &= + ~(start_mask(bit_start) & end_mask(bit_end)); + } else { + bitp[word_index++] &= ~start_mask(bit_start); + if (total_words > 2) { + int total_bytes; + + total_words -= 2; + total_bytes = total_words << 2; + + memset(&bitp[word_index], 0, total_bytes); + word_index += total_words; + } + bitp[word_index] &= ~end_mask(bit_end); + } + } +} + +static int pmem_free_bitmap(int id, int bitnum) +{ + /* caller should hold the lock on arena_mutex! */ + int i; + char currtask_name[FIELD_SIZEOF(struct task_struct, comm) + 1]; +/*HTC_START*/ + if (misc_msg_pmem_qcom) + pr_info("[PME][%s] pmem_free_bitmap, bitnum %d\n", pmem[id].name, bitnum); +/*HTC_END*/ + for (i = 0; i < pmem[id].allocator.bitmap.bitmap_allocs; i++) { + const int curr_bit = + pmem[id].allocator.bitmap.bitm_alloc[i].bit; + + if (curr_bit == bitnum) { + const int curr_quanta = + pmem[id].allocator.bitmap.bitm_alloc[i].quanta; + + bitmap_bits_clear_all(pmem[id].allocator.bitmap.bitmap, + curr_bit, curr_bit + curr_quanta); + pmem[id].allocator.bitmap.bitmap_free += curr_quanta; + pmem[id].allocator.bitmap.bitm_alloc[i].bit = -1; + return 0; + } + } + printk(KERN_ALERT "pmem: %s: Attempt to free unallocated index %d, id" + " %d, pid %d(%s)\n", __func__, bitnum, id, current->pid, + get_task_comm(currtask_name, current)); + + return -1; +} + +static int pmem_free_space_bitmap(int id, struct pmem_freespace *fs) +{ + int i, j; + int max_allocs = pmem[id].allocator.bitmap.bitmap_allocs; + int alloc_start = 0; + int next_alloc; + unsigned long size = 0; + + fs->total = 0; + fs->largest = 0; + + for (i = 0; i < max_allocs; i++) { + + int alloc_quanta = 0; + int alloc_idx = 0; + next_alloc = pmem[id].num_entries; + + /* Look for the lowest bit where next allocation starts */ + for (j = 0; j < max_allocs; j++) { + const int curr_alloc = pmem[id].allocator. + bitmap.bitm_alloc[j].bit; + if (curr_alloc != -1) { + if (alloc_start >= curr_alloc) + continue; + if (curr_alloc < next_alloc) { + next_alloc = curr_alloc; + alloc_idx = j; + } + } + } + alloc_quanta = pmem[id].allocator.bitmap. + bitm_alloc[alloc_idx].quanta; + size = (next_alloc - (alloc_start + alloc_quanta)) * + pmem[id].quantum; + + if (size > fs->largest) + fs->largest = size; + fs->total += size; + + if (next_alloc == pmem[id].num_entries) + break; + else + alloc_start = next_alloc; + } + + return 0; +} + +static void pmem_revoke(struct file *file, struct pmem_data *data); + +static int pmem_release(struct inode *inode, struct file *file) +{ + struct pmem_data *data = file->private_data; + struct pmem_region_node *region_node; + struct list_head *elt, *elt2; + int id = get_id(file), ret = 0; + +#if PMEM_DEBUG_MSGS + char currtask_name[FIELD_SIZEOF(struct task_struct, comm) + 1]; +#endif + DLOG("releasing memory pid %u(%s) file %p(%ld) dev %s(id: %d)\n", + current->pid, get_task_comm(currtask_name, current), + file, file_count(file), get_name(file), id); + mutex_lock(&pmem[id].data_list_mutex); + /* if this file is a master, revoke all the memory in the connected + * files */ + if (PMEM_FLAGS_MASTERMAP & data->flags) { + list_for_each(elt, &pmem[id].data_list) { + struct pmem_data *sub_data = + list_entry(elt, struct pmem_data, list); + int is_master; + + down_read(&sub_data->sem); + is_master = (PMEM_IS_SUBMAP(sub_data) && + file == sub_data->master_file); + up_read(&sub_data->sem); + + if (is_master) + pmem_revoke(file, sub_data); + } + } + list_del(&data->list); + mutex_unlock(&pmem[id].data_list_mutex); + + down_write(&data->sem); + + /* if it is not a connected file and it has an allocation, free it */ + if (!(PMEM_FLAGS_CONNECTED & data->flags) && has_allocation(file)) { + mutex_lock(&pmem[id].arena_mutex); + ret = pmem[id].free(id, data->index); + mutex_unlock(&pmem[id].arena_mutex); + } + + /* if this file is a submap (mapped, connected file), downref the + * task struct */ + if (PMEM_FLAGS_SUBMAP & data->flags) + if (data->task) { + put_task_struct(data->task); + data->task = NULL; + } + + file->private_data = NULL; + + list_for_each_safe(elt, elt2, &data->region_list) { + region_node = list_entry(elt, struct pmem_region_node, list); + list_del(elt); + kfree(region_node); + } + BUG_ON(!list_empty(&data->region_list)); + + up_write(&data->sem); + kfree(data); + if (pmem[id].release) + ret = pmem[id].release(inode, file); + + return ret; +} + +static int pmem_open(struct inode *inode, struct file *file) +{ + struct pmem_data *data; + int id = get_id(file); + int ret = 0; +#if PMEM_DEBUG_MSGS + char currtask_name[FIELD_SIZEOF(struct task_struct, comm) + 1]; +#endif + + if (pmem[id].memory_state == MEMORY_UNSTABLE_NO_MEMORY_ALLOCATED) + return -ENODEV; + DLOG("pid %u(%s) file %p(%ld) dev %s(id: %d)\n", + current->pid, get_task_comm(currtask_name, current), + file, file_count(file), get_name(file), id); + /* setup file->private_data to indicate its unmapped */ + /* you can only open a pmem device one time */ + if (file->private_data != NULL) + return -EINVAL; + data = kmalloc(sizeof(struct pmem_data), GFP_KERNEL); + if (!data) { + printk(KERN_ALERT "pmem: %s: unable to allocate memory for " + "pmem metadata.", __func__); + return -1; + } + data->flags = 0; + data->index = -1; + data->task = NULL; + data->vma = NULL; + data->pid = 0; + data->master_file = NULL; +#if PMEM_DEBUG + data->ref = 0; +#endif + INIT_LIST_HEAD(&data->region_list); + init_rwsem(&data->sem); + + file->private_data = data; + INIT_LIST_HEAD(&data->list); + + mutex_lock(&pmem[id].data_list_mutex); + list_add(&data->list, &pmem[id].data_list); + mutex_unlock(&pmem[id].data_list_mutex); + return ret; +} + +static unsigned long pmem_order(unsigned long len, int id) +{ + int i; + + len = (len + pmem[id].quantum - 1)/pmem[id].quantum; + len--; + for (i = 0; i < sizeof(len)*8; i++) + if (len >> i == 0) + break; + return i; +} + +static int pmem_allocator_all_or_nothing(const int id, + const unsigned long len, + const unsigned int align) +{ + /* caller should hold the lock on arena_mutex! */ + DLOG("all or nothing\n"); + if ((len > pmem[id].size) || + pmem[id].allocator.all_or_nothing.allocated) + return -1; + pmem[id].allocator.all_or_nothing.allocated = 1; + return len; +} + +static int pmem_allocator_buddy_bestfit(const int id, + const unsigned long len, + unsigned int align) +{ + /* caller should hold the lock on arena_mutex! */ + int curr; + int best_fit = -1; + unsigned long order; + + DLOG("buddy bestfit\n"); + order = pmem_order(len, id); + if (order > PMEM_MAX_ORDER) + goto out; + + DLOG("order %lx\n", order); + + /* Look through the bitmap. + * If a free slot of the correct order is found, use it. + * Otherwise, use the best fit (smallest with size > order) slot. + */ + for (curr = 0; + curr < pmem[id].num_entries; + curr = PMEM_BUDDY_NEXT_INDEX(id, curr)) + if (PMEM_IS_FREE_BUDDY(id, curr)) { + if (PMEM_BUDDY_ORDER(id, curr) == + (unsigned char)order) { + /* set the not free bit and clear others */ + best_fit = curr; + break; + } + if (PMEM_BUDDY_ORDER(id, curr) > + (unsigned char)order && + (best_fit < 0 || + PMEM_BUDDY_ORDER(id, curr) < + PMEM_BUDDY_ORDER(id, best_fit))) + best_fit = curr; + } + + /* if best_fit < 0, there are no suitable slots; return an error */ + if (best_fit < 0) { +#if PMEM_DEBUG + printk(KERN_ALERT "pmem: %s: no space left to allocate!\n", + __func__); +#endif + goto out; + } + + /* now partition the best fit: + * split the slot into 2 buddies of order - 1 + * repeat until the slot is of the correct order + */ + while (PMEM_BUDDY_ORDER(id, best_fit) > (unsigned char)order) { + int buddy; + PMEM_BUDDY_ORDER(id, best_fit) -= 1; + buddy = PMEM_BUDDY_INDEX(id, best_fit); + PMEM_BUDDY_ORDER(id, buddy) = PMEM_BUDDY_ORDER(id, best_fit); + } + pmem[id].allocator.buddy_bestfit.buddy_bitmap[best_fit].allocated = 1; +out: + return best_fit; +} + + +static inline unsigned long paddr_from_bit(const int id, const int bitnum) +{ + return pmem[id].base + pmem[id].quantum * bitnum; +} + +static inline unsigned long bit_from_paddr(const int id, + const unsigned long paddr) +{ + return (paddr - pmem[id].base) / pmem[id].quantum; +} + +static void bitmap_bits_set_all(uint32_t *bitp, int bit_start, int bit_end) +{ + int word_index = bit_start >> PMEM_32BIT_WORD_ORDER, total_words; + + total_words = compute_total_words(bit_end, word_index); + if (total_words > 0) { + if (total_words == 1) { + bitp[word_index] |= + (start_mask(bit_start) & end_mask(bit_end)); + } else { + bitp[word_index++] |= start_mask(bit_start); + if (total_words > 2) { + int total_bytes; + + total_words -= 2; + total_bytes = total_words << 2; + + memset(&bitp[word_index], ~0, total_bytes); + word_index += total_words; + } + bitp[word_index] |= end_mask(bit_end); + } + } +} + +static int +bitmap_allocate_contiguous(uint32_t *bitp, int num_bits_to_alloc, + int total_bits, int spacing) +{ + int bit_start, last_bit, word_index; + + if (num_bits_to_alloc <= 0) + return -1; + + for (bit_start = 0; ; + bit_start = (last_bit + + (word_index << PMEM_32BIT_WORD_ORDER) + spacing - 1) + & ~(spacing - 1)) { + int bit_end = bit_start + num_bits_to_alloc, total_words; + + if (bit_end > total_bits) + return -1; /* out of contiguous memory */ + + word_index = bit_start >> PMEM_32BIT_WORD_ORDER; + total_words = compute_total_words(bit_end, word_index); + + if (total_words <= 0) + return -1; + + if (total_words == 1) { + last_bit = fls(bitp[word_index] & + (start_mask(bit_start) & + end_mask(bit_end))); + if (last_bit) + continue; + } else { + int end_word = word_index + (total_words - 1); + last_bit = + fls(bitp[word_index] & start_mask(bit_start)); + if (last_bit) + continue; + + for (word_index++; + word_index < end_word; + word_index++) { + last_bit = fls(bitp[word_index]); + if (last_bit) + break; + } + if (last_bit) + continue; + + last_bit = fls(bitp[word_index] & end_mask(bit_end)); + if (last_bit) + continue; + } + bitmap_bits_set_all(bitp, bit_start, bit_end); + return bit_start; + } + return -1; +} + +static int reserve_quanta(const unsigned int quanta_needed, + const int id, + unsigned int align) +{ + /* alignment should be a valid power of 2 */ + int ret = -1, start_bit = 0, spacing = 1; + + /* Sanity check */ + if (quanta_needed > pmem[id].allocator.bitmap.bitmap_free) { +#if PMEM_DEBUG + printk(KERN_ALERT "pmem: %s: request (%d) too big for" + " available free (%d)\n", __func__, quanta_needed, + pmem[id].allocator.bitmap.bitmap_free); +#endif + return -1; + } + + start_bit = bit_from_paddr(id, + (pmem[id].base + align - 1) & ~(align - 1)); + if (start_bit <= -1) { +#if PMEM_DEBUG + printk(KERN_ALERT + "pmem: %s: bit_from_paddr fails for" + " %u alignment.\n", __func__, align); +#endif + return -1; + } + spacing = align / pmem[id].quantum; + spacing = spacing > 1 ? spacing : 1; + + ret = bitmap_allocate_contiguous(pmem[id].allocator.bitmap.bitmap, + quanta_needed, + (pmem[id].size + pmem[id].quantum - 1) / pmem[id].quantum, + spacing); + +#if PMEM_DEBUG + if (ret < 0) + printk(KERN_ALERT "pmem: %s: not enough contiguous bits free " + "in bitmap! Region memory is either too fragmented or" + " request is too large for available memory.\n", + __func__); +#endif + + return ret; +} + +static int pmem_allocator_bitmap(const int id, + const unsigned long len, + const unsigned int align) +{ + /* caller should hold the lock on arena_mutex! */ + int bitnum, i; + unsigned int quanta_needed; +/*HTC_START*/ + if (misc_msg_pmem_qcom) + pr_info("[PME][%s] pmem_allocator_bitmap, len %ld\n", pmem[id].name, len); + + if (!pmem[id].allocator.bitmap.bitm_alloc) { + if (misc_msg_pmem_qcom) { + printk(KERN_ALERT "[PME][%s] bitm_alloc not present! \n", + pmem[id].name); + } +/*HTC_END*/ + bitnum = -1; goto leave; + } + + quanta_needed = (len + pmem[id].quantum - 1) / pmem[id].quantum; +/*HTC_START*/ + if (misc_msg_pmem_qcom) { + pr_info("[PME][%s] quantum size %u quanta needed %u free %u\n", + pmem[id].name, pmem[id].quantum, quanta_needed, + pmem[id].allocator.bitmap.bitmap_free); + } + + if (pmem[id].allocator.bitmap.bitmap_free < quanta_needed) { + if (misc_msg_pmem_qcom) { + printk(KERN_ALERT "[PME][%s] memory allocation failure. " + "PMEM memory region exhausted." + " Unable to comply with allocation request.\n", pmem[id].name); + } +/*HTC_END*/ + bitnum = -1; goto leave; + } + + bitnum = reserve_quanta(quanta_needed, id, align); + if (bitnum == -1) + goto leave; + + for (i = 0; + i < pmem[id].allocator.bitmap.bitmap_allocs && + pmem[id].allocator.bitmap.bitm_alloc[i].bit != -1; + i++) + ; + + if (i >= pmem[id].allocator.bitmap.bitmap_allocs) { + void *temp; + int32_t new_bitmap_allocs = + pmem[id].allocator.bitmap.bitmap_allocs << 1; + int j; + + if (!new_bitmap_allocs) { /* failed sanity check!! */ +/*HTC_START*/ + if (misc_msg_pmem_qcom) { + pr_alert("[PME][%s] pmem: bitmap_allocs number" + " wrapped around to zero! Something " + "is VERY wrong.\n", pmem[id].name); + } + bitnum = -1; goto leave; + } + if (new_bitmap_allocs > pmem[id].num_entries) { + /* failed sanity check!! */ + if (misc_msg_pmem_qcom) { + pr_alert("[PME][%s] pmem: required bitmap_allocs" + " number exceeds maximum entries possible" + " for current quanta\n", pmem[id].name); + } + + bitnum = -1; goto leave; + } + temp = krealloc(pmem[id].allocator.bitmap.bitm_alloc, + new_bitmap_allocs * + sizeof(*pmem[id].allocator.bitmap.bitm_alloc), + GFP_KERNEL); + if (!temp) { + if (misc_msg_pmem_qcom) { + pr_alert("[PME][%s] can't realloc bitmap_allocs," + " current num bitmap allocs %d\n", + pmem[id].name, pmem[id].allocator.bitmap.bitmap_allocs); + } +/*HTC_END*/ + bitnum = -1; goto leave; + } + pmem[id].allocator.bitmap.bitmap_allocs = new_bitmap_allocs; + pmem[id].allocator.bitmap.bitm_alloc = temp; + + for (j = i; j < new_bitmap_allocs; j++) { + pmem[id].allocator.bitmap.bitm_alloc[j].bit = -1; + pmem[id].allocator.bitmap.bitm_alloc[i].quanta = 0; + } +/*HTC_START*/ + if (misc_msg_pmem_qcom) { + pr_info("[PME][%s] increased # of allocated regions to %d for \n", + pmem[id].name, pmem[id].allocator.bitmap.bitmap_allocs); + } + } + if (misc_msg_pmem_qcom) + pr_info("[PME][%s] bitnum %d, bitm_alloc index %d\n", pmem[id].name, bitnum, i); +/*HTC_END*/ + pmem[id].allocator.bitmap.bitmap_free -= quanta_needed; + pmem[id].allocator.bitmap.bitm_alloc[i].bit = bitnum; + pmem[id].allocator.bitmap.bitm_alloc[i].quanta = quanta_needed; +leave: + if (-1 == bitnum) { + pr_err("[PME][%s] error: pmem_allocator_bitmap failed\n", pmem[id].name); + for (i = 0; i < pmem[id].allocator.bitmap.bitmap_allocs; i++) { + if (pmem[id].allocator.bitmap.bitm_alloc[i].bit != -1) { + /*HTC_START*/ + if (misc_msg_pmem_qcom) { + pr_info("[PME][%s] bitm_alloc[%d].bit: %u bitm_alloc[%d].quanta: %u\n", + pmem[id].name, + i, + pmem[id].allocator.bitmap.bitm_alloc[i].bit, + i, + pmem[id].allocator.bitmap.bitm_alloc[i].quanta + ); + } + /*HTC_END*/ + } + } + } + return bitnum; +} + +static pgprot_t pmem_phys_mem_access_prot(struct file *file, pgprot_t vma_prot) +{ + int id = get_id(file); +#ifdef pgprot_writecombine + if (pmem[id].cached == 0 || file->f_flags & O_SYNC) + /* on ARMv6 and ARMv7 this expands to Normal Noncached */ + return pgprot_writecombine(vma_prot); +#endif +#ifdef pgprot_ext_buffered + else if (pmem[id].buffered) + return pgprot_ext_buffered(vma_prot); +#endif + return vma_prot; +} + +static unsigned long pmem_start_addr_all_or_nothing(int id, + struct pmem_data *data) +{ + return PMEM_START_ADDR(id, 0); +} + +static unsigned long pmem_start_addr_buddy_bestfit(int id, + struct pmem_data *data) +{ + return PMEM_START_ADDR(id, data->index); +} + +static unsigned long pmem_start_addr_bitmap(int id, struct pmem_data *data) +{ + return data->index * pmem[id].quantum + pmem[id].base; +} + +static void *pmem_start_vaddr(int id, struct pmem_data *data) +{ + return pmem[id].start_addr(id, data) - pmem[id].base + pmem[id].vbase; +} + +static unsigned long pmem_len_all_or_nothing(int id, struct pmem_data *data) +{ + return data->index; +} + +static unsigned long pmem_len_buddy_bestfit(int id, struct pmem_data *data) +{ + return PMEM_BUDDY_LEN(id, data->index); +} + +static unsigned long pmem_len_bitmap(int id, struct pmem_data *data) +{ + int i; + unsigned long ret = 0; + + mutex_lock(&pmem[id].arena_mutex); + + for (i = 0; i < pmem[id].allocator.bitmap.bitmap_allocs; i++) + if (pmem[id].allocator.bitmap.bitm_alloc[i].bit == + data->index) { + ret = pmem[id].allocator.bitmap.bitm_alloc[i].quanta * + pmem[id].quantum; + break; + } + + mutex_unlock(&pmem[id].arena_mutex); +#if PMEM_DEBUG + if (i >= pmem[id].allocator.bitmap.bitmap_allocs) + pr_alert("pmem: %s: can't find bitnum %d in " + "alloc'd array!\n", __func__, data->index); +#endif + return ret; +} + +static int pmem_map_garbage(int id, struct vm_area_struct *vma, + struct pmem_data *data, unsigned long offset, + unsigned long len) +{ + int i, garbage_pages = len >> PAGE_SHIFT; + + vma->vm_flags |= VM_IO | VM_RESERVED | VM_PFNMAP | VM_SHARED | VM_WRITE; + for (i = 0; i < garbage_pages; i++) { + if (vm_insert_pfn(vma, vma->vm_start + offset + (i * PAGE_SIZE), + pmem[id].garbage_pfn)) + return -EAGAIN; + } + return 0; +} + +static int pmem_unmap_pfn_range(int id, struct vm_area_struct *vma, + struct pmem_data *data, unsigned long offset, + unsigned long len) +{ + int garbage_pages; + DLOG("unmap offset %lx len %lx\n", offset, len); + + BUG_ON(!PMEM_IS_PAGE_ALIGNED(len)); + + garbage_pages = len >> PAGE_SHIFT; + zap_page_range(vma, vma->vm_start + offset, len, NULL); + pmem_map_garbage(id, vma, data, offset, len); + return 0; +} + +static int pmem_map_pfn_range(int id, struct vm_area_struct *vma, + struct pmem_data *data, unsigned long offset, + unsigned long len) +{ + int ret; + DLOG("map offset %lx len %lx\n", offset, len); + BUG_ON(!PMEM_IS_PAGE_ALIGNED(vma->vm_start)); + BUG_ON(!PMEM_IS_PAGE_ALIGNED(vma->vm_end)); + BUG_ON(!PMEM_IS_PAGE_ALIGNED(len)); + BUG_ON(!PMEM_IS_PAGE_ALIGNED(offset)); + + ret = io_remap_pfn_range(vma, vma->vm_start + offset, + (pmem[id].start_addr(id, data) + offset) >> PAGE_SHIFT, + len, vma->vm_page_prot); + if (ret) { +#if PMEM_DEBUG + pr_alert("pmem: %s: io_remap_pfn_range fails with " + "return value: %d!\n", __func__, ret); +#endif + + ret = -EAGAIN; + } + return ret; +} + +static int pmem_remap_pfn_range(int id, struct vm_area_struct *vma, + struct pmem_data *data, unsigned long offset, + unsigned long len) +{ + /* hold the mm semp for the vma you are modifying when you call this */ + BUG_ON(!vma); + zap_page_range(vma, vma->vm_start + offset, len, NULL); + return pmem_map_pfn_range(id, vma, data, offset, len); +} + +static void pmem_vma_open(struct vm_area_struct *vma) +{ + struct file *file = vma->vm_file; + struct pmem_data *data = file->private_data; + int id = get_id(file); + +#if PMEM_DEBUG_MSGS + char currtask_name[FIELD_SIZEOF(struct task_struct, comm) + 1]; +#endif + DLOG("Dev %s(id: %d) pid %u(%s) ppid %u file %p count %ld\n", + get_name(file), id, current->pid, + get_task_comm(currtask_name, current), + current->parent->pid, file, file_count(file)); + /* this should never be called as we don't support copying pmem + * ranges via fork */ + down_read(&data->sem); + BUG_ON(!has_allocation(file)); + /* remap the garbage pages, forkers don't get access to the data */ + pmem_unmap_pfn_range(id, vma, data, 0, vma->vm_start - vma->vm_end); + up_read(&data->sem); +} + +static void pmem_vma_close(struct vm_area_struct *vma) +{ + struct file *file = vma->vm_file; + struct pmem_data *data = file->private_data; + +#if PMEM_DEBUG_MSGS + char currtask_name[FIELD_SIZEOF(struct task_struct, comm) + 1]; +#endif + DLOG("Dev %s(id: %d) pid %u(%s) ppid %u file %p count %ld\n", + get_name(file), get_id(file), current->pid, + get_task_comm(currtask_name, current), + current->parent->pid, file, file_count(file)); + + if (unlikely(!is_pmem_file(file))) { + pr_warning("pmem: something is very wrong, you are " + "closing a vm backing an allocation that doesn't " + "exist!\n"); + return; + } + + down_write(&data->sem); + if (unlikely(!has_allocation(file))) { + up_write(&data->sem); + pr_warning("pmem: something is very wrong, you are " + "closing a vm backing an allocation that doesn't " + "exist!\n"); + return; + } + if (data->vma == vma) { + data->vma = NULL; + if ((data->flags & PMEM_FLAGS_CONNECTED) && + (data->flags & PMEM_FLAGS_SUBMAP)) + data->flags |= PMEM_FLAGS_UNSUBMAP; + } + /* the kernel is going to free this vma now anyway */ + up_write(&data->sem); +} + +static struct vm_operations_struct vm_ops = { + .open = pmem_vma_open, + .close = pmem_vma_close, +}; + +static int pmem_mmap(struct file *file, struct vm_area_struct *vma) +{ + struct pmem_data *data = file->private_data; + int index; + unsigned long vma_size = vma->vm_end - vma->vm_start; + int ret = 0, id = get_id(file); + +#if PMEM_DEBUG_MSGS + char currtask_name[FIELD_SIZEOF(struct task_struct, comm) + 1]; +#endif + DLOG("pid %u(%s) mmap vma_size %lu on dev %s(id: %d)\n", current->pid, + get_task_comm(currtask_name, current), vma_size, + get_name(file), id); + if (vma->vm_pgoff || !PMEM_IS_PAGE_ALIGNED(vma_size)) { +#if PMEM_DEBUG + pr_err("pmem: mmaps must be at offset zero, aligned" + " and a multiple of pages_size.\n"); +#endif + return -EINVAL; + } + + down_write(&data->sem); + /* check this file isn't already mmaped, for submaps check this file + * has never been mmaped */ + if ((data->flags & PMEM_FLAGS_MASTERMAP) || + (data->flags & PMEM_FLAGS_SUBMAP) || + (data->flags & PMEM_FLAGS_UNSUBMAP)) { +#if PMEM_DEBUG + pr_err("pmem: you can only mmap a pmem file once, " + "this file is already mmaped. %x\n", data->flags); +#endif + ret = -EINVAL; + goto error; + } + /* if file->private_data == unalloced, alloc*/ + if (data && data->index == -1) { + mutex_lock(&pmem[id].arena_mutex); + index = pmem[id].allocate(id, + vma->vm_end - vma->vm_start, + SZ_4K); + mutex_unlock(&pmem[id].arena_mutex); + data->index = index; + if (data->index < 0) { + pr_err("pmem: mmap unable to allocate memory" + "on %s\n", get_name(file)); + } + } + + /* either no space was available or an error occured */ + if (!has_allocation(file)) { + ret = -ENOMEM; + pr_err("pmem: could not find allocation for map.\n"); + goto error; + } + + if (pmem[id].len(id, data) < vma_size) { +#if PMEM_DEBUG + pr_err("pmem: mmap size [%lu] does not match" + " size of backing region [%lu].\n", vma_size, + pmem[id].len(id, data)); +#endif + ret = -EINVAL; + goto error; + } + + vma->vm_pgoff = pmem[id].start_addr(id, data) >> PAGE_SHIFT; + + vma->vm_page_prot = pmem_phys_mem_access_prot(file, vma->vm_page_prot); + + if (data->flags & PMEM_FLAGS_CONNECTED) { + struct pmem_region_node *region_node; + struct list_head *elt; + if (pmem_map_garbage(id, vma, data, 0, vma_size)) { + pr_alert("pmem: mmap failed in kernel!\n"); + ret = -EAGAIN; + goto error; + } + list_for_each(elt, &data->region_list) { + region_node = list_entry(elt, struct pmem_region_node, + list); + DLOG("remapping file: %p %lx %lx\n", file, + region_node->region.offset, + region_node->region.len); + if (pmem_remap_pfn_range(id, vma, data, + region_node->region.offset, + region_node->region.len)) { + ret = -EAGAIN; + goto error; + } + } + data->flags |= PMEM_FLAGS_SUBMAP; + get_task_struct(current->group_leader); + data->task = current->group_leader; + data->vma = vma; +#if PMEM_DEBUG + data->pid = current->pid; +#endif + DLOG("submmapped file %p vma %p pid %u\n", file, vma, + current->pid); + } else { + if (pmem_map_pfn_range(id, vma, data, 0, vma_size)) { + pr_err("pmem: mmap failed in kernel!\n"); + ret = -EAGAIN; + goto error; + } + data->flags |= PMEM_FLAGS_MASTERMAP; + data->pid = current->pid; + } + vma->vm_ops = &vm_ops; +error: + up_write(&data->sem); + return ret; +} + +/* the following are the api for accessing pmem regions by other drivers + * from inside the kernel */ +int get_pmem_user_addr(struct file *file, unsigned long *start, + unsigned long *len) +{ + int ret = -1; + + if (is_pmem_file(file)) { + struct pmem_data *data = file->private_data; + + down_read(&data->sem); + if (has_allocation(file)) { + if (data->vma) { + *start = data->vma->vm_start; + *len = data->vma->vm_end - data->vma->vm_start; + } else { + *start = *len = 0; +#if PMEM_DEBUG + pr_err("pmem: %s: no vma present.\n", + __func__); +#endif + } + ret = 0; + } + up_read(&data->sem); + } + +#if PMEM_DEBUG + if (ret) + pr_err("pmem: %s: requested pmem data from invalid" + "file.\n", __func__); +#endif + return ret; +} + +int get_pmem_addr(struct file *file, unsigned long *start, + unsigned long *vstart, unsigned long *len) +{ + int ret = -1; + + if (is_pmem_file(file)) { + struct pmem_data *data = file->private_data; + + down_read(&data->sem); + if (has_allocation(file)) { + int id = get_id(file); + + *start = pmem[id].start_addr(id, data); + *len = pmem[id].len(id, data); + *vstart = (unsigned long) + pmem_start_vaddr(id, data); + up_read(&data->sem); +#if PMEM_DEBUG + down_write(&data->sem); + data->ref++; + up_write(&data->sem); +#endif + DLOG("returning start %#lx len %lu " + "vstart %#lx\n", + *start, *len, *vstart); + ret = 0; + } else { + up_read(&data->sem); + } + } + return ret; +} + +int get_pmem_file(unsigned int fd, unsigned long *start, unsigned long *vstart, + unsigned long *len, struct file **filp) +{ + int ret = -1; + struct file *file = fget(fd); + + if (unlikely(file == NULL)) { + pr_err("pmem: %s: requested data from file " + "descriptor that doesn't exist.\n", __func__); + } else { +#if PMEM_DEBUG_MSGS + char currtask_name[FIELD_SIZEOF(struct task_struct, comm) + 1]; +#endif + DLOG("filp %p rdev %d pid %u(%s) file %p(%ld)" + " dev %s(id: %d)\n", filp, + file->f_dentry->d_inode->i_rdev, + current->pid, get_task_comm(currtask_name, current), + file, file_count(file), get_name(file), get_id(file)); + + if (!get_pmem_addr(file, start, vstart, len)) { + if (filp) + *filp = file; + ret = 0; + } else { + fput(file); + } + } + return ret; +} +EXPORT_SYMBOL(get_pmem_file); + +int get_pmem_fd(int fd, unsigned long *start, unsigned long *len) +{ + unsigned long vstart; + return get_pmem_file(fd, start, &vstart, len, NULL); +} +EXPORT_SYMBOL(get_pmem_fd); + +void put_pmem_file(struct file *file) +{ +#if PMEM_DEBUG_MSGS + char currtask_name[FIELD_SIZEOF(struct task_struct, comm) + 1]; +#endif + DLOG("rdev %d pid %u(%s) file %p(%ld)" " dev %s(id: %d)\n", + file->f_dentry->d_inode->i_rdev, current->pid, + get_task_comm(currtask_name, current), file, + file_count(file), get_name(file), get_id(file)); + if (is_pmem_file(file)) { +#if PMEM_DEBUG + struct pmem_data *data = file->private_data; + + down_write(&data->sem); + if (!data->ref--) { + data->ref++; + pr_alert("pmem: pmem_put > pmem_get %s " + "(pid %d)\n", + pmem[get_id(file)].dev.name, data->pid); + BUG(); + } + up_write(&data->sem); +#endif + fput(file); + } +} +EXPORT_SYMBOL(put_pmem_file); + +void put_pmem_fd(int fd) +{ + int put_needed; + struct file *file = fget_light(fd, &put_needed); + + if (file) { + put_pmem_file(file); + fput_light(file, put_needed); + } +} + +void flush_pmem_fd(int fd, unsigned long offset, unsigned long len) +{ + int fput_needed; + struct file *file = fget_light(fd, &fput_needed); + + if (file) { + flush_pmem_file(file, offset, len); + fput_light(file, fput_needed); + } +} + +void flush_pmem_file(struct file *file, unsigned long offset, unsigned long len) +{ + struct pmem_data *data; + int id; + void *vaddr; + struct pmem_region_node *region_node; + struct list_head *elt; + void *flush_start, *flush_end; +#ifdef CONFIG_OUTER_CACHE + unsigned long phy_start, phy_end; +#endif + if (!is_pmem_file(file)) + return; + + id = get_id(file); + if (!pmem[id].cached) + return; + + /* is_pmem_file fails if !file */ + data = file->private_data; + + down_read(&data->sem); + if (!has_allocation(file)) + goto end; + + vaddr = pmem_start_vaddr(id, data); + /* if this isn't a submmapped file, flush the whole thing */ + if (unlikely(!(data->flags & PMEM_FLAGS_CONNECTED))) { + dmac_flush_range(vaddr, vaddr + pmem[id].len(id, data)); +#ifdef CONFIG_OUTER_CACHE + phy_start = (unsigned long)vaddr - + (unsigned long)pmem[id].vbase + pmem[id].base; + + phy_end = phy_start + pmem[id].len(id, data); + + outer_flush_range(phy_start, phy_end); +#endif + goto end; + } + /* otherwise, flush the region of the file we are drawing */ + list_for_each(elt, &data->region_list) { + region_node = list_entry(elt, struct pmem_region_node, list); + if ((offset >= region_node->region.offset) && + ((offset + len) <= (region_node->region.offset + + region_node->region.len))) { + flush_start = vaddr + region_node->region.offset; + flush_end = flush_start + region_node->region.len; + dmac_flush_range(flush_start, flush_end); +#ifdef CONFIG_OUTER_CACHE + + phy_start = (unsigned long)flush_start - + (unsigned long)pmem[id].vbase + pmem[id].base; + + phy_end = phy_start + region_node->region.len; + + outer_flush_range(phy_start, phy_end); +#endif + break; + } + } +end: + up_read(&data->sem); +} + +int pmem_cache_maint(struct file *file, unsigned int cmd, + struct pmem_addr *pmem_addr) +{ + struct pmem_data *data; + int id; + unsigned long vaddr, paddr, length, offset, + pmem_len, pmem_start_addr; + + /* Called from kernel-space so file may be NULL */ + if (!file) + return -EBADF; + + data = file->private_data; + id = get_id(file); + + if (!pmem[id].cached) + return 0; + + offset = pmem_addr->offset; + length = pmem_addr->length; + + down_read(&data->sem); + if (!has_allocation(file)) { + up_read(&data->sem); + return -EINVAL; + } + pmem_len = pmem[id].len(id, data); + pmem_start_addr = pmem[id].start_addr(id, data); + up_read(&data->sem); + + if (offset + length > pmem_len) + return -EINVAL; + + vaddr = pmem_addr->vaddr; + paddr = pmem_start_addr + offset; + + DLOG("pmem cache maint on dev %s(id: %d)" + "(vaddr %lx paddr %lx len %lu bytes)\n", + get_name(file), id, vaddr, paddr, length); + if (cmd == PMEM_CLEAN_INV_CACHES) + clean_and_invalidate_caches(vaddr, + length, paddr); + else if (cmd == PMEM_CLEAN_CACHES) + clean_caches(vaddr, length, paddr); + else if (cmd == PMEM_INV_CACHES) + invalidate_caches(vaddr, length, paddr); + + return 0; +} +EXPORT_SYMBOL(pmem_cache_maint); + +int32_t pmem_kalloc(const size_t size, const uint32_t flags) +{ + int info_id, i, memtype, fallback = 0; + unsigned int align; + int32_t index = -1; + + switch (flags & PMEM_ALIGNMENT_MASK) { + case PMEM_ALIGNMENT_4K: + align = SZ_4K; + break; + case PMEM_ALIGNMENT_1M: + align = SZ_1M; + break; + default: + pr_alert("pmem: %s: Invalid alignment %#x\n", + __func__, (flags & PMEM_ALIGNMENT_MASK)); + return -EINVAL; + } + + memtype = flags & PMEM_MEMTYPE_MASK; +retry_memalloc: + info_id = -1; + for (i = 0; i < ARRAY_SIZE(kapi_memtypes); i++) + if (kapi_memtypes[i].memtype == memtype) { + info_id = kapi_memtypes[i].info_id; + break; + } + if (info_id < 0) { + pr_alert("pmem: %s: Kernel %#x memory arena is not " + "initialized. Check board file!\n", + __func__, (flags & PMEM_MEMTYPE_MASK)); + return -EINVAL; + } + + if (!pmem[info_id].allocate) { + pr_alert("pmem: %s: Attempt to allocate size %u, alignment %#x" + " from non-existent PMEM kernel region %d. " + "Driver/board setup is faulty!", + __func__, size, (flags & PMEM_ALIGNMENT_MASK), + info_id); + return -ENOMEM; + } + +#if PMEM_DEBUG + if (align != SZ_4K && + (pmem[info_id].allocator_type == + PMEM_ALLOCATORTYPE_ALLORNOTHING || + pmem[info_id].allocator_type == + PMEM_ALLOCATORTYPE_BUDDYBESTFIT)) + pr_warning("pmem: %s: alignment other than on 4K " + "pages not supported with %s allocator for PMEM " + "memory region '%s'. Memory will be aligned to 4K " + "boundary. Check your board file or allocation " + "invocation.\n", __func__, + (pmem[info_id].allocator_type == + PMEM_ALLOCATORTYPE_ALLORNOTHING ? + "'All Or Nothing'" + : + "'Buddy / Best Fit'"), + pmem[info_id].dev.name); +#endif + + mutex_lock(&pmem[info_id].arena_mutex); + index = pmem[info_id].allocate(info_id, size, align); + mutex_unlock(&pmem[info_id].arena_mutex); + + if (index < 0 && + !fallback && + kapi_memtypes[i].fallback_memtype != PMEM_INVALID_MEMTYPE) { + fallback = 1; + memtype = kapi_memtypes[i].fallback_memtype; + goto retry_memalloc; + } + + return index >= 0 ? + index * pmem[info_id].quantum + pmem[info_id].base : -ENOMEM; +} +EXPORT_SYMBOL(pmem_kalloc); + +static int pmem_kapi_free_index_allornothing(const int32_t physaddr, int id) +{ + return physaddr == pmem[id].base ? 0 : -1; +} + +static int pmem_kapi_free_index_buddybestfit(const int32_t physaddr, int id) +{ + return (physaddr >= pmem[id].base && + physaddr < (pmem[id].base + pmem[id].size && + !(physaddr % pmem[id].quantum))) ? + (physaddr - pmem[id].base) / pmem[id].quantum : -1; +} + +static int pmem_kapi_free_index_bitmap(const int32_t physaddr, int id) +{ + return (physaddr >= pmem[id].base && + physaddr < (pmem[id].base + pmem[id].size)) ? + bit_from_paddr(id, physaddr) : -1; +} + +int pmem_kfree(const int32_t physaddr) +{ + int i; + for (i = 0; i < ARRAY_SIZE(kapi_memtypes); i++) { + int index; + int id = kapi_memtypes[i].info_id; + + if (id < 0) + continue; + + if (!pmem[id].allocate) { +#if PMEM_DEBUG + pr_alert("pmem: %s: " + "Attempt to free physical address %#x " + "from unregistered PMEM kernel region" + " %d. Driver/board setup is faulty!", + __func__, physaddr, id); +#endif + return -EINVAL; + } + + index = pmem[id].kapi_free_index(physaddr, id); + if (index >= 0) + return pmem[id].free(id, index) ? -EINVAL : 0; + } +#if PMEM_DEBUG + pr_alert("pmem: %s: Failed to free physaddr %#x, does not " + "seem be value returned by pmem_kalloc()!", + __func__, physaddr); +#endif + return -EINVAL; +} +EXPORT_SYMBOL(pmem_kfree); + +static int pmem_connect(unsigned long connect, struct file *file) +{ + int ret = 0, put_needed; + struct file *src_file; + + if (!file) { + pr_err("pmem: %s: NULL file pointer passed in, " + "bailing out!\n", __func__); + ret = -EINVAL; + goto leave; + } + + src_file = fget_light(connect, &put_needed); + + if (!src_file) { + pr_err("pmem: %s: src file not found!\n", __func__); + ret = -EBADF; + goto leave; + } + + if (src_file == file) { /* degenerative case, operator error */ + pr_err("pmem: %s: src_file and passed in file are " + "the same; refusing to connect to self!\n", __func__); + ret = -EINVAL; + goto put_src_file; + } + + if (unlikely(!is_pmem_file(src_file))) { + pr_err("pmem: %s: src file is not a pmem file!\n", + __func__); + ret = -EINVAL; + goto put_src_file; + } else { + struct pmem_data *src_data = src_file->private_data; + + if (!src_data) { + pr_err("pmem: %s: src file pointer has no" + "private data, bailing out!\n", __func__); + ret = -EINVAL; + goto put_src_file; + } + + down_read(&src_data->sem); + + if (unlikely(!has_allocation(src_file))) { + up_read(&src_data->sem); + pr_err("pmem: %s: src file has no allocation!\n", + __func__); + ret = -EINVAL; + } else { + struct pmem_data *data; + int src_index = src_data->index; + + up_read(&src_data->sem); + + data = file->private_data; + if (!data) { + pr_err("pmem: %s: passed in file " + "pointer has no private data, bailing" + " out!\n", __func__); + ret = -EINVAL; + goto put_src_file; + } + + down_write(&data->sem); + if (has_allocation(file) && + (data->index != src_index)) { + up_write(&data->sem); + + pr_err("pmem: %s: file is already " + "mapped but doesn't match this " + "src_file!\n", __func__); + ret = -EINVAL; + } else { + data->index = src_index; + data->flags |= PMEM_FLAGS_CONNECTED; + data->master_fd = connect; + data->master_file = src_file; + + up_write(&data->sem); + + DLOG("connect %p to %p\n", file, src_file); + } + } + } +put_src_file: + fput_light(src_file, put_needed); +leave: + return ret; +} + +static void pmem_unlock_data_and_mm(struct pmem_data *data, + struct mm_struct *mm) +{ + up_write(&data->sem); + if (mm != NULL) { + up_write(&mm->mmap_sem); + mmput(mm); + } +} + +static int pmem_lock_data_and_mm(struct file *file, struct pmem_data *data, + struct mm_struct **locked_mm) +{ + int ret = 0; + struct mm_struct *mm = NULL; +#if PMEM_DEBUG_MSGS + char currtask_name[FIELD_SIZEOF(struct task_struct, comm) + 1]; +#endif + DLOG("pid %u(%s) file %p(%ld)\n", + current->pid, get_task_comm(currtask_name, current), + file, file_count(file)); + + *locked_mm = NULL; +lock_mm: + down_read(&data->sem); + if (PMEM_IS_SUBMAP(data)) { + mm = get_task_mm(data->task); + if (!mm) { + up_read(&data->sem); +#if PMEM_DEBUG + pr_alert("pmem: can't remap - task is gone!\n"); +#endif + return -1; + } + } + up_read(&data->sem); + + if (mm) + down_write(&mm->mmap_sem); + + down_write(&data->sem); + /* check that the file didn't get mmaped before we could take the + * data sem, this should be safe b/c you can only submap each file + * once */ + if (PMEM_IS_SUBMAP(data) && !mm) { + pmem_unlock_data_and_mm(data, mm); + DLOG("mapping contention, repeating mmap op\n"); + goto lock_mm; + } + /* now check that vma.mm is still there, it could have been + * deleted by vma_close before we could get the data->sem */ + if ((data->flags & PMEM_FLAGS_UNSUBMAP) && (mm != NULL)) { + /* might as well release this */ + if (data->flags & PMEM_FLAGS_SUBMAP) { + put_task_struct(data->task); + data->task = NULL; + /* lower the submap flag to show the mm is gone */ + data->flags &= ~(PMEM_FLAGS_SUBMAP); + } + pmem_unlock_data_and_mm(data, mm); +#if PMEM_DEBUG + pr_alert("pmem: vma.mm went away!\n"); +#endif + return -1; + } + *locked_mm = mm; + return ret; +} + +int pmem_remap(struct pmem_region *region, struct file *file, + unsigned operation) +{ + int ret; + struct pmem_region_node *region_node; + struct mm_struct *mm = NULL; + struct list_head *elt, *elt2; + int id = get_id(file); + struct pmem_data *data; + + DLOG("operation %#x, region offset %ld, region len %ld\n", + operation, region->offset, region->len); + + if (!is_pmem_file(file)) { +#if PMEM_DEBUG + pr_err("pmem: remap request for non-pmem file descriptor\n"); +#endif + return -EINVAL; + } + + /* is_pmem_file fails if !file */ + data = file->private_data; + + /* pmem region must be aligned on a page boundry */ + if (unlikely(!PMEM_IS_PAGE_ALIGNED(region->offset) || + !PMEM_IS_PAGE_ALIGNED(region->len))) { +#if PMEM_DEBUG + pr_err("pmem: request for unaligned pmem" + "suballocation %lx %lx\n", + region->offset, region->len); +#endif + return -EINVAL; + } + + /* if userspace requests a region of len 0, there's nothing to do */ + if (region->len == 0) + return 0; + + /* lock the mm and data */ + ret = pmem_lock_data_and_mm(file, data, &mm); + if (ret) + return 0; + + /* only the owner of the master file can remap the client fds + * that back in it */ + if (!is_master_owner(file)) { +#if PMEM_DEBUG + pr_err("pmem: remap requested from non-master process\n"); +#endif + ret = -EINVAL; + goto err; + } + + /* check that the requested range is within the src allocation */ + if (unlikely((region->offset > pmem[id].len(id, data)) || + (region->len > pmem[id].len(id, data)) || + (region->offset + region->len > pmem[id].len(id, data)))) { +#if PMEM_DEBUG + pr_err("pmem: suballoc doesn't fit in src_file!\n"); +#endif + ret = -EINVAL; + goto err; + } + + if (operation == PMEM_MAP) { + region_node = kmalloc(sizeof(struct pmem_region_node), + GFP_KERNEL); + if (!region_node) { + ret = -ENOMEM; +#if PMEM_DEBUG + pr_alert("pmem: No space to allocate remap metadata!"); +#endif + goto err; + } + region_node->region = *region; + list_add(®ion_node->list, &data->region_list); + } else if (operation == PMEM_UNMAP) { + int found = 0; + list_for_each_safe(elt, elt2, &data->region_list) { + region_node = list_entry(elt, struct pmem_region_node, + list); + if (region->len == 0 || + (region_node->region.offset == region->offset && + region_node->region.len == region->len)) { + list_del(elt); + kfree(region_node); + found = 1; + } + } + if (!found) { +#if PMEM_DEBUG + pr_err("pmem: Unmap region does not map any" + " mapped region!"); +#endif + ret = -EINVAL; + goto err; + } + } + + if (data->vma && PMEM_IS_SUBMAP(data)) { + if (operation == PMEM_MAP) + ret = pmem_remap_pfn_range(id, data->vma, data, + region->offset, region->len); + else if (operation == PMEM_UNMAP) + ret = pmem_unmap_pfn_range(id, data->vma, data, + region->offset, region->len); + } + +err: + pmem_unlock_data_and_mm(data, mm); + return ret; +} + +static void pmem_revoke(struct file *file, struct pmem_data *data) +{ + struct pmem_region_node *region_node; + struct list_head *elt, *elt2; + struct mm_struct *mm = NULL; + int id = get_id(file); + int ret = 0; + + data->master_file = NULL; + ret = pmem_lock_data_and_mm(file, data, &mm); + /* if lock_data_and_mm fails either the task that mapped the fd, or + * the vma that mapped it have already gone away, nothing more + * needs to be done */ + if (ret) + return; + /* unmap everything */ + /* delete the regions and region list nothing is mapped any more */ + if (data->vma) + list_for_each_safe(elt, elt2, &data->region_list) { + region_node = list_entry(elt, struct pmem_region_node, + list); + pmem_unmap_pfn_range(id, data->vma, data, + region_node->region.offset, + region_node->region.len); + list_del(elt); + kfree(region_node); + } + /* delete the master file */ + pmem_unlock_data_and_mm(data, mm); +} + +static void pmem_get_size(struct pmem_region *region, struct file *file) +{ + /* called via ioctl file op, so file guaranteed to be not NULL */ + struct pmem_data *data = file->private_data; + int id = get_id(file); + + down_read(&data->sem); + if (!has_allocation(file)) { + region->offset = 0; + region->len = 0; + } else { + region->offset = pmem[id].start_addr(id, data); + region->len = pmem[id].len(id, data); + } + up_read(&data->sem); + DLOG("offset 0x%lx len 0x%lx\n", region->offset, region->len); +} + + +static long pmem_ioctl(struct file *file, unsigned int cmd, unsigned long arg) +{ + /* called from user space as file op, so file guaranteed to be not + * NULL + */ + struct pmem_data *data = file->private_data; + int id = get_id(file); +#if PMEM_DEBUG_MSGS + char currtask_name[ + FIELD_SIZEOF(struct task_struct, comm) + 1]; +#endif + + DLOG("pid %u(%s) file %p(%ld) cmd %#x, dev %s(id: %d)\n", + current->pid, get_task_comm(currtask_name, current), + file, file_count(file), cmd, get_name(file), id); + + switch (cmd) { + case PMEM_GET_PHYS: + { + struct pmem_region region; + + DLOG("get_phys\n"); + down_read(&data->sem); + if (!has_allocation(file)) { + region.offset = 0; + region.len = 0; + } else { + region.offset = pmem[id].start_addr(id, data); + region.len = pmem[id].len(id, data); + } + up_read(&data->sem); + + if (copy_to_user((void __user *)arg, ®ion, + sizeof(struct pmem_region))) + return -EFAULT; + + DLOG("pmem: successful request for " + "physical address of pmem region id %d, " + "offset 0x%lx, len 0x%lx\n", + id, region.offset, region.len); + + break; + } + case PMEM_MAP: + { + struct pmem_region region; + DLOG("map\n"); + if (copy_from_user(®ion, (void __user *)arg, + sizeof(struct pmem_region))) + return -EFAULT; + return pmem_remap(®ion, file, PMEM_MAP); + } + break; + case PMEM_UNMAP: + { + struct pmem_region region; + DLOG("unmap\n"); + if (copy_from_user(®ion, (void __user *)arg, + sizeof(struct pmem_region))) + return -EFAULT; + return pmem_remap(®ion, file, PMEM_UNMAP); + break; + } + case PMEM_GET_SIZE: + { + struct pmem_region region; + DLOG("get_size\n"); + pmem_get_size(®ion, file); + if (copy_to_user((void __user *)arg, ®ion, + sizeof(struct pmem_region))) + return -EFAULT; + break; + } + case PMEM_GET_TOTAL_SIZE: + { + struct pmem_region region; + DLOG("get total size\n"); + region.offset = 0; + get_id(file); + region.len = pmem[id].size; + if (copy_to_user((void __user *)arg, ®ion, + sizeof(struct pmem_region))) + return -EFAULT; + break; + } + case PMEM_GET_FREE_SPACE: + { + struct pmem_freespace fs; + DLOG("get freespace on %s(id: %d)\n", + get_name(file), id); + + mutex_lock(&pmem[id].arena_mutex); + pmem[id].free_space(id, &fs); + mutex_unlock(&pmem[id].arena_mutex); + + DLOG("%s(id: %d) total free %lu, largest %lu\n", + get_name(file), id, fs.total, fs.largest); + + if (copy_to_user((void __user *)arg, &fs, + sizeof(struct pmem_freespace))) + return -EFAULT; + break; + } + + case PMEM_ALLOCATE: + { + int ret = 0; + DLOG("allocate, id %d\n", id); + down_write(&data->sem); + if (has_allocation(file)) { + pr_err("pmem: Existing allocation found on " + "this file descrpitor\n"); + up_write(&data->sem); + return -EINVAL; + } + + mutex_lock(&pmem[id].arena_mutex); + data->index = pmem[id].allocate(id, + arg, + SZ_4K); + mutex_unlock(&pmem[id].arena_mutex); + ret = data->index == -1 ? -ENOMEM : + data->index; + up_write(&data->sem); + return ret; + } + case PMEM_ALLOCATE_ALIGNED: + { + struct pmem_allocation alloc; + int ret = 0; + + if (copy_from_user(&alloc, (void __user *)arg, + sizeof(struct pmem_allocation))) + return -EFAULT; + DLOG("allocate id align %d %u\n", id, alloc.align); + down_write(&data->sem); + if (has_allocation(file)) { + pr_err("pmem: Existing allocation found on " + "this file descrpitor\n"); + up_write(&data->sem); + return -EINVAL; + } + + if (alloc.align & (alloc.align - 1)) { + pr_err("pmem: Alignment is not a power of 2\n"); + return -EINVAL; + } + + if (alloc.align != SZ_4K && + (pmem[id].allocator_type != + PMEM_ALLOCATORTYPE_BITMAP)) { + pr_err("pmem: Non 4k alignment requires bitmap" + " allocator on %s\n", pmem[id].name); + return -EINVAL; + } + + if (alloc.align > SZ_1M || + alloc.align < SZ_4K) { + pr_err("pmem: Invalid Alignment (%u) " + "specified\n", alloc.align); + return -EINVAL; + } + + mutex_lock(&pmem[id].arena_mutex); + data->index = pmem[id].allocate(id, + alloc.size, + alloc.align); + mutex_unlock(&pmem[id].arena_mutex); + ret = data->index == -1 ? -ENOMEM : + data->index; + up_write(&data->sem); + return ret; + } + case PMEM_CONNECT: + DLOG("connect\n"); + return pmem_connect(arg, file); + case PMEM_CLEAN_INV_CACHES: + case PMEM_CLEAN_CACHES: + case PMEM_INV_CACHES: + { + struct pmem_addr pmem_addr; + + if (copy_from_user(&pmem_addr, (void __user *)arg, + sizeof(struct pmem_addr))) + return -EFAULT; + + return pmem_cache_maint(file, cmd, &pmem_addr); + } + default: + if (pmem[id].ioctl) + return pmem[id].ioctl(file, cmd, arg); + + DLOG("ioctl invalid (%#x)\n", cmd); + return -EINVAL; + } + return 0; +} + +static void ioremap_pmem(int id) +{ + if (pmem[id].cached) + pmem[id].vbase = ioremap_cached(pmem[id].base, pmem[id].size); +#ifdef ioremap_ext_buffered + else if (pmem[id].buffered) + pmem[id].vbase = ioremap_ext_buffered(pmem[id].base, + pmem[id].size); +#endif + else + pmem[id].vbase = ioremap(pmem[id].base, pmem[id].size); +} + +#ifdef CONFIG_MEMORY_HOTPLUG +static int pmem_mapped_regions(int id) +{ + struct list_head *elt; + + mutex_lock(&pmem[id].data_list_mutex); + list_for_each(elt, &pmem[id].data_list) { + struct pmem_data *data = + list_entry(elt, struct pmem_data, list); + + if (data) { + mutex_unlock(&pmem[id].data_list_mutex); + return 1; + } + } + mutex_unlock(&pmem[id].data_list_mutex); + return 0; +} + +static int active_unstable_pmem(void) +{ + int id; + + for (id = 0; id < id_count; id++) { + if (pmem[id].memory_state == MEMORY_STABLE) + continue; + if (pmem_mapped_regions(id)) + return 1; + } + + return 0; +} + +static void reserve_unstable_pmem(unsigned long unstable_pmem_start, + unsigned long unstable_pmem_size) +{ + reserve_hotplug_pages(unstable_pmem_start >> PAGE_SHIFT, + unstable_pmem_size >> PAGE_SHIFT); +} + +static void unreserve_unstable_pmem(unsigned long unstable_pmem_start, + unsigned long unstable_pmem_size) +{ + unreserve_hotplug_pages(unstable_pmem_start >> PAGE_SHIFT, + unstable_pmem_size >> PAGE_SHIFT); +} + +static void pmem_setup_unstable_devices(unsigned long start_pfn, + unsigned long nr_pages) +{ + int id; + unsigned long tmp; + + unstable_pmem_start = start_pfn << PAGE_SHIFT; + tmp = unstable_pmem_start; + + for (id = 0; id < id_count; id++) { + if (pmem[id].memory_state == MEMORY_STABLE) + continue; + + pmem[id].base = tmp; + pr_info("reserving %lx bytes unstable memory at %lx \ + for %s\n", pmem[id].size, pmem[id].base, pmem[id].name); + tmp += pmem[id].size; + } + unstable_pmem_size = tmp - unstable_pmem_start; + + for (id = 0; id < id_count; id++) { + if (pmem[id].memory_state == + MEMORY_UNSTABLE_NO_MEMORY_ALLOCATED) { + ioremap_pmem(id); + pmem[id].garbage_pfn = + page_to_pfn(alloc_page(GFP_KERNEL)); + + if (pmem[id].vbase == 0) + continue; + pmem[id].memory_state = + MEMORY_UNSTABLE_MEMORY_ALLOCATED; + } + } +} + +static int pmem_mem_going_offline_callback(void *arg) +{ + struct memory_notify *marg = arg; + int id; + + if ((marg->start_pfn << PAGE_SHIFT) != unstable_pmem_start) + return 0; + + if (active_unstable_pmem()) { + pr_alert("unstable PMEM memory device in use \ + prevents memory hotremove!\n"); + return -EAGAIN; + } + + unreserve_unstable_pmem(unstable_pmem_start, unstable_pmem_size); + + for (id = 0; id < id_count; id++) { + if (pmem[id].memory_state == MEMORY_UNSTABLE_MEMORY_ALLOCATED) + pmem[id].memory_state = + MEMORY_UNSTABLE_NO_MEMORY_ALLOCATED; + } + return 0; +} + +static int pmem_mem_online_callback(void *arg) +{ + struct memory_notify *marg = arg; + int id; + + + if (unstable_pmem_present == UNSTABLE_UNINITIALIZED) { + pmem_setup_unstable_devices(marg->start_pfn, marg->nr_pages); + pr_alert("unstable pmem start %lx size %lx\n", + unstable_pmem_start, unstable_pmem_size); + unstable_pmem_present = UNSTABLE_INITIALIZED; + } + + if ((marg->start_pfn << PAGE_SHIFT) != unstable_pmem_start) + return 0; + + reserve_unstable_pmem(unstable_pmem_start, unstable_pmem_size); + + for (id = 0; id < id_count; id++) { + if (pmem[id].memory_state == + MEMORY_UNSTABLE_NO_MEMORY_ALLOCATED) { + if (pmem[id].vbase == 0) + ioremap_pmem(id); + if (pmem[id].vbase == 0) + continue; + pmem[id].memory_state = + MEMORY_UNSTABLE_MEMORY_ALLOCATED; + } + } + return 0; +} + +static int pmem_memory_callback(struct notifier_block *self, + unsigned long action, void *arg) +{ + int ret = 0; + + if (unstable_pmem_present == NO_UNSTABLE_MEMORY) + return 0; + + switch (action) { + case MEM_ONLINE: + ret = pmem_mem_online_callback(arg); + break; + case MEM_GOING_OFFLINE: + ret = pmem_mem_going_offline_callback(arg); + break; + case MEM_OFFLINE: + case MEM_GOING_ONLINE: + case MEM_CANCEL_ONLINE: + case MEM_CANCEL_OFFLINE: + break; + } + if (ret) + ret = notifier_from_errno(ret); + else + ret = NOTIFY_OK; + return ret; +} +#endif + +int pmem_setup(struct android_pmem_platform_data *pdata, + long (*ioctl)(struct file *, unsigned int, unsigned long), + int (*release)(struct inode *, struct file *)) +{ + int i, index = 0, kapi_memtype_idx = -1, id, is_kernel_memtype = 0; + + if (id_count >= PMEM_MAX_DEVICES) { + pr_alert("pmem: %s: unable to register driver(%s) - no more " + "devices available!\n", __func__, pdata->name); + goto err_no_mem; + } + + if (!pdata->size) { + pr_alert("pmem: %s: unable to register pmem driver(%s) - zero " + "size passed in!\n", __func__, pdata->name); + goto err_no_mem; + } + + id = id_count++; + + pmem[id].id = id; + + if (pmem[id].allocate) { + pr_alert("pmem: %s: unable to register pmem driver - " + "duplicate registration of %s!\n", + __func__, pdata->name); + goto err_no_mem; + } + + pmem[id].allocator_type = pdata->allocator_type; + + for (i = 0; i < ARRAY_SIZE(kapi_memtypes); i++) { + if (!strcmp(kapi_memtypes[i].name, pdata->name)) { + if (kapi_memtypes[i].info_id >= 0) { + pr_alert("Unable to register kernel pmem " + "driver - duplicate registration of " + "%s!\n", pdata->name); + goto err_no_mem; + } + if (pdata->cached) { + pr_alert("kernel arena memory must " + "NOT be configured as 'cached'. Check " + "and fix your board file. Failing " + "pmem driver %s registration!", + pdata->name); + goto err_no_mem; + } + + is_kernel_memtype = 1; + kapi_memtypes[i].info_id = id; + kapi_memtype_idx = i; + break; + } + } + + /* 'quantum' is a "hidden" variable that defaults to 0 in the board + * files */ + pmem[id].quantum = pdata->quantum ?: PMEM_MIN_ALLOC; + if (pmem[id].quantum < PMEM_MIN_ALLOC || + !is_power_of_2(pmem[id].quantum)) { + pr_alert("pmem: %s: unable to register pmem driver %s - " + "invalid quantum value (%#x)!\n", + __func__, pdata->name, pmem[id].quantum); + goto err_reset_pmem_info; + } + + if (pdata->start % pmem[id].quantum) { + /* bad alignment for start! */ + pr_alert("pmem: %s: Unable to register driver %s - " + "improperly aligned memory region start address " + "(%#lx) as checked against quantum value of %#x!\n", + __func__, pdata->name, pdata->start, + pmem[id].quantum); + goto err_reset_pmem_info; + } + + if (pdata->size % pmem[id].quantum) { + /* bad alignment for size! */ + pr_alert("pmem: %s: Unable to register driver %s - " + "memory region size (%#lx) is not a multiple of " + "quantum size(%#x)!\n", __func__, pdata->name, + pdata->size, pmem[id].quantum); + goto err_reset_pmem_info; + } + + pmem[id].cached = pdata->cached; + pmem[id].buffered = pdata->buffered; + pmem[id].base = pdata->start; + pmem[id].size = pdata->size; + strlcpy(pmem[id].name, pdata->name, PMEM_NAME_SIZE); + + if (pdata->unstable) { + pmem[id].memory_state = MEMORY_UNSTABLE_NO_MEMORY_ALLOCATED; + unstable_pmem_present = UNSTABLE_UNINITIALIZED; + } + + pmem[id].num_entries = pmem[id].size / pmem[id].quantum; + + memset(&pmem[id].kobj, 0, sizeof(pmem[0].kobj)); + pmem[id].kobj.kset = pmem_kset; + + switch (pmem[id].allocator_type) { + case PMEM_ALLOCATORTYPE_ALLORNOTHING: + pmem[id].allocate = pmem_allocator_all_or_nothing; + pmem[id].free = pmem_free_all_or_nothing; + pmem[id].free_space = pmem_free_space_all_or_nothing; + pmem[id].kapi_free_index = pmem_kapi_free_index_allornothing; + pmem[id].len = pmem_len_all_or_nothing; + pmem[id].start_addr = pmem_start_addr_all_or_nothing; + pmem[id].num_entries = 1; + pmem[id].quantum = pmem[id].size; + pmem[id].allocator.all_or_nothing.allocated = 0; + + if (kobject_init_and_add(&pmem[id].kobj, + &pmem_allornothing_ktype, NULL, + "%s", pdata->name)) + goto out_put_kobj; + + break; + + case PMEM_ALLOCATORTYPE_BUDDYBESTFIT: + pmem[id].allocator.buddy_bestfit.buddy_bitmap = kmalloc( + pmem[id].num_entries * sizeof(struct pmem_bits), + GFP_KERNEL); + if (!pmem[id].allocator.buddy_bestfit.buddy_bitmap) + goto err_reset_pmem_info; + + memset(pmem[id].allocator.buddy_bestfit.buddy_bitmap, 0, + sizeof(struct pmem_bits) * pmem[id].num_entries); + + for (i = sizeof(pmem[id].num_entries) * 8 - 1; i >= 0; i--) + if ((pmem[id].num_entries) & 1<name)) + goto out_put_kobj; + + break; + + case PMEM_ALLOCATORTYPE_BITMAP: /* 0, default if not explicit */ + pmem[id].allocator.bitmap.bitm_alloc = kmalloc( + PMEM_INITIAL_NUM_BITMAP_ALLOCATIONS * + sizeof(*pmem[id].allocator.bitmap.bitm_alloc), + GFP_KERNEL); + if (!pmem[id].allocator.bitmap.bitm_alloc) { + pr_alert("pmem: %s: Unable to register pmem " + "driver %s - can't allocate " + "bitm_alloc!\n", + __func__, pdata->name); + goto err_reset_pmem_info; + } + + if (kobject_init_and_add(&pmem[id].kobj, + &pmem_bitmap_ktype, NULL, + "%s", pdata->name)) + goto out_put_kobj; + + for (i = 0; i < PMEM_INITIAL_NUM_BITMAP_ALLOCATIONS; i++) { + pmem[id].allocator.bitmap.bitm_alloc[i].bit = -1; + pmem[id].allocator.bitmap.bitm_alloc[i].quanta = 0; + } + + pmem[id].allocator.bitmap.bitmap_allocs = + PMEM_INITIAL_NUM_BITMAP_ALLOCATIONS; + + pmem[id].allocator.bitmap.bitmap = + kcalloc((pmem[id].num_entries + 31) / 32, + sizeof(unsigned int), GFP_KERNEL); + if (!pmem[id].allocator.bitmap.bitmap) { + pr_alert("pmem: %s: Unable to register pmem " + "driver - can't allocate bitmap!\n", + __func__); + goto err_cant_register_device; + } + pmem[id].allocator.bitmap.bitmap_free = pmem[id].num_entries; + + pmem[id].allocate = pmem_allocator_bitmap; + pmem[id].free = pmem_free_bitmap; + pmem[id].free_space = pmem_free_space_bitmap; + pmem[id].kapi_free_index = pmem_kapi_free_index_bitmap; + pmem[id].len = pmem_len_bitmap; + pmem[id].start_addr = pmem_start_addr_bitmap; + + DLOG("bitmap allocator id %d (%s), num_entries %u, raw size " + "%lu, quanta size %u\n", + id, pdata->name, pmem[id].allocator.bitmap.bitmap_free, + pmem[id].size, pmem[id].quantum); + break; + + default: + pr_alert("Invalid allocator type (%d) for pmem driver\n", + pdata->allocator_type); + goto err_reset_pmem_info; + } + + pmem[id].ioctl = ioctl; + pmem[id].release = release; + mutex_init(&pmem[id].arena_mutex); + mutex_init(&pmem[id].data_list_mutex); + INIT_LIST_HEAD(&pmem[id].data_list); + + pmem[id].dev.name = pdata->name; + if (!is_kernel_memtype) { + pmem[id].dev.minor = id; + pmem[id].dev.fops = &pmem_fops; + pr_info("pmem: Initializing %s (user-space) as %s\n", + pdata->name, pdata->cached ? "cached" : "non-cached"); + + if (misc_register(&pmem[id].dev)) { + pr_alert("Unable to register pmem driver!\n"); + goto err_cant_register_device; + } + } else { /* kernel region, no user accessible device */ + pmem[id].dev.minor = -1; + pr_info("pmem: Initializing %s (in-kernel)\n", pdata->name); + } + + /* do not set up unstable pmem now, wait until first memory hotplug */ + if (pmem[id].memory_state == MEMORY_UNSTABLE_NO_MEMORY_ALLOCATED) + return 0; + + if (!is_kernel_memtype) { + ioremap_pmem(id); + if (pmem[id].vbase == 0) { + pr_err("pmem: ioremap failed for device %s\n", + pmem[id].name); + goto error_cant_remap; + } + } + + pmem[id].garbage_pfn = page_to_pfn(alloc_page(GFP_KERNEL)); + + return 0; + +error_cant_remap: + if (!is_kernel_memtype) + misc_deregister(&pmem[id].dev); +err_cant_register_device: +out_put_kobj: + kobject_put(&pmem[id].kobj); + if (pmem[id].allocator_type == PMEM_ALLOCATORTYPE_BUDDYBESTFIT) + kfree(pmem[id].allocator.buddy_bestfit.buddy_bitmap); + else if (pmem[id].allocator_type == PMEM_ALLOCATORTYPE_BITMAP) { + kfree(pmem[id].allocator.bitmap.bitmap); + kfree(pmem[id].allocator.bitmap.bitm_alloc); + } +err_reset_pmem_info: + pmem[id].allocate = 0; + pmem[id].dev.minor = -1; + if (kapi_memtype_idx >= 0) + kapi_memtypes[i].info_id = -1; +err_no_mem: + return -1; +} + +static int pmem_probe(struct platform_device *pdev) +{ + struct android_pmem_platform_data *pdata; + + if (!pdev || !pdev->dev.platform_data) { + pr_alert("Unable to probe pmem!\n"); + return -1; + } + pdata = pdev->dev.platform_data; + + pm_runtime_set_active(&pdev->dev); + pm_runtime_enable(&pdev->dev); + + return pmem_setup(pdata, NULL, NULL); +} + +static int pmem_remove(struct platform_device *pdev) +{ + int id = pdev->id; + __free_page(pfn_to_page(pmem[id].garbage_pfn)); + pm_runtime_disable(&pdev->dev); + misc_deregister(&pmem[id].dev); + return 0; +} + +static int pmem_runtime_suspend(struct device *dev) +{ + dev_dbg(dev, "pm_runtime: suspending...\n"); + return 0; +} + +static int pmem_runtime_resume(struct device *dev) +{ + dev_dbg(dev, "pm_runtime: resuming...\n"); + return 0; +} + +static const struct dev_pm_ops pmem_dev_pm_ops = { + .runtime_suspend = pmem_runtime_suspend, + .runtime_resume = pmem_runtime_resume, +}; + +static struct platform_driver pmem_driver = { + .probe = pmem_probe, + .remove = pmem_remove, + .driver = { .name = "android_pmem", + .pm = &pmem_dev_pm_ops, + } +}; + + +static int __init pmem_init(void) +{ + /* create /sys/kernel/ directory */ + pmem_kset = kset_create_and_add(PMEM_SYSFS_DIR_NAME, + NULL, kernel_kobj); + if (!pmem_kset) { + pr_err("pmem(%s):kset_create_and_add fail\n", __func__); + return -ENOMEM; + } +/*HTC_START*/ + +root = vidc_get_debugfs_root(); + if (root) { + vidc_debugfs_file_create(root, "misc_msg_pmem_qcom", + (u32 *) &misc_msg_pmem_qcom); + } +/*HTC_END*/ +#ifdef CONFIG_MEMORY_HOTPLUG + hotplug_memory_notifier(pmem_memory_callback, 0); +#endif + return platform_driver_register(&pmem_driver); +} + +static void __exit pmem_exit(void) +{ + platform_driver_unregister(&pmem_driver); +} + +module_init(pmem_init); +module_exit(pmem_exit); +#else + /* drivers/android/pmem.c * * Copyright (C) 2007 Google, Inc. @@ -250,7 +3292,8 @@ static int pmem_free(int id, int index) */ do { buddy = PMEM_BUDDY_INDEX(id, curr); - if (PMEM_IS_FREE(id, buddy) && + if (buddy < pmem[id].num_entries && + PMEM_IS_FREE(id, buddy) && PMEM_ORDER(id, buddy) == PMEM_ORDER(id, curr)) { PMEM_ORDER(id, buddy)++; PMEM_ORDER(id, curr)++; @@ -1343,3 +4386,4 @@ static void __exit pmem_exit(void) module_init(pmem_init); module_exit(pmem_exit); +#endif \ No newline at end of file diff --git a/drivers/video/Kconfig b/drivers/video/Kconfig index 576a74c3..7a43dff5 100644 --- a/drivers/video/Kconfig +++ b/drivers/video/Kconfig @@ -11,6 +11,8 @@ source "drivers/gpu/vga/Kconfig" source "drivers/gpu/drm/Kconfig" +source "drivers/gpu/msm/Kconfig" + config VGASTATE tristate default n diff --git a/drivers/video/msm/Kconfig b/drivers/video/msm/Kconfig index 1ca3622c..9ce28950 100644 --- a/drivers/video/msm/Kconfig +++ b/drivers/video/msm/Kconfig @@ -55,7 +55,7 @@ config MSM_ROTATOR_USE_IMEM block. Or some systems may want the iMem to be dedicated to a different function. -config MSM_KGSL_MMU +config GPU_MSM_KGSL_MMU bool "Turn on MMU for graphics driver " depends on GPU_MSM_KGSL && MMU default n diff --git a/drivers/video/msm/Makefile b/drivers/video/msm/Makefile index bb2447d1..a013cd57 100644 --- a/drivers/video/msm/Makefile +++ b/drivers/video/msm/Makefile @@ -34,10 +34,4 @@ obj-$(CONFIG_FB_MSM_LCDC) += mdp_lcdc.o obj-$(CONFIG_FB_MSM_TVOUT) += tvenc.o tvfb.o # Yamato GL driver -ifeq ($(CONFIG_ARCH_MSM7X30),y) -obj-$(CONFIG_GPU_MSM_KGSL) += gpu/kgsl_adreno205/ -else obj-$(CONFIG_GPU_MSM_KGSL) += gpu/kgsl/ -endif - -obj-$(CONFIG_MSM_HDMI) += hdmi/ diff --git a/drivers/video/msm/gpu/KGSL_SI/Untitled Project.IAB b/drivers/video/msm/gpu/KGSL_SI/Untitled Project.IAB new file mode 100755 index 00000000..e8d88646 Binary files /dev/null and b/drivers/video/msm/gpu/KGSL_SI/Untitled Project.IAB differ diff --git a/drivers/video/msm/gpu/KGSL_SI/Untitled Project.IAD b/drivers/video/msm/gpu/KGSL_SI/Untitled Project.IAD new file mode 100755 index 00000000..09e28dae Binary files /dev/null and b/drivers/video/msm/gpu/KGSL_SI/Untitled Project.IAD differ diff --git a/drivers/video/msm/gpu/KGSL_SI/Untitled Project.IMB b/drivers/video/msm/gpu/KGSL_SI/Untitled Project.IMB new file mode 100755 index 00000000..ae098b3b Binary files /dev/null and b/drivers/video/msm/gpu/KGSL_SI/Untitled Project.IMB differ diff --git a/drivers/video/msm/gpu/KGSL_SI/Untitled Project.IMD b/drivers/video/msm/gpu/KGSL_SI/Untitled Project.IMD new file mode 100755 index 00000000..89d25dd2 Binary files /dev/null and b/drivers/video/msm/gpu/KGSL_SI/Untitled Project.IMD differ diff --git a/drivers/video/msm/gpu/KGSL_SI/Untitled Project.PFI b/drivers/video/msm/gpu/KGSL_SI/Untitled Project.PFI new file mode 100755 index 00000000..4e77fed0 Binary files /dev/null and b/drivers/video/msm/gpu/KGSL_SI/Untitled Project.PFI differ diff --git a/drivers/video/msm/gpu/KGSL_SI/Untitled Project.PO b/drivers/video/msm/gpu/KGSL_SI/Untitled Project.PO new file mode 100755 index 00000000..30a745ff Binary files /dev/null and b/drivers/video/msm/gpu/KGSL_SI/Untitled Project.PO differ diff --git a/drivers/video/msm/gpu/KGSL_SI/Untitled Project.PR b/drivers/video/msm/gpu/KGSL_SI/Untitled Project.PR new file mode 100755 index 00000000..7a4ae3e4 Binary files /dev/null and b/drivers/video/msm/gpu/KGSL_SI/Untitled Project.PR differ diff --git a/drivers/video/msm/gpu/KGSL_SI/Untitled Project.PRI b/drivers/video/msm/gpu/KGSL_SI/Untitled Project.PRI new file mode 100755 index 00000000..65dfe059 Binary files /dev/null and b/drivers/video/msm/gpu/KGSL_SI/Untitled Project.PRI differ diff --git a/drivers/video/msm/gpu/KGSL_SI/Untitled Project.PS b/drivers/video/msm/gpu/KGSL_SI/Untitled Project.PS new file mode 100755 index 00000000..0d0fa1d5 Binary files /dev/null and b/drivers/video/msm/gpu/KGSL_SI/Untitled Project.PS differ diff --git a/drivers/video/msm/gpu/KGSL_SI/Untitled Project.SearchResults b/drivers/video/msm/gpu/KGSL_SI/Untitled Project.SearchResults new file mode 100755 index 00000000..71943fcd --- /dev/null +++ b/drivers/video/msm/gpu/KGSL_SI/Untitled Project.SearchResults @@ -0,0 +1,12 @@ +---- CONFIG_MSM_KGSL_MMU Matches (11 in 6 files) ---- +kgsl.c (\\jupiter\xinwang\hd2\kernel\msm8x50\20120427A\dorimanx-Dorimanx-HD2-2.6.32.X-69084da_hwa_mix\drivers\video\msm\gpu\kgsl):#ifdef CONFIG_MSM_KGSL_MMU +kgsl.c (\\jupiter\xinwang\hd2\kernel\msm8x50\20120427A\dorimanx-Dorimanx-HD2-2.6.32.X-69084da_hwa_mix\drivers\video\msm\gpu\kgsl):#ifdef CONFIG_MSM_KGSL_MMU +kgsl.c (\\jupiter\xinwang\hd2\kernel\msm8x50\20120427A\dorimanx-Dorimanx-HD2-2.6.32.X-69084da_hwa_mix\drivers\video\msm\gpu\kgsl):#ifdef CONFIG_MSM_KGSL_MMU +kgsl_device.h (\\jupiter\xinwang\hd2\kernel\msm8x50\20120427A\dorimanx-Dorimanx-HD2-2.6.32.X-69084da_hwa_mix\drivers\video\msm\gpu\kgsl):#ifdef CONFIG_MSM_KGSL_MMU +kgsl_log.c (\\jupiter\xinwang\hd2\kernel\msm8x50\20120427A\dorimanx-Dorimanx-HD2-2.6.32.X-69084da_hwa_mix\drivers\video\msm\gpu\kgsl):#ifdef CONFIG_MSM_KGSL_MMU +kgsl_log.c (\\jupiter\xinwang\hd2\kernel\msm8x50\20120427A\dorimanx-Dorimanx-HD2-2.6.32.X-69084da_hwa_mix\drivers\video\msm\gpu\kgsl):#ifdef CONFIG_MSM_KGSL_MMU +kgsl_mmu.c (\\jupiter\xinwang\hd2\kernel\msm8x50\20120427A\dorimanx-Dorimanx-HD2-2.6.32.X-69084da_hwa_mix\drivers\video\msm\gpu\kgsl):#ifndef CONFIG_MSM_KGSL_MMU +kgsl_mmu.c (\\jupiter\xinwang\hd2\kernel\msm8x50\20120427A\dorimanx-Dorimanx-HD2-2.6.32.X-69084da_hwa_mix\drivers\video\msm\gpu\kgsl):#ifdef CONFIG_MSM_KGSL_MMU +kgsl_mmu.h (\\jupiter\xinwang\hd2\kernel\msm8x50\20120427A\dorimanx-Dorimanx-HD2-2.6.32.X-69084da_hwa_mix\drivers\video\msm\gpu\kgsl):#ifdef CONFIG_MSM_KGSL_MMU +kgsl_yamato.c (\\jupiter\xinwang\hd2\kernel\msm8x50\20120427A\dorimanx-Dorimanx-HD2-2.6.32.X-69084da_hwa_mix\drivers\video\msm\gpu\kgsl):#ifdef CONFIG_MSM_KGSL_MMU +kgsl_yamato.c (\\jupiter\xinwang\hd2\kernel\msm8x50\20120427A\dorimanx-Dorimanx-HD2-2.6.32.X-69084da_hwa_mix\drivers\video\msm\gpu\kgsl):#ifdef CONFIG_MSM_KGSL_MMU diff --git a/drivers/video/msm/gpu/kgsl/kgsl.c b/drivers/video/msm/gpu/kgsl/kgsl.c index 4dde98e0..2d296514 100644 --- a/drivers/video/msm/gpu/kgsl/kgsl.c +++ b/drivers/video/msm/gpu/kgsl/kgsl.c @@ -51,7 +51,7 @@ struct kgsl_file_private { static void kgsl_put_phys_file(struct file *file); -#ifdef CONFIG_MSM_KGSL_MMU +#ifdef CONFIG_GPU_MSM_KGSL_MMU static long flush_l1_cache_range(unsigned long addr, int size) { struct page *page; @@ -190,11 +190,6 @@ static int kgsl_first_open_locked(void) kgsl_clk_enable(); - /* init memory apertures */ - result = kgsl_sharedmem_init(&kgsl_driver.shmem); - if (result != 0) - goto done; - /* init devices */ result = kgsl_yamato_init(&kgsl_driver.yamato_device, &kgsl_driver.yamato_config); @@ -221,9 +216,6 @@ static int kgsl_last_release_locked(void) /* close devices */ kgsl_yamato_close(&kgsl_driver.yamato_device); - /* shutdown memory apertures */ - kgsl_sharedmem_close(&kgsl_driver.shmem); - kgsl_clk_disable(); kgsl_driver.active = false; wake_unlock(&kgsl_driver.wake_lock); @@ -642,7 +634,7 @@ done: return result; } -#ifdef CONFIG_MSM_KGSL_MMU +#ifdef CONFIG_GPU_MSM_KGSL_MMU static int kgsl_ioctl_sharedmem_from_vmalloc(struct kgsl_file_private *private, void __user *arg) { @@ -888,7 +880,7 @@ error: return result; } -#ifdef CONFIG_MSM_KGSL_MMU +#ifdef CONFIG_GPU_MSM_KGSL_MMU /*This function flushes a graphics memory allocation from CPU cache *when caching is enabled with MMU*/ static int kgsl_ioctl_sharedmem_flush_cache(struct kgsl_file_private *private, @@ -1066,6 +1058,9 @@ static void kgsl_driver_cleanup(void) kgsl_driver.interrupt_num = 0; } + /* shutdown memory apertures */ + kgsl_sharedmem_close(&kgsl_driver.shmem); + if (kgsl_driver.grp_clk) { clk_put(kgsl_driver.grp_clk); kgsl_driver.grp_clk = NULL; @@ -1170,6 +1165,9 @@ static int __devinit kgsl_platform_probe(struct platform_device *pdev) kgsl_driver.shmem.physbase = res->start; kgsl_driver.shmem.size = resource_size(res); + /* init memory apertures */ + result = kgsl_sharedmem_init(&kgsl_driver.shmem); + done: if (result) kgsl_driver_cleanup(); diff --git a/drivers/video/msm/gpu/kgsl/kgsl_cmdstream.c b/drivers/video/msm/gpu/kgsl/kgsl_cmdstream.c index 9b7183cb..d8a16551 100644 --- a/drivers/video/msm/gpu/kgsl/kgsl_cmdstream.c +++ b/drivers/video/msm/gpu/kgsl/kgsl_cmdstream.c @@ -46,6 +46,8 @@ kgsl_cmdstream_readtimestamp(struct kgsl_device *device, KGSL_CMDSTREAM_GET_EOP_TIMESTAMP(device, (unsigned int *)×tamp); + rmb(); + KGSL_CMD_VDBG("return %d\n", timestamp); return timestamp; diff --git a/drivers/video/msm/gpu/kgsl/kgsl_device.h b/drivers/video/msm/gpu/kgsl/kgsl_device.h index dcbf4db7..c88e0cb5 100644 --- a/drivers/video/msm/gpu/kgsl/kgsl_device.h +++ b/drivers/video/msm/gpu/kgsl/kgsl_device.h @@ -129,7 +129,7 @@ int kgsl_yamato_setup_pt(struct kgsl_device *device, struct kgsl_pagetable *pagetable); int kgsl_yamato_cleanup_pt(struct kgsl_device *device, struct kgsl_pagetable *pagetable); -#ifdef CONFIG_MSM_KGSL_MMU +#ifdef CONFIG_GPU_MSM_KGSL_MMU int kgsl_yamato_setstate(struct kgsl_device *device, uint32_t flags); #else static inline int kgsl_yamato_setstate(struct kgsl_device *device, uint32_t flags) diff --git a/drivers/video/msm/gpu/kgsl/kgsl_log.c b/drivers/video/msm/gpu/kgsl/kgsl_log.c index 6308087d..d39f3447 100644 --- a/drivers/video/msm/gpu/kgsl/kgsl_log.c +++ b/drivers/video/msm/gpu/kgsl/kgsl_log.c @@ -237,7 +237,7 @@ static struct file_operations kgsl_mmu_regs_fops = { }; #endif /*DEBUG*/ -#ifdef CONFIG_MSM_KGSL_MMU +#ifdef CONFIG_GPU_MSM_KGSL_MMU static int kgsl_cache_enable_set(void *data, u64 val) { kgsl_cache_enable = (val != 0); @@ -282,7 +282,7 @@ int kgsl_debug_init(void) &kgsl_mmu_regs_fops); #endif -#ifdef CONFIG_MSM_KGSL_MMU +#ifdef CONFIG_GPU_MSM_KGSL_MMU debugfs_create_file("cache_enable", 0644, dent, 0, &kgsl_cache_enable_fops); #endif diff --git a/drivers/video/msm/gpu/kgsl/kgsl_mmu.c b/drivers/video/msm/gpu/kgsl/kgsl_mmu.c index c32558ff..eb7794ba 100644 --- a/drivers/video/msm/gpu/kgsl/kgsl_mmu.c +++ b/drivers/video/msm/gpu/kgsl/kgsl_mmu.c @@ -319,7 +319,7 @@ int kgsl_mmu_init(struct kgsl_device *device) mmu->device = device; -#ifndef CONFIG_MSM_KGSL_MMU +#ifndef CONFIG_GPU_MSM_KGSL_MMU mmu->config = 0x00000000; #endif @@ -396,6 +396,16 @@ int kgsl_mmu_init(struct kgsl_device *device) return -ENOMEM; } mmu->hwpagetable = mmu->defaultpagetable; + mmu->tlbflushfilter.size = (mmu->va_range / + (PAGE_SIZE * GSL_PT_SUPER_PTE * 8)) + 1; + mmu->tlbflushfilter.base = (unsigned int *) + kzalloc(mmu->tlbflushfilter.size, GFP_KERNEL); + if (!mmu->tlbflushfilter.base) { + KGSL_MEM_ERR("Failed to create tlbflushfilter\n"); + kgsl_mmu_close(device); + return -ENOMEM; + } + GSL_TLBFLUSH_FILTER_RESET(); kgsl_yamato_regwrite(device, REG_MH_MMU_PT_BASE, mmu->hwpagetable->base.gpuaddr); kgsl_yamato_regwrite(device, REG_MH_MMU_VA_RANGE, @@ -415,7 +425,7 @@ int kgsl_mmu_init(struct kgsl_device *device) return 0; } -#ifdef CONFIG_MSM_KGSL_MMU +#ifdef CONFIG_GPU_MSM_KGSL_MMU pte_t *kgsl_get_pte_from_vaddr(unsigned int vaddr) { pgd_t *pgd_ptr = NULL; @@ -456,7 +466,7 @@ int kgsl_mmu_map(struct kgsl_pagetable *pagetable, unsigned int flags) { int numpages; - unsigned int pte, superpte, ptefirst, ptelast, physaddr; + unsigned int pte, ptefirst, ptelast, physaddr; int flushtlb, alloc_size; struct kgsl_mmu *mmu = NULL; int phys_contiguous = flags & KGSL_MEMFLAGS_CONPHYS; @@ -514,15 +524,11 @@ int kgsl_mmu_map(struct kgsl_pagetable *pagetable, pte = ptefirst; flushtlb = 0; - superpte = ptefirst & (GSL_PT_SUPER_PTE - 1); - for (pte = superpte; pte < ptefirst; pte++) { - /* tlb needs to be flushed only when a dirty superPTE - gets backed */ - if (kgsl_pt_map_isdirty(pagetable, pte)) { - flushtlb = 1; - break; - } - } + /* tlb needs to be flushed when the first and last pte are not at + * superpte boundaries */ + if ((ptefirst & (GSL_PT_SUPER_PTE - 1)) != 0 || + ((ptelast + 1) & (GSL_PT_SUPER_PTE-1)) != 0) + flushtlb = 1; for (pte = ptefirst; pte < ptelast; pte++) { #ifdef VERBOSE_DEBUG @@ -530,8 +536,10 @@ int kgsl_mmu_map(struct kgsl_pagetable *pagetable, uint32_t val = kgsl_pt_map_getaddr(pagetable, pte); BUG_ON(val != 0 && val != GSL_PT_PAGE_DIRTY); #endif - if (kgsl_pt_map_isdirty(pagetable, pte)) + if ((pte & (GSL_PT_SUPER_PTE-1)) == 0) + if (GSL_TLBFLUSH_FILTER_ISDIRTY(pte / GSL_PT_SUPER_PTE)) flushtlb = 1; + /* mark pte as in use */ if (phys_contiguous) physaddr = address; @@ -552,17 +560,6 @@ int kgsl_mmu_map(struct kgsl_pagetable *pagetable, address += KGSL_PAGESIZE; } - /* set superpte to end of next superpte */ - superpte = (ptelast + (GSL_PT_SUPER_PTE - 1)) - & (GSL_PT_SUPER_PTE - 1); - for (pte = ptelast; pte < superpte; pte++) { - /* tlb needs to be flushed only when a dirty superPTE - gets backed */ - if (kgsl_pt_map_isdirty(pagetable, pte)) { - flushtlb = 1; - break; - } - } KGSL_MEM_INFO("pt %p p %08x g %08x pte f %d l %d n %d f %d\n", pagetable, address, *gpuaddr, ptefirst, ptelast, numpages, flushtlb); @@ -571,8 +568,10 @@ int kgsl_mmu_map(struct kgsl_pagetable *pagetable, /* Invalidate tlb only if current page table used by GPU is the * pagetable that we used to allocate */ - if (pagetable == mmu->hwpagetable) + if (flushtlb && (pagetable == mmu->hwpagetable)) { kgsl_yamato_setstate(mmu->device, KGSL_MMUFLAGS_TLBFLUSH); + GSL_TLBFLUSH_FILTER_RESET(); + } KGSL_MEM_VDBG("return %d\n", 0); @@ -585,7 +584,8 @@ kgsl_mmu_unmap(struct kgsl_pagetable *pagetable, unsigned int gpuaddr, int range) { unsigned int numpages; - unsigned int pte, ptefirst, ptelast; + unsigned int pte, ptefirst, ptelast, superpte; + struct kgsl_mmu *mmu = NULL; KGSL_MEM_VDBG("enter (pt=%p, gpuaddr=0x%08x, range=%d)\n", pagetable, gpuaddr, range); @@ -602,22 +602,24 @@ kgsl_mmu_unmap(struct kgsl_pagetable *pagetable, unsigned int gpuaddr, KGSL_MEM_INFO("pt %p gpu %08x pte first %d last %d numpages %d\n", pagetable, gpuaddr, ptefirst, ptelast, numpages); + mmu = pagetable->mmu; + + superpte = ptefirst - (ptefirst & (GSL_PT_SUPER_PTE-1)); + GSL_TLBFLUSH_FILTER_SETDIRTY(superpte / GSL_PT_SUPER_PTE); for (pte = ptefirst; pte < ptelast; pte++) { #ifdef VERBOSE_DEBUG /* check if PTE exists */ BUG_ON(!kgsl_pt_map_getaddr(pagetable, pte)); #endif kgsl_pt_map_set(pagetable, pte, GSL_PT_PAGE_DIRTY); + superpte = pte - (pte & (GSL_PT_SUPER_PTE - 1)); + if (pte == superpte) + GSL_TLBFLUSH_FILTER_SETDIRTY(superpte / + GSL_PT_SUPER_PTE); } dmb(); - /* Invalidate tlb only if current page table used by GPU is the - * pagetable that we used to allocate */ - if (pagetable == pagetable->mmu->hwpagetable) - kgsl_yamato_setstate(pagetable->mmu->device, - KGSL_MMUFLAGS_TLBFLUSH); - gen_pool_free(pagetable->pool, gpuaddr, range); KGSL_MEM_VDBG("return %d\n", 0); @@ -651,6 +653,12 @@ int kgsl_mmu_close(struct kgsl_device *device) if (mmu->dummyspace.gpuaddr) kgsl_sharedmem_free(&mmu->dummyspace); + if (mmu->tlbflushfilter.base) { + mmu->tlbflushfilter.size = 0; + kfree(mmu->tlbflushfilter.base); + mmu->tlbflushfilter.base = NULL; + } + mmu->flags &= ~KGSL_FLAGS_STARTED; mmu->flags &= ~KGSL_FLAGS_INITIALIZED; mmu->flags &= ~KGSL_FLAGS_INITIALIZED0; diff --git a/drivers/video/msm/gpu/kgsl/kgsl_mmu.h b/drivers/video/msm/gpu/kgsl/kgsl_mmu.h index d70f24a1..627a6b82 100644 --- a/drivers/video/msm/gpu/kgsl/kgsl_mmu.h +++ b/drivers/video/msm/gpu/kgsl/kgsl_mmu.h @@ -31,6 +31,21 @@ #define KGSL_MMUFLAGS_TLBFLUSH 0x10000000 #define KGSL_MMUFLAGS_PTUPDATE 0x20000000 +/* Macros to manage TLB flushing */ +#define GSL_TLBFLUSH_FILTER_ENTRY_NUMBITS (sizeof(unsigned char) * 8) +#define GSL_TLBFLUSH_FILTER_GET(superpte) \ + (*((unsigned char *) \ + (((unsigned int)mmu->tlbflushfilter.base) \ + + (superpte / GSL_TLBFLUSH_FILTER_ENTRY_NUMBITS)))) +#define GSL_TLBFLUSH_FILTER_SETDIRTY(superpte) \ + (GSL_TLBFLUSH_FILTER_GET((superpte)) |= 1 << \ + (superpte % GSL_TLBFLUSH_FILTER_ENTRY_NUMBITS)) +#define GSL_TLBFLUSH_FILTER_ISDIRTY(superpte) \ + (GSL_TLBFLUSH_FILTER_GET((superpte)) & \ + (1 << (superpte % GSL_TLBFLUSH_FILTER_ENTRY_NUMBITS))) +#define GSL_TLBFLUSH_FILTER_RESET() memset(mmu->tlbflushfilter.base,\ + 0, mmu->tlbflushfilter.size) + extern unsigned int kgsl_cache_enable; struct kgsl_device; @@ -68,6 +83,11 @@ struct kgsl_pagetable { struct gen_pool *pool; }; +struct kgsl_tlbflushfilter { + unsigned int *base; + unsigned int size; +}; + struct kgsl_mmu { unsigned int refcnt; uint32_t flags; @@ -81,6 +101,8 @@ struct kgsl_mmu { /* current page table object being used by device mmu */ struct kgsl_pagetable *defaultpagetable; struct kgsl_pagetable *hwpagetable; + /* Maintain filter to manage tlb flushing */ + struct kgsl_tlbflushfilter tlbflushfilter; }; @@ -102,7 +124,7 @@ int kgsl_mmu_destroypagetableobject(struct kgsl_pagetable *pagetable); int kgsl_mmu_setstate(struct kgsl_device *device, struct kgsl_pagetable *pagetable); -#ifdef CONFIG_MSM_KGSL_MMU +#ifdef CONFIG_GPU_MSM_KGSL_MMU int kgsl_mmu_map(struct kgsl_pagetable *pagetable, unsigned int address, int range, diff --git a/drivers/video/msm/gpu/kgsl/kgsl_yamato.c b/drivers/video/msm/gpu/kgsl/kgsl_yamato.c index af5a6ff2..b0d6eeec 100644 --- a/drivers/video/msm/gpu/kgsl/kgsl_yamato.c +++ b/drivers/video/msm/gpu/kgsl/kgsl_yamato.c @@ -325,7 +325,7 @@ error: } -#ifdef CONFIG_MSM_KGSL_MMU +#ifdef CONFIG_GPU_MSM_KGSL_MMU int kgsl_yamato_setstate(struct kgsl_device *device, uint32_t flags) { unsigned int link[32]; @@ -731,7 +731,7 @@ int kgsl_yamato_getproperty(struct kgsl_device *device, break; case KGSL_PROP_MMU_ENABLE: { -#ifdef CONFIG_MSM_KGSL_MMU +#ifdef CONFIG_GPU_MSM_KGSL_MMU int mmuProp = 1; #else int mmuProp = 0; diff --git a/include/linux/android_pmem.h b/include/linux/android_pmem.h index f633621f..6e37c0f8 100644 --- a/include/linux/android_pmem.h +++ b/include/linux/android_pmem.h @@ -1,3 +1,175 @@ +/* include/linux/android_pmem.h + * + * Copyright (C) 2007 Google, Inc. + * Copyright (c) 2009-2010, Code Aurora Forum. All rights reserved. + * + * This software is licensed under the terms of the GNU General Public + * License version 2, as published by the Free Software Foundation, and + * may be copied, distributed, and modified under those terms. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + */ +#ifdef CONFIG_MSM_KGSL +#ifndef _ANDROID_PMEM_H_ +#define _ANDROID_PMEM_H_ + +#include + +#define PMEM_KERNEL_TEST_MAGIC 0xc0 +#define PMEM_KERNEL_TEST_NOMINAL_TEST_IOCTL \ + _IO(PMEM_KERNEL_TEST_MAGIC, 1) +#define PMEM_KERNEL_TEST_ADVERSARIAL_TEST_IOCTL \ + _IO(PMEM_KERNEL_TEST_MAGIC, 2) +#define PMEM_KERNEL_TEST_HUGE_ALLOCATION_TEST_IOCTL \ + _IO(PMEM_KERNEL_TEST_MAGIC, 3) +#define PMEM_KERNEL_TEST_FREE_UNALLOCATED_TEST_IOCTL \ + _IO(PMEM_KERNEL_TEST_MAGIC, 4) +#define PMEM_KERNEL_TEST_LARGE_REGION_NUMBER_TEST_IOCTL \ + _IO(PMEM_KERNEL_TEST_MAGIC, 5) + +#define PMEM_IOCTL_MAGIC 'p' +#define PMEM_GET_PHYS _IOW(PMEM_IOCTL_MAGIC, 1, unsigned int) +#define PMEM_MAP _IOW(PMEM_IOCTL_MAGIC, 2, unsigned int) +#define PMEM_GET_SIZE _IOW(PMEM_IOCTL_MAGIC, 3, unsigned int) +#define PMEM_UNMAP _IOW(PMEM_IOCTL_MAGIC, 4, unsigned int) +/* This ioctl will allocate pmem space, backing the file, it will fail + * if the file already has an allocation, pass it the len as the argument + * to the ioctl */ +#define PMEM_ALLOCATE _IOW(PMEM_IOCTL_MAGIC, 5, unsigned int) +/* This will connect a one pmem file to another, pass the file that is already + * backed in memory as the argument to the ioctl + */ +#define PMEM_CONNECT _IOW(PMEM_IOCTL_MAGIC, 6, unsigned int) +/* Returns the total size of the pmem region it is sent to as a pmem_region + * struct (with offset set to 0). + */ +#define PMEM_GET_TOTAL_SIZE _IOW(PMEM_IOCTL_MAGIC, 7, unsigned int) +/* Revokes gpu registers and resets the gpu. Pass a pointer to the + * start of the mapped gpu regs (the vaddr returned by mmap) as the argument. + */ +#define HW3D_REVOKE_GPU _IOW(PMEM_IOCTL_MAGIC, 8, unsigned int) +#define PMEM_CACHE_FLUSH _IOW(PMEM_IOCTL_MAGIC, 8, unsigned int) +#define HW3D_GRANT_GPU _IOW(PMEM_IOCTL_MAGIC, 9, unsigned int) + +#define PMEM_CLEAN_INV_CACHES _IOW(PMEM_IOCTL_MAGIC, 11, unsigned int) +#define PMEM_CLEAN_CACHES _IOW(PMEM_IOCTL_MAGIC, 12, unsigned int) +#define PMEM_INV_CACHES _IOW(PMEM_IOCTL_MAGIC, 13, unsigned int) + +#define PMEM_GET_FREE_SPACE _IOW(PMEM_IOCTL_MAGIC, 14, unsigned int) +#define PMEM_ALLOCATE_ALIGNED _IOW(PMEM_IOCTL_MAGIC, 15, unsigned int) +struct pmem_region { + unsigned long offset; + unsigned long len; +}; + +struct pmem_addr { + unsigned long vaddr; + unsigned long offset; + unsigned long length; +}; + +struct pmem_freespace { + unsigned long total; + unsigned long largest; +}; + +struct pmem_allocation { + unsigned long size; + unsigned int align; +}; + +#ifdef __KERNEL__ +int get_pmem_file(unsigned int fd, unsigned long *start, unsigned long *vstart, + unsigned long *end, struct file **filp); +int get_pmem_fd(int fd, unsigned long *start, unsigned long *end); +int get_pmem_user_addr(struct file *file, unsigned long *start, + unsigned long *end); +void put_pmem_file(struct file* file); +void put_pmem_fd(int fd); +void flush_pmem_fd(int fd, unsigned long start, unsigned long len); +void flush_pmem_file(struct file *file, unsigned long start, unsigned long len); +int pmem_cache_maint(struct file *file, unsigned int cmd, + struct pmem_addr *pmem_addr); + +enum pmem_allocator_type { + /* Zero is a default in platform PMEM structures in the board files, + * when the "allocator_type" structure element is not explicitly + * defined + */ + PMEM_ALLOCATORTYPE_BITMAP = 0, /* forced to be zero here */ + + PMEM_ALLOCATORTYPE_ALLORNOTHING, + PMEM_ALLOCATORTYPE_BUDDYBESTFIT, + + PMEM_ALLOCATORTYPE_MAX, +}; + +#define PMEM_MEMTYPE_MASK 0x7 +#define PMEM_INVALID_MEMTYPE 0x0 +#define PMEM_MEMTYPE_EBI1 0x1 +#define PMEM_MEMTYPE_SMI 0x2 +#define PMEM_MEMTYPE_RESERVED_INVALID2 0x3 +#define PMEM_MEMTYPE_RESERVED_INVALID3 0x4 +#define PMEM_MEMTYPE_RESERVED_INVALID4 0x5 +#define PMEM_MEMTYPE_RESERVED_INVALID5 0x6 +#define PMEM_MEMTYPE_RESERVED_INVALID6 0x7 + +#define PMEM_ALIGNMENT_MASK 0x18 +#define PMEM_ALIGNMENT_RESERVED_INVALID1 0x0 +#define PMEM_ALIGNMENT_4K 0x8 /* the default */ +#define PMEM_ALIGNMENT_1M 0x10 +#define PMEM_ALIGNMENT_RESERVED_INVALID2 0x18 + +/* flags in the following function defined as above. */ +int32_t pmem_kalloc(const size_t size, const uint32_t flags); +int32_t pmem_kfree(const int32_t physaddr); + +/* kernel api names for board specific data structures */ +#define PMEM_KERNEL_EBI1_DATA_NAME "pmem_kernel_ebi1" +#define PMEM_KERNEL_SMI_DATA_NAME "pmem_kernel_smi" + +struct android_pmem_platform_data +{ + const char* name; + /* starting physical address of memory region */ + unsigned long start; + /* size of memory region */ + unsigned long size; + + enum pmem_allocator_type allocator_type; + /* treated as a 'hidden' variable in the board files. Can be + * set, but default is the system init value of 0 which becomes a + * quantum of 4K pages. + */ + unsigned int quantum; + + /* set to indicate maps of this region should be cached, if a mix of + * cached and uncached is desired, set this and open the device with + * O_SYNC to get an uncached region */ + unsigned cached; + /* The MSM7k has bits to enable a write buffer in the bus controller*/ + unsigned buffered; + /* This PMEM is on memory that may be powered off */ + unsigned unstable; +}; + +int pmem_setup(struct android_pmem_platform_data *pdata, + long (*ioctl)(struct file *, unsigned int, unsigned long), + int (*release)(struct inode *, struct file *)); + +int pmem_remap(struct pmem_region *region, struct file *file, + unsigned operation); +int is_pmem_file(struct file *file); + +#endif /* __KERNEL__ */ + +#endif //_ANDROID_PPP_H_ +#else + /* include/linux/android_pmem.h * * Copyright (C) 2007 Google, Inc. @@ -91,3 +263,4 @@ static inline int pmem_remap(struct pmem_region *region, struct file *file, #endif //_ANDROID_PPP_H_ +#endif \ No newline at end of file diff --git a/include/linux/ashmem.h b/include/linux/ashmem.h index 1976b10e..4d9b8315 100644 --- a/include/linux/ashmem.h +++ b/include/linux/ashmem.h @@ -44,5 +44,12 @@ struct ashmem_pin { #define ASHMEM_UNPIN _IOW(__ASHMEMIOC, 8, struct ashmem_pin) #define ASHMEM_GET_PIN_STATUS _IO(__ASHMEMIOC, 9) #define ASHMEM_PURGE_ALL_CACHES _IO(__ASHMEMIOC, 10) +#define ASHMEM_CACHE_FLUSH_RANGE _IO(__ASHMEMIOC, 11) +#define ASHMEM_CACHE_CLEAN_RANGE _IO(__ASHMEMIOC, 12) +#define ASHMEM_CACHE_INV_RANGE _IO(__ASHMEMIOC, 13) + +int get_ashmem_file(int fd, struct file **filp, struct file **vm_file, + unsigned long *len); +void put_ashmem_file(struct file *file); #endif /* _LINUX_ASHMEM_H */ diff --git a/include/linux/atomic.h b/include/linux/atomic.h new file mode 100755 index 00000000..96c038e4 --- /dev/null +++ b/include/linux/atomic.h @@ -0,0 +1,37 @@ +#ifndef _LINUX_ATOMIC_H +#define _LINUX_ATOMIC_H +#include + +/** + * atomic_inc_not_zero_hint - increment if not null + * @v: pointer of type atomic_t + * @hint: probable value of the atomic before the increment + * + * This version of atomic_inc_not_zero() gives a hint of probable + * value of the atomic. This helps processor to not read the memory + * before doing the atomic read/modify/write cycle, lowering + * number of bus transactions on some arches. + * + * Returns: 0 if increment was not done, 1 otherwise. + */ +#ifndef atomic_inc_not_zero_hint +static inline int atomic_inc_not_zero_hint(atomic_t *v, int hint) +{ + int val, c = hint; + + /* sanity test, should be removed by compiler if hint is a constant */ + if (!hint) + return atomic_inc_not_zero(v); + + do { + val = atomic_cmpxchg(v, c, c + 1); + if (val == c) + return 1; + c = val; + } while (c); + + return 0; +} +#endif + +#endif /* _LINUX_ATOMIC_H */ diff --git a/include/linux/bitmap.h b/include/linux/bitmap.h index 756d78b8..9a27ded6 100644 --- a/include/linux/bitmap.h +++ b/include/linux/bitmap.h @@ -42,6 +42,10 @@ * bitmap_empty(src, nbits) Are all bits zero in *src? * bitmap_full(src, nbits) Are all bits set in *src? * bitmap_weight(src, nbits) Hamming Weight: number set bits + * bitmap_set(dst, pos, nbits) Set specified bit area + * bitmap_clear(dst, pos, nbits) Clear specified bit area + * bitmap_find_next_zero_area(buf, len, pos, n, mask) Find bit free area + * bitmap_find_next_zero_area_off(buf, len, pos, n, mask) as above * bitmap_shift_right(dst, src, n, nbits) *dst = *src >> n * bitmap_shift_left(dst, src, n, nbits) *dst = *src << n * bitmap_remap(dst, src, old, new, nbits) *dst = map(old, new)(src) @@ -108,6 +112,27 @@ extern int __bitmap_subset(const unsigned long *bitmap1, const unsigned long *bitmap2, int bits); extern int __bitmap_weight(const unsigned long *bitmap, int bits); +extern void bitmap_set(unsigned long *map, int i, int len); +extern void bitmap_clear(unsigned long *map, int start, int nr); + +extern unsigned long bitmap_find_next_zero_area_off(unsigned long *map, + unsigned long size, + unsigned long start, + unsigned int nr, + unsigned long align_mask, + unsigned long align_offset); + +static inline unsigned long +bitmap_find_next_zero_area(unsigned long *map, + unsigned long size, + unsigned long start, + unsigned int nr, + unsigned long align_mask) +{ + return bitmap_find_next_zero_area_off(map, size, start, nr, + align_mask, 0); +} + extern int bitmap_scnprintf(char *buf, unsigned int len, const unsigned long *src, int nbits); extern int __bitmap_parse(const char *buf, unsigned int buflen, int is_user, @@ -118,6 +143,8 @@ extern int bitmap_scnlistprintf(char *buf, unsigned int len, const unsigned long *src, int nbits); extern int bitmap_parselist(const char *buf, unsigned long *maskp, int nmaskbits); +extern int bitmap_parselist_user(const char __user *ubuf, unsigned int ulen, + unsigned long *dst, int nbits); extern void bitmap_remap(unsigned long *dst, const unsigned long *src, const unsigned long *old, const unsigned long *new, int bits); extern int bitmap_bitremap(int oldbit, diff --git a/include/linux/memcopy.h b/include/linux/memcopy.h new file mode 100644 index 00000000..18bc8609 --- /dev/null +++ b/include/linux/memcopy.h @@ -0,0 +1,226 @@ +/* + * memcopy.h -- definitions for memory copy functions. Generic C version. + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the Free + * Software Foundation; either version 2 of the License, or (at your option) + * any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General + * Public License for more details. + * + * You should have received a copy of the GNU General Public License along + * with this program; if not, write to the Free Software Foundation, Inc., + * 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * The code is derived from the GNU C Library. + * Copyright (C) 1991, 1992, 1993, 1997, 2004 Free Software Foundation, Inc. + */ +#ifndef _LINUX_MEMCOPY_H_ +#define _LINUX_MEMCOPY_H_ + +/* + * The strategy of the memory functions is: + * + * 1. Copy bytes until the destination pointer is aligned. + * + * 2. Copy words in unrolled loops. If the source and destination + * are not aligned in the same way, use word memory operations, + * but shift and merge two read words before writing. + * + * 3. Copy the few remaining bytes. + * + * This is fast on processors that have at least 10 registers for + * allocation by GCC, and that can access memory at reg+const in one + * instruction. + */ + +#include +#include +#include + +/* + * The macros defined in this file are: + * + * BYTE_COPY_FWD(dst_beg_ptr, src_beg_ptr, nbytes_to_copy) + * + * BYTE_COPY_BWD(dst_end_ptr, src_end_ptr, nbytes_to_copy) + * + * WORD_COPY_FWD(dst_beg_ptr, src_beg_ptr, nbytes_remaining, nbytes_to_copy) + * + * WORD_COPY_BWD(dst_end_ptr, src_end_ptr, nbytes_remaining, nbytes_to_copy) + * + * MERGE(old_word, sh_1, new_word, sh_2) + * + * MEM_COPY_FWD(dst_beg_ptr, src_beg_ptr, nbytes_to_copy) + * + * MEM_COPY_BWD(dst_end_ptr, src_end_ptr, nbytes_to_copy) + */ + +#define OP_T_THRESHOLD 16 + +/* + * Type to use for aligned memory operations. + * This should normally be the biggest type supported by a single load + * and store. + */ +#define op_t unsigned long int +#define OPSIZ (sizeof(op_t)) + +/* Type to use for unaligned operations. */ +typedef unsigned char byte; + +#ifndef MERGE +# ifdef __LITTLE_ENDIAN +# define MERGE(w0, sh_1, w1, sh_2) (((w0) >> (sh_1)) | ((w1) << (sh_2))) +# elif defined(__BIG_ENDIAN) +# define MERGE(w0, sh_1, w1, sh_2) (((w0) << (sh_1)) | ((w1) >> (sh_2))) +# else +# error "Macro MERGE() hasn't defined!" +# endif +#endif + +/* + * Copy exactly NBYTES bytes from SRC_BP to DST_BP, + * without any assumptions about alignment of the pointers. + */ +#ifndef BYTE_COPY_FWD +#define BYTE_COPY_FWD(dst_bp, src_bp, nbytes) \ +do { \ + size_t __nbytes = (nbytes); \ + while (__nbytes > 0) { \ + byte __x = ((byte *) src_bp)[0]; \ + src_bp += 1; \ + __nbytes -= 1; \ + ((byte *) dst_bp)[0] = __x; \ + dst_bp += 1; \ + } \ +} while (0) +#endif + +/* + * Copy exactly NBYTES_TO_COPY bytes from SRC_END_PTR to DST_END_PTR, + * beginning at the bytes right before the pointers and continuing towards + * smaller addresses. Don't assume anything about alignment of the + * pointers. + */ +#ifndef BYTE_COPY_BWD +#define BYTE_COPY_BWD(dst_ep, src_ep, nbytes) \ +do { \ + size_t __nbytes = (nbytes); \ + while (__nbytes > 0) { \ + byte __x; \ + src_ep -= 1; \ + __x = ((byte *) src_ep)[0]; \ + dst_ep -= 1; \ + __nbytes -= 1; \ + ((byte *) dst_ep)[0] = __x; \ + } \ +} while (0) +#endif +/* + * Copy *up to* NBYTES bytes from SRC_BP to DST_BP, with + * the assumption that DST_BP is aligned on an OPSIZ multiple. If + * not all bytes could be easily copied, store remaining number of bytes + * in NBYTES_LEFT, otherwise store 0. + */ +extern void _wordcopy_fwd_aligned(long int, long int, size_t); +extern void _wordcopy_fwd_dest_aligned(long int, long int, size_t); +#ifndef WORD_COPY_FWD +#define WORD_COPY_FWD(dst_bp, src_bp, nbytes_left, nbytes) \ +do { \ + if (src_bp % OPSIZ == 0) \ + _wordcopy_fwd_aligned (dst_bp, src_bp, (nbytes) / OPSIZ); \ + else \ + _wordcopy_fwd_dest_aligned (dst_bp, src_bp, (nbytes) / OPSIZ);\ + \ + src_bp += (nbytes) & -OPSIZ; \ + dst_bp += (nbytes) & -OPSIZ; \ + (nbytes_left) = (nbytes) % OPSIZ; \ +} while (0) +#endif + +/* + * Copy *up to* NBYTES_TO_COPY bytes from SRC_END_PTR to DST_END_PTR, + * beginning at the words (of type op_t) right before the pointers and + * continuing towards smaller addresses. May take advantage of that + * DST_END_PTR is aligned on an OPSIZ multiple. If not all bytes could be + * easily copied, store remaining number of bytes in NBYTES_REMAINING, + * otherwise store 0. + */ +extern void _wordcopy_bwd_aligned(long int, long int, size_t); +extern void _wordcopy_bwd_dest_aligned(long int, long int, size_t); +#ifndef WORD_COPY_BWD +#define WORD_COPY_BWD(dst_ep, src_ep, nbytes_left, nbytes) \ +do { \ + if (src_ep % OPSIZ == 0) \ + _wordcopy_bwd_aligned (dst_ep, src_ep, (nbytes) / OPSIZ); \ + else \ + _wordcopy_bwd_dest_aligned (dst_ep, src_ep, (nbytes) / OPSIZ);\ + \ + src_ep -= (nbytes) & -OPSIZ; \ + dst_ep -= (nbytes) & -OPSIZ; \ + (nbytes_left) = (nbytes) % OPSIZ; \ +} while (0) +#endif + +/* Copy memory from the beginning to the end */ +#ifndef MEM_COPY_FWD +static __always_inline void mem_copy_fwd(unsigned long dstp, + unsigned long srcp, + size_t count) +{ + /* If there not too few bytes to copy, use word copy. */ + if (count >= OP_T_THRESHOLD) { + /* Copy just a few bytes to make dstp aligned. */ + count -= (-dstp) % OPSIZ; + BYTE_COPY_FWD(dstp, srcp, (-dstp) % OPSIZ); + + /* + * Copy from srcp to dstp taking advantage of the known + * alignment of dstp. Number if bytes remaining is put in + * the third argument. + */ + WORD_COPY_FWD(dstp, srcp, count, count); + + /* Fall out and copy the tail. */ + } + + /* There are just a few bytes to copy. Use byte memory operations. */ + BYTE_COPY_FWD(dstp, srcp, count); +} +#endif + +/* Copy memory from the end to the beginning. */ +#ifndef MEM_COPY_BWD +static __always_inline void mem_copy_bwd(unsigned long dstp, + unsigned long srcp, + size_t count) +{ + srcp += count; + dstp += count; + + /* If there not too few bytes to copy, use word copy. */ + if (count >= OP_T_THRESHOLD) { + /* Copy just a few bytes to make dstp aligned. */ + count -= dstp % OPSIZ; + BYTE_COPY_BWD(dstp, srcp, dstp % OPSIZ); + + /* + * Copy from srcp to dstp taking advantage of the known + * alignment of dstp. Number if bytes remaining is put in + * the third argument. + */ + WORD_COPY_BWD(dstp, srcp, count, count); + + /* Fall out and copy the tail. */ + } + + /* There are just a few bytes to copy. Use byte memory operations. */ + BYTE_COPY_BWD (dstp, srcp, count); +} +#endif + +#endif diff --git a/include/linux/memory_alloc.h b/include/linux/memory_alloc.h new file mode 100644 index 00000000..ec005a1c --- /dev/null +++ b/include/linux/memory_alloc.h @@ -0,0 +1,59 @@ +/* Copyright (c) 2011, Code Aurora Forum. All rights reserved. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 and + * only version 2 as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + */ + +#ifndef _LINUX_MEMALLOC_H +#define _LINUX_MEMALLOC_H + +#include +#include +#include + +struct mem_pool { + struct mutex pool_mutex; + struct gen_pool *gpool; + unsigned long paddr; + unsigned long size; + unsigned long free; + unsigned int id; +}; + +struct alloc { + struct rb_node rb_node; + void *vaddr; + unsigned long paddr; + struct mem_pool *mpool; + unsigned long len; + void *caller; +}; + +struct mem_pool *initialize_memory_pool(unsigned long start, + unsigned long size, int mem_type); + +void *allocate_contiguous_memory(unsigned long size, + int mem_type, unsigned long align, int cached); + +unsigned long _allocate_contiguous_memory_nomap(unsigned long size, + int mem_type, unsigned long align, void *caller); + +unsigned long allocate_contiguous_memory_nomap(unsigned long size, + int mem_type, unsigned long align); + +void free_contiguous_memory(void *addr); +void free_contiguous_memory_by_paddr(unsigned long paddr); + +unsigned long memory_pool_node_paddr(void *vaddr); + +unsigned long memory_pool_node_len(void *vaddr); + +int memory_pool_init(void); +#endif /* _LINUX_MEMALLOC_H */ + diff --git a/include/linux/msm_kgsl.h b/include/linux/msm_kgsl.h index 84bcebd7..86f6b5c5 100644 --- a/include/linux/msm_kgsl.h +++ b/include/linux/msm_kgsl.h @@ -1,3 +1,473 @@ +#ifdef CONFIG_MSM_KGSL +/* Copyright (c) 2002,2007-2011, Code Aurora Forum. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, and the entire permission notice in its entirety, + * including the disclaimer of warranties. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * 3. The name of the author may not be used to endorse or promote + * products derived from this software without specific prior + * written permission. + * + * ALTERNATIVELY, this product may be distributed under the terms of + * the GNU General Public License, version 2, in which case the provisions + * of the GPL version 2 are required INSTEAD OF the BSD license. + * + * THIS SOFTWARE IS PROVIDED ``AS IS'' AND ANY EXPRESS OR IMPLIED + * WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE, ALL OF + * WHICH ARE HEREBY DISCLAIMED. IN NO EVENT SHALL THE AUTHOR BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT + * OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR + * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF + * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE + * USE OF THIS SOFTWARE, EVEN IF NOT ADVISED OF THE POSSIBILITY OF SUCH + * DAMAGE. + * + */ +#ifndef _MSM_KGSL_H +#define _MSM_KGSL_H + +#define KGSL_VERSION_MAJOR 3 +#define KGSL_VERSION_MINOR 7 + +/*context flags */ +#define KGSL_CONTEXT_SAVE_GMEM 1 +#define KGSL_CONTEXT_NO_GMEM_ALLOC 2 +#define KGSL_CONTEXT_SUBMIT_IB_LIST 4 +#define KGSL_CONTEXT_CTX_SWITCH 8 + +/* Memory allocayion flags */ +#define KGSL_MEMFLAGS_GPUREADONLY 0x01000000 + +/* generic flag values */ +#define KGSL_FLAGS_NORMALMODE 0x00000000 +#define KGSL_FLAGS_SAFEMODE 0x00000001 +#define KGSL_FLAGS_INITIALIZED0 0x00000002 +#define KGSL_FLAGS_INITIALIZED 0x00000004 +#define KGSL_FLAGS_STARTED 0x00000008 +#define KGSL_FLAGS_ACTIVE 0x00000010 +#define KGSL_FLAGS_RESERVED0 0x00000020 +#define KGSL_FLAGS_RESERVED1 0x00000040 +#define KGSL_FLAGS_RESERVED2 0x00000080 +#define KGSL_FLAGS_SOFT_RESET 0x00000100 + +#define KGSL_MAX_PWRLEVELS 5 + +/* device id */ +enum kgsl_deviceid { + KGSL_DEVICE_3D0 = 0x00000000, + KGSL_DEVICE_2D0 = 0x00000001, + KGSL_DEVICE_2D1 = 0x00000002, + KGSL_DEVICE_MAX = 0x00000003 +}; + +enum kgsl_user_mem_type { + KGSL_USER_MEM_TYPE_PMEM = 0x00000000, + KGSL_USER_MEM_TYPE_ASHMEM = 0x00000001, + KGSL_USER_MEM_TYPE_ADDR = 0x00000002 +}; + +struct kgsl_devinfo { + + unsigned int device_id; + /* chip revision id + * coreid:8 majorrev:8 minorrev:8 patch:8 + */ + unsigned int chip_id; + unsigned int mmu_enabled; + unsigned int gmem_gpubaseaddr; + /* + * This field contains the adreno revision + * number 200, 205, 220, etc... + */ + unsigned int gpu_id; + unsigned int gmem_sizebytes; +}; + +/* this structure defines the region of memory that can be mmap()ed from this + driver. The timestamp fields are volatile because they are written by the + GPU +*/ +struct kgsl_devmemstore { + volatile unsigned int soptimestamp; + unsigned int sbz; + volatile unsigned int eoptimestamp; + unsigned int sbz2; + volatile unsigned int ts_cmp_enable; + unsigned int sbz3; + volatile unsigned int ref_wait_ts; + unsigned int sbz4; + unsigned int current_context; + unsigned int sbz5; +}; + +#define KGSL_DEVICE_MEMSTORE_OFFSET(field) \ + offsetof(struct kgsl_devmemstore, field) + + +/* timestamp id*/ +enum kgsl_timestamp_type { + KGSL_TIMESTAMP_CONSUMED = 0x00000001, /* start-of-pipeline timestamp */ + KGSL_TIMESTAMP_RETIRED = 0x00000002, /* end-of-pipeline timestamp*/ + KGSL_TIMESTAMP_MAX = 0x00000002, +}; + +/* property types - used with kgsl_device_getproperty */ +enum kgsl_property_type { + KGSL_PROP_DEVICE_INFO = 0x00000001, + KGSL_PROP_DEVICE_SHADOW = 0x00000002, + KGSL_PROP_DEVICE_POWER = 0x00000003, + KGSL_PROP_SHMEM = 0x00000004, + KGSL_PROP_SHMEM_APERTURES = 0x00000005, + KGSL_PROP_MMU_ENABLE = 0x00000006, + KGSL_PROP_INTERRUPT_WAITS = 0x00000007, + KGSL_PROP_VERSION = 0x00000008, +}; + +struct kgsl_shadowprop { + unsigned int gpuaddr; + unsigned int size; + unsigned int flags; /* contains KGSL_FLAGS_ values */ +}; + +struct kgsl_pwrlevel { + unsigned int gpu_freq; + unsigned int bus_freq; +}; + +struct kgsl_version { + unsigned int drv_major; + unsigned int drv_minor; + unsigned int dev_major; + unsigned int dev_minor; +}; + +#ifdef __KERNEL__ + +#define KGSL_3D0_REG_MEMORY "kgsl_3d0_reg_memory" +#define KGSL_3D0_IRQ "kgsl_3d0_irq" +#define KGSL_2D0_REG_MEMORY "kgsl_2d0_reg_memory" +#define KGSL_2D0_IRQ "kgsl_2d0_irq" +#define KGSL_2D1_REG_MEMORY "kgsl_2d1_reg_memory" +#define KGSL_2D1_IRQ "kgsl_2d1_irq" + +struct kgsl_grp_clk_name { + const char *clk; + const char *pclk; +}; + +struct kgsl_device_pwr_data { + struct kgsl_pwrlevel pwrlevel[KGSL_MAX_PWRLEVELS]; + int init_level; + int num_levels; + int (*set_grp_async)(void); + unsigned int idle_timeout; + unsigned int nap_allowed; + unsigned int idle_pass; +}; + +struct kgsl_clk_data { + struct kgsl_grp_clk_name name; + struct msm_bus_scale_pdata *bus_scale_table; +}; + +struct kgsl_device_platform_data { + struct kgsl_device_pwr_data pwr_data; + struct kgsl_clk_data clk; + /* imem_clk_name is for 3d only, not used in 2d devices */ + struct kgsl_grp_clk_name imem_clk_name; +}; + +#endif + +/* structure holds list of ibs */ +struct kgsl_ibdesc { + unsigned int gpuaddr; + void *hostptr; + unsigned int sizedwords; + unsigned int ctrl; +}; + +/* ioctls */ +#define KGSL_IOC_TYPE 0x09 + +/* get misc info about the GPU + type should be a value from enum kgsl_property_type + value points to a structure that varies based on type + sizebytes is sizeof() that structure + for KGSL_PROP_DEVICE_INFO, use struct kgsl_devinfo + this structure contaings hardware versioning info. + for KGSL_PROP_DEVICE_SHADOW, use struct kgsl_shadowprop + this is used to find mmap() offset and sizes for mapping + struct kgsl_memstore into userspace. +*/ +struct kgsl_device_getproperty { + unsigned int type; + void *value; + unsigned int sizebytes; +}; + +#define IOCTL_KGSL_DEVICE_GETPROPERTY \ + _IOWR(KGSL_IOC_TYPE, 0x2, struct kgsl_device_getproperty) + + +/* read a GPU register. + offsetwords it the 32 bit word offset from the beginning of the + GPU register space. + */ +struct kgsl_device_regread { + unsigned int offsetwords; + unsigned int value; /* output param */ +}; + +#define IOCTL_KGSL_DEVICE_REGREAD \ + _IOWR(KGSL_IOC_TYPE, 0x3, struct kgsl_device_regread) + + +/* block until the GPU has executed past a given timestamp + * timeout is in milliseconds. + */ +struct kgsl_device_waittimestamp { + unsigned int timestamp; + unsigned int timeout; +}; + +#define IOCTL_KGSL_DEVICE_WAITTIMESTAMP \ + _IOW(KGSL_IOC_TYPE, 0x6, struct kgsl_device_waittimestamp) + + +/* issue indirect commands to the GPU. + * drawctxt_id must have been created with IOCTL_KGSL_DRAWCTXT_CREATE + * ibaddr and sizedwords must specify a subset of a buffer created + * with IOCTL_KGSL_SHAREDMEM_FROM_PMEM + * flags may be a mask of KGSL_CONTEXT_ values + * timestamp is a returned counter value which can be passed to + * other ioctls to determine when the commands have been executed by + * the GPU. + */ +struct kgsl_ringbuffer_issueibcmds { + unsigned int drawctxt_id; + unsigned int ibdesc_addr; + unsigned int numibs; + unsigned int timestamp; /*output param */ + unsigned int flags; +}; + +#define IOCTL_KGSL_RINGBUFFER_ISSUEIBCMDS \ + _IOWR(KGSL_IOC_TYPE, 0x10, struct kgsl_ringbuffer_issueibcmds) + +/* read the most recently executed timestamp value + * type should be a value from enum kgsl_timestamp_type + */ +struct kgsl_cmdstream_readtimestamp { + unsigned int type; + unsigned int timestamp; /*output param */ +}; + +#define IOCTL_KGSL_CMDSTREAM_READTIMESTAMP_OLD \ + _IOR(KGSL_IOC_TYPE, 0x11, struct kgsl_cmdstream_readtimestamp) + +#define IOCTL_KGSL_CMDSTREAM_READTIMESTAMP \ + _IOWR(KGSL_IOC_TYPE, 0x11, struct kgsl_cmdstream_readtimestamp) + +/* free memory when the GPU reaches a given timestamp. + * gpuaddr specify a memory region created by a + * IOCTL_KGSL_SHAREDMEM_FROM_PMEM call + * type should be a value from enum kgsl_timestamp_type + */ +struct kgsl_cmdstream_freememontimestamp { + unsigned int gpuaddr; + unsigned int type; + unsigned int timestamp; +}; + +#define IOCTL_KGSL_CMDSTREAM_FREEMEMONTIMESTAMP \ + _IOW(KGSL_IOC_TYPE, 0x12, struct kgsl_cmdstream_freememontimestamp) + +/* Previous versions of this header had incorrectly defined + IOCTL_KGSL_CMDSTREAM_FREEMEMONTIMESTAMP as a read-only ioctl instead + of a write only ioctl. To ensure binary compatability, the following + #define will be used to intercept the incorrect ioctl +*/ + +#define IOCTL_KGSL_CMDSTREAM_FREEMEMONTIMESTAMP_OLD \ + _IOR(KGSL_IOC_TYPE, 0x12, struct kgsl_cmdstream_freememontimestamp) + +/* create a draw context, which is used to preserve GPU state. + * The flags field may contain a mask KGSL_CONTEXT_* values + */ +struct kgsl_drawctxt_create { + unsigned int flags; + unsigned int drawctxt_id; /*output param */ +}; + +#define IOCTL_KGSL_DRAWCTXT_CREATE \ + _IOWR(KGSL_IOC_TYPE, 0x13, struct kgsl_drawctxt_create) + +/* destroy a draw context */ +struct kgsl_drawctxt_destroy { + unsigned int drawctxt_id; +}; + +#define IOCTL_KGSL_DRAWCTXT_DESTROY \ + _IOW(KGSL_IOC_TYPE, 0x14, struct kgsl_drawctxt_destroy) + +/* add a block of pmem, fb, ashmem or user allocated address + * into the GPU address space */ +struct kgsl_map_user_mem { + int fd; + unsigned int gpuaddr; /*output param */ + unsigned int len; + unsigned int offset; + unsigned int hostptr; /*input param */ + enum kgsl_user_mem_type memtype; + unsigned int reserved; /* May be required to add + params for another mem type */ +}; + +#define IOCTL_KGSL_MAP_USER_MEM \ + _IOWR(KGSL_IOC_TYPE, 0x15, struct kgsl_map_user_mem) + +/* add a block of pmem or fb into the GPU address space */ +struct kgsl_sharedmem_from_pmem { + int pmem_fd; + unsigned int gpuaddr; /*output param */ + unsigned int len; + unsigned int offset; +}; + +#define IOCTL_KGSL_SHAREDMEM_FROM_PMEM \ + _IOWR(KGSL_IOC_TYPE, 0x20, struct kgsl_sharedmem_from_pmem) + +/* remove memory from the GPU's address space */ +struct kgsl_sharedmem_free { + unsigned int gpuaddr; +}; + +#define IOCTL_KGSL_SHAREDMEM_FREE \ + _IOW(KGSL_IOC_TYPE, 0x21, struct kgsl_sharedmem_free) + +struct kgsl_cff_user_event { + unsigned char cff_opcode; + unsigned int op1; + unsigned int op2; + unsigned int op3; + unsigned int op4; + unsigned int op5; + unsigned int __pad[2]; +}; + +#define IOCTL_KGSL_CFF_USER_EVENT \ + _IOW(KGSL_IOC_TYPE, 0x31, struct kgsl_cff_user_event) + +struct kgsl_gmem_desc { + unsigned int x; + unsigned int y; + unsigned int width; + unsigned int height; + unsigned int pitch; +}; + +struct kgsl_buffer_desc { + void *hostptr; + unsigned int gpuaddr; + int size; + unsigned int format; + unsigned int pitch; + unsigned int enabled; +}; + +struct kgsl_bind_gmem_shadow { + unsigned int drawctxt_id; + struct kgsl_gmem_desc gmem_desc; + unsigned int shadow_x; + unsigned int shadow_y; + struct kgsl_buffer_desc shadow_buffer; + unsigned int buffer_id; +}; + +#define IOCTL_KGSL_DRAWCTXT_BIND_GMEM_SHADOW \ + _IOW(KGSL_IOC_TYPE, 0x22, struct kgsl_bind_gmem_shadow) + +/* add a block of memory into the GPU address space */ +struct kgsl_sharedmem_from_vmalloc { + unsigned int gpuaddr; /*output param */ + unsigned int hostptr; + unsigned int flags; +}; + +#define IOCTL_KGSL_SHAREDMEM_FROM_VMALLOC \ + _IOWR(KGSL_IOC_TYPE, 0x23, struct kgsl_sharedmem_from_vmalloc) + +#define IOCTL_KGSL_SHAREDMEM_FLUSH_CACHE \ + _IOW(KGSL_IOC_TYPE, 0x24, struct kgsl_sharedmem_free) + +struct kgsl_drawctxt_set_bin_base_offset { + unsigned int drawctxt_id; + unsigned int offset; +}; + +#define IOCTL_KGSL_DRAWCTXT_SET_BIN_BASE_OFFSET \ + _IOW(KGSL_IOC_TYPE, 0x25, struct kgsl_drawctxt_set_bin_base_offset) + +enum kgsl_cmdwindow_type { + KGSL_CMDWINDOW_MIN = 0x00000000, + KGSL_CMDWINDOW_2D = 0x00000000, + KGSL_CMDWINDOW_3D = 0x00000001, /* legacy */ + KGSL_CMDWINDOW_MMU = 0x00000002, + KGSL_CMDWINDOW_ARBITER = 0x000000FF, + KGSL_CMDWINDOW_MAX = 0x000000FF, +}; + +/* write to the command window */ +struct kgsl_cmdwindow_write { + enum kgsl_cmdwindow_type target; + unsigned int addr; + unsigned int data; +}; + +#define IOCTL_KGSL_CMDWINDOW_WRITE \ + _IOW(KGSL_IOC_TYPE, 0x2e, struct kgsl_cmdwindow_write) + +struct kgsl_gpumem_alloc { + unsigned long gpuaddr; + size_t size; + unsigned int flags; +}; + +#define IOCTL_KGSL_GPUMEM_ALLOC \ + _IOWR(KGSL_IOC_TYPE, 0x2f, struct kgsl_gpumem_alloc) + +struct kgsl_cff_syncmem { + unsigned int gpuaddr; + unsigned int len; + unsigned int __pad[2]; /* For future binary compatibility */ +}; + +#define IOCTL_KGSL_CFF_SYNCMEM \ + _IOW(KGSL_IOC_TYPE, 0x30, struct kgsl_cff_syncmem) + +#ifdef __KERNEL__ +#ifdef CONFIG_MSM_KGSL_DRM +int kgsl_gem_obj_addr(int drm_fd, int handle, unsigned long *start, + unsigned long *len); +#else +#define kgsl_gem_obj_addr(...) 0 +#endif +#endif +#endif /* _MSM_KGSL_H */ + + +#else + /* * (C) Copyright Advanced Micro Devices, Inc. 2002, 2007 * Copyright (c) 2008-2009 QUALCOMM USA, INC. @@ -345,3 +815,4 @@ struct kgsl_cmdwindow_write { _IOW(KGSL_IOC_TYPE, 0x2e, struct kgsl_cmdwindow_write) #endif /* _MSM_KGSL_H */ +#endif diff --git a/include/linux/wait.h b/include/linux/wait.h index a48e16b7..0314550f 100644 --- a/include/linux/wait.h +++ b/include/linux/wait.h @@ -188,6 +188,75 @@ do { \ finish_wait(&wq, &__wait); \ } while (0) +/** + * wait_io_event_interruptible - sleep until an io condition gets true + * @wq: the waitqueue to wait on + * @condition: a C expression for the event to wait for + * + * The process is put to sleep (TASK_INTERRUPTIBLE) until the + * @condition evaluates to true or a signal is received. + * The @condition is checked each time the waitqueue @wq is woken up. + * + * wake_up() has to be called after changing any variable that could + * change the result of the wait condition. + * + * The function will return -ERESTARTSYS if it was interrupted by a + * signal and 0 if @condition evaluated to true. + */ +#define wait_io_event_interruptible(wq, condition) \ +({ \ + int __ret = 0; \ + if (!(condition)) \ + __wait_io_event_interruptible(wq, condition, __ret); \ + __ret; \ +}) + +#define __wait_io_event_interruptible_timeout(wq, condition, ret) \ +do { \ + DEFINE_WAIT(__wait); \ + \ + for (;;) { \ + prepare_to_wait(&wq, &__wait, TASK_INTERRUPTIBLE); \ + if (condition) \ + break; \ + if (!signal_pending(current)) { \ + ret = io_schedule_timeout(ret); \ + if (!ret) \ + break; \ + continue; \ + } \ + ret = -ERESTARTSYS; \ + break; \ + } \ + finish_wait(&wq, &__wait); \ +} while (0) + +/** + * wait_io_event_interruptible_timeout - sleep until an io condition gets true or a timeout elapses + * @wq: the waitqueue to wait on + * @condition: a C expression for the event to wait for + * @timeout: timeout, in jiffies + * + * The process is put to sleep (TASK_INTERRUPTIBLE) until the + * @condition evaluates to true or a signal is received. + * The @condition is checked each time the waitqueue @wq is woken up. + * + * wake_up() has to be called after changing any variable that could + * change the result of the wait condition. + * + * The function returns 0 if the @timeout elapsed, -ERESTARTSYS if it + * was interrupted by a signal, and the remaining jiffies otherwise + * if the condition evaluated to true before the timeout elapsed. + */ + +#define wait_io_event_interruptible_timeout(wq, condition, timeout) \ +({ \ + long __ret = timeout; \ + if (!(condition)) \ + __wait_io_event_interruptible_timeout(wq, condition, __ret); \ + __ret; \ +}) + /** * wait_event - sleep until a condition gets true * @wq: the waitqueue to wait on diff --git a/kernel/sched.c b/kernel/sched.c index add23595..5afa5476 100644 --- a/kernel/sched.c +++ b/kernel/sched.c @@ -121,7 +121,7 @@ static inline int rt_policy(int policy) { - if (unlikely(policy == SCHED_FIFO || policy == SCHED_RR)) + if (policy == SCHED_FIFO || policy == SCHED_RR) return 1; return 0; } @@ -2443,7 +2443,7 @@ out_running: if (p->sched_class->task_wake_up) p->sched_class->task_wake_up(rq, p); - if (unlikely(rq->idle_stamp)) { + if (rq->idle_stamp) { u64 delta = rq->clock - rq->idle_stamp; u64 max = 2*sysctl_sched_migration_cost; diff --git a/kernel/sched_rt.c b/kernel/sched_rt.c index a4d790cd..178db056 100644 --- a/kernel/sched_rt.c +++ b/kernel/sched_rt.c @@ -1052,7 +1052,7 @@ static struct task_struct *_pick_next_task_rt(struct rq *rq) rt_rq = &rq->rt; - if (unlikely(!rt_rq->rt_nr_running)) + if (!rt_rq->rt_nr_running) return NULL; if (rt_rq_throttled(rt_rq)) @@ -1472,7 +1472,7 @@ static int pull_rt_task(struct rq *this_rq) static void pre_schedule_rt(struct rq *rq, struct task_struct *prev) { /* Try to pull RT tasks here if we lower this rq's prio */ - if (unlikely(rt_task(prev)) && rq->rt.highest_prio.curr > prev->prio) + if (rq->rt.highest_prio.curr > prev->prio) pull_rt_task(rq); } diff --git a/lib/Makefile b/lib/Makefile index 452f188c..019e8ef7 100644 --- a/lib/Makefile +++ b/lib/Makefile @@ -10,9 +10,9 @@ endif lib-y := ctype.o string.o vsprintf.o cmdline.o \ rbtree.o radix-tree.o dump_stack.o \ idr.o int_sqrt.o extable.o prio_tree.o \ - sha1.o irq_regs.o reciprocal_div.o argv_split.o \ + sha1.o md5.o irq_regs.o reciprocal_div.o argv_split.o \ proportions.o prio_heap.o ratelimit.o show_mem.o \ - is_single_threaded.o plist.o decompress.o flex_array.o + is_single_threaded.o plist.o decompress.o flex_array.o memcopy.o lib-$(CONFIG_MMU) += ioremap.o lib-$(CONFIG_SMP) += cpumask.o diff --git a/lib/bitmap.c b/lib/bitmap.c index 70256582..a7326e40 100644 --- a/lib/bitmap.c +++ b/lib/bitmap.c @@ -271,6 +271,89 @@ int __bitmap_weight(const unsigned long *bitmap, int bits) } EXPORT_SYMBOL(__bitmap_weight); +#define BITMAP_FIRST_WORD_MASK(start) (~0UL << ((start) % BITS_PER_LONG)) + +void bitmap_set(unsigned long *map, int start, int nr) +{ + unsigned long *p = map + BIT_WORD(start); + const int size = start + nr; + int bits_to_set = BITS_PER_LONG - (start % BITS_PER_LONG); + unsigned long mask_to_set = BITMAP_FIRST_WORD_MASK(start); + + while (nr - bits_to_set >= 0) { + *p |= mask_to_set; + nr -= bits_to_set; + bits_to_set = BITS_PER_LONG; + mask_to_set = ~0UL; + p++; + } + if (nr) { + mask_to_set &= BITMAP_LAST_WORD_MASK(size); + *p |= mask_to_set; + } +} +EXPORT_SYMBOL(bitmap_set); + +void bitmap_clear(unsigned long *map, int start, int nr) +{ + unsigned long *p = map + BIT_WORD(start); + const int size = start + nr; + int bits_to_clear = BITS_PER_LONG - (start % BITS_PER_LONG); + unsigned long mask_to_clear = BITMAP_FIRST_WORD_MASK(start); + + while (nr - bits_to_clear >= 0) { + *p &= ~mask_to_clear; + nr -= bits_to_clear; + bits_to_clear = BITS_PER_LONG; + mask_to_clear = ~0UL; + p++; + } + if (nr) { + mask_to_clear &= BITMAP_LAST_WORD_MASK(size); + *p &= ~mask_to_clear; + } +} +EXPORT_SYMBOL(bitmap_clear); + +/** + * bitmap_find_next_zero_area - find a contiguous aligned zero area + * @map: The address to base the search on + * @size: The bitmap size in bits + * @start: The bitnumber to start searching at + * @nr: The number of zeroed bits we're looking for + * @align_mask: Alignment mask for zero area + * @align_offset: Alignment offset for zero area. + * + * The @align_mask should be one less than a power of 2; the effect is that + * the bit offset of all zero areas this function finds plus @align_offset + * is multiple of that power of 2. + */ +unsigned long bitmap_find_next_zero_area_off(unsigned long *map, + unsigned long size, + unsigned long start, + unsigned int nr, + unsigned long align_mask, + unsigned long align_offset) +{ + unsigned long index, end, i; +again: + index = find_next_zero_bit(map, size, start); + + /* Align allocation */ + index = __ALIGN_MASK(index + align_offset, align_mask) - align_offset; + + end = index + nr; + if (end > size) + return end; + i = find_next_bit(map, end, index); + if (i < end) { + start = i + 1; + goto again; + } + return index; +} +EXPORT_SYMBOL(bitmap_find_next_zero_area_off); + /* * Bitmap printing & parsing functions: first version by Bill Irwin, * second version by Paul Jackson, third by Joe Korty. diff --git a/lib/memcopy.c b/lib/memcopy.c new file mode 100644 index 00000000..92c300fa --- /dev/null +++ b/lib/memcopy.c @@ -0,0 +1,403 @@ +/* + * memcopy.c -- subroutines for memory copy functions. + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the Free + * Software Foundation; either version 2 of the License, or (at your option) + * any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General + * Public License for more details. + * + * You should have received a copy of the GNU General Public License along + * with this program; if not, write to the Free Software Foundation, Inc., + * 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * The code is derived from the GNU C Library. + * Copyright (C) 1991, 1992, 1993, 1997, 2004 Free Software Foundation, Inc. + */ + +/* BE VERY CAREFUL IF YOU CHANGE THIS CODE...! */ + +#include + +/* + * _wordcopy_fwd_aligned -- Copy block beginning at SRCP to block beginning + * at DSTP with LEN `op_t' words (not LEN bytes!). + * Both SRCP and DSTP should be aligned for memory operations on `op_t's. + */ +void _wordcopy_fwd_aligned (long int dstp, long int srcp, size_t len) +{ + op_t a0, a1; + + switch (len % 8) { + case 2: + a0 = ((op_t *) srcp)[0]; + srcp -= 6 * OPSIZ; + dstp -= 7 * OPSIZ; + len += 6; + goto do1; + case 3: + a1 = ((op_t *) srcp)[0]; + srcp -= 5 * OPSIZ; + dstp -= 6 * OPSIZ; + len += 5; + goto do2; + case 4: + a0 = ((op_t *) srcp)[0]; + srcp -= 4 * OPSIZ; + dstp -= 5 * OPSIZ; + len += 4; + goto do3; + case 5: + a1 = ((op_t *) srcp)[0]; + srcp -= 3 * OPSIZ; + dstp -= 4 * OPSIZ; + len += 3; + goto do4; + case 6: + a0 = ((op_t *) srcp)[0]; + srcp -= 2 * OPSIZ; + dstp -= 3 * OPSIZ; + len += 2; + goto do5; + case 7: + a1 = ((op_t *) srcp)[0]; + srcp -= 1 * OPSIZ; + dstp -= 2 * OPSIZ; + len += 1; + goto do6; + case 0: + if (OP_T_THRESHOLD <= 3 * OPSIZ && len == 0) + return; + a0 = ((op_t *) srcp)[0]; + srcp -= 0 * OPSIZ; + dstp -= 1 * OPSIZ; + goto do7; + case 1: + a1 = ((op_t *) srcp)[0]; + srcp -=-1 * OPSIZ; + dstp -= 0 * OPSIZ; + len -= 1; + if (OP_T_THRESHOLD <= 3 * OPSIZ && len == 0) + goto do0; + goto do8; /* No-op. */ + } + + do { +do8: + a0 = ((op_t *) srcp)[0]; + ((op_t *) dstp)[0] = a1; +do7: + a1 = ((op_t *) srcp)[1]; + ((op_t *) dstp)[1] = a0; +do6: + a0 = ((op_t *) srcp)[2]; + ((op_t *) dstp)[2] = a1; +do5: + a1 = ((op_t *) srcp)[3]; + ((op_t *) dstp)[3] = a0; +do4: + a0 = ((op_t *) srcp)[4]; + ((op_t *) dstp)[4] = a1; +do3: + a1 = ((op_t *) srcp)[5]; + ((op_t *) dstp)[5] = a0; +do2: + a0 = ((op_t *) srcp)[6]; + ((op_t *) dstp)[6] = a1; +do1: + a1 = ((op_t *) srcp)[7]; + ((op_t *) dstp)[7] = a0; + + srcp += 8 * OPSIZ; + dstp += 8 * OPSIZ; + len -= 8; + } while (len != 0); + + /* + * This is the right position for do0. Please don't move it into + * the loop. + */ +do0: + ((op_t *) dstp)[0] = a1; +} + +/* + * _wordcopy_fwd_dest_aligned -- Copy block beginning at SRCP to block + * beginning at DSTP with LEN `op_t' words (not LEN bytes!). DSTP should + * be aligned for memory operations on `op_t's, but SRCP must *not* be aligned. + */ + +void _wordcopy_fwd_dest_aligned (long int dstp, long int srcp, size_t len) +{ + op_t a0, a1, a2, a3; + int sh_1, sh_2; + + /* + * Calculate how to shift a word read at the memory operation aligned + * srcp to make it aligned for copy. + */ + sh_1 = 8 * (srcp % OPSIZ); + sh_2 = 8 * OPSIZ - sh_1; + + /* + * Make SRCP aligned by rounding it down to the beginning of the `op_t' + * it points in the middle of. + */ + srcp &= -OPSIZ; + + switch (len % 4) { + case 2: + a1 = ((op_t *) srcp)[0]; + a2 = ((op_t *) srcp)[1]; + srcp -= 1 * OPSIZ; + dstp -= 3 * OPSIZ; + len += 2; + goto do1; + case 3: + a0 = ((op_t *) srcp)[0]; + a1 = ((op_t *) srcp)[1]; + srcp -= 0 * OPSIZ; + dstp -= 2 * OPSIZ; + len += 1; + goto do2; + case 0: + if (OP_T_THRESHOLD <= 3 * OPSIZ && len == 0) + return; + a3 = ((op_t *) srcp)[0]; + a0 = ((op_t *) srcp)[1]; + srcp -=-1 * OPSIZ; + dstp -= 1 * OPSIZ; + len += 0; + goto do3; + case 1: + a2 = ((op_t *) srcp)[0]; + a3 = ((op_t *) srcp)[1]; + srcp -=-2 * OPSIZ; + dstp -= 0 * OPSIZ; + len -= 1; + if (OP_T_THRESHOLD <= 3 * OPSIZ && len == 0) + goto do0; + goto do4; /* No-op. */ + } + + do { +do4: + a0 = ((op_t *) srcp)[0]; + ((op_t *) dstp)[0] = MERGE (a2, sh_1, a3, sh_2); +do3: + a1 = ((op_t *) srcp)[1]; + ((op_t *) dstp)[1] = MERGE (a3, sh_1, a0, sh_2); +do2: + a2 = ((op_t *) srcp)[2]; + ((op_t *) dstp)[2] = MERGE (a0, sh_1, a1, sh_2); +do1: + a3 = ((op_t *) srcp)[3]; + ((op_t *) dstp)[3] = MERGE (a1, sh_1, a2, sh_2); + + srcp += 4 * OPSIZ; + dstp += 4 * OPSIZ; + len -= 4; + } while (len != 0); + + /* + * This is the right position for do0. Please don't move it into + * the loop. + */ +do0: + ((op_t *) dstp)[0] = MERGE (a2, sh_1, a3, sh_2); +} + +/* + * _wordcopy_bwd_aligned -- Copy block finishing right before + * SRCP to block finishing right before DSTP with LEN `op_t' words (not LEN + * bytes!). Both SRCP and DSTP should be aligned for memory operations + * on `op_t's. + */ +void _wordcopy_bwd_aligned (long int dstp, long int srcp, size_t len) +{ + op_t a0, a1; + + switch (len % 8) { + case 2: + srcp -= 2 * OPSIZ; + dstp -= 1 * OPSIZ; + a0 = ((op_t *) srcp)[1]; + len += 6; + goto do1; + case 3: + srcp -= 3 * OPSIZ; + dstp -= 2 * OPSIZ; + a1 = ((op_t *) srcp)[2]; + len += 5; + goto do2; + case 4: + srcp -= 4 * OPSIZ; + dstp -= 3 * OPSIZ; + a0 = ((op_t *) srcp)[3]; + len += 4; + goto do3; + case 5: + srcp -= 5 * OPSIZ; + dstp -= 4 * OPSIZ; + a1 = ((op_t *) srcp)[4]; + len += 3; + goto do4; + case 6: + srcp -= 6 * OPSIZ; + dstp -= 5 * OPSIZ; + a0 = ((op_t *) srcp)[5]; + len += 2; + goto do5; + case 7: + srcp -= 7 * OPSIZ; + dstp -= 6 * OPSIZ; + a1 = ((op_t *) srcp)[6]; + len += 1; + goto do6; + case 0: + if (OP_T_THRESHOLD <= 3 * OPSIZ && len == 0) + return; + srcp -= 8 * OPSIZ; + dstp -= 7 * OPSIZ; + a0 = ((op_t *) srcp)[7]; + goto do7; + case 1: + srcp -= 9 * OPSIZ; + dstp -= 8 * OPSIZ; + a1 = ((op_t *) srcp)[8]; + len -= 1; + if (OP_T_THRESHOLD <= 3 * OPSIZ && len == 0) + goto do0; + goto do8; /* No-op. */ + } + + do { +do8: + a0 = ((op_t *) srcp)[7]; + ((op_t *) dstp)[7] = a1; +do7: + a1 = ((op_t *) srcp)[6]; + ((op_t *) dstp)[6] = a0; +do6: + a0 = ((op_t *) srcp)[5]; + ((op_t *) dstp)[5] = a1; +do5: + a1 = ((op_t *) srcp)[4]; + ((op_t *) dstp)[4] = a0; +do4: + a0 = ((op_t *) srcp)[3]; + ((op_t *) dstp)[3] = a1; +do3: + a1 = ((op_t *) srcp)[2]; + ((op_t *) dstp)[2] = a0; +do2: + a0 = ((op_t *) srcp)[1]; + ((op_t *) dstp)[1] = a1; +do1: + a1 = ((op_t *) srcp)[0]; + ((op_t *) dstp)[0] = a0; + + srcp -= 8 * OPSIZ; + dstp -= 8 * OPSIZ; + len -= 8; + } while (len != 0); + + /* + * This is the right position for do0. Please don't move it into + * the loop. + */ +do0: + ((op_t *) dstp)[7] = a1; +} + +/* + * _wordcopy_bwd_dest_aligned -- Copy block finishing right before SRCP to + * block finishing right before DSTP with LEN `op_t' words (not LEN bytes!). + * DSTP should be aligned for memory operations on `op_t', but SRCP must *not* + * be aligned. + */ +void _wordcopy_bwd_dest_aligned (long int dstp, long int srcp, size_t len) +{ + op_t a0, a1, a2, a3; + int sh_1, sh_2; + + /* + * Calculate how to shift a word read at the memory operation aligned + * srcp to make it aligned for copy. + */ + + sh_1 = 8 * (srcp % OPSIZ); + sh_2 = 8 * OPSIZ - sh_1; + + /* + * Make srcp aligned by rounding it down to the beginning of the op_t + * it points in the middle of. + */ + srcp &= -OPSIZ; + srcp += OPSIZ; + + switch (len % 4) { + case 2: + srcp -= 3 * OPSIZ; + dstp -= 1 * OPSIZ; + a2 = ((op_t *) srcp)[2]; + a1 = ((op_t *) srcp)[1]; + len += 2; + goto do1; + case 3: + srcp -= 4 * OPSIZ; + dstp -= 2 * OPSIZ; + a3 = ((op_t *) srcp)[3]; + a2 = ((op_t *) srcp)[2]; + len += 1; + goto do2; + case 0: + if (OP_T_THRESHOLD <= 3 * OPSIZ && len == 0) + return; + srcp -= 5 * OPSIZ; + dstp -= 3 * OPSIZ; + a0 = ((op_t *) srcp)[4]; + a3 = ((op_t *) srcp)[3]; + goto do3; + case 1: + srcp -= 6 * OPSIZ; + dstp -= 4 * OPSIZ; + a1 = ((op_t *) srcp)[5]; + a0 = ((op_t *) srcp)[4]; + len -= 1; + if (OP_T_THRESHOLD <= 3 * OPSIZ && len == 0) + goto do0; + goto do4; /* No-op. */ + } + + do { +do4: + a3 = ((op_t *) srcp)[3]; + ((op_t *) dstp)[3] = MERGE (a0, sh_1, a1, sh_2); +do3: + a2 = ((op_t *) srcp)[2]; + ((op_t *) dstp)[2] = MERGE (a3, sh_1, a0, sh_2); +do2: + a1 = ((op_t *) srcp)[1]; + ((op_t *) dstp)[1] = MERGE (a2, sh_1, a3, sh_2); +do1: + a0 = ((op_t *) srcp)[0]; + ((op_t *) dstp)[0] = MERGE (a1, sh_1, a2, sh_2); + + srcp -= 4 * OPSIZ; + dstp -= 4 * OPSIZ; + len -= 4; + } while (len != 0); + + /* + * This is the right position for do0. Please don't move it into + * the loop. + */ +do0: + ((op_t *) dstp)[3] = MERGE (a0, sh_1, a1, sh_2); +} + diff --git a/lib/memory_alloc.c b/lib/memory_alloc.c new file mode 100644 index 00000000..7bfba29c --- /dev/null +++ b/lib/memory_alloc.c @@ -0,0 +1,426 @@ +/* Copyright (c) 2011, Code Aurora Forum. All rights reserved. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 and + * only version 2 as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + + +#define MAX_MEMPOOLS 8 + +struct mem_pool mpools[MAX_MEMPOOLS]; + +/* The tree contains all allocations over all memory pools */ +static struct rb_root alloc_root; +static struct mutex alloc_mutex; + +static void *s_start(struct seq_file *m, loff_t *pos) + __acquires(&alloc_mutex) +{ + loff_t n = *pos; + struct rb_node *r; + + mutex_lock(&alloc_mutex); + r = rb_first(&alloc_root); + + while (n > 0 && r) { + n--; + r = rb_next(r); + } + if (!n) + return r; + return NULL; +} + +static void *s_next(struct seq_file *m, void *p, loff_t *pos) +{ + struct rb_node *r = p; + ++*pos; + return rb_next(r); +} + +static void s_stop(struct seq_file *m, void *p) + __releases(&alloc_mutex) +{ + mutex_unlock(&alloc_mutex); +} + +static int s_show(struct seq_file *m, void *p) +{ + struct rb_node *r = p; + struct alloc *node = rb_entry(r, struct alloc, rb_node); + + seq_printf(m, "0x%lx 0x%p %ld %u %pS\n", node->paddr, node->vaddr, + node->len, node->mpool->id, node->caller); + return 0; +} + +static const struct seq_operations mempool_op = { + .start = s_start, + .next = s_next, + .stop = s_stop, + .show = s_show, +}; + +static int mempool_open(struct inode *inode, struct file *file) +{ + return seq_open(file, &mempool_op); +} + +static struct alloc *find_alloc(void *addr) +{ + struct rb_root *root = &alloc_root; + struct rb_node *p = root->rb_node; + + mutex_lock(&alloc_mutex); + + while (p) { + struct alloc *node; + + node = rb_entry(p, struct alloc, rb_node); + if (addr < node->vaddr) + p = p->rb_left; + else if (addr > node->vaddr) + p = p->rb_right; + else { + mutex_unlock(&alloc_mutex); + return node; + } + } + mutex_unlock(&alloc_mutex); + return NULL; +} + +static int add_alloc(struct alloc *node) +{ + struct rb_root *root = &alloc_root; + struct rb_node **p = &root->rb_node; + struct rb_node *parent = NULL; + + mutex_lock(&alloc_mutex); + while (*p) { + struct alloc *tmp; + parent = *p; + + tmp = rb_entry(parent, struct alloc, rb_node); + + if (node->vaddr < tmp->vaddr) + p = &(*p)->rb_left; + else if (node->vaddr > tmp->vaddr) + p = &(*p)->rb_right; + else { + WARN(1, "memory at %p already allocated", tmp->vaddr); + mutex_unlock(&alloc_mutex); + return -EINVAL; + } + } + rb_link_node(&node->rb_node, parent, p); + rb_insert_color(&node->rb_node, root); + mutex_unlock(&alloc_mutex); + return 0; +} + +static int remove_alloc(struct alloc *victim_node) +{ + struct rb_root *root = &alloc_root; + if (!victim_node) + return -EINVAL; + + mutex_lock(&alloc_mutex); + rb_erase(&victim_node->rb_node, root); + mutex_unlock(&alloc_mutex); + return 0; +} + +static struct gen_pool *initialize_gpool(unsigned long start, + unsigned long size) +{ + struct gen_pool *gpool; + + gpool = gen_pool_create(PAGE_SHIFT, -1); + + if (!gpool) + return NULL; + if (gen_pool_add(gpool, start, size, -1)) { + gen_pool_destroy(gpool); + return NULL; + } + + return gpool; +} + +static void *__alloc(struct mem_pool *mpool, unsigned long size, + unsigned long align, int cached, void *caller) +{ + unsigned long paddr; + void __iomem *vaddr; + + unsigned long aligned_size; + int log_align = ilog2(align); + + struct alloc *node; + + aligned_size = PFN_ALIGN(size); + paddr = gen_pool_alloc_aligned(mpool->gpool, aligned_size, log_align); + if (!paddr) + return NULL; + + node = kmalloc(sizeof(struct alloc), GFP_KERNEL); + if (!node) + goto out; + + if (cached) + vaddr = ioremap_cached(paddr, aligned_size); + else + vaddr = ioremap(paddr, aligned_size); + + if (!vaddr) + goto out_kfree; + + node->vaddr = vaddr; + node->paddr = paddr; + node->len = aligned_size; + node->mpool = mpool; + node->caller = caller; + if (add_alloc(node)) + goto out_kfree; + + mpool->free -= aligned_size; + + return vaddr; +out_kfree: + if (vaddr) + iounmap(vaddr); + kfree(node); +out: + gen_pool_free(mpool->gpool, paddr, aligned_size); + return NULL; +} + +static void __free(void *vaddr, bool unmap) +{ + struct alloc *node = find_alloc(vaddr); + + if (!node) + return; + + if (unmap) + iounmap(node->vaddr); + + gen_pool_free(node->mpool->gpool, node->paddr, node->len); + node->mpool->free += node->len; + + remove_alloc(node); + kfree(node); +} + +static struct mem_pool *mem_type_to_memory_pool(int mem_type) +{ + struct mem_pool *mpool = &mpools[mem_type]; + + if (!mpool->size) + return NULL; + + mutex_lock(&mpool->pool_mutex); + if (!mpool->gpool) + mpool->gpool = initialize_gpool(mpool->paddr, mpool->size); + mutex_unlock(&mpool->pool_mutex); + if (!mpool->gpool) + return NULL; + + return mpool; +} + +struct mem_pool *initialize_memory_pool(unsigned long start, + unsigned long size, int mem_type) +{ + int id = mem_type; + + if (id >= MAX_MEMPOOLS || size <= PAGE_SIZE || size % PAGE_SIZE) + return NULL; + + mutex_lock(&mpools[id].pool_mutex); + + mpools[id].paddr = start; + mpools[id].size = size; + mpools[id].free = size; + mpools[id].id = id; + mutex_unlock(&mpools[id].pool_mutex); + + pr_info("memory pool %d (start %lx size %lx) initialized\n", + id, start, size); + return &mpools[id]; +} +EXPORT_SYMBOL_GPL(initialize_memory_pool); + +void *allocate_contiguous_memory(unsigned long size, + int mem_type, unsigned long align, int cached) +{ + unsigned long aligned_size = PFN_ALIGN(size); + struct mem_pool *mpool; + + mpool = mem_type_to_memory_pool(mem_type); + if (!mpool) + return NULL; + return __alloc(mpool, aligned_size, align, cached, + __builtin_return_address(0)); + +} +EXPORT_SYMBOL_GPL(allocate_contiguous_memory); + +unsigned long _allocate_contiguous_memory_nomap(unsigned long size, + int mem_type, unsigned long align, void *caller) +{ + unsigned long paddr; + unsigned long aligned_size; + + struct alloc *node; + struct mem_pool *mpool; + int log_align = ilog2(align); + + mpool = mem_type_to_memory_pool(mem_type); + if (!mpool || !mpool->gpool) + return 0; + + aligned_size = PFN_ALIGN(size); + paddr = gen_pool_alloc_aligned(mpool->gpool, aligned_size, log_align); + if (!paddr) + return 0; + + node = kmalloc(sizeof(struct alloc), GFP_KERNEL); + if (!node) + goto out; + + node->paddr = paddr; + + /* We search the tree using node->vaddr, so set + * it to something unique even though we don't + * use it for physical allocation nodes. + * The virtual and physical address ranges + * are disjoint, so there won't be any chance of + * a duplicate node->vaddr value. + */ + node->vaddr = (void *)paddr; + node->len = aligned_size; + node->mpool = mpool; + node->caller = caller; + if (add_alloc(node)) + goto out_kfree; + + mpool->free -= aligned_size; + return paddr; +out_kfree: + kfree(node); +out: + gen_pool_free(mpool->gpool, paddr, aligned_size); + return 0; +} +EXPORT_SYMBOL_GPL(_allocate_contiguous_memory_nomap); + +unsigned long allocate_contiguous_memory_nomap(unsigned long size, + int mem_type, unsigned long align) +{ + return _allocate_contiguous_memory_nomap(size, mem_type, align, + __builtin_return_address(0)); +} +EXPORT_SYMBOL_GPL(allocate_contiguous_memory_nomap); + +void free_contiguous_memory(void *addr) +{ + if (!addr) + return; + __free(addr, true); + return; +} +EXPORT_SYMBOL_GPL(free_contiguous_memory); + +void free_contiguous_memory_by_paddr(unsigned long paddr) +{ + if (!paddr) + return; + __free((void *)paddr, false); + return; +} +EXPORT_SYMBOL_GPL(free_contiguous_memory_by_paddr); + +unsigned long memory_pool_node_paddr(void *vaddr) +{ + struct alloc *node = find_alloc(vaddr); + + if (!node) + return -EINVAL; + + return node->paddr; +} +EXPORT_SYMBOL_GPL(memory_pool_node_paddr); + +unsigned long memory_pool_node_len(void *vaddr) +{ + struct alloc *node = find_alloc(vaddr); + + if (!node) + return -EINVAL; + + return node->len; +} +EXPORT_SYMBOL_GPL(memory_pool_node_len); + +static const struct file_operations mempool_operations = { + .owner = THIS_MODULE, + .open = mempool_open, + .read = seq_read, + .llseek = seq_lseek, + .release = seq_release_private, +}; + +int __init memory_pool_init(void) +{ + int i; + + alloc_root = RB_ROOT; + mutex_init(&alloc_mutex); + for (i = 0; i < ARRAY_SIZE(mpools); i++) { + mutex_init(&mpools[i].pool_mutex); + mpools[i].gpool = NULL; + } + + return 0; +} + +static int __init debugfs_mempool_init(void) +{ + struct dentry *entry, *dir = debugfs_create_dir("mempool", NULL); + + if (!dir) { + pr_err("Cannot create /sys/kernel/debug/mempool"); + return -EINVAL; + } + + entry = debugfs_create_file("map", S_IRUSR, dir, + NULL, &mempool_operations); + + if (!entry) + pr_err("Cannot create /sys/kernel/debug/mempool/map"); + + return entry ? 0 : -EINVAL; +} + +module_init(debugfs_mempool_init); + diff --git a/mm/ashmem.c b/mm/ashmem.c index 70028c39..16058090 100644 --- a/mm/ashmem.c +++ b/mm/ashmem.c @@ -276,7 +276,7 @@ out: * chunks of ashmem regions LRU-wise one-at-a-time until we hit 'nr_to_scan' * pages freed. */ -static int ashmem_shrink(int nr_to_scan, gfp_t gfp_mask) +static int ashmem_shrink(struct shrinker *s, int nr_to_scan, gfp_t gfp_mask) { struct ashmem_range *range, *next; @@ -595,8 +595,8 @@ static long ashmem_ioctl(struct file *file, unsigned int cmd, unsigned long arg) case ASHMEM_PURGE_ALL_CACHES: ret = -EPERM; if (capable(CAP_SYS_ADMIN)) { - ret = ashmem_shrink(0, GFP_KERNEL); - ashmem_shrink(ret, GFP_KERNEL); + ret = ashmem_shrink(&ashmem_shrinker, 0, GFP_KERNEL); + ashmem_shrink(&ashmem_shrinker, ret, GFP_KERNEL); } break; } @@ -604,6 +604,59 @@ static long ashmem_ioctl(struct file *file, unsigned int cmd, unsigned long arg) return ret; } +static int is_ashmem_file(struct file *file) +{ + char fname[256], *name; + name = dentry_path(file->f_dentry, fname, 256); + return strcmp(name, "/ashmem") ? 0 : 1; +} + +int get_ashmem_file(int fd, struct file **filp, struct file **vm_file, + unsigned long *len) +{ + int ret = -1; + struct file *file = fget(fd); + *filp = NULL; + *vm_file = NULL; + if (unlikely(file == NULL)) { + pr_err("ashmem: %s: requested data from file " + "descriptor that doesn't exist.\n", __func__); + } else { + char currtask_name[FIELD_SIZEOF(struct task_struct, comm) + 1]; + pr_debug("filp %p rdev %d pid %u(%s) file %p(%ld)" + " dev id: %d\n", filp, + file->f_dentry->d_inode->i_rdev, + current->pid, get_task_comm(currtask_name, current), + file, file_count(file), + MINOR(file->f_dentry->d_inode->i_rdev)); + if (is_ashmem_file(file)) { + struct ashmem_area *asma = file->private_data; + *filp = file; + *vm_file = asma->file; + *len = asma->size; + ret = 0; + } else { + pr_err("file descriptor is not an ashmem " + "region fd: %d\n", fd); + fput(file); + } + } + return ret; +} +EXPORT_SYMBOL(get_ashmem_file); + +void put_ashmem_file(struct file *file) +{ + char currtask_name[FIELD_SIZEOF(struct task_struct, comm) + 1]; + pr_debug("rdev %d pid %u(%s) file %p(%ld)" " dev id: %d\n", + file->f_dentry->d_inode->i_rdev, current->pid, + get_task_comm(currtask_name, current), file, + file_count(file), MINOR(file->f_dentry->d_inode->i_rdev)); + if (file && is_ashmem_file(file)) + fput(file); +} +EXPORT_SYMBOL(put_ashmem_file); + static struct file_operations ashmem_fops = { .owner = THIS_MODULE, .open = ashmem_open, diff --git a/mm/page_alloc.c b/mm/page_alloc.c index 7d4406f0..a24fdf92 100644 --- a/mm/page_alloc.c +++ b/mm/page_alloc.c @@ -121,7 +121,7 @@ static char * const zone_names[MAX_NR_ZONES] = { "Movable", }; -int min_free_kbytes = 1024; +int min_free_kbytes = 5120; int min_free_order_shift = 1; static unsigned long __meminitdata nr_kernel_pages;