ion integration challengesconnect.linaro.org.s3.amazonaws.com/sfo17/presentations/... ·...

ION Integration ChallengesLinaro Connect SFO17

Agenda

- ION on Qualcomm SoCs- Downstream differences- Secure heaps- Virtualized platforms

1. ION on Qualcomm SoCs

uapi/ion.h

enum ion_heap_type {

ION_HEAP_TYPE_SYSTEM,

ION_HEAP_TYPE_SYSTEM_CONTIG,

ION_HEAP_TYPE_CARVEOUT,

ION_HEAP_TYPE_CHUNK,

ION_HEAP_TYPE_DMA,

ION_HEAP_TYPE_CUSTOM,

};

uapi/ion.h

uapi/msm_ion.h

enum ion_heap_type {

ION_HEAP_TYPE_SYSTEM,

ION_HEAP_TYPE_SYSTEM_CONTIG,

ION_HEAP_TYPE_CARVEOUT,

ION_HEAP_TYPE_CHUNK,

ION_HEAP_TYPE_DMA,

ION_HEAP_TYPE_CUSTOM,

};

enum msm_ion_heap_types {

ION_HEAP_TYPE_MSM_START = ION_HEAP_TYPE_CUSTOM + 1,

ION_HEAP_TYPE_SECURE_DMA = ION_HEAP_TYPE_MSM_START,

ION_HEAP_TYPE_SYSTEM_SECURE,

ION_HEAP_TYPE_HYP_CMA,

};

Gralloc

Display GPU

Camera Video

Gralloc

GPU

Camera Video

Prop/Misc

FastCV

VPP

GLES

WFD

QSECOM

FastRPC

Biometrics

Audio

Camera libs

SPSS

Display

Kernel

Display Camera Video

Audio QSECOM

Gralloc

GPU

Camera Video

Prop/Misc

FastCV

GLES

QSECOM

SPSS

VPP

WFD

FastRPC

Biometrics

Audio

Camera libs

Display

Gralloc

GPU

Camera Video

Prop/Misc

FastCV

GLES

QSECOM

SPSS

System

DMA

HYP_CMA

System Secure

Kernel


Audio QSECOM

VPP

WFD

FastRPC

Biometrics

Audio

Camera libs

Display

uapi/msm_ion.h

enum ion_heap_ids {

INVALID_HEAP_ID = -1,

ION_CP_MM_HEAP_ID = 8,

ION_SECURE_HEAP_ID = 9,

ION_SECURE_DISPLAY_HEAP_ID = 10,

ION_CP_MFC_HEAP_ID = 12,

ION_SPSS_HEAP_ID = 13,

ION_CP_WB_HEAP_ID = 16,

ION_CAMERA_HEAP_ID = 20,

ION_SYSTEM_CONTIG_HEAP_ID = 21,

ION_ADSP_HEAP_ID = 22,

ION_PIL1_HEAP_ID = 23,

ION_SF_HEAP_ID = 24,

ION_SYSTEM_HEAP_ID = 25,


ION_QSECOM_HEAP_ID = 27,

ION_AUDIO_HEAP_ID = 28,

ION_MM_FIRMWARE_HEAP_ID = 29,

ION_HEAP_ID_RESERVED = 31

};

uapi/msm_ion.h

System

DMA

HYP CMA

System Secure

Secure DMA

Carveout

CP?

System Contig

enum ion_heap_ids {

INVALID_HEAP_ID = -1,

ION_CP_MM_HEAP_ID = 8,

ION_SECURE_HEAP_ID = 9,

ION_SECURE_DISPLAY_HEAP_ID = 10,

ION_CP_MFC_HEAP_ID = 12,

ION_SPSS_HEAP_ID = 13,

ION_CP_WB_HEAP_ID = 16,

ION_CAMERA_HEAP_ID = 20,

ION_SYSTEM_CONTIG_HEAP_ID = 21,

ION_ADSP_HEAP_ID = 22,


ION_SF_HEAP_ID = 24,

ION_SYSTEM_HEAP_ID = 25,


ION_QSECOM_HEAP_ID = 27,

ION_AUDIO_HEAP_ID = 28,

ION_MM_FIRMWARE_HEAP_ID = 29,

ION_HEAP_ID_RESERVED = 31

};

qcom,ion {

compatible = "qcom,msm-ion";

#address-cells = <1>;

#size-cells = <0>;

system_heap: qcom,ion-heap@25 {

reg = <25>;

qcom,ion-heap-type = "SYSTEM";

};

qcom,ion-heap@22 { /* ADSP HEAP */

reg = <22>;

memory-region = <&adsp_mem>;

qcom,ion-heap-type = "DMA";

};

qcom,ion-heap@27 { /* QSEECOM HEAP */

reg = <27>;

memory-region = <&qseecom_mem>;


};

qcom,ion-heap@13 { /* SPSS HEAP */

reg = <13>;

memory-region = <&sp_mem>;


};

qcom,ion-heap@10 { /* SECURE DISPLAY HEAP */

reg = <10>;

memory-region = <&secure_display_memory>;

qcom,ion-heap-type = "HYP_CMA";

};

qcom,ion-heap@9 {

reg = <9>;

qcom,ion-heap-type = "SYSTEM_SECURE";

};

};

msm8998-ion.dtsi

msm8998-ion.dtsi

ION_SYSTEM_HEAP_ID

ION_ADSP_HEAP_ID

ION_QSECOM_HEAP_ID

ION_SPSS_HEAP_ID

ION_SECURE_DISPLAY_HEAP_ID

ION_SECURE_HEAP_ID

qcom,ion {



#size-cells = <0>;


reg = <25>;


};


reg = <22>;



};


reg = <27>;



};


reg = <13>;



};


reg = <10>;



};

qcom,ion-heap@9 {

reg = <9>;


};

};

msm8998-ion.dtsi

ION_SYSTEM_HEAP_ID

ION_ADSP_HEAP_ID

ION_QSECOM_HEAP_ID

ION_SPSS_HEAP_ID

ION_SECURE_DISPLAY_HEAP_ID

ION_SECURE_HEAP_ID

qcom,ion {



#size-cells = <0>;


reg = <25>;


};


reg = <22>;



};


reg = <27>;



};


reg = <13>;



};


reg = <10>;



};

qcom,ion-heap@9 {

reg = <9>;


};

};

reserved-memory {


#size-cells = <2>;

ranges;

...

adsp_mem: adsp_region {

compatible = "shared-dma-pool";

alloc-ranges = <0 0x00000000 0 0xffffffff>;

reusable;

alignment = <0 0x400000>;

size = <0 0x800000>;

};

qseecom_mem: qseecom_region {

compatible = "shared-dma-pool";

alloc-ranges = <0 0x00000000 0 0xffffffff>;

reusable;

alignment = <0 0x400000>;

size = <0 0x1400000>;

};

sp_mem: sp_region {

...

size = <0 0x800000>;

...

};

secure_display_memory: secure_region {

...

size = <0 0x5c00000>;

};

...

};

Example: VPP

Example: VPP

#include <linux/msm_ion.h>

uint32_t u32VppIon_Alloc(t_StVppCtx *pstCtx, uint32_t u32Len, uint32_t u32Secure,

t_StVppIonBuf *buf)

{

...

// Allocate ion memory

alloc_data.len = (u32Len + (ION_BUF_ALIGN - 1)) & ~(ION_BUF_ALIGN - 1);

if (u32Secure)

{

alloc_data.align = ION_BUF_ALIGN_SECURE;

alloc_data.flags = ION_FLAG_SECURE | ION_FLAG_CP_PIXEL;

alloc_data.heap_id_mask = ION_HEAP(ION_SECURE_HEAP_ID);

}

else

{

alloc_data.align = ION_BUF_ALIGN;

alloc_data.flags = 0;

alloc_data.heap_id_mask = ION_HEAP(ION_SYSTEM_HEAP_ID);

}

...

rc = ioctl(sStIonCb.fd_dev_ion, ION_IOC_ALLOC, &alloc_data);

if (rc < 0)

...

}

Example: Gralloc

Example: Gralloc

int IonController::allocate(alloc_data& data, int usage)

{

...

if (usage & GRALLOC_USAGE_PROTECTED) {

...

if (usage & GRALLOC_USAGE_PRIVATE_SECURE_DISPLAY) {

ionHeapId = ION_HEAP(ION_SECURE_DISPLAY_HEAP_ID);

ionFlags |= ION_FLAG_SECURE | ION_FLAG_CP_SEC_DISPLAY;

} else if (usage & GRALLOC_USAGE_HW_CAMERA_MASK) {

ionHeapId = ION_HEAP(ION_SECURE_DISPLAY_HEAP_ID);

...

} else {

ionHeapId = ION_HEAP(ION_SECURE_HEAP_ID);

...

}

} else if (usage & GRALLOC_USAGE_PRIVATE_MM_HEAP) {

...

}

if (usage & GRALLOC_USAGE_PRIVATE_CAMERA_HEAP)

ionHeapId |= ION_HEAP(ION_CAMERA_HEAP_ID);

if (usage & GRALLOC_USAGE_PRIVATE_ADSP_HEAP)

ionHeapId |= ION_HEAP(ION_ADSP_HEAP_ID);

if (!ionHeapId)

ionHeapId |= ION_HEAP(ION_SYSTEM_HEAP_ID);

...

allocate(ionHeapId, ionFlags);

...

}

Example: SCVE?

Example: SCVE?

int scve_rpc_ion_alloc(mmap_info *m)

{

...

/* heap_id_mask is only available for 8974/latest JB */

#if defined USE_CPMM_ION

alloc.flags = ION_HEAP(ION_CP_MM_HEAP_ID);

#elif defined USE_IOMMU_ION

alloc.align = 0x1000;

alloc.heap_id_mask = ION_HEAP(ION_SYSTEM_HEAP_ID);

alloc.flags = ION_FLAG_CACHED;

#elif defined USE_ADSP_ION

alloc.heap_id_mask = ION_HEAP(ION_ADSP_HEAP_ID);

alloc.flags = ION_FLAG_CACHED;

#else

#error "No ION HEAP ID defined. Check feature-config.mk"

#endif

if (ioctl(m->fd, ION_IOC_ALLOC, &alloc) < 0)

...

}

Problem:

Problem: Userspace needs to specify hardcoded heap IDs.


Solution:


Solution: API to query heaps IDs?

msm8998-ion.dtsi

qcom,ion {



#size-cells = <0>;


reg = <25>;


};


reg = <22>;



};


reg = <27>;



};


reg = <13>;



};


reg = <10>;



};

qcom,ion-heap@9 {

reg = <9>;


};

};

Gralloc

GPU

Camera Video

Prop/MiscGLES

QSECOM

SPSS

System

DMA

HYP_CMA

System Secure

Kernel


Audio QSECOM

FastCV

VPP

WFD

FastRPC

Biometrics

Audio

Camera libs

Display

Gralloc

GPU

Camera Video

Prop/MiscGLES

QSECOM

SPSS

System

DMA

HYP_CMA

System Secure

QSECOM

ADSP

SPSS

Kernel


Audio QSECOM

Multiple instances

FastCV

VPP

WFD

FastRPC

Biometrics

Audio

Camera libs

Display

Problem: Userspace needs to specify heap IDs.




- We may also need to query the heap name.



- We may also need to query the heap name.- But that’s okay.

2. Downstream kernel differences:


- Heap instantiation


- Heap instantiation: Are DT bindings for ION acceptable?


- Heap instantiation: Are DT bindings for ION acceptable?- Zeroing optimizations for system heaps

add to sglist

page pool?

alloc

size > 0?

Y

Y

endN

alloc_pagesN

(GFP_ZERO)

Upstream

add to sglist

page pool?

alloc

size > 0?

Y

Y

endN

alloc_pagesN

(GFP_ZERO)

add to page pool

zero pages

free

end

Upstream

add to sglist

page pool?

alloc

size > 0?

Y

Y

endN

alloc_pagesN

(GFP_ZERO)

page pool?

alloc

size > 0?

Y

Y

N

alloc_pagesN

(No GFP_ZERO) vmap / zero pages

add to sglist1

sync sglist2

Add to sglist2/save page to array

merge sglists

end

add to page pool

zero pages

free

end

Upstream Optimized

add to sglist

page pool?

alloc

size > 0?

Y

Y

endN

alloc_pagesN

(GFP_ZERO)

page pool?

alloc

size > 0?

Y

Y

N

alloc_pagesN

(No GFP_ZERO) vmap / zero pages

add to sglist1

sync sglist2

Add to sglist2/save page to array

merge sglists

end

add to page pool

zero pages

free

end

Upstream Optimized

dma_sync_sg_for_device

dma_sync_sg_for_device(ion_dev, t->sgl, t->nents, ..);


- Heap instantiation: Are DT bindings for ION acceptable?- Zeroing optimizations for system heaps: DMA Mapping API abuse.


- Heap instantiation: Are DT bindings for ION acceptable?- Zeroing optimizations for system heaps: DMA Mapping API abuse.- Custom caching IOCTLs vs DMA_BUF_IOCTL_SYNC

User

struct ion_custom_data d;

struct ion_flush_data flush_data;

flush_data.handle = handle;

flush_data.offset = offset;

...

d.arg = (unsigned long int)&flush_data;

...

d.cmd = ION_IOC_INV_CACHES;

ioctl(ion_fd, ION_IOC_CUSTOM, &d);

/* CPU access */

d.cmd = ION_IOC_CLEAN_CACHES;


static int ion_pages_cache_ops(...,

unsigned int offset,

unsigned int length,

unsigned int cmd)

{

...

switch (cmd) {

case ION_IOC_CLEAN_CACHES:

op = dmac_clean_range; break;

case ION_IOC_INV_CACHES:

op = dmac_inv_range; break;

...

}

...

for_each_sg(table->sgl, sg, table->nents, i) {

...

__do_cache_ops(sg_page(sg), sg_offset,size,

op);

...

}

}

struct ion_custom_data d;

struct ion_flush_data flush_data;

flush_data.handle = handle;

flush_data.offset = offset;

...

d.arg = (unsigned long int)&flush_data;

...

d.cmd = ION_IOC_INV_CACHES;


/* CPU access */

d.cmd = ION_IOC_CLEAN_CACHES;


User Kernel

struct dma_buf_sync sync;

sync.flags = DMA_BUF_SYNC_START |

DMA_BUF_SYNC_RW;

ioctl(dmabuf_fd, DMA_BUF_IOCTL_SYNC, &sync);

/* CPU access */

sync.flags = DMA_BUF_SYNC_END |

DMA_BUF_SYNC_RW;


User

struct dma_buf_sync sync;

sync.flags = DMA_BUF_SYNC_START |

DMA_BUF_SYNC_RW;


/* CPU access */

sync.flags = DMA_BUF_SYNC_END |

DMA_BUF_SYNC_RW;


User

static int

ion_dma_buf_begin_cpu_access(

struct dma_buf *dmabuf,

enum dma_data_direction d)

{

...

list_for_each_entry(a, &buffer->attachments,

list) }{

dma_sync_sg_for_cpu(a->dev, sgl, nents,

..);

}

...

}

Kernel

Some concerns:- No provision of flushing a specific region in the buffer.- ION buffer needs to be imported by at least one client to use this method.- Unnecessary multiple cache ops when buffer imported by more than one client?


- Heap instantiation: Are DT bindings for ION acceptable?- Zeroing optimizations for system heaps: DMA Mapping API abuse.- Custom caching IOCTLs vs DMA_BUF_IOCTL_SYNC: Mostly good.


- Heap instantiation: Are DT bindings for ION acceptable?- Zeroing optimizations for system heaps: DMA Mapping API abuse.- Custom caching IOCTLs vs DMA_BUF_IOCTL_SYNC: Mostly good.- Debug capabilities

msm8998:/ # ls -l /sys/kernel/debug/ion/heaps/

total 0

-rw-rw-r-- 1 root root 0 1970-01-01 00:00 adsp

-rw-rw-r-- 1 root root 0 1970-01-01 00:00 qsecom

-rw-rw-r-- 1 root root 0 1970-01-01 00:00 secure_display

-rw-rw-r-- 1 root root 0 1970-01-01 00:00 secure_heap

-rw-rw-r-- 1 root root 0 1970-01-01 00:00 spss

-rw-rw-r-- 1 root root 0 1970-01-01 00:00 system

msm8998:/ # cat /sys/kernel/debug/ion/heaps/qsecom

client pid size

----------------------------------------------------

keystore 1109 45056

seempd 1571 45056

gatekeeperd 1179 45056

mdtpd 1166 204800

qseecomd 604 4096

qseecomd 604 4096

qseecomd 604 20480

qsee_log 380 32768

qseecom-kernel 1 999424

seempd 1571 204800

seemp_healthd 1149 12288

qseecomd 604 28672

qseecomd 604 20480

qseecomd 604 20480

qseecomd 604 20480

qseecomd 604 8192

qseecomd 604 516096

qseecomd 604 4096

----------------------------------------------------

orphaned allocations (info is from last known client):

----------------------------------------------------

total orphaned 0

total 1236992

----------------------------------------------------

msm8998:/ # ls -l /sys/kernel/debug/ion/clients/

total 0

-rw-rw-r-- 1 root root 0 1970-01-01 01:34 1068-0

-rw-rw-r-- 1 root root 0 1970-01-01 01:34 1075-0

...

...

-rw-rw-r-- 1 root root 0 1970-01-01 01:34 604-0

-rw-rw-r-- 1 root root 0 1970-01-01 01:34 604-1

-rw-rw-r-- 1 root root 0 1970-01-01 01:34 604-2

-rw-rw-r-- 1 root root 0 1970-01-01 01:34 604-3

...

...

-rw-rw-r-- 1 root root 0 1970-01-01 00:00 adsprpc-smd-0

-rw-rw-r-- 1 root root 0 1970-01-01 01:38 asm_client-1

-rw-rw-r-- 1 root root 0 1970-01-01 01:34 audio_cal_client-0

-rw-rw-r-- 1 root root 0 1970-01-01 01:34 audio_cal_client-1

...

...

-rw-rw-r-- 1 root root 0 1970-01-01 00:00 c900000.qcom,mdss_mdp-0

-rw-rw-r-- 1 root root 0 1970-01-01 00:00 c900000.qcom,mdss_rotator-0

-rw-rw-r-- 1 root root 0 1970-01-01 00:00 msm_cam_generic_buf_mgr-0

-rw-rw-r-- 1 root root 0 1970-01-01 00:00 qsee_log-0

-rw-rw-r-- 1 root root 0 1970-01-01 00:00 qseecom-kernel-0

-rw-rw-r-- 1 root root 0 1970-01-01 00:00 spcom-0

-rw-rw-r-- 1 root root 0 1970-01-01 01:34 video_client-1

-rw-rw-r-- 1 root root 0 1970-01-01 01:38 video_client-2

-rw-rw-r-- 1 root root 0 1970-01-01 01:34 voc_cal-0

-rw-rw-r-- 1 root root 0 1970-01-01 01:34 voip_client-0

-rw-rw-r-- 1 root root 0 1970-01-01 01:34 voip_client-1

msm8998:/ # cat /sys/kernel/debug/ion/clients/604-*

heap_name: size_in_bytes : handle refcount : buffer

qsecom: 7000 : 1 : 0


qsecom: 5000 : 1 : 0


qsecom: 2000 : 1 : 0


qsecom: 5000 : 1 : 0


qsecom: 5000 : 1 : 0


qsecom: 7e000 : 1 : 0


qsecom: 1000 : 1 : 0


qsecom: 1000 : 1 : 0


qsecom: 5000 : 1 : 0


qsecom: 1000 : 1 : 0

msm8998:/ # ls -l /proc/604/fd

total 0

lrwx------ 1 root root 64 1970-01-01 01:56 0 -> /dev/null


lrwx------ 1 root root 64 1970-01-01 01:56 10 -> anon_inode:dmabuf

lrwx------ 1 root root 64 1970-01-01 01:56 11 -> /dev/qseecom

lr-x------ 1 root root 64 1970-01-01 01:56 12 -> /dev/ion


lrwx------ 1 root root 64 1970-01-01 01:56 14 -> socket:[19174]

















...

...

...


- Heap instantiation: Are DT bindings for ION acceptable?- Zeroing optimizations for system heaps: DMA Mapping API abuse.- Custom caching IOCTLs vs DMA_BUF_IOCTL_SYNC: Mostly good.- Debug capabilities: Needs to be re-done.


- Heap instantiation: Are DT bindings for ION acceptable?- Zeroing optimizations for system heaps: DMA Mapping API abuse.- Custom caching IOCTLs vs DMA_BUF_IOCTL_SYNC: Mostly good.- Debug capabilities: Needs to be re-done.- Secure heaps.

3. Secure heaps

3. Secure heaps

CP/Access control has evolved with every SoC generation

3. Secure heaps


ION_HEAP_TYPE_CP

ION_HEAP_TYPE_SECURE_DMA

ION_HEAP_TYPE_SYSTEM_SECURE

ION_HEAP_TYPE_HYP_CMA

3. Secure heaps


ION_HEAP_TYPE_CP




Allocation semantics vary for each heap type

ION_FLAG_ALLOW_NON_CONTIG

ION_FLAG_CP_TOUCH

ION_FLAG_CP_BITSTREAM

ION_FLAG_CP_PIXEL

ION_FLAG_CP_CAMERA

…

3. Secure heaps


ION_HEAP_TYPE_CP




ION_FLAG_ALLOW_NON_CONTIG

ION_FLAG_CP_TOUCH

ION_FLAG_CP_BITSTREAM

ION_FLAG_CP_PIXEL

ION_FLAG_CP_CAMERA

…

Additional IOCTLs for some secure heaps:ION_IOC_PREFETCH/DRAIN

Allocation semantics vary for each heap type

4. Virtualized platforms


- Early implementations start by carving out memory.


- Early implementations start by carving out memory.- Hardcoded heap IDs in userspace a problem. Ability to query heaps will be helpful here.


- Early implementations start by carving out memory.- Hardcoded heap IDs in userspace a problem. Ability to query heaps will be helpful here.- A carveout heap type which provides optimized page pools like with system heaps.


- Early implementations start by carving out memory.- Hardcoded heap IDs in userspace a problem. Ability to query heaps will be helpful here.- A carveout heap type which provides optimized page pools like with system heaps.- Memory allocation can vary based on the underlying Hypervisor.


- Early implementations start by carving out memory.- Hardcoded heap IDs in userspace a problem. Ability to query heaps will be helpful here.- A carveout heap type which provides optimized page pools like with system heaps.- Memory allocation can vary based on the underlying Hypervisor.- Conflict with hypervisor based secure heap implementations.

Summary:

- Kernel clients could be removed at a (minor) cost.- How do we instantiate heaps if DT is not an option?- Hardcoded heap IDs in userspace should finally go away.- Cache maintenance using dmabuf looks (mostly) good.- Ion debugging needs to be redone.- Secure heaps can leak out implementation specific details to userspace.- Need to keep things virtualization friendly.

ion integration challengesconnect.linaro.org.s3.amazonaws.com/sfo17/presentations/... ·...

Documents