From fb9bdbb294428e20981dd99af44fc038e8fb9e09 Mon Sep 17 00:00:00 2001 From: Waldemar Brodkorb Date: Tue, 25 Aug 2015 11:40:44 +0200 Subject: update 4.1.x to latest --- .../solidrun-imx6/patches/4.1.3/0001-xbian.patch | 171697 ------------------ .../solidrun-imx6/patches/4.1.6/0001-xbian.patch | 171697 ++++++++++++++++++ 2 files changed, 171697 insertions(+), 171697 deletions(-) delete mode 100644 target/arm/solidrun-imx6/patches/4.1.3/0001-xbian.patch create mode 100644 target/arm/solidrun-imx6/patches/4.1.6/0001-xbian.patch (limited to 'target/arm/solidrun-imx6/patches') diff --git a/target/arm/solidrun-imx6/patches/4.1.3/0001-xbian.patch b/target/arm/solidrun-imx6/patches/4.1.3/0001-xbian.patch deleted file mode 100644 index 6286c0868..000000000 --- a/target/arm/solidrun-imx6/patches/4.1.3/0001-xbian.patch +++ /dev/null @@ -1,171697 +0,0 @@ -diff -Nur linux-4.1.3/arch/arm/boot/dts/imx6dl.dtsi linux-xbian-imx6/arch/arm/boot/dts/imx6dl.dtsi ---- linux-4.1.3/arch/arm/boot/dts/imx6dl.dtsi 2015-07-21 19:10:33.000000000 +0200 -+++ linux-xbian-imx6/arch/arm/boot/dts/imx6dl.dtsi 2015-07-27 23:13:00.299912248 +0200 -@@ -60,17 +60,103 @@ - }; - - soc { -- ocram: sram@00900000 { -+ busfreq { /* BUSFREQ */ -+ compatible = "fsl,imx6_busfreq"; -+ clocks = <&clks 171>, <&clks 6>, <&clks 11>, <&clks 104>, <&clks 172>, <&clks 58>, -+ <&clks 18>, <&clks 60>, <&clks 20>, <&clks 3>, <&clks 22> , <&clks 8>; -+ clock-names = "pll2_bus", "pll2_pfd2_396m", "pll2_198m", "arm", "pll3_usb_otg", "periph", -+ "periph_pre", "periph_clk2", "periph_clk2_sel", "osc", "axi_sel", "pll3_pfd1_540m"; -+ interrupts = <0 107 0x04>, <0 112 0x4>; -+ interrupt-names = "irq_busfreq_0", "irq_busfreq_1"; -+ fsl,max_ddr_freq = <400000000>; -+ }; -+ -+ gpu@00130000 { -+ compatible = "fsl,imx6dl-gpu", "fsl,imx6q-gpu"; -+ reg = <0x00130000 0x4000>, <0x00134000 0x4000>, -+ <0x0 0x0>; -+ reg-names = "iobase_3d", "iobase_2d", -+ "phys_baseaddr"; -+ interrupts = <0 9 0x04>, <0 10 0x04>; -+ interrupt-names = "irq_3d", "irq_2d"; -+ clocks = <&clks 26>, <&clks 27>, -+ <&clks 121>, <&clks 122>, -+ <&clks 74>; -+ clock-names = "gpu2d_axi_clk", "gpu3d_axi_clk", -+ "gpu2d_clk", "gpu3d_clk", -+ "gpu3d_shader_clk"; -+ resets = <&src 0>, <&src 3>; -+ reset-names = "gpu3d", "gpu2d"; -+ power-domains = <&gpc 1>; -+ }; -+ -+ hdmi_core: hdmi_core@00120000 { -+ compatible = "fsl,imx6q-hdmi-core"; -+ reg = <0x00120000 0x9000>; -+ clocks = <&clks 124>, <&clks 123>; -+ clock-names = "hdmi_isfr", "hdmi_iahb"; -+ status = "disabled"; -+ }; -+ -+ hdmi_video: hdmi_video@020e0000 { -+ compatible = "fsl,imx6q-hdmi-video"; -+ reg = <0x020e0000 0x1000>; -+ reg-names = "hdmi_gpr"; -+ interrupts = <0 115 0x04>; -+ clocks = <&clks 124>, <&clks 123>; -+ clock-names = "hdmi_isfr", "hdmi_iahb"; -+ status = "disabled"; -+ }; -+ -+ hdmi_audio: hdmi_audio@00120000 { -+ compatible = "fsl,imx6q-hdmi-audio"; -+ clocks = <&clks 124>, <&clks 123>; -+ clock-names = "hdmi_isfr", "hdmi_iahb"; -+ dmas = <&sdma 2 23 0>; -+ dma-names = "tx"; -+ status = "disabled"; -+ }; -+ -+ hdmi_cec: hdmi_cec@00120000 { -+ compatible = "fsl,imx6q-hdmi-cec"; -+ interrupts = <0 115 0x04>; -+ status = "disabled"; -+ }; -+ -+ ocrams: sram@00900000 { -+ compatible = "fsl,lpm-sram"; -+ reg = <0x00900000 0x4000>; -+ clocks = <&clks IMX6QDL_CLK_OCRAM>; -+ }; -+ -+ ocrams_ddr: sram@00904000 { -+ compatible = "fsl,ddr-lpm-sram"; -+ reg = <0x00904000 0x1000>; -+ clocks = <&clks IMX6QDL_CLK_OCRAM>; -+ }; -+ -+ ocram: sram@00905000 { - compatible = "mmio-sram"; -- reg = <0x00900000 0x20000>; -+ reg = <0x00905000 0x1B000>; - clocks = <&clks IMX6QDL_CLK_OCRAM>; - }; - - aips1: aips-bus@02000000 { -+ vpu@02040000 { -+ iramsize = <0>; -+ status = "okay"; -+ }; -+ - iomuxc: iomuxc@020e0000 { - compatible = "fsl,imx6dl-iomuxc"; - }; - -+ dcic2: dcic@020e8000 { -+ clocks = <&clks IMX6QDL_CLK_DCIC1 >, -+ <&clks IMX6QDL_CLK_DCIC2>; /* DCIC2 depend on DCIC1 clock in imx6dl*/ -+ clock-names = "dcic", "disp-axi"; -+ }; -+ - pxp: pxp@020f0000 { - reg = <0x020f0000 0x4000>; - interrupts = <0 98 IRQ_TYPE_LEVEL_HIGH>; -@@ -99,26 +185,13 @@ - }; - }; - }; -- -- display-subsystem { -- compatible = "fsl,imx-display-subsystem"; -- ports = <&ipu1_di0>, <&ipu1_di1>; -- }; --}; -- --&hdmi { -- compatible = "fsl,imx6dl-hdmi"; - }; - - &ldb { -- clocks = <&clks IMX6QDL_CLK_LDB_DI0_SEL>, <&clks IMX6QDL_CLK_LDB_DI1_SEL>, -- <&clks IMX6QDL_CLK_IPU1_DI0_SEL>, <&clks IMX6QDL_CLK_IPU1_DI1_SEL>, -- <&clks IMX6QDL_CLK_LDB_DI0>, <&clks IMX6QDL_CLK_LDB_DI1>; -+ clocks = <&clks 33>, <&clks 34>, -+ <&clks 39>, <&clks 40>, -+ <&clks 135>, <&clks 136>; - clock-names = "di0_pll", "di1_pll", - "di0_sel", "di1_sel", - "di0", "di1"; - }; -- --&vpu { -- compatible = "fsl,imx6dl-vpu", "cnm,coda960"; --}; -diff -Nur linux-4.1.3/arch/arm/boot/dts/imx6qdl-cubox-i.dtsi linux-xbian-imx6/arch/arm/boot/dts/imx6qdl-cubox-i.dtsi ---- linux-4.1.3/arch/arm/boot/dts/imx6qdl-cubox-i.dtsi 2015-07-21 19:10:33.000000000 +0200 -+++ linux-xbian-imx6/arch/arm/boot/dts/imx6qdl-cubox-i.dtsi 2015-07-27 23:13:00.303898027 +0200 -@@ -45,11 +45,22 @@ - #include - - / { -+ chosen { -+ bootargs = "quiet console=ttymxc0,115200 root=/dev/mmcblk0p2 rw"; -+ }; -+ -+ aliases { -+ mmc0 = &usdhc2; -+ mmc1 = &usdhc1; -+ mxcfb0 = &mxcfb1; -+ }; -+ - ir_recv: ir-receiver { - compatible = "gpio-ir-receiver"; - gpios = <&gpio3 9 1>; - pinctrl-names = "default"; - pinctrl-0 = <&pinctrl_cubox_i_ir>; -+ linux,rc-map-name = "rc-rc6-mce"; - }; - - pwmleds { -@@ -78,6 +89,8 @@ - - reg_usbh1_vbus: usb-h1-vbus { - compatible = "regulator-fixed"; -+ regulator-boot-on; -+ regulator-always-on; - enable-active-high; - gpio = <&gpio1 0 0>; - pinctrl-names = "default"; -@@ -89,6 +102,8 @@ - - reg_usbotg_vbus: usb-otg-vbus { - compatible = "regulator-fixed"; -+ regulator-boot-on; -+ regulator-always-on; - enable-active-high; - gpio = <&gpio3 22 0>; - pinctrl-names = "default"; -@@ -101,8 +116,7 @@ - - sound-spdif { - compatible = "fsl,imx-audio-spdif"; -- model = "Integrated SPDIF"; -- /* IMX6 doesn't implement this yet */ -+ model = "imx-spdif"; - spdif-controller = <&spdif>; - spdif-out; - }; -@@ -118,12 +132,45 @@ - linux,code = ; - }; - }; -+ -+ sound-hdmi { -+ compatible = "fsl,imx6q-audio-hdmi", -+ "fsl,imx-audio-hdmi"; -+ model = "imx-audio-hdmi"; -+ hdmi-controller = <&hdmi_audio>; -+ }; -+ -+ mxcfb1: fb@0 { -+ compatible = "fsl,mxc_sdc_fb"; -+ disp_dev = "hdmi"; -+ interface_pix_fmt = "RGB24"; -+ mode_str ="1920x1080M@60"; -+ default_bpp = <32>; -+ int_clk = <0>; -+ late_init = <0>; -+ status = "okay"; -+ }; -+}; -+ -+&hdmi_core { -+ ipu_id = <0>; -+ disp_id = <0>; -+ status = "okay"; -+}; -+ -+&hdmi_video { -+ fsl,phy_reg_vlev = <0x0294>; -+ fsl,phy_reg_cksymtx = <0x800d>; -+ status = "okay"; -+}; -+ -+&hdmi_audio { -+ status = "okay"; - }; - --&hdmi { -+&hdmi_cec { - pinctrl-names = "default"; - pinctrl-0 = <&pinctrl_cubox_i_hdmi>; -- ddc-i2c-bus = <&i2c2>; - status = "okay"; - }; - -@@ -131,7 +178,13 @@ - clock-frequency = <100000>; - pinctrl-names = "default"; - pinctrl-0 = <&pinctrl_cubox_i_i2c2>; -+ - status = "okay"; -+ -+ ddc: imx6_hdmi_i2c@50 { -+ compatible = "fsl,imx6-hdmi-i2c"; -+ reg = <0x50>; -+ }; - }; - - &i2c3 { -@@ -228,6 +281,28 @@ - MX6QDL_PAD_EIM_DA8__GPIO3_IO08 0x17059 - >; - }; -+ -+ pinctrl_cubox_i_usdhc2_100mhz: cubox-i-usdhc2-100mhz { -+ fsl,pins = < -+ MX6QDL_PAD_SD2_CMD__SD2_CMD 0x170b9 -+ MX6QDL_PAD_SD2_CLK__SD2_CLK 0x100b9 -+ MX6QDL_PAD_SD2_DAT0__SD2_DATA0 0x170b9 -+ MX6QDL_PAD_SD2_DAT1__SD2_DATA1 0x170b9 -+ MX6QDL_PAD_SD2_DAT2__SD2_DATA2 0x170b9 -+ MX6QDL_PAD_SD2_DAT3__SD2_DATA3 0x130b9 -+ >; -+ }; -+ -+ pinctrl_cubox_i_usdhc2_200mhz: cubox-i-usdhc2-200mhz { -+ fsl,pins = < -+ MX6QDL_PAD_SD2_CMD__SD2_CMD 0x170f9 -+ MX6QDL_PAD_SD2_CLK__SD2_CLK 0x100f9 -+ MX6QDL_PAD_SD2_DAT0__SD2_DATA0 0x170f9 -+ MX6QDL_PAD_SD2_DAT1__SD2_DATA1 0x170f9 -+ MX6QDL_PAD_SD2_DAT2__SD2_DATA2 0x170f9 -+ MX6QDL_PAD_SD2_DAT3__SD2_DATA3 0x130f9 -+ >; -+ }; - }; - }; - -@@ -256,9 +331,24 @@ - }; - - &usdhc2 { -- pinctrl-names = "default"; -+ pinctrl-names = "default", "state_100mhz", "state_200mhz"; - pinctrl-0 = <&pinctrl_cubox_i_usdhc2_aux &pinctrl_cubox_i_usdhc2>; -+ pinctrl-1 = <&pinctrl_cubox_i_usdhc2_aux &pinctrl_cubox_i_usdhc2_100mhz>; -+ pinctrl-2 = <&pinctrl_cubox_i_usdhc2_aux &pinctrl_cubox_i_usdhc2_200mhz>; - vmmc-supply = <®_3p3v>; - cd-gpios = <&gpio1 4 0>; - status = "okay"; -+ no-1-8-v; -+}; -+ -+&dcic1 { -+ dcic_id = <0>; -+ dcic_mux = "dcic-hdmi"; -+ status = "okay"; -+}; -+ -+&dcic2 { -+ dcic_id = <1>; -+ dcic_mux = "dcic-lvds1"; -+ status = "okay"; - }; -diff -Nur linux-4.1.3/arch/arm/boot/dts/imx6qdl.dtsi linux-xbian-imx6/arch/arm/boot/dts/imx6qdl.dtsi ---- linux-4.1.3/arch/arm/boot/dts/imx6qdl.dtsi 2015-07-21 19:10:33.000000000 +0200 -+++ linux-xbian-imx6/arch/arm/boot/dts/imx6qdl.dtsi 2015-07-27 23:13:00.303898027 +0200 -@@ -14,6 +14,7 @@ - #include - - #include "skeleton.dtsi" -+#include - - / { - aliases { -@@ -30,6 +31,7 @@ - i2c0 = &i2c1; - i2c1 = &i2c2; - i2c2 = &i2c3; -+ ipu0 = &ipu1; - mmc0 = &usdhc1; - mmc1 = &usdhc2; - mmc2 = &usdhc3; -@@ -79,6 +81,10 @@ - }; - }; - -+ pu_dummy: pudummy_reg { -+ compatible = "fsl,imx6-dummy-pureg"; /* only used in ldo-bypass */ -+ }; -+ - soc { - #address-cells = <1>; - #size-cells = <1>; -@@ -86,6 +92,11 @@ - interrupt-parent = <&gpc>; - ranges; - -+ caam_sm: caam-sm@00100000 { -+ compatible = "fsl,imx6q-caam-sm"; -+ reg = <0x00100000 0x3fff>; -+ }; -+ - dma_apbh: dma-apbh@00110000 { - compatible = "fsl,imx6q-dma-apbh", "fsl,imx28-dma-apbh"; - reg = <0x00110000 0x2000>; -@@ -99,6 +110,12 @@ - clocks = <&clks IMX6QDL_CLK_APBH_DMA>; - }; - -+ irq_sec_vio: caam_secvio { -+ compatible = "fsl,imx6q-caam-secvio"; -+ interrupts = <0 20 0x04>; -+ secvio_src = <0x8000001d>; -+ }; -+ - gpmi: gpmi-nand@00112000 { - compatible = "fsl,imx6q-gpmi-nand"; - #address-cells = <1>; -@@ -190,16 +207,16 @@ - dmas = <&sdma 14 18 0>, - <&sdma 15 18 0>; - dma-names = "rx", "tx"; -- clocks = <&clks IMX6QDL_CLK_SPDIF>, <&clks IMX6QDL_CLK_OSC>, -- <&clks IMX6QDL_CLK_SPDIF>, <&clks IMX6QDL_CLK_DUMMY>, -- <&clks IMX6QDL_CLK_DUMMY>, <&clks IMX6QDL_CLK_DUMMY>, -- <&clks IMX6QDL_CLK_DUMMY>, <&clks IMX6QDL_CLK_DUMMY>, -- <&clks IMX6QDL_CLK_DUMMY>; -+ clocks = <&clks IMX6QDL_CLK_SPDIF_GCLK>, <&clks IMX6QDL_CLK_OSC>, -+ <&clks IMX6QDL_CLK_SPDIF>, <&clks IMX6QDL_CLK_ASRC>, -+ <&clks IMX6QDL_CLK_DUMMY>, <&clks IMX6QDL_CLK_ESAI_EXTAL>, -+ <&clks IMX6QDL_CLK_IPG>, <&clks IMX6QDL_CLK_MLB>, -+ <&clks IMX6QDL_CLK_DUMMY>, <&clks IMX6QDL_CLK_SPBA>; - clock-names = "core", "rxtx0", - "rxtx1", "rxtx2", - "rxtx3", "rxtx4", - "rxtx5", "rxtx6", -- "rxtx7"; -+ "rxtx7", "dma"; - status = "disabled"; - }; - -@@ -274,7 +291,12 @@ - esai: esai@02024000 { - reg = <0x02024000 0x4000>; - interrupts = <0 51 IRQ_TYPE_LEVEL_HIGH>; -- }; -+ compatible = "fsl,imx6q-esai"; -+ clocks = <&clks 118>; -+ fsl,esai-dma-events = <24 23>; -+ fsl,flags = <1>; -+ status = "disabled"; -+ }; - - ssi1: ssi@02028000 { - #sound-dai-cells = <0>; -@@ -325,8 +347,30 @@ - }; - - asrc: asrc@02034000 { -+ compatible = "fsl,imx53-asrc"; - reg = <0x02034000 0x4000>; - interrupts = <0 50 IRQ_TYPE_LEVEL_HIGH>; -+ clocks = <&clks IMX6QDL_CLK_ASRC_MEM>, -+ <&clks IMX6QDL_CLK_ASRC_IPG>, -+ <&clks IMX6QDL_CLK_SPDIF>, -+ <&clks IMX6QDL_CLK_SPBA>; -+ clock-names = "mem", "ipg", "asrck_0", "dma"; -+ dmas = <&sdma 17 20 1>, <&sdma 18 20 1>, <&sdma 19 20 1>, -+ <&sdma 20 20 1>, <&sdma 21 20 1>, <&sdma 22 20 1>; -+ dma-names = "rxa", "rxb", "rxc", -+ "txa", "txb", "txc"; -+ fsl,asrc-rate = <48000>; -+ fsl,asrc-width = <16>; -+ status = "okay"; -+ }; -+ -+ asrc_p2p: asrc_p2p { -+ compatible = "fsl,imx6q-asrc-p2p"; -+ fsl,output-rate = <48000>; -+ fsl,output-width = <16>; -+ fsl,asrc-dma-rx-events = <17 18 19>; -+ fsl,asrc-dma-tx-events = <20 21 22>; -+ status = "okay"; - }; - - spba@0203c000 { -@@ -335,16 +379,20 @@ - }; - - vpu: vpu@02040000 { -- compatible = "cnm,coda960"; -+ compatible = "cnm,coda960", "fsl,imx6-vpu"; - reg = <0x02040000 0x3c000>; -+ reg-names = "vpu_regs"; - interrupts = <0 12 IRQ_TYPE_LEVEL_HIGH>, - <0 3 IRQ_TYPE_LEVEL_HIGH>; - interrupt-names = "bit", "jpeg"; - clocks = <&clks IMX6QDL_CLK_VPU_AXI>, -- <&clks IMX6QDL_CLK_MMDC_CH0_AXI>; -- clock-names = "per", "ahb"; -- resets = <&src 1>; -+ <&clks IMX6QDL_CLK_MMDC_CH0_AXI>, -+ <&clks IMX6QDL_CLK_OCRAM>; -+ clock-names = "per", "ahb", "ocram"; -+ iramsize = <0x21000>; - iram = <&ocram>; -+ resets = <&src 1>; -+ power-domains = <&gpc 1>; - }; - - aipstz@0207c000 { /* AIPSTZ1 */ -@@ -552,20 +600,21 @@ - anatop-min-bit-val = <4>; - anatop-min-voltage = <800000>; - anatop-max-voltage = <1375000>; -+ anatop-enable-bit = <0>; - }; - -- regulator-3p0@120 { -+ reg_3p0: regulator-3p0@120 { - compatible = "fsl,anatop-regulator"; - regulator-name = "vdd3p0"; -- regulator-min-microvolt = <2800000>; -- regulator-max-microvolt = <3150000>; -- regulator-always-on; -+ regulator-min-microvolt = <2625000>; -+ regulator-max-microvolt = <3400000>; - anatop-reg-offset = <0x120>; - anatop-vol-bit-shift = <8>; - anatop-vol-bit-width = <5>; - anatop-min-bit-val = <0>; - anatop-min-voltage = <2625000>; - anatop-max-voltage = <3400000>; -+ anatop-enable-bit = <0>; - }; - - regulator-2p5@130 { -@@ -580,6 +629,7 @@ - anatop-min-bit-val = <0>; - anatop-min-voltage = <2000000>; - anatop-max-voltage = <2750000>; -+ anatop-enable-bit = <0>; - }; - - reg_arm: regulator-vddcore@140 { -@@ -647,6 +697,7 @@ - reg = <0x020c9000 0x1000>; - interrupts = <0 44 IRQ_TYPE_LEVEL_HIGH>; - clocks = <&clks IMX6QDL_CLK_USBPHY1>; -+ phy-3p0-supply = <®_3p0>; - fsl,anatop = <&anatop>; - }; - -@@ -655,9 +706,15 @@ - reg = <0x020ca000 0x1000>; - interrupts = <0 45 IRQ_TYPE_LEVEL_HIGH>; - clocks = <&clks IMX6QDL_CLK_USBPHY2>; -+ phy-3p0-supply = <®_3p0>; - fsl,anatop = <&anatop>; - }; - -+ caam_snvs: caam-snvs@020cc000 { -+ compatible = "fsl,imx6q-caam-snvs"; -+ reg = <0x020cc000 0x4000>; -+ }; -+ - snvs@020cc000 { - compatible = "fsl,sec-v4.0-mon", "simple-bus"; - #address-cells = <1>; -@@ -704,14 +761,12 @@ - interrupts = <0 89 IRQ_TYPE_LEVEL_HIGH>, - <0 90 IRQ_TYPE_LEVEL_HIGH>; - interrupt-parent = <&intc>; -- pu-supply = <®_pu>; -- clocks = <&clks IMX6QDL_CLK_GPU3D_CORE>, -- <&clks IMX6QDL_CLK_GPU3D_SHADER>, -- <&clks IMX6QDL_CLK_GPU2D_CORE>, -- <&clks IMX6QDL_CLK_GPU2D_AXI>, -- <&clks IMX6QDL_CLK_OPENVG_AXI>, -- <&clks IMX6QDL_CLK_VPU_AXI>; - #power-domain-cells = <1>; -+ clocks = <&clks 122>, <&clks 74>, <&clks 121>, -+ <&clks 26>, <&clks 143>, <&clks 168>; -+ clock-names = "gpu3d_core", "gpu3d_shader", "gpu2d_core", -+ "gpu2d_axi", "openvg_axi", "vpu_axi"; -+ pu-supply = <®_pu>; - }; - - gpr: iomuxc-gpr@020e0000 { -@@ -736,22 +791,6 @@ - #size-cells = <0>; - reg = <0>; - status = "disabled"; -- -- port@0 { -- reg = <0>; -- -- lvds0_mux_0: endpoint { -- remote-endpoint = <&ipu1_di0_lvds0>; -- }; -- }; -- -- port@1 { -- reg = <1>; -- -- lvds0_mux_1: endpoint { -- remote-endpoint = <&ipu1_di1_lvds0>; -- }; -- }; - }; - - lvds-channel@1 { -@@ -759,22 +798,6 @@ - #size-cells = <0>; - reg = <1>; - status = "disabled"; -- -- port@0 { -- reg = <0>; -- -- lvds1_mux_0: endpoint { -- remote-endpoint = <&ipu1_di0_lvds1>; -- }; -- }; -- -- port@1 { -- reg = <1>; -- -- lvds1_mux_1: endpoint { -- remote-endpoint = <&ipu1_di1_lvds1>; -- }; -- }; - }; - }; - -@@ -788,32 +811,26 @@ - <&clks IMX6QDL_CLK_HDMI_ISFR>; - clock-names = "iahb", "isfr"; - status = "disabled"; -- -- port@0 { -- reg = <0>; -- -- hdmi_mux_0: endpoint { -- remote-endpoint = <&ipu1_di0_hdmi>; -- }; -- }; -- -- port@1 { -- reg = <1>; -- -- hdmi_mux_1: endpoint { -- remote-endpoint = <&ipu1_di1_hdmi>; -- }; -- }; - }; - - dcic1: dcic@020e4000 { -+ compatible = "fsl,imx6q-dcic"; - reg = <0x020e4000 0x4000>; - interrupts = <0 124 IRQ_TYPE_LEVEL_HIGH>; -+ clocks = <&clks IMX6QDL_CLK_DCIC1>, <&clks IMX6QDL_CLK_DCIC1>; -+ clock-names = "dcic", "disp-axi"; -+ gpr = <&gpr>; -+ status = "disabled"; - }; - - dcic2: dcic@020e8000 { -+ compatible = "fsl,imx6q-dcic"; - reg = <0x020e8000 0x4000>; - interrupts = <0 125 IRQ_TYPE_LEVEL_HIGH>; -+ clocks = <&clks IMX6QDL_CLK_DCIC2>, <&clks IMX6QDL_CLK_DCIC2>; -+ clock-names = "dcic", "disp-axi"; -+ gpr = <&gpr>; -+ status = "disabled"; - }; - - sdma: sdma@020ec000 { -@@ -824,6 +841,7 @@ - <&clks IMX6QDL_CLK_SDMA>; - clock-names = "ipg", "ahb"; - #dma-cells = <3>; -+ iram = <&ocram>; - fsl,sdma-ram-script-name = "imx/sdma/sdma-imx6q.bin"; - }; - }; -@@ -835,10 +853,30 @@ - reg = <0x02100000 0x100000>; - ranges; - -- caam@02100000 { -- reg = <0x02100000 0x40000>; -- interrupts = <0 105 IRQ_TYPE_LEVEL_HIGH>, -- <0 106 IRQ_TYPE_LEVEL_HIGH>; -+ crypto: caam@2100000 { -+ compatible = "fsl,sec-v4.0"; -+ #address-cells = <1>; -+ #size-cells = <1>; -+ reg = <0x2100000 0x40000>; -+ ranges = <0 0x2100000 0x40000>; -+ interrupt-parent = <&intc>; /* interrupts = <0 92 0x4>; */ -+ interrupts = <0 92 0x4>; -+ clocks = <&clks 213>, <&clks 214>, <&clks 215> ,<&clks 196>; -+ clock-names = "caam_mem", "caam_aclk", "caam_ipg", "caam_emi_slow"; -+ -+ sec_jr0: jr0@1000 { -+ compatible = "fsl,sec-v4.0-job-ring"; -+ reg = <0x1000 0x1000>; -+ interrupt-parent = <&intc>; -+ interrupts = <0 105 IRQ_TYPE_LEVEL_HIGH>; -+ }; -+ -+ sec_jr1: jr1@2000 { -+ compatible = "fsl,sec-v4.0-job-ring"; -+ reg = <0x2000 0x1000>; -+ interrupt-parent = <&intc>; -+ interrupts = <0 106 IRQ_TYPE_LEVEL_HIGH>; -+ }; - }; - - aipstz@0217c000 { /* AIPSTZ2 */ -@@ -852,6 +890,7 @@ - clocks = <&clks IMX6QDL_CLK_USBOH3>; - fsl,usbphy = <&usbphy1>; - fsl,usbmisc = <&usbmisc 0>; -+ fsl,anatop = <&anatop>; - status = "disabled"; - }; - -@@ -903,14 +942,21 @@ - <&clks IMX6QDL_CLK_ENET>, - <&clks IMX6QDL_CLK_ENET_REF>; - clock-names = "ipg", "ahb", "ptp"; -- status = "disabled"; -+ phy-mode = "rgmii"; -+ fsl,magic-packet; -+ status = "okay"; - }; - -- mlb@0218c000 { -+ mlb: mlb@0218c000 { - reg = <0x0218c000 0x4000>; - interrupts = <0 53 IRQ_TYPE_LEVEL_HIGH>, - <0 117 IRQ_TYPE_LEVEL_HIGH>, - <0 126 IRQ_TYPE_LEVEL_HIGH>; -+ compatible = "fsl,imx6q-mlb150"; -+ clocks = <&clks 139>, <&clks 175>; -+ clock-names = "mlb", "pll8_mlb"; -+ iram = <&ocram>; -+ status = "disabled"; - }; - - usdhc1: usdhc@02190000 { -@@ -995,6 +1041,11 @@ - reg = <0x021ac000 0x4000>; - }; - -+ mmdc0-1@021b0000 { -+ compatible = "fsl,imx6q-mmdc-combine"; -+ reg = <0x021b0000 0x8000>; -+ }; -+ - mmdc0: mmdc@021b0000 { /* MMDC0 */ - compatible = "fsl,imx6q-mmdc"; - reg = <0x021b0000 0x4000>; -@@ -1011,11 +1062,17 @@ - clocks = <&clks IMX6QDL_CLK_EIM_SLOW>; - }; - -- ocotp: ocotp@021bc000 { -- compatible = "fsl,imx6q-ocotp", "syscon"; -+ ocotp: ocotp-ctrl@021bc000 { -+ compatible = "syscon"; - reg = <0x021bc000 0x4000>; - }; - -+ ocotp-fuse@021bc000 { -+ compatible = "fsl,imx6q-ocotp"; -+ reg = <0x021bc000 0x4000>; -+ clocks = <&clks 128>; -+ }; -+ - tzasc@021d0000 { /* TZASC1 */ - reg = <0x021d0000 0x4000>; - interrupts = <0 108 IRQ_TYPE_LEVEL_HIGH>; -@@ -1034,39 +1091,38 @@ - - mipi_csi: mipi@021dc000 { - reg = <0x021dc000 0x4000>; -+ compatible = "fsl,imx6q-mipi-csi2"; -+ interrupts = <0 100 0x04>, <0 101 0x04>; -+ clocks = <&clks IMX6QDL_CLK_HSI_TX>, -+ <&clks IMX6QDL_CLK_EIM_SEL>, -+ <&clks IMX6QDL_CLK_LVDS2_IN>; -+ /* Note: clks 138 is hsi_tx, however, the dphy_c -+ * hsi_tx and pll_refclk use the same clk gate. -+ * In current clk driver, open/close clk gate do -+ * use hsi_tx for a temporary debug purpose. -+ */ -+ clock-names = "dphy_clk", "pixel_clk", "cfg_clk"; -+ status = "disabled"; - }; - - mipi_dsi: mipi@021e0000 { -+ compatible = "fsl,imx6q-mipi-dsi"; - #address-cells = <1>; - #size-cells = <0>; - reg = <0x021e0000 0x4000>; - status = "disabled"; -- -- ports { -- #address-cells = <1>; -- #size-cells = <0>; -- -- port@0 { -- reg = <0>; -- -- mipi_mux_0: endpoint { -- remote-endpoint = <&ipu1_di0_mipi>; -- }; -- }; -- -- port@1 { -- reg = <1>; -- -- mipi_mux_1: endpoint { -- remote-endpoint = <&ipu1_di1_mipi>; -- }; -- }; -- }; -+ interrupts = <0 102 0x04>; -+ gpr = <&gpr>; -+ clocks = <&clks IMX6QDL_CLK_HSI_TX>, <&clks IMX6QDL_CLK_VIDEO_27M>; -+ clock-names = "mipi_pllref_clk", "mipi_cfg_clk"; - }; - - vdoa@021e4000 { -+ compatible = "fsl,imx6q-vdoa"; - reg = <0x021e4000 0x4000>; - interrupts = <0 18 IRQ_TYPE_LEVEL_HIGH>; -+ clocks = <&clks 202>; -+ iram = <&ocram>; - }; - - uart2: serial@021e8000 { -@@ -1127,67 +1183,14 @@ - <0 5 IRQ_TYPE_LEVEL_HIGH>; - clocks = <&clks IMX6QDL_CLK_IPU1>, - <&clks IMX6QDL_CLK_IPU1_DI0>, -- <&clks IMX6QDL_CLK_IPU1_DI1>; -- clock-names = "bus", "di0", "di1"; -+ <&clks IMX6QDL_CLK_IPU1_DI1>, -+ <&clks 39>, <&clks 40>, -+ <&clks 135>, <&clks 136>; -+ clock-names = "bus", "di0", "di1", -+ "di0_sel", "di1_sel", -+ "ldb_di0", "ldb_di1"; - resets = <&src 2>; -- -- ipu1_csi0: port@0 { -- reg = <0>; -- }; -- -- ipu1_csi1: port@1 { -- reg = <1>; -- }; -- -- ipu1_di0: port@2 { -- #address-cells = <1>; -- #size-cells = <0>; -- reg = <2>; -- -- ipu1_di0_disp0: endpoint@0 { -- }; -- -- ipu1_di0_hdmi: endpoint@1 { -- remote-endpoint = <&hdmi_mux_0>; -- }; -- -- ipu1_di0_mipi: endpoint@2 { -- remote-endpoint = <&mipi_mux_0>; -- }; -- -- ipu1_di0_lvds0: endpoint@3 { -- remote-endpoint = <&lvds0_mux_0>; -- }; -- -- ipu1_di0_lvds1: endpoint@4 { -- remote-endpoint = <&lvds1_mux_0>; -- }; -- }; -- -- ipu1_di1: port@3 { -- #address-cells = <1>; -- #size-cells = <0>; -- reg = <3>; -- -- ipu1_di0_disp1: endpoint@0 { -- }; -- -- ipu1_di1_hdmi: endpoint@1 { -- remote-endpoint = <&hdmi_mux_1>; -- }; -- -- ipu1_di1_mipi: endpoint@2 { -- remote-endpoint = <&mipi_mux_1>; -- }; -- -- ipu1_di1_lvds0: endpoint@3 { -- remote-endpoint = <&lvds0_mux_1>; -- }; -- -- ipu1_di1_lvds1: endpoint@4 { -- remote-endpoint = <&lvds1_mux_1>; -- }; -- }; -+ bypass_reset = <0>; - }; - }; - }; -diff -Nur linux-4.1.3/arch/arm/boot/dts/imx6qdl-hummingboard.dtsi linux-xbian-imx6/arch/arm/boot/dts/imx6qdl-hummingboard.dtsi ---- linux-4.1.3/arch/arm/boot/dts/imx6qdl-hummingboard.dtsi 2015-07-21 19:10:33.000000000 +0200 -+++ linux-xbian-imx6/arch/arm/boot/dts/imx6qdl-hummingboard.dtsi 2015-07-27 23:13:00.303898027 +0200 -@@ -43,8 +43,10 @@ - #include "imx6qdl-microsom-ar8035.dtsi" - - / { -- chosen { -- stdout-path = &uart1; -+ aliases { -+ mmc0 = &usdhc2; -+ mmc1 = &usdhc1; -+ mxcfb0 = &mxcfb1; - }; - - ir_recv: ir-receiver { -@@ -52,6 +54,7 @@ - gpios = <&gpio3 5 1>; - pinctrl-names = "default"; - pinctrl-0 = <&pinctrl_hummingboard_gpio3_5>; -+ linux,rc-map-name = "rc-rc6-mce"; - }; - - regulators { -@@ -98,32 +101,70 @@ - model = "On-board Codec"; - mux-ext-port = <5>; - mux-int-port = <1>; -+ cpu-dai = <&ssi1>; - ssi-controller = <&ssi1>; - }; - - sound-spdif { - compatible = "fsl,imx-audio-spdif"; -- model = "On-board SPDIF"; -+ model = "imx-spdif"; - /* IMX6 doesn't implement this yet */ - spdif-controller = <&spdif>; - spdif-out; - }; -+ -+ sound-hdmi { -+ compatible = "fsl,imx6q-audio-hdmi", -+ "fsl,imx-audio-hdmi"; -+ model = "imx-audio-hdmi"; -+ hdmi-controller = <&hdmi_audio>; -+ }; -+ -+ mxcfb1: fb@0 { -+ compatible = "fsl,mxc_sdc_fb"; -+ disp_dev = "hdmi"; -+ interface_pix_fmt = "RGB24"; -+ mode_str ="1920x1080M@60"; -+ default_bpp = <32>; -+ int_clk = <0>; -+ late_init = <0>; -+ status = "okay"; -+ }; - }; - - &audmux { - status = "okay"; - }; - --&can1 { -+/*&can1 { - pinctrl-names = "default"; - pinctrl-0 = <&pinctrl_hummingboard_flexcan1>; - status = "okay"; - }; -+*/ -+&hdmi_core { -+ ipu_id = <0>; -+ disp_id = <0>; -+ status = "okay"; -+}; -+ -+&hdmi_video { -+ fsl,phy_reg_vlev = <0x0294>; -+ fsl,phy_reg_cksymtx = <0x800d>; -+ status = "okay"; -+}; -+ -+&hdmi_audio { -+ status = "okay"; -+}; -+ -+&ocram { -+ status = "okay"; -+}; - --&hdmi { -+&hdmi_cec { - pinctrl-names = "default"; - pinctrl-0 = <&pinctrl_hummingboard_hdmi>; -- ddc-i2c-bus = <&i2c2>; - status = "okay"; - }; - -@@ -136,6 +177,7 @@ - rtc: pcf8523@68 { - compatible = "nxp,pcf8523"; - reg = <0x68>; -+ nxp,12p5_pf; - }; - - /* Pro baseboard model */ -@@ -155,20 +197,57 @@ - pinctrl-names = "default"; - pinctrl-0 = <&pinctrl_hummingboard_i2c2>; - status = "okay"; -+ -+ ddc: imx6_hdmi_i2c@50 { -+ compatible = "fsl,imx6-hdmi-i2c"; -+ reg = <0x50>; -+ }; - }; - - &iomuxc { -+ pinctrl-names = "default"; -+ pinctrl-0 = <&pinctrl_hog>; - hummingboard { -- pinctrl_hummingboard_flexcan1: hummingboard-flexcan1 { -+ pinctrl_hog: hoggrp { -+ fsl,pins = < -+ /* -+ * 26 pin header GPIO description. The pins. -+ * numbering as following - -+ * GPIO number | GPIO (bank,num) | PIN number -+ * ------------+-----------------+------------ -+ * gpio1 | (1,1) | IO7 -+ * gpio73 | (3,9) | IO11 -+ * gpio72 | (3,8) | IO12 -+ * gpio71 | (3,7) | IO13 -+ * gpio70 | (3,6) | IO15 -+ * gpio194 | (7,2) | IO16 -+ * gpio195 | (7,3) | IO18 -+ * gpio67 | (3,3) | IO22 -+ * -+ * Notice the gpioX and GPIO (Y,Z) mapping forumla : -+ * X = (Y-1) * 32 + Z -+ */ -+ MX6QDL_PAD_GPIO_1__GPIO1_IO01 0x400130b1 -+ MX6QDL_PAD_EIM_DA9__GPIO3_IO09 0x400130b1 -+ MX6QDL_PAD_EIM_DA8__GPIO3_IO08 0x400130b1 -+ MX6QDL_PAD_EIM_DA7__GPIO3_IO07 0x400130b1 -+ MX6QDL_PAD_EIM_DA6__GPIO3_IO06 0x400130b1 -+ MX6QDL_PAD_SD3_CMD__GPIO7_IO02 0x400130b1 -+ MX6QDL_PAD_SD3_CLK__GPIO7_IO03 0x400130b1 -+ MX6QDL_PAD_EIM_DA3__GPIO3_IO03 0x400130b1 -+ >; -+ }; -+ -+/* pinctrl_hummingboard_flexcan1: hummingboard-flexcan1 { - fsl,pins = < - MX6QDL_PAD_SD3_CLK__FLEXCAN1_RX 0x80000000 - MX6QDL_PAD_SD3_CMD__FLEXCAN1_TX 0x80000000 - >; - }; -- -+*/ - pinctrl_hummingboard_gpio3_5: hummingboard-gpio3_5 { - fsl,pins = < -- MX6QDL_PAD_EIM_DA5__GPIO3_IO05 0x1b0b1 -+ MX6QDL_PAD_EIM_DA5__GPIO3_IO05 0x80000000 - >; - }; - -@@ -198,10 +277,10 @@ - - pinctrl_hummingboard_sgtl5000: hummingboard-sgtl5000 { - fsl,pins = < -- MX6QDL_PAD_DISP0_DAT19__AUD5_RXD 0x130b0 -- MX6QDL_PAD_KEY_COL0__AUD5_TXC 0x130b0 -- MX6QDL_PAD_KEY_ROW0__AUD5_TXD 0x110b0 -- MX6QDL_PAD_KEY_COL1__AUD5_TXFS 0x130b0 -+ MX6QDL_PAD_DISP0_DAT19__AUD5_RXD 0x130b0 /*brk*/ -+ MX6QDL_PAD_KEY_COL0__AUD5_TXC 0x130b0 /*ok*/ -+ MX6QDL_PAD_KEY_ROW0__AUD5_TXD 0x110b0 /*brk*/ -+ MX6QDL_PAD_KEY_COL1__AUD5_TXFS 0x130b0 /*ok*/ - MX6QDL_PAD_GPIO_5__CCM_CLKO1 0x130b0 - >; - }; -@@ -219,7 +298,7 @@ - * Similar to pinctrl_usbotg_2, but we want it - * pulled down for a fixed host connection. - */ -- fsl,pins = ; -+ fsl,pins = ; - }; - - pinctrl_hummingboard_usbotg_vbus: hummingboard-usbotg-vbus { -@@ -242,6 +321,13 @@ - MX6QDL_PAD_SD2_DAT3__SD2_DATA3 0x13059 - >; - }; -+ -+ pinctrl_hummingboard_pcie_reset: hummingboard-pcie-reset { -+ fsl,pins = < -+ MX6QDL_PAD_EIM_DA4__GPIO3_IO04 0x80000000 -+ >; -+ }; -+ - }; - }; - -@@ -256,6 +342,14 @@ - status = "okay"; - }; - -+&pwm3 { -+ status = "disabled"; -+}; -+ -+&pwm4 { -+ status = "disabled"; -+}; -+ - &spdif { - pinctrl-names = "default"; - pinctrl-0 = <&pinctrl_hummingboard_spdif>; -@@ -291,3 +385,48 @@ - cd-gpios = <&gpio1 4 0>; - status = "okay"; - }; -+ -+&gpc { -+ fsl,cpu_pupscr_sw2iso = <0xf>; -+ fsl,cpu_pupscr_sw = <0xf>; -+ fsl,cpu_pdnscr_iso2sw = <0x1>; -+ fsl,cpu_pdnscr_iso = <0x1>; -+ status = "okay"; -+}; -+ -+&pcie { -+ pinctrl-names = "default"; -+ pinctrl-0 = < -+ &pinctrl_hummingboard_pcie_reset -+ >; -+ reset-gpio = <&gpio3 4 0>; -+ status = "okay"; -+ no-msi; -+}; -+ -+&ecspi1 { -+ status = "okay"; -+ fsl,spi-num-chipselects = <1>; -+}; -+ -+&ecspi2 { -+ status = "okay"; -+ fsl,spi-num-chipselects = <2>; -+}; -+ -+&ecspi3 { -+ status = "okay"; -+ fsl,spi-num-chipselects = <3>; -+}; -+ -+&dcic1 { -+ dcic_id = <0>; -+ dcic_mux = "dcic-hdmi"; -+ status = "okay"; -+}; -+ -+&dcic2 { -+ dcic_id = <1>; -+ dcic_mux = "dcic-lvds1"; -+ status = "okay"; -+}; -diff -Nur linux-4.1.3/arch/arm/boot/dts/imx6qdl-microsom.dtsi linux-xbian-imx6/arch/arm/boot/dts/imx6qdl-microsom.dtsi ---- linux-4.1.3/arch/arm/boot/dts/imx6qdl-microsom.dtsi 2015-07-21 19:10:33.000000000 +0200 -+++ linux-xbian-imx6/arch/arm/boot/dts/imx6qdl-microsom.dtsi 2015-07-27 23:13:00.303898027 +0200 -@@ -39,15 +39,98 @@ - * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR - * OTHER DEALINGS IN THE SOFTWARE. - */ -+#include -+/ { -+ clk_sdio: sdio-clock { -+ compatible = "gpio-gate-clock"; -+ #clock-cells = <0>; -+ pinctrl-names = "default"; -+ pinctrl-0 = <&pinctrl_microsom_brcm_osc>; -+ enable-gpios = <&gpio5 5 GPIO_ACTIVE_HIGH>; -+ }; -+ -+ regulators { -+ compatible = "simple-bus"; -+ -+ reg_brcm: brcm-reg { -+ compatible = "regulator-fixed"; -+ enable-active-high; -+ gpio = <&gpio3 19 0>; -+ pinctrl-names = "default"; -+ pinctrl-0 = <&pinctrl_microsom_brcm_reg>; -+ regulator-name = "brcm_reg"; -+ regulator-min-microvolt = <3300000>; -+ regulator-max-microvolt = <3300000>; -+ startup-delay-us = <200000>; -+ }; -+ }; -+ -+ usdhc1_pwrseq: usdhc1_pwrseq { -+ compatible = "mmc-pwrseq-simple"; -+ reset-gpios = <&gpio5 26 GPIO_ACTIVE_LOW>, -+ <&gpio6 0 GPIO_ACTIVE_LOW>; -+ clocks = <&clk_sdio>; -+ clock-names = "ext_clock"; -+ }; -+}; - - &iomuxc { - microsom { -+ pinctrl_microsom_brcm_bt: microsom-brcm-bt { -+ fsl,pins = < -+ MX6QDL_PAD_CSI0_DAT14__GPIO6_IO00 0x40013070 -+ MX6QDL_PAD_CSI0_DAT15__GPIO6_IO01 0x40013070 -+ MX6QDL_PAD_CSI0_DAT18__GPIO6_IO04 0x40013070 -+ >; -+ }; -+ -+ pinctrl_microsom_brcm_osc: microsom-brcm-osc { -+ fsl,pins = < -+ MX6QDL_PAD_DISP0_DAT11__GPIO5_IO05 0x40013070 -+ >; -+ }; -+ -+ pinctrl_microsom_brcm_reg: microsom-brcm-reg { -+ fsl,pins = < -+ MX6QDL_PAD_EIM_D19__GPIO3_IO19 0x40013070 -+ >; -+ }; -+ -+ pinctrl_microsom_brcm_wifi: microsom-brcm-wifi { -+ fsl,pins = < -+ MX6QDL_PAD_GPIO_8__XTALOSC_REF_CLK_32K 0x1b0b0 -+ MX6QDL_PAD_CSI0_DATA_EN__GPIO5_IO20 0x40013070 -+ MX6QDL_PAD_CSI0_DAT8__GPIO5_IO26 0x40013070 -+ MX6QDL_PAD_CSI0_DAT9__GPIO5_IO27 0x40013070 -+ >; -+ }; -+ - pinctrl_microsom_uart1: microsom-uart1 { - fsl,pins = < - MX6QDL_PAD_CSI0_DAT10__UART1_TX_DATA 0x1b0b1 - MX6QDL_PAD_CSI0_DAT11__UART1_RX_DATA 0x1b0b1 - >; - }; -+ -+ pinctrl_microsom_uart4: microsom-uart4 { -+ fsl,pins = < -+ MX6QDL_PAD_CSI0_DAT12__UART4_TX_DATA 0x1b0b1 -+ MX6QDL_PAD_CSI0_DAT13__UART4_RX_DATA 0x1b0b1 -+ MX6QDL_PAD_CSI0_DAT16__UART4_RTS_B 0x1b0b1 -+ MX6QDL_PAD_CSI0_DAT17__UART4_CTS_B 0x1b0b1 -+ >; -+ }; -+ -+ pinctrl_microsom_usdhc1: microsom-usdhc1 { -+ fsl,pins = < -+ MX6QDL_PAD_SD1_CMD__SD1_CMD 0x17059 -+ MX6QDL_PAD_SD1_CLK__SD1_CLK 0x10059 -+ MX6QDL_PAD_SD1_DAT0__SD1_DATA0 0x17059 -+ MX6QDL_PAD_SD1_DAT1__SD1_DATA1 0x17059 -+ MX6QDL_PAD_SD1_DAT2__SD1_DATA2 0x17059 -+ MX6QDL_PAD_SD1_DAT3__SD1_DATA3 0x17059 -+ >; -+ }; - }; - }; - -@@ -56,3 +139,23 @@ - pinctrl-0 = <&pinctrl_microsom_uart1>; - status = "okay"; - }; -+ -+/* UART4 - Connected to optional BRCM Wifi/BT/FM */ -+&uart4 { -+ pinctrl-names = "default"; -+ pinctrl-0 = <&pinctrl_microsom_brcm_bt &pinctrl_microsom_uart4>; -+ fsl,uart-has-rtscts; -+ status = "okay"; -+}; -+ -+/* USDHC1 - Connected to optional BRCM Wifi/BT/FM */ -+&usdhc1 { -+ pinctrl-names = "default"; -+ pinctrl-0 = <&pinctrl_microsom_brcm_wifi &pinctrl_microsom_usdhc1>; -+ bus-width = <4>; -+ mmc-pwrseq = <&usdhc1_pwrseq>; -+ keep-power-in-suspend; -+ non-removable; -+ vmmc-supply = <®_brcm>; -+ status = "okay"; -+}; -diff -Nur linux-4.1.3/arch/arm/boot/dts/imx6q.dtsi linux-xbian-imx6/arch/arm/boot/dts/imx6q.dtsi ---- linux-4.1.3/arch/arm/boot/dts/imx6q.dtsi 2015-07-21 19:10:33.000000000 +0200 -+++ linux-xbian-imx6/arch/arm/boot/dts/imx6q.dtsi 2015-07-27 23:13:00.303898027 +0200 -@@ -14,6 +14,7 @@ - - / { - aliases { -+ ipu1 = &ipu2; - spi4 = &ecspi5; - }; - -@@ -47,9 +48,12 @@ - <&clks IMX6QDL_CLK_PLL2_PFD2_396M>, - <&clks IMX6QDL_CLK_STEP>, - <&clks IMX6QDL_CLK_PLL1_SW>, -- <&clks IMX6QDL_CLK_PLL1_SYS>; -+ <&clks IMX6QDL_CLK_PLL1_SYS>, -+ <&clks IMX6QDL_PLL1_BYPASS>, -+ <&clks IMX6QDL_CLK_PLL1>, -+ <&clks IMX6QDL_PLL1_BYPASS_SRC> ; - clock-names = "arm", "pll2_pfd2_396m", "step", -- "pll1_sw", "pll1_sys"; -+ "pll1_sw", "pll1_sys", "pll1_bypass", "pll1", "pll1_bypass_src"; - arm-supply = <®_arm>; - pu-supply = <®_pu>; - soc-supply = <®_soc>; -@@ -78,9 +82,85 @@ - }; - - soc { -- ocram: sram@00900000 { -+ -+ busfreq { /* BUSFREQ */ -+ compatible = "fsl,imx6_busfreq"; -+ clocks = <&clks 171>, <&clks 6>, <&clks 11>, <&clks 104>, <&clks 172>, <&clks 58>, -+ <&clks 18>, <&clks 60>, <&clks 20>, <&clks 3>; -+ clock-names = "pll2_bus", "pll2_pfd2_396m", "pll2_198m", "arm", "pll3_usb_otg", "periph", -+ "periph_pre", "periph_clk2", "periph_clk2_sel", "osc"; -+ interrupts = <0 107 0x04>, <0 112 0x4>, <0 113 0x4>, <0 114 0x4>; -+ interrupt-names = "irq_busfreq_0", "irq_busfreq_1", "irq_busfreq_2", "irq_busfreq_3"; -+ fsl,max_ddr_freq = <528000000>; -+ }; -+ -+ gpu@00130000 { -+ compatible = "fsl,imx6q-gpu"; -+ reg = <0x00130000 0x4000>, <0x00134000 0x4000>, -+ <0x02204000 0x4000>, <0x0 0x0>; -+ reg-names = "iobase_3d", "iobase_2d", -+ "iobase_vg", "phys_baseaddr"; -+ interrupts = <0 9 0x04>, <0 10 0x04>,<0 11 0x04>; -+ interrupt-names = "irq_3d", "irq_2d", "irq_vg"; -+ clocks = <&clks 26>, <&clks 143>, -+ <&clks 27>, <&clks 121>, -+ <&clks 122>, <&clks 74>; -+ clock-names = "gpu2d_axi_clk", "openvg_axi_clk", -+ "gpu3d_axi_clk", "gpu2d_clk", -+ "gpu3d_clk", "gpu3d_shader_clk"; -+ resets = <&src 0>, <&src 3>, <&src 3>; -+ reset-names = "gpu3d", "gpu2d", "gpuvg"; -+ power-domains = <&gpc 1>; -+ }; -+ -+ hdmi_core: hdmi_core@00120000 { -+ compatible = "fsl,imx6q-hdmi-core"; -+ reg = <0x00120000 0x9000>; -+ clocks = <&clks 124>, <&clks 123>; -+ clock-names = "hdmi_isfr", "hdmi_iahb"; -+ status = "disabled"; -+ }; -+ -+ hdmi_video: hdmi_video@020e0000 { -+ compatible = "fsl,imx6q-hdmi-video"; -+ reg = <0x020e0000 0x1000>; -+ reg-names = "hdmi_gpr"; -+ interrupts = <0 115 0x04>; -+ clocks = <&clks 124>, <&clks 123>; -+ clock-names = "hdmi_isfr", "hdmi_iahb"; -+ status = "disabled"; -+ }; -+ -+ hdmi_audio: hdmi_audio@00120000 { -+ compatible = "fsl,imx6q-hdmi-audio"; -+ clocks = <&clks 124>, <&clks 123>; -+ clock-names = "hdmi_isfr", "hdmi_iahb"; -+ dmas = <&sdma 2 23 0>; -+ dma-names = "tx"; -+ status = "disabled"; -+ }; -+ -+ hdmi_cec: hdmi_cec@00120000 { -+ compatible = "fsl,imx6q-hdmi-cec"; -+ interrupts = <0 115 0x04>; -+ status = "disabled"; -+ }; -+ -+ ocrams: sram@00900000 { -+ compatible = "fsl,lpm-sram"; -+ reg = <0x00900000 0x4000>; -+ clocks = <&clks IMX6QDL_CLK_OCRAM>; -+ }; -+ -+ ocrams_ddr: sram@00904000 { -+ compatible = "fsl,ddr-lpm-sram"; -+ reg = <0x00904000 0x1000>; -+ clocks = <&clks IMX6QDL_CLK_OCRAM>; -+ }; -+ -+ ocram: sram@00905000 { - compatible = "mmio-sram"; -- reg = <0x00900000 0x40000>; -+ reg = <0x00905000 0x3B000>; - clocks = <&clks IMX6QDL_CLK_OCRAM>; - }; - -@@ -101,6 +181,10 @@ - }; - }; - -+ vpu@02040000 { -+ status = "okay"; -+ }; -+ - iomuxc: iomuxc@020e0000 { - compatible = "fsl,imx6q-iomuxc"; - -@@ -154,165 +238,33 @@ - }; - - ipu2: ipu@02800000 { -- #address-cells = <1>; -- #size-cells = <0>; - compatible = "fsl,imx6q-ipu"; - reg = <0x02800000 0x400000>; - interrupts = <0 8 IRQ_TYPE_LEVEL_HIGH>, - <0 7 IRQ_TYPE_LEVEL_HIGH>; -- clocks = <&clks IMX6QDL_CLK_IPU2>, -- <&clks IMX6QDL_CLK_IPU2_DI0>, -- <&clks IMX6QDL_CLK_IPU2_DI1>; -- clock-names = "bus", "di0", "di1"; -+ clocks = <&clks 133>, <&clks 134>, <&clks 137>, -+ <&clks 41>, <&clks 42>, -+ <&clks 135>, <&clks 136>; -+ clock-names = "bus", "di0", "di1", -+ "di0_sel", "di1_sel", -+ "ldb_di0", "ldb_di1"; - resets = <&src 4>; -- -- ipu2_csi0: port@0 { -- reg = <0>; -- }; -- -- ipu2_csi1: port@1 { -- reg = <1>; -- }; -- -- ipu2_di0: port@2 { -- #address-cells = <1>; -- #size-cells = <0>; -- reg = <2>; -- -- ipu2_di0_disp0: endpoint@0 { -- }; -- -- ipu2_di0_hdmi: endpoint@1 { -- remote-endpoint = <&hdmi_mux_2>; -- }; -- -- ipu2_di0_mipi: endpoint@2 { -- }; -- -- ipu2_di0_lvds0: endpoint@3 { -- remote-endpoint = <&lvds0_mux_2>; -- }; -- -- ipu2_di0_lvds1: endpoint@4 { -- remote-endpoint = <&lvds1_mux_2>; -- }; -- }; -- -- ipu2_di1: port@3 { -- #address-cells = <1>; -- #size-cells = <0>; -- reg = <3>; -- -- ipu2_di1_hdmi: endpoint@1 { -- remote-endpoint = <&hdmi_mux_3>; -- }; -- -- ipu2_di1_mipi: endpoint@2 { -- }; -- -- ipu2_di1_lvds0: endpoint@3 { -- remote-endpoint = <&lvds0_mux_3>; -- }; -- -- ipu2_di1_lvds1: endpoint@4 { -- remote-endpoint = <&lvds1_mux_3>; -- }; -- }; -- }; -- }; -- -- display-subsystem { -- compatible = "fsl,imx-display-subsystem"; -- ports = <&ipu1_di0>, <&ipu1_di1>, <&ipu2_di0>, <&ipu2_di1>; -- }; --}; -- --&hdmi { -- compatible = "fsl,imx6q-hdmi"; -- -- port@2 { -- reg = <2>; -- -- hdmi_mux_2: endpoint { -- remote-endpoint = <&ipu2_di0_hdmi>; -- }; -- }; -- -- port@3 { -- reg = <3>; -- -- hdmi_mux_3: endpoint { -- remote-endpoint = <&ipu2_di1_hdmi>; -+ bypass_reset = <0>; - }; - }; - }; - - &ldb { -- clocks = <&clks IMX6QDL_CLK_LDB_DI0_SEL>, <&clks IMX6QDL_CLK_LDB_DI1_SEL>, -+ clocks = <&clks IMX6QDL_CLK_LDB_DI0>, <&clks IMX6QDL_CLK_LDB_DI1>, - <&clks IMX6QDL_CLK_IPU1_DI0_SEL>, <&clks IMX6QDL_CLK_IPU1_DI1_SEL>, - <&clks IMX6QDL_CLK_IPU2_DI0_SEL>, <&clks IMX6QDL_CLK_IPU2_DI1_SEL>, -- <&clks IMX6QDL_CLK_LDB_DI0>, <&clks IMX6QDL_CLK_LDB_DI1>; -- clock-names = "di0_pll", "di1_pll", -- "di0_sel", "di1_sel", "di2_sel", "di3_sel", -- "di0", "di1"; -- -- lvds-channel@0 { -- port@2 { -- reg = <2>; -- -- lvds0_mux_2: endpoint { -- remote-endpoint = <&ipu2_di0_lvds0>; -- }; -- }; -- -- port@3 { -- reg = <3>; -- -- lvds0_mux_3: endpoint { -- remote-endpoint = <&ipu2_di1_lvds0>; -- }; -- }; -- }; -- -- lvds-channel@1 { -- port@2 { -- reg = <2>; -- -- lvds1_mux_2: endpoint { -- remote-endpoint = <&ipu2_di0_lvds1>; -- }; -- }; -- -- port@3 { -- reg = <3>; -- -- lvds1_mux_3: endpoint { -- remote-endpoint = <&ipu2_di1_lvds1>; -- }; -- }; -- }; --}; -- --&mipi_dsi { -- ports { -- port@2 { -- reg = <2>; -- -- mipi_mux_2: endpoint { -- remote-endpoint = <&ipu2_di0_mipi>; -- }; -- }; -- -- port@3 { -- reg = <3>; -- -- mipi_mux_3: endpoint { -- remote-endpoint = <&ipu2_di1_mipi>; -- }; -- }; -- }; --}; -- --&vpu { -- compatible = "fsl,imx6q-vpu", "cnm,coda960"; -+ <&clks IMX6QDL_CLK_LDB_DI0_DIV_3_5>, <&clks IMX6QDL_CLK_LDB_DI1_DIV_3_5>, -+ <&clks IMX6QDL_CLK_LDB_DI0_DIV_7>, <&clks IMX6QDL_CLK_LDB_DI1_DIV_7>, -+ <&clks IMX6QDL_CLK_LDB_DI0_DIV_SEL>, <&clks IMX6QDL_CLK_LDB_DI1_DIV_SEL>; -+ clock-names = "ldb_di0", "ldb_di1", -+ "di0_sel", "di1_sel", -+ "di2_sel", "di3_sel", -+ "ldb_di0_div_3_5", "ldb_di1_div_3_5", -+ "ldb_di0_div_7", "ldb_di1_div_7", -+ "ldb_di0_div_sel", "ldb_di1_div_sel"; - }; -diff -Nur linux-4.1.3/arch/arm/boot/dts/imx6q-hummingboard.dts linux-xbian-imx6/arch/arm/boot/dts/imx6q-hummingboard.dts ---- linux-4.1.3/arch/arm/boot/dts/imx6q-hummingboard.dts 2015-07-21 19:10:33.000000000 +0200 -+++ linux-xbian-imx6/arch/arm/boot/dts/imx6q-hummingboard.dts 2015-07-27 23:13:00.303898027 +0200 -@@ -57,3 +57,7 @@ - fsl,transmit-atten-16ths = <9>; - fsl,receive-eq-mdB = <3000>; - }; -+ -+&sgtl5000 { -+ status = "okay"; -+}; -diff -Nur linux-4.1.3/arch/arm/boot/dts/imx6sl.dtsi linux-xbian-imx6/arch/arm/boot/dts/imx6sl.dtsi ---- linux-4.1.3/arch/arm/boot/dts/imx6sl.dtsi 2015-07-21 19:10:33.000000000 +0200 -+++ linux-xbian-imx6/arch/arm/boot/dts/imx6sl.dtsi 2015-07-27 23:13:00.307883804 +0200 -@@ -457,20 +457,21 @@ - anatop-min-bit-val = <4>; - anatop-min-voltage = <800000>; - anatop-max-voltage = <1375000>; -+ anatop-enable-bit = <0>; - }; - -- regulator-3p0@120 { -+ reg_3p0: regulator-3p0@120 { - compatible = "fsl,anatop-regulator"; - regulator-name = "vdd3p0"; -- regulator-min-microvolt = <2800000>; -- regulator-max-microvolt = <3150000>; -- regulator-always-on; -+ regulator-min-microvolt = <2625000>; -+ regulator-max-microvolt = <3400000>; - anatop-reg-offset = <0x120>; - anatop-vol-bit-shift = <8>; - anatop-vol-bit-width = <5>; - anatop-min-bit-val = <0>; - anatop-min-voltage = <2625000>; - anatop-max-voltage = <3400000>; -+ anatop-enable-bit = <0>; - }; - - regulator-2p5@130 { -@@ -485,6 +486,7 @@ - anatop-min-bit-val = <0>; - anatop-min-voltage = <2100000>; - anatop-max-voltage = <2850000>; -+ anatop-enable-bit = <0>; - }; - - reg_arm: regulator-vddcore@140 { -@@ -552,6 +554,7 @@ - reg = <0x020c9000 0x1000>; - interrupts = <0 44 IRQ_TYPE_LEVEL_HIGH>; - clocks = <&clks IMX6SL_CLK_USBPHY1>; -+ phy-3p0-supply = <®_3p0>; - fsl,anatop = <&anatop>; - }; - -@@ -560,6 +563,7 @@ - reg = <0x020ca000 0x1000>; - interrupts = <0 45 IRQ_TYPE_LEVEL_HIGH>; - clocks = <&clks IMX6SL_CLK_USBPHY2>; -+ phy-3p0-supply = <®_3p0>; - fsl,anatop = <&anatop>; - }; - -diff -Nur linux-4.1.3/arch/arm/boot/dts/imx6sx.dtsi linux-xbian-imx6/arch/arm/boot/dts/imx6sx.dtsi ---- linux-4.1.3/arch/arm/boot/dts/imx6sx.dtsi 2015-07-21 19:10:33.000000000 +0200 -+++ linux-xbian-imx6/arch/arm/boot/dts/imx6sx.dtsi 2015-07-27 23:13:00.307883804 +0200 -@@ -556,20 +556,21 @@ - anatop-min-bit-val = <4>; - anatop-min-voltage = <800000>; - anatop-max-voltage = <1375000>; -+ anatop-enable-bit = <0>; - }; - -- regulator-3p0@120 { -+ reg_3p0: regulator-3p0@120 { - compatible = "fsl,anatop-regulator"; - regulator-name = "vdd3p0"; -- regulator-min-microvolt = <2800000>; -- regulator-max-microvolt = <3150000>; -- regulator-always-on; -+ regulator-min-microvolt = <2625000>; -+ regulator-max-microvolt = <3400000>; - anatop-reg-offset = <0x120>; - anatop-vol-bit-shift = <8>; - anatop-vol-bit-width = <5>; - anatop-min-bit-val = <0>; - anatop-min-voltage = <2625000>; - anatop-max-voltage = <3400000>; -+ anatop-enable-bit = <0>; - }; - - regulator-2p5@130 { -@@ -584,6 +585,7 @@ - anatop-min-bit-val = <0>; - anatop-min-voltage = <2100000>; - anatop-max-voltage = <2875000>; -+ anatop-enable-bit = <0>; - }; - - reg_arm: regulator-vddcore@140 { -@@ -650,6 +652,7 @@ - reg = <0x020c9000 0x1000>; - interrupts = ; - clocks = <&clks IMX6SX_CLK_USBPHY1>; -+ phy-3p0-supply = <®_3p0>; - fsl,anatop = <&anatop>; - }; - -@@ -658,6 +661,7 @@ - reg = <0x020ca000 0x1000>; - interrupts = ; - clocks = <&clks IMX6SX_CLK_USBPHY2>; -+ phy-3p0-supply = <®_3p0>; - fsl,anatop = <&anatop>; - }; - -diff -Nur linux-4.1.3/arch/arm/include/asm/glue-cache.h linux-xbian-imx6/arch/arm/include/asm/glue-cache.h ---- linux-4.1.3/arch/arm/include/asm/glue-cache.h 2015-07-21 19:10:33.000000000 +0200 -+++ linux-xbian-imx6/arch/arm/include/asm/glue-cache.h 2015-07-27 23:13:00.746319518 +0200 -@@ -102,19 +102,19 @@ - #endif - - #if defined(CONFIG_CPU_V6) || defined(CONFIG_CPU_V6K) --# ifdef _CACHE -+//# ifdef _CACHE - # define MULTI_CACHE 1 --# else --# define _CACHE v6 --# endif -+//# else -+//# define _CACHE v6 -+//# endif - #endif - - #if defined(CONFIG_CPU_V7) --# ifdef _CACHE -+//# ifdef _CACHE - # define MULTI_CACHE 1 --# else --# define _CACHE v7 --# endif -+//# else -+//# define _CACHE v7 -+//# endif - #endif - - #if defined(CONFIG_CPU_V7M) -diff -Nur linux-4.1.3/arch/arm/Kconfig linux-xbian-imx6/arch/arm/Kconfig ---- linux-4.1.3/arch/arm/Kconfig 2015-07-21 19:10:33.000000000 +0200 -+++ linux-xbian-imx6/arch/arm/Kconfig 2015-07-27 23:13:00.128523741 +0200 -@@ -1688,6 +1688,7 @@ - range 11 64 if ARCH_SHMOBILE_LEGACY - default "12" if SOC_AM33XX - default "9" if SA1111 || ARCH_EFM32 -+ default "14" if ARCH_MXC - default "11" - help - The kernel memory allocator divides physically contiguous memory -diff -Nur linux-4.1.3/arch/arm/mach-imx/busfreq_ddr3.c linux-xbian-imx6/arch/arm/mach-imx/busfreq_ddr3.c ---- linux-4.1.3/arch/arm/mach-imx/busfreq_ddr3.c 1970-01-01 01:00:00.000000000 +0100 -+++ linux-xbian-imx6/arch/arm/mach-imx/busfreq_ddr3.c 2015-07-27 23:13:01.073153409 +0200 -@@ -0,0 +1,514 @@ -+/* -+ * Copyright (C) 2011-2013 Freescale Semiconductor, Inc. All Rights Reserved. -+ */ -+ -+/* -+ * The code contained herein is licensed under the GNU General Public -+ * License. You may obtain a copy of the GNU General Public License -+ * Version 2 or later at the following locations: -+ * -+ * http://www.opensource.org/licenses/gpl-license.html -+ * http://www.gnu.org/copyleft/gpl.html -+ */ -+ -+/*! -+ * @file busfreq_ddr3.c -+ * -+ * @brief iMX6 DDR3 frequency change specific file. -+ * -+ * @ingroup PM -+ */ -+#define DEBUG -+ -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+ -+#include "hardware.h" -+ -+/* DDR settings */ -+static unsigned long (*iram_ddr_settings)[2]; -+static unsigned long (*normal_mmdc_settings)[2]; -+static unsigned long (*iram_iomux_settings)[2]; -+static void __iomem *mmdc_base; -+static void __iomem *iomux_base; -+static void __iomem *ccm_base; -+static void __iomem *l2_base; -+static void __iomem *gic_dist_base; -+static u32 *irqs_used; -+ -+static void *ddr_freq_change_iram_base; -+static int ddr_settings_size; -+static int iomux_settings_size; -+static volatile unsigned int cpus_in_wfe; -+static volatile bool wait_for_ddr_freq_update; -+static int curr_ddr_rate; -+ -+void (*mx6_change_ddr_freq)(u32 freq, void *ddr_settings, -+ bool dll_mode, void *iomux_offsets) = NULL; -+ -+extern unsigned int ddr_med_rate; -+extern unsigned int ddr_normal_rate; -+extern int low_bus_freq_mode; -+extern int audio_bus_freq_mode; -+extern void mx6_ddr3_freq_change(u32 freq, void *ddr_settings, -+ bool dll_mode, void *iomux_offsets); -+extern unsigned long save_ttbr1(void); -+extern void restore_ttbr1(unsigned long ttbr1); -+ -+#ifdef CONFIG_SMP -+extern void __iomem *imx_scu_base; -+static unsigned int online_cpus; -+#endif -+ -+#define MIN_DLL_ON_FREQ 333000000 -+#define MAX_DLL_OFF_FREQ 125000000 -+#define DDR_FREQ_CHANGE_SIZE 0x2000 -+ -+unsigned long ddr3_dll_mx6q[][2] = { -+ {0x0c, 0x0}, -+ {0x10, 0x0}, -+ {0x1C, 0x04088032}, -+ {0x1C, 0x0408803a}, -+ {0x1C, 0x08408030}, -+ {0x1C, 0x08408038}, -+ {0x818, 0x0}, -+}; -+ -+unsigned long ddr3_calibration[][2] = { -+ {0x83c, 0x0}, -+ {0x840, 0x0}, -+ {0x483c, 0x0}, -+ {0x4840, 0x0}, -+ {0x848, 0x0}, -+ {0x4848, 0x0}, -+ {0x850, 0x0}, -+ {0x4850, 0x0}, -+}; -+ -+unsigned long ddr3_dll_mx6dl[][2] = { -+ {0x0c, 0x0}, -+ {0x10, 0x0}, -+ {0x1C, 0x04008032}, -+ {0x1C, 0x0400803a}, -+ {0x1C, 0x07208030}, -+ {0x1C, 0x07208038}, -+ {0x818, 0x0}, -+}; -+ -+unsigned long iomux_offsets_mx6q[][2] = { -+ {0x5A8, 0x0}, -+ {0x5B0, 0x0}, -+ {0x524, 0x0}, -+ {0x51C, 0x0}, -+ {0x518, 0x0}, -+ {0x50C, 0x0}, -+ {0x5B8, 0x0}, -+ {0x5C0, 0x0}, -+}; -+ -+unsigned long iomux_offsets_mx6dl[][2] = { -+ {0x4BC, 0x0}, -+ {0x4C0, 0x0}, -+ {0x4C4, 0x0}, -+ {0x4C8, 0x0}, -+ {0x4CC, 0x0}, -+ {0x4D0, 0x0}, -+ {0x4D4, 0x0}, -+ {0x4D8, 0x0}, -+}; -+ -+unsigned long ddr3_400[][2] = { -+ {0x83c, 0x42490249}, -+ {0x840, 0x02470247}, -+ {0x483c, 0x42570257}, -+ {0x4840, 0x02400240}, -+ {0x848, 0x4039363C}, -+ {0x4848, 0x3A39333F}, -+ {0x850, 0x38414441}, -+ {0x4850, 0x472D4833} -+}; -+ -+int can_change_ddr_freq(void) -+{ -+ return 0; -+} -+ -+/* -+ * each active core apart from the one changing -+ * the DDR frequency will execute this function. -+ * the rest of the cores have to remain in WFE -+ * state until the frequency is changed. -+ */ -+irqreturn_t wait_in_wfe_irq(int irq, void *dev_id) -+{ -+ u32 me = smp_processor_id(); -+ -+ *((char *)(&cpus_in_wfe) + (u8)me) = 0xff; -+ -+ while (wait_for_ddr_freq_update) -+ wfe(); -+ -+ *((char *)(&cpus_in_wfe) + (u8)me) = 0; -+ -+ return IRQ_HANDLED; -+} -+ -+/* change the DDR frequency. */ -+int update_ddr_freq(int ddr_rate) -+{ -+ int i, j; -+ bool dll_off = false; -+ int me = 0; -+ unsigned long ttbr1; -+#ifdef CONFIG_SMP -+ unsigned int reg; -+ int cpu = 0; -+#endif -+ -+ if (!can_change_ddr_freq()) -+ return -1; -+ -+ if (ddr_rate == curr_ddr_rate) -+ return 0; -+ -+ printk(KERN_DEBUG "\nBus freq set to %d start...\n", ddr_rate); -+ -+ if (low_bus_freq_mode || audio_bus_freq_mode) -+ dll_off = true; -+ -+ iram_ddr_settings[0][0] = ddr_settings_size; -+ iram_iomux_settings[0][0] = iomux_settings_size; -+ if (ddr_rate == ddr_med_rate && cpu_is_imx6q() && -+ ddr_med_rate != ddr_normal_rate) { -+ for (i = 0; i < ARRAY_SIZE(ddr3_dll_mx6q); i++) { -+ iram_ddr_settings[i + 1][0] = -+ normal_mmdc_settings[i][0]; -+ iram_ddr_settings[i + 1][1] = -+ normal_mmdc_settings[i][1]; -+ } -+ for (j = 0, i = ARRAY_SIZE(ddr3_dll_mx6q); -+ i < iram_ddr_settings[0][0]; j++, i++) { -+ iram_ddr_settings[i + 1][0] = -+ ddr3_400[j][0]; -+ iram_ddr_settings[i + 1][1] = -+ ddr3_400[j][1]; -+ } -+ } else if (ddr_rate == ddr_normal_rate) { -+ for (i = 0; i < iram_ddr_settings[0][0]; i++) { -+ iram_ddr_settings[i + 1][0] = -+ normal_mmdc_settings[i][0]; -+ iram_ddr_settings[i + 1][1] = -+ normal_mmdc_settings[i][1]; -+ } -+ } -+ -+ /* ensure that all Cores are in WFE. */ -+ local_irq_disable(); -+ -+#ifdef CONFIG_SMP -+ me = smp_processor_id(); -+ -+ /* Make sure all the online cores are active */ -+ while (1) { -+ bool not_exited_busfreq = false; -+ for_each_online_cpu(cpu) { -+ u32 reg = __raw_readl(imx_scu_base + 0x08); -+ if (reg & (0x02 << (cpu * 8))) -+ not_exited_busfreq = true; -+ } -+ if (!not_exited_busfreq) -+ break; -+ } -+ -+ wmb(); -+ wait_for_ddr_freq_update = 1; -+ dsb(); -+ -+ online_cpus = readl_relaxed(imx_scu_base + 0x08); -+ for_each_online_cpu(cpu) { -+ *((char *)(&online_cpus) + (u8)cpu) = 0x02; -+ if (cpu != me) { -+ /* set the interrupt to be pending in the GIC. */ -+ reg = 1 << (irqs_used[cpu] % 32); -+ writel_relaxed(reg, gic_dist_base + GIC_DIST_PENDING_SET -+ + (irqs_used[cpu] / 32) * 4); -+ } -+ } -+ /* Wait for the other active CPUs to idle */ -+ while (1) { -+ u32 reg = readl_relaxed(imx_scu_base + 0x08); -+ reg |= (0x02 << (me * 8)); -+ if (reg == online_cpus) -+ break; -+ } -+#endif -+ -+ /* Ensure iram_tlb_phys_addr is flushed to DDR. */ -+ /*__cpuc_flush_dcache_area(&iram_tlb_phys_addr, sizeof(iram_tlb_phys_addr)); -+ outer_clean_range(virt_to_phys(&iram_tlb_phys_addr), virt_to_phys(&iram_tlb_phys_addr + 1));*/ -+ -+ /* -+ * Flush the TLB, to ensure no TLB maintenance occurs -+ * when DDR is in self-refresh. -+ */ -+ local_flush_tlb_all(); -+ -+ ttbr1 = save_ttbr1(); -+ /* Now we can change the DDR frequency. */ -+ mx6_change_ddr_freq(ddr_rate, iram_ddr_settings, -+ dll_off, iram_iomux_settings); -+ restore_ttbr1(ttbr1); -+ curr_ddr_rate = ddr_rate; -+ -+#ifdef CONFIG_SMP -+ wmb(); -+ /* DDR frequency change is done . */ -+ wait_for_ddr_freq_update = 0; -+ dsb(); -+ -+ /* wake up all the cores. */ -+ sev(); -+#endif -+ -+ local_irq_enable(); -+ -+ printk(KERN_DEBUG "Bus freq set to %d done! cpu=%d\n", ddr_rate, me); -+ -+ return 0; -+} -+ -+int init_mmdc_ddr3_settings(struct platform_device *busfreq_pdev) -+{ -+ struct device *dev = &busfreq_pdev->dev; -+ struct platform_device *ocram_dev; -+ unsigned int iram_paddr; -+ int i, err; -+ u32 cpu; -+ struct device_node *node; -+ struct gen_pool *iram_pool; -+ -+ node = of_find_compatible_node(NULL, NULL, "fsl,imx6q-mmdc-combine"); -+ if (!node) { -+ pr_err("failed to find imx6q-mmdc device tree data!\n"); -+ return -EINVAL; -+ } -+ mmdc_base = of_iomap(node, 0); -+ WARN(!mmdc_base, "unable to map mmdc registers\n"); -+ -+ node = NULL; -+ if (cpu_is_imx6q()) -+ node = of_find_compatible_node(NULL, NULL, "fsl,imx6q-iomuxc"); -+ if (cpu_is_imx6dl()) -+ node = of_find_compatible_node(NULL, NULL, -+ "fsl,imx6dl-iomuxc"); -+ if (!node) { -+ pr_err("failed to find imx6q-iomux device tree data!\n"); -+ return -EINVAL; -+ } -+ iomux_base = of_iomap(node, 0); -+ WARN(!iomux_base, "unable to map iomux registers\n"); -+ -+ node = of_find_compatible_node(NULL, NULL, "fsl,imx6q-ccm"); -+ if (!node) { -+ pr_err("failed to find imx6q-ccm device tree data!\n"); -+ return -EINVAL; -+ } -+ ccm_base = of_iomap(node, 0); -+ WARN(!ccm_base, "unable to map mmdc registers\n"); -+ -+ node = of_find_compatible_node(NULL, NULL, "arm,pl310-cache"); -+ if (!node) { -+ pr_err("failed to find imx6q-pl310-cache device tree data!\n"); -+ return -EINVAL; -+ } -+ l2_base = of_iomap(node, 0); -+ WARN(!ccm_base, "unable to map mmdc registers\n"); -+ -+ node = NULL; -+ node = of_find_compatible_node(NULL, NULL, "arm,cortex-a9-gic"); -+ if (!node) { -+ pr_err("failed to find imx6q-a9-gic device tree data!\n"); -+ return -EINVAL; -+ } -+ gic_dist_base = of_iomap(node, 0); -+ WARN(!gic_dist_base, "unable to map gic dist registers\n"); -+ -+ if (cpu_is_imx6q()) -+ ddr_settings_size = ARRAY_SIZE(ddr3_dll_mx6q) + -+ ARRAY_SIZE(ddr3_calibration); -+ if (cpu_is_imx6dl()) -+ ddr_settings_size = ARRAY_SIZE(ddr3_dll_mx6dl) + -+ ARRAY_SIZE(ddr3_calibration); -+ -+ normal_mmdc_settings = kmalloc((ddr_settings_size * 8), GFP_KERNEL); -+ if (cpu_is_imx6q()) { -+ memcpy(normal_mmdc_settings, ddr3_dll_mx6q, -+ sizeof(ddr3_dll_mx6q)); -+ memcpy(((char *)normal_mmdc_settings + sizeof(ddr3_dll_mx6q)), -+ ddr3_calibration, sizeof(ddr3_calibration)); -+ } -+ if (cpu_is_imx6dl()) { -+ memcpy(normal_mmdc_settings, ddr3_dll_mx6dl, -+ sizeof(ddr3_dll_mx6dl)); -+ memcpy(((char *)normal_mmdc_settings + sizeof(ddr3_dll_mx6dl)), -+ ddr3_calibration, sizeof(ddr3_calibration)); -+ } -+ /* store the original DDR settings at boot. */ -+ for (i = 0; i < ddr_settings_size; i++) { -+ /* -+ * writes via command mode register cannot be read back. -+ * hence hardcode them in the initial static array. -+ * this may require modification on a per customer basis. -+ */ -+ if (normal_mmdc_settings[i][0] != 0x1C) -+ normal_mmdc_settings[i][1] = -+ readl_relaxed(mmdc_base -+ + normal_mmdc_settings[i][0]); -+ } -+ -+ irqs_used = devm_kzalloc(dev, sizeof(u32) * num_present_cpus(), -+ GFP_KERNEL); -+ -+ for_each_online_cpu(cpu) { -+ int irq; -+ -+ /* -+ * set up a reserved interrupt to get all -+ * the active cores into a WFE state -+ * before changing the DDR frequency. -+ */ -+ irq = platform_get_irq(busfreq_pdev, cpu); -+ err = request_irq(irq, wait_in_wfe_irq, -+ IRQF_PERCPU, "mmdc_1", NULL); -+ if (err) { -+ dev_err(dev, -+ "Busfreq:request_irq failed %d, err = %d\n", -+ irq, err); -+ return err; -+ } -+ err = irq_set_affinity(irq, cpumask_of(cpu)); -+ if (err) { -+ dev_err(dev, -+ "Busfreq: Cannot set irq affinity irq=%d,\n", -+ irq); -+ return err; -+ } -+ irqs_used[cpu] = irq; -+ } -+ -+ node = NULL; -+ node = of_find_compatible_node(NULL, NULL, "mmio-sram"); -+ if (!node) { -+ dev_err(dev, "%s: failed to find ocram node\n", -+ __func__); -+ return -EINVAL; -+ } -+ -+ ocram_dev = of_find_device_by_node(node); -+ if (!ocram_dev) { -+ dev_err(dev, "failed to find ocram device!\n"); -+ return -EINVAL; -+ } -+ -+ iram_pool = dev_get_gen_pool(&ocram_dev->dev); -+ if (!iram_pool) { -+ dev_err(dev, "iram pool unavailable!\n"); -+ return -EINVAL; -+ } -+ -+ iomux_settings_size = ARRAY_SIZE(iomux_offsets_mx6q); -+ iram_iomux_settings = (void*)gen_pool_alloc(iram_pool, -+ (iomux_settings_size * 8) + 8); -+ if (!iram_iomux_settings) { -+ dev_err(dev, "unable to alloc iram for IOMUX settings!\n"); -+ return -ENOMEM; -+ } -+ -+ /* -+ * Allocate extra space to store the number of entries in the -+ * ddr_settings plus 4 extra regsiter information that needs -+ * to be passed to the frequency change code. -+ * sizeof(iram_ddr_settings) = sizeof(ddr_settings) + -+ * entries in ddr_settings + 16. -+ * The last 4 enties store the addresses of the registers: -+ * CCM_BASE_ADDR -+ * MMDC_BASE_ADDR -+ * IOMUX_BASE_ADDR -+ * L2X0_BASE_ADDR -+ */ -+ iram_ddr_settings = (void*)gen_pool_alloc(iram_pool, -+ (ddr_settings_size * 8) + 8 + 32); -+ if (!iram_ddr_settings) { -+ dev_err(dev, "unable to alloc iram for ddr settings!\n"); -+ return -ENOMEM; -+ } -+ i = ddr_settings_size + 1; -+ iram_ddr_settings[i][0] = (unsigned long)mmdc_base; -+ iram_ddr_settings[i+1][0] = (unsigned long)ccm_base; -+ iram_ddr_settings[i+2][0] = (unsigned long)iomux_base; -+ iram_ddr_settings[i+3][0] = (unsigned long)l2_base; -+ -+ if (cpu_is_imx6q()) { -+ /* store the IOMUX settings at boot. */ -+ for (i = 0; i < iomux_settings_size; i++) { -+ iomux_offsets_mx6q[i][1] = -+ readl_relaxed(iomux_base + -+ iomux_offsets_mx6q[i][0]); -+ iram_iomux_settings[i+1][0] = iomux_offsets_mx6q[i][0]; -+ iram_iomux_settings[i+1][1] = iomux_offsets_mx6q[i][1]; -+ } -+ } -+ -+ if (cpu_is_imx6dl()) { -+ for (i = 0; i < iomux_settings_size; i++) { -+ iomux_offsets_mx6dl[i][1] = -+ readl_relaxed(iomux_base + -+ iomux_offsets_mx6dl[i][0]); -+ iram_iomux_settings[i+1][0] = iomux_offsets_mx6dl[i][0]; -+ iram_iomux_settings[i+1][1] = iomux_offsets_mx6dl[i][1]; -+ } -+ } -+ -+ ddr_freq_change_iram_base = (void*)gen_pool_alloc(iram_pool, -+ DDR_FREQ_CHANGE_SIZE); -+ if (!ddr_freq_change_iram_base) { -+ dev_err(dev, "Cannot alloc iram for ddr freq change code!\n"); -+ return -ENOMEM; -+ } -+ -+ iram_paddr = gen_pool_virt_to_phys(iram_pool, -+ (unsigned long)ddr_freq_change_iram_base); -+ /* -+ * Need to remap the area here since we want -+ * the memory region to be executable. -+ */ -+ ddr_freq_change_iram_base = __arm_ioremap(iram_paddr, -+ DDR_FREQ_CHANGE_SIZE, -+ MT_MEMORY_RWX_NONCACHED); -+ mx6_change_ddr_freq = (void *)fncpy(ddr_freq_change_iram_base, -+ &mx6_ddr3_freq_change, DDR_FREQ_CHANGE_SIZE); -+ -+ curr_ddr_rate = ddr_normal_rate; -+ -+ return 0; -+} -diff -Nur linux-4.1.3/arch/arm/mach-imx/busfreq-imx6.c linux-xbian-imx6/arch/arm/mach-imx/busfreq-imx6.c ---- linux-4.1.3/arch/arm/mach-imx/busfreq-imx6.c 1970-01-01 01:00:00.000000000 +0100 -+++ linux-xbian-imx6/arch/arm/mach-imx/busfreq-imx6.c 2015-07-27 23:13:01.073153409 +0200 -@@ -0,0 +1,984 @@ -+/* -+ * Copyright (C) 2011-2013 Freescale Semiconductor, Inc. All Rights Reserved. -+ * -+ * This program is free software; you can redistribute it and/or modify -+ * it under the terms of the GNU General Public License as published by -+ * the Free Software Foundation; either version 2 of the License, or -+ * (at your option) any later version. -+ -+ * This program is distributed in the hope that it will be useful, -+ * but WITHOUT ANY WARRANTY; without even the implied warranty of -+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -+ * GNU General Public License for more details. -+ -+ * You should have received a copy of the GNU General Public License along -+ * with this program; if not, write to the Free Software Foundation, Inc., -+ * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. -+ */ -+ -+/*! -+ * @file busfreq-imx6.c -+ * -+ * @brief A common API for the Freescale Semiconductor iMX6 Busfreq API -+ * -+ * The APIs are for setting bus frequency to different values based on the -+ * highest freqeuncy requested. -+ * -+ * @ingroup PM -+ */ -+ -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include "hardware.h" -+ -+#define LPAPM_CLK 24000000 -+#define DDR3_AUDIO_CLK 50000000 -+#define LPDDR2_AUDIO_CLK 100000000 -+ -+int vpu352 = 0; -+ -+int high_bus_freq_mode; -+int med_bus_freq_mode; -+int audio_bus_freq_mode; -+int low_bus_freq_mode; -+int ultra_low_bus_freq_mode; -+unsigned int ddr_med_rate; -+unsigned int ddr_normal_rate; -+ -+#ifdef CONFIG_ARM_IMX6Q_CPUFREQ -+static int bus_freq_scaling_initialized; -+static struct device *busfreq_dev; -+static int busfreq_suspended; -+static u32 org_arm_rate; -+static int bus_freq_scaling_is_active; -+static int high_bus_count, med_bus_count, audio_bus_count, low_bus_count; -+static unsigned int ddr_low_rate; -+ -+extern int init_mmdc_lpddr2_settings(struct platform_device *dev); -+extern int init_mmdc_ddr3_settings(struct platform_device *dev); -+extern int update_ddr_freq(int ddr_rate); -+extern int update_lpddr2_freq(int ddr_rate); -+ -+DEFINE_MUTEX(bus_freq_mutex); -+static DEFINE_SPINLOCK(freq_lock); -+ -+static struct clk *pll2_400; -+static struct clk *periph_clk; -+static struct clk *periph_pre_clk; -+static struct clk *periph_clk2_sel; -+static struct clk *periph_clk2; -+static struct clk *osc_clk; -+static struct clk *cpu_clk; -+static struct clk *pll3; -+static struct clk *pll2; -+static struct clk *pll2_200; -+static struct clk *pll1_sys; -+static struct clk *periph2_clk; -+static struct clk *ocram_clk; -+static struct clk *ahb_clk; -+static struct clk *pll1_sw_clk; -+static struct clk *periph2_pre_clk; -+static struct clk *periph2_clk2_sel; -+static struct clk *periph2_clk2; -+static struct clk *step_clk; -+static struct clk *axi_sel_clk; -+static struct clk *pll3_pfd1_540m; -+ -+static u32 pll2_org_rate; -+static struct delayed_work low_bus_freq_handler; -+static struct delayed_work bus_freq_daemon; -+ -+static void enter_lpm_imx6sl(void) -+{ -+ unsigned long flags; -+ -+ if (high_bus_freq_mode) { -+ pll2_org_rate = clk_get_rate(pll2); -+ /* Set periph_clk to be sourced from OSC_CLK */ -+ clk_set_parent(periph_clk2_sel, osc_clk); -+ clk_set_parent(periph_clk, periph_clk2); -+ /* Ensure AHB/AXI clks are at 24MHz. */ -+ clk_set_rate(ahb_clk, LPAPM_CLK); -+ clk_set_rate(ocram_clk, LPAPM_CLK); -+ } -+ if (audio_bus_count) { -+ /* Set AHB to 8MHz to lower pwer.*/ -+ clk_set_rate(ahb_clk, LPAPM_CLK / 3); -+ -+ /* Set up DDR to 100MHz. */ -+ spin_lock_irqsave(&freq_lock, flags); -+ update_lpddr2_freq(LPDDR2_AUDIO_CLK); -+ spin_unlock_irqrestore(&freq_lock, flags); -+ -+ /* Fix the clock tree in kernel */ -+ clk_set_rate(pll2, pll2_org_rate); -+ clk_set_parent(periph2_pre_clk, pll2_200); -+ clk_set_parent(periph2_clk, periph2_pre_clk); -+ -+ if (low_bus_freq_mode || ultra_low_bus_freq_mode) { -+ /* -+ * Swtich ARM to run off PLL2_PFD2_400MHz -+ * since DDR is anyway at 100MHz. -+ */ -+ clk_set_parent(step_clk, pll2_400); -+ clk_set_parent(pll1_sw_clk, step_clk); -+ /* -+ * Ensure that the clock will be -+ * at original speed. -+ */ -+ clk_set_rate(cpu_clk, org_arm_rate); -+ } -+ low_bus_freq_mode = 0; -+ ultra_low_bus_freq_mode = 0; -+ audio_bus_freq_mode = 1; -+ } else { -+ u32 arm_div, pll1_rate; -+ org_arm_rate = clk_get_rate(cpu_clk); -+ if (low_bus_freq_mode && low_bus_count == 0) { -+ /* -+ * We are already in DDR @ 24MHz state, but -+ * no one but ARM needs the DDR. In this case, -+ * we can lower the DDR freq to 1MHz when ARM -+ * enters WFI in this state. Keep track of this state. -+ */ -+ ultra_low_bus_freq_mode = 1; -+ low_bus_freq_mode = 0; -+ audio_bus_freq_mode = 0; -+ } else { -+ if (!ultra_low_bus_freq_mode && !low_bus_freq_mode) { -+ /* -+ * Set DDR to 24MHz. -+ * Since we are going to bypass PLL2, -+ * we need to move ARM clk off PLL2_PFD2 -+ * to PLL1. Make sure the PLL1 is running -+ * at the lowest possible freq. -+ */ -+ clk_set_rate(pll1_sys, -+ clk_round_rate(pll1_sys, org_arm_rate)); -+ pll1_rate = clk_get_rate(pll1_sys); -+ arm_div = pll1_rate / org_arm_rate + 1; -+ /* -+ * Ensure ARM CLK is lower before -+ * changing the parent. -+ */ -+ clk_set_rate(cpu_clk, org_arm_rate / arm_div); -+ /* Now set the ARM clk parent to PLL1_SYS. */ -+ clk_set_parent(pll1_sw_clk, pll1_sys); -+ -+ /* -+ * Set STEP_CLK back to OSC to save power and -+ * also to maintain the parent.The WFI iram code -+ * will switch step_clk to osc, but the clock API -+ * is not aware of the change and when a new request -+ * to change the step_clk parent to pll2_pfd2_400M -+ * is requested sometime later, the change is ignored. -+ */ -+ clk_set_parent(step_clk, osc_clk); -+ /* Now set DDR to 24MHz. */ -+ spin_lock_irqsave(&freq_lock, flags); -+ update_lpddr2_freq(LPAPM_CLK); -+ spin_unlock_irqrestore(&freq_lock, flags); -+ -+ /* -+ * Fix the clock tree in kernel. -+ * Make sure PLL2 rate is updated as it gets -+ * bypassed in the DDR freq change code. -+ */ -+ clk_set_rate(pll2, LPAPM_CLK); -+ clk_set_parent(periph2_clk2_sel, pll2); -+ clk_set_parent(periph2_clk, periph2_clk2_sel); -+ -+ } -+ if (low_bus_count == 0) { -+ ultra_low_bus_freq_mode = 1; -+ low_bus_freq_mode = 0; -+ } else { -+ ultra_low_bus_freq_mode = 0; -+ low_bus_freq_mode = 1; -+ } -+ audio_bus_freq_mode = 0; -+ } -+ } -+} -+ -+static void exit_lpm_imx6sl(void) -+{ -+ unsigned long flags; -+ -+ spin_lock_irqsave(&freq_lock, flags); -+ /* Change DDR freq in IRAM. */ -+ update_lpddr2_freq(ddr_normal_rate); -+ spin_unlock_irqrestore(&freq_lock, flags); -+ -+ /* -+ * Fix the clock tree in kernel. -+ * Make sure PLL2 rate is updated as it gets -+ * un-bypassed in the DDR freq change code. -+ */ -+ clk_set_rate(pll2, pll2_org_rate); -+ clk_set_parent(periph2_pre_clk, pll2_400); -+ clk_set_parent(periph2_clk, periph2_pre_clk); -+ -+ /* Ensure that periph_clk is sourced from PLL2_400. */ -+ clk_set_parent(periph_pre_clk, pll2_400); -+ /* -+ * Before switching the perhiph_clk, ensure that the -+ * AHB/AXI will not be too fast. -+ */ -+ clk_set_rate(ahb_clk, LPAPM_CLK / 3); -+ clk_set_rate(ocram_clk, LPAPM_CLK / 2); -+ clk_set_parent(periph_clk, periph_pre_clk); -+ -+ if (low_bus_freq_mode || ultra_low_bus_freq_mode) { -+ /* Move ARM from PLL1_SW_CLK to PLL2_400. */ -+ clk_set_parent(step_clk, pll2_400); -+ clk_set_parent(pll1_sw_clk, step_clk); -+ clk_set_rate(cpu_clk, org_arm_rate); -+ ultra_low_bus_freq_mode = 0; -+ } -+} -+ -+int reduce_bus_freq(void) -+{ -+ int ret = 0; -+ clk_prepare_enable(pll3); -+ if (cpu_is_imx6sl()) -+ enter_lpm_imx6sl(); -+ else { -+ if (cpu_is_imx6dl() && (clk_get_parent(axi_sel_clk) -+ != periph_clk)) -+ /* Set axi to periph_clk */ -+ clk_set_parent(axi_sel_clk, periph_clk); -+ -+ if (audio_bus_count) { -+ /* Need to ensure that PLL2_PFD_400M is kept ON. */ -+ clk_prepare_enable(pll2_400); -+ update_ddr_freq(DDR3_AUDIO_CLK); -+ /* Make sure periph clk's parent also got updated */ -+ ret = clk_set_parent(periph_clk2_sel, pll3); -+ if (ret) -+ dev_WARN(busfreq_dev, -+ "%s: %d: clk set parent fail!\n", -+ __func__, __LINE__); -+ ret = clk_set_parent(periph_pre_clk, pll2_200); -+ if (ret) -+ dev_WARN(busfreq_dev, -+ "%s: %d: clk set parent fail!\n", -+ __func__, __LINE__); -+ ret = clk_set_parent(periph_clk, periph_pre_clk); -+ if (ret) -+ dev_WARN(busfreq_dev, -+ "%s: %d: clk set parent fail!\n", -+ __func__, __LINE__); -+ audio_bus_freq_mode = 1; -+ low_bus_freq_mode = 0; -+ } else { -+ update_ddr_freq(LPAPM_CLK); -+ /* Make sure periph clk's parent also got updated */ -+ ret = clk_set_parent(periph_clk2_sel, osc_clk); -+ if (ret) -+ dev_WARN(busfreq_dev, -+ "%s: %d: clk set parent fail!\n", -+ __func__, __LINE__); -+ /* Set periph_clk parent to OSC via periph_clk2_sel */ -+ ret = clk_set_parent(periph_clk, periph_clk2); -+ if (ret) -+ dev_WARN(busfreq_dev, -+ "%s: %d: clk set parent fail!\n", -+ __func__, __LINE__); -+ if (audio_bus_freq_mode) -+ clk_disable_unprepare(pll2_400); -+ low_bus_freq_mode = 1; -+ audio_bus_freq_mode = 0; -+ } -+ } -+ clk_disable_unprepare(pll3); -+ -+ med_bus_freq_mode = 0; -+ high_bus_freq_mode = 0; -+ -+ if (audio_bus_freq_mode) -+ dev_dbg(busfreq_dev, "Bus freq set to audio mode. Count:\ -+ high %d, med %d, audio %d\n", -+ high_bus_count, med_bus_count, audio_bus_count); -+ if (low_bus_freq_mode) -+ dev_dbg(busfreq_dev, "Bus freq set to low mode. Count:\ -+ high %d, med %d, audio %d\n", -+ high_bus_count, med_bus_count, audio_bus_count); -+ -+ return ret; -+} -+ -+static void reduce_bus_freq_handler(struct work_struct *work) -+{ -+ mutex_lock(&bus_freq_mutex); -+ -+ reduce_bus_freq(); -+ -+ mutex_unlock(&bus_freq_mutex); -+} -+ -+/* -+ * Set the DDR, AHB to 24MHz. -+ * This mode will be activated only when none of the modules that -+ * need a higher DDR or AHB frequency are active. -+ */ -+int set_low_bus_freq(void) -+{ -+ if (busfreq_suspended) -+ return 0; -+ -+ if (!bus_freq_scaling_initialized || !bus_freq_scaling_is_active) -+ return 0; -+ -+ /* -+ * Check to see if we need to got from -+ * low bus freq mode to audio bus freq mode. -+ * If so, the change needs to be done immediately. -+ */ -+ if (audio_bus_count && (low_bus_freq_mode || ultra_low_bus_freq_mode)) -+ reduce_bus_freq(); -+ else -+ /* -+ * Don't lower the frequency immediately. Instead -+ * scheduled a delayed work and drop the freq if -+ * the conditions still remain the same. -+ */ -+ schedule_delayed_work(&low_bus_freq_handler, -+ usecs_to_jiffies(3000000)); -+ return 0; -+} -+ -+/* -+ * Set the DDR to either 528MHz or 400MHz for iMX6qd -+ * or 400MHz for iMX6dl. -+ */ -+int set_high_bus_freq(int high_bus_freq) -+{ -+ int ret = 0; -+ struct clk *periph_clk_parent; -+ -+ if (bus_freq_scaling_initialized && bus_freq_scaling_is_active) -+ cancel_delayed_work_sync(&low_bus_freq_handler); -+ -+ if (busfreq_suspended) -+ return 0; -+ -+ if (cpu_is_imx6q()) -+ periph_clk_parent = pll2; -+ else -+ periph_clk_parent = pll2_400; -+ -+ if (!bus_freq_scaling_initialized || !bus_freq_scaling_is_active) -+ return 0; -+ -+ if (high_bus_freq_mode) -+ return 0; -+ -+ /* medium bus freq is only supported for MX6DQ */ -+ if (med_bus_freq_mode && !high_bus_freq) -+ return 0; -+ -+ clk_prepare_enable(pll3); -+ if (cpu_is_imx6sl()) -+ exit_lpm_imx6sl(); -+ else { -+ if (high_bus_freq) { -+ update_ddr_freq(ddr_normal_rate); -+ /* Make sure periph clk's parent also got updated */ -+ ret = clk_set_parent(periph_clk2_sel, pll3); -+ if (ret) -+ dev_WARN(busfreq_dev, -+ "%s: %d: clk set parent fail!\n", -+ __func__, __LINE__); -+ ret = clk_set_parent(periph_pre_clk, periph_clk_parent); -+ if (ret) -+ dev_WARN(busfreq_dev, -+ "%s: %d: clk set parent fail!\n", -+ __func__, __LINE__); -+ ret = clk_set_parent(periph_clk, periph_pre_clk); -+ if (ret) -+ dev_WARN(busfreq_dev, -+ "%s: %d: clk set parent fail!\n", -+ __func__, __LINE__); -+ if (cpu_is_imx6dl() && (clk_get_parent(axi_sel_clk) -+ != pll3_pfd1_540m)) -+ /* Set axi to pll3_pfd1_540m */ -+ clk_set_parent(axi_sel_clk, pll3_pfd1_540m); -+ } else { -+ update_ddr_freq(ddr_med_rate); -+ /* Make sure periph clk's parent also got updated */ -+ ret = clk_set_parent(periph_clk2_sel, pll3); -+ if (ret) -+ dev_WARN(busfreq_dev, -+ "%s: %d: clk set parent fail!\n", -+ __func__, __LINE__); -+ ret = clk_set_parent(periph_pre_clk, pll2_400); -+ if (ret) -+ dev_WARN(busfreq_dev, -+ "%s: %d: clk set parent fail!\n", -+ __func__, __LINE__); -+ ret = clk_set_parent(periph_clk, periph_pre_clk); -+ if (ret) -+ dev_WARN(busfreq_dev, -+ "%s: %d: clk set parent fail!\n", -+ __func__, __LINE__); -+ } -+ if (audio_bus_freq_mode) -+ clk_disable_unprepare(pll2_400); -+ } -+ -+ high_bus_freq_mode = 1; -+ med_bus_freq_mode = 0; -+ low_bus_freq_mode = 0; -+ audio_bus_freq_mode = 0; -+ -+ clk_disable_unprepare(pll3); -+ -+ if (high_bus_freq_mode) -+ dev_dbg(busfreq_dev, "Bus freq set to high mode. Count:\ -+ high %d, med %d, audio %d\n", -+ high_bus_count, med_bus_count, audio_bus_count); -+ if (med_bus_freq_mode) -+ dev_dbg(busfreq_dev, "Bus freq set to med mode. Count:\ -+ high %d, med %d, audio %d\n", -+ high_bus_count, med_bus_count, audio_bus_count); -+ -+ return 0; -+} -+#endif -+ -+void request_bus_freq(enum bus_freq_mode mode) -+{ -+#ifdef CONFIG_ARM_IMX6Q_CPUFREQ -+ mutex_lock(&bus_freq_mutex); -+ -+ if (mode == BUS_FREQ_HIGH) -+ high_bus_count++; -+ else if (mode == BUS_FREQ_MED) -+ med_bus_count++; -+ else if (mode == BUS_FREQ_AUDIO) -+ audio_bus_count++; -+ else if (mode == BUS_FREQ_LOW) -+ low_bus_count++; -+ -+ if (busfreq_suspended || !bus_freq_scaling_initialized || -+ !bus_freq_scaling_is_active) { -+ mutex_unlock(&bus_freq_mutex); -+ return; -+ } -+ cancel_delayed_work_sync(&low_bus_freq_handler); -+ -+ if (cpu_is_imx6dl()) { -+ /* No support for medium setpoint on MX6DL. */ -+ if (mode == BUS_FREQ_MED) { -+ high_bus_count++; -+ mode = BUS_FREQ_HIGH; -+ } -+ } -+ -+ if ((mode == BUS_FREQ_HIGH) && (!high_bus_freq_mode)) { -+ set_high_bus_freq(1); -+ mutex_unlock(&bus_freq_mutex); -+ return; -+ } -+ -+ if ((mode == BUS_FREQ_MED) && (!high_bus_freq_mode) && -+ (!med_bus_freq_mode)) { -+ set_high_bus_freq(0); -+ mutex_unlock(&bus_freq_mutex); -+ return; -+ } -+ if ((mode == BUS_FREQ_AUDIO) && (!high_bus_freq_mode) && -+ (!med_bus_freq_mode) && (!audio_bus_freq_mode)) { -+ set_low_bus_freq(); -+ mutex_unlock(&bus_freq_mutex); -+ return; -+ } -+ mutex_unlock(&bus_freq_mutex); -+#endif -+ return; -+} -+EXPORT_SYMBOL(request_bus_freq); -+ -+void release_bus_freq(enum bus_freq_mode mode) -+{ -+#ifdef CONFIG_ARM_IMX6Q_CPUFREQ -+ mutex_lock(&bus_freq_mutex); -+ -+ if (mode == BUS_FREQ_HIGH) { -+ if (high_bus_count == 0) { -+ dev_err(busfreq_dev, "high bus count mismatch!\n"); -+ dump_stack(); -+ mutex_unlock(&bus_freq_mutex); -+ return; -+ } -+ high_bus_count--; -+ } else if (mode == BUS_FREQ_MED) { -+ if (med_bus_count == 0) { -+ dev_err(busfreq_dev, "med bus count mismatch!\n"); -+ dump_stack(); -+ mutex_unlock(&bus_freq_mutex); -+ return; -+ } -+ med_bus_count--; -+ } else if (mode == BUS_FREQ_AUDIO) { -+ if (audio_bus_count == 0) { -+ dev_err(busfreq_dev, "audio bus count mismatch!\n"); -+ dump_stack(); -+ mutex_unlock(&bus_freq_mutex); -+ return; -+ } -+ audio_bus_count--; -+ } else if (mode == BUS_FREQ_LOW) { -+ if (low_bus_count == 0) { -+ dev_err(busfreq_dev, "low bus count mismatch!\n"); -+ dump_stack(); -+ mutex_unlock(&bus_freq_mutex); -+ return; -+ } -+ low_bus_count--; -+ } -+ -+ if (busfreq_suspended || !bus_freq_scaling_initialized || -+ !bus_freq_scaling_is_active) { -+ mutex_unlock(&bus_freq_mutex); -+ return; -+ } -+ -+ if (cpu_is_imx6dl()) { -+ /* No support for medium setpoint on MX6DL. */ -+ if (mode == BUS_FREQ_MED) { -+ high_bus_count--; -+ mode = BUS_FREQ_HIGH; -+ } -+ } -+ -+ if ((!audio_bus_freq_mode) && (high_bus_count == 0) && -+ (med_bus_count == 0) && (audio_bus_count != 0)) { -+ set_low_bus_freq(); -+ mutex_unlock(&bus_freq_mutex); -+ return; -+ } -+ if ((!low_bus_freq_mode) && (high_bus_count == 0) && -+ (med_bus_count == 0) && (audio_bus_count == 0) && -+ (low_bus_count != 0)) { -+ set_low_bus_freq(); -+ mutex_unlock(&bus_freq_mutex); -+ return; -+ } -+ if ((!ultra_low_bus_freq_mode) && (high_bus_count == 0) && -+ (med_bus_count == 0) && (audio_bus_count == 0) && -+ (low_bus_count == 0)) { -+ set_low_bus_freq(); -+ mutex_unlock(&bus_freq_mutex); -+ return; -+ } -+ -+ mutex_unlock(&bus_freq_mutex); -+#endif -+ return; -+} -+EXPORT_SYMBOL(release_bus_freq); -+ -+#ifdef CONFIG_ARM_IMX6Q_CPUFREQ -+static void bus_freq_daemon_handler(struct work_struct *work) -+{ -+ mutex_lock(&bus_freq_mutex); -+ if ((!low_bus_freq_mode) && (!ultra_low_bus_freq_mode) && (high_bus_count == 0) && -+ (med_bus_count == 0) && (audio_bus_count == 0)) -+ set_low_bus_freq(); -+ mutex_unlock(&bus_freq_mutex); -+} -+ -+static ssize_t bus_freq_scaling_enable_show(struct device *dev, -+ struct device_attribute *attr, char *buf) -+{ -+ if (bus_freq_scaling_is_active) -+ return sprintf(buf, "Bus frequency scaling is enabled\n"); -+ else -+ return sprintf(buf, "Bus frequency scaling is disabled\n"); -+} -+ -+static ssize_t vpu352_enable_show(struct device *dev, -+ struct device_attribute *attr, char *buf) -+{ -+ if (vpu352) -+ return sprintf(buf, "VPU352M is enabled\n"); -+ else -+ return sprintf(buf, "VPU352M is disabled\n"); -+} -+ -+static int vpu352_setup(char *options) -+{ -+ return kstrtol(options, 0, (long int *)&vpu352); -+} -+ -+static ssize_t bus_freq_scaling_enable_store(struct device *dev, -+ struct device_attribute *attr, -+ const char *buf, size_t size) -+{ -+ if (strncmp(buf, "1", 1) == 0) { -+ bus_freq_scaling_is_active = 1; -+ set_high_bus_freq(1); -+ /* -+ * We set bus freq to highest at the beginning, -+ * so we use this daemon thread to make sure system -+ * can enter low bus mode if -+ * there is no high bus request pending -+ */ -+ schedule_delayed_work(&bus_freq_daemon, -+ usecs_to_jiffies(5000000)); -+ } else if (strncmp(buf, "0", 1) == 0) { -+ if (bus_freq_scaling_is_active) -+ set_high_bus_freq(1); -+ bus_freq_scaling_is_active = 0; -+ } -+ return size; -+} -+ -+static int bus_freq_pm_notify(struct notifier_block *nb, unsigned long event, -+ void *dummy) -+{ -+ mutex_lock(&bus_freq_mutex); -+ -+ if (event == PM_SUSPEND_PREPARE) { -+ high_bus_count++; -+ set_high_bus_freq(1); -+ busfreq_suspended = 1; -+ } else if (event == PM_POST_SUSPEND) { -+ busfreq_suspended = 0; -+ high_bus_count--; -+ schedule_delayed_work(&bus_freq_daemon, -+ usecs_to_jiffies(5000000)); -+ } -+ -+ mutex_unlock(&bus_freq_mutex); -+ -+ return NOTIFY_OK; -+} -+ -+static int busfreq_reboot_notifier_event(struct notifier_block *this, -+ unsigned long event, void *ptr) -+{ -+ /* System is rebooting. Set the system into high_bus_freq_mode. */ -+ request_bus_freq(BUS_FREQ_HIGH); -+ -+ return 0; -+} -+ -+static struct notifier_block imx_bus_freq_pm_notifier = { -+ .notifier_call = bus_freq_pm_notify, -+}; -+ -+static struct notifier_block imx_busfreq_reboot_notifier = { -+ .notifier_call = busfreq_reboot_notifier_event, -+}; -+ -+ -+static DEVICE_ATTR(enable, 0644, bus_freq_scaling_enable_show, -+ bus_freq_scaling_enable_store); -+static DEVICE_ATTR(vpu352, 0444, vpu352_enable_show, -+ NULL); -+#endif -+ -+/*! -+ * This is the probe routine for the bus frequency driver. -+ * -+ * @param pdev The platform device structure -+ * -+ * @return The function returns 0 on success -+ * -+ */ -+ -+static int busfreq_probe(struct platform_device *pdev) -+{ -+#ifdef CONFIG_ARM_IMX6Q_CPUFREQ -+ u32 err; -+ -+ busfreq_dev = &pdev->dev; -+ -+ pll2_400 = devm_clk_get(&pdev->dev, "pll2_pfd2_396m"); -+ if (IS_ERR(pll2_400)) { -+ dev_err(busfreq_dev, "%s: failed to get pll2_pfd2_396m\n", -+ __func__); -+ return PTR_ERR(pll2_400); -+ } -+ -+ pll2_200 = devm_clk_get(&pdev->dev, "pll2_198m"); -+ if (IS_ERR(pll2_200)) { -+ dev_err(busfreq_dev, "%s: failed to get pll2_198m\n", -+ __func__); -+ return PTR_ERR(pll2_200); -+ } -+ -+ pll2 = devm_clk_get(&pdev->dev, "pll2_bus"); -+ if (IS_ERR(pll2)) { -+ dev_err(busfreq_dev, "%s: failed to get pll2_bus\n", -+ __func__); -+ return PTR_ERR(pll2); -+ } -+ -+ cpu_clk = devm_clk_get(&pdev->dev, "arm"); -+ if (IS_ERR(cpu_clk)) { -+ dev_err(busfreq_dev, "%s: failed to get cpu_clk\n", -+ __func__); -+ return PTR_ERR(cpu_clk); -+ } -+ -+ pll3 = devm_clk_get(&pdev->dev, "pll3_usb_otg"); -+ if (IS_ERR(pll3)) { -+ dev_err(busfreq_dev, "%s: failed to get pll3_usb_otg\n", -+ __func__); -+ return PTR_ERR(pll3); -+ } -+ -+ periph_clk = devm_clk_get(&pdev->dev, "periph"); -+ if (IS_ERR(periph_clk)) { -+ dev_err(busfreq_dev, "%s: failed to get periph\n", -+ __func__); -+ return PTR_ERR(periph_clk); -+ } -+ -+ periph_pre_clk = devm_clk_get(&pdev->dev, "periph_pre"); -+ if (IS_ERR(periph_pre_clk)) { -+ dev_err(busfreq_dev, "%s: failed to get periph_pre\n", -+ __func__); -+ return PTR_ERR(periph_pre_clk); -+ } -+ -+ periph_clk2 = devm_clk_get(&pdev->dev, "periph_clk2"); -+ if (IS_ERR(periph_clk2)) { -+ dev_err(busfreq_dev, "%s: failed to get periph_clk2\n", -+ __func__); -+ return PTR_ERR(periph_clk2); -+ } -+ -+ periph_clk2_sel = devm_clk_get(&pdev->dev, "periph_clk2_sel"); -+ if (IS_ERR(periph_clk2_sel)) { -+ dev_err(busfreq_dev, "%s: failed to get periph_clk2_sel\n", -+ __func__); -+ return PTR_ERR(periph_clk2_sel); -+ } -+ -+ osc_clk = devm_clk_get(&pdev->dev, "osc"); -+ if (IS_ERR(osc_clk)) { -+ dev_err(busfreq_dev, "%s: failed to get osc_clk\n", -+ __func__); -+ return PTR_ERR(osc_clk); -+ } -+ -+ if (cpu_is_imx6dl()) { -+ axi_sel_clk = devm_clk_get(&pdev->dev, "axi_sel"); -+ if (IS_ERR(axi_sel_clk)) { -+ dev_err(busfreq_dev, "%s: failed to get axi_sel_clk\n", -+ __func__); -+ return PTR_ERR(axi_sel_clk); -+ } -+ -+ pll3_pfd1_540m = devm_clk_get(&pdev->dev, "pll3_pfd1_540m"); -+ if (IS_ERR(pll3_pfd1_540m)) { -+ dev_err(busfreq_dev, -+ "%s: failed to get pll3_pfd1_540m\n", __func__); -+ return PTR_ERR(pll3_pfd1_540m); -+ } -+ } -+ -+ if (cpu_is_imx6sl()) { -+ pll1_sys = devm_clk_get(&pdev->dev, "pll1_sys"); -+ if (IS_ERR(pll1_sys)) { -+ dev_err(busfreq_dev, "%s: failed to get pll1_sys\n", -+ __func__); -+ return PTR_ERR(pll1_sys); -+ } -+ -+ ahb_clk = devm_clk_get(&pdev->dev, "ahb"); -+ if (IS_ERR(ahb_clk)) { -+ dev_err(busfreq_dev, "%s: failed to get ahb_clk\n", -+ __func__); -+ return PTR_ERR(ahb_clk); -+ } -+ -+ ocram_clk = devm_clk_get(&pdev->dev, "ocram"); -+ if (IS_ERR(ocram_clk)) { -+ dev_err(busfreq_dev, "%s: failed to get ocram_clk\n", -+ __func__); -+ return PTR_ERR(ocram_clk); -+ } -+ -+ pll1_sw_clk = devm_clk_get(&pdev->dev, "pll1_sw"); -+ if (IS_ERR(pll1_sw_clk)) { -+ dev_err(busfreq_dev, "%s: failed to get pll1_sw_clk\n", -+ __func__); -+ return PTR_ERR(pll1_sw_clk); -+ } -+ -+ periph2_clk = devm_clk_get(&pdev->dev, "periph2"); -+ if (IS_ERR(periph2_clk)) { -+ dev_err(busfreq_dev, "%s: failed to get periph2\n", -+ __func__); -+ return PTR_ERR(periph2_clk); -+ } -+ -+ periph2_pre_clk = devm_clk_get(&pdev->dev, "periph2_pre"); -+ if (IS_ERR(periph2_pre_clk)) { -+ dev_err(busfreq_dev, -+ "%s: failed to get periph2_pre_clk\n", -+ __func__); -+ return PTR_ERR(periph2_pre_clk); -+ } -+ -+ periph2_clk2 = devm_clk_get(&pdev->dev, "periph2_clk2"); -+ if (IS_ERR(periph2_clk2)) { -+ dev_err(busfreq_dev, -+ "%s: failed to get periph2_clk2\n", -+ __func__); -+ return PTR_ERR(periph2_clk2); -+ } -+ -+ periph2_clk2_sel = devm_clk_get(&pdev->dev, "periph2_clk2_sel"); -+ if (IS_ERR(periph2_clk2_sel)) { -+ dev_err(busfreq_dev, -+ "%s: failed to get periph2_clk2_sel\n", -+ __func__); -+ return PTR_ERR(periph2_clk2_sel); -+ } -+ -+ step_clk = devm_clk_get(&pdev->dev, "step"); -+ if (IS_ERR(step_clk)) { -+ dev_err(busfreq_dev, -+ "%s: failed to get step_clk\n", -+ __func__); -+ return PTR_ERR(periph2_clk2_sel); -+ } -+ -+ } -+ -+ err = sysfs_create_file(&busfreq_dev->kobj, &dev_attr_enable.attr); -+ if (err) { -+ dev_err(busfreq_dev, -+ "Unable to register sysdev entry for BUSFREQ"); -+ return err; -+ } -+ err = sysfs_create_file(&busfreq_dev->kobj, &dev_attr_vpu352.attr); -+ if (err) { -+ dev_err(busfreq_dev, -+ "Unable to register sysdev entry for BUSFREQ"); -+ return err; -+ } -+ -+ if (of_property_read_u32(pdev->dev.of_node, "fsl,max_ddr_freq", -+ &ddr_normal_rate)) { -+ dev_err(busfreq_dev, "max_ddr_freq entry missing\n"); -+ return -EINVAL; -+ } -+#endif -+ -+ high_bus_freq_mode = 1; -+ med_bus_freq_mode = 0; -+ low_bus_freq_mode = 0; -+ audio_bus_freq_mode = 0; -+ ultra_low_bus_freq_mode = 0; -+ -+#ifdef CONFIG_ARM_IMX6Q_CPUFREQ -+ bus_freq_scaling_is_active = 1; -+ bus_freq_scaling_initialized = 1; -+ -+ ddr_low_rate = LPAPM_CLK; -+ if (cpu_is_imx6q()) { -+ if (of_property_read_u32(pdev->dev.of_node, "fsl,med_ddr_freq", -+ &ddr_med_rate)) { -+ dev_info(busfreq_dev, -+ "DDR medium rate not supported.\n"); -+ ddr_med_rate = ddr_normal_rate; -+ } -+ } -+ -+ INIT_DELAYED_WORK(&low_bus_freq_handler, reduce_bus_freq_handler); -+ INIT_DELAYED_WORK(&bus_freq_daemon, bus_freq_daemon_handler); -+ register_pm_notifier(&imx_bus_freq_pm_notifier); -+ register_reboot_notifier(&imx_busfreq_reboot_notifier); -+ -+ if (cpu_is_imx6sl()) -+ err = init_mmdc_lpddr2_settings(pdev); -+ else -+ err = init_mmdc_ddr3_settings(pdev); -+ if (err) { -+ dev_err(busfreq_dev, "Busfreq init of MMDC failed\n"); -+ return err; -+ } -+#endif -+ return 0; -+} -+ -+static const struct of_device_id imx6_busfreq_ids[] = { -+ { .compatible = "fsl,imx6_busfreq", }, -+ { /* sentinel */ } -+}; -+ -+static struct platform_driver busfreq_driver = { -+ .driver = { -+ .name = "imx6_busfreq", -+ .owner = THIS_MODULE, -+ .of_match_table = imx6_busfreq_ids, -+ }, -+ .probe = busfreq_probe, -+}; -+ -+/*! -+ * Initialise the busfreq_driver. -+ * -+ * @return The function always returns 0. -+ */ -+ -+static int __init busfreq_init(void) -+{ -+ if (vpu352) { -+ printk(KERN_INFO "VPU@352Mhz activated. Bus freq driver module not loading\n"); -+ return 0; -+ } -+ -+ if (platform_driver_register(&busfreq_driver) != 0) -+ return -ENODEV; -+ -+ printk(KERN_INFO "Bus freq driver module loaded\n"); -+ -+ return 0; -+} -+ -+static void __exit busfreq_cleanup(void) -+{ -+#ifdef CONFIG_ARM_IMX6Q_CPUFREQ -+ sysfs_remove_file(&busfreq_dev->kobj, &dev_attr_enable.attr); -+ -+ bus_freq_scaling_initialized = 0; -+#endif -+ /* Unregister the device structure */ -+ platform_driver_unregister(&busfreq_driver); -+} -+ -+__setup("vpu352=", vpu352_setup); -+module_init(busfreq_init); -+module_exit(busfreq_cleanup); -+ -+MODULE_AUTHOR("Freescale Semiconductor, Inc."); -+MODULE_DESCRIPTION("BusFreq driver"); -+MODULE_LICENSE("GPL"); -diff -Nur linux-4.1.3/arch/arm/mach-imx/busfreq_lpddr2.c linux-xbian-imx6/arch/arm/mach-imx/busfreq_lpddr2.c ---- linux-4.1.3/arch/arm/mach-imx/busfreq_lpddr2.c 1970-01-01 01:00:00.000000000 +0100 -+++ linux-xbian-imx6/arch/arm/mach-imx/busfreq_lpddr2.c 2015-07-27 23:13:01.073153409 +0200 -@@ -0,0 +1,183 @@ -+/* -+ * Copyright (C) 2011-2013 Freescale Semiconductor, Inc. All Rights Reserved. -+ */ -+ -+/* -+ * The code contained herein is licensed under the GNU General Public -+ * License. You may obtain a copy of the GNU General Public License -+ * Version 2 or later at the following locations: -+ * -+ * http://www.opensource.org/licenses/gpl-license.html -+ * http://www.gnu.org/copyleft/gpl.html -+ */ -+ -+/*! -+ * @file busfreq_lpddr2.c -+ * -+ * @brief iMX6 LPDDR2 frequency change specific file. -+ * -+ * @ingroup PM -+ */ -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+ -+#include "hardware.h" -+ -+/* DDR settings */ -+static void __iomem *mmdc_base; -+static void __iomem *anatop_base; -+static void __iomem *ccm_base; -+static void __iomem *l2_base; -+static struct device *busfreq_dev; -+static void *ddr_freq_change_iram_base; -+static int curr_ddr_rate; -+ -+unsigned long reg_addrs[4]; -+ -+void (*mx6_change_lpddr2_freq)(u32 ddr_freq, int bus_freq_mode, -+ void *iram_addr) = NULL; -+ -+extern unsigned int ddr_normal_rate; -+extern int low_bus_freq_mode; -+extern int ultra_low_bus_freq_mode; -+extern void mx6_lpddr2_freq_change(u32 freq, int bus_freq_mode, -+ void *iram_addr); -+ -+ -+#define LPDDR2_FREQ_CHANGE_SIZE 0x1000 -+ -+ -+/* change the DDR frequency. */ -+int update_lpddr2_freq(int ddr_rate) -+{ -+ if (ddr_rate == curr_ddr_rate) -+ return 0; -+ -+ dev_dbg(busfreq_dev, "\nBus freq set to %d start...\n", ddr_rate); -+ -+ /* -+ * Flush the TLB, to ensure no TLB maintenance occurs -+ * when DDR is in self-refresh. -+ */ -+ local_flush_tlb_all(); -+ /* Now change DDR frequency. */ -+ mx6_change_lpddr2_freq(ddr_rate, -+ (low_bus_freq_mode | ultra_low_bus_freq_mode), -+ reg_addrs); -+ -+ curr_ddr_rate = ddr_rate; -+ -+ dev_dbg(busfreq_dev, "\nBus freq set to %d done...\n", ddr_rate); -+ -+ return 0; -+} -+ -+int init_mmdc_lpddr2_settings(struct platform_device *busfreq_pdev) -+{ -+ struct platform_device *ocram_dev; -+ unsigned int iram_paddr; -+ struct device_node *node; -+ struct gen_pool *iram_pool; -+ -+ busfreq_dev = &busfreq_pdev->dev; -+ node = of_find_compatible_node(NULL, NULL, "fsl,imx6sl-mmdc"); -+ if (!node) { -+ printk(KERN_ERR "failed to find imx6sl-mmdc device tree data!\n"); -+ return -EINVAL; -+ } -+ mmdc_base = of_iomap(node, 0); -+ WARN(!mmdc_base, "unable to map mmdc registers\n"); -+ -+ node = NULL; -+ node = of_find_compatible_node(NULL, NULL, "fsl,imx6sl-ccm"); -+ if (!node) { -+ printk(KERN_ERR "failed to find imx6sl-ccm device tree data!\n"); -+ return -EINVAL; -+ } -+ ccm_base = of_iomap(node, 0); -+ WARN(!ccm_base, "unable to map ccm registers\n"); -+ -+ node = of_find_compatible_node(NULL, NULL, "arm,pl310-cache"); -+ if (!node) { -+ printk(KERN_ERR "failed to find imx6sl-pl310-cache device tree data!\n"); -+ return -EINVAL; -+ } -+ l2_base = of_iomap(node, 0); -+ WARN(!l2_base, "unable to map PL310 registers\n"); -+ -+ node = of_find_compatible_node(NULL, NULL, "fsl,imx6sl-anatop"); -+ if (!node) { -+ printk(KERN_ERR "failed to find imx6sl-pl310-cache device tree data!\n"); -+ return -EINVAL; -+ } -+ anatop_base = of_iomap(node, 0); -+ WARN(!anatop_base, "unable to map anatop registers\n"); -+ -+ node = NULL; -+ node = of_find_compatible_node(NULL, NULL, "mmio-sram"); -+ if (!node) { -+ dev_err(busfreq_dev, "%s: failed to find ocram node\n", -+ __func__); -+ return -EINVAL; -+ } -+ -+ ocram_dev = of_find_device_by_node(node); -+ if (!ocram_dev) { -+ dev_err(busfreq_dev, "failed to find ocram device!\n"); -+ return -EINVAL; -+ } -+ -+ iram_pool = dev_get_gen_pool(&ocram_dev->dev); -+ if (!iram_pool) { -+ dev_err(busfreq_dev, "iram pool unavailable!\n"); -+ return -EINVAL; -+ } -+ -+ reg_addrs[0] = (unsigned long)anatop_base; -+ reg_addrs[1] = (unsigned long)ccm_base; -+ reg_addrs[2] = (unsigned long)mmdc_base; -+ reg_addrs[3] = (unsigned long)l2_base; -+ -+ ddr_freq_change_iram_base = (void *)gen_pool_alloc(iram_pool, -+ LPDDR2_FREQ_CHANGE_SIZE); -+ if (!ddr_freq_change_iram_base) { -+ dev_err(busfreq_dev, -+ "Cannot alloc iram for ddr freq change code!\n"); -+ return -ENOMEM; -+ } -+ -+ iram_paddr = gen_pool_virt_to_phys(iram_pool, -+ (unsigned long)ddr_freq_change_iram_base); -+ /* -+ * Need to remap the area here since we want -+ * the memory region to be executable. -+ */ -+ ddr_freq_change_iram_base = __arm_ioremap(iram_paddr, -+ LPDDR2_FREQ_CHANGE_SIZE, -+ MT_MEMORY_RWX_NONCACHED); -+ mx6_change_lpddr2_freq = (void *)fncpy(ddr_freq_change_iram_base, -+ &mx6_lpddr2_freq_change, LPDDR2_FREQ_CHANGE_SIZE); -+ -+ curr_ddr_rate = ddr_normal_rate; -+ -+ return 0; -+} -diff -Nur linux-4.1.3/arch/arm/mach-imx/clk.h linux-xbian-imx6/arch/arm/mach-imx/clk.h ---- linux-4.1.3/arch/arm/mach-imx/clk.h 2015-07-21 19:10:33.000000000 +0200 -+++ linux-xbian-imx6/arch/arm/mach-imx/clk.h 2015-07-27 23:13:01.085110746 +0200 -@@ -55,6 +55,34 @@ - shift, 0, &imx_ccm_lock, share_count); - } - -+static inline void imx_clk_prepare_enable(struct clk *clk) -+{ -+ int ret = clk_prepare_enable(clk); -+ -+ if (ret) -+ pr_err("failed to prepare and enable clk %s: %d\n", -+ __clk_get_name(clk), ret); -+} -+ -+static inline int imx_clk_set_parent(struct clk *clk, struct clk *parent) -+{ -+ int ret = clk_set_parent(clk, parent); -+ -+ if (ret) -+ pr_err("failed to set parent of clk %s to %s: %d\n", -+ __clk_get_name(clk), __clk_get_name(parent), ret); -+ return ret; -+} -+ -+static inline void imx_clk_set_rate(struct clk *clk, unsigned long rate) -+{ -+ int ret = clk_set_rate(clk, rate); -+ -+ if (ret) -+ pr_err("failed to set rate of clk %s to %ld: %d\n", -+ __clk_get_name(clk), rate, ret); -+} -+ - struct clk *imx_clk_pfd(const char *name, const char *parent_name, - void __iomem *reg, u8 idx); - -diff -Nur linux-4.1.3/arch/arm/mach-imx/clk-imx6q.c linux-xbian-imx6/arch/arm/mach-imx/clk-imx6q.c ---- linux-4.1.3/arch/arm/mach-imx/clk-imx6q.c 2015-07-21 19:10:33.000000000 +0200 -+++ linux-xbian-imx6/arch/arm/mach-imx/clk-imx6q.c 2015-07-27 23:13:01.081124967 +0200 -@@ -24,7 +24,6 @@ - #include "clk.h" - #include "common.h" - #include "hardware.h" -- - static const char *step_sels[] = { "osc", "pll2_pfd2_396m", }; - static const char *pll1_sw_sels[] = { "pll1_sys", "step", }; - static const char *periph_pre_sels[] = { "pll2_bus", "pll2_pfd2_396m", "pll2_pfd0_352m", "pll2_198m", }; -@@ -41,6 +40,8 @@ - static const char *ipu_sels[] = { "mmdc_ch0_axi", "pll2_pfd2_396m", "pll3_120m", "pll3_pfd1_540m", }; - static const char *ldb_di_sels[] = { "pll5_video_div", "pll2_pfd0_352m", "pll2_pfd2_396m", "mmdc_ch1_axi", "pll3_usb_otg", }; - static const char *ipu_di_pre_sels[] = { "mmdc_ch0_axi", "pll3_usb_otg", "pll5_video_div", "pll2_pfd0_352m", "pll2_pfd2_396m", "pll3_pfd1_540m", }; -+static const char *ldb_di0_div_sels[] = { "ldb_di0_div_3_5", "ldb_di0_div_7", }; -+static const char *ldb_di1_div_sels[] = { "ldb_di1_div_3_5", "ldb_di1_div_7", }; - static const char *ipu1_di0_sels[] = { "ipu1_di0_pre", "dummy", "dummy", "ldb_di0", "ldb_di1", }; - static const char *ipu1_di1_sels[] = { "ipu1_di1_pre", "dummy", "dummy", "ldb_di0", "ldb_di1", }; - static const char *ipu2_di0_sels[] = { "ipu2_di0_pre", "dummy", "dummy", "ldb_di0", "ldb_di1", }; -@@ -119,8 +120,118 @@ - static unsigned int share_count_ssi1; - static unsigned int share_count_ssi2; - static unsigned int share_count_ssi3; -+static unsigned int share_count_spdif; - static unsigned int share_count_mipi_core_cfg; - -+static void __iomem *ccm_base; -+ -+static void init_ldb_clks(struct clk *new_parent) -+{ -+ u32 reg; -+ -+ /* -+ * Need to follow a strict procedure when changing the LDB -+ * clock, else we can introduce a glitch. Things to keep in -+ * mind: -+ * 1. The current and new parent clocks must be disabled. -+ * 2. The default clock for ldb_dio_clk is mmdc_ch1 which has -+ * no CG bit. -+ * 3. In the RTL implementation of the LDB_DI_CLK_SEL mux -+ * the top four options are in one mux and the PLL3 option along -+ * with another option is in the second mux. There is third mux -+ * used to decide between the first and second mux. -+ * The code below switches the parent to the bottom mux first -+ * and then manipulates the top mux. This ensures that no glitch -+ * will enter the divider. -+ * -+ * Need to disable MMDC_CH1 clock manually as there is no CG bit -+ * for this clock. The only way to disable this clock is to move -+ * it topll3_sw_clk and then to disable pll3_sw_clk -+ * Make sure periph2_clk2_sel is set to pll3_sw_clk -+ */ -+ reg = readl_relaxed(ccm_base + 0x18); -+ reg &= ~(1 << 20); -+ writel_relaxed(reg, ccm_base + 0x18); -+ -+ /* -+ * Set MMDC_CH1 mask bit. -+ */ -+ reg = readl_relaxed(ccm_base + 0x4); -+ reg |= 1 << 16; -+ writel_relaxed(reg, ccm_base + 0x4); -+ -+ /* -+ * Set the periph2_clk_sel to the top mux so that -+ * mmdc_ch1 is from pll3_sw_clk. -+ */ -+ reg = readl_relaxed(ccm_base + 0x14); -+ reg |= 1 << 26; -+ writel_relaxed(reg, ccm_base + 0x14); -+ -+ /* -+ * Wait for the clock switch. -+ */ -+ while (readl_relaxed(ccm_base + 0x48)) -+ ; -+ -+ /* -+ * Disable pll3_sw_clk by selecting the bypass clock source. -+ */ -+ reg = readl_relaxed(ccm_base + 0xc); -+ reg |= 1 << 0; -+ writel_relaxed(reg, ccm_base + 0xc); -+ -+ /* -+ * Set the ldb_di0_clk and ldb_di1_clk to 111b. -+ */ -+ reg = readl_relaxed(ccm_base + 0x2c); -+ reg |= ((7 << 9) | (7 << 12)); -+ writel_relaxed(reg, ccm_base + 0x2c); -+ -+ /* -+ * Set the ldb_di0_clk and ldb_di1_clk to 100b. -+ */ -+ reg = readl_relaxed(ccm_base + 0x2c); -+ reg &= ~((7 << 9) | (7 << 12)); -+ reg |= ((4 << 9) | (4 << 12)); -+ writel_relaxed(reg, ccm_base + 0x2c); -+ -+ /* -+ * Perform the LDB parent clock switch. -+ */ -+ imx_clk_set_parent(clk[IMX6QDL_CLK_LDB_DI0_SEL], new_parent); -+ imx_clk_set_parent(clk[IMX6QDL_CLK_LDB_DI1_SEL], new_parent); -+ -+ /* -+ * Unbypass pll3_sw_clk. -+ */ -+ reg = readl_relaxed(ccm_base + 0xc); -+ reg &= ~(1 << 0); -+ writel_relaxed(reg, ccm_base + 0xc); -+ -+ /* -+ * Set the periph2_clk_sel back to the bottom mux so that -+ * mmdc_ch1 is from its original parent. -+ */ -+ reg = readl_relaxed(ccm_base + 0x14); -+ reg &= ~(1 << 26); -+ writel_relaxed(reg, ccm_base + 0x14); -+ -+ /* -+ * Wait for the clock switch. -+ */ -+ while (readl_relaxed(ccm_base + 0x48)) -+ ; -+ -+ /* -+ * Clear MMDC_CH1 mask bit. -+ */ -+ reg = readl_relaxed(ccm_base + 0x4); -+ reg &= ~(1 << 16); -+ writel_relaxed(reg, ccm_base + 0x4); -+ -+} -+ - static void __init imx6q_clocks_init(struct device_node *ccm_node) - { - struct device_node *np; -@@ -174,13 +285,13 @@ - clk[IMX6QDL_PLL7_BYPASS] = imx_clk_mux_flags("pll7_bypass", base + 0x20, 16, 1, pll7_bypass_sels, ARRAY_SIZE(pll7_bypass_sels), CLK_SET_RATE_PARENT); - - /* Do not bypass PLLs initially */ -- clk_set_parent(clk[IMX6QDL_PLL1_BYPASS], clk[IMX6QDL_CLK_PLL1]); -- clk_set_parent(clk[IMX6QDL_PLL2_BYPASS], clk[IMX6QDL_CLK_PLL2]); -- clk_set_parent(clk[IMX6QDL_PLL3_BYPASS], clk[IMX6QDL_CLK_PLL3]); -- clk_set_parent(clk[IMX6QDL_PLL4_BYPASS], clk[IMX6QDL_CLK_PLL4]); -- clk_set_parent(clk[IMX6QDL_PLL5_BYPASS], clk[IMX6QDL_CLK_PLL5]); -- clk_set_parent(clk[IMX6QDL_PLL6_BYPASS], clk[IMX6QDL_CLK_PLL6]); -- clk_set_parent(clk[IMX6QDL_PLL7_BYPASS], clk[IMX6QDL_CLK_PLL7]); -+ imx_clk_set_parent(clk[IMX6QDL_PLL1_BYPASS], clk[IMX6QDL_CLK_PLL1]); -+ imx_clk_set_parent(clk[IMX6QDL_PLL2_BYPASS], clk[IMX6QDL_CLK_PLL2]); -+ imx_clk_set_parent(clk[IMX6QDL_PLL3_BYPASS], clk[IMX6QDL_CLK_PLL3]); -+ imx_clk_set_parent(clk[IMX6QDL_PLL4_BYPASS], clk[IMX6QDL_CLK_PLL4]); -+ imx_clk_set_parent(clk[IMX6QDL_PLL5_BYPASS], clk[IMX6QDL_CLK_PLL5]); -+ imx_clk_set_parent(clk[IMX6QDL_PLL6_BYPASS], clk[IMX6QDL_CLK_PLL6]); -+ imx_clk_set_parent(clk[IMX6QDL_PLL7_BYPASS], clk[IMX6QDL_CLK_PLL7]); - - clk[IMX6QDL_CLK_PLL1_SYS] = imx_clk_gate("pll1_sys", "pll1_bypass", base + 0x00, 13); - clk[IMX6QDL_CLK_PLL2_BUS] = imx_clk_gate("pll2_bus", "pll2_bypass", base + 0x30, 13); -@@ -259,7 +370,7 @@ - clk[IMX6QDL_CLK_PLL5_VIDEO_DIV] = clk_register_divider_table(NULL, "pll5_video_div", "pll5_post_div", CLK_SET_RATE_PARENT, base + 0x170, 30, 2, 0, video_div_table, &imx_ccm_lock); - - np = ccm_node; -- base = of_iomap(np, 0); -+ ccm_base = base = of_iomap(np, 0); - WARN_ON(!base); - - imx6q_pm_set_ccm_base(base); -@@ -286,6 +397,8 @@ - clk[IMX6QDL_CLK_IPU2_SEL] = imx_clk_mux("ipu2_sel", base + 0x3c, 14, 2, ipu_sels, ARRAY_SIZE(ipu_sels)); - clk[IMX6QDL_CLK_LDB_DI0_SEL] = imx_clk_mux_flags("ldb_di0_sel", base + 0x2c, 9, 3, ldb_di_sels, ARRAY_SIZE(ldb_di_sels), CLK_SET_RATE_PARENT); - clk[IMX6QDL_CLK_LDB_DI1_SEL] = imx_clk_mux_flags("ldb_di1_sel", base + 0x2c, 12, 3, ldb_di_sels, ARRAY_SIZE(ldb_di_sels), CLK_SET_RATE_PARENT); -+ clk[IMX6QDL_CLK_LDB_DI0_DIV_SEL] = imx_clk_mux_flags("ldb_di0_div_sel", base + 0x20, 10, 1, ldb_di0_div_sels, ARRAY_SIZE(ldb_di0_div_sels), CLK_SET_RATE_PARENT); -+ clk[IMX6QDL_CLK_LDB_DI1_DIV_SEL] = imx_clk_mux_flags("ldb_di1_div_sel", base + 0x20, 11, 1, ldb_di1_div_sels, ARRAY_SIZE(ldb_di1_div_sels), CLK_SET_RATE_PARENT); - clk[IMX6QDL_CLK_IPU1_DI0_PRE_SEL] = imx_clk_mux_flags("ipu1_di0_pre_sel", base + 0x34, 6, 3, ipu_di_pre_sels, ARRAY_SIZE(ipu_di_pre_sels), CLK_SET_RATE_PARENT); - clk[IMX6QDL_CLK_IPU1_DI1_PRE_SEL] = imx_clk_mux_flags("ipu1_di1_pre_sel", base + 0x34, 15, 3, ipu_di_pre_sels, ARRAY_SIZE(ipu_di_pre_sels), CLK_SET_RATE_PARENT); - clk[IMX6QDL_CLK_IPU2_DI0_PRE_SEL] = imx_clk_mux_flags("ipu2_di0_pre_sel", base + 0x38, 6, 3, ipu_di_pre_sels, ARRAY_SIZE(ipu_di_pre_sels), CLK_SET_RATE_PARENT); -@@ -335,9 +448,9 @@ - clk[IMX6QDL_CLK_IPU1_PODF] = imx_clk_divider("ipu1_podf", "ipu1_sel", base + 0x3c, 11, 3); - clk[IMX6QDL_CLK_IPU2_PODF] = imx_clk_divider("ipu2_podf", "ipu2_sel", base + 0x3c, 16, 3); - clk[IMX6QDL_CLK_LDB_DI0_DIV_3_5] = imx_clk_fixed_factor("ldb_di0_div_3_5", "ldb_di0_sel", 2, 7); -- clk[IMX6QDL_CLK_LDB_DI0_PODF] = imx_clk_divider_flags("ldb_di0_podf", "ldb_di0_div_3_5", base + 0x20, 10, 1, 0); -+ clk[IMX6QDL_CLK_LDB_DI0_DIV_7] = imx_clk_fixed_factor("ldb_di0_div_7", "ldb_di0_sel", 1, 7); - clk[IMX6QDL_CLK_LDB_DI1_DIV_3_5] = imx_clk_fixed_factor("ldb_di1_div_3_5", "ldb_di1_sel", 2, 7); -- clk[IMX6QDL_CLK_LDB_DI1_PODF] = imx_clk_divider_flags("ldb_di1_podf", "ldb_di1_div_3_5", base + 0x20, 11, 1, 0); -+ clk[IMX6QDL_CLK_LDB_DI1_DIV_7] = imx_clk_fixed_factor("ldb_di1_div_7", "ldb_di1_sel", 1, 7); - clk[IMX6QDL_CLK_IPU1_DI0_PRE] = imx_clk_divider("ipu1_di0_pre", "ipu1_di0_pre_sel", base + 0x34, 3, 3); - clk[IMX6QDL_CLK_IPU1_DI1_PRE] = imx_clk_divider("ipu1_di1_pre", "ipu1_di1_pre_sel", base + 0x34, 12, 3); - clk[IMX6QDL_CLK_IPU2_DI0_PRE] = imx_clk_divider("ipu2_di0_pre", "ipu2_di0_pre_sel", base + 0x38, 3, 3); -@@ -378,6 +491,8 @@ - clk[IMX6QDL_CLK_CAN1_SERIAL] = imx_clk_gate2("can1_serial", "can_root", base + 0x68, 16); - clk[IMX6QDL_CLK_CAN2_IPG] = imx_clk_gate2("can2_ipg", "ipg", base + 0x68, 18); - clk[IMX6QDL_CLK_CAN2_SERIAL] = imx_clk_gate2("can2_serial", "can_root", base + 0x68, 20); -+ clk[IMX6QDL_CLK_DCIC1] = imx_clk_gate2("dcic1", "ipu1_podf", base + 0x68, 24); -+ clk[IMX6QDL_CLK_DCIC2] = imx_clk_gate2("dcic2", "ipu2_podf", base + 0x68, 26); - clk[IMX6QDL_CLK_ECSPI1] = imx_clk_gate2("ecspi1", "ecspi_root", base + 0x6c, 0); - clk[IMX6QDL_CLK_ECSPI2] = imx_clk_gate2("ecspi2", "ecspi_root", base + 0x6c, 2); - clk[IMX6QDL_CLK_ECSPI3] = imx_clk_gate2("ecspi3", "ecspi_root", base + 0x6c, 4); -@@ -414,9 +529,9 @@ - clk[IMX6QDL_CLK_IPU1_DI1] = imx_clk_gate2("ipu1_di1", "ipu1_di1_sel", base + 0x74, 4); - clk[IMX6QDL_CLK_IPU2] = imx_clk_gate2("ipu2", "ipu2_podf", base + 0x74, 6); - clk[IMX6QDL_CLK_IPU2_DI0] = imx_clk_gate2("ipu2_di0", "ipu2_di0_sel", base + 0x74, 8); -- clk[IMX6QDL_CLK_LDB_DI0] = imx_clk_gate2("ldb_di0", "ldb_di0_podf", base + 0x74, 12); -- clk[IMX6QDL_CLK_LDB_DI1] = imx_clk_gate2("ldb_di1", "ldb_di1_podf", base + 0x74, 14); - clk[IMX6QDL_CLK_IPU2_DI1] = imx_clk_gate2("ipu2_di1", "ipu2_di1_sel", base + 0x74, 10); -+ clk[IMX6QDL_CLK_LDB_DI0] = imx_clk_gate2("ldb_di0", "ldb_di0_div_sel", base + 0x74, 12); -+ clk[IMX6QDL_CLK_LDB_DI1] = imx_clk_gate2("ldb_di1", "ldb_di1_div_sel", base + 0x74, 14); - clk[IMX6QDL_CLK_HSI_TX] = imx_clk_gate2_shared("hsi_tx", "hsi_tx_podf", base + 0x74, 16, &share_count_mipi_core_cfg); - clk[IMX6QDL_CLK_MIPI_CORE_CFG] = imx_clk_gate2_shared("mipi_core_cfg", "video_27m", base + 0x74, 16, &share_count_mipi_core_cfg); - clk[IMX6QDL_CLK_MIPI_IPG] = imx_clk_gate2_shared("mipi_ipg", "ipg", base + 0x74, 16, &share_count_mipi_core_cfg); -@@ -446,7 +561,8 @@ - clk[IMX6QDL_CLK_SATA] = imx_clk_gate2("sata", "ahb", base + 0x7c, 4); - clk[IMX6QDL_CLK_SDMA] = imx_clk_gate2("sdma", "ahb", base + 0x7c, 6); - clk[IMX6QDL_CLK_SPBA] = imx_clk_gate2("spba", "ipg", base + 0x7c, 12); -- clk[IMX6QDL_CLK_SPDIF] = imx_clk_gate2("spdif", "spdif_podf", base + 0x7c, 14); -+ clk[IMX6QDL_CLK_SPDIF] = imx_clk_gate2_shared("spdif", "spdif_podf", base + 0x7c, 14, &share_count_spdif); -+ clk[IMX6QDL_CLK_SPDIF_GCLK] = imx_clk_gate2_shared("spdif_gclk", "ipg", base + 0x7c, 14, &share_count_spdif); - clk[IMX6QDL_CLK_SSI1_IPG] = imx_clk_gate2_shared("ssi1_ipg", "ipg", base + 0x7c, 18, &share_count_ssi1); - clk[IMX6QDL_CLK_SSI2_IPG] = imx_clk_gate2_shared("ssi2_ipg", "ipg", base + 0x7c, 20, &share_count_ssi2); - clk[IMX6QDL_CLK_SSI3_IPG] = imx_clk_gate2_shared("ssi3_ipg", "ipg", base + 0x7c, 22, &share_count_ssi3); -@@ -479,54 +595,104 @@ - clk_data.clk_num = ARRAY_SIZE(clk); - of_clk_add_provider(np, of_clk_src_onecell_get, &clk_data); - -+ clk_register_clkdev(clk[IMX6QDL_CLK_GPT_3M], "gpt_3m", "imx-gpt.0"); - clk_register_clkdev(clk[IMX6QDL_CLK_ENET_REF], "enet_ref", NULL); - - if ((imx_get_soc_revision() != IMX_CHIP_REVISION_1_0) || - cpu_is_imx6dl()) { -- clk_set_parent(clk[IMX6QDL_CLK_LDB_DI0_SEL], clk[IMX6QDL_CLK_PLL5_VIDEO_DIV]); -- clk_set_parent(clk[IMX6QDL_CLK_LDB_DI1_SEL], clk[IMX6QDL_CLK_PLL5_VIDEO_DIV]); -+ imx_clk_set_parent(clk[IMX6QDL_CLK_LDB_DI0_SEL], clk[IMX6QDL_CLK_PLL5_VIDEO_DIV]); -+ imx_clk_set_parent(clk[IMX6QDL_CLK_LDB_DI1_SEL], clk[IMX6QDL_CLK_PLL5_VIDEO_DIV]); - } - -- clk_set_parent(clk[IMX6QDL_CLK_IPU1_DI0_PRE_SEL], clk[IMX6QDL_CLK_PLL5_VIDEO_DIV]); -- clk_set_parent(clk[IMX6QDL_CLK_IPU1_DI1_PRE_SEL], clk[IMX6QDL_CLK_PLL5_VIDEO_DIV]); -- clk_set_parent(clk[IMX6QDL_CLK_IPU2_DI0_PRE_SEL], clk[IMX6QDL_CLK_PLL5_VIDEO_DIV]); -- clk_set_parent(clk[IMX6QDL_CLK_IPU2_DI1_PRE_SEL], clk[IMX6QDL_CLK_PLL5_VIDEO_DIV]); -- clk_set_parent(clk[IMX6QDL_CLK_IPU1_DI0_SEL], clk[IMX6QDL_CLK_IPU1_DI0_PRE]); -- clk_set_parent(clk[IMX6QDL_CLK_IPU1_DI1_SEL], clk[IMX6QDL_CLK_IPU1_DI1_PRE]); -- clk_set_parent(clk[IMX6QDL_CLK_IPU2_DI0_SEL], clk[IMX6QDL_CLK_IPU2_DI0_PRE]); -- clk_set_parent(clk[IMX6QDL_CLK_IPU2_DI1_SEL], clk[IMX6QDL_CLK_IPU2_DI1_PRE]); -+ init_ldb_clks(clk[IMX6QDL_CLK_PLL2_PFD0_352M]); -+ imx_clk_set_parent(clk[IMX6QDL_CLK_IPU1_DI0_PRE_SEL], clk[IMX6QDL_CLK_PLL5_VIDEO_DIV]); -+ imx_clk_set_parent(clk[IMX6QDL_CLK_IPU1_DI1_PRE_SEL], clk[IMX6QDL_CLK_PLL5_VIDEO_DIV]); -+ imx_clk_set_parent(clk[IMX6QDL_CLK_IPU2_DI0_PRE_SEL], clk[IMX6QDL_CLK_PLL5_VIDEO_DIV]); -+ imx_clk_set_parent(clk[IMX6QDL_CLK_IPU2_DI1_PRE_SEL], clk[IMX6QDL_CLK_PLL5_VIDEO_DIV]); -+ imx_clk_set_parent(clk[IMX6QDL_CLK_IPU1_DI0_SEL], clk[IMX6QDL_CLK_IPU1_DI0_PRE]); -+ imx_clk_set_parent(clk[IMX6QDL_CLK_IPU1_DI1_SEL], clk[IMX6QDL_CLK_IPU1_DI1_PRE]); -+ imx_clk_set_parent(clk[IMX6QDL_CLK_IPU2_DI0_SEL], clk[IMX6QDL_CLK_IPU2_DI0_PRE]); -+ imx_clk_set_parent(clk[IMX6QDL_CLK_IPU2_DI1_SEL], clk[IMX6QDL_CLK_IPU2_DI1_PRE]); -+ -+ if (cpu_is_imx6dl()) -+ imx_clk_set_parent(clk[IMX6QDL_CLK_IPU1_SEL], clk[IMX6QDL_CLK_PLL3_PFD1_540M]); - - /* - * The gpmi needs 100MHz frequency in the EDO/Sync mode, - * We can not get the 100MHz from the pll2_pfd0_352m. - * So choose pll2_pfd2_396m as enfc_sel's parent. - */ -- clk_set_parent(clk[IMX6QDL_CLK_ENFC_SEL], clk[IMX6QDL_CLK_PLL2_PFD2_396M]); -- -- for (i = 0; i < ARRAY_SIZE(clks_init_on); i++) -- clk_prepare_enable(clk[clks_init_on[i]]); -+ imx_clk_set_parent(clk[IMX6QDL_CLK_ENFC_SEL], clk[IMX6QDL_CLK_PLL2_PFD2_396M]); - -+ /* gpu clock initilazation */ -+ /* -+ * On mx6dl, 2d core clock sources(sel, podf) is from 3d -+ * shader core clock, but 3d shader clock multiplexer of -+ * mx6dl is different. For instance the equivalent of -+ * pll2_pfd_594M on mx6q is pll2_pfd_528M on mx6dl. -+ * Make a note here. -+ */ -+#if 0 -+ imx_clk_set_parent(clk[IMX6QDL_CLK_GPU3D_SHADER_SEL], clk[IMX6QDL_CLK_PLL2_PFD1_594M]); -+ if (cpu_is_imx6dl()) { -+ imx_clk_set_rate(clk[IMX6QDL_CLK_GPU3D_SHADER], 528000000); -+ /* for mx6dl, change gpu3d_core parent to 594_PFD*/ -+ imx_clk_set_parent(clk[IMX6QDL_CLK_GPU3D_CORE_SEL], clk[IMX6QDL_CLK_PLL2_PFD1_594M]); -+ imx_clk_set_rate(clk[IMX6QDL_CLK_GPU3D_CORE], 528000000); -+ /* for mx6dl, change gpu2d_core parent to 594_PFD*/ -+ imx_clk_set_parent(clk[IMX6QDL_CLK_GPU2D_CORE_SEL], clk[IMX6QDL_CLK_PLL2_PFD1_594M]); -+ imx_clk_set_rate(clk[IMX6QDL_CLK_GPU2D_CORE], 528000000); -+ } else if (cpu_is_imx6q()) { -+ imx_clk_set_rate(clk[IMX6QDL_CLK_GPU3D_SHADER], 594000000); -+ imx_clk_set_parent(clk[IMX6QDL_CLK_GPU3D_CORE_SEL], clk[IMX6QDL_CLK_MMDC_CH0_AXI]); -+ imx_clk_set_rate(clk[IMX6QDL_CLK_GPU3D_CORE], 528000000); -+ imx_clk_set_parent(clk[IMX6QDL_CLK_GPU2D_CORE_SEL], clk[IMX6QDL_CLK_PLL3_USB_OTG]); -+ imx_clk_set_rate(clk[IMX6QDL_CLK_GPU2D_CORE], 480000000); -+ } -+#endif - if (IS_ENABLED(CONFIG_USB_MXS_PHY)) { -- clk_prepare_enable(clk[IMX6QDL_CLK_USBPHY1_GATE]); -- clk_prepare_enable(clk[IMX6QDL_CLK_USBPHY2_GATE]); -+ imx_clk_prepare_enable(clk[IMX6QDL_CLK_USBPHY1_GATE]); -+ imx_clk_prepare_enable(clk[IMX6QDL_CLK_USBPHY2_GATE]); - } - - /* - * Let's initially set up CLKO with OSC24M, since this configuration - * is widely used by imx6q board designs to clock audio codec. - */ -- ret = clk_set_parent(clk[IMX6QDL_CLK_CKO2_SEL], clk[IMX6QDL_CLK_OSC]); -+ ret = imx_clk_set_parent(clk[IMX6QDL_CLK_CKO2_SEL], clk[IMX6QDL_CLK_OSC]); - if (!ret) -- ret = clk_set_parent(clk[IMX6QDL_CLK_CKO], clk[IMX6QDL_CLK_CKO2]); -+ ret = imx_clk_set_parent(clk[IMX6QDL_CLK_CKO], clk[IMX6QDL_CLK_CKO2]); - if (ret) - pr_warn("failed to set up CLKO: %d\n", ret); - - /* Audio-related clocks configuration */ -- clk_set_parent(clk[IMX6QDL_CLK_SPDIF_SEL], clk[IMX6QDL_CLK_PLL3_PFD3_454M]); -+ imx_clk_set_parent(clk[IMX6QDL_CLK_SPDIF_SEL], clk[IMX6QDL_CLK_PLL3_PFD3_454M]); - - /* All existing boards with PCIe use LVDS1 */ - if (IS_ENABLED(CONFIG_PCI_IMX6)) -- clk_set_parent(clk[IMX6QDL_CLK_LVDS1_SEL], clk[IMX6QDL_CLK_SATA_REF_100M]); -+ imx_clk_set_parent(clk[IMX6QDL_CLK_LVDS1_SEL], clk[IMX6QDL_CLK_SATA_REF_100M]); -+ -+ /* -+ * Enable clocks only after both parent and rate are all initialized -+ * as needed -+ */ -+ for (i = 0; i < ARRAY_SIZE(clks_init_on); i++) -+ imx_clk_prepare_enable(clk[clks_init_on[i]]); -+ -+ /* -+ * If VPU 352M is enabled, then PLL2_PDF2 need to be -+ * set to 352M, cpufreq will be disabled as VDDSOC/PU -+ * need to be at highest voltage, scaling cpu freq is -+ * not saving any power, and busfreq will be also disabled -+ * as the PLL2_PFD2 is not at default freq, in a word, -+ * all modules that sourceing clk from PLL2_PFD2 will -+ * be impacted. -+ */ -+ if (vpu352) { -+ clk_set_rate(clk[IMX6QDL_CLK_PLL2_PFD2_396M], 352000000); -+ clk_set_parent(clk[IMX6QDL_CLK_VPU_AXI_SEL], clk[IMX6QDL_CLK_PLL2_PFD2_396M]); -+ pr_info("VPU 352M is enabled!\n"); -+ } - - /* Set initial power mode */ - imx6q_set_lpm(WAIT_CLOCKED); -diff -Nur linux-4.1.3/arch/arm/mach-imx/clk-pllv3.c linux-xbian-imx6/arch/arm/mach-imx/clk-pllv3.c ---- linux-4.1.3/arch/arm/mach-imx/clk-pllv3.c 2015-07-21 19:10:33.000000000 +0200 -+++ linux-xbian-imx6/arch/arm/mach-imx/clk-pllv3.c 2015-07-27 23:13:01.085110746 +0200 -@@ -23,6 +23,7 @@ - #define PLL_DENOM_OFFSET 0x20 - - #define BM_PLL_POWER (0x1 << 12) -+#define BM_PLL_BYPASS (0x1 << 16) - #define BM_PLL_LOCK (0x1 << 31) - - /** -@@ -237,9 +238,10 @@ - struct clk_pllv3 *pll = to_clk_pllv3(hw); - unsigned long min_rate = parent_rate * 27; - unsigned long max_rate = parent_rate * 54; -- u32 val, div; -+ u32 val, newval, div; - u32 mfn, mfd = 1000000; - s64 temp64; -+ int ret; - - if (rate < min_rate || rate > max_rate) - return -EINVAL; -@@ -251,13 +253,27 @@ - mfn = temp64; - - val = readl_relaxed(pll->base); -- val &= ~pll->div_mask; -- val |= div; -- writel_relaxed(val, pll->base); -+ -+ /* set the PLL into bypass mode */ -+ newval = val | BM_PLL_BYPASS; -+ writel_relaxed(newval, pll->base); -+ -+ /* configure the new frequency */ -+ newval &= ~pll->div_mask; -+ newval |= div; -+ writel_relaxed(newval, pll->base); - writel_relaxed(mfn, pll->base + PLL_NUM_OFFSET); - writel_relaxed(mfd, pll->base + PLL_DENOM_OFFSET); - -- return clk_pllv3_wait_lock(pll); -+ ret = clk_pllv3_wait_lock(pll); -+ if (ret == 0 && val & BM_PLL_POWER) { -+ /* only if it locked can we switch back to the PLL */ -+ newval &= ~BM_PLL_BYPASS; -+ newval |= val & BM_PLL_BYPASS; -+ writel_relaxed(newval, pll->base); -+ } -+ -+ return ret; - } - - static const struct clk_ops clk_pllv3_av_ops = { -diff -Nur linux-4.1.3/arch/arm/mach-imx/common.h linux-xbian-imx6/arch/arm/mach-imx/common.h ---- linux-4.1.3/arch/arm/mach-imx/common.h 2015-07-21 19:10:33.000000000 +0200 -+++ linux-xbian-imx6/arch/arm/mach-imx/common.h 2015-07-27 23:13:01.085110746 +0200 -@@ -56,6 +56,7 @@ - void mxc_set_cpu_type(unsigned int type); - void mxc_restart(enum reboot_mode, const char *); - void mxc_arch_reset_init(void __iomem *); -+void mxc_arch_reset_init_dt(void); - int mx51_revision(void); - int mx53_revision(void); - void imx_set_aips(void __iomem *); -@@ -86,6 +87,8 @@ - MX3_SLEEP, - }; - -+extern int vpu352; -+ - void mx3_cpu_lp_set(enum mx3_cpu_pwr_mode mode); - void imx_print_silicon_rev(const char *cpu, int srev); - -@@ -102,6 +105,7 @@ - static inline void imx_smp_prepare(void) {} - #endif - void imx_src_init(void); -+ - void imx_gpc_pre_suspend(bool arm_power_off); - void imx_gpc_post_resume(void); - void imx_gpc_mask_all(void); -diff -Nur linux-4.1.3/arch/arm/mach-imx/ddr3_freq_imx6.S linux-xbian-imx6/arch/arm/mach-imx/ddr3_freq_imx6.S ---- linux-4.1.3/arch/arm/mach-imx/ddr3_freq_imx6.S 1970-01-01 01:00:00.000000000 +0100 -+++ linux-xbian-imx6/arch/arm/mach-imx/ddr3_freq_imx6.S 2015-07-27 23:13:01.089096525 +0200 -@@ -0,0 +1,893 @@ -+/* -+ * Copyright (C) 2011-2013 Freescale Semiconductor, Inc. All Rights Reserved. -+ * -+ * This program is free software; you can redistribute it and/or modify -+ * it under the terms of the GNU General Public License as published by -+ * the Free Software Foundation; either version 2 of the License, or -+ * (at your option) any later version. -+ -+ * This program is distributed in the hope that it will be useful, -+ * but WITHOUT ANY WARRANTY; without even the implied warranty of -+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -+ * GNU General Public License for more details. -+ -+ * You should have received a copy of the GNU General Public License along -+ * with this program; if not, write to the Free Software Foundation, Inc., -+ * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. -+ */ -+ -+#include -+ -+#define MMDC0_MDPDC 0x4 -+#define MMDC0_MDCF0 0x0c -+#define MMDC0_MDCF1 0x10 -+#define MMDC0_MDMISC 0x18 -+#define MMDC0_MDSCR 0x1c -+#define MMDC0_MAPSR 0x404 -+#define MMDC0_MADPCR0 0x410 -+#define MMDC0_MPZQHWCTRL 0x800 -+#define MMDC1_MPZQHWCTRL 0x4800 -+#define MMDC0_MPODTCTRL 0x818 -+#define MMDC1_MPODTCTRL 0x4818 -+#define MMDC0_MPDGCTRL0 0x83c -+#define MMDC1_MPDGCTRL0 0x483c -+#define MMDC0_MPMUR0 0x8b8 -+#define MMDC1_MPMUR0 0x48b8 -+ -+#define CCM_CBCDR 0x14 -+#define CCM_CBCMR 0x18 -+#define CCM_CSCMR1 0x1c -+#define CCM_CDHIPR 0x48 -+ -+#define L2_CACHE_SYNC 0x730 -+ -+ .align 3 -+ -+ .macro switch_to_528MHz -+ -+ /* check if periph_clk_sel is already set */ -+ ldr r0, [r6, #CCM_CBCDR] -+ and r0, r0, #(1 << 25) -+ cmp r0, #(1 << 25) -+ beq set_ahb_podf_before_switch -+ -+ /* change periph_clk to be sourced from pll3_clk. */ -+ ldr r0, [r6, #CCM_CBCMR] -+ bic r0, r0, #(3 << 12) -+ str r0, [r6, #CCM_CBCMR] -+ -+ ldr r0, [r6, #CCM_CBCDR] -+ bic r0, r0, #(0x38 << 20) -+ str r0, [r6, #CCM_CBCDR] -+ -+ /* -+ * set the AHB dividers before the switch, -+ * don't change AXI clock divider, -+ * set the MMDC_DIV=1, AXI_DIV = 2, AHB_DIV=4, -+ */ -+ ldr r0, [r6, #CCM_CBCDR] -+ ldr r2, =0x3f1f00 -+ bic r0, r0, r2 -+ orr r0, r0, #0xd00 -+ orr r0, r0, #(1 << 16) -+ str r0, [r6, #CCM_CBCDR] -+ -+wait_div_update528: -+ ldr r0, [r6, #CCM_CDHIPR] -+ cmp r0, #0 -+ bne wait_div_update528 -+ -+ /* now switch periph_clk to pll3_main_clk. */ -+ ldr r0, [r6, #CCM_CBCDR] -+ orr r0, r0, #(1 << 25) -+ str r0, [r6, #CCM_CBCDR] -+ -+periph_clk_switch3: -+ ldr r0, [r6, #CCM_CDHIPR] -+ cmp r0, #0 -+ bne periph_clk_switch3 -+ -+ b switch_pre_periph_clk_528 -+ -+set_ahb_podf_before_switch: -+ /* -+ * set the MMDC_DIV=1, AXI_DIV = 2, AHB_DIV=4, -+ */ -+ ldr r0, [r6, #CCM_CBCDR] -+ ldr r2, =0x3f1f00 -+ bic r0, r0, r2 -+ orr r0, r0, #0xd00 -+ orr r0, r0, #(1 << 16) -+ str r0, [r6, #CCM_CBCDR] -+ -+wait_div_update528_1: -+ ldr r0, [r6, #CCM_CDHIPR] -+ cmp r0, #0 -+ bne wait_div_update528_1 -+ -+switch_pre_periph_clk_528: -+ -+ /* now switch pre_periph_clk to PLL2_528MHz. */ -+ ldr r0, [r6, #CCM_CBCMR] -+ bic r0, r0, #(0xc << 16) -+ str r0, [r6, #CCM_CBCMR] -+ -+ /* now switch periph_clk back. */ -+ ldr r0, [r6, #CCM_CBCDR] -+ bic r0, r0, #(1 << 25) -+ str r0, [r6, #CCM_CBCDR] -+ -+periph_clk_switch4: -+ ldr r0, [r6, #CCM_CDHIPR] -+ cmp r0, #0 -+ bne periph_clk_switch4 -+ -+ .endm -+ -+ .macro switch_to_400MHz -+ -+ /* check if periph_clk_sel is already set. */ -+ ldr r0, [r6, #CCM_CBCDR] -+ and r0, r0, #(1 << 25) -+ cmp r0, #(1 << 25) -+ beq set_ahb_podf_before_switch1 -+ -+ /* change periph_clk to be sourced from pll3_clk. */ -+ ldr r0, [r6, #CCM_CBCMR] -+ bic r0, r0, #(3 << 12) -+ str r0, [r6, #CCM_CBCMR] -+ -+ ldr r0, [r6, #CCM_CBCDR] -+ bic r0, r0, #(0x38 << 24) -+ str r0, [r6, #CCM_CBCDR] -+ -+ /* now switch periph_clk to pll3_main_clk. */ -+ ldr r0, [r6, #CCM_CBCDR] -+ orr r0, r0, #(1 << 25) -+ str r0, [r6, #CCM_CBCDR] -+ -+periph_clk_switch5: -+ ldr r0, [r6, #CCM_CDHIPR] -+ cmp r0, #0 -+ bne periph_clk_switch5 -+ -+ b switch_pre_periph_clk_400 -+ -+set_ahb_podf_before_switch1: -+ /* -+ * set the MMDC_DIV=1, AXI_DIV = 2, AHB_DIV=4, -+ */ -+ ldr r0, [r6, #CCM_CBCDR] -+ ldr r2, =0x3f1f00 -+ bic r0, r0, r2 -+ orr r0, r0, #(0x9 << 8) -+ orr r0, r0, #(1 << 16) -+ str r0, [r6, #CCM_CBCDR] -+ -+wait_div_update400_1: -+ ldr r0, [r6, #CCM_CDHIPR] -+ cmp r0, #0 -+ bne wait_div_update400_1 -+ -+switch_pre_periph_clk_400: -+ -+ /* now switch pre_periph_clk to PFD_400MHz. */ -+ ldr r0, [r6, #CCM_CBCMR] -+ bic r0, r0, #(0xc << 16) -+ orr r0, r0, #(0x4 << 16) -+ str r0, [r6, #CCM_CBCMR] -+ -+ /* now switch periph_clk back. */ -+ ldr r0, [r6, #CCM_CBCDR] -+ bic r0, r0, #(1 << 25) -+ str r0, [r6, #CCM_CBCDR] -+ -+periph_clk_switch6: -+ ldr r0, [r6, #CCM_CDHIPR] -+ cmp r0, #0 -+ bne periph_clk_switch6 -+ -+ /* -+ * change AHB divider so that we are at 400/3=133MHz. -+ * don't change AXI clock divider. -+ * set the MMDC_DIV=1, AXI_DIV=2, AHB_DIV=3, -+ */ -+ ldr r0, [r6, #CCM_CBCDR] -+ ldr r2, =0x3f1f00 -+ bic r0, r0, r2 -+ orr r0, r0, #(0x9 << 8) -+ orr r0, r0, #(1 << 16) -+ str r0, [r6, #CCM_CBCDR] -+ -+wait_div_update400_2: -+ ldr r0, [r6, #CCM_CDHIPR] -+ cmp r0, #0 -+ bne wait_div_update400_2 -+ -+ .endm -+ -+ .macro switch_to_50MHz -+ -+ /* check if periph_clk_sel is already set. */ -+ ldr r0, [r6, #CCM_CBCDR] -+ and r0, r0, #(1 << 25) -+ cmp r0, #(1 << 25) -+ beq switch_pre_periph_clk_50 -+ -+ /* -+ * set the periph_clk to be sourced from PLL2_PFD_200M -+ * change periph_clk to be sourced from pll3_clk. -+ * ensure PLL3 is the source and set the divider to 1. -+ */ -+ ldr r0, [r6, #CCM_CBCMR] -+ bic r0, r0, #(0x3 << 12) -+ str r0, [r6, #CCM_CBCMR] -+ -+ ldr r0, [r6, #CCM_CBCDR] -+ bic r0, r0, #(0x38 << 24) -+ str r0, [r6, #CCM_CBCDR] -+ -+ /* now switch periph_clk to pll3_main_clk. */ -+ ldr r0, [r6, #CCM_CBCDR] -+ orr r0, r0, #(1 << 25) -+ str r0, [r6, #CCM_CBCDR] -+ -+periph_clk_switch_50: -+ ldr r0, [r6, #CCM_CDHIPR] -+ cmp r0, #0 -+ bne periph_clk_switch_50 -+ -+switch_pre_periph_clk_50: -+ -+ /* now switch pre_periph_clk to PFD_200MHz. */ -+ ldr r0, [r6, #CCM_CBCMR] -+ orr r0, r0, #(0xc << 16) -+ str r0, [r6, #CCM_CBCMR] -+ -+ /* -+ * set the MMDC_DIV=4, AXI_DIV = 4, AHB_DIV=8, -+ */ -+ ldr r0, [r6, #CCM_CBCDR] -+ ldr r2, =0x3f1f00 -+ bic r0, r0, r2 -+ orr r0, r0, #(0x18 << 16) -+ orr r0, r0, #(0x3 << 16) -+ -+ /* -+ * if changing AHB divider remember to change -+ * the IPGPER divider too below. -+ */ -+ orr r0, r0, #0x1d00 -+ str r0, [r6, #CCM_CBCDR] -+ -+wait_div_update_50: -+ ldr r0, [r6, #CCM_CDHIPR] -+ cmp r0, #0 -+ bne wait_div_update_50 -+ -+ /* now switch periph_clk back. */ -+ ldr r0, [r6, #CCM_CBCDR] -+ bic r0, r0, #(1 << 25) -+ str r0, [r6, #CCM_CBCDR] -+ -+periph_clk_switch2: -+ ldr r0, [r6, #CCM_CDHIPR] -+ cmp r0, #0 -+ bne periph_clk_switch2 -+ -+ .endm -+ -+ .macro switch_to_24MHz -+ /* -+ * change the freq now try setting DDR to 24MHz. -+ * source it from the periph_clk2 ensure the -+ * periph_clk2 is sourced from 24MHz and the -+ * divider is 1. -+ */ -+ -+ ldr r0, [r6, #CCM_CBCMR] -+ bic r0, r0, #(0x3 << 12) -+ orr r0, r0, #(1 << 12) -+ str r0, [r6, #CCM_CBCMR] -+ -+ ldr r0, [r6, #CCM_CBCDR] -+ bic r0, r0, #(0x38 << 24) -+ str r0, [r6, #CCM_CBCDR] -+ -+ /* now switch periph_clk to 24MHz. */ -+ ldr r0, [r6, #CCM_CBCDR] -+ orr r0, r0, #(1 << 25) -+ str r0, [r6, #CCM_CBCDR] -+ -+periph_clk_switch1: -+ ldr r0, [r6, #CCM_CDHIPR] -+ cmp r0, #0 -+ bne periph_clk_switch1 -+ -+ /* change all the dividers to 1. */ -+ ldr r0, [r6, #CCM_CBCDR] -+ ldr r2, =0x3f1f00 -+ bic r0, r0, r2 -+ orr r0, r0, #(1 << 8) -+ str r0, [r6, #CCM_CBCDR] -+ -+ /* Wait for the divider to change. */ -+wait_div_update: -+ ldr r0, [r6, #CCM_CDHIPR] -+ cmp r0, #0 -+ bne wait_div_update -+ -+ .endm -+ -+/* -+ * mx6_ddr3_freq_change -+ * -+ * idle the processor (eg, wait for interrupt). -+ * make sure DDR is in self-refresh. -+ * IRQs are already disabled. -+ */ -+ENTRY(mx6_ddr3_freq_change) -+ -+ stmfd sp!, {r4-r12} -+ -+ /* -+ * r5 -> mmdc_base -+ * r6 -> ccm_base -+ * r7 -> iomux_base -+ * r12 -> l2_base -+ */ -+ mov r4, r0 -+ mov r8, r1 -+ mov r9, r2 -+ mov r11, r3 -+ -+ /* -+ * Get the addresses of the registers. -+ * They are last few entries in the -+ * ddr_settings parameter. -+ * The first entry contains the count, -+ * and each entry is 2 words. -+ */ -+ ldr r0, [r1] -+ add r0, r0, #1 -+ lsl r0, r0, #3 -+ add r1, r0, r1 -+ /* mmdc_base. */ -+ ldr r5, [r1] -+ add r1, #8 -+ /* ccm_base */ -+ ldr r6, [r1] -+ add r1, #8 -+ /*iomux_base */ -+ ldr r7, [r1] -+ add r1, #8 -+ /*l2_base */ -+ ldr r12, [r1] -+ -+ddr_freq_change: -+ /* -+ * make sure no TLB miss will occur when -+ * the DDR is in self refresh. invalidate -+ * TLB single entry to ensure that the -+ * address is not already in the TLB. -+ */ -+ -+ adr r10, ddr_freq_change -+ -+ ldr r2, [r6] -+ ldr r2, [r5] -+ ldr r2, [r7] -+ ldr r2, [r8] -+ ldr r2, [r10] -+ ldr r2, [r11] -+ ldr r2, [r12] -+ -+#ifdef CONFIG_CACHE_L2X0 -+ /* -+ * Make sure the L2 buffers are drained. -+ * Sync operation on L2 drains the buffers. -+ */ -+ mov r1, #0x0 -+ str r1, [r12, #L2_CACHE_SYNC] -+#endif -+ -+ /* disable automatic power saving. */ -+ ldr r0, [r5, #MMDC0_MAPSR] -+ orr r0, r0, #0x01 -+ str r0, [r5, #MMDC0_MAPSR] -+ -+ /* disable MMDC power down timer. */ -+ ldr r0, [r5, #MMDC0_MDPDC] -+ bic r0, r0, #(0xff << 8) -+ str r0, [r5, #MMDC0_MDPDC] -+ -+ /* delay for a while */ -+ ldr r1, =4 -+delay1: -+ ldr r2, =0 -+cont1: -+ ldr r0, [r5, r2] -+ add r2, r2, #4 -+ cmp r2, #16 -+ bne cont1 -+ sub r1, r1, #1 -+ cmp r1, #0 -+ bgt delay1 -+ -+ /* set CON_REG */ -+ ldr r0, =0x8000 -+ str r0, [r5, #MMDC0_MDSCR] -+poll_conreq_set_1: -+ ldr r0, [r5, #MMDC0_MDSCR] -+ and r0, r0, #(0x4 << 12) -+ cmp r0, #(0x4 << 12) -+ bne poll_conreq_set_1 -+ -+ ldr r0, =0x00008050 -+ str r0, [r5, #MMDC0_MDSCR] -+ ldr r0, =0x00008058 -+ str r0, [r5, #MMDC0_MDSCR] -+ -+ /* -+ * if requested frequency is greater than -+ * 300MHz go to DLL on mode. -+ */ -+ ldr r1, =300000000 -+ cmp r4, r1 -+ bge dll_on_mode -+ -+dll_off_mode: -+ -+ /* if DLL is currently on, turn it off. */ -+ cmp r9, #1 -+ beq continue_dll_off_1 -+ -+ ldr r0, =0x00018031 -+ str r0, [r5, #MMDC0_MDSCR] -+ -+ ldr r0, =0x00018039 -+ str r0, [r5, #MMDC0_MDSCR] -+ -+ ldr r1, =10 -+delay1a: -+ ldr r2, =0 -+cont1a: -+ ldr r0, [r5, r2] -+ add r2, r2, #4 -+ cmp r2, #16 -+ bne cont1a -+ sub r1, r1, #1 -+ cmp r1, #0 -+ bgt delay1a -+ -+continue_dll_off_1: -+ /* set DVFS - enter self refresh mode */ -+ ldr r0, [r5, #MMDC0_MAPSR] -+ orr r0, r0, #(1 << 21) -+ str r0, [r5, #MMDC0_MAPSR] -+ -+ /* de-assert con_req */ -+ mov r0, #0x0 -+ str r0, [r5, #MMDC0_MDSCR] -+ -+poll_dvfs_set_1: -+ ldr r0, [r5, #MMDC0_MAPSR] -+ and r0, r0, #(1 << 25) -+ cmp r0, #(1 << 25) -+ bne poll_dvfs_set_1 -+ -+ ldr r1, =24000000 -+ cmp r4, r1 -+ beq switch_freq_24 -+ -+ switch_to_50MHz -+ b continue_dll_off_2 -+ -+switch_freq_24: -+ switch_to_24MHz -+ -+continue_dll_off_2: -+ -+ /* set SBS - block ddr accesses */ -+ ldr r0, [r5, #MMDC0_MADPCR0] -+ orr r0, r0, #(1 << 8) -+ str r0, [r5, #MMDC0_MADPCR0] -+ -+ /* clear DVFS - exit from self refresh mode */ -+ ldr r0, [r5, #MMDC0_MAPSR] -+ bic r0, r0, #(1 << 21) -+ str r0, [r5, #MMDC0_MAPSR] -+ -+poll_dvfs_clear_1: -+ ldr r0, [r5, #MMDC0_MAPSR] -+ and r0, r0, #(1 << 25) -+ cmp r0, #(1 << 25) -+ beq poll_dvfs_clear_1 -+ -+ /* if DLL was previously on, continue DLL off routine. */ -+ cmp r9, #1 -+ beq continue_dll_off_3 -+ -+ ldr r0, =0x00018031 -+ str r0, [r5, #MMDC0_MDSCR] -+ -+ ldr r0, =0x00018039 -+ str r0, [r5, #MMDC0_MDSCR] -+ -+ ldr r0, =0x08208030 -+ str r0, [r5, #MMDC0_MDSCR] -+ -+ ldr r0, =0x08208038 -+ str r0, [r5, #MMDC0_MDSCR] -+ -+ ldr r0, =0x00088032 -+ str r0, [r5, #MMDC0_MDSCR] -+ -+ ldr r0, =0x0008803A -+ str r0, [r5, #MMDC0_MDSCR] -+ -+ /* delay for a while. */ -+ ldr r1, =4 -+delay_1: -+ ldr r2, =0 -+cont_1: -+ ldr r0, [r5, r2] -+ add r2, r2, #4 -+ cmp r2, #16 -+ bne cont_1 -+ sub r1, r1, #1 -+ cmp r1, #0 -+ bgt delay_1 -+ -+ ldr r0, [r5, #MMDC0_MDCF0] -+ bic r0, r0, #0xf -+ orr r0, r0, #0x3 -+ str r0, [r5, #MMDC0_MDCF0] -+ -+ ldr r0, [r5, #MMDC0_MDCF1] -+ bic r0, r0, #0x7 -+ orr r0, r0, #0x4 -+ str r0, [r5, #MMDC0_MDCF1] -+ -+ ldr r0, =0x00011680 -+ str r0, [r5, #MMDC0_MDMISC] -+ -+ /* enable dqs pull down in the IOMUX. */ -+ ldr r1, [r11] -+ add r11, r11, #8 -+ ldr r2, =0x3028 -+update_iomux: -+ ldr r0, [r11, #0x0] -+ ldr r3, [r7, r0] -+ bic r3, r3, r2 -+ orr r3, r3, #(0x3 << 12) -+ orr r3, r3, #0x28 -+ str r3, [r7, r0] -+ add r11, r11, #8 -+ sub r1, r1, #1 -+ cmp r1, #0 -+ bgt update_iomux -+ -+ /* ODT disabled. */ -+ ldr r0, =0x0 -+ ldr r2, =MMDC0_MPODTCTRL -+ str r0, [r5, r2] -+ ldr r2, =MMDC1_MPODTCTRL -+ str r0, [r5, r2] -+ -+ /* DQS gating disabled. */ -+ ldr r2, =MMDC0_MPDGCTRL0 -+ ldr r0, [r5, r2] -+ orr r0, r0, #(1 << 29) -+ str r0, [r5, r2] -+ -+ ldr r2, =MMDC1_MPDGCTRL0 -+ ldr r0, [r5, r2] -+ orr r0, r0, #(0x1 << 29) -+ str r0, [r5, r2] -+ -+ /* MMDC0_MAPSR adopt power down enable. */ -+ ldr r0, [r5, #MMDC0_MAPSR] -+ bic r0, r0, #0x01 -+ str r0, [r5, #MMDC0_MAPSR] -+ -+ /* frc_msr + mu bypass */ -+ ldr r0, =0x00000060 -+ str r0, [r5, #MMDC0_MPMUR0] -+ ldr r2, =MMDC1_MPMUR0 -+ str r0, [r5, r2] -+ ldr r0, =0x00000460 -+ str r0, [r5, #MMDC0_MPMUR0] -+ ldr r2, =MMDC1_MPMUR0 -+ str r0, [r5, r2] -+ ldr r0, =0x00000c60 -+ str r0, [r5, #MMDC0_MPMUR0] -+ ldr r2, =MMDC1_MPMUR0 -+ str r0, [r5, r2] -+ -+continue_dll_off_3: -+ /* clear SBS - unblock accesses to DDR. */ -+ ldr r0, [r5, #MMDC0_MADPCR0] -+ bic r0, r0, #(0x1 << 8) -+ str r0, [r5, #MMDC0_MADPCR0] -+ -+ mov r0, #0x0 -+ str r0, [r5, #MMDC0_MDSCR] -+poll_conreq_clear_1: -+ ldr r0, [r5, #MMDC0_MDSCR] -+ and r0, r0, #(0x4 << 12) -+ cmp r0, #(0x4 << 12) -+ beq poll_conreq_clear_1 -+ -+ b done -+ -+dll_on_mode: -+ /* assert DVFS - enter self refresh mode. */ -+ ldr r0, [r5, #MMDC0_MAPSR] -+ orr r0, r0, #(1 << 21) -+ str r0, [r5, #MMDC0_MAPSR] -+ -+ /* de-assert CON_REQ. */ -+ mov r0, #0x0 -+ str r0, [r5, #MMDC0_MDSCR] -+ -+ /* poll DVFS ack. */ -+poll_dvfs_set_2: -+ ldr r0, [r5, #MMDC0_MAPSR] -+ and r0, r0, #(1 << 25) -+ cmp r0, #(1 << 25) -+ bne poll_dvfs_set_2 -+ -+ ldr r1, =528000000 -+ cmp r4, r1 -+ beq switch_freq_528 -+ -+ switch_to_400MHz -+ -+ b continue_dll_on -+ -+switch_freq_528: -+ switch_to_528MHz -+ -+continue_dll_on: -+ -+ /* set SBS step-by-step mode. */ -+ ldr r0, [r5, #MMDC0_MADPCR0] -+ orr r0, r0, #( 1 << 8) -+ str r0, [r5, #MMDC0_MADPCR0] -+ -+ /* clear DVFS - exit self refresh mode. */ -+ ldr r0, [r5, #MMDC0_MAPSR] -+ bic r0, r0, #(1 << 21) -+ str r0, [r5, #MMDC0_MAPSR] -+ -+poll_dvfs_clear_2: -+ ldr r0, [r5, #MMDC0_MAPSR] -+ and r0, r0, #(1 << 25) -+ cmp r0, #(1 << 25) -+ beq poll_dvfs_clear_2 -+ -+ /* if DLL is currently off, turn it back on. */ -+ cmp r9, #0 -+ beq update_calibration_only -+ -+ ldr r0, =0xa5390003 -+ str r0, [r5, #MMDC0_MPZQHWCTRL] -+ ldr r2, =MMDC1_MPZQHWCTRL -+ str r0, [r5, r2] -+ -+ /* enable DQS gating. */ -+ ldr r2, =MMDC0_MPDGCTRL0 -+ ldr r0, [r5, r2] -+ bic r0, r0, #(1 << 29) -+ str r0, [r5, r2] -+ -+ ldr r2, =MMDC1_MPDGCTRL0 -+ ldr r0, [r5, r2] -+ bic r0, r0, #(1 << 29) -+ str r0, [r5, r2] -+ -+ /* force measure. */ -+ ldr r0, =0x00000800 -+ str r0, [r5, #MMDC0_MPMUR0] -+ ldr r2, =MMDC1_MPMUR0 -+ str r0, [r5, r2] -+ -+ /* delay for while. */ -+ ldr r1, =4 -+delay5: -+ ldr r2, =0 -+cont5: -+ ldr r0, [r5, r2] -+ add r2, r2, #4 -+ cmp r2, #16 -+ bne cont5 -+ sub r1, r1, #1 -+ cmp r1, #0 -+ bgt delay5 -+ -+ /* disable dqs pull down in the IOMUX. */ -+ ldr r1, [r11] -+ add r11, r11, #8 -+update_iomux1: -+ ldr r0, [r11, #0x0] -+ ldr r3, [r11, #0x4] -+ str r3, [r7, r0] -+ add r11, r11, #8 -+ sub r1, r1, #1 -+ cmp r1, #0 -+ bgt update_iomux1 -+ -+ /* config MMDC timings to 528MHz. */ -+ ldr r9, [r8] -+ add r8, r8, #8 -+ ldr r0, [r8, #0x0] -+ ldr r3, [r8, #0x4] -+ str r3, [r5, r0] -+ add r8, r8, #8 -+ -+ ldr r0, [r8, #0x0] -+ ldr r3, [r8, #0x4] -+ str r3, [r5, r0] -+ add r8, r8, #8 -+ -+ /* update MISC register: WALAT, RALAT */ -+ ldr r0, =0x00001740 -+ str r0, [r5, #MMDC0_MDMISC] -+ -+ /* configure ddr devices to dll on, odt. */ -+ ldr r0, =0x00048031 -+ str r0, [r5, #MMDC0_MDSCR] -+ -+ ldr r0, =0x00048039 -+ str r0, [r5, #MMDC0_MDSCR] -+ -+ /* delay for while. */ -+ ldr r1, =4 -+delay7: -+ ldr r2, =0 -+cont7: -+ ldr r0, [r5, r2] -+ add r2, r2, #4 -+ cmp r2, #16 -+ bne cont7 -+ sub r1, r1, #1 -+ cmp r1, #0 -+ bgt delay7 -+ -+ /* reset dll. */ -+ ldr r0, =0x09408030 -+ str r0, [r5, #MMDC0_MDSCR] -+ -+ ldr r0, =0x09408038 -+ str r0, [r5, #MMDC0_MDSCR] -+ -+ /* delay for while. */ -+ ldr r1, =100 -+delay8: -+ ldr r2, =0 -+cont8: -+ ldr r0, [r5, r2] -+ add r2, r2, #4 -+ cmp r2, #16 -+ bne cont8 -+ sub r1, r1, #1 -+ cmp r1, #0 -+ bgt delay8 -+ -+ ldr r0, [r8, #0x0] -+ ldr r3, [r8, #0x4] -+ str r3, [r5, r0] -+ add r8, r8, #8 -+ -+ ldr r0, [r8, #0x0] -+ ldr r3, [r8, #0x4] -+ str r3, [r5, r0] -+ add r8, r8, #8 -+ -+ ldr r0, =0x00428031 -+ str r0, [r5, #MMDC0_MDSCR] -+ -+ ldr r0, =0x00428039 -+ str r0, [r5, #MMDC0_MDSCR] -+ -+ ldr r0, [r8, #0x0] -+ ldr r3, [r8, #0x4] -+ str r3, [r5, r0] -+ add r8, r8, #8 -+ -+ ldr r0, [r8, #0x0] -+ ldr r3, [r8, #0x4] -+ str r3, [r5, r0] -+ add r8, r8, #8 -+ -+ /* issue a zq command. */ -+ ldr r0, =0x04008040 -+ str r0, [r5, #MMDC0_MDSCR] -+ -+ ldr r0, =0x04008048 -+ str r0, [r5, #MMDC0_MDSCR] -+ -+ /* MMDC ODT enable. */ -+ ldr r0, [r8, #0x0] -+ ldr r3, [r8, #0x4] -+ str r3, [r5, r0] -+ add r8, r8, #8 -+ -+ ldr r2, =0x4818 -+ str r3, [r5, r2] -+ -+ /* delay for while. */ -+ ldr r1, =40 -+delay15: -+ ldr r2, =0 -+cont15: -+ ldr r0, [r5, r2] -+ add r2, r2, #4 -+ cmp r2, #16 -+ bne cont15 -+ sub r1, r1, #1 -+ cmp r1, #0 -+ bgt delay15 -+ -+ /* MMDC0_MAPSR adopt power down enable. */ -+ ldr r0, [r5, #MMDC0_MAPSR] -+ bic r0, r0, #0x01 -+ str r0, [r5, #MMDC0_MAPSR] -+ -+ /* enable MMDC power down timer. */ -+ ldr r0, [r5, #MMDC0_MDPDC] -+ orr r0, r0, #(0x55 << 8) -+ str r0, [r5, #MMDC0_MDPDC] -+ -+ b update_calibration -+ -+update_calibration_only: -+ ldr r1, [r8] -+ sub r1, r1, #7 -+ add r8, r8, #64 -+ b update_calib -+ -+update_calibration: -+ /* write the new calibration values. */ -+ mov r1, r9 -+ sub r1, r1, #7 -+ -+update_calib: -+ ldr r0, [r8, #0x0] -+ ldr r3, [r8, #0x4] -+ str r3, [r5, r0] -+ add r8, r8, #8 -+ sub r1, r1, #1 -+ cmp r1, #0 -+ bgt update_calib -+ -+ /* perform a force measurement. */ -+ ldr r0, =0x800 -+ str r0, [r5, #MMDC0_MPMUR0] -+ ldr r2, =MMDC1_MPMUR0 -+ str r0, [r5, r2] -+ -+ /* clear SBS - unblock DDR accesses. */ -+ ldr r0, [r5, #MMDC0_MADPCR0] -+ bic r0, r0, #(1 << 8) -+ str r0, [r5, #MMDC0_MADPCR0] -+ -+ mov r0, #0x0 -+ str r0, [r5, #MMDC0_MDSCR] -+poll_conreq_clear_2: -+ ldr r0, [r5, #MMDC0_MDSCR] -+ and r0, r0, #(0x4 << 12) -+ cmp r0, #(0x4 << 12) -+ beq poll_conreq_clear_2 -+ -+done: -+ /* restore registers */ -+ -+ ldmfd sp!, {r4-r12} -+ mov pc, lr -+ -+ .type mx6_do_ddr3_freq_change, #object -+ENTRY(mx6_do_ddr_freq_change) -+ .word mx6_ddr3_freq_change -+ .size mx6_ddr3_freq_change, . - mx6_ddr3_freq_change -diff -Nur linux-4.1.3/arch/arm/mach-imx/Kconfig linux-xbian-imx6/arch/arm/mach-imx/Kconfig ---- linux-4.1.3/arch/arm/mach-imx/Kconfig 2015-07-21 19:10:33.000000000 +0200 -+++ linux-xbian-imx6/arch/arm/mach-imx/Kconfig 2015-07-27 23:13:01.053224513 +0200 -@@ -1,5 +1,6 @@ - menuconfig ARCH_MXC - bool "Freescale i.MX family" if ARCH_MULTI_V4_V5 || ARCH_MULTI_V6_V7 -+ select ARCH_HAS_RESET_CONTROLLER - select ARCH_REQUIRE_GPIOLIB - select ARM_CPU_SUSPEND if PM - select CLKSRC_MMIO -@@ -8,6 +9,7 @@ - select PM_OPP if PM - select SOC_BUS - select SRAM -+ select ZONE_DMA - help - Support for Freescale MXC/iMX-based family of processors - -@@ -58,7 +60,6 @@ - - config HAVE_IMX_SRC - def_bool y if SMP -- select ARCH_HAS_RESET_CONTROLLER - - config IMX_HAVE_IOMUX_V1 - bool -diff -Nur linux-4.1.3/arch/arm/mach-imx/lpddr2_freq_imx6.S linux-xbian-imx6/arch/arm/mach-imx/lpddr2_freq_imx6.S ---- linux-4.1.3/arch/arm/mach-imx/lpddr2_freq_imx6.S 1970-01-01 01:00:00.000000000 +0100 -+++ linux-xbian-imx6/arch/arm/mach-imx/lpddr2_freq_imx6.S 2015-07-27 23:13:01.093082305 +0200 -@@ -0,0 +1,484 @@ -+/* -+ * Copyright (C) 2012-2013 Freescale Semiconductor, Inc. All Rights Reserved. -+ * -+ * This program is free software; you can redistribute it and/or modify -+ * it under the terms of the GNU General Public License as published by -+ * the Free Software Foundation; either version 2 of the License, or -+ * (at your option) any later version. -+ -+ * This program is distributed in the hope that it will be useful, -+ * but WITHOUT ANY WARRANTY; without even the implied warranty of -+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -+ * GNU General Public License for more details. -+ -+ * You should have received a copy of the GNU General Public License along -+ * with this program; if not, write to the Free Software Foundation, Inc., -+ * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. -+ */ -+ -+#include -+ -+ .macro mx6sl_switch_to_24MHz -+ -+ /* -+ * Set MMDC clock to be sourced from PLL3. -+ * Ensure first periph2_clk2 is sourced from PLL3. -+ * Set the PERIPH2_CLK2_PODF to divide by 2. -+ */ -+ ldr r6, [r2, #0x14] -+ bic r6, r6, #0x7 -+ orr r6, r6, #0x1 -+ str r6, [r2, #0x14] -+ -+ /* Select PLL3 to source MMDC. */ -+ ldr r6, [r2, #0x18] -+ bic r6, r6, #0x100000 -+ str r6, [r2, #0x18] -+ -+ /* Swtich periph2_clk_sel to run from PLL3. */ -+ ldr r6, [r2, #0x14] -+ orr r6, r6, #0x4000000 -+ str r6, [r2, #0x14] -+ -+periph2_clk_switch1: -+ ldr r6, [r2, #0x48] -+ cmp r6, #0 -+ bne periph2_clk_switch1 -+ -+ /* -+ * Need to clock gate the 528 PFDs before -+ * powering down PLL2. -+ * Only the PLL2_PFD2_400M should be ON -+ * at this time, so only clock gate that one. -+ */ -+ ldr r6, [r3, #0x100] -+ orr r6, r6, #0x800000 -+ str r6, [r3, #0x100] -+ -+ /* -+ * Set PLL2 to bypass state. We should be here -+ * only if MMDC is not sourced from PLL2. -+ */ -+ ldr r6, [r3, #0x30] -+ orr r6, r6, #0x10000 -+ str r6, [r3, #0x30] -+ -+ ldr r6, [r3, #0x30] -+ orr r6, r6, #0x1000 -+ str r6, [r3, #0x30] -+ -+ /* Ensure pre_periph2_clk_mux is set to pll2 */ -+ ldr r6, [r2, #0x18] -+ bic r6, r6, #0x600000 -+ str r6, [r2, #0x18] -+ -+ /* Set MMDC clock to be sourced from the bypassed PLL2. */ -+ ldr r6, [r2, #0x14] -+ bic r6, r6, #0x4000000 -+ str r6, [r2, #0x14] -+ -+periph2_clk_switch2: -+ ldr r6, [r2, #0x48] -+ cmp r6, #0 -+ bne periph2_clk_switch2 -+ -+ /* -+ * Now move MMDC back to periph2_clk2 source. -+ * after selecting PLL2 as the option. -+ * Select PLL2 as the source. -+ */ -+ ldr r6, [r2, #0x18] -+ orr r6, r6, #0x100000 -+ str r6, [r2, #0x18] -+ -+ /* set periph2_clk2_podf to divide by 1. */ -+ ldr r6, [r2, #0x14] -+ bic r6, r6, #0x7 -+ str r6, [r2, #0x14] -+ -+ /* Now move periph2_clk to periph2_clk2 source */ -+ ldr r6, [r2, #0x14] -+ orr r6, r6, #0x4000000 -+ str r6, [r2, #0x14] -+ -+periph2_clk_switch3: -+ ldr r6, [r2, #0x48] -+ cmp r6, #0 -+ bne periph2_clk_switch3 -+ -+ /* Now set the MMDC PODF back to 1.*/ -+ ldr r6, [r2, #0x14] -+ bic r6, r6, #0x38 -+ str r6, [r2, #0x14] -+ -+mmdc_podf0: -+ ldr r6, [r2, #0x48] -+ cmp r6, #0 -+ bne mmdc_podf0 -+ -+ .endm -+ -+ .macro ddr_switch_400MHz -+ -+ /* Set MMDC divider first, in case PLL3 is at 480MHz. */ -+ ldr r6, [r3, #0x10] -+ and r6, r6, #0x10000 -+ cmp r6, #0x10000 -+ beq pll3_in_bypass -+ -+ /* Set MMDC divder to divide by 2. */ -+ ldr r6, [r2, #0x14] -+ bic r6, r6, #0x38 -+ orr r6, r6, #0x8 -+ str r6, [r2, #0x14] -+ -+mmdc_podf: -+ ldr r6, [r2, #0x48] -+ cmp r6, #0 -+ bne mmdc_podf -+ -+pll3_in_bypass: -+ /* -+ * Check if we are switching between -+ * 400Mhz <-> 100MHz.If so, we should -+ * try to source MMDC from PLL2_200M. -+ */ -+ cmp r1, #0 -+ beq not_low_bus_freq -+ -+ /* Ensure that MMDC is sourced from PLL2 mux first. */ -+ ldr r6, [r2, #0x14] -+ bic r6, r6, #0x4000000 -+ str r6, [r2, #0x14] -+ -+periph2_clk_switch4: -+ ldr r6, [r2, #0x48] -+ cmp r6, #0 -+ bne periph2_clk_switch4 -+ -+not_low_bus_freq: -+ /* Now ensure periph2_clk2_sel mux is set to PLL3 */ -+ ldr r6, [r2, #0x18] -+ bic r6, r6, #0x100000 -+ str r6, [r2, #0x18] -+ -+ /* Now switch MMDC to PLL3. */ -+ ldr r6, [r2, #0x14] -+ orr r6, r6, #0x4000000 -+ str r6, [r2, #0x14] -+ -+periph2_clk_switch5: -+ ldr r6, [r2, #0x48] -+ cmp r6, #0 -+ bne periph2_clk_switch5 -+ -+ /* -+ * Check if PLL2 is already unlocked. -+ * If so do nothing with PLL2. -+ */ -+ cmp r1, #0 -+ beq pll2_already_on -+ -+ /* Now power up PLL2 and unbypass it. */ -+ ldr r6, [r3, #0x30] -+ bic r6, r6, #0x1000 -+ str r6, [r3, #0x30] -+ -+ /* Make sure PLL2 has locked.*/ -+wait_for_pll_lock: -+ ldr r6, [r3, #0x30] -+ and r6, r6, #0x80000000 -+ cmp r6, #0x80000000 -+ bne wait_for_pll_lock -+ -+ ldr r6, [r3, #0x30] -+ bic r6, r6, #0x10000 -+ str r6, [r3, #0x30] -+ -+ /* -+ * Need to enable the 528 PFDs after -+ * powering up PLL2. -+ * Only the PLL2_PFD2_400M should be ON -+ * as it feeds the MMDC. Rest should have -+ * been managed by clock code. -+ */ -+ ldr r6, [r3, #0x100] -+ bic r6, r6, #0x800000 -+ str r6, [r3, #0x100] -+ -+pll2_already_on: -+ /* -+ * Now switch MMDC clk back to pll2_mux option. -+ * Ensure pre_periph2_clk2 is set to pll2_pfd_400M. -+ * If switching to audio DDR freq, set the -+ * pre_periph2_clk2 to PLL2_PFD_200M -+ */ -+ ldr r6, =400000000 -+ cmp r6, r0 -+ bne use_pll2_pfd_200M -+ -+ ldr r6, [r2, #0x18] -+ bic r6, r6, #0x600000 -+ orr r6, r6, #0x200000 -+ str r6, [r2, #0x18] -+ ldr r6, =400000000 -+ b cont2 -+ -+use_pll2_pfd_200M: -+ ldr r6, [r2, #0x18] -+ orr r6, r6, #0x600000 -+ str r6, [r2, #0x18] -+ ldr r6, =200000000 -+ -+cont2: -+ ldr r4, [r2, #0x14] -+ bic r4, r4, #0x4000000 -+ str r4, [r2, #0x14] -+ -+periph2_clk_switch6: -+ ldr r4, [r2, #0x48] -+ cmp r4, #0 -+ bne periph2_clk_switch6 -+ -+change_divider_only: -+ /* -+ * Calculate the MMDC divider -+ * based on the requested freq. -+ */ -+ ldr r4, =0 -+Loop2: -+ sub r6, r6, r0 -+ cmp r6, r0 -+ blt Div_Found -+ add r4, r4, #1 -+ bgt Loop2 -+ -+ /* Shift divider into correct offset. */ -+ lsl r4, r4, #3 -+Div_Found: -+ /* Set the MMDC PODF. */ -+ ldr r6, [r2, #0x14] -+ bic r6, r6, #0x38 -+ orr r6, r6, r4 -+ str r6, [r2, #0x14] -+ -+mmdc_podf1: -+ ldr r6, [r2, #0x48] -+ cmp r6, #0 -+ bne mmdc_podf1 -+ -+ .endm -+ -+ .macro mmdc_clk_lower_100MHz -+ -+ /* -+ * Prior to reducing the DDR frequency (at 528/400 MHz), -+ * read the Measure unit count bits (MU_UNIT_DEL_NUM) -+ */ -+ ldr r5, =0x8B8 -+ ldr r6, [r8, r5] -+ /* Original MU unit count */ -+ mov r6, r6, LSR #16 -+ ldr r4, =0x3FF -+ and r6, r6, r4 -+ /* Original MU unit count * 2 */ -+ mov r7, r6, LSL #1 -+ /* -+ * Bypass the automatic measure unit when below 100 MHz -+ * by setting the Measure unit bypass enable bit (MU_BYP_EN) -+ */ -+ ldr r6, [r8, r5] -+ orr r6, r6, #0x400 -+ str r6, [r8, r5] -+ /* -+ * Double the measure count value read in step 1 and program it in the -+ * measurement bypass bits (MU_BYP_VAL) of the MMDC PHY Measure Unit -+ * Register for the reduced frequency operation below 100 MHz -+ */ -+ ldr r6, [r8, r5] -+ ldr r4, =0x3FF -+ bic r6, r6, r4 -+ orr r6, r6, r7 -+ str r6, [r8, r5] -+ /* Now perform a Force Measurement. */ -+ ldr r6, [r8, r5] -+ orr r6, r6, #0x800 -+ str r6, [r8, r5] -+ /* Wait for FRC_MSR to clear. */ -+force_measure: -+ ldr r6, [r8, r5] -+ and r6, r6, #0x800 -+ cmp r6, #0x0 -+ bne force_measure -+ -+ .endm -+ -+ .macro mmdc_clk_above_100MHz -+ -+ /* Make sure that the PHY measurement unit is NOT in bypass mode */ -+ ldr r5, =0x8B8 -+ ldr r6, [r8, r5] -+ bic r6, r6, #0x400 -+ str r6, [r8, r5] -+ /* Now perform a Force Measurement. */ -+ ldr r6, [r8, r5] -+ orr r6, r6, #0x800 -+ str r6, [r8, r5] -+ /* Wait for FRC_MSR to clear. */ -+force_measure1: -+ ldr r6, [r8, r5] -+ and r6, r6, #0x800 -+ cmp r6, #0x0 -+ bne force_measure1 -+ .endm -+ -+/* -+ * mx6_lpddr2_freq_change -+ * -+ * Make sure DDR is in self-refresh. -+ * IRQs are already disabled. -+ * r0 : DDR freq. -+ * r1: low_bus_freq_mode flag -+ * r2: Pointer to array containing addresses of registers. -+ */ -+ .align 3 -+ENTRY(mx6_lpddr2_freq_change) -+ -+ push {r4-r10} -+ -+ mov r4, r2 -+ ldr r3, [r4] @ANATOP_BASE_ADDR -+ ldr r2, [r4, #0x4] @CCM_BASE_ADDR -+ ldr r8, [r4, #0x8] @MMDC_P0_BASE_ADDR -+ ldr r7, [r4, #0xC] @L2_BASE_ADDR -+ -+lpddr2_freq_change: -+ adr r9, lpddr2_freq_change -+ -+ /* Prime all TLB entries. */ -+ ldr r6, [r9] -+ ldr r6, [r8] -+ ldr r6, [r3] -+ ldr r6, [r2] -+ -+ /* Drain all the L1 buffers. */ -+ dsb -+ -+#ifdef CONFIG_CACHE_L2X0 -+ /* -+ * Need to make sure the buffers in L2 are drained. -+ * Performing a sync operation does this. -+ */ -+ mov r6, #0x0 -+ str r6, [r7, #0x730] -+#endif -+ -+ /* -+ * The second dsb might be needed to keep cache sync (device write) -+ * ordering with the memory accesses before it. -+ */ -+ dsb -+ isb -+ -+ /* Disable Automatic power savings. */ -+ ldr r6, [r8, #0x404] -+ orr r6, r6, #0x01 -+ str r6, [r8, #0x404] -+ -+ /* MMDC0_MDPDC disable power down timer */ -+ ldr r6, [r8, #0x4] -+ bic r6, r6, #0xff00 -+ str r6, [r8, #0x4] -+ -+ /* Delay for a while */ -+ ldr r10, =10 -+delay1: -+ ldr r7, =0 -+cont1: -+ ldr r6, [r8, r7] -+ add r7, r7, #4 -+ cmp r7, #16 -+ bne cont1 -+ sub r10, r10, #1 -+ cmp r10, #0 -+ bgt delay1 -+ -+ /* Make the DDR explicitly enter self-refresh. */ -+ ldr r6, [r8, #0x404] -+ orr r6, r6, #0x200000 -+ str r6, [r8, #0x404] -+ -+poll_dvfs_set_1: -+ ldr r6, [r8, #0x404] -+ and r6, r6, #0x2000000 -+ cmp r6, #0x2000000 -+ bne poll_dvfs_set_1 -+ -+ /* set SBS step-by-step mode */ -+ ldr r6, [r8, #0x410] -+ orr r6, r6, #0x100 -+ str r6, [r8, #0x410] -+ -+ ldr r10, =100000000 -+ cmp r0, r10 -+ bgt set_ddr_mu_above_100 -+ mmdc_clk_lower_100MHz -+ -+set_ddr_mu_above_100: -+ ldr r10, =24000000 -+ cmp r0, r10 -+ beq set_to_24MHz -+ -+ ddr_switch_400MHz -+ -+ ldr r10,=100000000 -+ cmp r0, r10 -+ blt done -+ mmdc_clk_above_100MHz -+ -+ b done -+ -+set_to_24MHz: -+ mx6sl_switch_to_24MHz -+ -+done: -+ /* clear DVFS - exit from self refresh mode */ -+ ldr r6, [r8, #0x404] -+ bic r6, r6, #0x200000 -+ str r6, [r8, #0x404] -+ -+poll_dvfs_clear_1: -+ ldr r6, [r8, #0x404] -+ and r6, r6, #0x2000000 -+ cmp r6, #0x2000000 -+ beq poll_dvfs_clear_1 -+ -+ /* Enable Automatic power savings. */ -+ ldr r6, [r8, #0x404] -+ bic r6, r6, #0x01 -+ str r6, [r8, #0x404] -+ -+ ldr r10, =24000000 -+ cmp r0, r10 -+ beq skip_power_down -+ -+ /* Enable MMDC power down timer. */ -+ ldr r6, [r8, #0x4] -+ orr r6, r6, #0x5500 -+ str r6, [r8, #0x4] -+ -+skip_power_down: -+ /* clear SBS - unblock DDR accesses */ -+ ldr r6, [r8, #0x410] -+ bic r6, r6, #0x100 -+ str r6, [r8, #0x410] -+ -+ pop {r4-r10} -+ -+ /* Restore registers */ -+ mov pc, lr -+ -+ .type mx6_lpddr2_do_iram, #object -+ENTRY(mx6_lpddr2_do_iram) -+ .word mx6_lpddr2_freq_change -+ .size mx6_lpddr2_freq_change, . - mx6_lpddr2_freq_change -diff -Nur linux-4.1.3/arch/arm/mach-imx/mach-imx6q.c linux-xbian-imx6/arch/arm/mach-imx/mach-imx6q.c ---- linux-4.1.3/arch/arm/mach-imx/mach-imx6q.c 2015-07-21 19:10:33.000000000 +0200 -+++ linux-xbian-imx6/arch/arm/mach-imx/mach-imx6q.c 2015-07-27 23:13:01.097068084 +0200 -@@ -265,10 +265,12 @@ - static void __init imx6q_init_machine(void) - { - struct device *parent; -+ void __iomem *p; - - imx_print_silicon_rev(cpu_is_imx6dl() ? "i.MX6DL" : "i.MX6Q", - imx_get_soc_revision()); - -+ mxc_arch_reset_init_dt(); - parent = imx_soc_device_init(); - if (parent == NULL) - pr_warn("failed to initialize soc device\n"); -@@ -281,6 +283,12 @@ - cpu_is_imx6q() ? imx6q_pm_init() : imx6dl_pm_init(); - imx6q_1588_init(); - imx6q_axi_init(); -+ -+ p = ioremap(0x21b0000, SZ_4K); -+ if (p) { -+ writel(0x7f, p + 0x40); -+ iounmap(p); -+ } - } - - #define OCOTP_CFG3 0x440 -@@ -330,6 +338,12 @@ - if (dev_pm_opp_disable(cpu_dev, 852000000)) - pr_warn("failed to disable 852 MHz OPP\n"); - } -+ if (vpu352) { -+ if (dev_pm_opp_disable(cpu_dev, 396000000)) -+ pr_warn("VPU352: failed to disable 396MHz OPP\n"); -+ pr_info("VPU352: remove 396MHz OPP for VPU running at 352MHz!\n"); -+ } -+ - iounmap(base); - put_node: - of_node_put(np); -@@ -408,4 +422,5 @@ - .init_machine = imx6q_init_machine, - .init_late = imx6q_init_late, - .dt_compat = imx6q_dt_compat, -+ .restart = mxc_restart, - MACHINE_END -diff -Nur linux-4.1.3/arch/arm/mach-imx/Makefile linux-xbian-imx6/arch/arm/mach-imx/Makefile ---- linux-4.1.3/arch/arm/mach-imx/Makefile 2015-07-21 19:10:33.000000000 +0200 -+++ linux-xbian-imx6/arch/arm/mach-imx/Makefile 2015-07-27 23:13:01.053224513 +0200 -@@ -28,6 +28,12 @@ - obj-$(CONFIG_MXC_USE_EPIT) += epit.o - obj-$(CONFIG_MXC_DEBUG_BOARD) += 3ds_debugboard.o - -+obj-y += busfreq-imx6.o -+ifdef CONFIG_ARM_IMX6Q_CPUFREQ -+obj-$(CONFIG_SOC_IMX6Q) += ddr3_freq_imx6.o busfreq_ddr3.o -+obj-$(CONFIG_SOC_IMX6SL) += lpddr2_freq_imx6.o busfreq_lpddr2.o -+endif -+ - ifeq ($(CONFIG_CPU_IDLE),y) - obj-$(CONFIG_SOC_IMX5) += cpuidle-imx5.o - obj-$(CONFIG_SOC_IMX6Q) += cpuidle-imx6q.o -diff -Nur linux-4.1.3/arch/arm/mach-imx/src.c linux-xbian-imx6/arch/arm/mach-imx/src.c ---- linux-4.1.3/arch/arm/mach-imx/src.c 2015-07-21 19:10:33.000000000 +0200 -+++ linux-xbian-imx6/arch/arm/mach-imx/src.c 2015-07-27 23:13:01.097068084 +0200 -@@ -1,5 +1,5 @@ - /* -- * Copyright 2011 Freescale Semiconductor, Inc. -+ * Copyright 2011-2014 Freescale Semiconductor, Inc. - * Copyright 2011 Linaro Ltd. - * - * The code contained herein is licensed under the GNU General Public -@@ -18,6 +18,7 @@ - #include - #include - #include "common.h" -+#include "hardware.h" - - #define SRC_SCR 0x000 - #define SRC_GPR1 0x020 -@@ -32,6 +33,7 @@ - - static void __iomem *src_base; - static DEFINE_SPINLOCK(scr_lock); -+static bool m4_is_enabled; - - static const int sw_reset_bits[5] = { - BP_SRC_SCR_SW_GPU_RST, -@@ -41,6 +43,11 @@ - BP_SRC_SCR_SW_IPU2_RST - }; - -+bool imx_src_is_m4_enabled(void) -+{ -+ return m4_is_enabled; -+} -+ - static int imx_src_reset_module(struct reset_controller_dev *rcdev, - unsigned long sw_reset_idx) - { -@@ -136,6 +143,14 @@ - */ - spin_lock(&scr_lock); - val = readl_relaxed(src_base + SRC_SCR); -+ -+ /* bit 4 is m4c_non_sclr_rst on i.MX6SX */ -+ if (cpu_is_imx6sx() && ((val & -+ (1 << BP_SRC_SCR_SW_OPEN_VG_RST)) == 0)) -+ m4_is_enabled = true; -+ else -+ m4_is_enabled = false; -+ - val &= ~(1 << BP_SRC_SCR_WARM_RESET_ENABLE); - writel_relaxed(val, src_base + SRC_SCR); - spin_unlock(&scr_lock); -diff -Nur linux-4.1.3/arch/arm/mach-imx/system.c linux-xbian-imx6/arch/arm/mach-imx/system.c ---- linux-4.1.3/arch/arm/mach-imx/system.c 2015-07-21 19:10:33.000000000 +0200 -+++ linux-xbian-imx6/arch/arm/mach-imx/system.c 2015-07-27 23:13:01.101053863 +0200 -@@ -34,6 +34,7 @@ - - static void __iomem *wdog_base; - static struct clk *wdog_clk; -+static u32 wdog_source = 1; /* use WDOG1 default */ - - /* - * Reset the system. It is called by machine_restart(). -@@ -50,6 +51,17 @@ - - if (cpu_is_mx1()) - wcr_enable = (1 << 0); -+ /* -+ * Some i.MX6 boards use WDOG2 to reset external pmic in bypass mode, -+ * so do WDOG2 reset here. Do not set SRS, since we will -+ * trigger external POR later. Use WDOG1 to reset in ldo-enable -+ * mode. You can set it by "fsl,wdog-reset" in dts. -+ * For i.MX6SX we have to trigger wdog-reset to reset QSPI-NOR flash to -+ * workaround qspi-nor reboot issue whatever ldo-bypass or not. -+ */ -+ else if ((wdog_source == 2 && (cpu_is_imx6q() || cpu_is_imx6dl() || -+ cpu_is_imx6sl())) || cpu_is_imx6sx()) -+ wcr_enable = 0x14; - else - wcr_enable = (1 << 2); - -@@ -89,6 +101,41 @@ - clk_prepare(wdog_clk); - } - -+void __init mxc_arch_reset_init_dt(void) -+{ -+ struct device_node *np = NULL; -+ -+ if (cpu_is_imx6q() || cpu_is_imx6dl()) -+ np = of_find_compatible_node(NULL, NULL, "fsl,imx6q-gpc"); -+ else if (cpu_is_imx6sl()) -+ np = of_find_compatible_node(NULL, NULL, "fsl,imx6sl-gpc"); -+ -+ if (np) -+ of_property_read_u32(np, "fsl,wdog-reset", &wdog_source); -+ pr_info("Use WDOG%d as reset source\n", wdog_source); -+ -+ np = of_find_compatible_node(NULL, NULL, "fsl,imx21-wdt"); -+ wdog_base = of_iomap(np, 0); -+ WARN_ON(!wdog_base); -+ -+ /* Some i.MX6 boards use WDOG2 to reset board in ldo-bypass mode */ -+ if (wdog_source == 2 && (cpu_is_imx6q() || cpu_is_imx6dl() || -+ cpu_is_imx6sl())) { -+ np = of_find_compatible_node(np, NULL, "fsl,imx21-wdt"); -+ wdog_base = of_iomap(np, 0); -+ WARN_ON(!wdog_base); -+ } -+ -+ wdog_clk = of_clk_get(np, 0); -+ if (IS_ERR(wdog_clk)) { -+ pr_warn("%s: failed to get wdog clock\n", __func__); -+ wdog_clk = NULL; -+ return; -+ } -+ -+ clk_prepare(wdog_clk); -+} -+ - #ifdef CONFIG_CACHE_L2X0 - void __init imx_init_l2cache(void) - { -diff -Nur linux-4.1.3/arch/arm/mm/cache-v7.S linux-xbian-imx6/arch/arm/mm/cache-v7.S ---- linux-4.1.3/arch/arm/mm/cache-v7.S 2015-07-21 19:10:33.000000000 +0200 -+++ linux-xbian-imx6/arch/arm/mm/cache-v7.S 2015-07-27 23:13:01.503617563 +0200 -@@ -446,3 +446,5 @@ - - @ define struct cpu_cache_fns (see and proc-macros.S) - define_cache_functions v7 -+ -+ .long v7_dma_flush_range -diff -Nur linux-4.1.3/block/bfq-cgroup.c linux-xbian-imx6/block/bfq-cgroup.c ---- linux-4.1.3/block/bfq-cgroup.c 1970-01-01 01:00:00.000000000 +0100 -+++ linux-xbian-imx6/block/bfq-cgroup.c 2015-07-27 23:13:03.600137415 +0200 -@@ -0,0 +1,936 @@ -+/* -+ * BFQ: CGROUPS support. -+ * -+ * Based on ideas and code from CFQ: -+ * Copyright (C) 2003 Jens Axboe -+ * -+ * Copyright (C) 2008 Fabio Checconi -+ * Paolo Valente -+ * -+ * Copyright (C) 2010 Paolo Valente -+ * -+ * Licensed under the GPL-2 as detailed in the accompanying COPYING.BFQ -+ * file. -+ */ -+ -+#ifdef CONFIG_CGROUP_BFQIO -+ -+static DEFINE_MUTEX(bfqio_mutex); -+ -+static bool bfqio_is_removed(struct bfqio_cgroup *bgrp) -+{ -+ return bgrp ? !bgrp->online : false; -+} -+ -+static struct bfqio_cgroup bfqio_root_cgroup = { -+ .weight = BFQ_DEFAULT_GRP_WEIGHT, -+ .ioprio = BFQ_DEFAULT_GRP_IOPRIO, -+ .ioprio_class = BFQ_DEFAULT_GRP_CLASS, -+}; -+ -+static inline void bfq_init_entity(struct bfq_entity *entity, -+ struct bfq_group *bfqg) -+{ -+ entity->weight = entity->new_weight; -+ entity->orig_weight = entity->new_weight; -+ entity->ioprio = entity->new_ioprio; -+ entity->ioprio_class = entity->new_ioprio_class; -+ entity->parent = bfqg->my_entity; -+ entity->sched_data = &bfqg->sched_data; -+} -+ -+static struct bfqio_cgroup *css_to_bfqio(struct cgroup_subsys_state *css) -+{ -+ return css ? container_of(css, struct bfqio_cgroup, css) : NULL; -+} -+ -+/* -+ * Search the bfq_group for bfqd into the hash table (by now only a list) -+ * of bgrp. Must be called under rcu_read_lock(). -+ */ -+static struct bfq_group *bfqio_lookup_group(struct bfqio_cgroup *bgrp, -+ struct bfq_data *bfqd) -+{ -+ struct bfq_group *bfqg; -+ void *key; -+ -+ hlist_for_each_entry_rcu(bfqg, &bgrp->group_data, group_node) { -+ key = rcu_dereference(bfqg->bfqd); -+ if (key == bfqd) -+ return bfqg; -+ } -+ -+ return NULL; -+} -+ -+static inline void bfq_group_init_entity(struct bfqio_cgroup *bgrp, -+ struct bfq_group *bfqg) -+{ -+ struct bfq_entity *entity = &bfqg->entity; -+ -+ /* -+ * If the weight of the entity has never been set via the sysfs -+ * interface, then bgrp->weight == 0. In this case we initialize -+ * the weight from the current ioprio value. Otherwise, the group -+ * weight, if set, has priority over the ioprio value. -+ */ -+ if (bgrp->weight == 0) { -+ entity->new_weight = bfq_ioprio_to_weight(bgrp->ioprio); -+ entity->new_ioprio = bgrp->ioprio; -+ } else { -+ if (bgrp->weight < BFQ_MIN_WEIGHT || -+ bgrp->weight > BFQ_MAX_WEIGHT) { -+ printk(KERN_CRIT "bfq_group_init_entity: " -+ "bgrp->weight %d\n", bgrp->weight); -+ BUG(); -+ } -+ entity->new_weight = bgrp->weight; -+ entity->new_ioprio = bfq_weight_to_ioprio(bgrp->weight); -+ } -+ entity->orig_weight = entity->weight = entity->new_weight; -+ entity->ioprio = entity->new_ioprio; -+ entity->ioprio_class = entity->new_ioprio_class = bgrp->ioprio_class; -+ entity->my_sched_data = &bfqg->sched_data; -+ bfqg->active_entities = 0; -+} -+ -+static inline void bfq_group_set_parent(struct bfq_group *bfqg, -+ struct bfq_group *parent) -+{ -+ struct bfq_entity *entity; -+ -+ BUG_ON(parent == NULL); -+ BUG_ON(bfqg == NULL); -+ -+ entity = &bfqg->entity; -+ entity->parent = parent->my_entity; -+ entity->sched_data = &parent->sched_data; -+} -+ -+/** -+ * bfq_group_chain_alloc - allocate a chain of groups. -+ * @bfqd: queue descriptor. -+ * @css: the leaf cgroup_subsys_state this chain starts from. -+ * -+ * Allocate a chain of groups starting from the one belonging to -+ * @cgroup up to the root cgroup. Stop if a cgroup on the chain -+ * to the root has already an allocated group on @bfqd. -+ */ -+static struct bfq_group *bfq_group_chain_alloc(struct bfq_data *bfqd, -+ struct cgroup_subsys_state *css) -+{ -+ struct bfqio_cgroup *bgrp; -+ struct bfq_group *bfqg, *prev = NULL, *leaf = NULL; -+ -+ for (; css != NULL; css = css->parent) { -+ bgrp = css_to_bfqio(css); -+ -+ bfqg = bfqio_lookup_group(bgrp, bfqd); -+ if (bfqg != NULL) { -+ /* -+ * All the cgroups in the path from there to the -+ * root must have a bfq_group for bfqd, so we don't -+ * need any more allocations. -+ */ -+ break; -+ } -+ -+ bfqg = kzalloc(sizeof(*bfqg), GFP_ATOMIC); -+ if (bfqg == NULL) -+ goto cleanup; -+ -+ bfq_group_init_entity(bgrp, bfqg); -+ bfqg->my_entity = &bfqg->entity; -+ -+ if (leaf == NULL) { -+ leaf = bfqg; -+ prev = leaf; -+ } else { -+ bfq_group_set_parent(prev, bfqg); -+ /* -+ * Build a list of allocated nodes using the bfqd -+ * filed, that is still unused and will be -+ * initialized only after the node will be -+ * connected. -+ */ -+ prev->bfqd = bfqg; -+ prev = bfqg; -+ } -+ } -+ -+ return leaf; -+ -+cleanup: -+ while (leaf != NULL) { -+ prev = leaf; -+ leaf = leaf->bfqd; -+ kfree(prev); -+ } -+ -+ return NULL; -+} -+ -+/** -+ * bfq_group_chain_link - link an allocated group chain to a cgroup -+ * hierarchy. -+ * @bfqd: the queue descriptor. -+ * @css: the leaf cgroup_subsys_state to start from. -+ * @leaf: the leaf group (to be associated to @cgroup). -+ * -+ * Try to link a chain of groups to a cgroup hierarchy, connecting the -+ * nodes bottom-up, so we can be sure that when we find a cgroup in the -+ * hierarchy that already as a group associated to @bfqd all the nodes -+ * in the path to the root cgroup have one too. -+ * -+ * On locking: the queue lock protects the hierarchy (there is a hierarchy -+ * per device) while the bfqio_cgroup lock protects the list of groups -+ * belonging to the same cgroup. -+ */ -+static void bfq_group_chain_link(struct bfq_data *bfqd, -+ struct cgroup_subsys_state *css, -+ struct bfq_group *leaf) -+{ -+ struct bfqio_cgroup *bgrp; -+ struct bfq_group *bfqg, *next, *prev = NULL; -+ unsigned long flags; -+ -+ assert_spin_locked(bfqd->queue->queue_lock); -+ -+ for (; css != NULL && leaf != NULL; css = css->parent) { -+ bgrp = css_to_bfqio(css); -+ next = leaf->bfqd; -+ -+ bfqg = bfqio_lookup_group(bgrp, bfqd); -+ BUG_ON(bfqg != NULL); -+ -+ spin_lock_irqsave(&bgrp->lock, flags); -+ -+ rcu_assign_pointer(leaf->bfqd, bfqd); -+ hlist_add_head_rcu(&leaf->group_node, &bgrp->group_data); -+ hlist_add_head(&leaf->bfqd_node, &bfqd->group_list); -+ -+ spin_unlock_irqrestore(&bgrp->lock, flags); -+ -+ prev = leaf; -+ leaf = next; -+ } -+ -+ BUG_ON(css == NULL && leaf != NULL); -+ if (css != NULL && prev != NULL) { -+ bgrp = css_to_bfqio(css); -+ bfqg = bfqio_lookup_group(bgrp, bfqd); -+ bfq_group_set_parent(prev, bfqg); -+ } -+} -+ -+/** -+ * bfq_find_alloc_group - return the group associated to @bfqd in @cgroup. -+ * @bfqd: queue descriptor. -+ * @cgroup: cgroup being searched for. -+ * -+ * Return a group associated to @bfqd in @cgroup, allocating one if -+ * necessary. When a group is returned all the cgroups in the path -+ * to the root have a group associated to @bfqd. -+ * -+ * If the allocation fails, return the root group: this breaks guarantees -+ * but is a safe fallback. If this loss becomes a problem it can be -+ * mitigated using the equivalent weight (given by the product of the -+ * weights of the groups in the path from @group to the root) in the -+ * root scheduler. -+ * -+ * We allocate all the missing nodes in the path from the leaf cgroup -+ * to the root and we connect the nodes only after all the allocations -+ * have been successful. -+ */ -+static struct bfq_group *bfq_find_alloc_group(struct bfq_data *bfqd, -+ struct cgroup_subsys_state *css) -+{ -+ struct bfqio_cgroup *bgrp = css_to_bfqio(css); -+ struct bfq_group *bfqg; -+ -+ bfqg = bfqio_lookup_group(bgrp, bfqd); -+ if (bfqg != NULL) -+ return bfqg; -+ -+ bfqg = bfq_group_chain_alloc(bfqd, css); -+ if (bfqg != NULL) -+ bfq_group_chain_link(bfqd, css, bfqg); -+ else -+ bfqg = bfqd->root_group; -+ -+ return bfqg; -+} -+ -+/** -+ * bfq_bfqq_move - migrate @bfqq to @bfqg. -+ * @bfqd: queue descriptor. -+ * @bfqq: the queue to move. -+ * @entity: @bfqq's entity. -+ * @bfqg: the group to move to. -+ * -+ * Move @bfqq to @bfqg, deactivating it from its old group and reactivating -+ * it on the new one. Avoid putting the entity on the old group idle tree. -+ * -+ * Must be called under the queue lock; the cgroup owning @bfqg must -+ * not disappear (by now this just means that we are called under -+ * rcu_read_lock()). -+ */ -+static void bfq_bfqq_move(struct bfq_data *bfqd, struct bfq_queue *bfqq, -+ struct bfq_entity *entity, struct bfq_group *bfqg) -+{ -+ int busy, resume; -+ -+ busy = bfq_bfqq_busy(bfqq); -+ resume = !RB_EMPTY_ROOT(&bfqq->sort_list); -+ -+ BUG_ON(resume && !entity->on_st); -+ BUG_ON(busy && !resume && entity->on_st && -+ bfqq != bfqd->in_service_queue); -+ -+ if (busy) { -+ BUG_ON(atomic_read(&bfqq->ref) < 2); -+ -+ if (!resume) -+ bfq_del_bfqq_busy(bfqd, bfqq, 0); -+ else -+ bfq_deactivate_bfqq(bfqd, bfqq, 0); -+ } else if (entity->on_st) -+ bfq_put_idle_entity(bfq_entity_service_tree(entity), entity); -+ -+ /* -+ * Here we use a reference to bfqg. We don't need a refcounter -+ * as the cgroup reference will not be dropped, so that its -+ * destroy() callback will not be invoked. -+ */ -+ entity->parent = bfqg->my_entity; -+ entity->sched_data = &bfqg->sched_data; -+ -+ if (busy && resume) -+ bfq_activate_bfqq(bfqd, bfqq); -+ -+ if (bfqd->in_service_queue == NULL && !bfqd->rq_in_driver) -+ bfq_schedule_dispatch(bfqd); -+} -+ -+/** -+ * __bfq_bic_change_cgroup - move @bic to @cgroup. -+ * @bfqd: the queue descriptor. -+ * @bic: the bic to move. -+ * @cgroup: the cgroup to move to. -+ * -+ * Move bic to cgroup, assuming that bfqd->queue is locked; the caller -+ * has to make sure that the reference to cgroup is valid across the call. -+ * -+ * NOTE: an alternative approach might have been to store the current -+ * cgroup in bfqq and getting a reference to it, reducing the lookup -+ * time here, at the price of slightly more complex code. -+ */ -+static struct bfq_group *__bfq_bic_change_cgroup(struct bfq_data *bfqd, -+ struct bfq_io_cq *bic, -+ struct cgroup_subsys_state *css) -+{ -+ struct bfq_queue *async_bfqq = bic_to_bfqq(bic, 0); -+ struct bfq_queue *sync_bfqq = bic_to_bfqq(bic, 1); -+ struct bfq_entity *entity; -+ struct bfq_group *bfqg; -+ struct bfqio_cgroup *bgrp; -+ -+ bgrp = css_to_bfqio(css); -+ -+ bfqg = bfq_find_alloc_group(bfqd, css); -+ if (async_bfqq != NULL) { -+ entity = &async_bfqq->entity; -+ -+ if (entity->sched_data != &bfqg->sched_data) { -+ bic_set_bfqq(bic, NULL, 0); -+ bfq_log_bfqq(bfqd, async_bfqq, -+ "bic_change_group: %p %d", -+ async_bfqq, atomic_read(&async_bfqq->ref)); -+ bfq_put_queue(async_bfqq); -+ } -+ } -+ -+ if (sync_bfqq != NULL) { -+ entity = &sync_bfqq->entity; -+ if (entity->sched_data != &bfqg->sched_data) -+ bfq_bfqq_move(bfqd, sync_bfqq, entity, bfqg); -+ } -+ -+ return bfqg; -+} -+ -+/** -+ * bfq_bic_change_cgroup - move @bic to @cgroup. -+ * @bic: the bic being migrated. -+ * @cgroup: the destination cgroup. -+ * -+ * When the task owning @bic is moved to @cgroup, @bic is immediately -+ * moved into its new parent group. -+ */ -+static void bfq_bic_change_cgroup(struct bfq_io_cq *bic, -+ struct cgroup_subsys_state *css) -+{ -+ struct bfq_data *bfqd; -+ unsigned long uninitialized_var(flags); -+ -+ bfqd = bfq_get_bfqd_locked(&(bic->icq.q->elevator->elevator_data), -+ &flags); -+ if (bfqd != NULL) { -+ __bfq_bic_change_cgroup(bfqd, bic, css); -+ bfq_put_bfqd_unlock(bfqd, &flags); -+ } -+} -+ -+/** -+ * bfq_bic_update_cgroup - update the cgroup of @bic. -+ * @bic: the @bic to update. -+ * -+ * Make sure that @bic is enqueued in the cgroup of the current task. -+ * We need this in addition to moving bics during the cgroup attach -+ * phase because the task owning @bic could be at its first disk -+ * access or we may end up in the root cgroup as the result of a -+ * memory allocation failure and here we try to move to the right -+ * group. -+ * -+ * Must be called under the queue lock. It is safe to use the returned -+ * value even after the rcu_read_unlock() as the migration/destruction -+ * paths act under the queue lock too. IOW it is impossible to race with -+ * group migration/destruction and end up with an invalid group as: -+ * a) here cgroup has not yet been destroyed, nor its destroy callback -+ * has started execution, as current holds a reference to it, -+ * b) if it is destroyed after rcu_read_unlock() [after current is -+ * migrated to a different cgroup] its attach() callback will have -+ * taken care of remove all the references to the old cgroup data. -+ */ -+static struct bfq_group *bfq_bic_update_cgroup(struct bfq_io_cq *bic) -+{ -+ struct bfq_data *bfqd = bic_to_bfqd(bic); -+ struct bfq_group *bfqg; -+ struct cgroup_subsys_state *css; -+ -+ BUG_ON(bfqd == NULL); -+ -+ rcu_read_lock(); -+ css = task_css(current, bfqio_cgrp_id); -+ bfqg = __bfq_bic_change_cgroup(bfqd, bic, css); -+ rcu_read_unlock(); -+ -+ return bfqg; -+} -+ -+/** -+ * bfq_flush_idle_tree - deactivate any entity on the idle tree of @st. -+ * @st: the service tree being flushed. -+ */ -+static inline void bfq_flush_idle_tree(struct bfq_service_tree *st) -+{ -+ struct bfq_entity *entity = st->first_idle; -+ -+ for (; entity != NULL; entity = st->first_idle) -+ __bfq_deactivate_entity(entity, 0); -+} -+ -+/** -+ * bfq_reparent_leaf_entity - move leaf entity to the root_group. -+ * @bfqd: the device data structure with the root group. -+ * @entity: the entity to move. -+ */ -+static inline void bfq_reparent_leaf_entity(struct bfq_data *bfqd, -+ struct bfq_entity *entity) -+{ -+ struct bfq_queue *bfqq = bfq_entity_to_bfqq(entity); -+ -+ BUG_ON(bfqq == NULL); -+ bfq_bfqq_move(bfqd, bfqq, entity, bfqd->root_group); -+ return; -+} -+ -+/** -+ * bfq_reparent_active_entities - move to the root group all active -+ * entities. -+ * @bfqd: the device data structure with the root group. -+ * @bfqg: the group to move from. -+ * @st: the service tree with the entities. -+ * -+ * Needs queue_lock to be taken and reference to be valid over the call. -+ */ -+static inline void bfq_reparent_active_entities(struct bfq_data *bfqd, -+ struct bfq_group *bfqg, -+ struct bfq_service_tree *st) -+{ -+ struct rb_root *active = &st->active; -+ struct bfq_entity *entity = NULL; -+ -+ if (!RB_EMPTY_ROOT(&st->active)) -+ entity = bfq_entity_of(rb_first(active)); -+ -+ for (; entity != NULL; entity = bfq_entity_of(rb_first(active))) -+ bfq_reparent_leaf_entity(bfqd, entity); -+ -+ if (bfqg->sched_data.in_service_entity != NULL) -+ bfq_reparent_leaf_entity(bfqd, -+ bfqg->sched_data.in_service_entity); -+ -+ return; -+} -+ -+/** -+ * bfq_destroy_group - destroy @bfqg. -+ * @bgrp: the bfqio_cgroup containing @bfqg. -+ * @bfqg: the group being destroyed. -+ * -+ * Destroy @bfqg, making sure that it is not referenced from its parent. -+ */ -+static void bfq_destroy_group(struct bfqio_cgroup *bgrp, struct bfq_group *bfqg) -+{ -+ struct bfq_data *bfqd; -+ struct bfq_service_tree *st; -+ struct bfq_entity *entity = bfqg->my_entity; -+ unsigned long uninitialized_var(flags); -+ int i; -+ -+ hlist_del(&bfqg->group_node); -+ -+ /* -+ * Empty all service_trees belonging to this group before -+ * deactivating the group itself. -+ */ -+ for (i = 0; i < BFQ_IOPRIO_CLASSES; i++) { -+ st = bfqg->sched_data.service_tree + i; -+ -+ /* -+ * The idle tree may still contain bfq_queues belonging -+ * to exited task because they never migrated to a different -+ * cgroup from the one being destroyed now. No one else -+ * can access them so it's safe to act without any lock. -+ */ -+ bfq_flush_idle_tree(st); -+ -+ /* -+ * It may happen that some queues are still active -+ * (busy) upon group destruction (if the corresponding -+ * processes have been forced to terminate). We move -+ * all the leaf entities corresponding to these queues -+ * to the root_group. -+ * Also, it may happen that the group has an entity -+ * in service, which is disconnected from the active -+ * tree: it must be moved, too. -+ * There is no need to put the sync queues, as the -+ * scheduler has taken no reference. -+ */ -+ bfqd = bfq_get_bfqd_locked(&bfqg->bfqd, &flags); -+ if (bfqd != NULL) { -+ bfq_reparent_active_entities(bfqd, bfqg, st); -+ bfq_put_bfqd_unlock(bfqd, &flags); -+ } -+ BUG_ON(!RB_EMPTY_ROOT(&st->active)); -+ BUG_ON(!RB_EMPTY_ROOT(&st->idle)); -+ } -+ BUG_ON(bfqg->sched_data.next_in_service != NULL); -+ BUG_ON(bfqg->sched_data.in_service_entity != NULL); -+ -+ /* -+ * We may race with device destruction, take extra care when -+ * dereferencing bfqg->bfqd. -+ */ -+ bfqd = bfq_get_bfqd_locked(&bfqg->bfqd, &flags); -+ if (bfqd != NULL) { -+ hlist_del(&bfqg->bfqd_node); -+ __bfq_deactivate_entity(entity, 0); -+ bfq_put_async_queues(bfqd, bfqg); -+ bfq_put_bfqd_unlock(bfqd, &flags); -+ } -+ BUG_ON(entity->tree != NULL); -+ -+ /* -+ * No need to defer the kfree() to the end of the RCU grace -+ * period: we are called from the destroy() callback of our -+ * cgroup, so we can be sure that no one is a) still using -+ * this cgroup or b) doing lookups in it. -+ */ -+ kfree(bfqg); -+} -+ -+static void bfq_end_wr_async(struct bfq_data *bfqd) -+{ -+ struct hlist_node *tmp; -+ struct bfq_group *bfqg; -+ -+ hlist_for_each_entry_safe(bfqg, tmp, &bfqd->group_list, bfqd_node) -+ bfq_end_wr_async_queues(bfqd, bfqg); -+ bfq_end_wr_async_queues(bfqd, bfqd->root_group); -+} -+ -+/** -+ * bfq_disconnect_groups - disconnect @bfqd from all its groups. -+ * @bfqd: the device descriptor being exited. -+ * -+ * When the device exits we just make sure that no lookup can return -+ * the now unused group structures. They will be deallocated on cgroup -+ * destruction. -+ */ -+static void bfq_disconnect_groups(struct bfq_data *bfqd) -+{ -+ struct hlist_node *tmp; -+ struct bfq_group *bfqg; -+ -+ bfq_log(bfqd, "disconnect_groups beginning"); -+ hlist_for_each_entry_safe(bfqg, tmp, &bfqd->group_list, bfqd_node) { -+ hlist_del(&bfqg->bfqd_node); -+ -+ __bfq_deactivate_entity(bfqg->my_entity, 0); -+ -+ /* -+ * Don't remove from the group hash, just set an -+ * invalid key. No lookups can race with the -+ * assignment as bfqd is being destroyed; this -+ * implies also that new elements cannot be added -+ * to the list. -+ */ -+ rcu_assign_pointer(bfqg->bfqd, NULL); -+ -+ bfq_log(bfqd, "disconnect_groups: put async for group %p", -+ bfqg); -+ bfq_put_async_queues(bfqd, bfqg); -+ } -+} -+ -+static inline void bfq_free_root_group(struct bfq_data *bfqd) -+{ -+ struct bfqio_cgroup *bgrp = &bfqio_root_cgroup; -+ struct bfq_group *bfqg = bfqd->root_group; -+ -+ bfq_put_async_queues(bfqd, bfqg); -+ -+ spin_lock_irq(&bgrp->lock); -+ hlist_del_rcu(&bfqg->group_node); -+ spin_unlock_irq(&bgrp->lock); -+ -+ /* -+ * No need to synchronize_rcu() here: since the device is gone -+ * there cannot be any read-side access to its root_group. -+ */ -+ kfree(bfqg); -+} -+ -+static struct bfq_group *bfq_alloc_root_group(struct bfq_data *bfqd, int node) -+{ -+ struct bfq_group *bfqg; -+ struct bfqio_cgroup *bgrp; -+ int i; -+ -+ bfqg = kzalloc_node(sizeof(*bfqg), GFP_KERNEL, node); -+ if (bfqg == NULL) -+ return NULL; -+ -+ bfqg->entity.parent = NULL; -+ for (i = 0; i < BFQ_IOPRIO_CLASSES; i++) -+ bfqg->sched_data.service_tree[i] = BFQ_SERVICE_TREE_INIT; -+ -+ bgrp = &bfqio_root_cgroup; -+ spin_lock_irq(&bgrp->lock); -+ rcu_assign_pointer(bfqg->bfqd, bfqd); -+ hlist_add_head_rcu(&bfqg->group_node, &bgrp->group_data); -+ spin_unlock_irq(&bgrp->lock); -+ -+ return bfqg; -+} -+ -+#define SHOW_FUNCTION(__VAR) \ -+static u64 bfqio_cgroup_##__VAR##_read(struct cgroup_subsys_state *css, \ -+ struct cftype *cftype) \ -+{ \ -+ struct bfqio_cgroup *bgrp = css_to_bfqio(css); \ -+ u64 ret = -ENODEV; \ -+ \ -+ mutex_lock(&bfqio_mutex); \ -+ if (bfqio_is_removed(bgrp)) \ -+ goto out_unlock; \ -+ \ -+ spin_lock_irq(&bgrp->lock); \ -+ ret = bgrp->__VAR; \ -+ spin_unlock_irq(&bgrp->lock); \ -+ \ -+out_unlock: \ -+ mutex_unlock(&bfqio_mutex); \ -+ return ret; \ -+} -+ -+SHOW_FUNCTION(weight); -+SHOW_FUNCTION(ioprio); -+SHOW_FUNCTION(ioprio_class); -+#undef SHOW_FUNCTION -+ -+#define STORE_FUNCTION(__VAR, __MIN, __MAX) \ -+static int bfqio_cgroup_##__VAR##_write(struct cgroup_subsys_state *css,\ -+ struct cftype *cftype, \ -+ u64 val) \ -+{ \ -+ struct bfqio_cgroup *bgrp = css_to_bfqio(css); \ -+ struct bfq_group *bfqg; \ -+ int ret = -EINVAL; \ -+ \ -+ if (val < (__MIN) || val > (__MAX)) \ -+ return ret; \ -+ \ -+ ret = -ENODEV; \ -+ mutex_lock(&bfqio_mutex); \ -+ if (bfqio_is_removed(bgrp)) \ -+ goto out_unlock; \ -+ ret = 0; \ -+ \ -+ spin_lock_irq(&bgrp->lock); \ -+ bgrp->__VAR = (unsigned short)val; \ -+ hlist_for_each_entry(bfqg, &bgrp->group_data, group_node) { \ -+ /* \ -+ * Setting the ioprio_changed flag of the entity \ -+ * to 1 with new_##__VAR == ##__VAR would re-set \ -+ * the value of the weight to its ioprio mapping. \ -+ * Set the flag only if necessary. \ -+ */ \ -+ if ((unsigned short)val != bfqg->entity.new_##__VAR) { \ -+ bfqg->entity.new_##__VAR = (unsigned short)val; \ -+ /* \ -+ * Make sure that the above new value has been \ -+ * stored in bfqg->entity.new_##__VAR before \ -+ * setting the ioprio_changed flag. In fact, \ -+ * this flag may be read asynchronously (in \ -+ * critical sections protected by a different \ -+ * lock than that held here), and finding this \ -+ * flag set may cause the execution of the code \ -+ * for updating parameters whose value may \ -+ * depend also on bfqg->entity.new_##__VAR (in \ -+ * __bfq_entity_update_weight_prio). \ -+ * This barrier makes sure that the new value \ -+ * of bfqg->entity.new_##__VAR is correctly \ -+ * seen in that code. \ -+ */ \ -+ smp_wmb(); \ -+ bfqg->entity.ioprio_changed = 1; \ -+ } \ -+ } \ -+ spin_unlock_irq(&bgrp->lock); \ -+ \ -+out_unlock: \ -+ mutex_unlock(&bfqio_mutex); \ -+ return ret; \ -+} -+ -+STORE_FUNCTION(weight, BFQ_MIN_WEIGHT, BFQ_MAX_WEIGHT); -+STORE_FUNCTION(ioprio, 0, IOPRIO_BE_NR - 1); -+STORE_FUNCTION(ioprio_class, IOPRIO_CLASS_RT, IOPRIO_CLASS_IDLE); -+#undef STORE_FUNCTION -+ -+static struct cftype bfqio_files[] = { -+ { -+ .name = "weight", -+ .read_u64 = bfqio_cgroup_weight_read, -+ .write_u64 = bfqio_cgroup_weight_write, -+ }, -+ { -+ .name = "ioprio", -+ .read_u64 = bfqio_cgroup_ioprio_read, -+ .write_u64 = bfqio_cgroup_ioprio_write, -+ }, -+ { -+ .name = "ioprio_class", -+ .read_u64 = bfqio_cgroup_ioprio_class_read, -+ .write_u64 = bfqio_cgroup_ioprio_class_write, -+ }, -+ { }, /* terminate */ -+}; -+ -+static struct cgroup_subsys_state *bfqio_create(struct cgroup_subsys_state -+ *parent_css) -+{ -+ struct bfqio_cgroup *bgrp; -+ -+ if (parent_css != NULL) { -+ bgrp = kzalloc(sizeof(*bgrp), GFP_KERNEL); -+ if (bgrp == NULL) -+ return ERR_PTR(-ENOMEM); -+ } else -+ bgrp = &bfqio_root_cgroup; -+ -+ spin_lock_init(&bgrp->lock); -+ INIT_HLIST_HEAD(&bgrp->group_data); -+ bgrp->ioprio = BFQ_DEFAULT_GRP_IOPRIO; -+ bgrp->ioprio_class = BFQ_DEFAULT_GRP_CLASS; -+ -+ return &bgrp->css; -+} -+ -+/* -+ * We cannot support shared io contexts, as we have no means to support -+ * two tasks with the same ioc in two different groups without major rework -+ * of the main bic/bfqq data structures. By now we allow a task to change -+ * its cgroup only if it's the only owner of its ioc; the drawback of this -+ * behavior is that a group containing a task that forked using CLONE_IO -+ * will not be destroyed until the tasks sharing the ioc die. -+ */ -+static int bfqio_can_attach(struct cgroup_subsys_state *css, -+ struct cgroup_taskset *tset) -+{ -+ struct task_struct *task; -+ struct io_context *ioc; -+ int ret = 0; -+ -+ cgroup_taskset_for_each(task, tset) { -+ /* -+ * task_lock() is needed to avoid races with -+ * exit_io_context() -+ */ -+ task_lock(task); -+ ioc = task->io_context; -+ if (ioc != NULL && atomic_read(&ioc->nr_tasks) > 1) -+ /* -+ * ioc == NULL means that the task is either too -+ * young or exiting: if it has still no ioc the -+ * ioc can't be shared, if the task is exiting the -+ * attach will fail anyway, no matter what we -+ * return here. -+ */ -+ ret = -EINVAL; -+ task_unlock(task); -+ if (ret) -+ break; -+ } -+ -+ return ret; -+} -+ -+static void bfqio_attach(struct cgroup_subsys_state *css, -+ struct cgroup_taskset *tset) -+{ -+ struct task_struct *task; -+ struct io_context *ioc; -+ struct io_cq *icq; -+ -+ /* -+ * IMPORTANT NOTE: The move of more than one process at a time to a -+ * new group has not yet been tested. -+ */ -+ cgroup_taskset_for_each(task, tset) { -+ ioc = get_task_io_context(task, GFP_ATOMIC, NUMA_NO_NODE); -+ if (ioc) { -+ /* -+ * Handle cgroup change here. -+ */ -+ rcu_read_lock(); -+ hlist_for_each_entry_rcu(icq, &ioc->icq_list, ioc_node) -+ if (!strncmp( -+ icq->q->elevator->type->elevator_name, -+ "bfq", ELV_NAME_MAX)) -+ bfq_bic_change_cgroup(icq_to_bic(icq), -+ css); -+ rcu_read_unlock(); -+ put_io_context(ioc); -+ } -+ } -+} -+ -+static void bfqio_destroy(struct cgroup_subsys_state *css) -+{ -+ struct bfqio_cgroup *bgrp = css_to_bfqio(css); -+ struct hlist_node *tmp; -+ struct bfq_group *bfqg; -+ -+ /* -+ * Since we are destroying the cgroup, there are no more tasks -+ * referencing it, and all the RCU grace periods that may have -+ * referenced it are ended (as the destruction of the parent -+ * cgroup is RCU-safe); bgrp->group_data will not be accessed by -+ * anything else and we don't need any synchronization. -+ */ -+ hlist_for_each_entry_safe(bfqg, tmp, &bgrp->group_data, group_node) -+ bfq_destroy_group(bgrp, bfqg); -+ -+ BUG_ON(!hlist_empty(&bgrp->group_data)); -+ -+ kfree(bgrp); -+} -+ -+static int bfqio_css_online(struct cgroup_subsys_state *css) -+{ -+ struct bfqio_cgroup *bgrp = css_to_bfqio(css); -+ -+ mutex_lock(&bfqio_mutex); -+ bgrp->online = true; -+ mutex_unlock(&bfqio_mutex); -+ -+ return 0; -+} -+ -+static void bfqio_css_offline(struct cgroup_subsys_state *css) -+{ -+ struct bfqio_cgroup *bgrp = css_to_bfqio(css); -+ -+ mutex_lock(&bfqio_mutex); -+ bgrp->online = false; -+ mutex_unlock(&bfqio_mutex); -+} -+ -+struct cgroup_subsys bfqio_cgrp_subsys = { -+ .css_alloc = bfqio_create, -+ .css_online = bfqio_css_online, -+ .css_offline = bfqio_css_offline, -+ .can_attach = bfqio_can_attach, -+ .attach = bfqio_attach, -+ .css_free = bfqio_destroy, -+ .legacy_cftypes = bfqio_files, -+}; -+#else -+static inline void bfq_init_entity(struct bfq_entity *entity, -+ struct bfq_group *bfqg) -+{ -+ entity->weight = entity->new_weight; -+ entity->orig_weight = entity->new_weight; -+ entity->ioprio = entity->new_ioprio; -+ entity->ioprio_class = entity->new_ioprio_class; -+ entity->sched_data = &bfqg->sched_data; -+} -+ -+static inline struct bfq_group * -+bfq_bic_update_cgroup(struct bfq_io_cq *bic) -+{ -+ struct bfq_data *bfqd = bic_to_bfqd(bic); -+ return bfqd->root_group; -+} -+ -+static inline void bfq_bfqq_move(struct bfq_data *bfqd, -+ struct bfq_queue *bfqq, -+ struct bfq_entity *entity, -+ struct bfq_group *bfqg) -+{ -+} -+ -+static void bfq_end_wr_async(struct bfq_data *bfqd) -+{ -+ bfq_end_wr_async_queues(bfqd, bfqd->root_group); -+} -+ -+static inline void bfq_disconnect_groups(struct bfq_data *bfqd) -+{ -+ bfq_put_async_queues(bfqd, bfqd->root_group); -+} -+ -+static inline void bfq_free_root_group(struct bfq_data *bfqd) -+{ -+ kfree(bfqd->root_group); -+} -+ -+static struct bfq_group *bfq_alloc_root_group(struct bfq_data *bfqd, int node) -+{ -+ struct bfq_group *bfqg; -+ int i; -+ -+ bfqg = kmalloc_node(sizeof(*bfqg), GFP_KERNEL | __GFP_ZERO, node); -+ if (bfqg == NULL) -+ return NULL; -+ -+ for (i = 0; i < BFQ_IOPRIO_CLASSES; i++) -+ bfqg->sched_data.service_tree[i] = BFQ_SERVICE_TREE_INIT; -+ -+ return bfqg; -+} -+#endif -diff -Nur linux-4.1.3/block/bfq.h linux-xbian-imx6/block/bfq.h ---- linux-4.1.3/block/bfq.h 1970-01-01 01:00:00.000000000 +0100 -+++ linux-xbian-imx6/block/bfq.h 2015-07-27 23:13:03.604123194 +0200 -@@ -0,0 +1,811 @@ -+/* -+ * BFQ-v7r7 for 4.0.0: data structures and common functions prototypes. -+ * -+ * Based on ideas and code from CFQ: -+ * Copyright (C) 2003 Jens Axboe -+ * -+ * Copyright (C) 2008 Fabio Checconi -+ * Paolo Valente -+ * -+ * Copyright (C) 2010 Paolo Valente -+ */ -+ -+#ifndef _BFQ_H -+#define _BFQ_H -+ -+#include -+#include -+#include -+#include -+ -+#define BFQ_IOPRIO_CLASSES 3 -+#define BFQ_CL_IDLE_TIMEOUT (HZ/5) -+ -+#define BFQ_MIN_WEIGHT 1 -+#define BFQ_MAX_WEIGHT 1000 -+ -+#define BFQ_DEFAULT_QUEUE_IOPRIO 4 -+ -+#define BFQ_DEFAULT_GRP_WEIGHT 10 -+#define BFQ_DEFAULT_GRP_IOPRIO 0 -+#define BFQ_DEFAULT_GRP_CLASS IOPRIO_CLASS_BE -+ -+struct bfq_entity; -+ -+/** -+ * struct bfq_service_tree - per ioprio_class service tree. -+ * @active: tree for active entities (i.e., those backlogged). -+ * @idle: tree for idle entities (i.e., those not backlogged, with V <= F_i). -+ * @first_idle: idle entity with minimum F_i. -+ * @last_idle: idle entity with maximum F_i. -+ * @vtime: scheduler virtual time. -+ * @wsum: scheduler weight sum; active and idle entities contribute to it. -+ * -+ * Each service tree represents a B-WF2Q+ scheduler on its own. Each -+ * ioprio_class has its own independent scheduler, and so its own -+ * bfq_service_tree. All the fields are protected by the queue lock -+ * of the containing bfqd. -+ */ -+struct bfq_service_tree { -+ struct rb_root active; -+ struct rb_root idle; -+ -+ struct bfq_entity *first_idle; -+ struct bfq_entity *last_idle; -+ -+ u64 vtime; -+ unsigned long wsum; -+}; -+ -+/** -+ * struct bfq_sched_data - multi-class scheduler. -+ * @in_service_entity: entity in service. -+ * @next_in_service: head-of-the-line entity in the scheduler. -+ * @service_tree: array of service trees, one per ioprio_class. -+ * -+ * bfq_sched_data is the basic scheduler queue. It supports three -+ * ioprio_classes, and can be used either as a toplevel queue or as -+ * an intermediate queue on a hierarchical setup. -+ * @next_in_service points to the active entity of the sched_data -+ * service trees that will be scheduled next. -+ * -+ * The supported ioprio_classes are the same as in CFQ, in descending -+ * priority order, IOPRIO_CLASS_RT, IOPRIO_CLASS_BE, IOPRIO_CLASS_IDLE. -+ * Requests from higher priority queues are served before all the -+ * requests from lower priority queues; among requests of the same -+ * queue requests are served according to B-WF2Q+. -+ * All the fields are protected by the queue lock of the containing bfqd. -+ */ -+struct bfq_sched_data { -+ struct bfq_entity *in_service_entity; -+ struct bfq_entity *next_in_service; -+ struct bfq_service_tree service_tree[BFQ_IOPRIO_CLASSES]; -+}; -+ -+/** -+ * struct bfq_weight_counter - counter of the number of all active entities -+ * with a given weight. -+ * @weight: weight of the entities that this counter refers to. -+ * @num_active: number of active entities with this weight. -+ * @weights_node: weights tree member (see bfq_data's @queue_weights_tree -+ * and @group_weights_tree). -+ */ -+struct bfq_weight_counter { -+ short int weight; -+ unsigned int num_active; -+ struct rb_node weights_node; -+}; -+ -+/** -+ * struct bfq_entity - schedulable entity. -+ * @rb_node: service_tree member. -+ * @weight_counter: pointer to the weight counter associated with this entity. -+ * @on_st: flag, true if the entity is on a tree (either the active or -+ * the idle one of its service_tree). -+ * @finish: B-WF2Q+ finish timestamp (aka F_i). -+ * @start: B-WF2Q+ start timestamp (aka S_i). -+ * @tree: tree the entity is enqueued into; %NULL if not on a tree. -+ * @min_start: minimum start time of the (active) subtree rooted at -+ * this entity; used for O(log N) lookups into active trees. -+ * @service: service received during the last round of service. -+ * @budget: budget used to calculate F_i; F_i = S_i + @budget / @weight. -+ * @weight: weight of the queue -+ * @parent: parent entity, for hierarchical scheduling. -+ * @my_sched_data: for non-leaf nodes in the cgroup hierarchy, the -+ * associated scheduler queue, %NULL on leaf nodes. -+ * @sched_data: the scheduler queue this entity belongs to. -+ * @ioprio: the ioprio in use. -+ * @new_weight: when a weight change is requested, the new weight value. -+ * @orig_weight: original weight, used to implement weight boosting -+ * @new_ioprio: when an ioprio change is requested, the new ioprio value. -+ * @ioprio_class: the ioprio_class in use. -+ * @new_ioprio_class: when an ioprio_class change is requested, the new -+ * ioprio_class value. -+ * @ioprio_changed: flag, true when the user requested a weight, ioprio or -+ * ioprio_class change. -+ * -+ * A bfq_entity is used to represent either a bfq_queue (leaf node in the -+ * cgroup hierarchy) or a bfq_group into the upper level scheduler. Each -+ * entity belongs to the sched_data of the parent group in the cgroup -+ * hierarchy. Non-leaf entities have also their own sched_data, stored -+ * in @my_sched_data. -+ * -+ * Each entity stores independently its priority values; this would -+ * allow different weights on different devices, but this -+ * functionality is not exported to userspace by now. Priorities and -+ * weights are updated lazily, first storing the new values into the -+ * new_* fields, then setting the @ioprio_changed flag. As soon as -+ * there is a transition in the entity state that allows the priority -+ * update to take place the effective and the requested priority -+ * values are synchronized. -+ * -+ * Unless cgroups are used, the weight value is calculated from the -+ * ioprio to export the same interface as CFQ. When dealing with -+ * ``well-behaved'' queues (i.e., queues that do not spend too much -+ * time to consume their budget and have true sequential behavior, and -+ * when there are no external factors breaking anticipation) the -+ * relative weights at each level of the cgroups hierarchy should be -+ * guaranteed. All the fields are protected by the queue lock of the -+ * containing bfqd. -+ */ -+struct bfq_entity { -+ struct rb_node rb_node; -+ struct bfq_weight_counter *weight_counter; -+ -+ int on_st; -+ -+ u64 finish; -+ u64 start; -+ -+ struct rb_root *tree; -+ -+ u64 min_start; -+ -+ unsigned long service, budget; -+ unsigned short weight, new_weight; -+ unsigned short orig_weight; -+ -+ struct bfq_entity *parent; -+ -+ struct bfq_sched_data *my_sched_data; -+ struct bfq_sched_data *sched_data; -+ -+ unsigned short ioprio, new_ioprio; -+ unsigned short ioprio_class, new_ioprio_class; -+ -+ int ioprio_changed; -+}; -+ -+struct bfq_group; -+ -+/** -+ * struct bfq_queue - leaf schedulable entity. -+ * @ref: reference counter. -+ * @bfqd: parent bfq_data. -+ * @new_bfqq: shared bfq_queue if queue is cooperating with -+ * one or more other queues. -+ * @pos_node: request-position tree member (see bfq_data's @rq_pos_tree). -+ * @pos_root: request-position tree root (see bfq_data's @rq_pos_tree). -+ * @sort_list: sorted list of pending requests. -+ * @next_rq: if fifo isn't expired, next request to serve. -+ * @queued: nr of requests queued in @sort_list. -+ * @allocated: currently allocated requests. -+ * @meta_pending: pending metadata requests. -+ * @fifo: fifo list of requests in sort_list. -+ * @entity: entity representing this queue in the scheduler. -+ * @max_budget: maximum budget allowed from the feedback mechanism. -+ * @budget_timeout: budget expiration (in jiffies). -+ * @dispatched: number of requests on the dispatch list or inside driver. -+ * @flags: status flags. -+ * @bfqq_list: node for active/idle bfqq list inside our bfqd. -+ * @burst_list_node: node for the device's burst list. -+ * @seek_samples: number of seeks sampled -+ * @seek_total: sum of the distances of the seeks sampled -+ * @seek_mean: mean seek distance -+ * @last_request_pos: position of the last request enqueued -+ * @requests_within_timer: number of consecutive pairs of request completion -+ * and arrival, such that the queue becomes idle -+ * after the completion, but the next request arrives -+ * within an idle time slice; used only if the queue's -+ * IO_bound has been cleared. -+ * @pid: pid of the process owning the queue, used for logging purposes. -+ * @last_wr_start_finish: start time of the current weight-raising period if -+ * the @bfq-queue is being weight-raised, otherwise -+ * finish time of the last weight-raising period -+ * @wr_cur_max_time: current max raising time for this queue -+ * @soft_rt_next_start: minimum time instant such that, only if a new -+ * request is enqueued after this time instant in an -+ * idle @bfq_queue with no outstanding requests, then -+ * the task associated with the queue it is deemed as -+ * soft real-time (see the comments to the function -+ * bfq_bfqq_softrt_next_start()) -+ * @last_idle_bklogged: time of the last transition of the @bfq_queue from -+ * idle to backlogged -+ * @service_from_backlogged: cumulative service received from the @bfq_queue -+ * since the last transition from idle to -+ * backlogged -+ * @bic: pointer to the bfq_io_cq owning the bfq_queue, set to %NULL if the -+ * queue is shared -+ * -+ * A bfq_queue is a leaf request queue; it can be associated with an -+ * io_context or more, if it is async or shared between cooperating -+ * processes. @cgroup holds a reference to the cgroup, to be sure that it -+ * does not disappear while a bfqq still references it (mostly to avoid -+ * races between request issuing and task migration followed by cgroup -+ * destruction). -+ * All the fields are protected by the queue lock of the containing bfqd. -+ */ -+struct bfq_queue { -+ atomic_t ref; -+ struct bfq_data *bfqd; -+ -+ /* fields for cooperating queues handling */ -+ struct bfq_queue *new_bfqq; -+ struct rb_node pos_node; -+ struct rb_root *pos_root; -+ -+ struct rb_root sort_list; -+ struct request *next_rq; -+ int queued[2]; -+ int allocated[2]; -+ int meta_pending; -+ struct list_head fifo; -+ -+ struct bfq_entity entity; -+ -+ unsigned long max_budget; -+ unsigned long budget_timeout; -+ -+ int dispatched; -+ -+ unsigned int flags; -+ -+ struct list_head bfqq_list; -+ -+ struct hlist_node burst_list_node; -+ -+ unsigned int seek_samples; -+ u64 seek_total; -+ sector_t seek_mean; -+ sector_t last_request_pos; -+ -+ unsigned int requests_within_timer; -+ -+ pid_t pid; -+ struct bfq_io_cq *bic; -+ -+ /* weight-raising fields */ -+ unsigned long wr_cur_max_time; -+ unsigned long soft_rt_next_start; -+ unsigned long last_wr_start_finish; -+ unsigned int wr_coeff; -+ unsigned long last_idle_bklogged; -+ unsigned long service_from_backlogged; -+}; -+ -+/** -+ * struct bfq_ttime - per process thinktime stats. -+ * @ttime_total: total process thinktime -+ * @ttime_samples: number of thinktime samples -+ * @ttime_mean: average process thinktime -+ */ -+struct bfq_ttime { -+ unsigned long last_end_request; -+ -+ unsigned long ttime_total; -+ unsigned long ttime_samples; -+ unsigned long ttime_mean; -+}; -+ -+/** -+ * struct bfq_io_cq - per (request_queue, io_context) structure. -+ * @icq: associated io_cq structure -+ * @bfqq: array of two process queues, the sync and the async -+ * @ttime: associated @bfq_ttime struct -+ * @wr_time_left: snapshot of the time left before weight raising ends -+ * for the sync queue associated to this process; this -+ * snapshot is taken to remember this value while the weight -+ * raising is suspended because the queue is merged with a -+ * shared queue, and is used to set @raising_cur_max_time -+ * when the queue is split from the shared queue and its -+ * weight is raised again -+ * @saved_idle_window: same purpose as the previous field for the idle -+ * window -+ * @saved_IO_bound: same purpose as the previous two fields for the I/O -+ * bound classification of a queue -+ * @saved_in_large_burst: same purpose as the previous fields for the -+ * value of the field keeping the queue's belonging -+ * to a large burst -+ * @was_in_burst_list: true if the queue belonged to a burst list -+ * before its merge with another cooperating queue -+ * @cooperations: counter of consecutive successful queue merges underwent -+ * by any of the process' @bfq_queues -+ * @failed_cooperations: counter of consecutive failed queue merges of any -+ * of the process' @bfq_queues -+ */ -+struct bfq_io_cq { -+ struct io_cq icq; /* must be the first member */ -+ struct bfq_queue *bfqq[2]; -+ struct bfq_ttime ttime; -+ int ioprio; -+ -+ unsigned int wr_time_left; -+ bool saved_idle_window; -+ bool saved_IO_bound; -+ -+ bool saved_in_large_burst; -+ bool was_in_burst_list; -+ -+ unsigned int cooperations; -+ unsigned int failed_cooperations; -+}; -+ -+enum bfq_device_speed { -+ BFQ_BFQD_FAST, -+ BFQ_BFQD_SLOW, -+}; -+ -+/** -+ * struct bfq_data - per device data structure. -+ * @queue: request queue for the managed device. -+ * @root_group: root bfq_group for the device. -+ * @rq_pos_tree: rbtree sorted by next_request position, used when -+ * determining if two or more queues have interleaving -+ * requests (see bfq_close_cooperator()). -+ * @active_numerous_groups: number of bfq_groups containing more than one -+ * active @bfq_entity. -+ * @queue_weights_tree: rbtree of weight counters of @bfq_queues, sorted by -+ * weight. Used to keep track of whether all @bfq_queues -+ * have the same weight. The tree contains one counter -+ * for each distinct weight associated to some active -+ * and not weight-raised @bfq_queue (see the comments to -+ * the functions bfq_weights_tree_[add|remove] for -+ * further details). -+ * @group_weights_tree: rbtree of non-queue @bfq_entity weight counters, sorted -+ * by weight. Used to keep track of whether all -+ * @bfq_groups have the same weight. The tree contains -+ * one counter for each distinct weight associated to -+ * some active @bfq_group (see the comments to the -+ * functions bfq_weights_tree_[add|remove] for further -+ * details). -+ * @busy_queues: number of bfq_queues containing requests (including the -+ * queue in service, even if it is idling). -+ * @busy_in_flight_queues: number of @bfq_queues containing pending or -+ * in-flight requests, plus the @bfq_queue in -+ * service, even if idle but waiting for the -+ * possible arrival of its next sync request. This -+ * field is updated only if the device is rotational, -+ * but used only if the device is also NCQ-capable. -+ * The reason why the field is updated also for non- -+ * NCQ-capable rotational devices is related to the -+ * fact that the value of @hw_tag may be set also -+ * later than when busy_in_flight_queues may need to -+ * be incremented for the first time(s). Taking also -+ * this possibility into account, to avoid unbalanced -+ * increments/decrements, would imply more overhead -+ * than just updating busy_in_flight_queues -+ * regardless of the value of @hw_tag. -+ * @const_seeky_busy_in_flight_queues: number of constantly-seeky @bfq_queues -+ * (that is, seeky queues that expired -+ * for budget timeout at least once) -+ * containing pending or in-flight -+ * requests, including the in-service -+ * @bfq_queue if constantly seeky. This -+ * field is updated only if the device -+ * is rotational, but used only if the -+ * device is also NCQ-capable (see the -+ * comments to @busy_in_flight_queues). -+ * @wr_busy_queues: number of weight-raised busy @bfq_queues. -+ * @queued: number of queued requests. -+ * @rq_in_driver: number of requests dispatched and waiting for completion. -+ * @sync_flight: number of sync requests in the driver. -+ * @max_rq_in_driver: max number of reqs in driver in the last -+ * @hw_tag_samples completed requests. -+ * @hw_tag_samples: nr of samples used to calculate hw_tag. -+ * @hw_tag: flag set to one if the driver is showing a queueing behavior. -+ * @budgets_assigned: number of budgets assigned. -+ * @idle_slice_timer: timer set when idling for the next sequential request -+ * from the queue in service. -+ * @unplug_work: delayed work to restart dispatching on the request queue. -+ * @in_service_queue: bfq_queue in service. -+ * @in_service_bic: bfq_io_cq (bic) associated with the @in_service_queue. -+ * @last_position: on-disk position of the last served request. -+ * @last_budget_start: beginning of the last budget. -+ * @last_idling_start: beginning of the last idle slice. -+ * @peak_rate: peak transfer rate observed for a budget. -+ * @peak_rate_samples: number of samples used to calculate @peak_rate. -+ * @bfq_max_budget: maximum budget allotted to a bfq_queue before -+ * rescheduling. -+ * @group_list: list of all the bfq_groups active on the device. -+ * @active_list: list of all the bfq_queues active on the device. -+ * @idle_list: list of all the bfq_queues idle on the device. -+ * @bfq_quantum: max number of requests dispatched per dispatch round. -+ * @bfq_fifo_expire: timeout for async/sync requests; when it expires -+ * requests are served in fifo order. -+ * @bfq_back_penalty: weight of backward seeks wrt forward ones. -+ * @bfq_back_max: maximum allowed backward seek. -+ * @bfq_slice_idle: maximum idling time. -+ * @bfq_user_max_budget: user-configured max budget value -+ * (0 for auto-tuning). -+ * @bfq_max_budget_async_rq: maximum budget (in nr of requests) allotted to -+ * async queues. -+ * @bfq_timeout: timeout for bfq_queues to consume their budget; used to -+ * to prevent seeky queues to impose long latencies to well -+ * behaved ones (this also implies that seeky queues cannot -+ * receive guarantees in the service domain; after a timeout -+ * they are charged for the whole allocated budget, to try -+ * to preserve a behavior reasonably fair among them, but -+ * without service-domain guarantees). -+ * @bfq_coop_thresh: number of queue merges after which a @bfq_queue is -+ * no more granted any weight-raising. -+ * @bfq_failed_cooperations: number of consecutive failed cooperation -+ * chances after which weight-raising is restored -+ * to a queue subject to more than bfq_coop_thresh -+ * queue merges. -+ * @bfq_requests_within_timer: number of consecutive requests that must be -+ * issued within the idle time slice to set -+ * again idling to a queue which was marked as -+ * non-I/O-bound (see the definition of the -+ * IO_bound flag for further details). -+ * @last_ins_in_burst: last time at which a queue entered the current -+ * burst of queues being activated shortly after -+ * each other; for more details about this and the -+ * following parameters related to a burst of -+ * activations, see the comments to the function -+ * @bfq_handle_burst. -+ * @bfq_burst_interval: reference time interval used to decide whether a -+ * queue has been activated shortly after -+ * @last_ins_in_burst. -+ * @burst_size: number of queues in the current burst of queue activations. -+ * @bfq_large_burst_thresh: maximum burst size above which the current -+ * queue-activation burst is deemed as 'large'. -+ * @large_burst: true if a large queue-activation burst is in progress. -+ * @burst_list: head of the burst list (as for the above fields, more details -+ * in the comments to the function bfq_handle_burst). -+ * @low_latency: if set to true, low-latency heuristics are enabled. -+ * @bfq_wr_coeff: maximum factor by which the weight of a weight-raised -+ * queue is multiplied. -+ * @bfq_wr_max_time: maximum duration of a weight-raising period (jiffies). -+ * @bfq_wr_rt_max_time: maximum duration for soft real-time processes. -+ * @bfq_wr_min_idle_time: minimum idle period after which weight-raising -+ * may be reactivated for a queue (in jiffies). -+ * @bfq_wr_min_inter_arr_async: minimum period between request arrivals -+ * after which weight-raising may be -+ * reactivated for an already busy queue -+ * (in jiffies). -+ * @bfq_wr_max_softrt_rate: max service-rate for a soft real-time queue, -+ * sectors per seconds. -+ * @RT_prod: cached value of the product R*T used for computing the maximum -+ * duration of the weight raising automatically. -+ * @device_speed: device-speed class for the low-latency heuristic. -+ * @oom_bfqq: fallback dummy bfqq for extreme OOM conditions. -+ * -+ * All the fields are protected by the @queue lock. -+ */ -+struct bfq_data { -+ struct request_queue *queue; -+ -+ struct bfq_group *root_group; -+ struct rb_root rq_pos_tree; -+ -+#ifdef CONFIG_CGROUP_BFQIO -+ int active_numerous_groups; -+#endif -+ -+ struct rb_root queue_weights_tree; -+ struct rb_root group_weights_tree; -+ -+ int busy_queues; -+ int busy_in_flight_queues; -+ int const_seeky_busy_in_flight_queues; -+ int wr_busy_queues; -+ int queued; -+ int rq_in_driver; -+ int sync_flight; -+ -+ int max_rq_in_driver; -+ int hw_tag_samples; -+ int hw_tag; -+ -+ int budgets_assigned; -+ -+ struct timer_list idle_slice_timer; -+ struct work_struct unplug_work; -+ -+ struct bfq_queue *in_service_queue; -+ struct bfq_io_cq *in_service_bic; -+ -+ sector_t last_position; -+ -+ ktime_t last_budget_start; -+ ktime_t last_idling_start; -+ int peak_rate_samples; -+ u64 peak_rate; -+ unsigned long bfq_max_budget; -+ -+ struct hlist_head group_list; -+ struct list_head active_list; -+ struct list_head idle_list; -+ -+ unsigned int bfq_quantum; -+ unsigned int bfq_fifo_expire[2]; -+ unsigned int bfq_back_penalty; -+ unsigned int bfq_back_max; -+ unsigned int bfq_slice_idle; -+ u64 bfq_class_idle_last_service; -+ -+ unsigned int bfq_user_max_budget; -+ unsigned int bfq_max_budget_async_rq; -+ unsigned int bfq_timeout[2]; -+ -+ unsigned int bfq_coop_thresh; -+ unsigned int bfq_failed_cooperations; -+ unsigned int bfq_requests_within_timer; -+ -+ unsigned long last_ins_in_burst; -+ unsigned long bfq_burst_interval; -+ int burst_size; -+ unsigned long bfq_large_burst_thresh; -+ bool large_burst; -+ struct hlist_head burst_list; -+ -+ bool low_latency; -+ -+ /* parameters of the low_latency heuristics */ -+ unsigned int bfq_wr_coeff; -+ unsigned int bfq_wr_max_time; -+ unsigned int bfq_wr_rt_max_time; -+ unsigned int bfq_wr_min_idle_time; -+ unsigned long bfq_wr_min_inter_arr_async; -+ unsigned int bfq_wr_max_softrt_rate; -+ u64 RT_prod; -+ enum bfq_device_speed device_speed; -+ -+ struct bfq_queue oom_bfqq; -+}; -+ -+enum bfqq_state_flags { -+ BFQ_BFQQ_FLAG_busy = 0, /* has requests or is in service */ -+ BFQ_BFQQ_FLAG_wait_request, /* waiting for a request */ -+ BFQ_BFQQ_FLAG_must_alloc, /* must be allowed rq alloc */ -+ BFQ_BFQQ_FLAG_fifo_expire, /* FIFO checked in this slice */ -+ BFQ_BFQQ_FLAG_idle_window, /* slice idling enabled */ -+ BFQ_BFQQ_FLAG_prio_changed, /* task priority has changed */ -+ BFQ_BFQQ_FLAG_sync, /* synchronous queue */ -+ BFQ_BFQQ_FLAG_budget_new, /* no completion with this budget */ -+ BFQ_BFQQ_FLAG_IO_bound, /* -+ * bfqq has timed-out at least once -+ * having consumed at most 2/10 of -+ * its budget -+ */ -+ BFQ_BFQQ_FLAG_in_large_burst, /* -+ * bfqq activated in a large burst, -+ * see comments to bfq_handle_burst. -+ */ -+ BFQ_BFQQ_FLAG_constantly_seeky, /* -+ * bfqq has proved to be slow and -+ * seeky until budget timeout -+ */ -+ BFQ_BFQQ_FLAG_softrt_update, /* -+ * may need softrt-next-start -+ * update -+ */ -+ BFQ_BFQQ_FLAG_coop, /* bfqq is shared */ -+ BFQ_BFQQ_FLAG_split_coop, /* shared bfqq will be split */ -+ BFQ_BFQQ_FLAG_just_split, /* queue has just been split */ -+}; -+ -+#define BFQ_BFQQ_FNS(name) \ -+static inline void bfq_mark_bfqq_##name(struct bfq_queue *bfqq) \ -+{ \ -+ (bfqq)->flags |= (1 << BFQ_BFQQ_FLAG_##name); \ -+} \ -+static inline void bfq_clear_bfqq_##name(struct bfq_queue *bfqq) \ -+{ \ -+ (bfqq)->flags &= ~(1 << BFQ_BFQQ_FLAG_##name); \ -+} \ -+static inline int bfq_bfqq_##name(const struct bfq_queue *bfqq) \ -+{ \ -+ return ((bfqq)->flags & (1 << BFQ_BFQQ_FLAG_##name)) != 0; \ -+} -+ -+BFQ_BFQQ_FNS(busy); -+BFQ_BFQQ_FNS(wait_request); -+BFQ_BFQQ_FNS(must_alloc); -+BFQ_BFQQ_FNS(fifo_expire); -+BFQ_BFQQ_FNS(idle_window); -+BFQ_BFQQ_FNS(prio_changed); -+BFQ_BFQQ_FNS(sync); -+BFQ_BFQQ_FNS(budget_new); -+BFQ_BFQQ_FNS(IO_bound); -+BFQ_BFQQ_FNS(in_large_burst); -+BFQ_BFQQ_FNS(constantly_seeky); -+BFQ_BFQQ_FNS(coop); -+BFQ_BFQQ_FNS(split_coop); -+BFQ_BFQQ_FNS(just_split); -+BFQ_BFQQ_FNS(softrt_update); -+#undef BFQ_BFQQ_FNS -+ -+/* Logging facilities. */ -+#define bfq_log_bfqq(bfqd, bfqq, fmt, args...) \ -+ blk_add_trace_msg((bfqd)->queue, "bfq%d " fmt, (bfqq)->pid, ##args) -+ -+#define bfq_log(bfqd, fmt, args...) \ -+ blk_add_trace_msg((bfqd)->queue, "bfq " fmt, ##args) -+ -+/* Expiration reasons. */ -+enum bfqq_expiration { -+ BFQ_BFQQ_TOO_IDLE = 0, /* -+ * queue has been idling for -+ * too long -+ */ -+ BFQ_BFQQ_BUDGET_TIMEOUT, /* budget took too long to be used */ -+ BFQ_BFQQ_BUDGET_EXHAUSTED, /* budget consumed */ -+ BFQ_BFQQ_NO_MORE_REQUESTS, /* the queue has no more requests */ -+}; -+ -+#ifdef CONFIG_CGROUP_BFQIO -+/** -+ * struct bfq_group - per (device, cgroup) data structure. -+ * @entity: schedulable entity to insert into the parent group sched_data. -+ * @sched_data: own sched_data, to contain child entities (they may be -+ * both bfq_queues and bfq_groups). -+ * @group_node: node to be inserted into the bfqio_cgroup->group_data -+ * list of the containing cgroup's bfqio_cgroup. -+ * @bfqd_node: node to be inserted into the @bfqd->group_list list -+ * of the groups active on the same device; used for cleanup. -+ * @bfqd: the bfq_data for the device this group acts upon. -+ * @async_bfqq: array of async queues for all the tasks belonging to -+ * the group, one queue per ioprio value per ioprio_class, -+ * except for the idle class that has only one queue. -+ * @async_idle_bfqq: async queue for the idle class (ioprio is ignored). -+ * @my_entity: pointer to @entity, %NULL for the toplevel group; used -+ * to avoid too many special cases during group creation/ -+ * migration. -+ * @active_entities: number of active entities belonging to the group; -+ * unused for the root group. Used to know whether there -+ * are groups with more than one active @bfq_entity -+ * (see the comments to the function -+ * bfq_bfqq_must_not_expire()). -+ * -+ * Each (device, cgroup) pair has its own bfq_group, i.e., for each cgroup -+ * there is a set of bfq_groups, each one collecting the lower-level -+ * entities belonging to the group that are acting on the same device. -+ * -+ * Locking works as follows: -+ * o @group_node is protected by the bfqio_cgroup lock, and is accessed -+ * via RCU from its readers. -+ * o @bfqd is protected by the queue lock, RCU is used to access it -+ * from the readers. -+ * o All the other fields are protected by the @bfqd queue lock. -+ */ -+struct bfq_group { -+ struct bfq_entity entity; -+ struct bfq_sched_data sched_data; -+ -+ struct hlist_node group_node; -+ struct hlist_node bfqd_node; -+ -+ void *bfqd; -+ -+ struct bfq_queue *async_bfqq[2][IOPRIO_BE_NR]; -+ struct bfq_queue *async_idle_bfqq; -+ -+ struct bfq_entity *my_entity; -+ -+ int active_entities; -+}; -+ -+/** -+ * struct bfqio_cgroup - bfq cgroup data structure. -+ * @css: subsystem state for bfq in the containing cgroup. -+ * @online: flag marked when the subsystem is inserted. -+ * @weight: cgroup weight. -+ * @ioprio: cgroup ioprio. -+ * @ioprio_class: cgroup ioprio_class. -+ * @lock: spinlock that protects @ioprio, @ioprio_class and @group_data. -+ * @group_data: list containing the bfq_group belonging to this cgroup. -+ * -+ * @group_data is accessed using RCU, with @lock protecting the updates, -+ * @ioprio and @ioprio_class are protected by @lock. -+ */ -+struct bfqio_cgroup { -+ struct cgroup_subsys_state css; -+ bool online; -+ -+ unsigned short weight, ioprio, ioprio_class; -+ -+ spinlock_t lock; -+ struct hlist_head group_data; -+}; -+#else -+struct bfq_group { -+ struct bfq_sched_data sched_data; -+ -+ struct bfq_queue *async_bfqq[2][IOPRIO_BE_NR]; -+ struct bfq_queue *async_idle_bfqq; -+}; -+#endif -+ -+static inline struct bfq_service_tree * -+bfq_entity_service_tree(struct bfq_entity *entity) -+{ -+ struct bfq_sched_data *sched_data = entity->sched_data; -+ unsigned int idx = entity->ioprio_class - 1; -+ -+ BUG_ON(idx >= BFQ_IOPRIO_CLASSES); -+ BUG_ON(sched_data == NULL); -+ -+ return sched_data->service_tree + idx; -+} -+ -+static inline struct bfq_queue *bic_to_bfqq(struct bfq_io_cq *bic, -+ bool is_sync) -+{ -+ return bic->bfqq[is_sync]; -+} -+ -+static inline void bic_set_bfqq(struct bfq_io_cq *bic, -+ struct bfq_queue *bfqq, bool is_sync) -+{ -+ bic->bfqq[is_sync] = bfqq; -+} -+ -+static inline struct bfq_data *bic_to_bfqd(struct bfq_io_cq *bic) -+{ -+ return bic->icq.q->elevator->elevator_data; -+} -+ -+/** -+ * bfq_get_bfqd_locked - get a lock to a bfqd using a RCU protected pointer. -+ * @ptr: a pointer to a bfqd. -+ * @flags: storage for the flags to be saved. -+ * -+ * This function allows bfqg->bfqd to be protected by the -+ * queue lock of the bfqd they reference; the pointer is dereferenced -+ * under RCU, so the storage for bfqd is assured to be safe as long -+ * as the RCU read side critical section does not end. After the -+ * bfqd->queue->queue_lock is taken the pointer is rechecked, to be -+ * sure that no other writer accessed it. If we raced with a writer, -+ * the function returns NULL, with the queue unlocked, otherwise it -+ * returns the dereferenced pointer, with the queue locked. -+ */ -+static inline struct bfq_data *bfq_get_bfqd_locked(void **ptr, -+ unsigned long *flags) -+{ -+ struct bfq_data *bfqd; -+ -+ rcu_read_lock(); -+ bfqd = rcu_dereference(*(struct bfq_data **)ptr); -+ -+ if (bfqd != NULL) { -+ spin_lock_irqsave(bfqd->queue->queue_lock, *flags); -+ if (*ptr == bfqd) -+ goto out; -+ spin_unlock_irqrestore(bfqd->queue->queue_lock, *flags); -+ } -+ -+ bfqd = NULL; -+out: -+ rcu_read_unlock(); -+ return bfqd; -+} -+ -+static inline void bfq_put_bfqd_unlock(struct bfq_data *bfqd, -+ unsigned long *flags) -+{ -+ spin_unlock_irqrestore(bfqd->queue->queue_lock, *flags); -+} -+ -+static void bfq_changed_ioprio(struct bfq_io_cq *bic); -+static void bfq_put_queue(struct bfq_queue *bfqq); -+static void bfq_dispatch_insert(struct request_queue *q, struct request *rq); -+static struct bfq_queue *bfq_get_queue(struct bfq_data *bfqd, -+ struct bfq_group *bfqg, int is_sync, -+ struct bfq_io_cq *bic, gfp_t gfp_mask); -+static void bfq_end_wr_async_queues(struct bfq_data *bfqd, -+ struct bfq_group *bfqg); -+static void bfq_put_async_queues(struct bfq_data *bfqd, struct bfq_group *bfqg); -+static void bfq_exit_bfqq(struct bfq_data *bfqd, struct bfq_queue *bfqq); -+ -+#endif /* _BFQ_H */ -diff -Nur linux-4.1.3/block/bfq-ioc.c linux-xbian-imx6/block/bfq-ioc.c ---- linux-4.1.3/block/bfq-ioc.c 1970-01-01 01:00:00.000000000 +0100 -+++ linux-xbian-imx6/block/bfq-ioc.c 2015-07-27 23:13:03.600137415 +0200 -@@ -0,0 +1,36 @@ -+/* -+ * BFQ: I/O context handling. -+ * -+ * Based on ideas and code from CFQ: -+ * Copyright (C) 2003 Jens Axboe -+ * -+ * Copyright (C) 2008 Fabio Checconi -+ * Paolo Valente -+ * -+ * Copyright (C) 2010 Paolo Valente -+ */ -+ -+/** -+ * icq_to_bic - convert iocontext queue structure to bfq_io_cq. -+ * @icq: the iocontext queue. -+ */ -+static inline struct bfq_io_cq *icq_to_bic(struct io_cq *icq) -+{ -+ /* bic->icq is the first member, %NULL will convert to %NULL */ -+ return container_of(icq, struct bfq_io_cq, icq); -+} -+ -+/** -+ * bfq_bic_lookup - search into @ioc a bic associated to @bfqd. -+ * @bfqd: the lookup key. -+ * @ioc: the io_context of the process doing I/O. -+ * -+ * Queue lock must be held. -+ */ -+static inline struct bfq_io_cq *bfq_bic_lookup(struct bfq_data *bfqd, -+ struct io_context *ioc) -+{ -+ if (ioc) -+ return icq_to_bic(ioc_lookup_icq(ioc, bfqd->queue)); -+ return NULL; -+} -diff -Nur linux-4.1.3/block/bfq-iosched.c linux-xbian-imx6/block/bfq-iosched.c ---- linux-4.1.3/block/bfq-iosched.c 1970-01-01 01:00:00.000000000 +0100 -+++ linux-xbian-imx6/block/bfq-iosched.c 2015-07-27 23:13:03.600137415 +0200 -@@ -0,0 +1,4223 @@ -+/* -+ * Budget Fair Queueing (BFQ) disk scheduler. -+ * -+ * Based on ideas and code from CFQ: -+ * Copyright (C) 2003 Jens Axboe -+ * -+ * Copyright (C) 2008 Fabio Checconi -+ * Paolo Valente -+ * -+ * Copyright (C) 2010 Paolo Valente -+ * -+ * Licensed under the GPL-2 as detailed in the accompanying COPYING.BFQ -+ * file. -+ * -+ * BFQ is a proportional-share storage-I/O scheduling algorithm based on -+ * the slice-by-slice service scheme of CFQ. But BFQ assigns budgets, -+ * measured in number of sectors, to processes instead of time slices. The -+ * device is not granted to the in-service process for a given time slice, -+ * but until it has exhausted its assigned budget. This change from the time -+ * to the service domain allows BFQ to distribute the device throughput -+ * among processes as desired, without any distortion due to ZBR, workload -+ * fluctuations or other factors. BFQ uses an ad hoc internal scheduler, -+ * called B-WF2Q+, to schedule processes according to their budgets. More -+ * precisely, BFQ schedules queues associated to processes. Thanks to the -+ * accurate policy of B-WF2Q+, BFQ can afford to assign high budgets to -+ * I/O-bound processes issuing sequential requests (to boost the -+ * throughput), and yet guarantee a low latency to interactive and soft -+ * real-time applications. -+ * -+ * BFQ is described in [1], where also a reference to the initial, more -+ * theoretical paper on BFQ can be found. The interested reader can find -+ * in the latter paper full details on the main algorithm, as well as -+ * formulas of the guarantees and formal proofs of all the properties. -+ * With respect to the version of BFQ presented in these papers, this -+ * implementation adds a few more heuristics, such as the one that -+ * guarantees a low latency to soft real-time applications, and a -+ * hierarchical extension based on H-WF2Q+. -+ * -+ * B-WF2Q+ is based on WF2Q+, that is described in [2], together with -+ * H-WF2Q+, while the augmented tree used to implement B-WF2Q+ with O(log N) -+ * complexity derives from the one introduced with EEVDF in [3]. -+ * -+ * [1] P. Valente and M. Andreolini, ``Improving Application Responsiveness -+ * with the BFQ Disk I/O Scheduler'', -+ * Proceedings of the 5th Annual International Systems and Storage -+ * Conference (SYSTOR '12), June 2012. -+ * -+ * http://algogroup.unimo.it/people/paolo/disk_sched/bf1-v1-suite-results.pdf -+ * -+ * [2] Jon C.R. Bennett and H. Zhang, ``Hierarchical Packet Fair Queueing -+ * Algorithms,'' IEEE/ACM Transactions on Networking, 5(5):675-689, -+ * Oct 1997. -+ * -+ * http://www.cs.cmu.edu/~hzhang/papers/TON-97-Oct.ps.gz -+ * -+ * [3] I. Stoica and H. Abdel-Wahab, ``Earliest Eligible Virtual Deadline -+ * First: A Flexible and Accurate Mechanism for Proportional Share -+ * Resource Allocation,'' technical report. -+ * -+ * http://www.cs.berkeley.edu/~istoica/papers/eevdf-tr-95.pdf -+ */ -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include "bfq.h" -+#include "blk.h" -+ -+/* Max number of dispatches in one round of service. */ -+static const int bfq_quantum = 4; -+ -+/* Expiration time of sync (0) and async (1) requests, in jiffies. */ -+static const int bfq_fifo_expire[2] = { HZ / 4, HZ / 8 }; -+ -+/* Maximum backwards seek, in KiB. */ -+static const int bfq_back_max = 16 * 1024; -+ -+/* Penalty of a backwards seek, in number of sectors. */ -+static const int bfq_back_penalty = 2; -+ -+/* Idling period duration, in jiffies. */ -+static int bfq_slice_idle = HZ / 125; -+ -+/* Default maximum budget values, in sectors and number of requests. */ -+static const int bfq_default_max_budget = 16 * 1024; -+static const int bfq_max_budget_async_rq = 4; -+ -+/* -+ * Async to sync throughput distribution is controlled as follows: -+ * when an async request is served, the entity is charged the number -+ * of sectors of the request, multiplied by the factor below -+ */ -+static const int bfq_async_charge_factor = 10; -+ -+/* Default timeout values, in jiffies, approximating CFQ defaults. */ -+static const int bfq_timeout_sync = HZ / 8; -+static int bfq_timeout_async = HZ / 25; -+ -+struct kmem_cache *bfq_pool; -+ -+/* Below this threshold (in ms), we consider thinktime immediate. */ -+#define BFQ_MIN_TT 2 -+ -+/* hw_tag detection: parallel requests threshold and min samples needed. */ -+#define BFQ_HW_QUEUE_THRESHOLD 4 -+#define BFQ_HW_QUEUE_SAMPLES 32 -+ -+#define BFQQ_SEEK_THR (sector_t)(8 * 1024) -+#define BFQQ_SEEKY(bfqq) ((bfqq)->seek_mean > BFQQ_SEEK_THR) -+ -+/* Min samples used for peak rate estimation (for autotuning). */ -+#define BFQ_PEAK_RATE_SAMPLES 32 -+ -+/* Shift used for peak rate fixed precision calculations. */ -+#define BFQ_RATE_SHIFT 16 -+ -+/* -+ * By default, BFQ computes the duration of the weight raising for -+ * interactive applications automatically, using the following formula: -+ * duration = (R / r) * T, where r is the peak rate of the device, and -+ * R and T are two reference parameters. -+ * In particular, R is the peak rate of the reference device (see below), -+ * and T is a reference time: given the systems that are likely to be -+ * installed on the reference device according to its speed class, T is -+ * about the maximum time needed, under BFQ and while reading two files in -+ * parallel, to load typical large applications on these systems. -+ * In practice, the slower/faster the device at hand is, the more/less it -+ * takes to load applications with respect to the reference device. -+ * Accordingly, the longer/shorter BFQ grants weight raising to interactive -+ * applications. -+ * -+ * BFQ uses four different reference pairs (R, T), depending on: -+ * . whether the device is rotational or non-rotational; -+ * . whether the device is slow, such as old or portable HDDs, as well as -+ * SD cards, or fast, such as newer HDDs and SSDs. -+ * -+ * The device's speed class is dynamically (re)detected in -+ * bfq_update_peak_rate() every time the estimated peak rate is updated. -+ * -+ * In the following definitions, R_slow[0]/R_fast[0] and T_slow[0]/T_fast[0] -+ * are the reference values for a slow/fast rotational device, whereas -+ * R_slow[1]/R_fast[1] and T_slow[1]/T_fast[1] are the reference values for -+ * a slow/fast non-rotational device. Finally, device_speed_thresh are the -+ * thresholds used to switch between speed classes. -+ * Both the reference peak rates and the thresholds are measured in -+ * sectors/usec, left-shifted by BFQ_RATE_SHIFT. -+ */ -+static int R_slow[2] = {1536, 10752}; -+static int R_fast[2] = {17415, 34791}; -+/* -+ * To improve readability, a conversion function is used to initialize the -+ * following arrays, which entails that they can be initialized only in a -+ * function. -+ */ -+static int T_slow[2]; -+static int T_fast[2]; -+static int device_speed_thresh[2]; -+ -+#define BFQ_SERVICE_TREE_INIT ((struct bfq_service_tree) \ -+ { RB_ROOT, RB_ROOT, NULL, NULL, 0, 0 }) -+ -+#define RQ_BIC(rq) ((struct bfq_io_cq *) (rq)->elv.priv[0]) -+#define RQ_BFQQ(rq) ((rq)->elv.priv[1]) -+ -+static inline void bfq_schedule_dispatch(struct bfq_data *bfqd); -+ -+#include "bfq-ioc.c" -+#include "bfq-sched.c" -+#include "bfq-cgroup.c" -+ -+#define bfq_class_idle(bfqq) ((bfqq)->entity.ioprio_class ==\ -+ IOPRIO_CLASS_IDLE) -+#define bfq_class_rt(bfqq) ((bfqq)->entity.ioprio_class ==\ -+ IOPRIO_CLASS_RT) -+ -+#define bfq_sample_valid(samples) ((samples) > 80) -+ -+/* -+ * We regard a request as SYNC, if either it's a read or has the SYNC bit -+ * set (in which case it could also be a direct WRITE). -+ */ -+static inline int bfq_bio_sync(struct bio *bio) -+{ -+ if (bio_data_dir(bio) == READ || (bio->bi_rw & REQ_SYNC)) -+ return 1; -+ -+ return 0; -+} -+ -+/* -+ * Scheduler run of queue, if there are requests pending and no one in the -+ * driver that will restart queueing. -+ */ -+static inline void bfq_schedule_dispatch(struct bfq_data *bfqd) -+{ -+ if (bfqd->queued != 0) { -+ bfq_log(bfqd, "schedule dispatch"); -+ kblockd_schedule_work(&bfqd->unplug_work); -+ } -+} -+ -+/* -+ * Lifted from AS - choose which of rq1 and rq2 that is best served now. -+ * We choose the request that is closesr to the head right now. Distance -+ * behind the head is penalized and only allowed to a certain extent. -+ */ -+static struct request *bfq_choose_req(struct bfq_data *bfqd, -+ struct request *rq1, -+ struct request *rq2, -+ sector_t last) -+{ -+ sector_t s1, s2, d1 = 0, d2 = 0; -+ unsigned long back_max; -+#define BFQ_RQ1_WRAP 0x01 /* request 1 wraps */ -+#define BFQ_RQ2_WRAP 0x02 /* request 2 wraps */ -+ unsigned wrap = 0; /* bit mask: requests behind the disk head? */ -+ -+ if (rq1 == NULL || rq1 == rq2) -+ return rq2; -+ if (rq2 == NULL) -+ return rq1; -+ -+ if (rq_is_sync(rq1) && !rq_is_sync(rq2)) -+ return rq1; -+ else if (rq_is_sync(rq2) && !rq_is_sync(rq1)) -+ return rq2; -+ if ((rq1->cmd_flags & REQ_META) && !(rq2->cmd_flags & REQ_META)) -+ return rq1; -+ else if ((rq2->cmd_flags & REQ_META) && !(rq1->cmd_flags & REQ_META)) -+ return rq2; -+ -+ s1 = blk_rq_pos(rq1); -+ s2 = blk_rq_pos(rq2); -+ -+ /* -+ * By definition, 1KiB is 2 sectors. -+ */ -+ back_max = bfqd->bfq_back_max * 2; -+ -+ /* -+ * Strict one way elevator _except_ in the case where we allow -+ * short backward seeks which are biased as twice the cost of a -+ * similar forward seek. -+ */ -+ if (s1 >= last) -+ d1 = s1 - last; -+ else if (s1 + back_max >= last) -+ d1 = (last - s1) * bfqd->bfq_back_penalty; -+ else -+ wrap |= BFQ_RQ1_WRAP; -+ -+ if (s2 >= last) -+ d2 = s2 - last; -+ else if (s2 + back_max >= last) -+ d2 = (last - s2) * bfqd->bfq_back_penalty; -+ else -+ wrap |= BFQ_RQ2_WRAP; -+ -+ /* Found required data */ -+ -+ /* -+ * By doing switch() on the bit mask "wrap" we avoid having to -+ * check two variables for all permutations: --> faster! -+ */ -+ switch (wrap) { -+ case 0: /* common case for CFQ: rq1 and rq2 not wrapped */ -+ if (d1 < d2) -+ return rq1; -+ else if (d2 < d1) -+ return rq2; -+ else { -+ if (s1 >= s2) -+ return rq1; -+ else -+ return rq2; -+ } -+ -+ case BFQ_RQ2_WRAP: -+ return rq1; -+ case BFQ_RQ1_WRAP: -+ return rq2; -+ case (BFQ_RQ1_WRAP|BFQ_RQ2_WRAP): /* both rqs wrapped */ -+ default: -+ /* -+ * Since both rqs are wrapped, -+ * start with the one that's further behind head -+ * (--> only *one* back seek required), -+ * since back seek takes more time than forward. -+ */ -+ if (s1 <= s2) -+ return rq1; -+ else -+ return rq2; -+ } -+} -+ -+static struct bfq_queue * -+bfq_rq_pos_tree_lookup(struct bfq_data *bfqd, struct rb_root *root, -+ sector_t sector, struct rb_node **ret_parent, -+ struct rb_node ***rb_link) -+{ -+ struct rb_node **p, *parent; -+ struct bfq_queue *bfqq = NULL; -+ -+ parent = NULL; -+ p = &root->rb_node; -+ while (*p) { -+ struct rb_node **n; -+ -+ parent = *p; -+ bfqq = rb_entry(parent, struct bfq_queue, pos_node); -+ -+ /* -+ * Sort strictly based on sector. Smallest to the left, -+ * largest to the right. -+ */ -+ if (sector > blk_rq_pos(bfqq->next_rq)) -+ n = &(*p)->rb_right; -+ else if (sector < blk_rq_pos(bfqq->next_rq)) -+ n = &(*p)->rb_left; -+ else -+ break; -+ p = n; -+ bfqq = NULL; -+ } -+ -+ *ret_parent = parent; -+ if (rb_link) -+ *rb_link = p; -+ -+ bfq_log(bfqd, "rq_pos_tree_lookup %llu: returning %d", -+ (long long unsigned)sector, -+ bfqq != NULL ? bfqq->pid : 0); -+ -+ return bfqq; -+} -+ -+static void bfq_rq_pos_tree_add(struct bfq_data *bfqd, struct bfq_queue *bfqq) -+{ -+ struct rb_node **p, *parent; -+ struct bfq_queue *__bfqq; -+ -+ if (bfqq->pos_root != NULL) { -+ rb_erase(&bfqq->pos_node, bfqq->pos_root); -+ bfqq->pos_root = NULL; -+ } -+ -+ if (bfq_class_idle(bfqq)) -+ return; -+ if (!bfqq->next_rq) -+ return; -+ -+ bfqq->pos_root = &bfqd->rq_pos_tree; -+ __bfqq = bfq_rq_pos_tree_lookup(bfqd, bfqq->pos_root, -+ blk_rq_pos(bfqq->next_rq), &parent, &p); -+ if (__bfqq == NULL) { -+ rb_link_node(&bfqq->pos_node, parent, p); -+ rb_insert_color(&bfqq->pos_node, bfqq->pos_root); -+ } else -+ bfqq->pos_root = NULL; -+} -+ -+/* -+ * Tell whether there are active queues or groups with differentiated weights. -+ */ -+static inline bool bfq_differentiated_weights(struct bfq_data *bfqd) -+{ -+ BUG_ON(!bfqd->hw_tag); -+ /* -+ * For weights to differ, at least one of the trees must contain -+ * at least two nodes. -+ */ -+ return (!RB_EMPTY_ROOT(&bfqd->queue_weights_tree) && -+ (bfqd->queue_weights_tree.rb_node->rb_left || -+ bfqd->queue_weights_tree.rb_node->rb_right) -+#ifdef CONFIG_CGROUP_BFQIO -+ ) || -+ (!RB_EMPTY_ROOT(&bfqd->group_weights_tree) && -+ (bfqd->group_weights_tree.rb_node->rb_left || -+ bfqd->group_weights_tree.rb_node->rb_right) -+#endif -+ ); -+} -+ -+/* -+ * If the weight-counter tree passed as input contains no counter for -+ * the weight of the input entity, then add that counter; otherwise just -+ * increment the existing counter. -+ * -+ * Note that weight-counter trees contain few nodes in mostly symmetric -+ * scenarios. For example, if all queues have the same weight, then the -+ * weight-counter tree for the queues may contain at most one node. -+ * This holds even if low_latency is on, because weight-raised queues -+ * are not inserted in the tree. -+ * In most scenarios, the rate at which nodes are created/destroyed -+ * should be low too. -+ */ -+static void bfq_weights_tree_add(struct bfq_data *bfqd, -+ struct bfq_entity *entity, -+ struct rb_root *root) -+{ -+ struct rb_node **new = &(root->rb_node), *parent = NULL; -+ -+ /* -+ * Do not insert if: -+ * - the device does not support queueing; -+ * - the entity is already associated with a counter, which happens if: -+ * 1) the entity is associated with a queue, 2) a request arrival -+ * has caused the queue to become both non-weight-raised, and hence -+ * change its weight, and backlogged; in this respect, each -+ * of the two events causes an invocation of this function, -+ * 3) this is the invocation of this function caused by the second -+ * event. This second invocation is actually useless, and we handle -+ * this fact by exiting immediately. More efficient or clearer -+ * solutions might possibly be adopted. -+ */ -+ if (!bfqd->hw_tag || entity->weight_counter) -+ return; -+ -+ while (*new) { -+ struct bfq_weight_counter *__counter = container_of(*new, -+ struct bfq_weight_counter, -+ weights_node); -+ parent = *new; -+ -+ if (entity->weight == __counter->weight) { -+ entity->weight_counter = __counter; -+ goto inc_counter; -+ } -+ if (entity->weight < __counter->weight) -+ new = &((*new)->rb_left); -+ else -+ new = &((*new)->rb_right); -+ } -+ -+ entity->weight_counter = kzalloc(sizeof(struct bfq_weight_counter), -+ GFP_ATOMIC); -+ entity->weight_counter->weight = entity->weight; -+ rb_link_node(&entity->weight_counter->weights_node, parent, new); -+ rb_insert_color(&entity->weight_counter->weights_node, root); -+ -+inc_counter: -+ entity->weight_counter->num_active++; -+} -+ -+/* -+ * Decrement the weight counter associated with the entity, and, if the -+ * counter reaches 0, remove the counter from the tree. -+ * See the comments to the function bfq_weights_tree_add() for considerations -+ * about overhead. -+ */ -+static void bfq_weights_tree_remove(struct bfq_data *bfqd, -+ struct bfq_entity *entity, -+ struct rb_root *root) -+{ -+ /* -+ * Check whether the entity is actually associated with a counter. -+ * In fact, the device may not be considered NCQ-capable for a while, -+ * which implies that no insertion in the weight trees is performed, -+ * after which the device may start to be deemed NCQ-capable, and hence -+ * this function may start to be invoked. This may cause the function -+ * to be invoked for entities that are not associated with any counter. -+ */ -+ if (!entity->weight_counter) -+ return; -+ -+ BUG_ON(RB_EMPTY_ROOT(root)); -+ BUG_ON(entity->weight_counter->weight != entity->weight); -+ -+ BUG_ON(!entity->weight_counter->num_active); -+ entity->weight_counter->num_active--; -+ if (entity->weight_counter->num_active > 0) -+ goto reset_entity_pointer; -+ -+ rb_erase(&entity->weight_counter->weights_node, root); -+ kfree(entity->weight_counter); -+ -+reset_entity_pointer: -+ entity->weight_counter = NULL; -+} -+ -+static struct request *bfq_find_next_rq(struct bfq_data *bfqd, -+ struct bfq_queue *bfqq, -+ struct request *last) -+{ -+ struct rb_node *rbnext = rb_next(&last->rb_node); -+ struct rb_node *rbprev = rb_prev(&last->rb_node); -+ struct request *next = NULL, *prev = NULL; -+ -+ BUG_ON(RB_EMPTY_NODE(&last->rb_node)); -+ -+ if (rbprev != NULL) -+ prev = rb_entry_rq(rbprev); -+ -+ if (rbnext != NULL) -+ next = rb_entry_rq(rbnext); -+ else { -+ rbnext = rb_first(&bfqq->sort_list); -+ if (rbnext && rbnext != &last->rb_node) -+ next = rb_entry_rq(rbnext); -+ } -+ -+ return bfq_choose_req(bfqd, next, prev, blk_rq_pos(last)); -+} -+ -+/* see the definition of bfq_async_charge_factor for details */ -+static inline unsigned long bfq_serv_to_charge(struct request *rq, -+ struct bfq_queue *bfqq) -+{ -+ return blk_rq_sectors(rq) * -+ (1 + ((!bfq_bfqq_sync(bfqq)) * (bfqq->wr_coeff == 1) * -+ bfq_async_charge_factor)); -+} -+ -+/** -+ * bfq_updated_next_req - update the queue after a new next_rq selection. -+ * @bfqd: the device data the queue belongs to. -+ * @bfqq: the queue to update. -+ * -+ * If the first request of a queue changes we make sure that the queue -+ * has enough budget to serve at least its first request (if the -+ * request has grown). We do this because if the queue has not enough -+ * budget for its first request, it has to go through two dispatch -+ * rounds to actually get it dispatched. -+ */ -+static void bfq_updated_next_req(struct bfq_data *bfqd, -+ struct bfq_queue *bfqq) -+{ -+ struct bfq_entity *entity = &bfqq->entity; -+ struct bfq_service_tree *st = bfq_entity_service_tree(entity); -+ struct request *next_rq = bfqq->next_rq; -+ unsigned long new_budget; -+ -+ if (next_rq == NULL) -+ return; -+ -+ if (bfqq == bfqd->in_service_queue) -+ /* -+ * In order not to break guarantees, budgets cannot be -+ * changed after an entity has been selected. -+ */ -+ return; -+ -+ BUG_ON(entity->tree != &st->active); -+ BUG_ON(entity == entity->sched_data->in_service_entity); -+ -+ new_budget = max_t(unsigned long, bfqq->max_budget, -+ bfq_serv_to_charge(next_rq, bfqq)); -+ if (entity->budget != new_budget) { -+ entity->budget = new_budget; -+ bfq_log_bfqq(bfqd, bfqq, "updated next rq: new budget %lu", -+ new_budget); -+ bfq_activate_bfqq(bfqd, bfqq); -+ } -+} -+ -+static inline unsigned int bfq_wr_duration(struct bfq_data *bfqd) -+{ -+ u64 dur; -+ -+ if (bfqd->bfq_wr_max_time > 0) -+ return bfqd->bfq_wr_max_time; -+ -+ dur = bfqd->RT_prod; -+ do_div(dur, bfqd->peak_rate); -+ -+ return dur; -+} -+ -+static inline unsigned -+bfq_bfqq_cooperations(struct bfq_queue *bfqq) -+{ -+ return bfqq->bic ? bfqq->bic->cooperations : 0; -+} -+ -+static inline void -+bfq_bfqq_resume_state(struct bfq_queue *bfqq, struct bfq_io_cq *bic) -+{ -+ if (bic->saved_idle_window) -+ bfq_mark_bfqq_idle_window(bfqq); -+ else -+ bfq_clear_bfqq_idle_window(bfqq); -+ if (bic->saved_IO_bound) -+ bfq_mark_bfqq_IO_bound(bfqq); -+ else -+ bfq_clear_bfqq_IO_bound(bfqq); -+ /* Assuming that the flag in_large_burst is already correctly set */ -+ if (bic->wr_time_left && bfqq->bfqd->low_latency && -+ !bfq_bfqq_in_large_burst(bfqq) && -+ bic->cooperations < bfqq->bfqd->bfq_coop_thresh) { -+ /* -+ * Start a weight raising period with the duration given by -+ * the raising_time_left snapshot. -+ */ -+ if (bfq_bfqq_busy(bfqq)) -+ bfqq->bfqd->wr_busy_queues++; -+ bfqq->wr_coeff = bfqq->bfqd->bfq_wr_coeff; -+ bfqq->wr_cur_max_time = bic->wr_time_left; -+ bfqq->last_wr_start_finish = jiffies; -+ bfqq->entity.ioprio_changed = 1; -+ } -+ /* -+ * Clear wr_time_left to prevent bfq_bfqq_save_state() from -+ * getting confused about the queue's need of a weight-raising -+ * period. -+ */ -+ bic->wr_time_left = 0; -+} -+ -+/* Must be called with the queue_lock held. */ -+static int bfqq_process_refs(struct bfq_queue *bfqq) -+{ -+ int process_refs, io_refs; -+ -+ io_refs = bfqq->allocated[READ] + bfqq->allocated[WRITE]; -+ process_refs = atomic_read(&bfqq->ref) - io_refs - bfqq->entity.on_st; -+ BUG_ON(process_refs < 0); -+ return process_refs; -+} -+ -+/* Empty burst list and add just bfqq (see comments to bfq_handle_burst) */ -+static inline void bfq_reset_burst_list(struct bfq_data *bfqd, -+ struct bfq_queue *bfqq) -+{ -+ struct bfq_queue *item; -+ struct hlist_node *n; -+ -+ hlist_for_each_entry_safe(item, n, &bfqd->burst_list, burst_list_node) -+ hlist_del_init(&item->burst_list_node); -+ hlist_add_head(&bfqq->burst_list_node, &bfqd->burst_list); -+ bfqd->burst_size = 1; -+} -+ -+/* Add bfqq to the list of queues in current burst (see bfq_handle_burst) */ -+static void bfq_add_to_burst(struct bfq_data *bfqd, struct bfq_queue *bfqq) -+{ -+ /* Increment burst size to take into account also bfqq */ -+ bfqd->burst_size++; -+ -+ if (bfqd->burst_size == bfqd->bfq_large_burst_thresh) { -+ struct bfq_queue *pos, *bfqq_item; -+ struct hlist_node *n; -+ -+ /* -+ * Enough queues have been activated shortly after each -+ * other to consider this burst as large. -+ */ -+ bfqd->large_burst = true; -+ -+ /* -+ * We can now mark all queues in the burst list as -+ * belonging to a large burst. -+ */ -+ hlist_for_each_entry(bfqq_item, &bfqd->burst_list, -+ burst_list_node) -+ bfq_mark_bfqq_in_large_burst(bfqq_item); -+ bfq_mark_bfqq_in_large_burst(bfqq); -+ -+ /* -+ * From now on, and until the current burst finishes, any -+ * new queue being activated shortly after the last queue -+ * was inserted in the burst can be immediately marked as -+ * belonging to a large burst. So the burst list is not -+ * needed any more. Remove it. -+ */ -+ hlist_for_each_entry_safe(pos, n, &bfqd->burst_list, -+ burst_list_node) -+ hlist_del_init(&pos->burst_list_node); -+ } else /* burst not yet large: add bfqq to the burst list */ -+ hlist_add_head(&bfqq->burst_list_node, &bfqd->burst_list); -+} -+ -+/* -+ * If many queues happen to become active shortly after each other, then, -+ * to help the processes associated to these queues get their job done as -+ * soon as possible, it is usually better to not grant either weight-raising -+ * or device idling to these queues. In this comment we describe, firstly, -+ * the reasons why this fact holds, and, secondly, the next function, which -+ * implements the main steps needed to properly mark these queues so that -+ * they can then be treated in a different way. -+ * -+ * As for the terminology, we say that a queue becomes active, i.e., -+ * switches from idle to backlogged, either when it is created (as a -+ * consequence of the arrival of an I/O request), or, if already existing, -+ * when a new request for the queue arrives while the queue is idle. -+ * Bursts of activations, i.e., activations of different queues occurring -+ * shortly after each other, are typically caused by services or applications -+ * that spawn or reactivate many parallel threads/processes. Examples are -+ * systemd during boot or git grep. -+ * -+ * These services or applications benefit mostly from a high throughput: -+ * the quicker the requests of the activated queues are cumulatively served, -+ * the sooner the target job of these queues gets completed. As a consequence, -+ * weight-raising any of these queues, which also implies idling the device -+ * for it, is almost always counterproductive: in most cases it just lowers -+ * throughput. -+ * -+ * On the other hand, a burst of activations may be also caused by the start -+ * of an application that does not consist in a lot of parallel I/O-bound -+ * threads. In fact, with a complex application, the burst may be just a -+ * consequence of the fact that several processes need to be executed to -+ * start-up the application. To start an application as quickly as possible, -+ * the best thing to do is to privilege the I/O related to the application -+ * with respect to all other I/O. Therefore, the best strategy to start as -+ * quickly as possible an application that causes a burst of activations is -+ * to weight-raise all the queues activated during the burst. This is the -+ * exact opposite of the best strategy for the other type of bursts. -+ * -+ * In the end, to take the best action for each of the two cases, the two -+ * types of bursts need to be distinguished. Fortunately, this seems -+ * relatively easy to do, by looking at the sizes of the bursts. In -+ * particular, we found a threshold such that bursts with a larger size -+ * than that threshold are apparently caused only by services or commands -+ * such as systemd or git grep. For brevity, hereafter we call just 'large' -+ * these bursts. BFQ *does not* weight-raise queues whose activations occur -+ * in a large burst. In addition, for each of these queues BFQ performs or -+ * does not perform idling depending on which choice boosts the throughput -+ * most. The exact choice depends on the device and request pattern at -+ * hand. -+ * -+ * Turning back to the next function, it implements all the steps needed -+ * to detect the occurrence of a large burst and to properly mark all the -+ * queues belonging to it (so that they can then be treated in a different -+ * way). This goal is achieved by maintaining a special "burst list" that -+ * holds, temporarily, the queues that belong to the burst in progress. The -+ * list is then used to mark these queues as belonging to a large burst if -+ * the burst does become large. The main steps are the following. -+ * -+ * . when the very first queue is activated, the queue is inserted into the -+ * list (as it could be the first queue in a possible burst) -+ * -+ * . if the current burst has not yet become large, and a queue Q that does -+ * not yet belong to the burst is activated shortly after the last time -+ * at which a new queue entered the burst list, then the function appends -+ * Q to the burst list -+ * -+ * . if, as a consequence of the previous step, the burst size reaches -+ * the large-burst threshold, then -+ * -+ * . all the queues in the burst list are marked as belonging to a -+ * large burst -+ * -+ * . the burst list is deleted; in fact, the burst list already served -+ * its purpose (keeping temporarily track of the queues in a burst, -+ * so as to be able to mark them as belonging to a large burst in the -+ * previous sub-step), and now is not needed any more -+ * -+ * . the device enters a large-burst mode -+ * -+ * . if a queue Q that does not belong to the burst is activated while -+ * the device is in large-burst mode and shortly after the last time -+ * at which a queue either entered the burst list or was marked as -+ * belonging to the current large burst, then Q is immediately marked -+ * as belonging to a large burst. -+ * -+ * . if a queue Q that does not belong to the burst is activated a while -+ * later, i.e., not shortly after, than the last time at which a queue -+ * either entered the burst list or was marked as belonging to the -+ * current large burst, then the current burst is deemed as finished and: -+ * -+ * . the large-burst mode is reset if set -+ * -+ * . the burst list is emptied -+ * -+ * . Q is inserted in the burst list, as Q may be the first queue -+ * in a possible new burst (then the burst list contains just Q -+ * after this step). -+ */ -+static void bfq_handle_burst(struct bfq_data *bfqd, struct bfq_queue *bfqq, -+ bool idle_for_long_time) -+{ -+ /* -+ * If bfqq happened to be activated in a burst, but has been idle -+ * for at least as long as an interactive queue, then we assume -+ * that, in the overall I/O initiated in the burst, the I/O -+ * associated to bfqq is finished. So bfqq does not need to be -+ * treated as a queue belonging to a burst anymore. Accordingly, -+ * we reset bfqq's in_large_burst flag if set, and remove bfqq -+ * from the burst list if it's there. We do not decrement instead -+ * burst_size, because the fact that bfqq does not need to belong -+ * to the burst list any more does not invalidate the fact that -+ * bfqq may have been activated during the current burst. -+ */ -+ if (idle_for_long_time) { -+ hlist_del_init(&bfqq->burst_list_node); -+ bfq_clear_bfqq_in_large_burst(bfqq); -+ } -+ -+ /* -+ * If bfqq is already in the burst list or is part of a large -+ * burst, then there is nothing else to do. -+ */ -+ if (!hlist_unhashed(&bfqq->burst_list_node) || -+ bfq_bfqq_in_large_burst(bfqq)) -+ return; -+ -+ /* -+ * If bfqq's activation happens late enough, then the current -+ * burst is finished, and related data structures must be reset. -+ * -+ * In this respect, consider the special case where bfqq is the very -+ * first queue being activated. In this case, last_ins_in_burst is -+ * not yet significant when we get here. But it is easy to verify -+ * that, whether or not the following condition is true, bfqq will -+ * end up being inserted into the burst list. In particular the -+ * list will happen to contain only bfqq. And this is exactly what -+ * has to happen, as bfqq may be the first queue in a possible -+ * burst. -+ */ -+ if (time_is_before_jiffies(bfqd->last_ins_in_burst + -+ bfqd->bfq_burst_interval)) { -+ bfqd->large_burst = false; -+ bfq_reset_burst_list(bfqd, bfqq); -+ return; -+ } -+ -+ /* -+ * If we get here, then bfqq is being activated shortly after the -+ * last queue. So, if the current burst is also large, we can mark -+ * bfqq as belonging to this large burst immediately. -+ */ -+ if (bfqd->large_burst) { -+ bfq_mark_bfqq_in_large_burst(bfqq); -+ return; -+ } -+ -+ /* -+ * If we get here, then a large-burst state has not yet been -+ * reached, but bfqq is being activated shortly after the last -+ * queue. Then we add bfqq to the burst. -+ */ -+ bfq_add_to_burst(bfqd, bfqq); -+} -+ -+static void bfq_add_request(struct request *rq) -+{ -+ struct bfq_queue *bfqq = RQ_BFQQ(rq); -+ struct bfq_entity *entity = &bfqq->entity; -+ struct bfq_data *bfqd = bfqq->bfqd; -+ struct request *next_rq, *prev; -+ unsigned long old_wr_coeff = bfqq->wr_coeff; -+ bool interactive = false; -+ -+ bfq_log_bfqq(bfqd, bfqq, "add_request %d", rq_is_sync(rq)); -+ bfqq->queued[rq_is_sync(rq)]++; -+ bfqd->queued++; -+ -+ elv_rb_add(&bfqq->sort_list, rq); -+ -+ /* -+ * Check if this request is a better next-serve candidate. -+ */ -+ prev = bfqq->next_rq; -+ next_rq = bfq_choose_req(bfqd, bfqq->next_rq, rq, bfqd->last_position); -+ BUG_ON(next_rq == NULL); -+ bfqq->next_rq = next_rq; -+ -+ /* -+ * Adjust priority tree position, if next_rq changes. -+ */ -+ if (prev != bfqq->next_rq) -+ bfq_rq_pos_tree_add(bfqd, bfqq); -+ -+ if (!bfq_bfqq_busy(bfqq)) { -+ bool soft_rt, coop_or_in_burst, -+ idle_for_long_time = time_is_before_jiffies( -+ bfqq->budget_timeout + -+ bfqd->bfq_wr_min_idle_time); -+ -+ if (bfq_bfqq_sync(bfqq)) { -+ bool already_in_burst = -+ !hlist_unhashed(&bfqq->burst_list_node) || -+ bfq_bfqq_in_large_burst(bfqq); -+ bfq_handle_burst(bfqd, bfqq, idle_for_long_time); -+ /* -+ * If bfqq was not already in the current burst, -+ * then, at this point, bfqq either has been -+ * added to the current burst or has caused the -+ * current burst to terminate. In particular, in -+ * the second case, bfqq has become the first -+ * queue in a possible new burst. -+ * In both cases last_ins_in_burst needs to be -+ * moved forward. -+ */ -+ if (!already_in_burst) -+ bfqd->last_ins_in_burst = jiffies; -+ } -+ -+ coop_or_in_burst = bfq_bfqq_in_large_burst(bfqq) || -+ bfq_bfqq_cooperations(bfqq) >= bfqd->bfq_coop_thresh; -+ soft_rt = bfqd->bfq_wr_max_softrt_rate > 0 && -+ !coop_or_in_burst && -+ time_is_before_jiffies(bfqq->soft_rt_next_start); -+ interactive = !coop_or_in_burst && idle_for_long_time; -+ entity->budget = max_t(unsigned long, bfqq->max_budget, -+ bfq_serv_to_charge(next_rq, bfqq)); -+ -+ if (!bfq_bfqq_IO_bound(bfqq)) { -+ if (time_before(jiffies, -+ RQ_BIC(rq)->ttime.last_end_request + -+ bfqd->bfq_slice_idle)) { -+ bfqq->requests_within_timer++; -+ if (bfqq->requests_within_timer >= -+ bfqd->bfq_requests_within_timer) -+ bfq_mark_bfqq_IO_bound(bfqq); -+ } else -+ bfqq->requests_within_timer = 0; -+ } -+ -+ if (!bfqd->low_latency) -+ goto add_bfqq_busy; -+ -+ if (bfq_bfqq_just_split(bfqq)) -+ goto set_ioprio_changed; -+ -+ /* -+ * If the queue: -+ * - is not being boosted, -+ * - has been idle for enough time, -+ * - is not a sync queue or is linked to a bfq_io_cq (it is -+ * shared "for its nature" or it is not shared and its -+ * requests have not been redirected to a shared queue) -+ * start a weight-raising period. -+ */ -+ if (old_wr_coeff == 1 && (interactive || soft_rt) && -+ (!bfq_bfqq_sync(bfqq) || bfqq->bic != NULL)) { -+ bfqq->wr_coeff = bfqd->bfq_wr_coeff; -+ if (interactive) -+ bfqq->wr_cur_max_time = bfq_wr_duration(bfqd); -+ else -+ bfqq->wr_cur_max_time = -+ bfqd->bfq_wr_rt_max_time; -+ bfq_log_bfqq(bfqd, bfqq, -+ "wrais starting at %lu, rais_max_time %u", -+ jiffies, -+ jiffies_to_msecs(bfqq->wr_cur_max_time)); -+ } else if (old_wr_coeff > 1) { -+ if (interactive) -+ bfqq->wr_cur_max_time = bfq_wr_duration(bfqd); -+ else if (coop_or_in_burst || -+ (bfqq->wr_cur_max_time == -+ bfqd->bfq_wr_rt_max_time && -+ !soft_rt)) { -+ bfqq->wr_coeff = 1; -+ bfq_log_bfqq(bfqd, bfqq, -+ "wrais ending at %lu, rais_max_time %u", -+ jiffies, -+ jiffies_to_msecs(bfqq-> -+ wr_cur_max_time)); -+ } else if (time_before( -+ bfqq->last_wr_start_finish + -+ bfqq->wr_cur_max_time, -+ jiffies + -+ bfqd->bfq_wr_rt_max_time) && -+ soft_rt) { -+ /* -+ * -+ * The remaining weight-raising time is lower -+ * than bfqd->bfq_wr_rt_max_time, which means -+ * that the application is enjoying weight -+ * raising either because deemed soft-rt in -+ * the near past, or because deemed interactive -+ * a long ago. -+ * In both cases, resetting now the current -+ * remaining weight-raising time for the -+ * application to the weight-raising duration -+ * for soft rt applications would not cause any -+ * latency increase for the application (as the -+ * new duration would be higher than the -+ * remaining time). -+ * -+ * In addition, the application is now meeting -+ * the requirements for being deemed soft rt. -+ * In the end we can correctly and safely -+ * (re)charge the weight-raising duration for -+ * the application with the weight-raising -+ * duration for soft rt applications. -+ * -+ * In particular, doing this recharge now, i.e., -+ * before the weight-raising period for the -+ * application finishes, reduces the probability -+ * of the following negative scenario: -+ * 1) the weight of a soft rt application is -+ * raised at startup (as for any newly -+ * created application), -+ * 2) since the application is not interactive, -+ * at a certain time weight-raising is -+ * stopped for the application, -+ * 3) at that time the application happens to -+ * still have pending requests, and hence -+ * is destined to not have a chance to be -+ * deemed soft rt before these requests are -+ * completed (see the comments to the -+ * function bfq_bfqq_softrt_next_start() -+ * for details on soft rt detection), -+ * 4) these pending requests experience a high -+ * latency because the application is not -+ * weight-raised while they are pending. -+ */ -+ bfqq->last_wr_start_finish = jiffies; -+ bfqq->wr_cur_max_time = -+ bfqd->bfq_wr_rt_max_time; -+ } -+ } -+set_ioprio_changed: -+ if (old_wr_coeff != bfqq->wr_coeff) -+ entity->ioprio_changed = 1; -+add_bfqq_busy: -+ bfqq->last_idle_bklogged = jiffies; -+ bfqq->service_from_backlogged = 0; -+ bfq_clear_bfqq_softrt_update(bfqq); -+ bfq_add_bfqq_busy(bfqd, bfqq); -+ } else { -+ if (bfqd->low_latency && old_wr_coeff == 1 && !rq_is_sync(rq) && -+ time_is_before_jiffies( -+ bfqq->last_wr_start_finish + -+ bfqd->bfq_wr_min_inter_arr_async)) { -+ bfqq->wr_coeff = bfqd->bfq_wr_coeff; -+ bfqq->wr_cur_max_time = bfq_wr_duration(bfqd); -+ -+ bfqd->wr_busy_queues++; -+ entity->ioprio_changed = 1; -+ bfq_log_bfqq(bfqd, bfqq, -+ "non-idle wrais starting at %lu, rais_max_time %u", -+ jiffies, -+ jiffies_to_msecs(bfqq->wr_cur_max_time)); -+ } -+ if (prev != bfqq->next_rq) -+ bfq_updated_next_req(bfqd, bfqq); -+ } -+ -+ if (bfqd->low_latency && -+ (old_wr_coeff == 1 || bfqq->wr_coeff == 1 || interactive)) -+ bfqq->last_wr_start_finish = jiffies; -+} -+ -+static struct request *bfq_find_rq_fmerge(struct bfq_data *bfqd, -+ struct bio *bio) -+{ -+ struct task_struct *tsk = current; -+ struct bfq_io_cq *bic; -+ struct bfq_queue *bfqq; -+ -+ bic = bfq_bic_lookup(bfqd, tsk->io_context); -+ if (bic == NULL) -+ return NULL; -+ -+ bfqq = bic_to_bfqq(bic, bfq_bio_sync(bio)); -+ if (bfqq != NULL) -+ return elv_rb_find(&bfqq->sort_list, bio_end_sector(bio)); -+ -+ return NULL; -+} -+ -+static void bfq_activate_request(struct request_queue *q, struct request *rq) -+{ -+ struct bfq_data *bfqd = q->elevator->elevator_data; -+ -+ bfqd->rq_in_driver++; -+ bfqd->last_position = blk_rq_pos(rq) + blk_rq_sectors(rq); -+ bfq_log(bfqd, "activate_request: new bfqd->last_position %llu", -+ (long long unsigned)bfqd->last_position); -+} -+ -+static inline void bfq_deactivate_request(struct request_queue *q, -+ struct request *rq) -+{ -+ struct bfq_data *bfqd = q->elevator->elevator_data; -+ -+ BUG_ON(bfqd->rq_in_driver == 0); -+ bfqd->rq_in_driver--; -+} -+ -+static void bfq_remove_request(struct request *rq) -+{ -+ struct bfq_queue *bfqq = RQ_BFQQ(rq); -+ struct bfq_data *bfqd = bfqq->bfqd; -+ const int sync = rq_is_sync(rq); -+ -+ if (bfqq->next_rq == rq) { -+ bfqq->next_rq = bfq_find_next_rq(bfqd, bfqq, rq); -+ bfq_updated_next_req(bfqd, bfqq); -+ } -+ -+ list_del_init(&rq->queuelist); -+ BUG_ON(bfqq->queued[sync] == 0); -+ bfqq->queued[sync]--; -+ bfqd->queued--; -+ elv_rb_del(&bfqq->sort_list, rq); -+ -+ if (RB_EMPTY_ROOT(&bfqq->sort_list)) { -+ if (bfq_bfqq_busy(bfqq) && bfqq != bfqd->in_service_queue) -+ bfq_del_bfqq_busy(bfqd, bfqq, 1); -+ /* -+ * Remove queue from request-position tree as it is empty. -+ */ -+ if (bfqq->pos_root != NULL) { -+ rb_erase(&bfqq->pos_node, bfqq->pos_root); -+ bfqq->pos_root = NULL; -+ } -+ } -+ -+ if (rq->cmd_flags & REQ_META) { -+ BUG_ON(bfqq->meta_pending == 0); -+ bfqq->meta_pending--; -+ } -+} -+ -+static int bfq_merge(struct request_queue *q, struct request **req, -+ struct bio *bio) -+{ -+ struct bfq_data *bfqd = q->elevator->elevator_data; -+ struct request *__rq; -+ -+ __rq = bfq_find_rq_fmerge(bfqd, bio); -+ if (__rq != NULL && elv_rq_merge_ok(__rq, bio)) { -+ *req = __rq; -+ return ELEVATOR_FRONT_MERGE; -+ } -+ -+ return ELEVATOR_NO_MERGE; -+} -+ -+static void bfq_merged_request(struct request_queue *q, struct request *req, -+ int type) -+{ -+ if (type == ELEVATOR_FRONT_MERGE && -+ rb_prev(&req->rb_node) && -+ blk_rq_pos(req) < -+ blk_rq_pos(container_of(rb_prev(&req->rb_node), -+ struct request, rb_node))) { -+ struct bfq_queue *bfqq = RQ_BFQQ(req); -+ struct bfq_data *bfqd = bfqq->bfqd; -+ struct request *prev, *next_rq; -+ -+ /* Reposition request in its sort_list */ -+ elv_rb_del(&bfqq->sort_list, req); -+ elv_rb_add(&bfqq->sort_list, req); -+ /* Choose next request to be served for bfqq */ -+ prev = bfqq->next_rq; -+ next_rq = bfq_choose_req(bfqd, bfqq->next_rq, req, -+ bfqd->last_position); -+ BUG_ON(next_rq == NULL); -+ bfqq->next_rq = next_rq; -+ /* -+ * If next_rq changes, update both the queue's budget to -+ * fit the new request and the queue's position in its -+ * rq_pos_tree. -+ */ -+ if (prev != bfqq->next_rq) { -+ bfq_updated_next_req(bfqd, bfqq); -+ bfq_rq_pos_tree_add(bfqd, bfqq); -+ } -+ } -+} -+ -+static void bfq_merged_requests(struct request_queue *q, struct request *rq, -+ struct request *next) -+{ -+ struct bfq_queue *bfqq = RQ_BFQQ(rq); -+ -+ /* -+ * Reposition in fifo if next is older than rq. -+ */ -+ if (!list_empty(&rq->queuelist) && !list_empty(&next->queuelist) && -+ time_before(next->fifo_time, rq->fifo_time)) { -+ list_move(&rq->queuelist, &next->queuelist); -+ rq->fifo_time = next->fifo_time; -+ } -+ -+ if (bfqq->next_rq == next) -+ bfqq->next_rq = rq; -+ -+ bfq_remove_request(next); -+} -+ -+/* Must be called with bfqq != NULL */ -+static inline void bfq_bfqq_end_wr(struct bfq_queue *bfqq) -+{ -+ BUG_ON(bfqq == NULL); -+ if (bfq_bfqq_busy(bfqq)) -+ bfqq->bfqd->wr_busy_queues--; -+ bfqq->wr_coeff = 1; -+ bfqq->wr_cur_max_time = 0; -+ /* Trigger a weight change on the next activation of the queue */ -+ bfqq->entity.ioprio_changed = 1; -+} -+ -+static void bfq_end_wr_async_queues(struct bfq_data *bfqd, -+ struct bfq_group *bfqg) -+{ -+ int i, j; -+ -+ for (i = 0; i < 2; i++) -+ for (j = 0; j < IOPRIO_BE_NR; j++) -+ if (bfqg->async_bfqq[i][j] != NULL) -+ bfq_bfqq_end_wr(bfqg->async_bfqq[i][j]); -+ if (bfqg->async_idle_bfqq != NULL) -+ bfq_bfqq_end_wr(bfqg->async_idle_bfqq); -+} -+ -+static void bfq_end_wr(struct bfq_data *bfqd) -+{ -+ struct bfq_queue *bfqq; -+ -+ spin_lock_irq(bfqd->queue->queue_lock); -+ -+ list_for_each_entry(bfqq, &bfqd->active_list, bfqq_list) -+ bfq_bfqq_end_wr(bfqq); -+ list_for_each_entry(bfqq, &bfqd->idle_list, bfqq_list) -+ bfq_bfqq_end_wr(bfqq); -+ bfq_end_wr_async(bfqd); -+ -+ spin_unlock_irq(bfqd->queue->queue_lock); -+} -+ -+static inline sector_t bfq_io_struct_pos(void *io_struct, bool request) -+{ -+ if (request) -+ return blk_rq_pos(io_struct); -+ else -+ return ((struct bio *)io_struct)->bi_iter.bi_sector; -+} -+ -+static inline sector_t bfq_dist_from(sector_t pos1, -+ sector_t pos2) -+{ -+ if (pos1 >= pos2) -+ return pos1 - pos2; -+ else -+ return pos2 - pos1; -+} -+ -+static inline int bfq_rq_close_to_sector(void *io_struct, bool request, -+ sector_t sector) -+{ -+ return bfq_dist_from(bfq_io_struct_pos(io_struct, request), sector) <= -+ BFQQ_SEEK_THR; -+} -+ -+static struct bfq_queue *bfqq_close(struct bfq_data *bfqd, sector_t sector) -+{ -+ struct rb_root *root = &bfqd->rq_pos_tree; -+ struct rb_node *parent, *node; -+ struct bfq_queue *__bfqq; -+ -+ if (RB_EMPTY_ROOT(root)) -+ return NULL; -+ -+ /* -+ * First, if we find a request starting at the end of the last -+ * request, choose it. -+ */ -+ __bfqq = bfq_rq_pos_tree_lookup(bfqd, root, sector, &parent, NULL); -+ if (__bfqq != NULL) -+ return __bfqq; -+ -+ /* -+ * If the exact sector wasn't found, the parent of the NULL leaf -+ * will contain the closest sector (rq_pos_tree sorted by -+ * next_request position). -+ */ -+ __bfqq = rb_entry(parent, struct bfq_queue, pos_node); -+ if (bfq_rq_close_to_sector(__bfqq->next_rq, true, sector)) -+ return __bfqq; -+ -+ if (blk_rq_pos(__bfqq->next_rq) < sector) -+ node = rb_next(&__bfqq->pos_node); -+ else -+ node = rb_prev(&__bfqq->pos_node); -+ if (node == NULL) -+ return NULL; -+ -+ __bfqq = rb_entry(node, struct bfq_queue, pos_node); -+ if (bfq_rq_close_to_sector(__bfqq->next_rq, true, sector)) -+ return __bfqq; -+ -+ return NULL; -+} -+ -+/* -+ * bfqd - obvious -+ * cur_bfqq - passed in so that we don't decide that the current queue -+ * is closely cooperating with itself -+ * sector - used as a reference point to search for a close queue -+ */ -+static struct bfq_queue *bfq_close_cooperator(struct bfq_data *bfqd, -+ struct bfq_queue *cur_bfqq, -+ sector_t sector) -+{ -+ struct bfq_queue *bfqq; -+ -+ if (bfq_class_idle(cur_bfqq)) -+ return NULL; -+ if (!bfq_bfqq_sync(cur_bfqq)) -+ return NULL; -+ if (BFQQ_SEEKY(cur_bfqq)) -+ return NULL; -+ -+ /* If device has only one backlogged bfq_queue, don't search. */ -+ if (bfqd->busy_queues == 1) -+ return NULL; -+ -+ /* -+ * We should notice if some of the queues are cooperating, e.g. -+ * working closely on the same area of the disk. In that case, -+ * we can group them together and don't waste time idling. -+ */ -+ bfqq = bfqq_close(bfqd, sector); -+ if (bfqq == NULL || bfqq == cur_bfqq) -+ return NULL; -+ -+ /* -+ * Do not merge queues from different bfq_groups. -+ */ -+ if (bfqq->entity.parent != cur_bfqq->entity.parent) -+ return NULL; -+ -+ /* -+ * It only makes sense to merge sync queues. -+ */ -+ if (!bfq_bfqq_sync(bfqq)) -+ return NULL; -+ if (BFQQ_SEEKY(bfqq)) -+ return NULL; -+ -+ /* -+ * Do not merge queues of different priority classes. -+ */ -+ if (bfq_class_rt(bfqq) != bfq_class_rt(cur_bfqq)) -+ return NULL; -+ -+ return bfqq; -+} -+ -+static struct bfq_queue * -+bfq_setup_merge(struct bfq_queue *bfqq, struct bfq_queue *new_bfqq) -+{ -+ int process_refs, new_process_refs; -+ struct bfq_queue *__bfqq; -+ -+ /* -+ * If there are no process references on the new_bfqq, then it is -+ * unsafe to follow the ->new_bfqq chain as other bfqq's in the chain -+ * may have dropped their last reference (not just their last process -+ * reference). -+ */ -+ if (!bfqq_process_refs(new_bfqq)) -+ return NULL; -+ -+ /* Avoid a circular list and skip interim queue merges. */ -+ while ((__bfqq = new_bfqq->new_bfqq)) { -+ if (__bfqq == bfqq) -+ return NULL; -+ new_bfqq = __bfqq; -+ } -+ -+ process_refs = bfqq_process_refs(bfqq); -+ new_process_refs = bfqq_process_refs(new_bfqq); -+ /* -+ * If the process for the bfqq has gone away, there is no -+ * sense in merging the queues. -+ */ -+ if (process_refs == 0 || new_process_refs == 0) -+ return NULL; -+ -+ bfq_log_bfqq(bfqq->bfqd, bfqq, "scheduling merge with queue %d", -+ new_bfqq->pid); -+ -+ /* -+ * Merging is just a redirection: the requests of the process -+ * owning one of the two queues are redirected to the other queue. -+ * The latter queue, in its turn, is set as shared if this is the -+ * first time that the requests of some process are redirected to -+ * it. -+ * -+ * We redirect bfqq to new_bfqq and not the opposite, because we -+ * are in the context of the process owning bfqq, hence we have -+ * the io_cq of this process. So we can immediately configure this -+ * io_cq to redirect the requests of the process to new_bfqq. -+ * -+ * NOTE, even if new_bfqq coincides with the in-service queue, the -+ * io_cq of new_bfqq is not available, because, if the in-service -+ * queue is shared, bfqd->in_service_bic may not point to the -+ * io_cq of the in-service queue. -+ * Redirecting the requests of the process owning bfqq to the -+ * currently in-service queue is in any case the best option, as -+ * we feed the in-service queue with new requests close to the -+ * last request served and, by doing so, hopefully increase the -+ * throughput. -+ */ -+ bfqq->new_bfqq = new_bfqq; -+ atomic_add(process_refs, &new_bfqq->ref); -+ return new_bfqq; -+} -+ -+/* -+ * Attempt to schedule a merge of bfqq with the currently in-service queue -+ * or with a close queue among the scheduled queues. -+ * Return NULL if no merge was scheduled, a pointer to the shared bfq_queue -+ * structure otherwise. -+ * -+ * The OOM queue is not allowed to participate to cooperation: in fact, since -+ * the requests temporarily redirected to the OOM queue could be redirected -+ * again to dedicated queues at any time, the state needed to correctly -+ * handle merging with the OOM queue would be quite complex and expensive -+ * to maintain. Besides, in such a critical condition as an out of memory, -+ * the benefits of queue merging may be little relevant, or even negligible. -+ */ -+static struct bfq_queue * -+bfq_setup_cooperator(struct bfq_data *bfqd, struct bfq_queue *bfqq, -+ void *io_struct, bool request) -+{ -+ struct bfq_queue *in_service_bfqq, *new_bfqq; -+ -+ if (bfqq->new_bfqq) -+ return bfqq->new_bfqq; -+ -+ if (!io_struct || unlikely(bfqq == &bfqd->oom_bfqq)) -+ return NULL; -+ -+ in_service_bfqq = bfqd->in_service_queue; -+ -+ if (in_service_bfqq == NULL || in_service_bfqq == bfqq || -+ !bfqd->in_service_bic || -+ unlikely(in_service_bfqq == &bfqd->oom_bfqq)) -+ goto check_scheduled; -+ -+ if (bfq_class_idle(in_service_bfqq) || bfq_class_idle(bfqq)) -+ goto check_scheduled; -+ -+ if (bfq_class_rt(in_service_bfqq) != bfq_class_rt(bfqq)) -+ goto check_scheduled; -+ -+ if (in_service_bfqq->entity.parent != bfqq->entity.parent) -+ goto check_scheduled; -+ -+ if (bfq_rq_close_to_sector(io_struct, request, bfqd->last_position) && -+ bfq_bfqq_sync(in_service_bfqq) && bfq_bfqq_sync(bfqq)) { -+ new_bfqq = bfq_setup_merge(bfqq, in_service_bfqq); -+ if (new_bfqq != NULL) -+ return new_bfqq; /* Merge with in-service queue */ -+ } -+ -+ /* -+ * Check whether there is a cooperator among currently scheduled -+ * queues. The only thing we need is that the bio/request is not -+ * NULL, as we need it to establish whether a cooperator exists. -+ */ -+check_scheduled: -+ new_bfqq = bfq_close_cooperator(bfqd, bfqq, -+ bfq_io_struct_pos(io_struct, request)); -+ if (new_bfqq && likely(new_bfqq != &bfqd->oom_bfqq)) -+ return bfq_setup_merge(bfqq, new_bfqq); -+ -+ return NULL; -+} -+ -+static inline void -+bfq_bfqq_save_state(struct bfq_queue *bfqq) -+{ -+ /* -+ * If bfqq->bic == NULL, the queue is already shared or its requests -+ * have already been redirected to a shared queue; both idle window -+ * and weight raising state have already been saved. Do nothing. -+ */ -+ if (bfqq->bic == NULL) -+ return; -+ if (bfqq->bic->wr_time_left) -+ /* -+ * This is the queue of a just-started process, and would -+ * deserve weight raising: we set wr_time_left to the full -+ * weight-raising duration to trigger weight-raising when -+ * and if the queue is split and the first request of the -+ * queue is enqueued. -+ */ -+ bfqq->bic->wr_time_left = bfq_wr_duration(bfqq->bfqd); -+ else if (bfqq->wr_coeff > 1) { -+ unsigned long wr_duration = -+ jiffies - bfqq->last_wr_start_finish; -+ /* -+ * It may happen that a queue's weight raising period lasts -+ * longer than its wr_cur_max_time, as weight raising is -+ * handled only when a request is enqueued or dispatched (it -+ * does not use any timer). If the weight raising period is -+ * about to end, don't save it. -+ */ -+ if (bfqq->wr_cur_max_time <= wr_duration) -+ bfqq->bic->wr_time_left = 0; -+ else -+ bfqq->bic->wr_time_left = -+ bfqq->wr_cur_max_time - wr_duration; -+ /* -+ * The bfq_queue is becoming shared or the requests of the -+ * process owning the queue are being redirected to a shared -+ * queue. Stop the weight raising period of the queue, as in -+ * both cases it should not be owned by an interactive or -+ * soft real-time application. -+ */ -+ bfq_bfqq_end_wr(bfqq); -+ } else -+ bfqq->bic->wr_time_left = 0; -+ bfqq->bic->saved_idle_window = bfq_bfqq_idle_window(bfqq); -+ bfqq->bic->saved_IO_bound = bfq_bfqq_IO_bound(bfqq); -+ bfqq->bic->saved_in_large_burst = bfq_bfqq_in_large_burst(bfqq); -+ bfqq->bic->was_in_burst_list = !hlist_unhashed(&bfqq->burst_list_node); -+ bfqq->bic->cooperations++; -+ bfqq->bic->failed_cooperations = 0; -+} -+ -+static inline void -+bfq_get_bic_reference(struct bfq_queue *bfqq) -+{ -+ /* -+ * If bfqq->bic has a non-NULL value, the bic to which it belongs -+ * is about to begin using a shared bfq_queue. -+ */ -+ if (bfqq->bic) -+ atomic_long_inc(&bfqq->bic->icq.ioc->refcount); -+} -+ -+static void -+bfq_merge_bfqqs(struct bfq_data *bfqd, struct bfq_io_cq *bic, -+ struct bfq_queue *bfqq, struct bfq_queue *new_bfqq) -+{ -+ bfq_log_bfqq(bfqd, bfqq, "merging with queue %lu", -+ (long unsigned)new_bfqq->pid); -+ /* Save weight raising and idle window of the merged queues */ -+ bfq_bfqq_save_state(bfqq); -+ bfq_bfqq_save_state(new_bfqq); -+ if (bfq_bfqq_IO_bound(bfqq)) -+ bfq_mark_bfqq_IO_bound(new_bfqq); -+ bfq_clear_bfqq_IO_bound(bfqq); -+ /* -+ * Grab a reference to the bic, to prevent it from being destroyed -+ * before being possibly touched by a bfq_split_bfqq(). -+ */ -+ bfq_get_bic_reference(bfqq); -+ bfq_get_bic_reference(new_bfqq); -+ /* -+ * Merge queues (that is, let bic redirect its requests to new_bfqq) -+ */ -+ bic_set_bfqq(bic, new_bfqq, 1); -+ bfq_mark_bfqq_coop(new_bfqq); -+ /* -+ * new_bfqq now belongs to at least two bics (it is a shared queue): -+ * set new_bfqq->bic to NULL. bfqq either: -+ * - does not belong to any bic any more, and hence bfqq->bic must -+ * be set to NULL, or -+ * - is a queue whose owning bics have already been redirected to a -+ * different queue, hence the queue is destined to not belong to -+ * any bic soon and bfqq->bic is already NULL (therefore the next -+ * assignment causes no harm). -+ */ -+ new_bfqq->bic = NULL; -+ bfqq->bic = NULL; -+ bfq_put_queue(bfqq); -+} -+ -+static inline void bfq_bfqq_increase_failed_cooperations(struct bfq_queue *bfqq) -+{ -+ struct bfq_io_cq *bic = bfqq->bic; -+ struct bfq_data *bfqd = bfqq->bfqd; -+ -+ if (bic && bfq_bfqq_cooperations(bfqq) >= bfqd->bfq_coop_thresh) { -+ bic->failed_cooperations++; -+ if (bic->failed_cooperations >= bfqd->bfq_failed_cooperations) -+ bic->cooperations = 0; -+ } -+} -+ -+static int bfq_allow_merge(struct request_queue *q, struct request *rq, -+ struct bio *bio) -+{ -+ struct bfq_data *bfqd = q->elevator->elevator_data; -+ struct bfq_io_cq *bic; -+ struct bfq_queue *bfqq, *new_bfqq; -+ -+ /* -+ * Disallow merge of a sync bio into an async request. -+ */ -+ if (bfq_bio_sync(bio) && !rq_is_sync(rq)) -+ return 0; -+ -+ /* -+ * Lookup the bfqq that this bio will be queued with. Allow -+ * merge only if rq is queued there. -+ * Queue lock is held here. -+ */ -+ bic = bfq_bic_lookup(bfqd, current->io_context); -+ if (bic == NULL) -+ return 0; -+ -+ bfqq = bic_to_bfqq(bic, bfq_bio_sync(bio)); -+ /* -+ * We take advantage of this function to perform an early merge -+ * of the queues of possible cooperating processes. -+ */ -+ if (bfqq != NULL) { -+ new_bfqq = bfq_setup_cooperator(bfqd, bfqq, bio, false); -+ if (new_bfqq != NULL) { -+ bfq_merge_bfqqs(bfqd, bic, bfqq, new_bfqq); -+ /* -+ * If we get here, the bio will be queued in the -+ * shared queue, i.e., new_bfqq, so use new_bfqq -+ * to decide whether bio and rq can be merged. -+ */ -+ bfqq = new_bfqq; -+ } else -+ bfq_bfqq_increase_failed_cooperations(bfqq); -+ } -+ -+ return bfqq == RQ_BFQQ(rq); -+} -+ -+static void __bfq_set_in_service_queue(struct bfq_data *bfqd, -+ struct bfq_queue *bfqq) -+{ -+ if (bfqq != NULL) { -+ bfq_mark_bfqq_must_alloc(bfqq); -+ bfq_mark_bfqq_budget_new(bfqq); -+ bfq_clear_bfqq_fifo_expire(bfqq); -+ -+ bfqd->budgets_assigned = (bfqd->budgets_assigned*7 + 256) / 8; -+ -+ bfq_log_bfqq(bfqd, bfqq, -+ "set_in_service_queue, cur-budget = %lu", -+ bfqq->entity.budget); -+ } -+ -+ bfqd->in_service_queue = bfqq; -+} -+ -+/* -+ * Get and set a new queue for service. -+ */ -+static struct bfq_queue *bfq_set_in_service_queue(struct bfq_data *bfqd) -+{ -+ struct bfq_queue *bfqq = bfq_get_next_queue(bfqd); -+ -+ __bfq_set_in_service_queue(bfqd, bfqq); -+ return bfqq; -+} -+ -+/* -+ * If enough samples have been computed, return the current max budget -+ * stored in bfqd, which is dynamically updated according to the -+ * estimated disk peak rate; otherwise return the default max budget -+ */ -+static inline unsigned long bfq_max_budget(struct bfq_data *bfqd) -+{ -+ if (bfqd->budgets_assigned < 194) -+ return bfq_default_max_budget; -+ else -+ return bfqd->bfq_max_budget; -+} -+ -+/* -+ * Return min budget, which is a fraction of the current or default -+ * max budget (trying with 1/32) -+ */ -+static inline unsigned long bfq_min_budget(struct bfq_data *bfqd) -+{ -+ if (bfqd->budgets_assigned < 194) -+ return bfq_default_max_budget / 32; -+ else -+ return bfqd->bfq_max_budget / 32; -+} -+ -+static void bfq_arm_slice_timer(struct bfq_data *bfqd) -+{ -+ struct bfq_queue *bfqq = bfqd->in_service_queue; -+ struct bfq_io_cq *bic; -+ unsigned long sl; -+ -+ BUG_ON(!RB_EMPTY_ROOT(&bfqq->sort_list)); -+ -+ /* Processes have exited, don't wait. */ -+ bic = bfqd->in_service_bic; -+ if (bic == NULL || atomic_read(&bic->icq.ioc->active_ref) == 0) -+ return; -+ -+ bfq_mark_bfqq_wait_request(bfqq); -+ -+ /* -+ * We don't want to idle for seeks, but we do want to allow -+ * fair distribution of slice time for a process doing back-to-back -+ * seeks. So allow a little bit of time for him to submit a new rq. -+ * -+ * To prevent processes with (partly) seeky workloads from -+ * being too ill-treated, grant them a small fraction of the -+ * assigned budget before reducing the waiting time to -+ * BFQ_MIN_TT. This happened to help reduce latency. -+ */ -+ sl = bfqd->bfq_slice_idle; -+ /* -+ * Unless the queue is being weight-raised, grant only minimum idle -+ * time if the queue either has been seeky for long enough or has -+ * already proved to be constantly seeky. -+ */ -+ if (bfq_sample_valid(bfqq->seek_samples) && -+ ((BFQQ_SEEKY(bfqq) && bfqq->entity.service > -+ bfq_max_budget(bfqq->bfqd) / 8) || -+ bfq_bfqq_constantly_seeky(bfqq)) && bfqq->wr_coeff == 1) -+ sl = min(sl, msecs_to_jiffies(BFQ_MIN_TT)); -+ else if (bfqq->wr_coeff > 1) -+ sl = sl * 3; -+ bfqd->last_idling_start = ktime_get(); -+ mod_timer(&bfqd->idle_slice_timer, jiffies + sl); -+ bfq_log(bfqd, "arm idle: %u/%u ms", -+ jiffies_to_msecs(sl), jiffies_to_msecs(bfqd->bfq_slice_idle)); -+} -+ -+/* -+ * Set the maximum time for the in-service queue to consume its -+ * budget. This prevents seeky processes from lowering the disk -+ * throughput (always guaranteed with a time slice scheme as in CFQ). -+ */ -+static void bfq_set_budget_timeout(struct bfq_data *bfqd) -+{ -+ struct bfq_queue *bfqq = bfqd->in_service_queue; -+ unsigned int timeout_coeff; -+ if (bfqq->wr_cur_max_time == bfqd->bfq_wr_rt_max_time) -+ timeout_coeff = 1; -+ else -+ timeout_coeff = bfqq->entity.weight / bfqq->entity.orig_weight; -+ -+ bfqd->last_budget_start = ktime_get(); -+ -+ bfq_clear_bfqq_budget_new(bfqq); -+ bfqq->budget_timeout = jiffies + -+ bfqd->bfq_timeout[bfq_bfqq_sync(bfqq)] * timeout_coeff; -+ -+ bfq_log_bfqq(bfqd, bfqq, "set budget_timeout %u", -+ jiffies_to_msecs(bfqd->bfq_timeout[bfq_bfqq_sync(bfqq)] * -+ timeout_coeff)); -+} -+ -+/* -+ * Move request from internal lists to the request queue dispatch list. -+ */ -+static void bfq_dispatch_insert(struct request_queue *q, struct request *rq) -+{ -+ struct bfq_data *bfqd = q->elevator->elevator_data; -+ struct bfq_queue *bfqq = RQ_BFQQ(rq); -+ -+ /* -+ * For consistency, the next instruction should have been executed -+ * after removing the request from the queue and dispatching it. -+ * We execute instead this instruction before bfq_remove_request() -+ * (and hence introduce a temporary inconsistency), for efficiency. -+ * In fact, in a forced_dispatch, this prevents two counters related -+ * to bfqq->dispatched to risk to be uselessly decremented if bfqq -+ * is not in service, and then to be incremented again after -+ * incrementing bfqq->dispatched. -+ */ -+ bfqq->dispatched++; -+ bfq_remove_request(rq); -+ elv_dispatch_sort(q, rq); -+ -+ if (bfq_bfqq_sync(bfqq)) -+ bfqd->sync_flight++; -+} -+ -+/* -+ * Return expired entry, or NULL to just start from scratch in rbtree. -+ */ -+static struct request *bfq_check_fifo(struct bfq_queue *bfqq) -+{ -+ struct request *rq = NULL; -+ -+ if (bfq_bfqq_fifo_expire(bfqq)) -+ return NULL; -+ -+ bfq_mark_bfqq_fifo_expire(bfqq); -+ -+ if (list_empty(&bfqq->fifo)) -+ return NULL; -+ -+ rq = rq_entry_fifo(bfqq->fifo.next); -+ -+ if (time_before(jiffies, rq->fifo_time)) -+ return NULL; -+ -+ return rq; -+} -+ -+static inline unsigned long bfq_bfqq_budget_left(struct bfq_queue *bfqq) -+{ -+ struct bfq_entity *entity = &bfqq->entity; -+ return entity->budget - entity->service; -+} -+ -+static void __bfq_bfqq_expire(struct bfq_data *bfqd, struct bfq_queue *bfqq) -+{ -+ BUG_ON(bfqq != bfqd->in_service_queue); -+ -+ __bfq_bfqd_reset_in_service(bfqd); -+ -+ /* -+ * If this bfqq is shared between multiple processes, check -+ * to make sure that those processes are still issuing I/Os -+ * within the mean seek distance. If not, it may be time to -+ * break the queues apart again. -+ */ -+ if (bfq_bfqq_coop(bfqq) && BFQQ_SEEKY(bfqq)) -+ bfq_mark_bfqq_split_coop(bfqq); -+ -+ if (RB_EMPTY_ROOT(&bfqq->sort_list)) { -+ /* -+ * Overloading budget_timeout field to store the time -+ * at which the queue remains with no backlog; used by -+ * the weight-raising mechanism. -+ */ -+ bfqq->budget_timeout = jiffies; -+ bfq_del_bfqq_busy(bfqd, bfqq, 1); -+ } else { -+ bfq_activate_bfqq(bfqd, bfqq); -+ /* -+ * Resort priority tree of potential close cooperators. -+ */ -+ bfq_rq_pos_tree_add(bfqd, bfqq); -+ } -+} -+ -+/** -+ * __bfq_bfqq_recalc_budget - try to adapt the budget to the @bfqq behavior. -+ * @bfqd: device data. -+ * @bfqq: queue to update. -+ * @reason: reason for expiration. -+ * -+ * Handle the feedback on @bfqq budget. See the body for detailed -+ * comments. -+ */ -+static void __bfq_bfqq_recalc_budget(struct bfq_data *bfqd, -+ struct bfq_queue *bfqq, -+ enum bfqq_expiration reason) -+{ -+ struct request *next_rq; -+ unsigned long budget, min_budget; -+ -+ budget = bfqq->max_budget; -+ min_budget = bfq_min_budget(bfqd); -+ -+ BUG_ON(bfqq != bfqd->in_service_queue); -+ -+ bfq_log_bfqq(bfqd, bfqq, "recalc_budg: last budg %lu, budg left %lu", -+ bfqq->entity.budget, bfq_bfqq_budget_left(bfqq)); -+ bfq_log_bfqq(bfqd, bfqq, "recalc_budg: last max_budg %lu, min budg %lu", -+ budget, bfq_min_budget(bfqd)); -+ bfq_log_bfqq(bfqd, bfqq, "recalc_budg: sync %d, seeky %d", -+ bfq_bfqq_sync(bfqq), BFQQ_SEEKY(bfqd->in_service_queue)); -+ -+ if (bfq_bfqq_sync(bfqq)) { -+ switch (reason) { -+ /* -+ * Caveat: in all the following cases we trade latency -+ * for throughput. -+ */ -+ case BFQ_BFQQ_TOO_IDLE: -+ /* -+ * This is the only case where we may reduce -+ * the budget: if there is no request of the -+ * process still waiting for completion, then -+ * we assume (tentatively) that the timer has -+ * expired because the batch of requests of -+ * the process could have been served with a -+ * smaller budget. Hence, betting that -+ * process will behave in the same way when it -+ * becomes backlogged again, we reduce its -+ * next budget. As long as we guess right, -+ * this budget cut reduces the latency -+ * experienced by the process. -+ * -+ * However, if there are still outstanding -+ * requests, then the process may have not yet -+ * issued its next request just because it is -+ * still waiting for the completion of some of -+ * the still outstanding ones. So in this -+ * subcase we do not reduce its budget, on the -+ * contrary we increase it to possibly boost -+ * the throughput, as discussed in the -+ * comments to the BUDGET_TIMEOUT case. -+ */ -+ if (bfqq->dispatched > 0) /* still outstanding reqs */ -+ budget = min(budget * 2, bfqd->bfq_max_budget); -+ else { -+ if (budget > 5 * min_budget) -+ budget -= 4 * min_budget; -+ else -+ budget = min_budget; -+ } -+ break; -+ case BFQ_BFQQ_BUDGET_TIMEOUT: -+ /* -+ * We double the budget here because: 1) it -+ * gives the chance to boost the throughput if -+ * this is not a seeky process (which may have -+ * bumped into this timeout because of, e.g., -+ * ZBR), 2) together with charge_full_budget -+ * it helps give seeky processes higher -+ * timestamps, and hence be served less -+ * frequently. -+ */ -+ budget = min(budget * 2, bfqd->bfq_max_budget); -+ break; -+ case BFQ_BFQQ_BUDGET_EXHAUSTED: -+ /* -+ * The process still has backlog, and did not -+ * let either the budget timeout or the disk -+ * idling timeout expire. Hence it is not -+ * seeky, has a short thinktime and may be -+ * happy with a higher budget too. So -+ * definitely increase the budget of this good -+ * candidate to boost the disk throughput. -+ */ -+ budget = min(budget * 4, bfqd->bfq_max_budget); -+ break; -+ case BFQ_BFQQ_NO_MORE_REQUESTS: -+ /* -+ * Leave the budget unchanged. -+ */ -+ default: -+ return; -+ } -+ } else /* async queue */ -+ /* async queues get always the maximum possible budget -+ * (their ability to dispatch is limited by -+ * @bfqd->bfq_max_budget_async_rq). -+ */ -+ budget = bfqd->bfq_max_budget; -+ -+ bfqq->max_budget = budget; -+ -+ if (bfqd->budgets_assigned >= 194 && bfqd->bfq_user_max_budget == 0 && -+ bfqq->max_budget > bfqd->bfq_max_budget) -+ bfqq->max_budget = bfqd->bfq_max_budget; -+ -+ /* -+ * Make sure that we have enough budget for the next request. -+ * Since the finish time of the bfqq must be kept in sync with -+ * the budget, be sure to call __bfq_bfqq_expire() after the -+ * update. -+ */ -+ next_rq = bfqq->next_rq; -+ if (next_rq != NULL) -+ bfqq->entity.budget = max_t(unsigned long, bfqq->max_budget, -+ bfq_serv_to_charge(next_rq, bfqq)); -+ else -+ bfqq->entity.budget = bfqq->max_budget; -+ -+ bfq_log_bfqq(bfqd, bfqq, "head sect: %u, new budget %lu", -+ next_rq != NULL ? blk_rq_sectors(next_rq) : 0, -+ bfqq->entity.budget); -+} -+ -+static unsigned long bfq_calc_max_budget(u64 peak_rate, u64 timeout) -+{ -+ unsigned long max_budget; -+ -+ /* -+ * The max_budget calculated when autotuning is equal to the -+ * amount of sectors transfered in timeout_sync at the -+ * estimated peak rate. -+ */ -+ max_budget = (unsigned long)(peak_rate * 1000 * -+ timeout >> BFQ_RATE_SHIFT); -+ -+ return max_budget; -+} -+ -+/* -+ * In addition to updating the peak rate, checks whether the process -+ * is "slow", and returns 1 if so. This slow flag is used, in addition -+ * to the budget timeout, to reduce the amount of service provided to -+ * seeky processes, and hence reduce their chances to lower the -+ * throughput. See the code for more details. -+ */ -+static int bfq_update_peak_rate(struct bfq_data *bfqd, struct bfq_queue *bfqq, -+ int compensate, enum bfqq_expiration reason) -+{ -+ u64 bw, usecs, expected, timeout; -+ ktime_t delta; -+ int update = 0; -+ -+ if (!bfq_bfqq_sync(bfqq) || bfq_bfqq_budget_new(bfqq)) -+ return 0; -+ -+ if (compensate) -+ delta = bfqd->last_idling_start; -+ else -+ delta = ktime_get(); -+ delta = ktime_sub(delta, bfqd->last_budget_start); -+ usecs = ktime_to_us(delta); -+ -+ /* Don't trust short/unrealistic values. */ -+ if (usecs < 100 || usecs >= LONG_MAX) -+ return 0; -+ -+ /* -+ * Calculate the bandwidth for the last slice. We use a 64 bit -+ * value to store the peak rate, in sectors per usec in fixed -+ * point math. We do so to have enough precision in the estimate -+ * and to avoid overflows. -+ */ -+ bw = (u64)bfqq->entity.service << BFQ_RATE_SHIFT; -+ do_div(bw, (unsigned long)usecs); -+ -+ timeout = jiffies_to_msecs(bfqd->bfq_timeout[BLK_RW_SYNC]); -+ -+ /* -+ * Use only long (> 20ms) intervals to filter out spikes for -+ * the peak rate estimation. -+ */ -+ if (usecs > 20000) { -+ if (bw > bfqd->peak_rate || -+ (!BFQQ_SEEKY(bfqq) && -+ reason == BFQ_BFQQ_BUDGET_TIMEOUT)) { -+ bfq_log(bfqd, "measured bw =%llu", bw); -+ /* -+ * To smooth oscillations use a low-pass filter with -+ * alpha=7/8, i.e., -+ * new_rate = (7/8) * old_rate + (1/8) * bw -+ */ -+ do_div(bw, 8); -+ if (bw == 0) -+ return 0; -+ bfqd->peak_rate *= 7; -+ do_div(bfqd->peak_rate, 8); -+ bfqd->peak_rate += bw; -+ update = 1; -+ bfq_log(bfqd, "new peak_rate=%llu", bfqd->peak_rate); -+ } -+ -+ update |= bfqd->peak_rate_samples == BFQ_PEAK_RATE_SAMPLES - 1; -+ -+ if (bfqd->peak_rate_samples < BFQ_PEAK_RATE_SAMPLES) -+ bfqd->peak_rate_samples++; -+ -+ if (bfqd->peak_rate_samples == BFQ_PEAK_RATE_SAMPLES && -+ update) { -+ int dev_type = blk_queue_nonrot(bfqd->queue); -+ if (bfqd->bfq_user_max_budget == 0) { -+ bfqd->bfq_max_budget = -+ bfq_calc_max_budget(bfqd->peak_rate, -+ timeout); -+ bfq_log(bfqd, "new max_budget=%lu", -+ bfqd->bfq_max_budget); -+ } -+ if (bfqd->device_speed == BFQ_BFQD_FAST && -+ bfqd->peak_rate < device_speed_thresh[dev_type]) { -+ bfqd->device_speed = BFQ_BFQD_SLOW; -+ bfqd->RT_prod = R_slow[dev_type] * -+ T_slow[dev_type]; -+ } else if (bfqd->device_speed == BFQ_BFQD_SLOW && -+ bfqd->peak_rate > device_speed_thresh[dev_type]) { -+ bfqd->device_speed = BFQ_BFQD_FAST; -+ bfqd->RT_prod = R_fast[dev_type] * -+ T_fast[dev_type]; -+ } -+ } -+ } -+ -+ /* -+ * If the process has been served for a too short time -+ * interval to let its possible sequential accesses prevail on -+ * the initial seek time needed to move the disk head on the -+ * first sector it requested, then give the process a chance -+ * and for the moment return false. -+ */ -+ if (bfqq->entity.budget <= bfq_max_budget(bfqd) / 8) -+ return 0; -+ -+ /* -+ * A process is considered ``slow'' (i.e., seeky, so that we -+ * cannot treat it fairly in the service domain, as it would -+ * slow down too much the other processes) if, when a slice -+ * ends for whatever reason, it has received service at a -+ * rate that would not be high enough to complete the budget -+ * before the budget timeout expiration. -+ */ -+ expected = bw * 1000 * timeout >> BFQ_RATE_SHIFT; -+ -+ /* -+ * Caveat: processes doing IO in the slower disk zones will -+ * tend to be slow(er) even if not seeky. And the estimated -+ * peak rate will actually be an average over the disk -+ * surface. Hence, to not be too harsh with unlucky processes, -+ * we keep a budget/3 margin of safety before declaring a -+ * process slow. -+ */ -+ return expected > (4 * bfqq->entity.budget) / 3; -+} -+ -+/* -+ * To be deemed as soft real-time, an application must meet two -+ * requirements. First, the application must not require an average -+ * bandwidth higher than the approximate bandwidth required to playback or -+ * record a compressed high-definition video. -+ * The next function is invoked on the completion of the last request of a -+ * batch, to compute the next-start time instant, soft_rt_next_start, such -+ * that, if the next request of the application does not arrive before -+ * soft_rt_next_start, then the above requirement on the bandwidth is met. -+ * -+ * The second requirement is that the request pattern of the application is -+ * isochronous, i.e., that, after issuing a request or a batch of requests, -+ * the application stops issuing new requests until all its pending requests -+ * have been completed. After that, the application may issue a new batch, -+ * and so on. -+ * For this reason the next function is invoked to compute -+ * soft_rt_next_start only for applications that meet this requirement, -+ * whereas soft_rt_next_start is set to infinity for applications that do -+ * not. -+ * -+ * Unfortunately, even a greedy application may happen to behave in an -+ * isochronous way if the CPU load is high. In fact, the application may -+ * stop issuing requests while the CPUs are busy serving other processes, -+ * then restart, then stop again for a while, and so on. In addition, if -+ * the disk achieves a low enough throughput with the request pattern -+ * issued by the application (e.g., because the request pattern is random -+ * and/or the device is slow), then the application may meet the above -+ * bandwidth requirement too. To prevent such a greedy application to be -+ * deemed as soft real-time, a further rule is used in the computation of -+ * soft_rt_next_start: soft_rt_next_start must be higher than the current -+ * time plus the maximum time for which the arrival of a request is waited -+ * for when a sync queue becomes idle, namely bfqd->bfq_slice_idle. -+ * This filters out greedy applications, as the latter issue instead their -+ * next request as soon as possible after the last one has been completed -+ * (in contrast, when a batch of requests is completed, a soft real-time -+ * application spends some time processing data). -+ * -+ * Unfortunately, the last filter may easily generate false positives if -+ * only bfqd->bfq_slice_idle is used as a reference time interval and one -+ * or both the following cases occur: -+ * 1) HZ is so low that the duration of a jiffy is comparable to or higher -+ * than bfqd->bfq_slice_idle. This happens, e.g., on slow devices with -+ * HZ=100. -+ * 2) jiffies, instead of increasing at a constant rate, may stop increasing -+ * for a while, then suddenly 'jump' by several units to recover the lost -+ * increments. This seems to happen, e.g., inside virtual machines. -+ * To address this issue, we do not use as a reference time interval just -+ * bfqd->bfq_slice_idle, but bfqd->bfq_slice_idle plus a few jiffies. In -+ * particular we add the minimum number of jiffies for which the filter -+ * seems to be quite precise also in embedded systems and KVM/QEMU virtual -+ * machines. -+ */ -+static inline unsigned long bfq_bfqq_softrt_next_start(struct bfq_data *bfqd, -+ struct bfq_queue *bfqq) -+{ -+ return max(bfqq->last_idle_bklogged + -+ HZ * bfqq->service_from_backlogged / -+ bfqd->bfq_wr_max_softrt_rate, -+ jiffies + bfqq->bfqd->bfq_slice_idle + 4); -+} -+ -+/* -+ * Return the largest-possible time instant such that, for as long as possible, -+ * the current time will be lower than this time instant according to the macro -+ * time_is_before_jiffies(). -+ */ -+static inline unsigned long bfq_infinity_from_now(unsigned long now) -+{ -+ return now + ULONG_MAX / 2; -+} -+ -+/** -+ * bfq_bfqq_expire - expire a queue. -+ * @bfqd: device owning the queue. -+ * @bfqq: the queue to expire. -+ * @compensate: if true, compensate for the time spent idling. -+ * @reason: the reason causing the expiration. -+ * -+ * -+ * If the process associated to the queue is slow (i.e., seeky), or in -+ * case of budget timeout, or, finally, if it is async, we -+ * artificially charge it an entire budget (independently of the -+ * actual service it received). As a consequence, the queue will get -+ * higher timestamps than the correct ones upon reactivation, and -+ * hence it will be rescheduled as if it had received more service -+ * than what it actually received. In the end, this class of processes -+ * will receive less service in proportion to how slowly they consume -+ * their budgets (and hence how seriously they tend to lower the -+ * throughput). -+ * -+ * In contrast, when a queue expires because it has been idling for -+ * too much or because it exhausted its budget, we do not touch the -+ * amount of service it has received. Hence when the queue will be -+ * reactivated and its timestamps updated, the latter will be in sync -+ * with the actual service received by the queue until expiration. -+ * -+ * Charging a full budget to the first type of queues and the exact -+ * service to the others has the effect of using the WF2Q+ policy to -+ * schedule the former on a timeslice basis, without violating the -+ * service domain guarantees of the latter. -+ */ -+static void bfq_bfqq_expire(struct bfq_data *bfqd, -+ struct bfq_queue *bfqq, -+ int compensate, -+ enum bfqq_expiration reason) -+{ -+ int slow; -+ BUG_ON(bfqq != bfqd->in_service_queue); -+ -+ /* Update disk peak rate for autotuning and check whether the -+ * process is slow (see bfq_update_peak_rate). -+ */ -+ slow = bfq_update_peak_rate(bfqd, bfqq, compensate, reason); -+ -+ /* -+ * As above explained, 'punish' slow (i.e., seeky), timed-out -+ * and async queues, to favor sequential sync workloads. -+ * -+ * Processes doing I/O in the slower disk zones will tend to be -+ * slow(er) even if not seeky. Hence, since the estimated peak -+ * rate is actually an average over the disk surface, these -+ * processes may timeout just for bad luck. To avoid punishing -+ * them we do not charge a full budget to a process that -+ * succeeded in consuming at least 2/3 of its budget. -+ */ -+ if (slow || (reason == BFQ_BFQQ_BUDGET_TIMEOUT && -+ bfq_bfqq_budget_left(bfqq) >= bfqq->entity.budget / 3)) -+ bfq_bfqq_charge_full_budget(bfqq); -+ -+ bfqq->service_from_backlogged += bfqq->entity.service; -+ -+ if (BFQQ_SEEKY(bfqq) && reason == BFQ_BFQQ_BUDGET_TIMEOUT && -+ !bfq_bfqq_constantly_seeky(bfqq)) { -+ bfq_mark_bfqq_constantly_seeky(bfqq); -+ if (!blk_queue_nonrot(bfqd->queue)) -+ bfqd->const_seeky_busy_in_flight_queues++; -+ } -+ -+ if (reason == BFQ_BFQQ_TOO_IDLE && -+ bfqq->entity.service <= 2 * bfqq->entity.budget / 10 ) -+ bfq_clear_bfqq_IO_bound(bfqq); -+ -+ if (bfqd->low_latency && bfqq->wr_coeff == 1) -+ bfqq->last_wr_start_finish = jiffies; -+ -+ if (bfqd->low_latency && bfqd->bfq_wr_max_softrt_rate > 0 && -+ RB_EMPTY_ROOT(&bfqq->sort_list)) { -+ /* -+ * If we get here, and there are no outstanding requests, -+ * then the request pattern is isochronous (see the comments -+ * to the function bfq_bfqq_softrt_next_start()). Hence we -+ * can compute soft_rt_next_start. If, instead, the queue -+ * still has outstanding requests, then we have to wait -+ * for the completion of all the outstanding requests to -+ * discover whether the request pattern is actually -+ * isochronous. -+ */ -+ if (bfqq->dispatched == 0) -+ bfqq->soft_rt_next_start = -+ bfq_bfqq_softrt_next_start(bfqd, bfqq); -+ else { -+ /* -+ * The application is still waiting for the -+ * completion of one or more requests: -+ * prevent it from possibly being incorrectly -+ * deemed as soft real-time by setting its -+ * soft_rt_next_start to infinity. In fact, -+ * without this assignment, the application -+ * would be incorrectly deemed as soft -+ * real-time if: -+ * 1) it issued a new request before the -+ * completion of all its in-flight -+ * requests, and -+ * 2) at that time, its soft_rt_next_start -+ * happened to be in the past. -+ */ -+ bfqq->soft_rt_next_start = -+ bfq_infinity_from_now(jiffies); -+ /* -+ * Schedule an update of soft_rt_next_start to when -+ * the task may be discovered to be isochronous. -+ */ -+ bfq_mark_bfqq_softrt_update(bfqq); -+ } -+ } -+ -+ bfq_log_bfqq(bfqd, bfqq, -+ "expire (%d, slow %d, num_disp %d, idle_win %d)", reason, -+ slow, bfqq->dispatched, bfq_bfqq_idle_window(bfqq)); -+ -+ /* -+ * Increase, decrease or leave budget unchanged according to -+ * reason. -+ */ -+ __bfq_bfqq_recalc_budget(bfqd, bfqq, reason); -+ __bfq_bfqq_expire(bfqd, bfqq); -+} -+ -+/* -+ * Budget timeout is not implemented through a dedicated timer, but -+ * just checked on request arrivals and completions, as well as on -+ * idle timer expirations. -+ */ -+static int bfq_bfqq_budget_timeout(struct bfq_queue *bfqq) -+{ -+ if (bfq_bfqq_budget_new(bfqq) || -+ time_before(jiffies, bfqq->budget_timeout)) -+ return 0; -+ return 1; -+} -+ -+/* -+ * If we expire a queue that is waiting for the arrival of a new -+ * request, we may prevent the fictitious timestamp back-shifting that -+ * allows the guarantees of the queue to be preserved (see [1] for -+ * this tricky aspect). Hence we return true only if this condition -+ * does not hold, or if the queue is slow enough to deserve only to be -+ * kicked off for preserving a high throughput. -+*/ -+static inline int bfq_may_expire_for_budg_timeout(struct bfq_queue *bfqq) -+{ -+ bfq_log_bfqq(bfqq->bfqd, bfqq, -+ "may_budget_timeout: wait_request %d left %d timeout %d", -+ bfq_bfqq_wait_request(bfqq), -+ bfq_bfqq_budget_left(bfqq) >= bfqq->entity.budget / 3, -+ bfq_bfqq_budget_timeout(bfqq)); -+ -+ return (!bfq_bfqq_wait_request(bfqq) || -+ bfq_bfqq_budget_left(bfqq) >= bfqq->entity.budget / 3) -+ && -+ bfq_bfqq_budget_timeout(bfqq); -+} -+ -+/* -+ * Device idling is allowed only for the queues for which this function -+ * returns true. For this reason, the return value of this function plays a -+ * critical role for both throughput boosting and service guarantees. The -+ * return value is computed through a logical expression. In this rather -+ * long comment, we try to briefly describe all the details and motivations -+ * behind the components of this logical expression. -+ * -+ * First, the expression is false if bfqq is not sync, or if: bfqq happened -+ * to become active during a large burst of queue activations, and the -+ * pattern of requests bfqq contains boosts the throughput if bfqq is -+ * expired. In fact, queues that became active during a large burst benefit -+ * only from throughput, as discussed in the comments to bfq_handle_burst. -+ * In this respect, expiring bfqq certainly boosts the throughput on NCQ- -+ * capable flash-based devices, whereas, on rotational devices, it boosts -+ * the throughput only if bfqq contains random requests. -+ * -+ * On the opposite end, if (a) bfqq is sync, (b) the above burst-related -+ * condition does not hold, and (c) bfqq is being weight-raised, then the -+ * expression always evaluates to true, as device idling is instrumental -+ * for preserving low-latency guarantees (see [1]). If, instead, conditions -+ * (a) and (b) do hold, but (c) does not, then the expression evaluates to -+ * true only if: (1) bfqq is I/O-bound and has a non-null idle window, and -+ * (2) at least one of the following two conditions holds. -+ * The first condition is that the device is not performing NCQ, because -+ * idling the device most certainly boosts the throughput if this condition -+ * holds and bfqq is I/O-bound and has been granted a non-null idle window. -+ * The second compound condition is made of the logical AND of two components. -+ * -+ * The first component is true only if there is no weight-raised busy -+ * queue. This guarantees that the device is not idled for a sync non- -+ * weight-raised queue when there are busy weight-raised queues. The former -+ * is then expired immediately if empty. Combined with the timestamping -+ * rules of BFQ (see [1] for details), this causes sync non-weight-raised -+ * queues to get a lower number of requests served, and hence to ask for a -+ * lower number of requests from the request pool, before the busy weight- -+ * raised queues get served again. -+ * -+ * This is beneficial for the processes associated with weight-raised -+ * queues, when the request pool is saturated (e.g., in the presence of -+ * write hogs). In fact, if the processes associated with the other queues -+ * ask for requests at a lower rate, then weight-raised processes have a -+ * higher probability to get a request from the pool immediately (or at -+ * least soon) when they need one. Hence they have a higher probability to -+ * actually get a fraction of the disk throughput proportional to their -+ * high weight. This is especially true with NCQ-capable drives, which -+ * enqueue several requests in advance and further reorder internally- -+ * queued requests. -+ * -+ * In the end, mistreating non-weight-raised queues when there are busy -+ * weight-raised queues seems to mitigate starvation problems in the -+ * presence of heavy write workloads and NCQ, and hence to guarantee a -+ * higher application and system responsiveness in these hostile scenarios. -+ * -+ * If the first component of the compound condition is instead true, i.e., -+ * there is no weight-raised busy queue, then the second component of the -+ * compound condition takes into account service-guarantee and throughput -+ * issues related to NCQ (recall that the compound condition is evaluated -+ * only if the device is detected as supporting NCQ). -+ * -+ * As for service guarantees, allowing the drive to enqueue more than one -+ * request at a time, and hence delegating de facto final scheduling -+ * decisions to the drive's internal scheduler, causes loss of control on -+ * the actual request service order. In this respect, when the drive is -+ * allowed to enqueue more than one request at a time, the service -+ * distribution enforced by the drive's internal scheduler is likely to -+ * coincide with the desired device-throughput distribution only in the -+ * following, perfectly symmetric, scenario: -+ * 1) all active queues have the same weight, -+ * 2) all active groups at the same level in the groups tree have the same -+ * weight, -+ * 3) all active groups at the same level in the groups tree have the same -+ * number of children. -+ * -+ * Even in such a scenario, sequential I/O may still receive a preferential -+ * treatment, but this is not likely to be a big issue with flash-based -+ * devices, because of their non-dramatic loss of throughput with random -+ * I/O. Things do differ with HDDs, for which additional care is taken, as -+ * explained after completing the discussion for flash-based devices. -+ * -+ * Unfortunately, keeping the necessary state for evaluating exactly the -+ * above symmetry conditions would be quite complex and time-consuming. -+ * Therefore BFQ evaluates instead the following stronger sub-conditions, -+ * for which it is much easier to maintain the needed state: -+ * 1) all active queues have the same weight, -+ * 2) all active groups have the same weight, -+ * 3) all active groups have at most one active child each. -+ * In particular, the last two conditions are always true if hierarchical -+ * support and the cgroups interface are not enabled, hence no state needs -+ * to be maintained in this case. -+ * -+ * According to the above considerations, the second component of the -+ * compound condition evaluates to true if any of the above symmetry -+ * sub-condition does not hold, or the device is not flash-based. Therefore, -+ * if also the first component is true, then idling is allowed for a sync -+ * queue. These are the only sub-conditions considered if the device is -+ * flash-based, as, for such a device, it is sensible to force idling only -+ * for service-guarantee issues. In fact, as for throughput, idling -+ * NCQ-capable flash-based devices would not boost the throughput even -+ * with sequential I/O; rather it would lower the throughput in proportion -+ * to how fast the device is. In the end, (only) if all the three -+ * sub-conditions hold and the device is flash-based, the compound -+ * condition evaluates to false and therefore no idling is performed. -+ * -+ * As already said, things change with a rotational device, where idling -+ * boosts the throughput with sequential I/O (even with NCQ). Hence, for -+ * such a device the second component of the compound condition evaluates -+ * to true also if the following additional sub-condition does not hold: -+ * the queue is constantly seeky. Unfortunately, this different behavior -+ * with respect to flash-based devices causes an additional asymmetry: if -+ * some sync queues enjoy idling and some other sync queues do not, then -+ * the latter get a low share of the device throughput, simply because the -+ * former get many requests served after being set as in service, whereas -+ * the latter do not. As a consequence, to guarantee the desired throughput -+ * distribution, on HDDs the compound expression evaluates to true (and -+ * hence device idling is performed) also if the following last symmetry -+ * condition does not hold: no other queue is benefiting from idling. Also -+ * this last condition is actually replaced with a simpler-to-maintain and -+ * stronger condition: there is no busy queue which is not constantly seeky -+ * (and hence may also benefit from idling). -+ * -+ * To sum up, when all the required symmetry and throughput-boosting -+ * sub-conditions hold, the second component of the compound condition -+ * evaluates to false, and hence no idling is performed. This helps to -+ * keep the drives' internal queues full on NCQ-capable devices, and hence -+ * to boost the throughput, without causing 'almost' any loss of service -+ * guarantees. The 'almost' follows from the fact that, if the internal -+ * queue of one such device is filled while all the sub-conditions hold, -+ * but at some point in time some sub-condition stops to hold, then it may -+ * become impossible to let requests be served in the new desired order -+ * until all the requests already queued in the device have been served. -+ */ -+static inline bool bfq_bfqq_must_not_expire(struct bfq_queue *bfqq) -+{ -+ struct bfq_data *bfqd = bfqq->bfqd; -+#ifdef CONFIG_CGROUP_BFQIO -+#define symmetric_scenario (!bfqd->active_numerous_groups && \ -+ !bfq_differentiated_weights(bfqd)) -+#else -+#define symmetric_scenario (!bfq_differentiated_weights(bfqd)) -+#endif -+#define cond_for_seeky_on_ncq_hdd (bfq_bfqq_constantly_seeky(bfqq) && \ -+ bfqd->busy_in_flight_queues == \ -+ bfqd->const_seeky_busy_in_flight_queues) -+ -+#define cond_for_expiring_in_burst (bfq_bfqq_in_large_burst(bfqq) && \ -+ bfqd->hw_tag && \ -+ (blk_queue_nonrot(bfqd->queue) || \ -+ bfq_bfqq_constantly_seeky(bfqq))) -+ -+/* -+ * Condition for expiring a non-weight-raised queue (and hence not idling -+ * the device). -+ */ -+#define cond_for_expiring_non_wr (bfqd->hw_tag && \ -+ (bfqd->wr_busy_queues > 0 || \ -+ (symmetric_scenario && \ -+ (blk_queue_nonrot(bfqd->queue) || \ -+ cond_for_seeky_on_ncq_hdd)))) -+ -+ return bfq_bfqq_sync(bfqq) && -+ !cond_for_expiring_in_burst && -+ (bfqq->wr_coeff > 1 || -+ (bfq_bfqq_IO_bound(bfqq) && bfq_bfqq_idle_window(bfqq) && -+ !cond_for_expiring_non_wr) -+ ); -+} -+ -+/* -+ * If the in-service queue is empty but sync, and the function -+ * bfq_bfqq_must_not_expire returns true, then: -+ * 1) the queue must remain in service and cannot be expired, and -+ * 2) the disk must be idled to wait for the possible arrival of a new -+ * request for the queue. -+ * See the comments to the function bfq_bfqq_must_not_expire for the reasons -+ * why performing device idling is the best choice to boost the throughput -+ * and preserve service guarantees when bfq_bfqq_must_not_expire itself -+ * returns true. -+ */ -+static inline bool bfq_bfqq_must_idle(struct bfq_queue *bfqq) -+{ -+ struct bfq_data *bfqd = bfqq->bfqd; -+ -+ return RB_EMPTY_ROOT(&bfqq->sort_list) && bfqd->bfq_slice_idle != 0 && -+ bfq_bfqq_must_not_expire(bfqq); -+} -+ -+/* -+ * Select a queue for service. If we have a current queue in service, -+ * check whether to continue servicing it, or retrieve and set a new one. -+ */ -+static struct bfq_queue *bfq_select_queue(struct bfq_data *bfqd) -+{ -+ struct bfq_queue *bfqq; -+ struct request *next_rq; -+ enum bfqq_expiration reason = BFQ_BFQQ_BUDGET_TIMEOUT; -+ -+ bfqq = bfqd->in_service_queue; -+ if (bfqq == NULL) -+ goto new_queue; -+ -+ bfq_log_bfqq(bfqd, bfqq, "select_queue: already in-service queue"); -+ -+ if (bfq_may_expire_for_budg_timeout(bfqq) && -+ !timer_pending(&bfqd->idle_slice_timer) && -+ !bfq_bfqq_must_idle(bfqq)) -+ goto expire; -+ -+ next_rq = bfqq->next_rq; -+ /* -+ * If bfqq has requests queued and it has enough budget left to -+ * serve them, keep the queue, otherwise expire it. -+ */ -+ if (next_rq != NULL) { -+ if (bfq_serv_to_charge(next_rq, bfqq) > -+ bfq_bfqq_budget_left(bfqq)) { -+ reason = BFQ_BFQQ_BUDGET_EXHAUSTED; -+ goto expire; -+ } else { -+ /* -+ * The idle timer may be pending because we may -+ * not disable disk idling even when a new request -+ * arrives. -+ */ -+ if (timer_pending(&bfqd->idle_slice_timer)) { -+ /* -+ * If we get here: 1) at least a new request -+ * has arrived but we have not disabled the -+ * timer because the request was too small, -+ * 2) then the block layer has unplugged -+ * the device, causing the dispatch to be -+ * invoked. -+ * -+ * Since the device is unplugged, now the -+ * requests are probably large enough to -+ * provide a reasonable throughput. -+ * So we disable idling. -+ */ -+ bfq_clear_bfqq_wait_request(bfqq); -+ del_timer(&bfqd->idle_slice_timer); -+ } -+ goto keep_queue; -+ } -+ } -+ -+ /* -+ * No requests pending. If the in-service queue still has requests -+ * in flight (possibly waiting for a completion) or is idling for a -+ * new request, then keep it. -+ */ -+ if (timer_pending(&bfqd->idle_slice_timer) || -+ (bfqq->dispatched != 0 && bfq_bfqq_must_not_expire(bfqq))) { -+ bfqq = NULL; -+ goto keep_queue; -+ } -+ -+ reason = BFQ_BFQQ_NO_MORE_REQUESTS; -+expire: -+ bfq_bfqq_expire(bfqd, bfqq, 0, reason); -+new_queue: -+ bfqq = bfq_set_in_service_queue(bfqd); -+ bfq_log(bfqd, "select_queue: new queue %d returned", -+ bfqq != NULL ? bfqq->pid : 0); -+keep_queue: -+ return bfqq; -+} -+ -+static void bfq_update_wr_data(struct bfq_data *bfqd, struct bfq_queue *bfqq) -+{ -+ struct bfq_entity *entity = &bfqq->entity; -+ if (bfqq->wr_coeff > 1) { /* queue is being weight-raised */ -+ bfq_log_bfqq(bfqd, bfqq, -+ "raising period dur %u/%u msec, old coeff %u, w %d(%d)", -+ jiffies_to_msecs(jiffies - bfqq->last_wr_start_finish), -+ jiffies_to_msecs(bfqq->wr_cur_max_time), -+ bfqq->wr_coeff, -+ bfqq->entity.weight, bfqq->entity.orig_weight); -+ -+ BUG_ON(bfqq != bfqd->in_service_queue && entity->weight != -+ entity->orig_weight * bfqq->wr_coeff); -+ if (entity->ioprio_changed) -+ bfq_log_bfqq(bfqd, bfqq, "WARN: pending prio change"); -+ -+ /* -+ * If the queue was activated in a burst, or -+ * too much time has elapsed from the beginning -+ * of this weight-raising period, or the queue has -+ * exceeded the acceptable number of cooperations, -+ * then end weight raising. -+ */ -+ if (bfq_bfqq_in_large_burst(bfqq) || -+ bfq_bfqq_cooperations(bfqq) >= bfqd->bfq_coop_thresh || -+ time_is_before_jiffies(bfqq->last_wr_start_finish + -+ bfqq->wr_cur_max_time)) { -+ bfqq->last_wr_start_finish = jiffies; -+ bfq_log_bfqq(bfqd, bfqq, -+ "wrais ending at %lu, rais_max_time %u", -+ bfqq->last_wr_start_finish, -+ jiffies_to_msecs(bfqq->wr_cur_max_time)); -+ bfq_bfqq_end_wr(bfqq); -+ } -+ } -+ /* Update weight both if it must be raised and if it must be lowered */ -+ if ((entity->weight > entity->orig_weight) != (bfqq->wr_coeff > 1)) -+ __bfq_entity_update_weight_prio( -+ bfq_entity_service_tree(entity), -+ entity); -+} -+ -+/* -+ * Dispatch one request from bfqq, moving it to the request queue -+ * dispatch list. -+ */ -+static int bfq_dispatch_request(struct bfq_data *bfqd, -+ struct bfq_queue *bfqq) -+{ -+ int dispatched = 0; -+ struct request *rq; -+ unsigned long service_to_charge; -+ -+ BUG_ON(RB_EMPTY_ROOT(&bfqq->sort_list)); -+ -+ /* Follow expired path, else get first next available. */ -+ rq = bfq_check_fifo(bfqq); -+ if (rq == NULL) -+ rq = bfqq->next_rq; -+ service_to_charge = bfq_serv_to_charge(rq, bfqq); -+ -+ if (service_to_charge > bfq_bfqq_budget_left(bfqq)) { -+ /* -+ * This may happen if the next rq is chosen in fifo order -+ * instead of sector order. The budget is properly -+ * dimensioned to be always sufficient to serve the next -+ * request only if it is chosen in sector order. The reason -+ * is that it would be quite inefficient and little useful -+ * to always make sure that the budget is large enough to -+ * serve even the possible next rq in fifo order. -+ * In fact, requests are seldom served in fifo order. -+ * -+ * Expire the queue for budget exhaustion, and make sure -+ * that the next act_budget is enough to serve the next -+ * request, even if it comes from the fifo expired path. -+ */ -+ bfqq->next_rq = rq; -+ /* -+ * Since this dispatch is failed, make sure that -+ * a new one will be performed -+ */ -+ if (!bfqd->rq_in_driver) -+ bfq_schedule_dispatch(bfqd); -+ goto expire; -+ } -+ -+ /* Finally, insert request into driver dispatch list. */ -+ bfq_bfqq_served(bfqq, service_to_charge); -+ bfq_dispatch_insert(bfqd->queue, rq); -+ -+ bfq_update_wr_data(bfqd, bfqq); -+ -+ bfq_log_bfqq(bfqd, bfqq, -+ "dispatched %u sec req (%llu), budg left %lu", -+ blk_rq_sectors(rq), -+ (long long unsigned)blk_rq_pos(rq), -+ bfq_bfqq_budget_left(bfqq)); -+ -+ dispatched++; -+ -+ if (bfqd->in_service_bic == NULL) { -+ atomic_long_inc(&RQ_BIC(rq)->icq.ioc->refcount); -+ bfqd->in_service_bic = RQ_BIC(rq); -+ } -+ -+ if (bfqd->busy_queues > 1 && ((!bfq_bfqq_sync(bfqq) && -+ dispatched >= bfqd->bfq_max_budget_async_rq) || -+ bfq_class_idle(bfqq))) -+ goto expire; -+ -+ return dispatched; -+ -+expire: -+ bfq_bfqq_expire(bfqd, bfqq, 0, BFQ_BFQQ_BUDGET_EXHAUSTED); -+ return dispatched; -+} -+ -+static int __bfq_forced_dispatch_bfqq(struct bfq_queue *bfqq) -+{ -+ int dispatched = 0; -+ -+ while (bfqq->next_rq != NULL) { -+ bfq_dispatch_insert(bfqq->bfqd->queue, bfqq->next_rq); -+ dispatched++; -+ } -+ -+ BUG_ON(!list_empty(&bfqq->fifo)); -+ return dispatched; -+} -+ -+/* -+ * Drain our current requests. -+ * Used for barriers and when switching io schedulers on-the-fly. -+ */ -+static int bfq_forced_dispatch(struct bfq_data *bfqd) -+{ -+ struct bfq_queue *bfqq, *n; -+ struct bfq_service_tree *st; -+ int dispatched = 0; -+ -+ bfqq = bfqd->in_service_queue; -+ if (bfqq != NULL) -+ __bfq_bfqq_expire(bfqd, bfqq); -+ -+ /* -+ * Loop through classes, and be careful to leave the scheduler -+ * in a consistent state, as feedback mechanisms and vtime -+ * updates cannot be disabled during the process. -+ */ -+ list_for_each_entry_safe(bfqq, n, &bfqd->active_list, bfqq_list) { -+ st = bfq_entity_service_tree(&bfqq->entity); -+ -+ dispatched += __bfq_forced_dispatch_bfqq(bfqq); -+ bfqq->max_budget = bfq_max_budget(bfqd); -+ -+ bfq_forget_idle(st); -+ } -+ -+ BUG_ON(bfqd->busy_queues != 0); -+ -+ return dispatched; -+} -+ -+static int bfq_dispatch_requests(struct request_queue *q, int force) -+{ -+ struct bfq_data *bfqd = q->elevator->elevator_data; -+ struct bfq_queue *bfqq; -+ int max_dispatch; -+ -+ bfq_log(bfqd, "dispatch requests: %d busy queues", bfqd->busy_queues); -+ if (bfqd->busy_queues == 0) -+ return 0; -+ -+ if (unlikely(force)) -+ return bfq_forced_dispatch(bfqd); -+ -+ bfqq = bfq_select_queue(bfqd); -+ if (bfqq == NULL) -+ return 0; -+ -+ max_dispatch = bfqd->bfq_quantum; -+ if (bfq_class_idle(bfqq)) -+ max_dispatch = 1; -+ -+ if (!bfq_bfqq_sync(bfqq)) -+ max_dispatch = bfqd->bfq_max_budget_async_rq; -+ -+ if (bfqq->dispatched >= max_dispatch) { -+ if (bfqd->busy_queues > 1) -+ return 0; -+ if (bfqq->dispatched >= 4 * max_dispatch) -+ return 0; -+ } -+ -+ if (bfqd->sync_flight != 0 && !bfq_bfqq_sync(bfqq)) -+ return 0; -+ -+ bfq_clear_bfqq_wait_request(bfqq); -+ BUG_ON(timer_pending(&bfqd->idle_slice_timer)); -+ -+ if (!bfq_dispatch_request(bfqd, bfqq)) -+ return 0; -+ -+ bfq_log_bfqq(bfqd, bfqq, "dispatched one request of %d (max_disp %d)", -+ bfqq->pid, max_dispatch); -+ -+ return 1; -+} -+ -+/* -+ * Task holds one reference to the queue, dropped when task exits. Each rq -+ * in-flight on this queue also holds a reference, dropped when rq is freed. -+ * -+ * Queue lock must be held here. -+ */ -+static void bfq_put_queue(struct bfq_queue *bfqq) -+{ -+ struct bfq_data *bfqd = bfqq->bfqd; -+ -+ BUG_ON(atomic_read(&bfqq->ref) <= 0); -+ -+ bfq_log_bfqq(bfqd, bfqq, "put_queue: %p %d", bfqq, -+ atomic_read(&bfqq->ref)); -+ if (!atomic_dec_and_test(&bfqq->ref)) -+ return; -+ -+ BUG_ON(rb_first(&bfqq->sort_list) != NULL); -+ BUG_ON(bfqq->allocated[READ] + bfqq->allocated[WRITE] != 0); -+ BUG_ON(bfqq->entity.tree != NULL); -+ BUG_ON(bfq_bfqq_busy(bfqq)); -+ BUG_ON(bfqd->in_service_queue == bfqq); -+ -+ if (bfq_bfqq_sync(bfqq)) -+ /* -+ * The fact that this queue is being destroyed does not -+ * invalidate the fact that this queue may have been -+ * activated during the current burst. As a consequence, -+ * although the queue does not exist anymore, and hence -+ * needs to be removed from the burst list if there, -+ * the burst size has not to be decremented. -+ */ -+ hlist_del_init(&bfqq->burst_list_node); -+ -+ bfq_log_bfqq(bfqd, bfqq, "put_queue: %p freed", bfqq); -+ -+ kmem_cache_free(bfq_pool, bfqq); -+} -+ -+static void bfq_put_cooperator(struct bfq_queue *bfqq) -+{ -+ struct bfq_queue *__bfqq, *next; -+ -+ /* -+ * If this queue was scheduled to merge with another queue, be -+ * sure to drop the reference taken on that queue (and others in -+ * the merge chain). See bfq_setup_merge and bfq_merge_bfqqs. -+ */ -+ __bfqq = bfqq->new_bfqq; -+ while (__bfqq) { -+ if (__bfqq == bfqq) -+ break; -+ next = __bfqq->new_bfqq; -+ bfq_put_queue(__bfqq); -+ __bfqq = next; -+ } -+} -+ -+static void bfq_exit_bfqq(struct bfq_data *bfqd, struct bfq_queue *bfqq) -+{ -+ if (bfqq == bfqd->in_service_queue) { -+ __bfq_bfqq_expire(bfqd, bfqq); -+ bfq_schedule_dispatch(bfqd); -+ } -+ -+ bfq_log_bfqq(bfqd, bfqq, "exit_bfqq: %p, %d", bfqq, -+ atomic_read(&bfqq->ref)); -+ -+ bfq_put_cooperator(bfqq); -+ -+ bfq_put_queue(bfqq); -+} -+ -+static inline void bfq_init_icq(struct io_cq *icq) -+{ -+ struct bfq_io_cq *bic = icq_to_bic(icq); -+ -+ bic->ttime.last_end_request = jiffies; -+ /* -+ * A newly created bic indicates that the process has just -+ * started doing I/O, and is probably mapping into memory its -+ * executable and libraries: it definitely needs weight raising. -+ * There is however the possibility that the process performs, -+ * for a while, I/O close to some other process. EQM intercepts -+ * this behavior and may merge the queue corresponding to the -+ * process with some other queue, BEFORE the weight of the queue -+ * is raised. Merged queues are not weight-raised (they are assumed -+ * to belong to processes that benefit only from high throughput). -+ * If the merge is basically the consequence of an accident, then -+ * the queue will be split soon and will get back its old weight. -+ * It is then important to write down somewhere that this queue -+ * does need weight raising, even if it did not make it to get its -+ * weight raised before being merged. To this purpose, we overload -+ * the field raising_time_left and assign 1 to it, to mark the queue -+ * as needing weight raising. -+ */ -+ bic->wr_time_left = 1; -+} -+ -+static void bfq_exit_icq(struct io_cq *icq) -+{ -+ struct bfq_io_cq *bic = icq_to_bic(icq); -+ struct bfq_data *bfqd = bic_to_bfqd(bic); -+ -+ if (bic->bfqq[BLK_RW_ASYNC]) { -+ bfq_exit_bfqq(bfqd, bic->bfqq[BLK_RW_ASYNC]); -+ bic->bfqq[BLK_RW_ASYNC] = NULL; -+ } -+ -+ if (bic->bfqq[BLK_RW_SYNC]) { -+ /* -+ * If the bic is using a shared queue, put the reference -+ * taken on the io_context when the bic started using a -+ * shared bfq_queue. -+ */ -+ if (bfq_bfqq_coop(bic->bfqq[BLK_RW_SYNC])) -+ put_io_context(icq->ioc); -+ bfq_exit_bfqq(bfqd, bic->bfqq[BLK_RW_SYNC]); -+ bic->bfqq[BLK_RW_SYNC] = NULL; -+ } -+} -+ -+/* -+ * Update the entity prio values; note that the new values will not -+ * be used until the next (re)activation. -+ */ -+static void bfq_init_prio_data(struct bfq_queue *bfqq, struct bfq_io_cq *bic) -+{ -+ struct task_struct *tsk = current; -+ int ioprio_class; -+ -+ if (!bfq_bfqq_prio_changed(bfqq)) -+ return; -+ -+ ioprio_class = IOPRIO_PRIO_CLASS(bic->ioprio); -+ switch (ioprio_class) { -+ default: -+ dev_err(bfqq->bfqd->queue->backing_dev_info.dev, -+ "bfq: bad prio class %d\n", ioprio_class); -+ case IOPRIO_CLASS_NONE: -+ /* -+ * No prio set, inherit CPU scheduling settings. -+ */ -+ bfqq->entity.new_ioprio = task_nice_ioprio(tsk); -+ bfqq->entity.new_ioprio_class = task_nice_ioclass(tsk); -+ break; -+ case IOPRIO_CLASS_RT: -+ bfqq->entity.new_ioprio = IOPRIO_PRIO_DATA(bic->ioprio); -+ bfqq->entity.new_ioprio_class = IOPRIO_CLASS_RT; -+ break; -+ case IOPRIO_CLASS_BE: -+ bfqq->entity.new_ioprio = IOPRIO_PRIO_DATA(bic->ioprio); -+ bfqq->entity.new_ioprio_class = IOPRIO_CLASS_BE; -+ break; -+ case IOPRIO_CLASS_IDLE: -+ bfqq->entity.new_ioprio_class = IOPRIO_CLASS_IDLE; -+ bfqq->entity.new_ioprio = 7; -+ bfq_clear_bfqq_idle_window(bfqq); -+ break; -+ } -+ -+ if (bfqq->entity.new_ioprio < 0 || -+ bfqq->entity.new_ioprio >= IOPRIO_BE_NR) { -+ printk(KERN_CRIT "bfq_init_prio_data: new_ioprio %d\n", -+ bfqq->entity.new_ioprio); -+ BUG(); -+ } -+ -+ bfqq->entity.ioprio_changed = 1; -+ -+ bfq_clear_bfqq_prio_changed(bfqq); -+} -+ -+static void bfq_changed_ioprio(struct bfq_io_cq *bic) -+{ -+ struct bfq_data *bfqd; -+ struct bfq_queue *bfqq, *new_bfqq; -+ struct bfq_group *bfqg; -+ unsigned long uninitialized_var(flags); -+ int ioprio = bic->icq.ioc->ioprio; -+ -+ bfqd = bfq_get_bfqd_locked(&(bic->icq.q->elevator->elevator_data), -+ &flags); -+ /* -+ * This condition may trigger on a newly created bic, be sure to -+ * drop the lock before returning. -+ */ -+ if (unlikely(bfqd == NULL) || likely(bic->ioprio == ioprio)) -+ goto out; -+ -+ bfqq = bic->bfqq[BLK_RW_ASYNC]; -+ if (bfqq != NULL) { -+ bfqg = container_of(bfqq->entity.sched_data, struct bfq_group, -+ sched_data); -+ new_bfqq = bfq_get_queue(bfqd, bfqg, BLK_RW_ASYNC, bic, -+ GFP_ATOMIC); -+ if (new_bfqq != NULL) { -+ bic->bfqq[BLK_RW_ASYNC] = new_bfqq; -+ bfq_log_bfqq(bfqd, bfqq, -+ "changed_ioprio: bfqq %p %d", -+ bfqq, atomic_read(&bfqq->ref)); -+ bfq_put_queue(bfqq); -+ } -+ } -+ -+ bfqq = bic->bfqq[BLK_RW_SYNC]; -+ if (bfqq != NULL) -+ bfq_mark_bfqq_prio_changed(bfqq); -+ -+ bic->ioprio = ioprio; -+ -+out: -+ bfq_put_bfqd_unlock(bfqd, &flags); -+} -+ -+static void bfq_init_bfqq(struct bfq_data *bfqd, struct bfq_queue *bfqq, -+ pid_t pid, int is_sync) -+{ -+ RB_CLEAR_NODE(&bfqq->entity.rb_node); -+ INIT_LIST_HEAD(&bfqq->fifo); -+ INIT_HLIST_NODE(&bfqq->burst_list_node); -+ -+ atomic_set(&bfqq->ref, 0); -+ bfqq->bfqd = bfqd; -+ -+ bfq_mark_bfqq_prio_changed(bfqq); -+ -+ if (is_sync) { -+ if (!bfq_class_idle(bfqq)) -+ bfq_mark_bfqq_idle_window(bfqq); -+ bfq_mark_bfqq_sync(bfqq); -+ } -+ bfq_mark_bfqq_IO_bound(bfqq); -+ -+ /* Tentative initial value to trade off between thr and lat */ -+ bfqq->max_budget = (2 * bfq_max_budget(bfqd)) / 3; -+ bfqq->pid = pid; -+ -+ bfqq->wr_coeff = 1; -+ bfqq->last_wr_start_finish = 0; -+ /* -+ * Set to the value for which bfqq will not be deemed as -+ * soft rt when it becomes backlogged. -+ */ -+ bfqq->soft_rt_next_start = bfq_infinity_from_now(jiffies); -+} -+ -+static struct bfq_queue *bfq_find_alloc_queue(struct bfq_data *bfqd, -+ struct bfq_group *bfqg, -+ int is_sync, -+ struct bfq_io_cq *bic, -+ gfp_t gfp_mask) -+{ -+ struct bfq_queue *bfqq, *new_bfqq = NULL; -+ -+retry: -+ /* bic always exists here */ -+ bfqq = bic_to_bfqq(bic, is_sync); -+ -+ /* -+ * Always try a new alloc if we fall back to the OOM bfqq -+ * originally, since it should just be a temporary situation. -+ */ -+ if (bfqq == NULL || bfqq == &bfqd->oom_bfqq) { -+ bfqq = NULL; -+ if (new_bfqq != NULL) { -+ bfqq = new_bfqq; -+ new_bfqq = NULL; -+ } else if (gfp_mask & __GFP_WAIT) { -+ spin_unlock_irq(bfqd->queue->queue_lock); -+ new_bfqq = kmem_cache_alloc_node(bfq_pool, -+ gfp_mask | __GFP_ZERO, -+ bfqd->queue->node); -+ spin_lock_irq(bfqd->queue->queue_lock); -+ if (new_bfqq != NULL) -+ goto retry; -+ } else { -+ bfqq = kmem_cache_alloc_node(bfq_pool, -+ gfp_mask | __GFP_ZERO, -+ bfqd->queue->node); -+ } -+ -+ if (bfqq != NULL) { -+ bfq_init_bfqq(bfqd, bfqq, current->pid, is_sync); -+ bfq_init_prio_data(bfqq, bic); -+ bfq_init_entity(&bfqq->entity, bfqg); -+ bfq_log_bfqq(bfqd, bfqq, "allocated"); -+ } else { -+ bfqq = &bfqd->oom_bfqq; -+ bfq_log_bfqq(bfqd, bfqq, "using oom bfqq"); -+ } -+ } -+ -+ if (new_bfqq != NULL) -+ kmem_cache_free(bfq_pool, new_bfqq); -+ -+ return bfqq; -+} -+ -+static struct bfq_queue **bfq_async_queue_prio(struct bfq_data *bfqd, -+ struct bfq_group *bfqg, -+ int ioprio_class, int ioprio) -+{ -+ switch (ioprio_class) { -+ case IOPRIO_CLASS_RT: -+ return &bfqg->async_bfqq[0][ioprio]; -+ case IOPRIO_CLASS_NONE: -+ ioprio = IOPRIO_NORM; -+ /* fall through */ -+ case IOPRIO_CLASS_BE: -+ return &bfqg->async_bfqq[1][ioprio]; -+ case IOPRIO_CLASS_IDLE: -+ return &bfqg->async_idle_bfqq; -+ default: -+ BUG(); -+ } -+} -+ -+static struct bfq_queue *bfq_get_queue(struct bfq_data *bfqd, -+ struct bfq_group *bfqg, int is_sync, -+ struct bfq_io_cq *bic, gfp_t gfp_mask) -+{ -+ const int ioprio = IOPRIO_PRIO_DATA(bic->ioprio); -+ const int ioprio_class = IOPRIO_PRIO_CLASS(bic->ioprio); -+ struct bfq_queue **async_bfqq = NULL; -+ struct bfq_queue *bfqq = NULL; -+ -+ if (!is_sync) { -+ async_bfqq = bfq_async_queue_prio(bfqd, bfqg, ioprio_class, -+ ioprio); -+ bfqq = *async_bfqq; -+ } -+ -+ if (bfqq == NULL) -+ bfqq = bfq_find_alloc_queue(bfqd, bfqg, is_sync, bic, gfp_mask); -+ -+ /* -+ * Pin the queue now that it's allocated, scheduler exit will -+ * prune it. -+ */ -+ if (!is_sync && *async_bfqq == NULL) { -+ atomic_inc(&bfqq->ref); -+ bfq_log_bfqq(bfqd, bfqq, "get_queue, bfqq not in async: %p, %d", -+ bfqq, atomic_read(&bfqq->ref)); -+ *async_bfqq = bfqq; -+ } -+ -+ atomic_inc(&bfqq->ref); -+ bfq_log_bfqq(bfqd, bfqq, "get_queue, at end: %p, %d", bfqq, -+ atomic_read(&bfqq->ref)); -+ return bfqq; -+} -+ -+static void bfq_update_io_thinktime(struct bfq_data *bfqd, -+ struct bfq_io_cq *bic) -+{ -+ unsigned long elapsed = jiffies - bic->ttime.last_end_request; -+ unsigned long ttime = min(elapsed, 2UL * bfqd->bfq_slice_idle); -+ -+ bic->ttime.ttime_samples = (7*bic->ttime.ttime_samples + 256) / 8; -+ bic->ttime.ttime_total = (7*bic->ttime.ttime_total + 256*ttime) / 8; -+ bic->ttime.ttime_mean = (bic->ttime.ttime_total + 128) / -+ bic->ttime.ttime_samples; -+} -+ -+static void bfq_update_io_seektime(struct bfq_data *bfqd, -+ struct bfq_queue *bfqq, -+ struct request *rq) -+{ -+ sector_t sdist; -+ u64 total; -+ -+ if (bfqq->last_request_pos < blk_rq_pos(rq)) -+ sdist = blk_rq_pos(rq) - bfqq->last_request_pos; -+ else -+ sdist = bfqq->last_request_pos - blk_rq_pos(rq); -+ -+ /* -+ * Don't allow the seek distance to get too large from the -+ * odd fragment, pagein, etc. -+ */ -+ if (bfqq->seek_samples == 0) /* first request, not really a seek */ -+ sdist = 0; -+ else if (bfqq->seek_samples <= 60) /* second & third seek */ -+ sdist = min(sdist, (bfqq->seek_mean * 4) + 2*1024*1024); -+ else -+ sdist = min(sdist, (bfqq->seek_mean * 4) + 2*1024*64); -+ -+ bfqq->seek_samples = (7*bfqq->seek_samples + 256) / 8; -+ bfqq->seek_total = (7*bfqq->seek_total + (u64)256*sdist) / 8; -+ total = bfqq->seek_total + (bfqq->seek_samples/2); -+ do_div(total, bfqq->seek_samples); -+ bfqq->seek_mean = (sector_t)total; -+ -+ bfq_log_bfqq(bfqd, bfqq, "dist=%llu mean=%llu", (u64)sdist, -+ (u64)bfqq->seek_mean); -+} -+ -+/* -+ * Disable idle window if the process thinks too long or seeks so much that -+ * it doesn't matter. -+ */ -+static void bfq_update_idle_window(struct bfq_data *bfqd, -+ struct bfq_queue *bfqq, -+ struct bfq_io_cq *bic) -+{ -+ int enable_idle; -+ -+ /* Don't idle for async or idle io prio class. */ -+ if (!bfq_bfqq_sync(bfqq) || bfq_class_idle(bfqq)) -+ return; -+ -+ /* Idle window just restored, statistics are meaningless. */ -+ if (bfq_bfqq_just_split(bfqq)) -+ return; -+ -+ enable_idle = bfq_bfqq_idle_window(bfqq); -+ -+ if (atomic_read(&bic->icq.ioc->active_ref) == 0 || -+ bfqd->bfq_slice_idle == 0 || -+ (bfqd->hw_tag && BFQQ_SEEKY(bfqq) && -+ bfqq->wr_coeff == 1)) -+ enable_idle = 0; -+ else if (bfq_sample_valid(bic->ttime.ttime_samples)) { -+ if (bic->ttime.ttime_mean > bfqd->bfq_slice_idle && -+ bfqq->wr_coeff == 1) -+ enable_idle = 0; -+ else -+ enable_idle = 1; -+ } -+ bfq_log_bfqq(bfqd, bfqq, "update_idle_window: enable_idle %d", -+ enable_idle); -+ -+ if (enable_idle) -+ bfq_mark_bfqq_idle_window(bfqq); -+ else -+ bfq_clear_bfqq_idle_window(bfqq); -+} -+ -+/* -+ * Called when a new fs request (rq) is added to bfqq. Check if there's -+ * something we should do about it. -+ */ -+static void bfq_rq_enqueued(struct bfq_data *bfqd, struct bfq_queue *bfqq, -+ struct request *rq) -+{ -+ struct bfq_io_cq *bic = RQ_BIC(rq); -+ -+ if (rq->cmd_flags & REQ_META) -+ bfqq->meta_pending++; -+ -+ bfq_update_io_thinktime(bfqd, bic); -+ bfq_update_io_seektime(bfqd, bfqq, rq); -+ if (!BFQQ_SEEKY(bfqq) && bfq_bfqq_constantly_seeky(bfqq)) { -+ bfq_clear_bfqq_constantly_seeky(bfqq); -+ if (!blk_queue_nonrot(bfqd->queue)) { -+ BUG_ON(!bfqd->const_seeky_busy_in_flight_queues); -+ bfqd->const_seeky_busy_in_flight_queues--; -+ } -+ } -+ if (bfqq->entity.service > bfq_max_budget(bfqd) / 8 || -+ !BFQQ_SEEKY(bfqq)) -+ bfq_update_idle_window(bfqd, bfqq, bic); -+ bfq_clear_bfqq_just_split(bfqq); -+ -+ bfq_log_bfqq(bfqd, bfqq, -+ "rq_enqueued: idle_window=%d (seeky %d, mean %llu)", -+ bfq_bfqq_idle_window(bfqq), BFQQ_SEEKY(bfqq), -+ (long long unsigned)bfqq->seek_mean); -+ -+ bfqq->last_request_pos = blk_rq_pos(rq) + blk_rq_sectors(rq); -+ -+ if (bfqq == bfqd->in_service_queue && bfq_bfqq_wait_request(bfqq)) { -+ int small_req = bfqq->queued[rq_is_sync(rq)] == 1 && -+ blk_rq_sectors(rq) < 32; -+ int budget_timeout = bfq_bfqq_budget_timeout(bfqq); -+ -+ /* -+ * There is just this request queued: if the request -+ * is small and the queue is not to be expired, then -+ * just exit. -+ * -+ * In this way, if the disk is being idled to wait for -+ * a new request from the in-service queue, we avoid -+ * unplugging the device and committing the disk to serve -+ * just a small request. On the contrary, we wait for -+ * the block layer to decide when to unplug the device: -+ * hopefully, new requests will be merged to this one -+ * quickly, then the device will be unplugged and -+ * larger requests will be dispatched. -+ */ -+ if (small_req && !budget_timeout) -+ return; -+ -+ /* -+ * A large enough request arrived, or the queue is to -+ * be expired: in both cases disk idling is to be -+ * stopped, so clear wait_request flag and reset -+ * timer. -+ */ -+ bfq_clear_bfqq_wait_request(bfqq); -+ del_timer(&bfqd->idle_slice_timer); -+ -+ /* -+ * The queue is not empty, because a new request just -+ * arrived. Hence we can safely expire the queue, in -+ * case of budget timeout, without risking that the -+ * timestamps of the queue are not updated correctly. -+ * See [1] for more details. -+ */ -+ if (budget_timeout) -+ bfq_bfqq_expire(bfqd, bfqq, 0, BFQ_BFQQ_BUDGET_TIMEOUT); -+ -+ /* -+ * Let the request rip immediately, or let a new queue be -+ * selected if bfqq has just been expired. -+ */ -+ __blk_run_queue(bfqd->queue); -+ } -+} -+ -+static void bfq_insert_request(struct request_queue *q, struct request *rq) -+{ -+ struct bfq_data *bfqd = q->elevator->elevator_data; -+ struct bfq_queue *bfqq = RQ_BFQQ(rq), *new_bfqq; -+ -+ assert_spin_locked(bfqd->queue->queue_lock); -+ -+ /* -+ * An unplug may trigger a requeue of a request from the device -+ * driver: make sure we are in process context while trying to -+ * merge two bfq_queues. -+ */ -+ if (!in_interrupt()) { -+ new_bfqq = bfq_setup_cooperator(bfqd, bfqq, rq, true); -+ if (new_bfqq != NULL) { -+ if (bic_to_bfqq(RQ_BIC(rq), 1) != bfqq) -+ new_bfqq = bic_to_bfqq(RQ_BIC(rq), 1); -+ /* -+ * Release the request's reference to the old bfqq -+ * and make sure one is taken to the shared queue. -+ */ -+ new_bfqq->allocated[rq_data_dir(rq)]++; -+ bfqq->allocated[rq_data_dir(rq)]--; -+ atomic_inc(&new_bfqq->ref); -+ bfq_put_queue(bfqq); -+ if (bic_to_bfqq(RQ_BIC(rq), 1) == bfqq) -+ bfq_merge_bfqqs(bfqd, RQ_BIC(rq), -+ bfqq, new_bfqq); -+ rq->elv.priv[1] = new_bfqq; -+ bfqq = new_bfqq; -+ } else -+ bfq_bfqq_increase_failed_cooperations(bfqq); -+ } -+ -+ bfq_init_prio_data(bfqq, RQ_BIC(rq)); -+ -+ bfq_add_request(rq); -+ -+ /* -+ * Here a newly-created bfq_queue has already started a weight-raising -+ * period: clear raising_time_left to prevent bfq_bfqq_save_state() -+ * from assigning it a full weight-raising period. See the detailed -+ * comments about this field in bfq_init_icq(). -+ */ -+ if (bfqq->bic != NULL) -+ bfqq->bic->wr_time_left = 0; -+ rq->fifo_time = jiffies + bfqd->bfq_fifo_expire[rq_is_sync(rq)]; -+ list_add_tail(&rq->queuelist, &bfqq->fifo); -+ -+ bfq_rq_enqueued(bfqd, bfqq, rq); -+} -+ -+static void bfq_update_hw_tag(struct bfq_data *bfqd) -+{ -+ bfqd->max_rq_in_driver = max(bfqd->max_rq_in_driver, -+ bfqd->rq_in_driver); -+ -+ if (bfqd->hw_tag == 1) -+ return; -+ -+ /* -+ * This sample is valid if the number of outstanding requests -+ * is large enough to allow a queueing behavior. Note that the -+ * sum is not exact, as it's not taking into account deactivated -+ * requests. -+ */ -+ if (bfqd->rq_in_driver + bfqd->queued < BFQ_HW_QUEUE_THRESHOLD) -+ return; -+ -+ if (bfqd->hw_tag_samples++ < BFQ_HW_QUEUE_SAMPLES) -+ return; -+ -+ bfqd->hw_tag = bfqd->max_rq_in_driver > BFQ_HW_QUEUE_THRESHOLD; -+ bfqd->max_rq_in_driver = 0; -+ bfqd->hw_tag_samples = 0; -+} -+ -+static void bfq_completed_request(struct request_queue *q, struct request *rq) -+{ -+ struct bfq_queue *bfqq = RQ_BFQQ(rq); -+ struct bfq_data *bfqd = bfqq->bfqd; -+ bool sync = bfq_bfqq_sync(bfqq); -+ -+ bfq_log_bfqq(bfqd, bfqq, "completed one req with %u sects left (%d)", -+ blk_rq_sectors(rq), sync); -+ -+ bfq_update_hw_tag(bfqd); -+ -+ BUG_ON(!bfqd->rq_in_driver); -+ BUG_ON(!bfqq->dispatched); -+ bfqd->rq_in_driver--; -+ bfqq->dispatched--; -+ -+ if (!bfqq->dispatched && !bfq_bfqq_busy(bfqq)) { -+ bfq_weights_tree_remove(bfqd, &bfqq->entity, -+ &bfqd->queue_weights_tree); -+ if (!blk_queue_nonrot(bfqd->queue)) { -+ BUG_ON(!bfqd->busy_in_flight_queues); -+ bfqd->busy_in_flight_queues--; -+ if (bfq_bfqq_constantly_seeky(bfqq)) { -+ BUG_ON(!bfqd-> -+ const_seeky_busy_in_flight_queues); -+ bfqd->const_seeky_busy_in_flight_queues--; -+ } -+ } -+ } -+ -+ if (sync) { -+ bfqd->sync_flight--; -+ RQ_BIC(rq)->ttime.last_end_request = jiffies; -+ } -+ -+ /* -+ * If we are waiting to discover whether the request pattern of the -+ * task associated with the queue is actually isochronous, and -+ * both requisites for this condition to hold are satisfied, then -+ * compute soft_rt_next_start (see the comments to the function -+ * bfq_bfqq_softrt_next_start()). -+ */ -+ if (bfq_bfqq_softrt_update(bfqq) && bfqq->dispatched == 0 && -+ RB_EMPTY_ROOT(&bfqq->sort_list)) -+ bfqq->soft_rt_next_start = -+ bfq_bfqq_softrt_next_start(bfqd, bfqq); -+ -+ /* -+ * If this is the in-service queue, check if it needs to be expired, -+ * or if we want to idle in case it has no pending requests. -+ */ -+ if (bfqd->in_service_queue == bfqq) { -+ if (bfq_bfqq_budget_new(bfqq)) -+ bfq_set_budget_timeout(bfqd); -+ -+ if (bfq_bfqq_must_idle(bfqq)) { -+ bfq_arm_slice_timer(bfqd); -+ goto out; -+ } else if (bfq_may_expire_for_budg_timeout(bfqq)) -+ bfq_bfqq_expire(bfqd, bfqq, 0, BFQ_BFQQ_BUDGET_TIMEOUT); -+ else if (RB_EMPTY_ROOT(&bfqq->sort_list) && -+ (bfqq->dispatched == 0 || -+ !bfq_bfqq_must_not_expire(bfqq))) -+ bfq_bfqq_expire(bfqd, bfqq, 0, -+ BFQ_BFQQ_NO_MORE_REQUESTS); -+ } -+ -+ if (!bfqd->rq_in_driver) -+ bfq_schedule_dispatch(bfqd); -+ -+out: -+ return; -+} -+ -+static inline int __bfq_may_queue(struct bfq_queue *bfqq) -+{ -+ if (bfq_bfqq_wait_request(bfqq) && bfq_bfqq_must_alloc(bfqq)) { -+ bfq_clear_bfqq_must_alloc(bfqq); -+ return ELV_MQUEUE_MUST; -+ } -+ -+ return ELV_MQUEUE_MAY; -+} -+ -+static int bfq_may_queue(struct request_queue *q, int rw) -+{ -+ struct bfq_data *bfqd = q->elevator->elevator_data; -+ struct task_struct *tsk = current; -+ struct bfq_io_cq *bic; -+ struct bfq_queue *bfqq; -+ -+ /* -+ * Don't force setup of a queue from here, as a call to may_queue -+ * does not necessarily imply that a request actually will be -+ * queued. So just lookup a possibly existing queue, or return -+ * 'may queue' if that fails. -+ */ -+ bic = bfq_bic_lookup(bfqd, tsk->io_context); -+ if (bic == NULL) -+ return ELV_MQUEUE_MAY; -+ -+ bfqq = bic_to_bfqq(bic, rw_is_sync(rw)); -+ if (bfqq != NULL) { -+ bfq_init_prio_data(bfqq, bic); -+ -+ return __bfq_may_queue(bfqq); -+ } -+ -+ return ELV_MQUEUE_MAY; -+} -+ -+/* -+ * Queue lock held here. -+ */ -+static void bfq_put_request(struct request *rq) -+{ -+ struct bfq_queue *bfqq = RQ_BFQQ(rq); -+ -+ if (bfqq != NULL) { -+ const int rw = rq_data_dir(rq); -+ -+ BUG_ON(!bfqq->allocated[rw]); -+ bfqq->allocated[rw]--; -+ -+ rq->elv.priv[0] = NULL; -+ rq->elv.priv[1] = NULL; -+ -+ bfq_log_bfqq(bfqq->bfqd, bfqq, "put_request %p, %d", -+ bfqq, atomic_read(&bfqq->ref)); -+ bfq_put_queue(bfqq); -+ } -+} -+ -+/* -+ * Returns NULL if a new bfqq should be allocated, or the old bfqq if this -+ * was the last process referring to said bfqq. -+ */ -+static struct bfq_queue * -+bfq_split_bfqq(struct bfq_io_cq *bic, struct bfq_queue *bfqq) -+{ -+ bfq_log_bfqq(bfqq->bfqd, bfqq, "splitting queue"); -+ -+ put_io_context(bic->icq.ioc); -+ -+ if (bfqq_process_refs(bfqq) == 1) { -+ bfqq->pid = current->pid; -+ bfq_clear_bfqq_coop(bfqq); -+ bfq_clear_bfqq_split_coop(bfqq); -+ return bfqq; -+ } -+ -+ bic_set_bfqq(bic, NULL, 1); -+ -+ bfq_put_cooperator(bfqq); -+ -+ bfq_put_queue(bfqq); -+ return NULL; -+} -+ -+/* -+ * Allocate bfq data structures associated with this request. -+ */ -+static int bfq_set_request(struct request_queue *q, struct request *rq, -+ struct bio *bio, gfp_t gfp_mask) -+{ -+ struct bfq_data *bfqd = q->elevator->elevator_data; -+ struct bfq_io_cq *bic = icq_to_bic(rq->elv.icq); -+ const int rw = rq_data_dir(rq); -+ const int is_sync = rq_is_sync(rq); -+ struct bfq_queue *bfqq; -+ struct bfq_group *bfqg; -+ unsigned long flags; -+ bool split = false; -+ -+ might_sleep_if(gfp_mask & __GFP_WAIT); -+ -+ bfq_changed_ioprio(bic); -+ -+ spin_lock_irqsave(q->queue_lock, flags); -+ -+ if (bic == NULL) -+ goto queue_fail; -+ -+ bfqg = bfq_bic_update_cgroup(bic); -+ -+new_queue: -+ bfqq = bic_to_bfqq(bic, is_sync); -+ if (bfqq == NULL || bfqq == &bfqd->oom_bfqq) { -+ bfqq = bfq_get_queue(bfqd, bfqg, is_sync, bic, gfp_mask); -+ bic_set_bfqq(bic, bfqq, is_sync); -+ if (split && is_sync) { -+ if ((bic->was_in_burst_list && bfqd->large_burst) || -+ bic->saved_in_large_burst) -+ bfq_mark_bfqq_in_large_burst(bfqq); -+ else { -+ bfq_clear_bfqq_in_large_burst(bfqq); -+ if (bic->was_in_burst_list) -+ hlist_add_head(&bfqq->burst_list_node, -+ &bfqd->burst_list); -+ } -+ } -+ } else { -+ /* If the queue was seeky for too long, break it apart. */ -+ if (bfq_bfqq_coop(bfqq) && bfq_bfqq_split_coop(bfqq)) { -+ bfq_log_bfqq(bfqd, bfqq, "breaking apart bfqq"); -+ bfqq = bfq_split_bfqq(bic, bfqq); -+ split = true; -+ if (!bfqq) -+ goto new_queue; -+ } -+ } -+ -+ bfqq->allocated[rw]++; -+ atomic_inc(&bfqq->ref); -+ bfq_log_bfqq(bfqd, bfqq, "set_request: bfqq %p, %d", bfqq, -+ atomic_read(&bfqq->ref)); -+ -+ rq->elv.priv[0] = bic; -+ rq->elv.priv[1] = bfqq; -+ -+ /* -+ * If a bfq_queue has only one process reference, it is owned -+ * by only one bfq_io_cq: we can set the bic field of the -+ * bfq_queue to the address of that structure. Also, if the -+ * queue has just been split, mark a flag so that the -+ * information is available to the other scheduler hooks. -+ */ -+ if (likely(bfqq != &bfqd->oom_bfqq) && bfqq_process_refs(bfqq) == 1) { -+ bfqq->bic = bic; -+ if (split) { -+ bfq_mark_bfqq_just_split(bfqq); -+ /* -+ * If the queue has just been split from a shared -+ * queue, restore the idle window and the possible -+ * weight raising period. -+ */ -+ bfq_bfqq_resume_state(bfqq, bic); -+ } -+ } -+ -+ spin_unlock_irqrestore(q->queue_lock, flags); -+ -+ return 0; -+ -+queue_fail: -+ bfq_schedule_dispatch(bfqd); -+ spin_unlock_irqrestore(q->queue_lock, flags); -+ -+ return 1; -+} -+ -+static void bfq_kick_queue(struct work_struct *work) -+{ -+ struct bfq_data *bfqd = -+ container_of(work, struct bfq_data, unplug_work); -+ struct request_queue *q = bfqd->queue; -+ -+ spin_lock_irq(q->queue_lock); -+ __blk_run_queue(q); -+ spin_unlock_irq(q->queue_lock); -+} -+ -+/* -+ * Handler of the expiration of the timer running if the in-service queue -+ * is idling inside its time slice. -+ */ -+static void bfq_idle_slice_timer(unsigned long data) -+{ -+ struct bfq_data *bfqd = (struct bfq_data *)data; -+ struct bfq_queue *bfqq; -+ unsigned long flags; -+ enum bfqq_expiration reason; -+ -+ spin_lock_irqsave(bfqd->queue->queue_lock, flags); -+ -+ bfqq = bfqd->in_service_queue; -+ /* -+ * Theoretical race here: the in-service queue can be NULL or -+ * different from the queue that was idling if the timer handler -+ * spins on the queue_lock and a new request arrives for the -+ * current queue and there is a full dispatch cycle that changes -+ * the in-service queue. This can hardly happen, but in the worst -+ * case we just expire a queue too early. -+ */ -+ if (bfqq != NULL) { -+ bfq_log_bfqq(bfqd, bfqq, "slice_timer expired"); -+ if (bfq_bfqq_budget_timeout(bfqq)) -+ /* -+ * Also here the queue can be safely expired -+ * for budget timeout without wasting -+ * guarantees -+ */ -+ reason = BFQ_BFQQ_BUDGET_TIMEOUT; -+ else if (bfqq->queued[0] == 0 && bfqq->queued[1] == 0) -+ /* -+ * The queue may not be empty upon timer expiration, -+ * because we may not disable the timer when the -+ * first request of the in-service queue arrives -+ * during disk idling. -+ */ -+ reason = BFQ_BFQQ_TOO_IDLE; -+ else -+ goto schedule_dispatch; -+ -+ bfq_bfqq_expire(bfqd, bfqq, 1, reason); -+ } -+ -+schedule_dispatch: -+ bfq_schedule_dispatch(bfqd); -+ -+ spin_unlock_irqrestore(bfqd->queue->queue_lock, flags); -+} -+ -+static void bfq_shutdown_timer_wq(struct bfq_data *bfqd) -+{ -+ del_timer_sync(&bfqd->idle_slice_timer); -+ cancel_work_sync(&bfqd->unplug_work); -+} -+ -+static inline void __bfq_put_async_bfqq(struct bfq_data *bfqd, -+ struct bfq_queue **bfqq_ptr) -+{ -+ struct bfq_group *root_group = bfqd->root_group; -+ struct bfq_queue *bfqq = *bfqq_ptr; -+ -+ bfq_log(bfqd, "put_async_bfqq: %p", bfqq); -+ if (bfqq != NULL) { -+ bfq_bfqq_move(bfqd, bfqq, &bfqq->entity, root_group); -+ bfq_log_bfqq(bfqd, bfqq, "put_async_bfqq: putting %p, %d", -+ bfqq, atomic_read(&bfqq->ref)); -+ bfq_put_queue(bfqq); -+ *bfqq_ptr = NULL; -+ } -+} -+ -+/* -+ * Release all the bfqg references to its async queues. If we are -+ * deallocating the group these queues may still contain requests, so -+ * we reparent them to the root cgroup (i.e., the only one that will -+ * exist for sure until all the requests on a device are gone). -+ */ -+static void bfq_put_async_queues(struct bfq_data *bfqd, struct bfq_group *bfqg) -+{ -+ int i, j; -+ -+ for (i = 0; i < 2; i++) -+ for (j = 0; j < IOPRIO_BE_NR; j++) -+ __bfq_put_async_bfqq(bfqd, &bfqg->async_bfqq[i][j]); -+ -+ __bfq_put_async_bfqq(bfqd, &bfqg->async_idle_bfqq); -+} -+ -+static void bfq_exit_queue(struct elevator_queue *e) -+{ -+ struct bfq_data *bfqd = e->elevator_data; -+ struct request_queue *q = bfqd->queue; -+ struct bfq_queue *bfqq, *n; -+ -+ bfq_shutdown_timer_wq(bfqd); -+ -+ spin_lock_irq(q->queue_lock); -+ -+ BUG_ON(bfqd->in_service_queue != NULL); -+ list_for_each_entry_safe(bfqq, n, &bfqd->idle_list, bfqq_list) -+ bfq_deactivate_bfqq(bfqd, bfqq, 0); -+ -+ bfq_disconnect_groups(bfqd); -+ spin_unlock_irq(q->queue_lock); -+ -+ bfq_shutdown_timer_wq(bfqd); -+ -+ synchronize_rcu(); -+ -+ BUG_ON(timer_pending(&bfqd->idle_slice_timer)); -+ -+ bfq_free_root_group(bfqd); -+ kfree(bfqd); -+} -+ -+static int bfq_init_queue(struct request_queue *q, struct elevator_type *e) -+{ -+ struct bfq_group *bfqg; -+ struct bfq_data *bfqd; -+ struct elevator_queue *eq; -+ -+ eq = elevator_alloc(q, e); -+ if (eq == NULL) -+ return -ENOMEM; -+ -+ bfqd = kzalloc_node(sizeof(*bfqd), GFP_KERNEL, q->node); -+ if (bfqd == NULL) { -+ kobject_put(&eq->kobj); -+ return -ENOMEM; -+ } -+ eq->elevator_data = bfqd; -+ -+ /* -+ * Our fallback bfqq if bfq_find_alloc_queue() runs into OOM issues. -+ * Grab a permanent reference to it, so that the normal code flow -+ * will not attempt to free it. -+ */ -+ bfq_init_bfqq(bfqd, &bfqd->oom_bfqq, 1, 0); -+ atomic_inc(&bfqd->oom_bfqq.ref); -+ bfqd->oom_bfqq.entity.new_ioprio = BFQ_DEFAULT_QUEUE_IOPRIO; -+ bfqd->oom_bfqq.entity.new_ioprio_class = IOPRIO_CLASS_BE; -+ /* -+ * Trigger weight initialization, according to ioprio, at the -+ * oom_bfqq's first activation. The oom_bfqq's ioprio and ioprio -+ * class won't be changed any more. -+ */ -+ bfqd->oom_bfqq.entity.ioprio_changed = 1; -+ -+ bfqd->queue = q; -+ -+ spin_lock_irq(q->queue_lock); -+ q->elevator = eq; -+ spin_unlock_irq(q->queue_lock); -+ -+ bfqg = bfq_alloc_root_group(bfqd, q->node); -+ if (bfqg == NULL) { -+ kfree(bfqd); -+ kobject_put(&eq->kobj); -+ return -ENOMEM; -+ } -+ -+ bfqd->root_group = bfqg; -+ bfq_init_entity(&bfqd->oom_bfqq.entity, bfqd->root_group); -+#ifdef CONFIG_CGROUP_BFQIO -+ bfqd->active_numerous_groups = 0; -+#endif -+ -+ init_timer(&bfqd->idle_slice_timer); -+ bfqd->idle_slice_timer.function = bfq_idle_slice_timer; -+ bfqd->idle_slice_timer.data = (unsigned long)bfqd; -+ -+ bfqd->rq_pos_tree = RB_ROOT; -+ bfqd->queue_weights_tree = RB_ROOT; -+ bfqd->group_weights_tree = RB_ROOT; -+ -+ INIT_WORK(&bfqd->unplug_work, bfq_kick_queue); -+ -+ INIT_LIST_HEAD(&bfqd->active_list); -+ INIT_LIST_HEAD(&bfqd->idle_list); -+ INIT_HLIST_HEAD(&bfqd->burst_list); -+ -+ bfqd->hw_tag = -1; -+ -+ bfqd->bfq_max_budget = bfq_default_max_budget; -+ -+ bfqd->bfq_quantum = bfq_quantum; -+ bfqd->bfq_fifo_expire[0] = bfq_fifo_expire[0]; -+ bfqd->bfq_fifo_expire[1] = bfq_fifo_expire[1]; -+ bfqd->bfq_back_max = bfq_back_max; -+ bfqd->bfq_back_penalty = bfq_back_penalty; -+ bfqd->bfq_slice_idle = bfq_slice_idle; -+ bfqd->bfq_class_idle_last_service = 0; -+ bfqd->bfq_max_budget_async_rq = bfq_max_budget_async_rq; -+ bfqd->bfq_timeout[BLK_RW_ASYNC] = bfq_timeout_async; -+ bfqd->bfq_timeout[BLK_RW_SYNC] = bfq_timeout_sync; -+ -+ bfqd->bfq_coop_thresh = 2; -+ bfqd->bfq_failed_cooperations = 7000; -+ bfqd->bfq_requests_within_timer = 120; -+ -+ bfqd->bfq_large_burst_thresh = 11; -+ bfqd->bfq_burst_interval = msecs_to_jiffies(500); -+ -+ bfqd->low_latency = true; -+ -+ bfqd->bfq_wr_coeff = 20; -+ bfqd->bfq_wr_rt_max_time = msecs_to_jiffies(300); -+ bfqd->bfq_wr_max_time = 0; -+ bfqd->bfq_wr_min_idle_time = msecs_to_jiffies(2000); -+ bfqd->bfq_wr_min_inter_arr_async = msecs_to_jiffies(500); -+ bfqd->bfq_wr_max_softrt_rate = 7000; /* -+ * Approximate rate required -+ * to playback or record a -+ * high-definition compressed -+ * video. -+ */ -+ bfqd->wr_busy_queues = 0; -+ bfqd->busy_in_flight_queues = 0; -+ bfqd->const_seeky_busy_in_flight_queues = 0; -+ -+ /* -+ * Begin by assuming, optimistically, that the device peak rate is -+ * equal to the highest reference rate. -+ */ -+ bfqd->RT_prod = R_fast[blk_queue_nonrot(bfqd->queue)] * -+ T_fast[blk_queue_nonrot(bfqd->queue)]; -+ bfqd->peak_rate = R_fast[blk_queue_nonrot(bfqd->queue)]; -+ bfqd->device_speed = BFQ_BFQD_FAST; -+ -+ return 0; -+} -+ -+static void bfq_slab_kill(void) -+{ -+ if (bfq_pool != NULL) -+ kmem_cache_destroy(bfq_pool); -+} -+ -+static int __init bfq_slab_setup(void) -+{ -+ bfq_pool = KMEM_CACHE(bfq_queue, 0); -+ if (bfq_pool == NULL) -+ return -ENOMEM; -+ return 0; -+} -+ -+static ssize_t bfq_var_show(unsigned int var, char *page) -+{ -+ return sprintf(page, "%d\n", var); -+} -+ -+static ssize_t bfq_var_store(unsigned long *var, const char *page, -+ size_t count) -+{ -+ unsigned long new_val; -+ int ret = kstrtoul(page, 10, &new_val); -+ -+ if (ret == 0) -+ *var = new_val; -+ -+ return count; -+} -+ -+static ssize_t bfq_wr_max_time_show(struct elevator_queue *e, char *page) -+{ -+ struct bfq_data *bfqd = e->elevator_data; -+ return sprintf(page, "%d\n", bfqd->bfq_wr_max_time > 0 ? -+ jiffies_to_msecs(bfqd->bfq_wr_max_time) : -+ jiffies_to_msecs(bfq_wr_duration(bfqd))); -+} -+ -+static ssize_t bfq_weights_show(struct elevator_queue *e, char *page) -+{ -+ struct bfq_queue *bfqq; -+ struct bfq_data *bfqd = e->elevator_data; -+ ssize_t num_char = 0; -+ -+ num_char += sprintf(page + num_char, "Tot reqs queued %d\n\n", -+ bfqd->queued); -+ -+ spin_lock_irq(bfqd->queue->queue_lock); -+ -+ num_char += sprintf(page + num_char, "Active:\n"); -+ list_for_each_entry(bfqq, &bfqd->active_list, bfqq_list) { -+ num_char += sprintf(page + num_char, -+ "pid%d: weight %hu, nr_queued %d %d, dur %d/%u\n", -+ bfqq->pid, -+ bfqq->entity.weight, -+ bfqq->queued[0], -+ bfqq->queued[1], -+ jiffies_to_msecs(jiffies - bfqq->last_wr_start_finish), -+ jiffies_to_msecs(bfqq->wr_cur_max_time)); -+ } -+ -+ num_char += sprintf(page + num_char, "Idle:\n"); -+ list_for_each_entry(bfqq, &bfqd->idle_list, bfqq_list) { -+ num_char += sprintf(page + num_char, -+ "pid%d: weight %hu, dur %d/%u\n", -+ bfqq->pid, -+ bfqq->entity.weight, -+ jiffies_to_msecs(jiffies - -+ bfqq->last_wr_start_finish), -+ jiffies_to_msecs(bfqq->wr_cur_max_time)); -+ } -+ -+ spin_unlock_irq(bfqd->queue->queue_lock); -+ -+ return num_char; -+} -+ -+#define SHOW_FUNCTION(__FUNC, __VAR, __CONV) \ -+static ssize_t __FUNC(struct elevator_queue *e, char *page) \ -+{ \ -+ struct bfq_data *bfqd = e->elevator_data; \ -+ unsigned int __data = __VAR; \ -+ if (__CONV) \ -+ __data = jiffies_to_msecs(__data); \ -+ return bfq_var_show(__data, (page)); \ -+} -+SHOW_FUNCTION(bfq_quantum_show, bfqd->bfq_quantum, 0); -+SHOW_FUNCTION(bfq_fifo_expire_sync_show, bfqd->bfq_fifo_expire[1], 1); -+SHOW_FUNCTION(bfq_fifo_expire_async_show, bfqd->bfq_fifo_expire[0], 1); -+SHOW_FUNCTION(bfq_back_seek_max_show, bfqd->bfq_back_max, 0); -+SHOW_FUNCTION(bfq_back_seek_penalty_show, bfqd->bfq_back_penalty, 0); -+SHOW_FUNCTION(bfq_slice_idle_show, bfqd->bfq_slice_idle, 1); -+SHOW_FUNCTION(bfq_max_budget_show, bfqd->bfq_user_max_budget, 0); -+SHOW_FUNCTION(bfq_max_budget_async_rq_show, -+ bfqd->bfq_max_budget_async_rq, 0); -+SHOW_FUNCTION(bfq_timeout_sync_show, bfqd->bfq_timeout[BLK_RW_SYNC], 1); -+SHOW_FUNCTION(bfq_timeout_async_show, bfqd->bfq_timeout[BLK_RW_ASYNC], 1); -+SHOW_FUNCTION(bfq_low_latency_show, bfqd->low_latency, 0); -+SHOW_FUNCTION(bfq_wr_coeff_show, bfqd->bfq_wr_coeff, 0); -+SHOW_FUNCTION(bfq_wr_rt_max_time_show, bfqd->bfq_wr_rt_max_time, 1); -+SHOW_FUNCTION(bfq_wr_min_idle_time_show, bfqd->bfq_wr_min_idle_time, 1); -+SHOW_FUNCTION(bfq_wr_min_inter_arr_async_show, bfqd->bfq_wr_min_inter_arr_async, -+ 1); -+SHOW_FUNCTION(bfq_wr_max_softrt_rate_show, bfqd->bfq_wr_max_softrt_rate, 0); -+#undef SHOW_FUNCTION -+ -+#define STORE_FUNCTION(__FUNC, __PTR, MIN, MAX, __CONV) \ -+static ssize_t \ -+__FUNC(struct elevator_queue *e, const char *page, size_t count) \ -+{ \ -+ struct bfq_data *bfqd = e->elevator_data; \ -+ unsigned long uninitialized_var(__data); \ -+ int ret = bfq_var_store(&__data, (page), count); \ -+ if (__data < (MIN)) \ -+ __data = (MIN); \ -+ else if (__data > (MAX)) \ -+ __data = (MAX); \ -+ if (__CONV) \ -+ *(__PTR) = msecs_to_jiffies(__data); \ -+ else \ -+ *(__PTR) = __data; \ -+ return ret; \ -+} -+STORE_FUNCTION(bfq_quantum_store, &bfqd->bfq_quantum, 1, INT_MAX, 0); -+STORE_FUNCTION(bfq_fifo_expire_sync_store, &bfqd->bfq_fifo_expire[1], 1, -+ INT_MAX, 1); -+STORE_FUNCTION(bfq_fifo_expire_async_store, &bfqd->bfq_fifo_expire[0], 1, -+ INT_MAX, 1); -+STORE_FUNCTION(bfq_back_seek_max_store, &bfqd->bfq_back_max, 0, INT_MAX, 0); -+STORE_FUNCTION(bfq_back_seek_penalty_store, &bfqd->bfq_back_penalty, 1, -+ INT_MAX, 0); -+STORE_FUNCTION(bfq_slice_idle_store, &bfqd->bfq_slice_idle, 0, INT_MAX, 1); -+STORE_FUNCTION(bfq_max_budget_async_rq_store, &bfqd->bfq_max_budget_async_rq, -+ 1, INT_MAX, 0); -+STORE_FUNCTION(bfq_timeout_async_store, &bfqd->bfq_timeout[BLK_RW_ASYNC], 0, -+ INT_MAX, 1); -+STORE_FUNCTION(bfq_wr_coeff_store, &bfqd->bfq_wr_coeff, 1, INT_MAX, 0); -+STORE_FUNCTION(bfq_wr_max_time_store, &bfqd->bfq_wr_max_time, 0, INT_MAX, 1); -+STORE_FUNCTION(bfq_wr_rt_max_time_store, &bfqd->bfq_wr_rt_max_time, 0, INT_MAX, -+ 1); -+STORE_FUNCTION(bfq_wr_min_idle_time_store, &bfqd->bfq_wr_min_idle_time, 0, -+ INT_MAX, 1); -+STORE_FUNCTION(bfq_wr_min_inter_arr_async_store, -+ &bfqd->bfq_wr_min_inter_arr_async, 0, INT_MAX, 1); -+STORE_FUNCTION(bfq_wr_max_softrt_rate_store, &bfqd->bfq_wr_max_softrt_rate, 0, -+ INT_MAX, 0); -+#undef STORE_FUNCTION -+ -+/* do nothing for the moment */ -+static ssize_t bfq_weights_store(struct elevator_queue *e, -+ const char *page, size_t count) -+{ -+ return count; -+} -+ -+static inline unsigned long bfq_estimated_max_budget(struct bfq_data *bfqd) -+{ -+ u64 timeout = jiffies_to_msecs(bfqd->bfq_timeout[BLK_RW_SYNC]); -+ -+ if (bfqd->peak_rate_samples >= BFQ_PEAK_RATE_SAMPLES) -+ return bfq_calc_max_budget(bfqd->peak_rate, timeout); -+ else -+ return bfq_default_max_budget; -+} -+ -+static ssize_t bfq_max_budget_store(struct elevator_queue *e, -+ const char *page, size_t count) -+{ -+ struct bfq_data *bfqd = e->elevator_data; -+ unsigned long uninitialized_var(__data); -+ int ret = bfq_var_store(&__data, (page), count); -+ -+ if (__data == 0) -+ bfqd->bfq_max_budget = bfq_estimated_max_budget(bfqd); -+ else { -+ if (__data > INT_MAX) -+ __data = INT_MAX; -+ bfqd->bfq_max_budget = __data; -+ } -+ -+ bfqd->bfq_user_max_budget = __data; -+ -+ return ret; -+} -+ -+static ssize_t bfq_timeout_sync_store(struct elevator_queue *e, -+ const char *page, size_t count) -+{ -+ struct bfq_data *bfqd = e->elevator_data; -+ unsigned long uninitialized_var(__data); -+ int ret = bfq_var_store(&__data, (page), count); -+ -+ if (__data < 1) -+ __data = 1; -+ else if (__data > INT_MAX) -+ __data = INT_MAX; -+ -+ bfqd->bfq_timeout[BLK_RW_SYNC] = msecs_to_jiffies(__data); -+ if (bfqd->bfq_user_max_budget == 0) -+ bfqd->bfq_max_budget = bfq_estimated_max_budget(bfqd); -+ -+ return ret; -+} -+ -+static ssize_t bfq_low_latency_store(struct elevator_queue *e, -+ const char *page, size_t count) -+{ -+ struct bfq_data *bfqd = e->elevator_data; -+ unsigned long uninitialized_var(__data); -+ int ret = bfq_var_store(&__data, (page), count); -+ -+ if (__data > 1) -+ __data = 1; -+ if (__data == 0 && bfqd->low_latency != 0) -+ bfq_end_wr(bfqd); -+ bfqd->low_latency = __data; -+ -+ return ret; -+} -+ -+#define BFQ_ATTR(name) \ -+ __ATTR(name, S_IRUGO|S_IWUSR, bfq_##name##_show, bfq_##name##_store) -+ -+static struct elv_fs_entry bfq_attrs[] = { -+ BFQ_ATTR(quantum), -+ BFQ_ATTR(fifo_expire_sync), -+ BFQ_ATTR(fifo_expire_async), -+ BFQ_ATTR(back_seek_max), -+ BFQ_ATTR(back_seek_penalty), -+ BFQ_ATTR(slice_idle), -+ BFQ_ATTR(max_budget), -+ BFQ_ATTR(max_budget_async_rq), -+ BFQ_ATTR(timeout_sync), -+ BFQ_ATTR(timeout_async), -+ BFQ_ATTR(low_latency), -+ BFQ_ATTR(wr_coeff), -+ BFQ_ATTR(wr_max_time), -+ BFQ_ATTR(wr_rt_max_time), -+ BFQ_ATTR(wr_min_idle_time), -+ BFQ_ATTR(wr_min_inter_arr_async), -+ BFQ_ATTR(wr_max_softrt_rate), -+ BFQ_ATTR(weights), -+ __ATTR_NULL -+}; -+ -+static struct elevator_type iosched_bfq = { -+ .ops = { -+ .elevator_merge_fn = bfq_merge, -+ .elevator_merged_fn = bfq_merged_request, -+ .elevator_merge_req_fn = bfq_merged_requests, -+ .elevator_allow_merge_fn = bfq_allow_merge, -+ .elevator_dispatch_fn = bfq_dispatch_requests, -+ .elevator_add_req_fn = bfq_insert_request, -+ .elevator_activate_req_fn = bfq_activate_request, -+ .elevator_deactivate_req_fn = bfq_deactivate_request, -+ .elevator_completed_req_fn = bfq_completed_request, -+ .elevator_former_req_fn = elv_rb_former_request, -+ .elevator_latter_req_fn = elv_rb_latter_request, -+ .elevator_init_icq_fn = bfq_init_icq, -+ .elevator_exit_icq_fn = bfq_exit_icq, -+ .elevator_set_req_fn = bfq_set_request, -+ .elevator_put_req_fn = bfq_put_request, -+ .elevator_may_queue_fn = bfq_may_queue, -+ .elevator_init_fn = bfq_init_queue, -+ .elevator_exit_fn = bfq_exit_queue, -+ }, -+ .icq_size = sizeof(struct bfq_io_cq), -+ .icq_align = __alignof__(struct bfq_io_cq), -+ .elevator_attrs = bfq_attrs, -+ .elevator_name = "bfq", -+ .elevator_owner = THIS_MODULE, -+}; -+ -+static int __init bfq_init(void) -+{ -+ /* -+ * Can be 0 on HZ < 1000 setups. -+ */ -+ if (bfq_slice_idle == 0) -+ bfq_slice_idle = 1; -+ -+ if (bfq_timeout_async == 0) -+ bfq_timeout_async = 1; -+ -+ if (bfq_slab_setup()) -+ return -ENOMEM; -+ -+ /* -+ * Times to load large popular applications for the typical systems -+ * installed on the reference devices (see the comments before the -+ * definitions of the two arrays). -+ */ -+ T_slow[0] = msecs_to_jiffies(2600); -+ T_slow[1] = msecs_to_jiffies(1000); -+ T_fast[0] = msecs_to_jiffies(5500); -+ T_fast[1] = msecs_to_jiffies(2000); -+ -+ /* -+ * Thresholds that determine the switch between speed classes (see -+ * the comments before the definition of the array). -+ */ -+ device_speed_thresh[0] = (R_fast[0] + R_slow[0]) / 2; -+ device_speed_thresh[1] = (R_fast[1] + R_slow[1]) / 2; -+ -+ elv_register(&iosched_bfq); -+ pr_info("BFQ I/O-scheduler version: v7r7"); -+ -+ return 0; -+} -+ -+static void __exit bfq_exit(void) -+{ -+ elv_unregister(&iosched_bfq); -+ bfq_slab_kill(); -+} -+ -+module_init(bfq_init); -+module_exit(bfq_exit); -+ -+MODULE_AUTHOR("Fabio Checconi, Paolo Valente"); -+MODULE_LICENSE("GPL"); -diff -Nur linux-4.1.3/block/bfq-sched.c linux-xbian-imx6/block/bfq-sched.c ---- linux-4.1.3/block/bfq-sched.c 1970-01-01 01:00:00.000000000 +0100 -+++ linux-xbian-imx6/block/bfq-sched.c 2015-07-27 23:13:03.604123194 +0200 -@@ -0,0 +1,1186 @@ -+/* -+ * BFQ: Hierarchical B-WF2Q+ scheduler. -+ * -+ * Based on ideas and code from CFQ: -+ * Copyright (C) 2003 Jens Axboe -+ * -+ * Copyright (C) 2008 Fabio Checconi -+ * Paolo Valente -+ * -+ * Copyright (C) 2010 Paolo Valente -+ */ -+ -+#ifdef CONFIG_CGROUP_BFQIO -+#define for_each_entity(entity) \ -+ for (; entity != NULL; entity = entity->parent) -+ -+#define for_each_entity_safe(entity, parent) \ -+ for (; entity && ({ parent = entity->parent; 1; }); entity = parent) -+ -+static struct bfq_entity *bfq_lookup_next_entity(struct bfq_sched_data *sd, -+ int extract, -+ struct bfq_data *bfqd); -+ -+static inline void bfq_update_budget(struct bfq_entity *next_in_service) -+{ -+ struct bfq_entity *bfqg_entity; -+ struct bfq_group *bfqg; -+ struct bfq_sched_data *group_sd; -+ -+ BUG_ON(next_in_service == NULL); -+ -+ group_sd = next_in_service->sched_data; -+ -+ bfqg = container_of(group_sd, struct bfq_group, sched_data); -+ /* -+ * bfq_group's my_entity field is not NULL only if the group -+ * is not the root group. We must not touch the root entity -+ * as it must never become an in-service entity. -+ */ -+ bfqg_entity = bfqg->my_entity; -+ if (bfqg_entity != NULL) -+ bfqg_entity->budget = next_in_service->budget; -+} -+ -+static int bfq_update_next_in_service(struct bfq_sched_data *sd) -+{ -+ struct bfq_entity *next_in_service; -+ -+ if (sd->in_service_entity != NULL) -+ /* will update/requeue at the end of service */ -+ return 0; -+ -+ /* -+ * NOTE: this can be improved in many ways, such as returning -+ * 1 (and thus propagating upwards the update) only when the -+ * budget changes, or caching the bfqq that will be scheduled -+ * next from this subtree. By now we worry more about -+ * correctness than about performance... -+ */ -+ next_in_service = bfq_lookup_next_entity(sd, 0, NULL); -+ sd->next_in_service = next_in_service; -+ -+ if (next_in_service != NULL) -+ bfq_update_budget(next_in_service); -+ -+ return 1; -+} -+ -+static inline void bfq_check_next_in_service(struct bfq_sched_data *sd, -+ struct bfq_entity *entity) -+{ -+ BUG_ON(sd->next_in_service != entity); -+} -+#else -+#define for_each_entity(entity) \ -+ for (; entity != NULL; entity = NULL) -+ -+#define for_each_entity_safe(entity, parent) \ -+ for (parent = NULL; entity != NULL; entity = parent) -+ -+static inline int bfq_update_next_in_service(struct bfq_sched_data *sd) -+{ -+ return 0; -+} -+ -+static inline void bfq_check_next_in_service(struct bfq_sched_data *sd, -+ struct bfq_entity *entity) -+{ -+} -+ -+static inline void bfq_update_budget(struct bfq_entity *next_in_service) -+{ -+} -+#endif -+ -+/* -+ * Shift for timestamp calculations. This actually limits the maximum -+ * service allowed in one timestamp delta (small shift values increase it), -+ * the maximum total weight that can be used for the queues in the system -+ * (big shift values increase it), and the period of virtual time -+ * wraparounds. -+ */ -+#define WFQ_SERVICE_SHIFT 22 -+ -+/** -+ * bfq_gt - compare two timestamps. -+ * @a: first ts. -+ * @b: second ts. -+ * -+ * Return @a > @b, dealing with wrapping correctly. -+ */ -+static inline int bfq_gt(u64 a, u64 b) -+{ -+ return (s64)(a - b) > 0; -+} -+ -+static inline struct bfq_queue *bfq_entity_to_bfqq(struct bfq_entity *entity) -+{ -+ struct bfq_queue *bfqq = NULL; -+ -+ BUG_ON(entity == NULL); -+ -+ if (entity->my_sched_data == NULL) -+ bfqq = container_of(entity, struct bfq_queue, entity); -+ -+ return bfqq; -+} -+ -+ -+/** -+ * bfq_delta - map service into the virtual time domain. -+ * @service: amount of service. -+ * @weight: scale factor (weight of an entity or weight sum). -+ */ -+static inline u64 bfq_delta(unsigned long service, -+ unsigned long weight) -+{ -+ u64 d = (u64)service << WFQ_SERVICE_SHIFT; -+ -+ do_div(d, weight); -+ return d; -+} -+ -+/** -+ * bfq_calc_finish - assign the finish time to an entity. -+ * @entity: the entity to act upon. -+ * @service: the service to be charged to the entity. -+ */ -+static inline void bfq_calc_finish(struct bfq_entity *entity, -+ unsigned long service) -+{ -+ struct bfq_queue *bfqq = bfq_entity_to_bfqq(entity); -+ -+ BUG_ON(entity->weight == 0); -+ -+ entity->finish = entity->start + -+ bfq_delta(service, entity->weight); -+ -+ if (bfqq != NULL) { -+ bfq_log_bfqq(bfqq->bfqd, bfqq, -+ "calc_finish: serv %lu, w %d", -+ service, entity->weight); -+ bfq_log_bfqq(bfqq->bfqd, bfqq, -+ "calc_finish: start %llu, finish %llu, delta %llu", -+ entity->start, entity->finish, -+ bfq_delta(service, entity->weight)); -+ } -+} -+ -+/** -+ * bfq_entity_of - get an entity from a node. -+ * @node: the node field of the entity. -+ * -+ * Convert a node pointer to the relative entity. This is used only -+ * to simplify the logic of some functions and not as the generic -+ * conversion mechanism because, e.g., in the tree walking functions, -+ * the check for a %NULL value would be redundant. -+ */ -+static inline struct bfq_entity *bfq_entity_of(struct rb_node *node) -+{ -+ struct bfq_entity *entity = NULL; -+ -+ if (node != NULL) -+ entity = rb_entry(node, struct bfq_entity, rb_node); -+ -+ return entity; -+} -+ -+/** -+ * bfq_extract - remove an entity from a tree. -+ * @root: the tree root. -+ * @entity: the entity to remove. -+ */ -+static inline void bfq_extract(struct rb_root *root, -+ struct bfq_entity *entity) -+{ -+ BUG_ON(entity->tree != root); -+ -+ entity->tree = NULL; -+ rb_erase(&entity->rb_node, root); -+} -+ -+/** -+ * bfq_idle_extract - extract an entity from the idle tree. -+ * @st: the service tree of the owning @entity. -+ * @entity: the entity being removed. -+ */ -+static void bfq_idle_extract(struct bfq_service_tree *st, -+ struct bfq_entity *entity) -+{ -+ struct bfq_queue *bfqq = bfq_entity_to_bfqq(entity); -+ struct rb_node *next; -+ -+ BUG_ON(entity->tree != &st->idle); -+ -+ if (entity == st->first_idle) { -+ next = rb_next(&entity->rb_node); -+ st->first_idle = bfq_entity_of(next); -+ } -+ -+ if (entity == st->last_idle) { -+ next = rb_prev(&entity->rb_node); -+ st->last_idle = bfq_entity_of(next); -+ } -+ -+ bfq_extract(&st->idle, entity); -+ -+ if (bfqq != NULL) -+ list_del(&bfqq->bfqq_list); -+} -+ -+/** -+ * bfq_insert - generic tree insertion. -+ * @root: tree root. -+ * @entity: entity to insert. -+ * -+ * This is used for the idle and the active tree, since they are both -+ * ordered by finish time. -+ */ -+static void bfq_insert(struct rb_root *root, struct bfq_entity *entity) -+{ -+ struct bfq_entity *entry; -+ struct rb_node **node = &root->rb_node; -+ struct rb_node *parent = NULL; -+ -+ BUG_ON(entity->tree != NULL); -+ -+ while (*node != NULL) { -+ parent = *node; -+ entry = rb_entry(parent, struct bfq_entity, rb_node); -+ -+ if (bfq_gt(entry->finish, entity->finish)) -+ node = &parent->rb_left; -+ else -+ node = &parent->rb_right; -+ } -+ -+ rb_link_node(&entity->rb_node, parent, node); -+ rb_insert_color(&entity->rb_node, root); -+ -+ entity->tree = root; -+} -+ -+/** -+ * bfq_update_min - update the min_start field of a entity. -+ * @entity: the entity to update. -+ * @node: one of its children. -+ * -+ * This function is called when @entity may store an invalid value for -+ * min_start due to updates to the active tree. The function assumes -+ * that the subtree rooted at @node (which may be its left or its right -+ * child) has a valid min_start value. -+ */ -+static inline void bfq_update_min(struct bfq_entity *entity, -+ struct rb_node *node) -+{ -+ struct bfq_entity *child; -+ -+ if (node != NULL) { -+ child = rb_entry(node, struct bfq_entity, rb_node); -+ if (bfq_gt(entity->min_start, child->min_start)) -+ entity->min_start = child->min_start; -+ } -+} -+ -+/** -+ * bfq_update_active_node - recalculate min_start. -+ * @node: the node to update. -+ * -+ * @node may have changed position or one of its children may have moved, -+ * this function updates its min_start value. The left and right subtrees -+ * are assumed to hold a correct min_start value. -+ */ -+static inline void bfq_update_active_node(struct rb_node *node) -+{ -+ struct bfq_entity *entity = rb_entry(node, struct bfq_entity, rb_node); -+ -+ entity->min_start = entity->start; -+ bfq_update_min(entity, node->rb_right); -+ bfq_update_min(entity, node->rb_left); -+} -+ -+/** -+ * bfq_update_active_tree - update min_start for the whole active tree. -+ * @node: the starting node. -+ * -+ * @node must be the deepest modified node after an update. This function -+ * updates its min_start using the values held by its children, assuming -+ * that they did not change, and then updates all the nodes that may have -+ * changed in the path to the root. The only nodes that may have changed -+ * are the ones in the path or their siblings. -+ */ -+static void bfq_update_active_tree(struct rb_node *node) -+{ -+ struct rb_node *parent; -+ -+up: -+ bfq_update_active_node(node); -+ -+ parent = rb_parent(node); -+ if (parent == NULL) -+ return; -+ -+ if (node == parent->rb_left && parent->rb_right != NULL) -+ bfq_update_active_node(parent->rb_right); -+ else if (parent->rb_left != NULL) -+ bfq_update_active_node(parent->rb_left); -+ -+ node = parent; -+ goto up; -+} -+ -+static void bfq_weights_tree_add(struct bfq_data *bfqd, -+ struct bfq_entity *entity, -+ struct rb_root *root); -+ -+static void bfq_weights_tree_remove(struct bfq_data *bfqd, -+ struct bfq_entity *entity, -+ struct rb_root *root); -+ -+ -+/** -+ * bfq_active_insert - insert an entity in the active tree of its -+ * group/device. -+ * @st: the service tree of the entity. -+ * @entity: the entity being inserted. -+ * -+ * The active tree is ordered by finish time, but an extra key is kept -+ * per each node, containing the minimum value for the start times of -+ * its children (and the node itself), so it's possible to search for -+ * the eligible node with the lowest finish time in logarithmic time. -+ */ -+static void bfq_active_insert(struct bfq_service_tree *st, -+ struct bfq_entity *entity) -+{ -+ struct bfq_queue *bfqq = bfq_entity_to_bfqq(entity); -+ struct rb_node *node = &entity->rb_node; -+#ifdef CONFIG_CGROUP_BFQIO -+ struct bfq_sched_data *sd = NULL; -+ struct bfq_group *bfqg = NULL; -+ struct bfq_data *bfqd = NULL; -+#endif -+ -+ bfq_insert(&st->active, entity); -+ -+ if (node->rb_left != NULL) -+ node = node->rb_left; -+ else if (node->rb_right != NULL) -+ node = node->rb_right; -+ -+ bfq_update_active_tree(node); -+ -+#ifdef CONFIG_CGROUP_BFQIO -+ sd = entity->sched_data; -+ bfqg = container_of(sd, struct bfq_group, sched_data); -+ BUG_ON(!bfqg); -+ bfqd = (struct bfq_data *)bfqg->bfqd; -+#endif -+ if (bfqq != NULL) -+ list_add(&bfqq->bfqq_list, &bfqq->bfqd->active_list); -+#ifdef CONFIG_CGROUP_BFQIO -+ else { /* bfq_group */ -+ BUG_ON(!bfqd); -+ bfq_weights_tree_add(bfqd, entity, &bfqd->group_weights_tree); -+ } -+ if (bfqg != bfqd->root_group) { -+ BUG_ON(!bfqg); -+ BUG_ON(!bfqd); -+ bfqg->active_entities++; -+ if (bfqg->active_entities == 2) -+ bfqd->active_numerous_groups++; -+ } -+#endif -+} -+ -+/** -+ * bfq_ioprio_to_weight - calc a weight from an ioprio. -+ * @ioprio: the ioprio value to convert. -+ */ -+static inline unsigned short bfq_ioprio_to_weight(int ioprio) -+{ -+ BUG_ON(ioprio < 0 || ioprio >= IOPRIO_BE_NR); -+ return IOPRIO_BE_NR - ioprio; -+} -+ -+/** -+ * bfq_weight_to_ioprio - calc an ioprio from a weight. -+ * @weight: the weight value to convert. -+ * -+ * To preserve as mush as possible the old only-ioprio user interface, -+ * 0 is used as an escape ioprio value for weights (numerically) equal or -+ * larger than IOPRIO_BE_NR -+ */ -+static inline unsigned short bfq_weight_to_ioprio(int weight) -+{ -+ BUG_ON(weight < BFQ_MIN_WEIGHT || weight > BFQ_MAX_WEIGHT); -+ return IOPRIO_BE_NR - weight < 0 ? 0 : IOPRIO_BE_NR - weight; -+} -+ -+static inline void bfq_get_entity(struct bfq_entity *entity) -+{ -+ struct bfq_queue *bfqq = bfq_entity_to_bfqq(entity); -+ -+ if (bfqq != NULL) { -+ atomic_inc(&bfqq->ref); -+ bfq_log_bfqq(bfqq->bfqd, bfqq, "get_entity: %p %d", -+ bfqq, atomic_read(&bfqq->ref)); -+ } -+} -+ -+/** -+ * bfq_find_deepest - find the deepest node that an extraction can modify. -+ * @node: the node being removed. -+ * -+ * Do the first step of an extraction in an rb tree, looking for the -+ * node that will replace @node, and returning the deepest node that -+ * the following modifications to the tree can touch. If @node is the -+ * last node in the tree return %NULL. -+ */ -+static struct rb_node *bfq_find_deepest(struct rb_node *node) -+{ -+ struct rb_node *deepest; -+ -+ if (node->rb_right == NULL && node->rb_left == NULL) -+ deepest = rb_parent(node); -+ else if (node->rb_right == NULL) -+ deepest = node->rb_left; -+ else if (node->rb_left == NULL) -+ deepest = node->rb_right; -+ else { -+ deepest = rb_next(node); -+ if (deepest->rb_right != NULL) -+ deepest = deepest->rb_right; -+ else if (rb_parent(deepest) != node) -+ deepest = rb_parent(deepest); -+ } -+ -+ return deepest; -+} -+ -+/** -+ * bfq_active_extract - remove an entity from the active tree. -+ * @st: the service_tree containing the tree. -+ * @entity: the entity being removed. -+ */ -+static void bfq_active_extract(struct bfq_service_tree *st, -+ struct bfq_entity *entity) -+{ -+ struct bfq_queue *bfqq = bfq_entity_to_bfqq(entity); -+ struct rb_node *node; -+#ifdef CONFIG_CGROUP_BFQIO -+ struct bfq_sched_data *sd = NULL; -+ struct bfq_group *bfqg = NULL; -+ struct bfq_data *bfqd = NULL; -+#endif -+ -+ node = bfq_find_deepest(&entity->rb_node); -+ bfq_extract(&st->active, entity); -+ -+ if (node != NULL) -+ bfq_update_active_tree(node); -+ -+#ifdef CONFIG_CGROUP_BFQIO -+ sd = entity->sched_data; -+ bfqg = container_of(sd, struct bfq_group, sched_data); -+ BUG_ON(!bfqg); -+ bfqd = (struct bfq_data *)bfqg->bfqd; -+#endif -+ if (bfqq != NULL) -+ list_del(&bfqq->bfqq_list); -+#ifdef CONFIG_CGROUP_BFQIO -+ else { /* bfq_group */ -+ BUG_ON(!bfqd); -+ bfq_weights_tree_remove(bfqd, entity, -+ &bfqd->group_weights_tree); -+ } -+ if (bfqg != bfqd->root_group) { -+ BUG_ON(!bfqg); -+ BUG_ON(!bfqd); -+ BUG_ON(!bfqg->active_entities); -+ bfqg->active_entities--; -+ if (bfqg->active_entities == 1) { -+ BUG_ON(!bfqd->active_numerous_groups); -+ bfqd->active_numerous_groups--; -+ } -+ } -+#endif -+} -+ -+/** -+ * bfq_idle_insert - insert an entity into the idle tree. -+ * @st: the service tree containing the tree. -+ * @entity: the entity to insert. -+ */ -+static void bfq_idle_insert(struct bfq_service_tree *st, -+ struct bfq_entity *entity) -+{ -+ struct bfq_queue *bfqq = bfq_entity_to_bfqq(entity); -+ struct bfq_entity *first_idle = st->first_idle; -+ struct bfq_entity *last_idle = st->last_idle; -+ -+ if (first_idle == NULL || bfq_gt(first_idle->finish, entity->finish)) -+ st->first_idle = entity; -+ if (last_idle == NULL || bfq_gt(entity->finish, last_idle->finish)) -+ st->last_idle = entity; -+ -+ bfq_insert(&st->idle, entity); -+ -+ if (bfqq != NULL) -+ list_add(&bfqq->bfqq_list, &bfqq->bfqd->idle_list); -+} -+ -+/** -+ * bfq_forget_entity - remove an entity from the wfq trees. -+ * @st: the service tree. -+ * @entity: the entity being removed. -+ * -+ * Update the device status and forget everything about @entity, putting -+ * the device reference to it, if it is a queue. Entities belonging to -+ * groups are not refcounted. -+ */ -+static void bfq_forget_entity(struct bfq_service_tree *st, -+ struct bfq_entity *entity) -+{ -+ struct bfq_queue *bfqq = bfq_entity_to_bfqq(entity); -+ struct bfq_sched_data *sd; -+ -+ BUG_ON(!entity->on_st); -+ -+ entity->on_st = 0; -+ st->wsum -= entity->weight; -+ if (bfqq != NULL) { -+ sd = entity->sched_data; -+ bfq_log_bfqq(bfqq->bfqd, bfqq, "forget_entity: %p %d", -+ bfqq, atomic_read(&bfqq->ref)); -+ bfq_put_queue(bfqq); -+ } -+} -+ -+/** -+ * bfq_put_idle_entity - release the idle tree ref of an entity. -+ * @st: service tree for the entity. -+ * @entity: the entity being released. -+ */ -+static void bfq_put_idle_entity(struct bfq_service_tree *st, -+ struct bfq_entity *entity) -+{ -+ bfq_idle_extract(st, entity); -+ bfq_forget_entity(st, entity); -+} -+ -+/** -+ * bfq_forget_idle - update the idle tree if necessary. -+ * @st: the service tree to act upon. -+ * -+ * To preserve the global O(log N) complexity we only remove one entry here; -+ * as the idle tree will not grow indefinitely this can be done safely. -+ */ -+static void bfq_forget_idle(struct bfq_service_tree *st) -+{ -+ struct bfq_entity *first_idle = st->first_idle; -+ struct bfq_entity *last_idle = st->last_idle; -+ -+ if (RB_EMPTY_ROOT(&st->active) && last_idle != NULL && -+ !bfq_gt(last_idle->finish, st->vtime)) { -+ /* -+ * Forget the whole idle tree, increasing the vtime past -+ * the last finish time of idle entities. -+ */ -+ st->vtime = last_idle->finish; -+ } -+ -+ if (first_idle != NULL && !bfq_gt(first_idle->finish, st->vtime)) -+ bfq_put_idle_entity(st, first_idle); -+} -+ -+static struct bfq_service_tree * -+__bfq_entity_update_weight_prio(struct bfq_service_tree *old_st, -+ struct bfq_entity *entity) -+{ -+ struct bfq_service_tree *new_st = old_st; -+ -+ if (entity->ioprio_changed) { -+ struct bfq_queue *bfqq = bfq_entity_to_bfqq(entity); -+ unsigned short prev_weight, new_weight; -+ struct bfq_data *bfqd = NULL; -+ struct rb_root *root; -+#ifdef CONFIG_CGROUP_BFQIO -+ struct bfq_sched_data *sd; -+ struct bfq_group *bfqg; -+#endif -+ -+ if (bfqq != NULL) -+ bfqd = bfqq->bfqd; -+#ifdef CONFIG_CGROUP_BFQIO -+ else { -+ sd = entity->my_sched_data; -+ bfqg = container_of(sd, struct bfq_group, sched_data); -+ BUG_ON(!bfqg); -+ bfqd = (struct bfq_data *)bfqg->bfqd; -+ BUG_ON(!bfqd); -+ } -+#endif -+ -+ BUG_ON(old_st->wsum < entity->weight); -+ old_st->wsum -= entity->weight; -+ -+ if (entity->new_weight != entity->orig_weight) { -+ if (entity->new_weight < BFQ_MIN_WEIGHT || -+ entity->new_weight > BFQ_MAX_WEIGHT) { -+ printk(KERN_CRIT "update_weight_prio: " -+ "new_weight %d\n", -+ entity->new_weight); -+ BUG(); -+ } -+ entity->orig_weight = entity->new_weight; -+ entity->ioprio = -+ bfq_weight_to_ioprio(entity->orig_weight); -+ } else if (entity->new_ioprio != entity->ioprio) { -+ entity->ioprio = entity->new_ioprio; -+ entity->orig_weight = -+ bfq_ioprio_to_weight(entity->ioprio); -+ } else -+ entity->new_weight = entity->orig_weight = -+ bfq_ioprio_to_weight(entity->ioprio); -+ -+ entity->ioprio_class = entity->new_ioprio_class; -+ entity->ioprio_changed = 0; -+ -+ /* -+ * NOTE: here we may be changing the weight too early, -+ * this will cause unfairness. The correct approach -+ * would have required additional complexity to defer -+ * weight changes to the proper time instants (i.e., -+ * when entity->finish <= old_st->vtime). -+ */ -+ new_st = bfq_entity_service_tree(entity); -+ -+ prev_weight = entity->weight; -+ new_weight = entity->orig_weight * -+ (bfqq != NULL ? bfqq->wr_coeff : 1); -+ /* -+ * If the weight of the entity changes, remove the entity -+ * from its old weight counter (if there is a counter -+ * associated with the entity), and add it to the counter -+ * associated with its new weight. -+ */ -+ if (prev_weight != new_weight) { -+ root = bfqq ? &bfqd->queue_weights_tree : -+ &bfqd->group_weights_tree; -+ bfq_weights_tree_remove(bfqd, entity, root); -+ } -+ entity->weight = new_weight; -+ /* -+ * Add the entity to its weights tree only if it is -+ * not associated with a weight-raised queue. -+ */ -+ if (prev_weight != new_weight && -+ (bfqq ? bfqq->wr_coeff == 1 : 1)) -+ /* If we get here, root has been initialized. */ -+ bfq_weights_tree_add(bfqd, entity, root); -+ -+ new_st->wsum += entity->weight; -+ -+ if (new_st != old_st) -+ entity->start = new_st->vtime; -+ } -+ -+ return new_st; -+} -+ -+/** -+ * bfq_bfqq_served - update the scheduler status after selection for -+ * service. -+ * @bfqq: the queue being served. -+ * @served: bytes to transfer. -+ * -+ * NOTE: this can be optimized, as the timestamps of upper level entities -+ * are synchronized every time a new bfqq is selected for service. By now, -+ * we keep it to better check consistency. -+ */ -+static void bfq_bfqq_served(struct bfq_queue *bfqq, unsigned long served) -+{ -+ struct bfq_entity *entity = &bfqq->entity; -+ struct bfq_service_tree *st; -+ -+ for_each_entity(entity) { -+ st = bfq_entity_service_tree(entity); -+ -+ entity->service += served; -+ BUG_ON(entity->service > entity->budget); -+ BUG_ON(st->wsum == 0); -+ -+ st->vtime += bfq_delta(served, st->wsum); -+ bfq_forget_idle(st); -+ } -+ bfq_log_bfqq(bfqq->bfqd, bfqq, "bfqq_served %lu secs", served); -+} -+ -+/** -+ * bfq_bfqq_charge_full_budget - set the service to the entity budget. -+ * @bfqq: the queue that needs a service update. -+ * -+ * When it's not possible to be fair in the service domain, because -+ * a queue is not consuming its budget fast enough (the meaning of -+ * fast depends on the timeout parameter), we charge it a full -+ * budget. In this way we should obtain a sort of time-domain -+ * fairness among all the seeky/slow queues. -+ */ -+static inline void bfq_bfqq_charge_full_budget(struct bfq_queue *bfqq) -+{ -+ struct bfq_entity *entity = &bfqq->entity; -+ -+ bfq_log_bfqq(bfqq->bfqd, bfqq, "charge_full_budget"); -+ -+ bfq_bfqq_served(bfqq, entity->budget - entity->service); -+} -+ -+/** -+ * __bfq_activate_entity - activate an entity. -+ * @entity: the entity being activated. -+ * -+ * Called whenever an entity is activated, i.e., it is not active and one -+ * of its children receives a new request, or has to be reactivated due to -+ * budget exhaustion. It uses the current budget of the entity (and the -+ * service received if @entity is active) of the queue to calculate its -+ * timestamps. -+ */ -+static void __bfq_activate_entity(struct bfq_entity *entity) -+{ -+ struct bfq_sched_data *sd = entity->sched_data; -+ struct bfq_service_tree *st = bfq_entity_service_tree(entity); -+ -+ if (entity == sd->in_service_entity) { -+ BUG_ON(entity->tree != NULL); -+ /* -+ * If we are requeueing the current entity we have -+ * to take care of not charging to it service it has -+ * not received. -+ */ -+ bfq_calc_finish(entity, entity->service); -+ entity->start = entity->finish; -+ sd->in_service_entity = NULL; -+ } else if (entity->tree == &st->active) { -+ /* -+ * Requeueing an entity due to a change of some -+ * next_in_service entity below it. We reuse the -+ * old start time. -+ */ -+ bfq_active_extract(st, entity); -+ } else if (entity->tree == &st->idle) { -+ /* -+ * Must be on the idle tree, bfq_idle_extract() will -+ * check for that. -+ */ -+ bfq_idle_extract(st, entity); -+ entity->start = bfq_gt(st->vtime, entity->finish) ? -+ st->vtime : entity->finish; -+ } else { -+ /* -+ * The finish time of the entity may be invalid, and -+ * it is in the past for sure, otherwise the queue -+ * would have been on the idle tree. -+ */ -+ entity->start = st->vtime; -+ st->wsum += entity->weight; -+ bfq_get_entity(entity); -+ -+ BUG_ON(entity->on_st); -+ entity->on_st = 1; -+ } -+ -+ st = __bfq_entity_update_weight_prio(st, entity); -+ bfq_calc_finish(entity, entity->budget); -+ bfq_active_insert(st, entity); -+} -+ -+/** -+ * bfq_activate_entity - activate an entity and its ancestors if necessary. -+ * @entity: the entity to activate. -+ * -+ * Activate @entity and all the entities on the path from it to the root. -+ */ -+static void bfq_activate_entity(struct bfq_entity *entity) -+{ -+ struct bfq_sched_data *sd; -+ -+ for_each_entity(entity) { -+ __bfq_activate_entity(entity); -+ -+ sd = entity->sched_data; -+ if (!bfq_update_next_in_service(sd)) -+ /* -+ * No need to propagate the activation to the -+ * upper entities, as they will be updated when -+ * the in-service entity is rescheduled. -+ */ -+ break; -+ } -+} -+ -+/** -+ * __bfq_deactivate_entity - deactivate an entity from its service tree. -+ * @entity: the entity to deactivate. -+ * @requeue: if false, the entity will not be put into the idle tree. -+ * -+ * Deactivate an entity, independently from its previous state. If the -+ * entity was not on a service tree just return, otherwise if it is on -+ * any scheduler tree, extract it from that tree, and if necessary -+ * and if the caller did not specify @requeue, put it on the idle tree. -+ * -+ * Return %1 if the caller should update the entity hierarchy, i.e., -+ * if the entity was in service or if it was the next_in_service for -+ * its sched_data; return %0 otherwise. -+ */ -+static int __bfq_deactivate_entity(struct bfq_entity *entity, int requeue) -+{ -+ struct bfq_sched_data *sd = entity->sched_data; -+ struct bfq_service_tree *st = bfq_entity_service_tree(entity); -+ int was_in_service = entity == sd->in_service_entity; -+ int ret = 0; -+ -+ if (!entity->on_st) -+ return 0; -+ -+ BUG_ON(was_in_service && entity->tree != NULL); -+ -+ if (was_in_service) { -+ bfq_calc_finish(entity, entity->service); -+ sd->in_service_entity = NULL; -+ } else if (entity->tree == &st->active) -+ bfq_active_extract(st, entity); -+ else if (entity->tree == &st->idle) -+ bfq_idle_extract(st, entity); -+ else if (entity->tree != NULL) -+ BUG(); -+ -+ if (was_in_service || sd->next_in_service == entity) -+ ret = bfq_update_next_in_service(sd); -+ -+ if (!requeue || !bfq_gt(entity->finish, st->vtime)) -+ bfq_forget_entity(st, entity); -+ else -+ bfq_idle_insert(st, entity); -+ -+ BUG_ON(sd->in_service_entity == entity); -+ BUG_ON(sd->next_in_service == entity); -+ -+ return ret; -+} -+ -+/** -+ * bfq_deactivate_entity - deactivate an entity. -+ * @entity: the entity to deactivate. -+ * @requeue: true if the entity can be put on the idle tree -+ */ -+static void bfq_deactivate_entity(struct bfq_entity *entity, int requeue) -+{ -+ struct bfq_sched_data *sd; -+ struct bfq_entity *parent; -+ -+ for_each_entity_safe(entity, parent) { -+ sd = entity->sched_data; -+ -+ if (!__bfq_deactivate_entity(entity, requeue)) -+ /* -+ * The parent entity is still backlogged, and -+ * we don't need to update it as it is still -+ * in service. -+ */ -+ break; -+ -+ if (sd->next_in_service != NULL) -+ /* -+ * The parent entity is still backlogged and -+ * the budgets on the path towards the root -+ * need to be updated. -+ */ -+ goto update; -+ -+ /* -+ * If we reach there the parent is no more backlogged and -+ * we want to propagate the dequeue upwards. -+ */ -+ requeue = 1; -+ } -+ -+ return; -+ -+update: -+ entity = parent; -+ for_each_entity(entity) { -+ __bfq_activate_entity(entity); -+ -+ sd = entity->sched_data; -+ if (!bfq_update_next_in_service(sd)) -+ break; -+ } -+} -+ -+/** -+ * bfq_update_vtime - update vtime if necessary. -+ * @st: the service tree to act upon. -+ * -+ * If necessary update the service tree vtime to have at least one -+ * eligible entity, skipping to its start time. Assumes that the -+ * active tree of the device is not empty. -+ * -+ * NOTE: this hierarchical implementation updates vtimes quite often, -+ * we may end up with reactivated processes getting timestamps after a -+ * vtime skip done because we needed a ->first_active entity on some -+ * intermediate node. -+ */ -+static void bfq_update_vtime(struct bfq_service_tree *st) -+{ -+ struct bfq_entity *entry; -+ struct rb_node *node = st->active.rb_node; -+ -+ entry = rb_entry(node, struct bfq_entity, rb_node); -+ if (bfq_gt(entry->min_start, st->vtime)) { -+ st->vtime = entry->min_start; -+ bfq_forget_idle(st); -+ } -+} -+ -+/** -+ * bfq_first_active_entity - find the eligible entity with -+ * the smallest finish time -+ * @st: the service tree to select from. -+ * -+ * This function searches the first schedulable entity, starting from the -+ * root of the tree and going on the left every time on this side there is -+ * a subtree with at least one eligible (start >= vtime) entity. The path on -+ * the right is followed only if a) the left subtree contains no eligible -+ * entities and b) no eligible entity has been found yet. -+ */ -+static struct bfq_entity *bfq_first_active_entity(struct bfq_service_tree *st) -+{ -+ struct bfq_entity *entry, *first = NULL; -+ struct rb_node *node = st->active.rb_node; -+ -+ while (node != NULL) { -+ entry = rb_entry(node, struct bfq_entity, rb_node); -+left: -+ if (!bfq_gt(entry->start, st->vtime)) -+ first = entry; -+ -+ BUG_ON(bfq_gt(entry->min_start, st->vtime)); -+ -+ if (node->rb_left != NULL) { -+ entry = rb_entry(node->rb_left, -+ struct bfq_entity, rb_node); -+ if (!bfq_gt(entry->min_start, st->vtime)) { -+ node = node->rb_left; -+ goto left; -+ } -+ } -+ if (first != NULL) -+ break; -+ node = node->rb_right; -+ } -+ -+ BUG_ON(first == NULL && !RB_EMPTY_ROOT(&st->active)); -+ return first; -+} -+ -+/** -+ * __bfq_lookup_next_entity - return the first eligible entity in @st. -+ * @st: the service tree. -+ * -+ * Update the virtual time in @st and return the first eligible entity -+ * it contains. -+ */ -+static struct bfq_entity *__bfq_lookup_next_entity(struct bfq_service_tree *st, -+ bool force) -+{ -+ struct bfq_entity *entity, *new_next_in_service = NULL; -+ -+ if (RB_EMPTY_ROOT(&st->active)) -+ return NULL; -+ -+ bfq_update_vtime(st); -+ entity = bfq_first_active_entity(st); -+ BUG_ON(bfq_gt(entity->start, st->vtime)); -+ -+ /* -+ * If the chosen entity does not match with the sched_data's -+ * next_in_service and we are forcedly serving the IDLE priority -+ * class tree, bubble up budget update. -+ */ -+ if (unlikely(force && entity != entity->sched_data->next_in_service)) { -+ new_next_in_service = entity; -+ for_each_entity(new_next_in_service) -+ bfq_update_budget(new_next_in_service); -+ } -+ -+ return entity; -+} -+ -+/** -+ * bfq_lookup_next_entity - return the first eligible entity in @sd. -+ * @sd: the sched_data. -+ * @extract: if true the returned entity will be also extracted from @sd. -+ * -+ * NOTE: since we cache the next_in_service entity at each level of the -+ * hierarchy, the complexity of the lookup can be decreased with -+ * absolutely no effort just returning the cached next_in_service value; -+ * we prefer to do full lookups to test the consistency of * the data -+ * structures. -+ */ -+static struct bfq_entity *bfq_lookup_next_entity(struct bfq_sched_data *sd, -+ int extract, -+ struct bfq_data *bfqd) -+{ -+ struct bfq_service_tree *st = sd->service_tree; -+ struct bfq_entity *entity; -+ int i = 0; -+ -+ BUG_ON(sd->in_service_entity != NULL); -+ -+ if (bfqd != NULL && -+ jiffies - bfqd->bfq_class_idle_last_service > BFQ_CL_IDLE_TIMEOUT) { -+ entity = __bfq_lookup_next_entity(st + BFQ_IOPRIO_CLASSES - 1, -+ true); -+ if (entity != NULL) { -+ i = BFQ_IOPRIO_CLASSES - 1; -+ bfqd->bfq_class_idle_last_service = jiffies; -+ sd->next_in_service = entity; -+ } -+ } -+ for (; i < BFQ_IOPRIO_CLASSES; i++) { -+ entity = __bfq_lookup_next_entity(st + i, false); -+ if (entity != NULL) { -+ if (extract) { -+ bfq_check_next_in_service(sd, entity); -+ bfq_active_extract(st + i, entity); -+ sd->in_service_entity = entity; -+ sd->next_in_service = NULL; -+ } -+ break; -+ } -+ } -+ -+ return entity; -+} -+ -+/* -+ * Get next queue for service. -+ */ -+static struct bfq_queue *bfq_get_next_queue(struct bfq_data *bfqd) -+{ -+ struct bfq_entity *entity = NULL; -+ struct bfq_sched_data *sd; -+ struct bfq_queue *bfqq; -+ -+ BUG_ON(bfqd->in_service_queue != NULL); -+ -+ if (bfqd->busy_queues == 0) -+ return NULL; -+ -+ sd = &bfqd->root_group->sched_data; -+ for (; sd != NULL; sd = entity->my_sched_data) { -+ entity = bfq_lookup_next_entity(sd, 1, bfqd); -+ BUG_ON(entity == NULL); -+ entity->service = 0; -+ } -+ -+ bfqq = bfq_entity_to_bfqq(entity); -+ BUG_ON(bfqq == NULL); -+ -+ return bfqq; -+} -+ -+static void __bfq_bfqd_reset_in_service(struct bfq_data *bfqd) -+{ -+ if (bfqd->in_service_bic != NULL) { -+ put_io_context(bfqd->in_service_bic->icq.ioc); -+ bfqd->in_service_bic = NULL; -+ } -+ -+ bfqd->in_service_queue = NULL; -+ del_timer(&bfqd->idle_slice_timer); -+} -+ -+static void bfq_deactivate_bfqq(struct bfq_data *bfqd, struct bfq_queue *bfqq, -+ int requeue) -+{ -+ struct bfq_entity *entity = &bfqq->entity; -+ -+ if (bfqq == bfqd->in_service_queue) -+ __bfq_bfqd_reset_in_service(bfqd); -+ -+ bfq_deactivate_entity(entity, requeue); -+} -+ -+static void bfq_activate_bfqq(struct bfq_data *bfqd, struct bfq_queue *bfqq) -+{ -+ struct bfq_entity *entity = &bfqq->entity; -+ -+ bfq_activate_entity(entity); -+} -+ -+/* -+ * Called when the bfqq no longer has requests pending, remove it from -+ * the service tree. -+ */ -+static void bfq_del_bfqq_busy(struct bfq_data *bfqd, struct bfq_queue *bfqq, -+ int requeue) -+{ -+ BUG_ON(!bfq_bfqq_busy(bfqq)); -+ BUG_ON(!RB_EMPTY_ROOT(&bfqq->sort_list)); -+ -+ bfq_log_bfqq(bfqd, bfqq, "del from busy"); -+ -+ bfq_clear_bfqq_busy(bfqq); -+ -+ BUG_ON(bfqd->busy_queues == 0); -+ bfqd->busy_queues--; -+ -+ if (!bfqq->dispatched) { -+ bfq_weights_tree_remove(bfqd, &bfqq->entity, -+ &bfqd->queue_weights_tree); -+ if (!blk_queue_nonrot(bfqd->queue)) { -+ BUG_ON(!bfqd->busy_in_flight_queues); -+ bfqd->busy_in_flight_queues--; -+ if (bfq_bfqq_constantly_seeky(bfqq)) { -+ BUG_ON(!bfqd-> -+ const_seeky_busy_in_flight_queues); -+ bfqd->const_seeky_busy_in_flight_queues--; -+ } -+ } -+ } -+ if (bfqq->wr_coeff > 1) -+ bfqd->wr_busy_queues--; -+ -+ bfq_deactivate_bfqq(bfqd, bfqq, requeue); -+} -+ -+/* -+ * Called when an inactive queue receives a new request. -+ */ -+static void bfq_add_bfqq_busy(struct bfq_data *bfqd, struct bfq_queue *bfqq) -+{ -+ BUG_ON(bfq_bfqq_busy(bfqq)); -+ BUG_ON(bfqq == bfqd->in_service_queue); -+ -+ bfq_log_bfqq(bfqd, bfqq, "add to busy"); -+ -+ bfq_activate_bfqq(bfqd, bfqq); -+ -+ bfq_mark_bfqq_busy(bfqq); -+ bfqd->busy_queues++; -+ -+ if (!bfqq->dispatched) { -+ if (bfqq->wr_coeff == 1) -+ bfq_weights_tree_add(bfqd, &bfqq->entity, -+ &bfqd->queue_weights_tree); -+ if (!blk_queue_nonrot(bfqd->queue)) { -+ bfqd->busy_in_flight_queues++; -+ if (bfq_bfqq_constantly_seeky(bfqq)) -+ bfqd->const_seeky_busy_in_flight_queues++; -+ } -+ } -+ if (bfqq->wr_coeff > 1) -+ bfqd->wr_busy_queues++; -+} -diff -Nur linux-4.1.3/block/Kconfig.iosched linux-xbian-imx6/block/Kconfig.iosched ---- linux-4.1.3/block/Kconfig.iosched 2015-07-21 19:10:33.000000000 +0200 -+++ linux-xbian-imx6/block/Kconfig.iosched 2015-07-27 23:13:03.600137415 +0200 -@@ -39,6 +39,27 @@ - ---help--- - Enable group IO scheduling in CFQ. - -+config IOSCHED_BFQ -+ tristate "BFQ I/O scheduler" -+ default n -+ ---help--- -+ The BFQ I/O scheduler tries to distribute bandwidth among -+ all processes according to their weights. -+ It aims at distributing the bandwidth as desired, independently of -+ the disk parameters and with any workload. It also tries to -+ guarantee low latency to interactive and soft real-time -+ applications. If compiled built-in (saying Y here), BFQ can -+ be configured to support hierarchical scheduling. -+ -+config CGROUP_BFQIO -+ bool "BFQ hierarchical scheduling support" -+ depends on CGROUPS && IOSCHED_BFQ=y -+ default n -+ ---help--- -+ Enable hierarchical scheduling in BFQ, using the cgroups -+ filesystem interface. The name of the subsystem will be -+ bfqio. -+ - choice - prompt "Default I/O scheduler" - default DEFAULT_CFQ -@@ -52,6 +73,16 @@ - config DEFAULT_CFQ - bool "CFQ" if IOSCHED_CFQ=y - -+ config DEFAULT_BFQ -+ bool "BFQ" if IOSCHED_BFQ=y -+ help -+ Selects BFQ as the default I/O scheduler which will be -+ used by default for all block devices. -+ The BFQ I/O scheduler aims at distributing the bandwidth -+ as desired, independently of the disk parameters and with -+ any workload. It also tries to guarantee low latency to -+ interactive and soft real-time applications. -+ - config DEFAULT_NOOP - bool "No-op" - -@@ -61,6 +92,7 @@ - string - default "deadline" if DEFAULT_DEADLINE - default "cfq" if DEFAULT_CFQ -+ default "bfq" if DEFAULT_BFQ - default "noop" if DEFAULT_NOOP - - endmenu -diff -Nur linux-4.1.3/block/Makefile linux-xbian-imx6/block/Makefile ---- linux-4.1.3/block/Makefile 2015-07-21 19:10:33.000000000 +0200 -+++ linux-xbian-imx6/block/Makefile 2015-07-27 23:13:03.600137415 +0200 -@@ -18,6 +18,7 @@ - obj-$(CONFIG_IOSCHED_NOOP) += noop-iosched.o - obj-$(CONFIG_IOSCHED_DEADLINE) += deadline-iosched.o - obj-$(CONFIG_IOSCHED_CFQ) += cfq-iosched.o -+obj-$(CONFIG_IOSCHED_BFQ) += bfq-iosched.o - - obj-$(CONFIG_BLOCK_COMPAT) += compat_ioctl.o - obj-$(CONFIG_BLK_CMDLINE_PARSER) += cmdline-parser.o -diff -Nur linux-4.1.3/Documentation/devicetree/bindings/mmc/mmc.txt linux-xbian-imx6/Documentation/devicetree/bindings/mmc/mmc.txt ---- linux-4.1.3/Documentation/devicetree/bindings/mmc/mmc.txt 2015-07-21 19:10:33.000000000 +0200 -+++ linux-xbian-imx6/Documentation/devicetree/bindings/mmc/mmc.txt 2015-07-27 23:12:59.335353684 +0200 -@@ -5,6 +5,8 @@ - Interpreted by the OF core: - - reg: Registers location and length. - - interrupts: Interrupts used by the MMC controller. -+- clocks: Clocks needed for the host controller, if any. -+- clock-names: Goes with clocks above. - - Card detection: - If no property below is supplied, host native card detect is used. -@@ -43,6 +45,15 @@ - - dsr: Value the card's (optional) Driver Stage Register (DSR) should be - programmed with. Valid range: [0 .. 0xffff]. - -+Card power and reset control: -+The following properties can be specified for cases where the MMC -+peripheral needs additional reset, regulator and clock lines. It is for -+example common for WiFi/BT adapters to have these separate from the main -+MMC bus: -+ - card-reset-gpios: Specify GPIOs for card reset (reset active low) -+ - card-external-vcc-supply: Regulator to drive (independent) card VCC -+ - clock with name "card_ext_clock": External clock provided to the card -+ - *NOTE* on CD and WP polarity. To use common for all SD/MMC host controllers line - polarity properties, we have to fix the meaning of the "normal" and "inverted" - line levels. We choose to follow the SDHCI standard, which specifies both those -diff -Nur linux-4.1.3/Documentation/devicetree/bindings/vendor-prefixes.txt linux-xbian-imx6/Documentation/devicetree/bindings/vendor-prefixes.txt ---- linux-4.1.3/Documentation/devicetree/bindings/vendor-prefixes.txt 2015-07-21 19:10:33.000000000 +0200 -+++ linux-xbian-imx6/Documentation/devicetree/bindings/vendor-prefixes.txt 2015-07-27 23:12:59.518699524 +0200 -@@ -201,6 +201,7 @@ - variscite Variscite Ltd. - via VIA Technologies, Inc. - virtio Virtual I/O Device Specification, developed by the OASIS consortium -+vivante Vivante Corporation - voipac Voipac Technologies s.r.o. - winbond Winbond Electronics corp. - wlf Wolfson Microelectronics -diff -Nur linux-4.1.3/drivers/char/frandom.c linux-xbian-imx6/drivers/char/frandom.c ---- linux-4.1.3/drivers/char/frandom.c 1970-01-01 01:00:00.000000000 +0100 -+++ linux-xbian-imx6/drivers/char/frandom.c 2015-07-27 23:13:03.986757999 +0200 -@@ -0,0 +1,415 @@ -+/* -+** frandom.c -+** Fast pseudo-random generator -+** -+** (c) Copyright 2003-2011 Eli Billauer -+** http://www.billauer.co.il -+** -+** This program is free software; you can redistribute it and/or modify -+** it under the terms of the GNU General Public License as published by -+** the Free Software Foundation; either version 2 of the License, or -+** (at your option) any later version. -+** -+** -+*/ -+ -+#include -+#include -+#include -+ -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+ -+#include -+#include -+#include -+#include -+ -+#define INTERNAL_SEED 0 -+#define EXTERNAL_SEED 1 -+ -+#define FRANDOM_MAJOR 235 -+#define FRANDOM_MINOR 11 -+#define ERANDOM_MINOR 12 -+ -+static struct file_operations frandom_fops; /* Values assigned below */ -+ -+static int erandom_seeded = 0; /* Internal flag */ -+ -+static int frandom_major = FRANDOM_MAJOR; -+static int frandom_minor = FRANDOM_MINOR; -+static int erandom_minor = ERANDOM_MINOR; -+static int frandom_bufsize = 256; -+static int frandom_chunklimit = 0; /* =0 means unlimited */ -+ -+static struct cdev frandom_cdev; -+static struct cdev erandom_cdev; -+static struct class *frandom_class; -+struct device *frandom_device; -+struct device *erandom_device; -+ -+MODULE_DESCRIPTION("Fast pseudo-random number generator"); -+MODULE_LICENSE("GPL"); -+MODULE_AUTHOR("Eli Billauer"); -+module_param(frandom_major, int, 0); -+module_param(frandom_minor, int, 0); -+module_param(erandom_minor, int, 0); -+module_param(frandom_bufsize, int, 0); -+module_param(frandom_chunklimit, int, 0); -+ -+MODULE_PARM_DESC(frandom_major,"Major number of /dev/frandom and /dev/erandom"); -+MODULE_PARM_DESC(frandom_minor,"Minor number of /dev/frandom"); -+MODULE_PARM_DESC(erandom_minor,"Minor number of /dev/erandom"); -+MODULE_PARM_DESC(frandom_bufsize,"Internal buffer size in bytes. Default is 256. Must be >= 256"); -+MODULE_PARM_DESC(frandom_chunklimit,"Limit for read() blocks size. 0 (default) is unlimited, otherwise must be >= 256"); -+ -+struct frandom_state -+{ -+ struct semaphore sem; /* Semaphore on the state structure */ -+ -+ u8 S[256]; /* The state array */ -+ u8 i; -+ u8 j; -+ -+ char *buf; -+}; -+ -+static struct frandom_state *erandom_state; -+ -+static inline void swap_byte(u8 *a, u8 *b) -+{ -+ u8 swapByte; -+ -+ swapByte = *a; -+ *a = *b; -+ *b = swapByte; -+} -+ -+static void init_rand_state(struct frandom_state *state, int seedflag); -+ -+void erandom_get_random_bytes(char *buf, size_t count) -+{ -+ struct frandom_state *state = erandom_state; -+ int k; -+ -+ unsigned int i; -+ unsigned int j; -+ u8 *S; -+ -+ /* If we fail to get the semaphore, we revert to external random data. -+ Since semaphore blocking is expected to be very rare, and interrupts -+ during these rare and very short periods of time even less frequent, -+ we take the better-safe-than-sorry approach, and fill the buffer -+ some expensive random data, in case the caller wasn't aware of this -+ possibility, and expects random data anyhow. -+ */ -+ -+ if (down_interruptible(&state->sem)) { -+ get_random_bytes(buf, count); -+ return; -+ } -+ -+ /* We seed erandom as late as possible, hoping that the kernel's main -+ RNG is already restored in the boot sequence (not critical, but -+ better. -+ */ -+ -+ if (!erandom_seeded) { -+ erandom_seeded = 1; -+ init_rand_state(state, EXTERNAL_SEED); -+ printk(KERN_INFO "frandom: Seeded global generator now (used by erandom)\n"); -+ } -+ -+ i = state->i; -+ j = state->j; -+ S = state->S; -+ -+ for (k=0; ki = i; -+ state->j = j; -+ -+ up(&state->sem); -+} -+ -+static void init_rand_state(struct frandom_state *state, int seedflag) -+{ -+ unsigned int i, j, k; -+ u8 *S; -+ u8 *seed = state->buf; -+ -+ if (seedflag == INTERNAL_SEED) -+ erandom_get_random_bytes(seed, 256); -+ else -+ get_random_bytes(seed, 256); -+ -+ S = state->S; -+ for (i=0; i<256; i++) -+ *S++=i; -+ -+ j=0; -+ S = state->S; -+ -+ for (i=0; i<256; i++) { -+ j = (j + S[i] + *seed++) & 0xff; -+ swap_byte(&S[i], &S[j]); -+ } -+ -+ /* It's considered good practice to discard the first 256 bytes -+ generated. So we do it: -+ */ -+ -+ i=0; j=0; -+ for (k=0; k<256; k++) { -+ i = (i + 1) & 0xff; -+ j = (j + S[i]) & 0xff; -+ swap_byte(&S[i], &S[j]); -+ } -+ -+ state->i = i; /* Save state */ -+ state->j = j; -+} -+ -+static int frandom_open(struct inode *inode, struct file *filp) -+{ -+ -+ struct frandom_state *state; -+ -+ int num = iminor(inode); -+ -+ /* This should never happen, now when the minors are regsitered -+ * explicitly -+ */ -+ if ((num != frandom_minor) && (num != erandom_minor)) return -ENODEV; -+ -+ state = kmalloc(sizeof(struct frandom_state), GFP_KERNEL); -+ if (!state) -+ return -ENOMEM; -+ -+ state->buf = kmalloc(frandom_bufsize, GFP_KERNEL); -+ if (!state->buf) { -+ kfree(state); -+ return -ENOMEM; -+ } -+ -+ sema_init(&state->sem, 1); /* Init semaphore as a mutex */ -+ -+ if (num == frandom_minor) -+ init_rand_state(state, EXTERNAL_SEED); -+ else -+ init_rand_state(state, INTERNAL_SEED); -+ -+ filp->private_data = state; -+ -+ return 0; /* Success */ -+} -+ -+static int frandom_release(struct inode *inode, struct file *filp) -+{ -+ -+ struct frandom_state *state = filp->private_data; -+ -+ kfree(state->buf); -+ kfree(state); -+ -+ return 0; -+} -+ -+static ssize_t frandom_read(struct file *filp, char *buf, size_t count, -+ loff_t *f_pos) -+{ -+ struct frandom_state *state = filp->private_data; -+ ssize_t ret; -+ int dobytes, k; -+ char *localbuf; -+ -+ unsigned int i; -+ unsigned int j; -+ u8 *S; -+ -+ if (down_interruptible(&state->sem)) -+ return -ERESTARTSYS; -+ -+ if ((frandom_chunklimit > 0) && (count > frandom_chunklimit)) -+ count = frandom_chunklimit; -+ -+ ret = count; /* It's either everything or an error... */ -+ -+ i = state->i; -+ j = state->j; -+ S = state->S; -+ -+ while (count) { -+ if (count > frandom_bufsize) -+ dobytes = frandom_bufsize; -+ else -+ dobytes = count; -+ -+ localbuf = state->buf; -+ -+ for (k=0; kbuf, dobytes)) { -+ ret = -EFAULT; -+ goto out; -+ } -+ -+ buf += dobytes; -+ count -= dobytes; -+ } -+ -+ out: -+ state->i = i; -+ state->j = j; -+ -+ up(&state->sem); -+ return ret; -+} -+ -+static struct file_operations frandom_fops = { -+ read: frandom_read, -+ open: frandom_open, -+ release: frandom_release, -+}; -+ -+static void frandom_cleanup_module(void) { -+ unregister_chrdev_region(MKDEV(frandom_major, erandom_minor), 1); -+ cdev_del(&erandom_cdev); -+ device_destroy(frandom_class, MKDEV(frandom_major, erandom_minor)); -+ -+ unregister_chrdev_region(MKDEV(frandom_major, frandom_minor), 1); -+ cdev_del(&frandom_cdev); -+ device_destroy(frandom_class, MKDEV(frandom_major, frandom_minor)); -+ class_destroy(frandom_class); -+ -+ kfree(erandom_state->buf); -+ kfree(erandom_state); -+} -+ -+ -+static int frandom_init_module(void) -+{ -+ int result; -+ -+ /* The buffer size MUST be at least 256 bytes, because we assume that -+ minimal length in init_rand_state(). -+ */ -+ if (frandom_bufsize < 256) { -+ printk(KERN_ERR "frandom: Refused to load because frandom_bufsize=%d < 256\n",frandom_bufsize); -+ return -EINVAL; -+ } -+ if ((frandom_chunklimit != 0) && (frandom_chunklimit < 256)) { -+ printk(KERN_ERR "frandom: Refused to load because frandom_chunklimit=%d < 256 and != 0\n",frandom_chunklimit); -+ return -EINVAL; -+ } -+ -+ erandom_state = kmalloc(sizeof(struct frandom_state), GFP_KERNEL); -+ if (!erandom_state) -+ return -ENOMEM; -+ -+ /* This specific buffer is only used for seeding, so we need -+ 256 bytes exactly */ -+ erandom_state->buf = kmalloc(256, GFP_KERNEL); -+ if (!erandom_state->buf) { -+ kfree(erandom_state); -+ return -ENOMEM; -+ } -+ -+ sema_init(&erandom_state->sem, 1); /* Init semaphore as a mutex */ -+ -+ erandom_seeded = 0; -+ -+ frandom_class = class_create(THIS_MODULE, "fastrng"); -+ if (IS_ERR(frandom_class)) { -+ result = PTR_ERR(frandom_class); -+ printk(KERN_WARNING "frandom: Failed to register class fastrng\n"); -+ goto error0; -+ } -+ -+ /* -+ * Register your major, and accept a dynamic number. This is the -+ * first thing to do, in order to avoid releasing other module's -+ * fops in frandom_cleanup_module() -+ */ -+ -+ cdev_init(&frandom_cdev, &frandom_fops); -+ frandom_cdev.owner = THIS_MODULE; -+ result = cdev_add(&frandom_cdev, MKDEV(frandom_major, frandom_minor), 1); -+ if (result) { -+ printk(KERN_WARNING "frandom: Failed to add cdev for /dev/frandom\n"); -+ goto error1; -+ } -+ -+ result = register_chrdev_region(MKDEV(frandom_major, frandom_minor), 1, "/dev/frandom"); -+ if (result < 0) { -+ printk(KERN_WARNING "frandom: can't get major/minor %d/%d\n", frandom_major, frandom_minor); -+ goto error2; -+ } -+ -+ frandom_device = device_create(frandom_class, NULL, MKDEV(frandom_major, frandom_minor), NULL, "frandom"); -+ -+ if (IS_ERR(frandom_device)) { -+ printk(KERN_WARNING "frandom: Failed to create frandom device\n"); -+ goto error3; -+ } -+ -+ cdev_init(&erandom_cdev, &frandom_fops); -+ erandom_cdev.owner = THIS_MODULE; -+ result = cdev_add(&erandom_cdev, MKDEV(frandom_major, erandom_minor), 1); -+ if (result) { -+ printk(KERN_WARNING "frandom: Failed to add cdev for /dev/erandom\n"); -+ goto error4; -+ } -+ -+ result = register_chrdev_region(MKDEV(frandom_major, erandom_minor), 1, "/dev/erandom"); -+ if (result < 0) { -+ printk(KERN_WARNING "frandom: can't get major/minor %d/%d\n", frandom_major, erandom_minor); -+ goto error5; -+ } -+ -+ erandom_device = device_create(frandom_class, NULL, MKDEV(frandom_major, erandom_minor), NULL, "erandom"); -+ -+ if (IS_ERR(erandom_device)) { -+ printk(KERN_WARNING "frandom: Failed to create erandom device\n"); -+ goto error6; -+ } -+ return 0; /* succeed */ -+ -+ error6: -+ unregister_chrdev_region(MKDEV(frandom_major, erandom_minor), 1); -+ error5: -+ cdev_del(&erandom_cdev); -+ error4: -+ device_destroy(frandom_class, MKDEV(frandom_major, frandom_minor)); -+ error3: -+ unregister_chrdev_region(MKDEV(frandom_major, frandom_minor), 1); -+ error2: -+ cdev_del(&frandom_cdev); -+ error1: -+ class_destroy(frandom_class); -+ error0: -+ kfree(erandom_state->buf); -+ kfree(erandom_state); -+ -+ return result; -+} -+ -+module_init(frandom_init_module); -+module_exit(frandom_cleanup_module); -+ -+EXPORT_SYMBOL(erandom_get_random_bytes); -diff -Nur linux-4.1.3/drivers/char/Makefile linux-xbian-imx6/drivers/char/Makefile ---- linux-4.1.3/drivers/char/Makefile 2015-07-21 19:10:33.000000000 +0200 -+++ linux-xbian-imx6/drivers/char/Makefile 2015-07-27 23:13:03.982772220 +0200 -@@ -2,6 +2,7 @@ - # Makefile for the kernel character device drivers. - # - -+obj-m += frandom.o - obj-y += mem.o random.o - obj-$(CONFIG_TTY_PRINTK) += ttyprintk.o - obj-y += misc.o -diff -Nur linux-4.1.3/drivers/cpufreq/imx6q-cpufreq.c linux-xbian-imx6/drivers/cpufreq/imx6q-cpufreq.c ---- linux-4.1.3/drivers/cpufreq/imx6q-cpufreq.c 2015-07-21 19:10:33.000000000 +0200 -+++ linux-xbian-imx6/drivers/cpufreq/imx6q-cpufreq.c 2015-07-27 23:13:04.158146502 +0200 -@@ -20,6 +20,8 @@ - #define PU_SOC_VOLTAGE_HIGH 1275000 - #define FREQ_1P2_GHZ 1200000000 - -+extern int vpu352; -+ - static struct regulator *arm_reg; - static struct regulator *pu_reg; - static struct regulator *soc_reg; -@@ -251,6 +253,10 @@ - unsigned long volt = be32_to_cpup(val++); - if (freq_table[j].frequency == freq) { - imx6_soc_volt[soc_opp_count++] = volt; -+ if (vpu352 && freq == 792000) { -+ pr_info("VPU352: increase SOC/PU voltage for VPU352MHz\n"); -+ imx6_soc_volt[soc_opp_count-1] = 1250000; -+ } - break; - } - } -diff -Nur linux-4.1.3/drivers/crypto/caam/caamalg.c linux-xbian-imx6/drivers/crypto/caam/caamalg.c ---- linux-4.1.3/drivers/crypto/caam/caamalg.c 2015-07-21 19:10:33.000000000 +0200 -+++ linux-xbian-imx6/drivers/crypto/caam/caamalg.c 2015-07-27 23:13:04.205975852 +0200 -@@ -1,7 +1,7 @@ - /* - * caam - Freescale FSL CAAM support for crypto API - * -- * Copyright 2008-2011 Freescale Semiconductor, Inc. -+ * Copyright (C) 2008-2013 Freescale Semiconductor, Inc. - * - * Based on talitos crypto API driver. - * -@@ -53,6 +53,7 @@ - #include "error.h" - #include "sg_sw_sec4.h" - #include "key_gen.h" -+#include - - /* - * crypto alg -@@ -60,68 +61,42 @@ - #define CAAM_CRA_PRIORITY 3000 - /* max key is sum of AES_MAX_KEY_SIZE, max split key size */ - #define CAAM_MAX_KEY_SIZE (AES_MAX_KEY_SIZE + \ -- CTR_RFC3686_NONCE_SIZE + \ - SHA512_DIGEST_SIZE * 2) - /* max IV is max of AES_BLOCK_SIZE, DES3_EDE_BLOCK_SIZE */ - #define CAAM_MAX_IV_LENGTH 16 - - /* length of descriptors text */ -+#define DESC_JOB_IO_LEN (CAAM_CMD_SZ * 5 + CAAM_PTR_SZ * 3) -+ - #define DESC_AEAD_BASE (4 * CAAM_CMD_SZ) --#define DESC_AEAD_ENC_LEN (DESC_AEAD_BASE + 15 * CAAM_CMD_SZ) --#define DESC_AEAD_DEC_LEN (DESC_AEAD_BASE + 18 * CAAM_CMD_SZ) -+#define DESC_AEAD_ENC_LEN (DESC_AEAD_BASE + 16 * CAAM_CMD_SZ) -+#define DESC_AEAD_DEC_LEN (DESC_AEAD_BASE + 21 * CAAM_CMD_SZ) - #define DESC_AEAD_GIVENC_LEN (DESC_AEAD_ENC_LEN + 7 * CAAM_CMD_SZ) - --/* Note: Nonce is counted in enckeylen */ --#define DESC_AEAD_CTR_RFC3686_LEN (6 * CAAM_CMD_SZ) -- --#define DESC_AEAD_NULL_BASE (3 * CAAM_CMD_SZ) --#define DESC_AEAD_NULL_ENC_LEN (DESC_AEAD_NULL_BASE + 14 * CAAM_CMD_SZ) --#define DESC_AEAD_NULL_DEC_LEN (DESC_AEAD_NULL_BASE + 17 * CAAM_CMD_SZ) -- --#define DESC_GCM_BASE (3 * CAAM_CMD_SZ) --#define DESC_GCM_ENC_LEN (DESC_GCM_BASE + 23 * CAAM_CMD_SZ) --#define DESC_GCM_DEC_LEN (DESC_GCM_BASE + 19 * CAAM_CMD_SZ) -- --#define DESC_RFC4106_BASE (3 * CAAM_CMD_SZ) --#define DESC_RFC4106_ENC_LEN (DESC_RFC4106_BASE + 15 * CAAM_CMD_SZ) --#define DESC_RFC4106_DEC_LEN (DESC_RFC4106_BASE + 14 * CAAM_CMD_SZ) --#define DESC_RFC4106_GIVENC_LEN (DESC_RFC4106_BASE + 21 * CAAM_CMD_SZ) -- --#define DESC_RFC4543_BASE (3 * CAAM_CMD_SZ) --#define DESC_RFC4543_ENC_LEN (DESC_RFC4543_BASE + 25 * CAAM_CMD_SZ) --#define DESC_RFC4543_DEC_LEN (DESC_RFC4543_BASE + 27 * CAAM_CMD_SZ) --#define DESC_RFC4543_GIVENC_LEN (DESC_RFC4543_BASE + 30 * CAAM_CMD_SZ) -- - #define DESC_ABLKCIPHER_BASE (3 * CAAM_CMD_SZ) - #define DESC_ABLKCIPHER_ENC_LEN (DESC_ABLKCIPHER_BASE + \ - 20 * CAAM_CMD_SZ) - #define DESC_ABLKCIPHER_DEC_LEN (DESC_ABLKCIPHER_BASE + \ - 15 * CAAM_CMD_SZ) - --#define DESC_MAX_USED_BYTES (DESC_RFC4543_GIVENC_LEN + \ -+#define DESC_MAX_USED_BYTES (DESC_AEAD_GIVENC_LEN + \ - CAAM_MAX_KEY_SIZE) - #define DESC_MAX_USED_LEN (DESC_MAX_USED_BYTES / CAAM_CMD_SZ) - - #ifdef DEBUG - /* for print_hex_dumps with line references */ -+#define xstr(s) str(s) -+#define str(s) #s - #define debug(format, arg...) printk(format, arg) - #else - #define debug(format, arg...) - #endif --static struct list_head alg_list; - - /* Set DK bit in class 1 operation if shared */ - static inline void append_dec_op1(u32 *desc, u32 type) - { - u32 *jump_cmd, *uncond_jump_cmd; - -- /* DK bit is valid only for AES */ -- if ((type & OP_ALG_ALGSEL_MASK) != OP_ALG_ALGSEL_AES) { -- append_operation(desc, type | OP_ALG_AS_INITFINAL | -- OP_ALG_DECRYPT); -- return; -- } -- - jump_cmd = append_jump(desc, JUMP_TEST_ALL | JUMP_COND_SHRD); - append_operation(desc, type | OP_ALG_AS_INITFINAL | - OP_ALG_DECRYPT); -@@ -133,26 +108,37 @@ - } - - /* -+ * Wait for completion of class 1 key loading before allowing -+ * error propagation -+ */ -+static inline void append_dec_shr_done(u32 *desc) -+{ -+ u32 *jump_cmd; -+ -+ jump_cmd = append_jump(desc, JUMP_CLASS_CLASS1 | JUMP_TEST_ALL); -+ set_jump_tgt_here(desc, jump_cmd); -+ append_cmd(desc, SET_OK_NO_PROP_ERRORS | CMD_LOAD); -+} -+ -+/* - * For aead functions, read payload and write payload, - * both of which are specified in req->src and req->dst - */ - static inline void aead_append_src_dst(u32 *desc, u32 msg_type) - { -- append_seq_fifo_store(desc, 0, FIFOST_TYPE_MESSAGE_DATA | KEY_VLF); - append_seq_fifo_load(desc, 0, FIFOLD_CLASS_BOTH | - KEY_VLF | msg_type | FIFOLD_TYPE_LASTBOTH); -+ append_seq_fifo_store(desc, 0, FIFOST_TYPE_MESSAGE_DATA | KEY_VLF); - } - - /* - * For aead encrypt and decrypt, read iv for both classes - */ --static inline void aead_append_ld_iv(u32 *desc, int ivsize, int ivoffset) -+static inline void aead_append_ld_iv(u32 *desc, int ivsize) - { -- append_seq_load(desc, ivsize, LDST_CLASS_1_CCB | -- LDST_SRCDST_BYTE_CONTEXT | -- (ivoffset << LDST_OFFSET_SHIFT)); -- append_move(desc, MOVE_SRC_CLASS1CTX | MOVE_DEST_CLASS2INFIFO | -- (ivoffset << MOVE_OFFSET_SHIFT) | ivsize); -+ append_cmd(desc, CMD_SEQ_LOAD | LDST_SRCDST_BYTE_CONTEXT | -+ LDST_CLASS_1_CCB | ivsize); -+ append_move(desc, MOVE_SRC_CLASS1CTX | MOVE_DEST_CLASS2INFIFO | ivsize); - } - - /* -@@ -198,309 +184,68 @@ - }; - - static void append_key_aead(u32 *desc, struct caam_ctx *ctx, -- int keys_fit_inline, bool is_rfc3686) -+ int keys_fit_inline) - { -- u32 *nonce; -- unsigned int enckeylen = ctx->enckeylen; -- -- /* -- * RFC3686 specific: -- * | ctx->key = {AUTH_KEY, ENC_KEY, NONCE} -- * | enckeylen = encryption key size + nonce size -- */ -- if (is_rfc3686) -- enckeylen -= CTR_RFC3686_NONCE_SIZE; -- - if (keys_fit_inline) { - append_key_as_imm(desc, ctx->key, ctx->split_key_pad_len, - ctx->split_key_len, CLASS_2 | - KEY_DEST_MDHA_SPLIT | KEY_ENC); - append_key_as_imm(desc, (void *)ctx->key + -- ctx->split_key_pad_len, enckeylen, -- enckeylen, CLASS_1 | KEY_DEST_CLASS_REG); -+ ctx->split_key_pad_len, ctx->enckeylen, -+ ctx->enckeylen, CLASS_1 | KEY_DEST_CLASS_REG); - } else { - append_key(desc, ctx->key_dma, ctx->split_key_len, CLASS_2 | - KEY_DEST_MDHA_SPLIT | KEY_ENC); - append_key(desc, ctx->key_dma + ctx->split_key_pad_len, -- enckeylen, CLASS_1 | KEY_DEST_CLASS_REG); -- } -- -- /* Load Counter into CONTEXT1 reg */ -- if (is_rfc3686) { -- nonce = (u32 *)((void *)ctx->key + ctx->split_key_pad_len + -- enckeylen); -- append_load_imm_u32(desc, *nonce, LDST_CLASS_IND_CCB | -- LDST_SRCDST_BYTE_OUTFIFO | LDST_IMM); -- append_move(desc, -- MOVE_SRC_OUTFIFO | -- MOVE_DEST_CLASS1CTX | -- (16 << MOVE_OFFSET_SHIFT) | -- (CTR_RFC3686_NONCE_SIZE << MOVE_LEN_SHIFT)); -+ ctx->enckeylen, CLASS_1 | KEY_DEST_CLASS_REG); - } - } - - static void init_sh_desc_key_aead(u32 *desc, struct caam_ctx *ctx, -- int keys_fit_inline, bool is_rfc3686) -+ int keys_fit_inline) - { - u32 *key_jump_cmd; - -- /* Note: Context registers are saved. */ -- init_sh_desc(desc, HDR_SHARE_SERIAL | HDR_SAVECTX); -- -- /* Skip if already shared */ -- key_jump_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL | -- JUMP_COND_SHRD); -- -- append_key_aead(desc, ctx, keys_fit_inline, is_rfc3686); -- -- set_jump_tgt_here(desc, key_jump_cmd); --} -- --static int aead_null_set_sh_desc(struct crypto_aead *aead) --{ -- struct aead_tfm *tfm = &aead->base.crt_aead; -- struct caam_ctx *ctx = crypto_aead_ctx(aead); -- struct device *jrdev = ctx->jrdev; -- bool keys_fit_inline = false; -- u32 *key_jump_cmd, *jump_cmd, *read_move_cmd, *write_move_cmd; -- u32 *desc; -- -- /* -- * Job Descriptor and Shared Descriptors -- * must all fit into the 64-word Descriptor h/w Buffer -- */ -- if (DESC_AEAD_NULL_ENC_LEN + DESC_JOB_IO_LEN + -- ctx->split_key_pad_len <= CAAM_DESC_BYTES_MAX) -- keys_fit_inline = true; -- -- /* aead_encrypt shared descriptor */ -- desc = ctx->sh_desc_enc; -- - init_sh_desc(desc, HDR_SHARE_SERIAL); - - /* Skip if already shared */ - key_jump_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL | - JUMP_COND_SHRD); -- if (keys_fit_inline) -- append_key_as_imm(desc, ctx->key, ctx->split_key_pad_len, -- ctx->split_key_len, CLASS_2 | -- KEY_DEST_MDHA_SPLIT | KEY_ENC); -- else -- append_key(desc, ctx->key_dma, ctx->split_key_len, CLASS_2 | -- KEY_DEST_MDHA_SPLIT | KEY_ENC); -- set_jump_tgt_here(desc, key_jump_cmd); -- -- /* cryptlen = seqoutlen - authsize */ -- append_math_sub_imm_u32(desc, REG3, SEQOUTLEN, IMM, ctx->authsize); -- -- /* -- * NULL encryption; IV is zero -- * assoclen = (assoclen + cryptlen) - cryptlen -- */ -- append_math_sub(desc, VARSEQINLEN, SEQINLEN, REG3, CAAM_CMD_SZ); -- -- /* read assoc before reading payload */ -- append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS2 | FIFOLD_TYPE_MSG | -- KEY_VLF); -- -- /* Prepare to read and write cryptlen bytes */ -- append_math_add(desc, VARSEQINLEN, ZERO, REG3, CAAM_CMD_SZ); -- append_math_add(desc, VARSEQOUTLEN, ZERO, REG3, CAAM_CMD_SZ); -- -- /* -- * MOVE_LEN opcode is not available in all SEC HW revisions, -- * thus need to do some magic, i.e. self-patch the descriptor -- * buffer. -- */ -- read_move_cmd = append_move(desc, MOVE_SRC_DESCBUF | -- MOVE_DEST_MATH3 | -- (0x6 << MOVE_LEN_SHIFT)); -- write_move_cmd = append_move(desc, MOVE_SRC_MATH3 | -- MOVE_DEST_DESCBUF | -- MOVE_WAITCOMP | -- (0x8 << MOVE_LEN_SHIFT)); -- -- /* Class 2 operation */ -- append_operation(desc, ctx->class2_alg_type | -- OP_ALG_AS_INITFINAL | OP_ALG_ENCRYPT); -- -- /* Read and write cryptlen bytes */ -- aead_append_src_dst(desc, FIFOLD_TYPE_MSG | FIFOLD_TYPE_FLUSH1); -- -- set_move_tgt_here(desc, read_move_cmd); -- set_move_tgt_here(desc, write_move_cmd); -- append_cmd(desc, CMD_LOAD | DISABLE_AUTO_INFO_FIFO); -- append_move(desc, MOVE_SRC_INFIFO_CL | MOVE_DEST_OUTFIFO | -- MOVE_AUX_LS); -- -- /* Write ICV */ -- append_seq_store(desc, ctx->authsize, LDST_CLASS_2_CCB | -- LDST_SRCDST_BYTE_CONTEXT); -- -- ctx->sh_desc_enc_dma = dma_map_single(jrdev, desc, -- desc_bytes(desc), -- DMA_TO_DEVICE); -- if (dma_mapping_error(jrdev, ctx->sh_desc_enc_dma)) { -- dev_err(jrdev, "unable to map shared descriptor\n"); -- return -ENOMEM; -- } --#ifdef DEBUG -- print_hex_dump(KERN_ERR, -- "aead null enc shdesc@"__stringify(__LINE__)": ", -- DUMP_PREFIX_ADDRESS, 16, 4, desc, -- desc_bytes(desc), 1); --#endif -- -- /* -- * Job Descriptor and Shared Descriptors -- * must all fit into the 64-word Descriptor h/w Buffer -- */ -- keys_fit_inline = false; -- if (DESC_AEAD_NULL_DEC_LEN + DESC_JOB_IO_LEN + -- ctx->split_key_pad_len <= CAAM_DESC_BYTES_MAX) -- keys_fit_inline = true; -- -- desc = ctx->sh_desc_dec; - -- /* aead_decrypt shared descriptor */ -- init_sh_desc(desc, HDR_SHARE_SERIAL); -+ append_key_aead(desc, ctx, keys_fit_inline); - -- /* Skip if already shared */ -- key_jump_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL | -- JUMP_COND_SHRD); -- if (keys_fit_inline) -- append_key_as_imm(desc, ctx->key, ctx->split_key_pad_len, -- ctx->split_key_len, CLASS_2 | -- KEY_DEST_MDHA_SPLIT | KEY_ENC); -- else -- append_key(desc, ctx->key_dma, ctx->split_key_len, CLASS_2 | -- KEY_DEST_MDHA_SPLIT | KEY_ENC); - set_jump_tgt_here(desc, key_jump_cmd); - -- /* Class 2 operation */ -- append_operation(desc, ctx->class2_alg_type | -- OP_ALG_AS_INITFINAL | OP_ALG_DECRYPT | OP_ALG_ICV_ON); -- -- /* assoclen + cryptlen = seqinlen - ivsize - authsize */ -- append_math_sub_imm_u32(desc, REG3, SEQINLEN, IMM, -- ctx->authsize + tfm->ivsize); -- /* assoclen = (assoclen + cryptlen) - cryptlen */ -- append_math_sub(desc, REG2, SEQOUTLEN, REG0, CAAM_CMD_SZ); -- append_math_sub(desc, VARSEQINLEN, REG3, REG2, CAAM_CMD_SZ); -- -- /* read assoc before reading payload */ -- append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS2 | FIFOLD_TYPE_MSG | -- KEY_VLF); -- -- /* Prepare to read and write cryptlen bytes */ -- append_math_add(desc, VARSEQINLEN, ZERO, REG2, CAAM_CMD_SZ); -- append_math_add(desc, VARSEQOUTLEN, ZERO, REG2, CAAM_CMD_SZ); -- -- /* -- * MOVE_LEN opcode is not available in all SEC HW revisions, -- * thus need to do some magic, i.e. self-patch the descriptor -- * buffer. -- */ -- read_move_cmd = append_move(desc, MOVE_SRC_DESCBUF | -- MOVE_DEST_MATH2 | -- (0x6 << MOVE_LEN_SHIFT)); -- write_move_cmd = append_move(desc, MOVE_SRC_MATH2 | -- MOVE_DEST_DESCBUF | -- MOVE_WAITCOMP | -- (0x8 << MOVE_LEN_SHIFT)); -- -- /* Read and write cryptlen bytes */ -- aead_append_src_dst(desc, FIFOLD_TYPE_MSG | FIFOLD_TYPE_FLUSH1); -- -- /* -- * Insert a NOP here, since we need at least 4 instructions between -- * code patching the descriptor buffer and the location being patched. -- */ -- jump_cmd = append_jump(desc, JUMP_TEST_ALL); -- set_jump_tgt_here(desc, jump_cmd); -- -- set_move_tgt_here(desc, read_move_cmd); -- set_move_tgt_here(desc, write_move_cmd); -- append_cmd(desc, CMD_LOAD | DISABLE_AUTO_INFO_FIFO); -- append_move(desc, MOVE_SRC_INFIFO_CL | MOVE_DEST_OUTFIFO | -- MOVE_AUX_LS); -- append_cmd(desc, CMD_LOAD | ENABLE_AUTO_INFO_FIFO); -- -- /* Load ICV */ -- append_seq_fifo_load(desc, ctx->authsize, FIFOLD_CLASS_CLASS2 | -- FIFOLD_TYPE_LAST2 | FIFOLD_TYPE_ICV); -- -- ctx->sh_desc_dec_dma = dma_map_single(jrdev, desc, -- desc_bytes(desc), -- DMA_TO_DEVICE); -- if (dma_mapping_error(jrdev, ctx->sh_desc_dec_dma)) { -- dev_err(jrdev, "unable to map shared descriptor\n"); -- return -ENOMEM; -- } --#ifdef DEBUG -- print_hex_dump(KERN_ERR, -- "aead null dec shdesc@"__stringify(__LINE__)": ", -- DUMP_PREFIX_ADDRESS, 16, 4, desc, -- desc_bytes(desc), 1); --#endif -- -- return 0; -+ /* Propagate errors from shared to job descriptor */ -+ append_cmd(desc, SET_OK_NO_PROP_ERRORS | CMD_LOAD); - } - - static int aead_set_sh_desc(struct crypto_aead *aead) - { - struct aead_tfm *tfm = &aead->base.crt_aead; - struct caam_ctx *ctx = crypto_aead_ctx(aead); -- struct crypto_tfm *ctfm = crypto_aead_tfm(aead); -- const char *alg_name = crypto_tfm_alg_name(ctfm); - struct device *jrdev = ctx->jrdev; -- bool keys_fit_inline; -+ bool keys_fit_inline = false; -+ u32 *key_jump_cmd, *jump_cmd; - u32 geniv, moveiv; -- u32 ctx1_iv_off = 0; - u32 *desc; -- const bool ctr_mode = ((ctx->class1_alg_type & OP_ALG_AAI_MASK) == -- OP_ALG_AAI_CTR_MOD128); -- const bool is_rfc3686 = (ctr_mode && -- (strstr(alg_name, "rfc3686") != NULL)); - -- if (!ctx->authsize) -+ if (!ctx->enckeylen || !ctx->authsize) - return 0; - -- /* NULL encryption / decryption */ -- if (!ctx->enckeylen) -- return aead_null_set_sh_desc(aead); -- -- /* -- * AES-CTR needs to load IV in CONTEXT1 reg -- * at an offset of 128bits (16bytes) -- * CONTEXT1[255:128] = IV -- */ -- if (ctr_mode) -- ctx1_iv_off = 16; -- -- /* -- * RFC3686 specific: -- * CONTEXT1[255:128] = {NONCE, IV, COUNTER} -- */ -- if (is_rfc3686) -- ctx1_iv_off = 16 + CTR_RFC3686_NONCE_SIZE; -- - /* - * Job Descriptor and Shared Descriptors - * must all fit into the 64-word Descriptor h/w Buffer - */ -- keys_fit_inline = false; - if (DESC_AEAD_ENC_LEN + DESC_JOB_IO_LEN + -- ctx->split_key_pad_len + ctx->enckeylen + -- (is_rfc3686 ? DESC_AEAD_CTR_RFC3686_LEN : 0) <= -+ ctx->split_key_pad_len + ctx->enckeylen <= - CAAM_DESC_BYTES_MAX) - keys_fit_inline = true; - - /* aead_encrypt shared descriptor */ - desc = ctx->sh_desc_enc; - -- /* Note: Context registers are saved. */ -- init_sh_desc_key_aead(desc, ctx, keys_fit_inline, is_rfc3686); -+ init_sh_desc_key_aead(desc, ctx, keys_fit_inline); - - /* Class 2 operation */ - append_operation(desc, ctx->class2_alg_type | -@@ -512,21 +257,13 @@ - /* assoclen + cryptlen = seqinlen - ivsize */ - append_math_sub_imm_u32(desc, REG2, SEQINLEN, IMM, tfm->ivsize); - -- /* assoclen = (assoclen + cryptlen) - cryptlen */ -+ /* assoclen + cryptlen = (assoclen + cryptlen) - cryptlen */ - append_math_sub(desc, VARSEQINLEN, REG2, REG3, CAAM_CMD_SZ); - - /* read assoc before reading payload */ - append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS2 | FIFOLD_TYPE_MSG | - KEY_VLF); -- aead_append_ld_iv(desc, tfm->ivsize, ctx1_iv_off); -- -- /* Load Counter into CONTEXT1 reg */ -- if (is_rfc3686) -- append_load_imm_u32(desc, be32_to_cpu(1), LDST_IMM | -- LDST_CLASS_1_CCB | -- LDST_SRCDST_BYTE_CONTEXT | -- ((ctx1_iv_off + CTR_RFC3686_IV_SIZE) << -- LDST_OFFSET_SHIFT)); -+ aead_append_ld_iv(desc, tfm->ivsize); - - /* Class 1 operation */ - append_operation(desc, ctx->class1_alg_type | -@@ -549,35 +286,46 @@ - return -ENOMEM; - } - #ifdef DEBUG -- print_hex_dump(KERN_ERR, "aead enc shdesc@"__stringify(__LINE__)": ", -+ print_hex_dump(KERN_ERR, "aead enc shdesc@"xstr(__LINE__)": ", - DUMP_PREFIX_ADDRESS, 16, 4, desc, - desc_bytes(desc), 1); - #endif - -+ dma_sync_single_for_cpu(jrdev, ctx->sh_desc_enc_dma, desc_bytes(desc), -+ DMA_TO_DEVICE); - /* - * Job Descriptor and Shared Descriptors - * must all fit into the 64-word Descriptor h/w Buffer - */ -- keys_fit_inline = false; - if (DESC_AEAD_DEC_LEN + DESC_JOB_IO_LEN + -- ctx->split_key_pad_len + ctx->enckeylen + -- (is_rfc3686 ? DESC_AEAD_CTR_RFC3686_LEN : 0) <= -+ ctx->split_key_pad_len + ctx->enckeylen <= - CAAM_DESC_BYTES_MAX) - keys_fit_inline = true; - -- /* aead_decrypt shared descriptor */ - desc = ctx->sh_desc_dec; - -- /* Note: Context registers are saved. */ -- init_sh_desc_key_aead(desc, ctx, keys_fit_inline, is_rfc3686); -+ /* aead_decrypt shared descriptor */ -+ init_sh_desc(desc, HDR_SHARE_SERIAL); -+ -+ /* Skip if already shared */ -+ key_jump_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL | -+ JUMP_COND_SHRD); -+ -+ append_key_aead(desc, ctx, keys_fit_inline); -+ -+ /* Only propagate error immediately if shared */ -+ jump_cmd = append_jump(desc, JUMP_TEST_ALL); -+ set_jump_tgt_here(desc, key_jump_cmd); -+ append_cmd(desc, SET_OK_NO_PROP_ERRORS | CMD_LOAD); -+ set_jump_tgt_here(desc, jump_cmd); - - /* Class 2 operation */ - append_operation(desc, ctx->class2_alg_type | - OP_ALG_AS_INITFINAL | OP_ALG_DECRYPT | OP_ALG_ICV_ON); - -- /* assoclen + cryptlen = seqinlen - ivsize - authsize */ -+ /* assoclen + cryptlen = seqinlen - ivsize */ - append_math_sub_imm_u32(desc, REG3, SEQINLEN, IMM, -- ctx->authsize + tfm->ivsize); -+ ctx->authsize + tfm->ivsize) - /* assoclen = (assoclen + cryptlen) - cryptlen */ - append_math_sub(desc, REG2, SEQOUTLEN, REG0, CAAM_CMD_SZ); - append_math_sub(desc, VARSEQINLEN, REG3, REG2, CAAM_CMD_SZ); -@@ -586,22 +334,9 @@ - append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS2 | FIFOLD_TYPE_MSG | - KEY_VLF); - -- aead_append_ld_iv(desc, tfm->ivsize, ctx1_iv_off); -+ aead_append_ld_iv(desc, tfm->ivsize); - -- /* Load Counter into CONTEXT1 reg */ -- if (is_rfc3686) -- append_load_imm_u32(desc, be32_to_cpu(1), LDST_IMM | -- LDST_CLASS_1_CCB | -- LDST_SRCDST_BYTE_CONTEXT | -- ((ctx1_iv_off + CTR_RFC3686_IV_SIZE) << -- LDST_OFFSET_SHIFT)); -- -- /* Choose operation */ -- if (ctr_mode) -- append_operation(desc, ctx->class1_alg_type | -- OP_ALG_AS_INITFINAL | OP_ALG_DECRYPT); -- else -- append_dec_op1(desc, ctx->class1_alg_type); -+ append_dec_op1(desc, ctx->class1_alg_type); - - /* Read and write cryptlen bytes */ - append_math_add(desc, VARSEQINLEN, ZERO, REG2, CAAM_CMD_SZ); -@@ -611,6 +346,7 @@ - /* Load ICV */ - append_seq_fifo_load(desc, ctx->authsize, FIFOLD_CLASS_CLASS2 | - FIFOLD_TYPE_LAST2 | FIFOLD_TYPE_ICV); -+ append_dec_shr_done(desc); - - ctx->sh_desc_dec_dma = dma_map_single(jrdev, desc, - desc_bytes(desc), -@@ -620,27 +356,26 @@ - return -ENOMEM; - } - #ifdef DEBUG -- print_hex_dump(KERN_ERR, "aead dec shdesc@"__stringify(__LINE__)": ", -+ print_hex_dump(KERN_ERR, "aead dec shdesc@"xstr(__LINE__)": ", - DUMP_PREFIX_ADDRESS, 16, 4, desc, - desc_bytes(desc), 1); - #endif -+ dma_sync_single_for_cpu(jrdev, ctx->sh_desc_dec_dma, desc_bytes(desc), -+ DMA_TO_DEVICE); - - /* - * Job Descriptor and Shared Descriptors - * must all fit into the 64-word Descriptor h/w Buffer - */ -- keys_fit_inline = false; - if (DESC_AEAD_GIVENC_LEN + DESC_JOB_IO_LEN + -- ctx->split_key_pad_len + ctx->enckeylen + -- (is_rfc3686 ? DESC_AEAD_CTR_RFC3686_LEN : 0) <= -+ ctx->split_key_pad_len + ctx->enckeylen <= - CAAM_DESC_BYTES_MAX) - keys_fit_inline = true; - - /* aead_givencrypt shared descriptor */ - desc = ctx->sh_desc_givenc; - -- /* Note: Context registers are saved. */ -- init_sh_desc_key_aead(desc, ctx, keys_fit_inline, is_rfc3686); -+ init_sh_desc_key_aead(desc, ctx, keys_fit_inline); - - /* Generate IV */ - geniv = NFIFOENTRY_STYPE_PAD | NFIFOENTRY_DEST_DECO | -@@ -649,16 +384,13 @@ - append_load_imm_u32(desc, geniv, LDST_CLASS_IND_CCB | - LDST_SRCDST_WORD_INFO_FIFO | LDST_IMM); - append_cmd(desc, CMD_LOAD | DISABLE_AUTO_INFO_FIFO); -- append_move(desc, MOVE_WAITCOMP | -- MOVE_SRC_INFIFO | MOVE_DEST_CLASS1CTX | -- (ctx1_iv_off << MOVE_OFFSET_SHIFT) | -- (tfm->ivsize << MOVE_LEN_SHIFT)); -+ append_move(desc, MOVE_SRC_INFIFO | -+ MOVE_DEST_CLASS1CTX | (tfm->ivsize << MOVE_LEN_SHIFT)); - append_cmd(desc, CMD_LOAD | ENABLE_AUTO_INFO_FIFO); - - /* Copy IV to class 1 context */ -- append_move(desc, MOVE_SRC_CLASS1CTX | MOVE_DEST_OUTFIFO | -- (ctx1_iv_off << MOVE_OFFSET_SHIFT) | -- (tfm->ivsize << MOVE_LEN_SHIFT)); -+ append_move(desc, MOVE_SRC_CLASS1CTX | -+ MOVE_DEST_OUTFIFO | (tfm->ivsize << MOVE_LEN_SHIFT)); - - /* Return to encryption */ - append_operation(desc, ctx->class2_alg_type | -@@ -674,7 +406,7 @@ - append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS2 | FIFOLD_TYPE_MSG | - KEY_VLF); - -- /* Copy iv from outfifo to class 2 fifo */ -+ /* Copy iv from class 1 ctx to class 2 fifo*/ - moveiv = NFIFOENTRY_STYPE_OFIFO | NFIFOENTRY_DEST_CLASS2 | - NFIFOENTRY_DTYPE_MSG | (tfm->ivsize << NFIFOENTRY_DLEN_SHIFT); - append_load_imm_u32(desc, moveiv, LDST_CLASS_IND_CCB | -@@ -682,14 +414,6 @@ - append_load_imm_u32(desc, tfm->ivsize, LDST_CLASS_2_CCB | - LDST_SRCDST_WORD_DATASZ_REG | LDST_IMM); - -- /* Load Counter into CONTEXT1 reg */ -- if (is_rfc3686) -- append_load_imm_u32(desc, be32_to_cpu(1), LDST_IMM | -- LDST_CLASS_1_CCB | -- LDST_SRCDST_BYTE_CONTEXT | -- ((ctx1_iv_off + CTR_RFC3686_IV_SIZE) << -- LDST_OFFSET_SHIFT)); -- - /* Class 1 operation */ - append_operation(desc, ctx->class1_alg_type | - OP_ALG_AS_INITFINAL | OP_ALG_ENCRYPT); -@@ -717,10 +441,12 @@ - return -ENOMEM; - } - #ifdef DEBUG -- print_hex_dump(KERN_ERR, "aead givenc shdesc@"__stringify(__LINE__)": ", -+ print_hex_dump(KERN_ERR, "aead givenc shdesc@"xstr(__LINE__)": ", - DUMP_PREFIX_ADDRESS, 16, 4, desc, - desc_bytes(desc), 1); - #endif -+ dma_sync_single_for_cpu(jrdev, ctx->sh_desc_givenc_dma, -+ desc_bytes(desc), DMA_TO_DEVICE); - - return 0; - } -@@ -736,977 +462,84 @@ - return 0; - } - --static int gcm_set_sh_desc(struct crypto_aead *aead) -+static u32 gen_split_aead_key(struct caam_ctx *ctx, const u8 *key_in, -+ u32 authkeylen) - { -- struct aead_tfm *tfm = &aead->base.crt_aead; -+ return gen_split_key(ctx->jrdev, ctx->key, ctx->split_key_len, -+ ctx->split_key_pad_len, key_in, authkeylen, -+ ctx->alg_op); -+} -+ -+static int aead_setkey(struct crypto_aead *aead, -+ const u8 *key, unsigned int keylen) -+{ -+ /* Sizes for MDHA pads (*not* keys): MD5, SHA1, 224, 256, 384, 512 */ -+ static const u8 mdpadlen[] = { 16, 20, 32, 32, 64, 64 }; - struct caam_ctx *ctx = crypto_aead_ctx(aead); - struct device *jrdev = ctx->jrdev; -- bool keys_fit_inline = false; -- u32 *key_jump_cmd, *zero_payload_jump_cmd, -- *zero_assoc_jump_cmd1, *zero_assoc_jump_cmd2; -- u32 *desc; -- -- if (!ctx->enckeylen || !ctx->authsize) -- return 0; -- -- /* -- * AES GCM encrypt shared descriptor -- * Job Descriptor and Shared Descriptor -- * must fit into the 64-word Descriptor h/w Buffer -- */ -- if (DESC_GCM_ENC_LEN + DESC_JOB_IO_LEN + -- ctx->enckeylen <= CAAM_DESC_BYTES_MAX) -- keys_fit_inline = true; -+ struct rtattr *rta = (void *)key; -+ struct crypto_authenc_key_param *param; -+ unsigned int authkeylen; -+ unsigned int enckeylen; -+ int ret = 0; - -- desc = ctx->sh_desc_enc; -+ param = RTA_DATA(rta); -+ enckeylen = be32_to_cpu(param->enckeylen); - -- init_sh_desc(desc, HDR_SHARE_SERIAL); -+ key += RTA_ALIGN(rta->rta_len); -+ keylen -= RTA_ALIGN(rta->rta_len); - -- /* skip key loading if they are loaded due to sharing */ -- key_jump_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL | -- JUMP_COND_SHRD | JUMP_COND_SELF); -- if (keys_fit_inline) -- append_key_as_imm(desc, (void *)ctx->key, ctx->enckeylen, -- ctx->enckeylen, CLASS_1 | KEY_DEST_CLASS_REG); -- else -- append_key(desc, ctx->key_dma, ctx->enckeylen, -- CLASS_1 | KEY_DEST_CLASS_REG); -- set_jump_tgt_here(desc, key_jump_cmd); -+ if (keylen < enckeylen) -+ goto badkey; - -- /* class 1 operation */ -- append_operation(desc, ctx->class1_alg_type | -- OP_ALG_AS_INITFINAL | OP_ALG_ENCRYPT); -+ authkeylen = keylen - enckeylen; - -- /* cryptlen = seqoutlen - authsize */ -- append_math_sub_imm_u32(desc, REG3, SEQOUTLEN, IMM, ctx->authsize); -+ if (keylen > CAAM_MAX_KEY_SIZE) -+ goto badkey; - -- /* assoclen + cryptlen = seqinlen - ivsize */ -- append_math_sub_imm_u32(desc, REG2, SEQINLEN, IMM, tfm->ivsize); -+ /* Pick class 2 key length from algorithm submask */ -+ ctx->split_key_len = mdpadlen[(ctx->alg_op & OP_ALG_ALGSEL_SUBMASK) >> -+ OP_ALG_ALGSEL_SHIFT] * 2; -+ ctx->split_key_pad_len = ALIGN(ctx->split_key_len, 16); - -- /* assoclen = (assoclen + cryptlen) - cryptlen */ -- append_math_sub(desc, REG1, REG2, REG3, CAAM_CMD_SZ); -+#ifdef DEBUG -+ printk(KERN_ERR "keylen %d enckeylen %d authkeylen %d\n", -+ keylen, enckeylen, authkeylen); -+ printk(KERN_ERR "split_key_len %d split_key_pad_len %d\n", -+ ctx->split_key_len, ctx->split_key_pad_len); -+ print_hex_dump(KERN_ERR, "key in @"xstr(__LINE__)": ", -+ DUMP_PREFIX_ADDRESS, 16, 4, key, keylen, 1); -+#endif - -- /* if cryptlen is ZERO jump to zero-payload commands */ -- append_math_add(desc, VARSEQOUTLEN, ZERO, REG3, CAAM_CMD_SZ); -- zero_payload_jump_cmd = append_jump(desc, JUMP_TEST_ALL | -- JUMP_COND_MATH_Z); -- /* read IV */ -- append_seq_fifo_load(desc, tfm->ivsize, FIFOLD_CLASS_CLASS1 | -- FIFOLD_TYPE_IV | FIFOLD_TYPE_FLUSH1); -- -- /* if assoclen is ZERO, skip reading the assoc data */ -- append_math_add(desc, VARSEQINLEN, ZERO, REG1, CAAM_CMD_SZ); -- zero_assoc_jump_cmd1 = append_jump(desc, JUMP_TEST_ALL | -- JUMP_COND_MATH_Z); -- -- /* read assoc data */ -- append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS1 | FIFOLDST_VLF | -- FIFOLD_TYPE_AAD | FIFOLD_TYPE_FLUSH1); -- set_jump_tgt_here(desc, zero_assoc_jump_cmd1); -+ ret = gen_split_aead_key(ctx, key, authkeylen); -+ if (ret) { -+ goto badkey; -+ } - -- append_math_add(desc, VARSEQINLEN, ZERO, REG3, CAAM_CMD_SZ); -+ /* postpend encryption key to auth split key */ -+ memcpy(ctx->key + ctx->split_key_pad_len, key + authkeylen, enckeylen); - -- /* write encrypted data */ -- append_seq_fifo_store(desc, 0, FIFOST_TYPE_MESSAGE_DATA | FIFOLDST_VLF); -+ ctx->key_dma = dma_map_single(jrdev, ctx->key, ctx->split_key_pad_len + -+ enckeylen, DMA_TO_DEVICE); -+ if (dma_mapping_error(jrdev, ctx->key_dma)) { -+ dev_err(jrdev, "unable to map key i/o memory\n"); -+ return -ENOMEM; -+ } -+#ifdef DEBUG -+ print_hex_dump(KERN_ERR, "ctx.key@"xstr(__LINE__)": ", -+ DUMP_PREFIX_ADDRESS, 16, 4, ctx->key, -+ ctx->split_key_pad_len + enckeylen, 1); -+#endif -+ dma_sync_single_for_device(jrdev, ctx->key_dma, -+ ctx->split_key_pad_len + enckeylen, -+ DMA_TO_DEVICE); - -- /* read payload data */ -- append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS1 | FIFOLDST_VLF | -- FIFOLD_TYPE_MSG | FIFOLD_TYPE_LAST1); -- -- /* jump the zero-payload commands */ -- append_jump(desc, JUMP_TEST_ALL | 7); -- -- /* zero-payload commands */ -- set_jump_tgt_here(desc, zero_payload_jump_cmd); -- -- /* if assoclen is ZERO, jump to IV reading - is the only input data */ -- append_math_add(desc, VARSEQINLEN, ZERO, REG1, CAAM_CMD_SZ); -- zero_assoc_jump_cmd2 = append_jump(desc, JUMP_TEST_ALL | -- JUMP_COND_MATH_Z); -- /* read IV */ -- append_seq_fifo_load(desc, tfm->ivsize, FIFOLD_CLASS_CLASS1 | -- FIFOLD_TYPE_IV | FIFOLD_TYPE_FLUSH1); -- -- /* read assoc data */ -- append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS1 | FIFOLDST_VLF | -- FIFOLD_TYPE_AAD | FIFOLD_TYPE_LAST1); -- -- /* jump to ICV writing */ -- append_jump(desc, JUMP_TEST_ALL | 2); -- -- /* read IV - is the only input data */ -- set_jump_tgt_here(desc, zero_assoc_jump_cmd2); -- append_seq_fifo_load(desc, tfm->ivsize, FIFOLD_CLASS_CLASS1 | -- FIFOLD_TYPE_IV | FIFOLD_TYPE_FLUSH1 | -- FIFOLD_TYPE_LAST1); -- -- /* write ICV */ -- append_seq_store(desc, ctx->authsize, LDST_CLASS_1_CCB | -- LDST_SRCDST_BYTE_CONTEXT); -- -- ctx->sh_desc_enc_dma = dma_map_single(jrdev, desc, -- desc_bytes(desc), -- DMA_TO_DEVICE); -- if (dma_mapping_error(jrdev, ctx->sh_desc_enc_dma)) { -- dev_err(jrdev, "unable to map shared descriptor\n"); -- return -ENOMEM; -- } --#ifdef DEBUG -- print_hex_dump(KERN_ERR, "gcm enc shdesc@"__stringify(__LINE__)": ", -- DUMP_PREFIX_ADDRESS, 16, 4, desc, -- desc_bytes(desc), 1); --#endif -- -- /* -- * Job Descriptor and Shared Descriptors -- * must all fit into the 64-word Descriptor h/w Buffer -- */ -- keys_fit_inline = false; -- if (DESC_GCM_DEC_LEN + DESC_JOB_IO_LEN + -- ctx->enckeylen <= CAAM_DESC_BYTES_MAX) -- keys_fit_inline = true; -- -- desc = ctx->sh_desc_dec; -- -- init_sh_desc(desc, HDR_SHARE_SERIAL); -- -- /* skip key loading if they are loaded due to sharing */ -- key_jump_cmd = append_jump(desc, JUMP_JSL | -- JUMP_TEST_ALL | JUMP_COND_SHRD | -- JUMP_COND_SELF); -- if (keys_fit_inline) -- append_key_as_imm(desc, (void *)ctx->key, ctx->enckeylen, -- ctx->enckeylen, CLASS_1 | KEY_DEST_CLASS_REG); -- else -- append_key(desc, ctx->key_dma, ctx->enckeylen, -- CLASS_1 | KEY_DEST_CLASS_REG); -- set_jump_tgt_here(desc, key_jump_cmd); -- -- /* class 1 operation */ -- append_operation(desc, ctx->class1_alg_type | -- OP_ALG_AS_INITFINAL | OP_ALG_DECRYPT | OP_ALG_ICV_ON); -- -- /* assoclen + cryptlen = seqinlen - ivsize - icvsize */ -- append_math_sub_imm_u32(desc, REG3, SEQINLEN, IMM, -- ctx->authsize + tfm->ivsize); -- -- /* assoclen = (assoclen + cryptlen) - cryptlen */ -- append_math_sub(desc, REG2, SEQOUTLEN, REG0, CAAM_CMD_SZ); -- append_math_sub(desc, REG1, REG3, REG2, CAAM_CMD_SZ); -- -- /* read IV */ -- append_seq_fifo_load(desc, tfm->ivsize, FIFOLD_CLASS_CLASS1 | -- FIFOLD_TYPE_IV | FIFOLD_TYPE_FLUSH1); -- -- /* jump to zero-payload command if cryptlen is zero */ -- append_math_add(desc, VARSEQOUTLEN, ZERO, REG2, CAAM_CMD_SZ); -- zero_payload_jump_cmd = append_jump(desc, JUMP_TEST_ALL | -- JUMP_COND_MATH_Z); -- -- append_math_add(desc, VARSEQINLEN, ZERO, REG1, CAAM_CMD_SZ); -- /* if asoclen is ZERO, skip reading assoc data */ -- zero_assoc_jump_cmd1 = append_jump(desc, JUMP_TEST_ALL | -- JUMP_COND_MATH_Z); -- /* read assoc data */ -- append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS1 | FIFOLDST_VLF | -- FIFOLD_TYPE_AAD | FIFOLD_TYPE_FLUSH1); -- set_jump_tgt_here(desc, zero_assoc_jump_cmd1); -- -- append_math_add(desc, VARSEQINLEN, ZERO, REG2, CAAM_CMD_SZ); -- -- /* store encrypted data */ -- append_seq_fifo_store(desc, 0, FIFOST_TYPE_MESSAGE_DATA | FIFOLDST_VLF); -- -- /* read payload data */ -- append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS1 | FIFOLDST_VLF | -- FIFOLD_TYPE_MSG | FIFOLD_TYPE_FLUSH1); -- -- /* jump the zero-payload commands */ -- append_jump(desc, JUMP_TEST_ALL | 4); -- -- /* zero-payload command */ -- set_jump_tgt_here(desc, zero_payload_jump_cmd); -- -- /* if assoclen is ZERO, jump to ICV reading */ -- append_math_add(desc, VARSEQINLEN, ZERO, REG1, CAAM_CMD_SZ); -- zero_assoc_jump_cmd2 = append_jump(desc, JUMP_TEST_ALL | -- JUMP_COND_MATH_Z); -- /* read assoc data */ -- append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS1 | FIFOLDST_VLF | -- FIFOLD_TYPE_AAD | FIFOLD_TYPE_FLUSH1); -- set_jump_tgt_here(desc, zero_assoc_jump_cmd2); -- -- /* read ICV */ -- append_seq_fifo_load(desc, ctx->authsize, FIFOLD_CLASS_CLASS1 | -- FIFOLD_TYPE_ICV | FIFOLD_TYPE_LAST1); -- -- ctx->sh_desc_dec_dma = dma_map_single(jrdev, desc, -- desc_bytes(desc), -- DMA_TO_DEVICE); -- if (dma_mapping_error(jrdev, ctx->sh_desc_dec_dma)) { -- dev_err(jrdev, "unable to map shared descriptor\n"); -- return -ENOMEM; -- } --#ifdef DEBUG -- print_hex_dump(KERN_ERR, "gcm dec shdesc@"__stringify(__LINE__)": ", -- DUMP_PREFIX_ADDRESS, 16, 4, desc, -- desc_bytes(desc), 1); --#endif -- -- return 0; --} -- --static int gcm_setauthsize(struct crypto_aead *authenc, unsigned int authsize) --{ -- struct caam_ctx *ctx = crypto_aead_ctx(authenc); -- -- ctx->authsize = authsize; -- gcm_set_sh_desc(authenc); -- -- return 0; --} -- --static int rfc4106_set_sh_desc(struct crypto_aead *aead) --{ -- struct aead_tfm *tfm = &aead->base.crt_aead; -- struct caam_ctx *ctx = crypto_aead_ctx(aead); -- struct device *jrdev = ctx->jrdev; -- bool keys_fit_inline = false; -- u32 *key_jump_cmd, *move_cmd, *write_iv_cmd; -- u32 *desc; -- u32 geniv; -- -- if (!ctx->enckeylen || !ctx->authsize) -- return 0; -- -- /* -- * RFC4106 encrypt shared descriptor -- * Job Descriptor and Shared Descriptor -- * must fit into the 64-word Descriptor h/w Buffer -- */ -- if (DESC_RFC4106_ENC_LEN + DESC_JOB_IO_LEN + -- ctx->enckeylen <= CAAM_DESC_BYTES_MAX) -- keys_fit_inline = true; -- -- desc = ctx->sh_desc_enc; -- -- init_sh_desc(desc, HDR_SHARE_SERIAL); -- -- /* Skip key loading if it is loaded due to sharing */ -- key_jump_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL | -- JUMP_COND_SHRD); -- if (keys_fit_inline) -- append_key_as_imm(desc, (void *)ctx->key, ctx->enckeylen, -- ctx->enckeylen, CLASS_1 | KEY_DEST_CLASS_REG); -- else -- append_key(desc, ctx->key_dma, ctx->enckeylen, -- CLASS_1 | KEY_DEST_CLASS_REG); -- set_jump_tgt_here(desc, key_jump_cmd); -- -- /* Class 1 operation */ -- append_operation(desc, ctx->class1_alg_type | -- OP_ALG_AS_INITFINAL | OP_ALG_ENCRYPT); -- -- /* cryptlen = seqoutlen - authsize */ -- append_math_sub_imm_u32(desc, REG3, SEQOUTLEN, IMM, ctx->authsize); -- append_math_add(desc, VARSEQOUTLEN, ZERO, REG3, CAAM_CMD_SZ); -- -- /* assoclen + cryptlen = seqinlen - ivsize */ -- append_math_sub_imm_u32(desc, REG2, SEQINLEN, IMM, tfm->ivsize); -- -- /* assoclen = (assoclen + cryptlen) - cryptlen */ -- append_math_sub(desc, VARSEQINLEN, REG2, REG3, CAAM_CMD_SZ); -- -- /* Read Salt */ -- append_fifo_load_as_imm(desc, (void *)(ctx->key + ctx->enckeylen), -- 4, FIFOLD_CLASS_CLASS1 | FIFOLD_TYPE_IV); -- /* Read AES-GCM-ESP IV */ -- append_seq_fifo_load(desc, tfm->ivsize, FIFOLD_CLASS_CLASS1 | -- FIFOLD_TYPE_IV | FIFOLD_TYPE_FLUSH1); -- -- /* Read assoc data */ -- append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS1 | FIFOLDST_VLF | -- FIFOLD_TYPE_AAD | FIFOLD_TYPE_FLUSH1); -- -- /* Will read cryptlen bytes */ -- append_math_add(desc, VARSEQINLEN, ZERO, REG3, CAAM_CMD_SZ); -- -- /* Write encrypted data */ -- append_seq_fifo_store(desc, 0, FIFOST_TYPE_MESSAGE_DATA | FIFOLDST_VLF); -- -- /* Read payload data */ -- append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS1 | FIFOLDST_VLF | -- FIFOLD_TYPE_MSG | FIFOLD_TYPE_LAST1); -- -- /* Write ICV */ -- append_seq_store(desc, ctx->authsize, LDST_CLASS_1_CCB | -- LDST_SRCDST_BYTE_CONTEXT); -- -- ctx->sh_desc_enc_dma = dma_map_single(jrdev, desc, -- desc_bytes(desc), -- DMA_TO_DEVICE); -- if (dma_mapping_error(jrdev, ctx->sh_desc_enc_dma)) { -- dev_err(jrdev, "unable to map shared descriptor\n"); -- return -ENOMEM; -- } --#ifdef DEBUG -- print_hex_dump(KERN_ERR, "rfc4106 enc shdesc@"__stringify(__LINE__)": ", -- DUMP_PREFIX_ADDRESS, 16, 4, desc, -- desc_bytes(desc), 1); --#endif -- -- /* -- * Job Descriptor and Shared Descriptors -- * must all fit into the 64-word Descriptor h/w Buffer -- */ -- keys_fit_inline = false; -- if (DESC_RFC4106_DEC_LEN + DESC_JOB_IO_LEN + -- ctx->enckeylen <= CAAM_DESC_BYTES_MAX) -- keys_fit_inline = true; -- -- desc = ctx->sh_desc_dec; -- -- init_sh_desc(desc, HDR_SHARE_SERIAL); -- -- /* Skip key loading if it is loaded due to sharing */ -- key_jump_cmd = append_jump(desc, JUMP_JSL | -- JUMP_TEST_ALL | JUMP_COND_SHRD); -- if (keys_fit_inline) -- append_key_as_imm(desc, (void *)ctx->key, ctx->enckeylen, -- ctx->enckeylen, CLASS_1 | KEY_DEST_CLASS_REG); -- else -- append_key(desc, ctx->key_dma, ctx->enckeylen, -- CLASS_1 | KEY_DEST_CLASS_REG); -- set_jump_tgt_here(desc, key_jump_cmd); -- -- /* Class 1 operation */ -- append_operation(desc, ctx->class1_alg_type | -- OP_ALG_AS_INITFINAL | OP_ALG_DECRYPT | OP_ALG_ICV_ON); -- -- /* assoclen + cryptlen = seqinlen - ivsize - icvsize */ -- append_math_sub_imm_u32(desc, REG3, SEQINLEN, IMM, -- ctx->authsize + tfm->ivsize); -- -- /* assoclen = (assoclen + cryptlen) - cryptlen */ -- append_math_sub(desc, REG2, SEQOUTLEN, REG0, CAAM_CMD_SZ); -- append_math_sub(desc, VARSEQINLEN, REG3, REG2, CAAM_CMD_SZ); -- -- /* Will write cryptlen bytes */ -- append_math_sub(desc, VARSEQOUTLEN, SEQOUTLEN, REG0, CAAM_CMD_SZ); -- -- /* Read Salt */ -- append_fifo_load_as_imm(desc, (void *)(ctx->key + ctx->enckeylen), -- 4, FIFOLD_CLASS_CLASS1 | FIFOLD_TYPE_IV); -- /* Read AES-GCM-ESP IV */ -- append_seq_fifo_load(desc, tfm->ivsize, FIFOLD_CLASS_CLASS1 | -- FIFOLD_TYPE_IV | FIFOLD_TYPE_FLUSH1); -- -- /* Read assoc data */ -- append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS1 | FIFOLDST_VLF | -- FIFOLD_TYPE_AAD | FIFOLD_TYPE_FLUSH1); -- -- /* Will read cryptlen bytes */ -- append_math_add(desc, VARSEQINLEN, ZERO, REG2, CAAM_CMD_SZ); -- -- /* Store payload data */ -- append_seq_fifo_store(desc, 0, FIFOST_TYPE_MESSAGE_DATA | FIFOLDST_VLF); -- -- /* Read encrypted data */ -- append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS1 | FIFOLDST_VLF | -- FIFOLD_TYPE_MSG | FIFOLD_TYPE_FLUSH1); -- -- /* Read ICV */ -- append_seq_fifo_load(desc, ctx->authsize, FIFOLD_CLASS_CLASS1 | -- FIFOLD_TYPE_ICV | FIFOLD_TYPE_LAST1); -- -- ctx->sh_desc_dec_dma = dma_map_single(jrdev, desc, -- desc_bytes(desc), -- DMA_TO_DEVICE); -- if (dma_mapping_error(jrdev, ctx->sh_desc_dec_dma)) { -- dev_err(jrdev, "unable to map shared descriptor\n"); -- return -ENOMEM; -- } --#ifdef DEBUG -- print_hex_dump(KERN_ERR, "rfc4106 dec shdesc@"__stringify(__LINE__)": ", -- DUMP_PREFIX_ADDRESS, 16, 4, desc, -- desc_bytes(desc), 1); --#endif -- -- /* -- * Job Descriptor and Shared Descriptors -- * must all fit into the 64-word Descriptor h/w Buffer -- */ -- keys_fit_inline = false; -- if (DESC_RFC4106_GIVENC_LEN + DESC_JOB_IO_LEN + -- ctx->split_key_pad_len + ctx->enckeylen <= -- CAAM_DESC_BYTES_MAX) -- keys_fit_inline = true; -- -- /* rfc4106_givencrypt shared descriptor */ -- desc = ctx->sh_desc_givenc; -- -- init_sh_desc(desc, HDR_SHARE_SERIAL); -- -- /* Skip key loading if it is loaded due to sharing */ -- key_jump_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL | -- JUMP_COND_SHRD); -- if (keys_fit_inline) -- append_key_as_imm(desc, (void *)ctx->key, ctx->enckeylen, -- ctx->enckeylen, CLASS_1 | KEY_DEST_CLASS_REG); -- else -- append_key(desc, ctx->key_dma, ctx->enckeylen, -- CLASS_1 | KEY_DEST_CLASS_REG); -- set_jump_tgt_here(desc, key_jump_cmd); -- -- /* Generate IV */ -- geniv = NFIFOENTRY_STYPE_PAD | NFIFOENTRY_DEST_DECO | -- NFIFOENTRY_DTYPE_MSG | NFIFOENTRY_LC1 | -- NFIFOENTRY_PTYPE_RND | (tfm->ivsize << NFIFOENTRY_DLEN_SHIFT); -- append_load_imm_u32(desc, geniv, LDST_CLASS_IND_CCB | -- LDST_SRCDST_WORD_INFO_FIFO | LDST_IMM); -- append_cmd(desc, CMD_LOAD | DISABLE_AUTO_INFO_FIFO); -- move_cmd = append_move(desc, MOVE_SRC_INFIFO | MOVE_DEST_DESCBUF | -- (tfm->ivsize << MOVE_LEN_SHIFT)); -- append_cmd(desc, CMD_LOAD | ENABLE_AUTO_INFO_FIFO); -- -- /* Copy generated IV to OFIFO */ -- write_iv_cmd = append_move(desc, MOVE_SRC_DESCBUF | MOVE_DEST_OUTFIFO | -- (tfm->ivsize << MOVE_LEN_SHIFT)); -- -- /* Class 1 operation */ -- append_operation(desc, ctx->class1_alg_type | -- OP_ALG_AS_INITFINAL | OP_ALG_ENCRYPT); -- -- /* ivsize + cryptlen = seqoutlen - authsize */ -- append_math_sub_imm_u32(desc, REG3, SEQOUTLEN, IMM, ctx->authsize); -- -- /* assoclen = seqinlen - (ivsize + cryptlen) */ -- append_math_sub(desc, VARSEQINLEN, SEQINLEN, REG3, CAAM_CMD_SZ); -- -- /* Will write ivsize + cryptlen */ -- append_math_add(desc, VARSEQOUTLEN, REG3, REG0, CAAM_CMD_SZ); -- -- /* Read Salt and generated IV */ -- append_cmd(desc, CMD_FIFO_LOAD | FIFOLD_CLASS_CLASS1 | FIFOLD_TYPE_IV | -- FIFOLD_TYPE_FLUSH1 | IMMEDIATE | 12); -- /* Append Salt */ -- append_data(desc, (void *)(ctx->key + ctx->enckeylen), 4); -- set_move_tgt_here(desc, move_cmd); -- set_move_tgt_here(desc, write_iv_cmd); -- /* Blank commands. Will be overwritten by generated IV. */ -- append_cmd(desc, 0x00000000); -- append_cmd(desc, 0x00000000); -- /* End of blank commands */ -- -- /* No need to reload iv */ -- append_seq_fifo_load(desc, tfm->ivsize, FIFOLD_CLASS_SKIP); -- -- /* Read assoc data */ -- append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS1 | FIFOLDST_VLF | -- FIFOLD_TYPE_AAD | FIFOLD_TYPE_FLUSH1); -- -- /* Will read cryptlen */ -- append_math_add(desc, VARSEQINLEN, SEQINLEN, REG0, CAAM_CMD_SZ); -- -- /* Store generated IV and encrypted data */ -- append_seq_fifo_store(desc, 0, FIFOST_TYPE_MESSAGE_DATA | FIFOLDST_VLF); -- -- /* Read payload data */ -- append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS1 | FIFOLDST_VLF | -- FIFOLD_TYPE_MSG | FIFOLD_TYPE_LAST1); -- -- /* Write ICV */ -- append_seq_store(desc, ctx->authsize, LDST_CLASS_1_CCB | -- LDST_SRCDST_BYTE_CONTEXT); -- -- ctx->sh_desc_givenc_dma = dma_map_single(jrdev, desc, -- desc_bytes(desc), -- DMA_TO_DEVICE); -- if (dma_mapping_error(jrdev, ctx->sh_desc_givenc_dma)) { -- dev_err(jrdev, "unable to map shared descriptor\n"); -- return -ENOMEM; -- } --#ifdef DEBUG -- print_hex_dump(KERN_ERR, -- "rfc4106 givenc shdesc@"__stringify(__LINE__)": ", -- DUMP_PREFIX_ADDRESS, 16, 4, desc, -- desc_bytes(desc), 1); --#endif -- -- return 0; --} -- --static int rfc4106_setauthsize(struct crypto_aead *authenc, -- unsigned int authsize) --{ -- struct caam_ctx *ctx = crypto_aead_ctx(authenc); -- -- ctx->authsize = authsize; -- rfc4106_set_sh_desc(authenc); -- -- return 0; --} -- --static int rfc4543_set_sh_desc(struct crypto_aead *aead) --{ -- struct aead_tfm *tfm = &aead->base.crt_aead; -- struct caam_ctx *ctx = crypto_aead_ctx(aead); -- struct device *jrdev = ctx->jrdev; -- bool keys_fit_inline = false; -- u32 *key_jump_cmd, *write_iv_cmd, *write_aad_cmd; -- u32 *read_move_cmd, *write_move_cmd; -- u32 *desc; -- u32 geniv; -- -- if (!ctx->enckeylen || !ctx->authsize) -- return 0; -- -- /* -- * RFC4543 encrypt shared descriptor -- * Job Descriptor and Shared Descriptor -- * must fit into the 64-word Descriptor h/w Buffer -- */ -- if (DESC_RFC4543_ENC_LEN + DESC_JOB_IO_LEN + -- ctx->enckeylen <= CAAM_DESC_BYTES_MAX) -- keys_fit_inline = true; -- -- desc = ctx->sh_desc_enc; -- -- init_sh_desc(desc, HDR_SHARE_SERIAL); -- -- /* Skip key loading if it is loaded due to sharing */ -- key_jump_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL | -- JUMP_COND_SHRD); -- if (keys_fit_inline) -- append_key_as_imm(desc, (void *)ctx->key, ctx->enckeylen, -- ctx->enckeylen, CLASS_1 | KEY_DEST_CLASS_REG); -- else -- append_key(desc, ctx->key_dma, ctx->enckeylen, -- CLASS_1 | KEY_DEST_CLASS_REG); -- set_jump_tgt_here(desc, key_jump_cmd); -- -- /* Class 1 operation */ -- append_operation(desc, ctx->class1_alg_type | -- OP_ALG_AS_INITFINAL | OP_ALG_ENCRYPT); -- -- /* Load AES-GMAC ESP IV into Math1 register */ -- append_cmd(desc, CMD_SEQ_LOAD | LDST_SRCDST_WORD_DECO_MATH1 | -- LDST_CLASS_DECO | tfm->ivsize); -- -- /* Wait the DMA transaction to finish */ -- append_jump(desc, JUMP_TEST_ALL | JUMP_COND_CALM | -- (1 << JUMP_OFFSET_SHIFT)); -- -- /* Overwrite blank immediate AES-GMAC ESP IV data */ -- write_iv_cmd = append_move(desc, MOVE_SRC_MATH1 | MOVE_DEST_DESCBUF | -- (tfm->ivsize << MOVE_LEN_SHIFT)); -- -- /* Overwrite blank immediate AAD data */ -- write_aad_cmd = append_move(desc, MOVE_SRC_MATH1 | MOVE_DEST_DESCBUF | -- (tfm->ivsize << MOVE_LEN_SHIFT)); -- -- /* cryptlen = seqoutlen - authsize */ -- append_math_sub_imm_u32(desc, REG3, SEQOUTLEN, IMM, ctx->authsize); -- -- /* assoclen = (seqinlen - ivsize) - cryptlen */ -- append_math_sub(desc, VARSEQINLEN, SEQINLEN, REG3, CAAM_CMD_SZ); -- -- /* Read Salt and AES-GMAC ESP IV */ -- append_cmd(desc, CMD_FIFO_LOAD | FIFOLD_CLASS_CLASS1 | IMMEDIATE | -- FIFOLD_TYPE_IV | FIFOLD_TYPE_FLUSH1 | (4 + tfm->ivsize)); -- /* Append Salt */ -- append_data(desc, (void *)(ctx->key + ctx->enckeylen), 4); -- set_move_tgt_here(desc, write_iv_cmd); -- /* Blank commands. Will be overwritten by AES-GMAC ESP IV. */ -- append_cmd(desc, 0x00000000); -- append_cmd(desc, 0x00000000); -- /* End of blank commands */ -- -- /* Read assoc data */ -- append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS1 | FIFOLDST_VLF | -- FIFOLD_TYPE_AAD); -- -- /* Will read cryptlen bytes */ -- append_math_add(desc, VARSEQINLEN, ZERO, REG3, CAAM_CMD_SZ); -- -- /* Will write cryptlen bytes */ -- append_math_add(desc, VARSEQOUTLEN, ZERO, REG3, CAAM_CMD_SZ); -- -- /* -- * MOVE_LEN opcode is not available in all SEC HW revisions, -- * thus need to do some magic, i.e. self-patch the descriptor -- * buffer. -- */ -- read_move_cmd = append_move(desc, MOVE_SRC_DESCBUF | MOVE_DEST_MATH3 | -- (0x6 << MOVE_LEN_SHIFT)); -- write_move_cmd = append_move(desc, MOVE_SRC_MATH3 | MOVE_DEST_DESCBUF | -- (0x8 << MOVE_LEN_SHIFT)); -- -- /* Authenticate AES-GMAC ESP IV */ -- append_cmd(desc, CMD_FIFO_LOAD | FIFOLD_CLASS_CLASS1 | IMMEDIATE | -- FIFOLD_TYPE_AAD | tfm->ivsize); -- set_move_tgt_here(desc, write_aad_cmd); -- /* Blank commands. Will be overwritten by AES-GMAC ESP IV. */ -- append_cmd(desc, 0x00000000); -- append_cmd(desc, 0x00000000); -- /* End of blank commands */ -- -- /* Read and write cryptlen bytes */ -- aead_append_src_dst(desc, FIFOLD_TYPE_AAD); -- -- set_move_tgt_here(desc, read_move_cmd); -- set_move_tgt_here(desc, write_move_cmd); -- append_cmd(desc, CMD_LOAD | DISABLE_AUTO_INFO_FIFO); -- /* Move payload data to OFIFO */ -- append_move(desc, MOVE_SRC_INFIFO_CL | MOVE_DEST_OUTFIFO); -- -- /* Write ICV */ -- append_seq_store(desc, ctx->authsize, LDST_CLASS_1_CCB | -- LDST_SRCDST_BYTE_CONTEXT); -- -- ctx->sh_desc_enc_dma = dma_map_single(jrdev, desc, -- desc_bytes(desc), -- DMA_TO_DEVICE); -- if (dma_mapping_error(jrdev, ctx->sh_desc_enc_dma)) { -- dev_err(jrdev, "unable to map shared descriptor\n"); -- return -ENOMEM; -- } --#ifdef DEBUG -- print_hex_dump(KERN_ERR, "rfc4543 enc shdesc@"__stringify(__LINE__)": ", -- DUMP_PREFIX_ADDRESS, 16, 4, desc, -- desc_bytes(desc), 1); --#endif -- -- /* -- * Job Descriptor and Shared Descriptors -- * must all fit into the 64-word Descriptor h/w Buffer -- */ -- keys_fit_inline = false; -- if (DESC_RFC4543_DEC_LEN + DESC_JOB_IO_LEN + -- ctx->enckeylen <= CAAM_DESC_BYTES_MAX) -- keys_fit_inline = true; -- -- desc = ctx->sh_desc_dec; -- -- init_sh_desc(desc, HDR_SHARE_SERIAL); -- -- /* Skip key loading if it is loaded due to sharing */ -- key_jump_cmd = append_jump(desc, JUMP_JSL | -- JUMP_TEST_ALL | JUMP_COND_SHRD); -- if (keys_fit_inline) -- append_key_as_imm(desc, (void *)ctx->key, ctx->enckeylen, -- ctx->enckeylen, CLASS_1 | KEY_DEST_CLASS_REG); -- else -- append_key(desc, ctx->key_dma, ctx->enckeylen, -- CLASS_1 | KEY_DEST_CLASS_REG); -- set_jump_tgt_here(desc, key_jump_cmd); -- -- /* Class 1 operation */ -- append_operation(desc, ctx->class1_alg_type | -- OP_ALG_AS_INITFINAL | OP_ALG_DECRYPT | OP_ALG_ICV_ON); -- -- /* Load AES-GMAC ESP IV into Math1 register */ -- append_cmd(desc, CMD_SEQ_LOAD | LDST_SRCDST_WORD_DECO_MATH1 | -- LDST_CLASS_DECO | tfm->ivsize); -- -- /* Wait the DMA transaction to finish */ -- append_jump(desc, JUMP_TEST_ALL | JUMP_COND_CALM | -- (1 << JUMP_OFFSET_SHIFT)); -- -- /* assoclen + cryptlen = (seqinlen - ivsize) - icvsize */ -- append_math_sub_imm_u32(desc, REG3, SEQINLEN, IMM, ctx->authsize); -- -- /* Overwrite blank immediate AES-GMAC ESP IV data */ -- write_iv_cmd = append_move(desc, MOVE_SRC_MATH1 | MOVE_DEST_DESCBUF | -- (tfm->ivsize << MOVE_LEN_SHIFT)); -- -- /* Overwrite blank immediate AAD data */ -- write_aad_cmd = append_move(desc, MOVE_SRC_MATH1 | MOVE_DEST_DESCBUF | -- (tfm->ivsize << MOVE_LEN_SHIFT)); -- -- /* assoclen = (assoclen + cryptlen) - cryptlen */ -- append_math_sub(desc, REG2, SEQOUTLEN, REG0, CAAM_CMD_SZ); -- append_math_sub(desc, VARSEQINLEN, REG3, REG2, CAAM_CMD_SZ); -- -- /* -- * MOVE_LEN opcode is not available in all SEC HW revisions, -- * thus need to do some magic, i.e. self-patch the descriptor -- * buffer. -- */ -- read_move_cmd = append_move(desc, MOVE_SRC_DESCBUF | MOVE_DEST_MATH3 | -- (0x6 << MOVE_LEN_SHIFT)); -- write_move_cmd = append_move(desc, MOVE_SRC_MATH3 | MOVE_DEST_DESCBUF | -- (0x8 << MOVE_LEN_SHIFT)); -- -- /* Read Salt and AES-GMAC ESP IV */ -- append_cmd(desc, CMD_FIFO_LOAD | FIFOLD_CLASS_CLASS1 | IMMEDIATE | -- FIFOLD_TYPE_IV | FIFOLD_TYPE_FLUSH1 | (4 + tfm->ivsize)); -- /* Append Salt */ -- append_data(desc, (void *)(ctx->key + ctx->enckeylen), 4); -- set_move_tgt_here(desc, write_iv_cmd); -- /* Blank commands. Will be overwritten by AES-GMAC ESP IV. */ -- append_cmd(desc, 0x00000000); -- append_cmd(desc, 0x00000000); -- /* End of blank commands */ -- -- /* Read assoc data */ -- append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS1 | FIFOLDST_VLF | -- FIFOLD_TYPE_AAD); -- -- /* Will read cryptlen bytes */ -- append_math_add(desc, VARSEQINLEN, ZERO, REG2, CAAM_CMD_SZ); -- -- /* Will write cryptlen bytes */ -- append_math_add(desc, VARSEQOUTLEN, ZERO, REG2, CAAM_CMD_SZ); -- -- /* Authenticate AES-GMAC ESP IV */ -- append_cmd(desc, CMD_FIFO_LOAD | FIFOLD_CLASS_CLASS1 | IMMEDIATE | -- FIFOLD_TYPE_AAD | tfm->ivsize); -- set_move_tgt_here(desc, write_aad_cmd); -- /* Blank commands. Will be overwritten by AES-GMAC ESP IV. */ -- append_cmd(desc, 0x00000000); -- append_cmd(desc, 0x00000000); -- /* End of blank commands */ -- -- /* Store payload data */ -- append_seq_fifo_store(desc, 0, FIFOST_TYPE_MESSAGE_DATA | FIFOLDST_VLF); -- -- /* In-snoop cryptlen data */ -- append_seq_fifo_load(desc, 0, FIFOLD_CLASS_BOTH | FIFOLDST_VLF | -- FIFOLD_TYPE_AAD | FIFOLD_TYPE_LAST2FLUSH1); -- -- set_move_tgt_here(desc, read_move_cmd); -- set_move_tgt_here(desc, write_move_cmd); -- append_cmd(desc, CMD_LOAD | DISABLE_AUTO_INFO_FIFO); -- /* Move payload data to OFIFO */ -- append_move(desc, MOVE_SRC_INFIFO_CL | MOVE_DEST_OUTFIFO); -- append_cmd(desc, CMD_LOAD | ENABLE_AUTO_INFO_FIFO); -- -- /* Read ICV */ -- append_seq_fifo_load(desc, ctx->authsize, FIFOLD_CLASS_CLASS1 | -- FIFOLD_TYPE_ICV | FIFOLD_TYPE_LAST1); -- -- ctx->sh_desc_dec_dma = dma_map_single(jrdev, desc, -- desc_bytes(desc), -- DMA_TO_DEVICE); -- if (dma_mapping_error(jrdev, ctx->sh_desc_dec_dma)) { -- dev_err(jrdev, "unable to map shared descriptor\n"); -- return -ENOMEM; -- } --#ifdef DEBUG -- print_hex_dump(KERN_ERR, "rfc4543 dec shdesc@"__stringify(__LINE__)": ", -- DUMP_PREFIX_ADDRESS, 16, 4, desc, -- desc_bytes(desc), 1); --#endif -- -- /* -- * Job Descriptor and Shared Descriptors -- * must all fit into the 64-word Descriptor h/w Buffer -- */ -- keys_fit_inline = false; -- if (DESC_RFC4543_GIVENC_LEN + DESC_JOB_IO_LEN + -- ctx->enckeylen <= CAAM_DESC_BYTES_MAX) -- keys_fit_inline = true; -- -- /* rfc4543_givencrypt shared descriptor */ -- desc = ctx->sh_desc_givenc; -- -- init_sh_desc(desc, HDR_SHARE_SERIAL); -- -- /* Skip key loading if it is loaded due to sharing */ -- key_jump_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL | -- JUMP_COND_SHRD); -- if (keys_fit_inline) -- append_key_as_imm(desc, (void *)ctx->key, ctx->enckeylen, -- ctx->enckeylen, CLASS_1 | KEY_DEST_CLASS_REG); -- else -- append_key(desc, ctx->key_dma, ctx->enckeylen, -- CLASS_1 | KEY_DEST_CLASS_REG); -- set_jump_tgt_here(desc, key_jump_cmd); -- -- /* Generate IV */ -- geniv = NFIFOENTRY_STYPE_PAD | NFIFOENTRY_DEST_DECO | -- NFIFOENTRY_DTYPE_MSG | NFIFOENTRY_LC1 | -- NFIFOENTRY_PTYPE_RND | (tfm->ivsize << NFIFOENTRY_DLEN_SHIFT); -- append_load_imm_u32(desc, geniv, LDST_CLASS_IND_CCB | -- LDST_SRCDST_WORD_INFO_FIFO | LDST_IMM); -- append_cmd(desc, CMD_LOAD | DISABLE_AUTO_INFO_FIFO); -- /* Move generated IV to Math1 register */ -- append_move(desc, MOVE_SRC_INFIFO | MOVE_DEST_MATH1 | -- (tfm->ivsize << MOVE_LEN_SHIFT)); -- append_cmd(desc, CMD_LOAD | ENABLE_AUTO_INFO_FIFO); -- -- /* Overwrite blank immediate AES-GMAC IV data */ -- write_iv_cmd = append_move(desc, MOVE_SRC_MATH1 | MOVE_DEST_DESCBUF | -- (tfm->ivsize << MOVE_LEN_SHIFT)); -- -- /* Overwrite blank immediate AAD data */ -- write_aad_cmd = append_move(desc, MOVE_SRC_MATH1 | MOVE_DEST_DESCBUF | -- (tfm->ivsize << MOVE_LEN_SHIFT)); -- -- /* Copy generated IV to OFIFO */ -- append_move(desc, MOVE_SRC_MATH1 | MOVE_DEST_OUTFIFO | -- (tfm->ivsize << MOVE_LEN_SHIFT)); -- -- /* Class 1 operation */ -- append_operation(desc, ctx->class1_alg_type | -- OP_ALG_AS_INITFINAL | OP_ALG_ENCRYPT); -- -- /* ivsize + cryptlen = seqoutlen - authsize */ -- append_math_sub_imm_u32(desc, REG3, SEQOUTLEN, IMM, ctx->authsize); -- -- /* assoclen = seqinlen - (ivsize + cryptlen) */ -- append_math_sub(desc, VARSEQINLEN, SEQINLEN, REG3, CAAM_CMD_SZ); -- -- /* Will write ivsize + cryptlen */ -- append_math_add(desc, VARSEQOUTLEN, REG3, REG0, CAAM_CMD_SZ); -- -- /* -- * MOVE_LEN opcode is not available in all SEC HW revisions, -- * thus need to do some magic, i.e. self-patch the descriptor -- * buffer. -- */ -- read_move_cmd = append_move(desc, MOVE_SRC_DESCBUF | MOVE_DEST_MATH3 | -- (0x6 << MOVE_LEN_SHIFT)); -- write_move_cmd = append_move(desc, MOVE_SRC_MATH3 | MOVE_DEST_DESCBUF | -- (0x8 << MOVE_LEN_SHIFT)); -- -- /* Read Salt and AES-GMAC generated IV */ -- append_cmd(desc, CMD_FIFO_LOAD | FIFOLD_CLASS_CLASS1 | IMMEDIATE | -- FIFOLD_TYPE_IV | FIFOLD_TYPE_FLUSH1 | (4 + tfm->ivsize)); -- /* Append Salt */ -- append_data(desc, (void *)(ctx->key + ctx->enckeylen), 4); -- set_move_tgt_here(desc, write_iv_cmd); -- /* Blank commands. Will be overwritten by AES-GMAC generated IV. */ -- append_cmd(desc, 0x00000000); -- append_cmd(desc, 0x00000000); -- /* End of blank commands */ -- -- /* No need to reload iv */ -- append_seq_fifo_load(desc, tfm->ivsize, FIFOLD_CLASS_SKIP); -- -- /* Read assoc data */ -- append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS1 | FIFOLDST_VLF | -- FIFOLD_TYPE_AAD); -- -- /* Will read cryptlen */ -- append_math_add(desc, VARSEQINLEN, SEQINLEN, REG0, CAAM_CMD_SZ); -- -- /* Authenticate AES-GMAC IV */ -- append_cmd(desc, CMD_FIFO_LOAD | FIFOLD_CLASS_CLASS1 | IMMEDIATE | -- FIFOLD_TYPE_AAD | tfm->ivsize); -- set_move_tgt_here(desc, write_aad_cmd); -- /* Blank commands. Will be overwritten by AES-GMAC IV. */ -- append_cmd(desc, 0x00000000); -- append_cmd(desc, 0x00000000); -- /* End of blank commands */ -- -- /* Read and write cryptlen bytes */ -- aead_append_src_dst(desc, FIFOLD_TYPE_AAD); -- -- set_move_tgt_here(desc, read_move_cmd); -- set_move_tgt_here(desc, write_move_cmd); -- append_cmd(desc, CMD_LOAD | DISABLE_AUTO_INFO_FIFO); -- /* Move payload data to OFIFO */ -- append_move(desc, MOVE_SRC_INFIFO_CL | MOVE_DEST_OUTFIFO); -- -- /* Write ICV */ -- append_seq_store(desc, ctx->authsize, LDST_CLASS_1_CCB | -- LDST_SRCDST_BYTE_CONTEXT); -- -- ctx->sh_desc_givenc_dma = dma_map_single(jrdev, desc, -- desc_bytes(desc), -- DMA_TO_DEVICE); -- if (dma_mapping_error(jrdev, ctx->sh_desc_givenc_dma)) { -- dev_err(jrdev, "unable to map shared descriptor\n"); -- return -ENOMEM; -- } --#ifdef DEBUG -- print_hex_dump(KERN_ERR, -- "rfc4543 givenc shdesc@"__stringify(__LINE__)": ", -- DUMP_PREFIX_ADDRESS, 16, 4, desc, -- desc_bytes(desc), 1); --#endif -- -- return 0; --} -- --static int rfc4543_setauthsize(struct crypto_aead *authenc, -- unsigned int authsize) --{ -- struct caam_ctx *ctx = crypto_aead_ctx(authenc); -- -- ctx->authsize = authsize; -- rfc4543_set_sh_desc(authenc); -- -- return 0; --} -- --static u32 gen_split_aead_key(struct caam_ctx *ctx, const u8 *key_in, -- u32 authkeylen) --{ -- return gen_split_key(ctx->jrdev, ctx->key, ctx->split_key_len, -- ctx->split_key_pad_len, key_in, authkeylen, -- ctx->alg_op); --} -- --static int aead_setkey(struct crypto_aead *aead, -- const u8 *key, unsigned int keylen) --{ -- /* Sizes for MDHA pads (*not* keys): MD5, SHA1, 224, 256, 384, 512 */ -- static const u8 mdpadlen[] = { 16, 20, 32, 32, 64, 64 }; -- struct caam_ctx *ctx = crypto_aead_ctx(aead); -- struct device *jrdev = ctx->jrdev; -- struct crypto_authenc_keys keys; -- int ret = 0; -- -- if (crypto_authenc_extractkeys(&keys, key, keylen) != 0) -- goto badkey; -- -- /* Pick class 2 key length from algorithm submask */ -- ctx->split_key_len = mdpadlen[(ctx->alg_op & OP_ALG_ALGSEL_SUBMASK) >> -- OP_ALG_ALGSEL_SHIFT] * 2; -- ctx->split_key_pad_len = ALIGN(ctx->split_key_len, 16); -- -- if (ctx->split_key_pad_len + keys.enckeylen > CAAM_MAX_KEY_SIZE) -- goto badkey; -- --#ifdef DEBUG -- printk(KERN_ERR "keylen %d enckeylen %d authkeylen %d\n", -- keys.authkeylen + keys.enckeylen, keys.enckeylen, -- keys.authkeylen); -- printk(KERN_ERR "split_key_len %d split_key_pad_len %d\n", -- ctx->split_key_len, ctx->split_key_pad_len); -- print_hex_dump(KERN_ERR, "key in @"__stringify(__LINE__)": ", -- DUMP_PREFIX_ADDRESS, 16, 4, key, keylen, 1); --#endif -- -- ret = gen_split_aead_key(ctx, keys.authkey, keys.authkeylen); -- if (ret) { -- goto badkey; -- } -- -- /* postpend encryption key to auth split key */ -- memcpy(ctx->key + ctx->split_key_pad_len, keys.enckey, keys.enckeylen); -- -- ctx->key_dma = dma_map_single(jrdev, ctx->key, ctx->split_key_pad_len + -- keys.enckeylen, DMA_TO_DEVICE); -- if (dma_mapping_error(jrdev, ctx->key_dma)) { -- dev_err(jrdev, "unable to map key i/o memory\n"); -- return -ENOMEM; -- } --#ifdef DEBUG -- print_hex_dump(KERN_ERR, "ctx.key@"__stringify(__LINE__)": ", -- DUMP_PREFIX_ADDRESS, 16, 4, ctx->key, -- ctx->split_key_pad_len + keys.enckeylen, 1); --#endif -- -- ctx->enckeylen = keys.enckeylen; -+ ctx->enckeylen = enckeylen; - - ret = aead_set_sh_desc(aead); - if (ret) { - dma_unmap_single(jrdev, ctx->key_dma, ctx->split_key_pad_len + -- keys.enckeylen, DMA_TO_DEVICE); -+ enckeylen, DMA_TO_DEVICE); - } - - return ret; -@@ -1715,154 +548,20 @@ - return -EINVAL; - } - --static int gcm_setkey(struct crypto_aead *aead, -- const u8 *key, unsigned int keylen) --{ -- struct caam_ctx *ctx = crypto_aead_ctx(aead); -- struct device *jrdev = ctx->jrdev; -- int ret = 0; -- --#ifdef DEBUG -- print_hex_dump(KERN_ERR, "key in @"__stringify(__LINE__)": ", -- DUMP_PREFIX_ADDRESS, 16, 4, key, keylen, 1); --#endif -- -- memcpy(ctx->key, key, keylen); -- ctx->key_dma = dma_map_single(jrdev, ctx->key, keylen, -- DMA_TO_DEVICE); -- if (dma_mapping_error(jrdev, ctx->key_dma)) { -- dev_err(jrdev, "unable to map key i/o memory\n"); -- return -ENOMEM; -- } -- ctx->enckeylen = keylen; -- -- ret = gcm_set_sh_desc(aead); -- if (ret) { -- dma_unmap_single(jrdev, ctx->key_dma, ctx->enckeylen, -- DMA_TO_DEVICE); -- } -- -- return ret; --} -- --static int rfc4106_setkey(struct crypto_aead *aead, -- const u8 *key, unsigned int keylen) --{ -- struct caam_ctx *ctx = crypto_aead_ctx(aead); -- struct device *jrdev = ctx->jrdev; -- int ret = 0; -- -- if (keylen < 4) -- return -EINVAL; -- --#ifdef DEBUG -- print_hex_dump(KERN_ERR, "key in @"__stringify(__LINE__)": ", -- DUMP_PREFIX_ADDRESS, 16, 4, key, keylen, 1); --#endif -- -- memcpy(ctx->key, key, keylen); -- -- /* -- * The last four bytes of the key material are used as the salt value -- * in the nonce. Update the AES key length. -- */ -- ctx->enckeylen = keylen - 4; -- -- ctx->key_dma = dma_map_single(jrdev, ctx->key, ctx->enckeylen, -- DMA_TO_DEVICE); -- if (dma_mapping_error(jrdev, ctx->key_dma)) { -- dev_err(jrdev, "unable to map key i/o memory\n"); -- return -ENOMEM; -- } -- -- ret = rfc4106_set_sh_desc(aead); -- if (ret) { -- dma_unmap_single(jrdev, ctx->key_dma, ctx->enckeylen, -- DMA_TO_DEVICE); -- } -- -- return ret; --} -- --static int rfc4543_setkey(struct crypto_aead *aead, -- const u8 *key, unsigned int keylen) --{ -- struct caam_ctx *ctx = crypto_aead_ctx(aead); -- struct device *jrdev = ctx->jrdev; -- int ret = 0; -- -- if (keylen < 4) -- return -EINVAL; -- --#ifdef DEBUG -- print_hex_dump(KERN_ERR, "key in @"__stringify(__LINE__)": ", -- DUMP_PREFIX_ADDRESS, 16, 4, key, keylen, 1); --#endif -- -- memcpy(ctx->key, key, keylen); -- -- /* -- * The last four bytes of the key material are used as the salt value -- * in the nonce. Update the AES key length. -- */ -- ctx->enckeylen = keylen - 4; -- -- ctx->key_dma = dma_map_single(jrdev, ctx->key, ctx->enckeylen, -- DMA_TO_DEVICE); -- if (dma_mapping_error(jrdev, ctx->key_dma)) { -- dev_err(jrdev, "unable to map key i/o memory\n"); -- return -ENOMEM; -- } -- -- ret = rfc4543_set_sh_desc(aead); -- if (ret) { -- dma_unmap_single(jrdev, ctx->key_dma, ctx->enckeylen, -- DMA_TO_DEVICE); -- } -- -- return ret; --} -- - static int ablkcipher_setkey(struct crypto_ablkcipher *ablkcipher, - const u8 *key, unsigned int keylen) - { - struct caam_ctx *ctx = crypto_ablkcipher_ctx(ablkcipher); -- struct ablkcipher_tfm *crt = &ablkcipher->base.crt_ablkcipher; -- struct crypto_tfm *tfm = crypto_ablkcipher_tfm(ablkcipher); -- const char *alg_name = crypto_tfm_alg_name(tfm); -+ struct ablkcipher_tfm *tfm = &ablkcipher->base.crt_ablkcipher; - struct device *jrdev = ctx->jrdev; - int ret = 0; -- u32 *key_jump_cmd; -+ u32 *key_jump_cmd, *jump_cmd; - u32 *desc; -- u32 *nonce; -- u32 geniv; -- u32 ctx1_iv_off = 0; -- const bool ctr_mode = ((ctx->class1_alg_type & OP_ALG_AAI_MASK) == -- OP_ALG_AAI_CTR_MOD128); -- const bool is_rfc3686 = (ctr_mode && -- (strstr(alg_name, "rfc3686") != NULL)); - - #ifdef DEBUG -- print_hex_dump(KERN_ERR, "key in @"__stringify(__LINE__)": ", -+ print_hex_dump(KERN_ERR, "key in @"xstr(__LINE__)": ", - DUMP_PREFIX_ADDRESS, 16, 4, key, keylen, 1); - #endif -- /* -- * AES-CTR needs to load IV in CONTEXT1 reg -- * at an offset of 128bits (16bytes) -- * CONTEXT1[255:128] = IV -- */ -- if (ctr_mode) -- ctx1_iv_off = 16; -- -- /* -- * RFC3686 specific: -- * | CONTEXT1[255:128] = {NONCE, IV, COUNTER} -- * | *key = {KEY, NONCE} -- */ -- if (is_rfc3686) { -- ctx1_iv_off = 16 + CTR_RFC3686_NONCE_SIZE; -- keylen -= CTR_RFC3686_NONCE_SIZE; -- } - - memcpy(ctx->key, key, keylen); - ctx->key_dma = dma_map_single(jrdev, ctx->key, keylen, -@@ -1872,10 +571,11 @@ - return -ENOMEM; - } - ctx->enckeylen = keylen; -+ dma_sync_single_for_device(jrdev, ctx->key_dma, keylen, DMA_TO_DEVICE); - - /* ablkcipher_encrypt shared descriptor */ - desc = ctx->sh_desc_enc; -- init_sh_desc(desc, HDR_SHARE_SERIAL | HDR_SAVECTX); -+ init_sh_desc(desc, HDR_SHARE_SERIAL); - /* Skip if already shared */ - key_jump_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL | - JUMP_COND_SHRD); -@@ -1885,32 +585,20 @@ - ctx->enckeylen, CLASS_1 | - KEY_DEST_CLASS_REG); - -- /* Load nonce into CONTEXT1 reg */ -- if (is_rfc3686) { -- nonce = (u32 *)(key + keylen); -- append_load_imm_u32(desc, *nonce, LDST_CLASS_IND_CCB | -- LDST_SRCDST_BYTE_OUTFIFO | LDST_IMM); -- append_move(desc, MOVE_WAITCOMP | -- MOVE_SRC_OUTFIFO | -- MOVE_DEST_CLASS1CTX | -- (16 << MOVE_OFFSET_SHIFT) | -- (CTR_RFC3686_NONCE_SIZE << MOVE_LEN_SHIFT)); -- } -- - set_jump_tgt_here(desc, key_jump_cmd); - -- /* Load iv */ -- append_seq_load(desc, crt->ivsize, LDST_SRCDST_BYTE_CONTEXT | -- LDST_CLASS_1_CCB | (ctx1_iv_off << LDST_OFFSET_SHIFT)); -- -- /* Load counter into CONTEXT1 reg */ -- if (is_rfc3686) -- append_load_imm_u32(desc, be32_to_cpu(1), LDST_IMM | -- LDST_CLASS_1_CCB | -- LDST_SRCDST_BYTE_CONTEXT | -- ((ctx1_iv_off + CTR_RFC3686_IV_SIZE) << -- LDST_OFFSET_SHIFT)); -+ /* Propagate errors from shared to job descriptor */ -+ append_cmd(desc, SET_OK_NO_PROP_ERRORS | CMD_LOAD); - -+ /* load IV */ -+ if (strncmp(ablkcipher->base.__crt_alg->cra_name, "ctr(aes)", 8) == 0) { -+ append_cmd(desc, CMD_SEQ_LOAD | LDST_SRCDST_BYTE_CONTEXT | -+ LDST_CLASS_1_CCB | tfm->ivsize | -+ (16 << LDST_OFFSET_SHIFT)); -+ } else { -+ append_cmd(desc, CMD_SEQ_LOAD | LDST_SRCDST_BYTE_CONTEXT | -+ LDST_CLASS_1_CCB | tfm->ivsize); -+ } - /* Load operation */ - append_operation(desc, ctx->class1_alg_type | - OP_ALG_AS_INITFINAL | OP_ALG_ENCRYPT); -@@ -1926,15 +614,17 @@ - return -ENOMEM; - } - #ifdef DEBUG -- print_hex_dump(KERN_ERR, -- "ablkcipher enc shdesc@"__stringify(__LINE__)": ", -+ print_hex_dump(KERN_ERR, "ablkcipher enc shdesc@"xstr(__LINE__)": ", - DUMP_PREFIX_ADDRESS, 16, 4, desc, - desc_bytes(desc), 1); - #endif -+ dma_sync_single_for_device(jrdev, ctx->sh_desc_enc_dma, -+ desc_bytes(desc), DMA_TO_DEVICE); -+ - /* ablkcipher_decrypt shared descriptor */ - desc = ctx->sh_desc_dec; - -- init_sh_desc(desc, HDR_SHARE_SERIAL | HDR_SAVECTX); -+ init_sh_desc(desc, HDR_SHARE_SERIAL); - /* Skip if already shared */ - key_jump_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL | - JUMP_COND_SHRD); -@@ -1944,133 +634,49 @@ - ctx->enckeylen, CLASS_1 | - KEY_DEST_CLASS_REG); - -- /* Load nonce into CONTEXT1 reg */ -- if (is_rfc3686) { -- nonce = (u32 *)(key + keylen); -- append_load_imm_u32(desc, *nonce, LDST_CLASS_IND_CCB | -- LDST_SRCDST_BYTE_OUTFIFO | LDST_IMM); -- append_move(desc, MOVE_WAITCOMP | -- MOVE_SRC_OUTFIFO | -- MOVE_DEST_CLASS1CTX | -- (16 << MOVE_OFFSET_SHIFT) | -- (CTR_RFC3686_NONCE_SIZE << MOVE_LEN_SHIFT)); -- } -- -+ /* For aead, only propagate error immediately if shared */ -+ jump_cmd = append_jump(desc, JUMP_TEST_ALL); - set_jump_tgt_here(desc, key_jump_cmd); -+ append_cmd(desc, SET_OK_NO_PROP_ERRORS | CMD_LOAD); -+ set_jump_tgt_here(desc, jump_cmd); - - /* load IV */ -- append_seq_load(desc, crt->ivsize, LDST_SRCDST_BYTE_CONTEXT | -- LDST_CLASS_1_CCB | (ctx1_iv_off << LDST_OFFSET_SHIFT)); -- -- /* Load counter into CONTEXT1 reg */ -- if (is_rfc3686) -- append_load_imm_u32(desc, be32_to_cpu(1), LDST_IMM | -- LDST_CLASS_1_CCB | -- LDST_SRCDST_BYTE_CONTEXT | -- ((ctx1_iv_off + CTR_RFC3686_IV_SIZE) << -- LDST_OFFSET_SHIFT)); -+ if (strncmp(ablkcipher->base.__crt_alg->cra_name, "ctr(aes)", 8) == 0) { -+ append_cmd(desc, CMD_SEQ_LOAD | LDST_SRCDST_BYTE_CONTEXT | -+ LDST_CLASS_1_CCB | tfm->ivsize | -+ (16 << LDST_OFFSET_SHIFT)); - -- /* Choose operation */ -- if (ctr_mode) - append_operation(desc, ctx->class1_alg_type | -- OP_ALG_AS_INITFINAL | OP_ALG_DECRYPT); -- else -+ OP_ALG_AS_INITFINAL | OP_ALG_DECRYPT); -+ } else { -+ append_cmd(desc, CMD_SEQ_LOAD | LDST_SRCDST_BYTE_CONTEXT | -+ LDST_CLASS_1_CCB | tfm->ivsize); -+ -+ /* Choose operation */ - append_dec_op1(desc, ctx->class1_alg_type); -+ } - - /* Perform operation */ - ablkcipher_append_src_dst(desc); - -+ /* Wait for key to load before allowing propagating error */ -+ append_dec_shr_done(desc); -+ - ctx->sh_desc_dec_dma = dma_map_single(jrdev, desc, - desc_bytes(desc), - DMA_TO_DEVICE); -- if (dma_mapping_error(jrdev, ctx->sh_desc_dec_dma)) { -+ if (dma_mapping_error(jrdev, ctx->sh_desc_enc_dma)) { - dev_err(jrdev, "unable to map shared descriptor\n"); - return -ENOMEM; - } - - #ifdef DEBUG -- print_hex_dump(KERN_ERR, -- "ablkcipher dec shdesc@"__stringify(__LINE__)": ", -- DUMP_PREFIX_ADDRESS, 16, 4, desc, -- desc_bytes(desc), 1); --#endif -- /* ablkcipher_givencrypt shared descriptor */ -- desc = ctx->sh_desc_givenc; -- -- init_sh_desc(desc, HDR_SHARE_SERIAL | HDR_SAVECTX); -- /* Skip if already shared */ -- key_jump_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL | -- JUMP_COND_SHRD); -- -- /* Load class1 key only */ -- append_key_as_imm(desc, (void *)ctx->key, ctx->enckeylen, -- ctx->enckeylen, CLASS_1 | -- KEY_DEST_CLASS_REG); -- -- /* Load Nonce into CONTEXT1 reg */ -- if (is_rfc3686) { -- nonce = (u32 *)(key + keylen); -- append_load_imm_u32(desc, *nonce, LDST_CLASS_IND_CCB | -- LDST_SRCDST_BYTE_OUTFIFO | LDST_IMM); -- append_move(desc, MOVE_WAITCOMP | -- MOVE_SRC_OUTFIFO | -- MOVE_DEST_CLASS1CTX | -- (16 << MOVE_OFFSET_SHIFT) | -- (CTR_RFC3686_NONCE_SIZE << MOVE_LEN_SHIFT)); -- } -- set_jump_tgt_here(desc, key_jump_cmd); -- -- /* Generate IV */ -- geniv = NFIFOENTRY_STYPE_PAD | NFIFOENTRY_DEST_DECO | -- NFIFOENTRY_DTYPE_MSG | NFIFOENTRY_LC1 | -- NFIFOENTRY_PTYPE_RND | (crt->ivsize << NFIFOENTRY_DLEN_SHIFT); -- append_load_imm_u32(desc, geniv, LDST_CLASS_IND_CCB | -- LDST_SRCDST_WORD_INFO_FIFO | LDST_IMM); -- append_cmd(desc, CMD_LOAD | DISABLE_AUTO_INFO_FIFO); -- append_move(desc, MOVE_WAITCOMP | -- MOVE_SRC_INFIFO | -- MOVE_DEST_CLASS1CTX | -- (crt->ivsize << MOVE_LEN_SHIFT) | -- (ctx1_iv_off << MOVE_OFFSET_SHIFT)); -- append_cmd(desc, CMD_LOAD | ENABLE_AUTO_INFO_FIFO); -- -- /* Copy generated IV to memory */ -- append_seq_store(desc, crt->ivsize, -- LDST_SRCDST_BYTE_CONTEXT | LDST_CLASS_1_CCB | -- (ctx1_iv_off << LDST_OFFSET_SHIFT)); -- -- /* Load Counter into CONTEXT1 reg */ -- if (is_rfc3686) -- append_load_imm_u32(desc, (u32)1, LDST_IMM | -- LDST_CLASS_1_CCB | -- LDST_SRCDST_BYTE_CONTEXT | -- ((ctx1_iv_off + CTR_RFC3686_IV_SIZE) << -- LDST_OFFSET_SHIFT)); -- -- if (ctx1_iv_off) -- append_jump(desc, JUMP_JSL | JUMP_TEST_ALL | JUMP_COND_NCP | -- (1 << JUMP_OFFSET_SHIFT)); -- -- /* Load operation */ -- append_operation(desc, ctx->class1_alg_type | -- OP_ALG_AS_INITFINAL | OP_ALG_ENCRYPT); -- -- /* Perform operation */ -- ablkcipher_append_src_dst(desc); -- -- ctx->sh_desc_givenc_dma = dma_map_single(jrdev, desc, -- desc_bytes(desc), -- DMA_TO_DEVICE); -- if (dma_mapping_error(jrdev, ctx->sh_desc_givenc_dma)) { -- dev_err(jrdev, "unable to map shared descriptor\n"); -- return -ENOMEM; -- } --#ifdef DEBUG -- print_hex_dump(KERN_ERR, -- "ablkcipher givenc shdesc@" __stringify(__LINE__) ": ", -+ print_hex_dump(KERN_ERR, "ablkcipher dec shdesc@"xstr(__LINE__)": ", - DUMP_PREFIX_ADDRESS, 16, 4, desc, - desc_bytes(desc), 1); - #endif -+ dma_sync_single_for_device(jrdev, ctx->sh_desc_dec_dma, -+ desc_bytes(desc), DMA_TO_DEVICE); - - return ret; - } -@@ -2195,19 +801,22 @@ - edesc = (struct aead_edesc *)((char *)desc - - offsetof(struct aead_edesc, hw_desc)); - -- if (err) -- caam_jr_strstatus(jrdev, err); -+ if (err) { -+ char tmp[CAAM_ERROR_STR_MAX]; -+ -+ dev_err(jrdev, "%08x: %s\n", err, caam_jr_strstatus(tmp, err)); -+ } - - aead_unmap(jrdev, edesc, req); - - #ifdef DEBUG -- print_hex_dump(KERN_ERR, "assoc @"__stringify(__LINE__)": ", -+ print_hex_dump(KERN_ERR, "assoc @"xstr(__LINE__)": ", - DUMP_PREFIX_ADDRESS, 16, 4, sg_virt(req->assoc), - req->assoclen , 1); -- print_hex_dump(KERN_ERR, "dstiv @"__stringify(__LINE__)": ", -+ print_hex_dump(KERN_ERR, "dstiv @"xstr(__LINE__)": ", - DUMP_PREFIX_ADDRESS, 16, 4, sg_virt(req->src) - ivsize, - edesc->src_nents ? 100 : ivsize, 1); -- print_hex_dump(KERN_ERR, "dst @"__stringify(__LINE__)": ", -+ print_hex_dump(KERN_ERR, "dst @"xstr(__LINE__)": ", - DUMP_PREFIX_ADDRESS, 16, 4, sg_virt(req->src), - edesc->src_nents ? 100 : req->cryptlen + - ctx->authsize + 4, 1); -@@ -2235,16 +844,19 @@ - offsetof(struct aead_edesc, hw_desc)); - - #ifdef DEBUG -- print_hex_dump(KERN_ERR, "dstiv @"__stringify(__LINE__)": ", -+ print_hex_dump(KERN_ERR, "dstiv @"xstr(__LINE__)": ", - DUMP_PREFIX_ADDRESS, 16, 4, req->iv, - ivsize, 1); -- print_hex_dump(KERN_ERR, "dst @"__stringify(__LINE__)": ", -+ print_hex_dump(KERN_ERR, "dst @"xstr(__LINE__)": ", - DUMP_PREFIX_ADDRESS, 16, 4, sg_virt(req->dst), -- req->cryptlen - ctx->authsize, 1); -+ req->cryptlen, 1); - #endif - -- if (err) -- caam_jr_strstatus(jrdev, err); -+ if (err) { -+ char tmp[CAAM_ERROR_STR_MAX]; -+ -+ dev_err(jrdev, "%08x: %s\n", err, caam_jr_strstatus(tmp, err)); -+ } - - aead_unmap(jrdev, edesc, req); - -@@ -2255,7 +867,7 @@ - err = -EBADMSG; - - #ifdef DEBUG -- print_hex_dump(KERN_ERR, "iphdrout@"__stringify(__LINE__)": ", -+ print_hex_dump(KERN_ERR, "iphdrout@"xstr(__LINE__)": ", - DUMP_PREFIX_ADDRESS, 16, 4, - ((char *)sg_virt(req->assoc) - sizeof(struct iphdr)), - sizeof(struct iphdr) + req->assoclen + -@@ -2263,7 +875,7 @@ - ctx->authsize + 36, 1); - if (!err && edesc->sec4_sg_bytes) { - struct scatterlist *sg = sg_last(req->src, edesc->src_nents); -- print_hex_dump(KERN_ERR, "sglastout@"__stringify(__LINE__)": ", -+ print_hex_dump(KERN_ERR, "sglastout@"xstr(__LINE__)": ", - DUMP_PREFIX_ADDRESS, 16, 4, sg_virt(sg), - sg->length + ctx->authsize + 16, 1); - } -@@ -2289,14 +901,17 @@ - edesc = (struct ablkcipher_edesc *)((char *)desc - - offsetof(struct ablkcipher_edesc, hw_desc)); - -- if (err) -- caam_jr_strstatus(jrdev, err); -+ if (err) { -+ char tmp[CAAM_ERROR_STR_MAX]; -+ -+ dev_err(jrdev, "%08x: %s\n", err, caam_jr_strstatus(tmp, err)); -+ } - - #ifdef DEBUG -- print_hex_dump(KERN_ERR, "dstiv @"__stringify(__LINE__)": ", -+ print_hex_dump(KERN_ERR, "dstiv @"xstr(__LINE__)": ", - DUMP_PREFIX_ADDRESS, 16, 4, req->info, - edesc->src_nents > 1 ? 100 : ivsize, 1); -- print_hex_dump(KERN_ERR, "dst @"__stringify(__LINE__)": ", -+ print_hex_dump(KERN_ERR, "dst @"xstr(__LINE__)": ", - DUMP_PREFIX_ADDRESS, 16, 4, sg_virt(req->src), - edesc->dst_nents > 1 ? 100 : req->nbytes, 1); - #endif -@@ -2321,14 +936,17 @@ - - edesc = (struct ablkcipher_edesc *)((char *)desc - - offsetof(struct ablkcipher_edesc, hw_desc)); -- if (err) -- caam_jr_strstatus(jrdev, err); -+ if (err) { -+ char tmp[CAAM_ERROR_STR_MAX]; -+ -+ dev_err(jrdev, "%08x: %s\n", err, caam_jr_strstatus(tmp, err)); -+ } - - #ifdef DEBUG -- print_hex_dump(KERN_ERR, "dstiv @"__stringify(__LINE__)": ", -+ print_hex_dump(KERN_ERR, "dstiv @"xstr(__LINE__)": ", - DUMP_PREFIX_ADDRESS, 16, 4, req->info, - ivsize, 1); -- print_hex_dump(KERN_ERR, "dst @"__stringify(__LINE__)": ", -+ print_hex_dump(KERN_ERR, "dst @"xstr(__LINE__)": ", - DUMP_PREFIX_ADDRESS, 16, 4, sg_virt(req->src), - edesc->dst_nents > 1 ? 100 : req->nbytes, 1); - #endif -@@ -2355,38 +973,29 @@ - u32 out_options = 0, in_options; - dma_addr_t dst_dma, src_dma; - int len, sec4_sg_index = 0; -- bool is_gcm = false; - - #ifdef DEBUG - debug("assoclen %d cryptlen %d authsize %d\n", - req->assoclen, req->cryptlen, authsize); -- print_hex_dump(KERN_ERR, "assoc @"__stringify(__LINE__)": ", -+ print_hex_dump(KERN_ERR, "assoc @"xstr(__LINE__)": ", - DUMP_PREFIX_ADDRESS, 16, 4, sg_virt(req->assoc), - req->assoclen , 1); -- print_hex_dump(KERN_ERR, "presciv@"__stringify(__LINE__)": ", -+ print_hex_dump(KERN_ERR, "presciv@"xstr(__LINE__)": ", - DUMP_PREFIX_ADDRESS, 16, 4, req->iv, - edesc->src_nents ? 100 : ivsize, 1); -- print_hex_dump(KERN_ERR, "src @"__stringify(__LINE__)": ", -+ print_hex_dump(KERN_ERR, "src @"xstr(__LINE__)": ", - DUMP_PREFIX_ADDRESS, 16, 4, sg_virt(req->src), - edesc->src_nents ? 100 : req->cryptlen, 1); -- print_hex_dump(KERN_ERR, "shrdesc@"__stringify(__LINE__)": ", -+ print_hex_dump(KERN_ERR, "shrdesc@"xstr(__LINE__)": ", - DUMP_PREFIX_ADDRESS, 16, 4, sh_desc, - desc_bytes(sh_desc), 1); - #endif - -- if (((ctx->class1_alg_type & OP_ALG_ALGSEL_MASK) == -- OP_ALG_ALGSEL_AES) && -- ((ctx->class1_alg_type & OP_ALG_AAI_MASK) == OP_ALG_AAI_GCM)) -- is_gcm = true; -- - len = desc_len(sh_desc); - init_job_desc_shared(desc, ptr, len, HDR_SHARE_DEFER | HDR_REVERSE); - - if (all_contig) { -- if (is_gcm) -- src_dma = edesc->iv_dma; -- else -- src_dma = sg_dma_address(req->assoc); -+ src_dma = sg_dma_address(req->assoc); - in_options = 0; - } else { - src_dma = edesc->sec4_sg_dma; -@@ -2394,9 +1003,12 @@ - (edesc->src_nents ? : 1); - in_options = LDST_SGF; - } -- -- append_seq_in_ptr(desc, src_dma, req->assoclen + ivsize + req->cryptlen, -- in_options); -+ if (encrypt) -+ append_seq_in_ptr(desc, src_dma, req->assoclen + ivsize + -+ req->cryptlen - authsize, in_options); -+ else -+ append_seq_in_ptr(desc, src_dma, req->assoclen + ivsize + -+ req->cryptlen, in_options); - - if (likely(req->src == req->dst)) { - if (all_contig) { -@@ -2417,8 +1029,7 @@ - } - } - if (encrypt) -- append_seq_out_ptr(desc, dst_dma, req->cryptlen + authsize, -- out_options); -+ append_seq_out_ptr(desc, dst_dma, req->cryptlen, out_options); - else - append_seq_out_ptr(desc, dst_dma, req->cryptlen - authsize, - out_options); -@@ -2440,53 +1051,43 @@ - u32 out_options = 0, in_options; - dma_addr_t dst_dma, src_dma; - int len, sec4_sg_index = 0; -- bool is_gcm = false; - - #ifdef DEBUG - debug("assoclen %d cryptlen %d authsize %d\n", - req->assoclen, req->cryptlen, authsize); -- print_hex_dump(KERN_ERR, "assoc @"__stringify(__LINE__)": ", -+ print_hex_dump(KERN_ERR, "assoc @"xstr(__LINE__)": ", - DUMP_PREFIX_ADDRESS, 16, 4, sg_virt(req->assoc), - req->assoclen , 1); -- print_hex_dump(KERN_ERR, "presciv@"__stringify(__LINE__)": ", -+ print_hex_dump(KERN_ERR, "presciv@"xstr(__LINE__)": ", - DUMP_PREFIX_ADDRESS, 16, 4, req->iv, ivsize, 1); -- print_hex_dump(KERN_ERR, "src @"__stringify(__LINE__)": ", -+ print_hex_dump(KERN_ERR, "src @"xstr(__LINE__)": ", - DUMP_PREFIX_ADDRESS, 16, 4, sg_virt(req->src), - edesc->src_nents > 1 ? 100 : req->cryptlen, 1); -- print_hex_dump(KERN_ERR, "shrdesc@"__stringify(__LINE__)": ", -+ print_hex_dump(KERN_ERR, "shrdesc@"xstr(__LINE__)": ", - DUMP_PREFIX_ADDRESS, 16, 4, sh_desc, - desc_bytes(sh_desc), 1); - #endif - -- if (((ctx->class1_alg_type & OP_ALG_ALGSEL_MASK) == -- OP_ALG_ALGSEL_AES) && -- ((ctx->class1_alg_type & OP_ALG_AAI_MASK) == OP_ALG_AAI_GCM)) -- is_gcm = true; -- - len = desc_len(sh_desc); - init_job_desc_shared(desc, ptr, len, HDR_SHARE_DEFER | HDR_REVERSE); - - if (contig & GIV_SRC_CONTIG) { -- if (is_gcm) -- src_dma = edesc->iv_dma; -- else -- src_dma = sg_dma_address(req->assoc); -+ src_dma = sg_dma_address(req->assoc); - in_options = 0; - } else { - src_dma = edesc->sec4_sg_dma; - sec4_sg_index += edesc->assoc_nents + 1 + edesc->src_nents; - in_options = LDST_SGF; - } -- append_seq_in_ptr(desc, src_dma, req->assoclen + ivsize + req->cryptlen, -- in_options); -+ append_seq_in_ptr(desc, src_dma, req->assoclen + ivsize + -+ req->cryptlen - authsize, in_options); - - if (contig & GIV_DST_CONTIG) { - dst_dma = edesc->iv_dma; - } else { - if (likely(req->src == req->dst)) { - dst_dma = src_dma + sizeof(struct sec4_sg_entry) * -- (edesc->assoc_nents + -- (is_gcm ? 1 + edesc->src_nents : 0)); -+ edesc->assoc_nents; - out_options = LDST_SGF; - } else { - dst_dma = edesc->sec4_sg_dma + -@@ -2496,8 +1097,7 @@ - } - } - -- append_seq_out_ptr(desc, dst_dma, ivsize + req->cryptlen + authsize, -- out_options); -+ append_seq_out_ptr(desc, dst_dma, ivsize + req->cryptlen, out_options); - } - - /* -@@ -2516,10 +1116,10 @@ - int len, sec4_sg_index = 0; - - #ifdef DEBUG -- print_hex_dump(KERN_ERR, "presciv@"__stringify(__LINE__)": ", -+ print_hex_dump(KERN_ERR, "presciv@"xstr(__LINE__)": ", - DUMP_PREFIX_ADDRESS, 16, 4, req->info, - ivsize, 1); -- print_hex_dump(KERN_ERR, "src @"__stringify(__LINE__)": ", -+ print_hex_dump(KERN_ERR, "src @"xstr(__LINE__)": ", - DUMP_PREFIX_ADDRESS, 16, 4, sg_virt(req->src), - edesc->src_nents ? 100 : req->nbytes, 1); - #endif -@@ -2532,7 +1132,7 @@ - in_options = 0; - } else { - src_dma = edesc->sec4_sg_dma; -- sec4_sg_index += edesc->src_nents + 1; -+ sec4_sg_index += (iv_contig ? 0 : 1) + edesc->src_nents; - in_options = LDST_SGF; - } - append_seq_in_ptr(desc, src_dma, req->nbytes + ivsize, in_options); -@@ -2558,59 +1158,10 @@ - } - - /* -- * Fill in ablkcipher givencrypt job descriptor -- */ --static void init_ablkcipher_giv_job(u32 *sh_desc, dma_addr_t ptr, -- struct ablkcipher_edesc *edesc, -- struct ablkcipher_request *req, -- bool iv_contig) --{ -- struct crypto_ablkcipher *ablkcipher = crypto_ablkcipher_reqtfm(req); -- int ivsize = crypto_ablkcipher_ivsize(ablkcipher); -- u32 *desc = edesc->hw_desc; -- u32 out_options, in_options; -- dma_addr_t dst_dma, src_dma; -- int len, sec4_sg_index = 0; -- --#ifdef DEBUG -- print_hex_dump(KERN_ERR, "presciv@" __stringify(__LINE__) ": ", -- DUMP_PREFIX_ADDRESS, 16, 4, req->info, -- ivsize, 1); -- print_hex_dump(KERN_ERR, "src @" __stringify(__LINE__) ": ", -- DUMP_PREFIX_ADDRESS, 16, 4, sg_virt(req->src), -- edesc->src_nents ? 100 : req->nbytes, 1); --#endif -- -- len = desc_len(sh_desc); -- init_job_desc_shared(desc, ptr, len, HDR_SHARE_DEFER | HDR_REVERSE); -- -- if (!edesc->src_nents) { -- src_dma = sg_dma_address(req->src); -- in_options = 0; -- } else { -- src_dma = edesc->sec4_sg_dma; -- sec4_sg_index += edesc->src_nents; -- in_options = LDST_SGF; -- } -- append_seq_in_ptr(desc, src_dma, req->nbytes, in_options); -- -- if (iv_contig) { -- dst_dma = edesc->iv_dma; -- out_options = 0; -- } else { -- dst_dma = edesc->sec4_sg_dma + -- sec4_sg_index * sizeof(struct sec4_sg_entry); -- out_options = LDST_SGF; -- } -- append_seq_out_ptr(desc, dst_dma, req->nbytes + ivsize, out_options); --} -- --/* - * allocate and map the aead extended descriptor - */ - static struct aead_edesc *aead_edesc_alloc(struct aead_request *req, -- int desc_bytes, bool *all_contig_ptr, -- bool encrypt) -+ int desc_bytes, bool *all_contig_ptr) - { - struct crypto_aead *aead = crypto_aead_reqtfm(req); - struct caam_ctx *ctx = crypto_aead_ctx(aead); -@@ -2625,26 +1176,15 @@ - bool assoc_chained = false, src_chained = false, dst_chained = false; - int ivsize = crypto_aead_ivsize(aead); - int sec4_sg_index, sec4_sg_len = 0, sec4_sg_bytes; -- unsigned int authsize = ctx->authsize; -- bool is_gcm = false; - - assoc_nents = sg_count(req->assoc, req->assoclen, &assoc_chained); -+ src_nents = sg_count(req->src, req->cryptlen, &src_chained); - -- if (unlikely(req->dst != req->src)) { -- src_nents = sg_count(req->src, req->cryptlen, &src_chained); -- dst_nents = sg_count(req->dst, -- req->cryptlen + -- (encrypt ? authsize : (-authsize)), -- &dst_chained); -- } else { -- src_nents = sg_count(req->src, -- req->cryptlen + -- (encrypt ? authsize : 0), -- &src_chained); -- } -+ if (unlikely(req->dst != req->src)) -+ dst_nents = sg_count(req->dst, req->cryptlen, &dst_chained); - - sgc = dma_map_sg_chained(jrdev, req->assoc, assoc_nents ? : 1, -- DMA_TO_DEVICE, assoc_chained); -+ DMA_BIDIRECTIONAL, assoc_chained); - if (likely(req->src == req->dst)) { - sgc = dma_map_sg_chained(jrdev, req->src, src_nents ? : 1, - DMA_BIDIRECTIONAL, src_chained); -@@ -2655,43 +1195,23 @@ - DMA_FROM_DEVICE, dst_chained); - } - -+ /* Check if data are contiguous */ - iv_dma = dma_map_single(jrdev, req->iv, ivsize, DMA_TO_DEVICE); -- if (dma_mapping_error(jrdev, iv_dma)) { -- dev_err(jrdev, "unable to map IV\n"); -- return ERR_PTR(-ENOMEM); -- } -- -- if (((ctx->class1_alg_type & OP_ALG_ALGSEL_MASK) == -- OP_ALG_ALGSEL_AES) && -- ((ctx->class1_alg_type & OP_ALG_AAI_MASK) == OP_ALG_AAI_GCM)) -- is_gcm = true; -- -- /* -- * Check if data are contiguous. -- * GCM expected input sequence: IV, AAD, text -- * All other - expected input sequence: AAD, IV, text -- */ -- if (is_gcm) -- all_contig = (!assoc_nents && -- iv_dma + ivsize == sg_dma_address(req->assoc) && -- !src_nents && sg_dma_address(req->assoc) + -- req->assoclen == sg_dma_address(req->src)); -- else -- all_contig = (!assoc_nents && sg_dma_address(req->assoc) + -- req->assoclen == iv_dma && !src_nents && -- iv_dma + ivsize == sg_dma_address(req->src)); -- if (!all_contig) { -+ if (assoc_nents || sg_dma_address(req->assoc) + req->assoclen != -+ iv_dma || src_nents || iv_dma + ivsize != -+ sg_dma_address(req->src)) { -+ all_contig = false; - assoc_nents = assoc_nents ? : 1; - src_nents = src_nents ? : 1; - sec4_sg_len = assoc_nents + 1 + src_nents; - } -- - sec4_sg_len += dst_nents; - - sec4_sg_bytes = sec4_sg_len * sizeof(struct sec4_sg_entry); -+ dma_sync_single_for_device(jrdev, iv_dma, ivsize, DMA_TO_DEVICE); - - /* allocate space for base edesc and hw desc commands, link tables */ -- edesc = kmalloc(sizeof(struct aead_edesc) + desc_bytes + -+ edesc = kzalloc(sizeof(struct aead_edesc) + desc_bytes + - sec4_sg_bytes, GFP_DMA | flags); - if (!edesc) { - dev_err(jrdev, "could not allocate extended descriptor\n"); -@@ -2708,46 +1228,32 @@ - edesc->sec4_sg_bytes = sec4_sg_bytes; - edesc->sec4_sg = (void *)edesc + sizeof(struct aead_edesc) + - desc_bytes; -+ edesc->sec4_sg_dma = dma_map_single(jrdev, edesc->sec4_sg, -+ sec4_sg_bytes, DMA_TO_DEVICE); - *all_contig_ptr = all_contig; - - sec4_sg_index = 0; - if (!all_contig) { -- if (!is_gcm) { -- sg_to_sec4_sg(req->assoc, -- assoc_nents, -- edesc->sec4_sg + -- sec4_sg_index, 0); -- sec4_sg_index += assoc_nents; -- } -- -+ sg_to_sec4_sg(req->assoc, -+ (assoc_nents ? : 1), -+ edesc->sec4_sg + -+ sec4_sg_index, 0); -+ sec4_sg_index += assoc_nents ? : 1; - dma_to_sec4_sg_one(edesc->sec4_sg + sec4_sg_index, - iv_dma, ivsize, 0); - sec4_sg_index += 1; -- -- if (is_gcm) { -- sg_to_sec4_sg(req->assoc, -- assoc_nents, -- edesc->sec4_sg + -- sec4_sg_index, 0); -- sec4_sg_index += assoc_nents; -- } -- - sg_to_sec4_sg_last(req->src, -- src_nents, -+ (src_nents ? : 1), - edesc->sec4_sg + - sec4_sg_index, 0); -- sec4_sg_index += src_nents; -+ sec4_sg_index += src_nents ? : 1; - } - if (dst_nents) { - sg_to_sec4_sg_last(req->dst, dst_nents, - edesc->sec4_sg + sec4_sg_index, 0); - } -- edesc->sec4_sg_dma = dma_map_single(jrdev, edesc->sec4_sg, -- sec4_sg_bytes, DMA_TO_DEVICE); -- if (dma_mapping_error(jrdev, edesc->sec4_sg_dma)) { -- dev_err(jrdev, "unable to map S/G table\n"); -- return ERR_PTR(-ENOMEM); -- } -+ dma_sync_single_for_device(jrdev, edesc->sec4_sg_dma, sec4_sg_bytes, -+ DMA_TO_DEVICE); - - return edesc; - } -@@ -2762,9 +1268,11 @@ - u32 *desc; - int ret = 0; - -+ req->cryptlen += ctx->authsize; -+ - /* allocate extended descriptor */ - edesc = aead_edesc_alloc(req, DESC_JOB_IO_LEN * -- CAAM_CMD_SZ, &all_contig, true); -+ CAAM_CMD_SZ, &all_contig); - if (IS_ERR(edesc)) - return PTR_ERR(edesc); - -@@ -2772,7 +1280,7 @@ - init_aead_job(ctx->sh_desc_enc, ctx->sh_desc_enc_dma, edesc, req, - all_contig, true); - #ifdef DEBUG -- print_hex_dump(KERN_ERR, "aead jobdesc@"__stringify(__LINE__)": ", -+ print_hex_dump(KERN_ERR, "aead jobdesc@"xstr(__LINE__)": ", - DUMP_PREFIX_ADDRESS, 16, 4, edesc->hw_desc, - desc_bytes(edesc->hw_desc), 1); - #endif -@@ -2801,12 +1309,12 @@ - - /* allocate extended descriptor */ - edesc = aead_edesc_alloc(req, DESC_JOB_IO_LEN * -- CAAM_CMD_SZ, &all_contig, false); -+ CAAM_CMD_SZ, &all_contig); - if (IS_ERR(edesc)) - return PTR_ERR(edesc); - - #ifdef DEBUG -- print_hex_dump(KERN_ERR, "dec src@"__stringify(__LINE__)": ", -+ print_hex_dump(KERN_ERR, "dec src@"xstr(__LINE__)": ", - DUMP_PREFIX_ADDRESS, 16, 4, sg_virt(req->src), - req->cryptlen, 1); - #endif -@@ -2815,7 +1323,7 @@ - init_aead_job(ctx->sh_desc_dec, - ctx->sh_desc_dec_dma, edesc, req, all_contig, false); - #ifdef DEBUG -- print_hex_dump(KERN_ERR, "aead jobdesc@"__stringify(__LINE__)": ", -+ print_hex_dump(KERN_ERR, "aead jobdesc@"xstr(__LINE__)": ", - DUMP_PREFIX_ADDRESS, 16, 4, edesc->hw_desc, - desc_bytes(edesc->hw_desc), 1); - #endif -@@ -2853,17 +1361,15 @@ - int ivsize = crypto_aead_ivsize(aead); - bool assoc_chained = false, src_chained = false, dst_chained = false; - int sec4_sg_index, sec4_sg_len = 0, sec4_sg_bytes; -- bool is_gcm = false; - - assoc_nents = sg_count(req->assoc, req->assoclen, &assoc_chained); - src_nents = sg_count(req->src, req->cryptlen, &src_chained); - - if (unlikely(req->dst != req->src)) -- dst_nents = sg_count(req->dst, req->cryptlen + ctx->authsize, -- &dst_chained); -+ dst_nents = sg_count(req->dst, req->cryptlen, &dst_chained); - - sgc = dma_map_sg_chained(jrdev, req->assoc, assoc_nents ? : 1, -- DMA_TO_DEVICE, assoc_chained); -+ DMA_BIDIRECTIONAL, assoc_chained); - if (likely(req->src == req->dst)) { - sgc = dma_map_sg_chained(jrdev, req->src, src_nents ? : 1, - DMA_BIDIRECTIONAL, src_chained); -@@ -2874,64 +1380,32 @@ - DMA_FROM_DEVICE, dst_chained); - } - -+ /* Check if data are contiguous */ - iv_dma = dma_map_single(jrdev, greq->giv, ivsize, DMA_TO_DEVICE); -- if (dma_mapping_error(jrdev, iv_dma)) { -- dev_err(jrdev, "unable to map IV\n"); -- return ERR_PTR(-ENOMEM); -- } -- -- if (((ctx->class1_alg_type & OP_ALG_ALGSEL_MASK) == -- OP_ALG_ALGSEL_AES) && -- ((ctx->class1_alg_type & OP_ALG_AAI_MASK) == OP_ALG_AAI_GCM)) -- is_gcm = true; -- -- /* -- * Check if data are contiguous. -- * GCM expected input sequence: IV, AAD, text -- * All other - expected input sequence: AAD, IV, text -- */ -- -- if (is_gcm) { -- if (assoc_nents || iv_dma + ivsize != -- sg_dma_address(req->assoc) || src_nents || -- sg_dma_address(req->assoc) + req->assoclen != -- sg_dma_address(req->src)) -- contig &= ~GIV_SRC_CONTIG; -- } else { -- if (assoc_nents || -- sg_dma_address(req->assoc) + req->assoclen != iv_dma || -- src_nents || iv_dma + ivsize != sg_dma_address(req->src)) -- contig &= ~GIV_SRC_CONTIG; -- } -- -+ if (assoc_nents || sg_dma_address(req->assoc) + req->assoclen != -+ iv_dma || src_nents || iv_dma + ivsize != sg_dma_address(req->src)) -+ contig &= ~GIV_SRC_CONTIG; - if (dst_nents || iv_dma + ivsize != sg_dma_address(req->dst)) - contig &= ~GIV_DST_CONTIG; -- -+ if (unlikely(req->src != req->dst)) { -+ dst_nents = dst_nents ? : 1; -+ sec4_sg_len += 1; -+ } - if (!(contig & GIV_SRC_CONTIG)) { - assoc_nents = assoc_nents ? : 1; - src_nents = src_nents ? : 1; - sec4_sg_len += assoc_nents + 1 + src_nents; -- if (req->src == req->dst && -- (src_nents || iv_dma + ivsize != sg_dma_address(req->src))) -- contig &= ~GIV_DST_CONTIG; -- } -- -- /* -- * Add new sg entries for GCM output sequence. -- * Expected output sequence: IV, encrypted text. -- */ -- if (is_gcm && req->src == req->dst && !(contig & GIV_DST_CONTIG)) -- sec4_sg_len += 1 + src_nents; -- -- if (unlikely(req->src != req->dst)) { -- dst_nents = dst_nents ? : 1; -- sec4_sg_len += 1 + dst_nents; -+ if (likely(req->src == req->dst)) -+ contig &= ~GIV_DST_CONTIG; - } -+ sec4_sg_len += dst_nents; - - sec4_sg_bytes = sec4_sg_len * sizeof(struct sec4_sg_entry); - -+ dma_sync_single_for_device(jrdev, iv_dma, ivsize, DMA_TO_DEVICE); -+ - /* allocate space for base edesc and hw desc commands, link tables */ -- edesc = kmalloc(sizeof(struct aead_edesc) + desc_bytes + -+ edesc = kzalloc(sizeof(struct aead_edesc) + desc_bytes + - sec4_sg_bytes, GFP_DMA | flags); - if (!edesc) { - dev_err(jrdev, "could not allocate extended descriptor\n"); -@@ -2948,40 +1422,24 @@ - edesc->sec4_sg_bytes = sec4_sg_bytes; - edesc->sec4_sg = (void *)edesc + sizeof(struct aead_edesc) + - desc_bytes; -+ edesc->sec4_sg_dma = dma_map_single(jrdev, edesc->sec4_sg, -+ sec4_sg_bytes, DMA_TO_DEVICE); - *contig_ptr = contig; - - sec4_sg_index = 0; - if (!(contig & GIV_SRC_CONTIG)) { -- if (!is_gcm) { -- sg_to_sec4_sg(req->assoc, assoc_nents, -- edesc->sec4_sg + sec4_sg_index, 0); -- sec4_sg_index += assoc_nents; -- } -- -+ sg_to_sec4_sg(req->assoc, assoc_nents, -+ edesc->sec4_sg + -+ sec4_sg_index, 0); -+ sec4_sg_index += assoc_nents; - dma_to_sec4_sg_one(edesc->sec4_sg + sec4_sg_index, - iv_dma, ivsize, 0); - sec4_sg_index += 1; -- -- if (is_gcm) { -- sg_to_sec4_sg(req->assoc, assoc_nents, -- edesc->sec4_sg + sec4_sg_index, 0); -- sec4_sg_index += assoc_nents; -- } -- - sg_to_sec4_sg_last(req->src, src_nents, - edesc->sec4_sg + - sec4_sg_index, 0); - sec4_sg_index += src_nents; - } -- -- if (is_gcm && req->src == req->dst && !(contig & GIV_DST_CONTIG)) { -- dma_to_sec4_sg_one(edesc->sec4_sg + sec4_sg_index, -- iv_dma, ivsize, 0); -- sec4_sg_index += 1; -- sg_to_sec4_sg_last(req->src, src_nents, -- edesc->sec4_sg + sec4_sg_index, 0); -- } -- - if (unlikely(req->src != req->dst && !(contig & GIV_DST_CONTIG))) { - dma_to_sec4_sg_one(edesc->sec4_sg + sec4_sg_index, - iv_dma, ivsize, 0); -@@ -2989,12 +1447,8 @@ - sg_to_sec4_sg_last(req->dst, dst_nents, - edesc->sec4_sg + sec4_sg_index, 0); - } -- edesc->sec4_sg_dma = dma_map_single(jrdev, edesc->sec4_sg, -- sec4_sg_bytes, DMA_TO_DEVICE); -- if (dma_mapping_error(jrdev, edesc->sec4_sg_dma)) { -- dev_err(jrdev, "unable to map S/G table\n"); -- return ERR_PTR(-ENOMEM); -- } -+ dma_sync_single_for_device(jrdev, edesc->sec4_sg_dma, sec4_sg_bytes, -+ DMA_TO_DEVICE); - - return edesc; - } -@@ -3010,6 +1464,8 @@ - u32 *desc; - int ret = 0; - -+ req->cryptlen += ctx->authsize; -+ - /* allocate extended descriptor */ - edesc = aead_giv_edesc_alloc(areq, DESC_JOB_IO_LEN * - CAAM_CMD_SZ, &contig); -@@ -3018,7 +1474,7 @@ - return PTR_ERR(edesc); - - #ifdef DEBUG -- print_hex_dump(KERN_ERR, "giv src@"__stringify(__LINE__)": ", -+ print_hex_dump(KERN_ERR, "giv src@"xstr(__LINE__)": ", - DUMP_PREFIX_ADDRESS, 16, 4, sg_virt(req->src), - req->cryptlen, 1); - #endif -@@ -3027,7 +1483,7 @@ - init_aead_giv_job(ctx->sh_desc_givenc, - ctx->sh_desc_givenc_dma, edesc, req, contig); - #ifdef DEBUG -- print_hex_dump(KERN_ERR, "aead jobdesc@"__stringify(__LINE__)": ", -+ print_hex_dump(KERN_ERR, "aead jobdesc@"xstr(__LINE__)": ", - DUMP_PREFIX_ADDRESS, 16, 4, edesc->hw_desc, - desc_bytes(edesc->hw_desc), 1); - #endif -@@ -3044,11 +1500,6 @@ - return ret; - } - --static int aead_null_givencrypt(struct aead_givcrypt_request *areq) --{ -- return aead_encrypt(&areq->areq); --} -- - /* - * allocate and map the ablkcipher extended descriptor for ablkcipher - */ -@@ -3086,16 +1537,12 @@ - DMA_FROM_DEVICE, dst_chained); - } - -- iv_dma = dma_map_single(jrdev, req->info, ivsize, DMA_TO_DEVICE); -- if (dma_mapping_error(jrdev, iv_dma)) { -- dev_err(jrdev, "unable to map IV\n"); -- return ERR_PTR(-ENOMEM); -- } -- - /* - * Check if iv can be contiguous with source and destination. - * If so, include it. If not, create scatterlist. - */ -+ iv_dma = dma_map_single(jrdev, req->info, ivsize, DMA_TO_DEVICE); -+ dma_sync_single_for_device(jrdev, iv_dma, ivsize, DMA_TO_DEVICE); - if (!src_nents && iv_dma + ivsize == sg_dma_address(req->src)) - iv_contig = true; - else -@@ -3104,7 +1551,7 @@ - sizeof(struct sec4_sg_entry); - - /* allocate space for base edesc and hw desc commands, link tables */ -- edesc = kmalloc(sizeof(struct ablkcipher_edesc) + desc_bytes + -+ edesc = kzalloc(sizeof(struct ablkcipher_edesc) + desc_bytes + - sec4_sg_bytes, GFP_DMA | flags); - if (!edesc) { - dev_err(jrdev, "could not allocate extended descriptor\n"); -@@ -3134,15 +1581,13 @@ - - edesc->sec4_sg_dma = dma_map_single(jrdev, edesc->sec4_sg, - sec4_sg_bytes, DMA_TO_DEVICE); -- if (dma_mapping_error(jrdev, edesc->sec4_sg_dma)) { -- dev_err(jrdev, "unable to map S/G table\n"); -- return ERR_PTR(-ENOMEM); -- } -- - edesc->iv_dma = iv_dma; - -+ dma_sync_single_for_device(jrdev, edesc->sec4_sg_dma, sec4_sg_bytes, -+ DMA_TO_DEVICE); -+ - #ifdef DEBUG -- print_hex_dump(KERN_ERR, "ablkcipher sec4_sg@"__stringify(__LINE__)": ", -+ print_hex_dump(KERN_ERR, "ablkcipher sec4_sg@"xstr(__LINE__)": ", - DUMP_PREFIX_ADDRESS, 16, 4, edesc->sec4_sg, - sec4_sg_bytes, 1); - #endif -@@ -3171,7 +1616,7 @@ - init_ablkcipher_job(ctx->sh_desc_enc, - ctx->sh_desc_enc_dma, edesc, req, iv_contig); - #ifdef DEBUG -- print_hex_dump(KERN_ERR, "ablkcipher jobdesc@"__stringify(__LINE__)": ", -+ print_hex_dump(KERN_ERR, "ablkcipher jobdesc@"xstr(__LINE__)": ", - DUMP_PREFIX_ADDRESS, 16, 4, edesc->hw_desc, - desc_bytes(edesc->hw_desc), 1); - #endif -@@ -3209,7 +1654,7 @@ - ctx->sh_desc_dec_dma, edesc, req, iv_contig); - desc = edesc->hw_desc; - #ifdef DEBUG -- print_hex_dump(KERN_ERR, "ablkcipher jobdesc@"__stringify(__LINE__)": ", -+ print_hex_dump(KERN_ERR, "ablkcipher jobdesc@"xstr(__LINE__)": ", - DUMP_PREFIX_ADDRESS, 16, 4, edesc->hw_desc, - desc_bytes(edesc->hw_desc), 1); - #endif -@@ -3225,291 +1670,28 @@ - return ret; - } - --/* -- * allocate and map the ablkcipher extended descriptor -- * for ablkcipher givencrypt -- */ --static struct ablkcipher_edesc *ablkcipher_giv_edesc_alloc( -- struct skcipher_givcrypt_request *greq, -- int desc_bytes, -- bool *iv_contig_out) --{ -- struct ablkcipher_request *req = &greq->creq; -- struct crypto_ablkcipher *ablkcipher = crypto_ablkcipher_reqtfm(req); -- struct caam_ctx *ctx = crypto_ablkcipher_ctx(ablkcipher); -- struct device *jrdev = ctx->jrdev; -- gfp_t flags = (req->base.flags & (CRYPTO_TFM_REQ_MAY_BACKLOG | -- CRYPTO_TFM_REQ_MAY_SLEEP)) ? -- GFP_KERNEL : GFP_ATOMIC; -- int src_nents, dst_nents = 0, sec4_sg_bytes; -- struct ablkcipher_edesc *edesc; -- dma_addr_t iv_dma = 0; -- bool iv_contig = false; -- int sgc; -- int ivsize = crypto_ablkcipher_ivsize(ablkcipher); -- bool src_chained = false, dst_chained = false; -- int sec4_sg_index; -- -- src_nents = sg_count(req->src, req->nbytes, &src_chained); -- -- if (unlikely(req->dst != req->src)) -- dst_nents = sg_count(req->dst, req->nbytes, &dst_chained); -- -- if (likely(req->src == req->dst)) { -- sgc = dma_map_sg_chained(jrdev, req->src, src_nents ? : 1, -- DMA_BIDIRECTIONAL, src_chained); -- } else { -- sgc = dma_map_sg_chained(jrdev, req->src, src_nents ? : 1, -- DMA_TO_DEVICE, src_chained); -- sgc = dma_map_sg_chained(jrdev, req->dst, dst_nents ? : 1, -- DMA_FROM_DEVICE, dst_chained); -- } -- -- /* -- * Check if iv can be contiguous with source and destination. -- * If so, include it. If not, create scatterlist. -- */ -- iv_dma = dma_map_single(jrdev, greq->giv, ivsize, DMA_TO_DEVICE); -- if (dma_mapping_error(jrdev, iv_dma)) { -- dev_err(jrdev, "unable to map IV\n"); -- return ERR_PTR(-ENOMEM); -- } -- -- if (!dst_nents && iv_dma + ivsize == sg_dma_address(req->dst)) -- iv_contig = true; -- else -- dst_nents = dst_nents ? : 1; -- sec4_sg_bytes = ((iv_contig ? 0 : 1) + src_nents + dst_nents) * -- sizeof(struct sec4_sg_entry); -- -- /* allocate space for base edesc and hw desc commands, link tables */ -- edesc = kmalloc(sizeof(*edesc) + desc_bytes + -- sec4_sg_bytes, GFP_DMA | flags); -- if (!edesc) { -- dev_err(jrdev, "could not allocate extended descriptor\n"); -- return ERR_PTR(-ENOMEM); -- } -- -- edesc->src_nents = src_nents; -- edesc->src_chained = src_chained; -- edesc->dst_nents = dst_nents; -- edesc->dst_chained = dst_chained; -- edesc->sec4_sg_bytes = sec4_sg_bytes; -- edesc->sec4_sg = (void *)edesc + sizeof(struct ablkcipher_edesc) + -- desc_bytes; -- -- sec4_sg_index = 0; -- if (src_nents) { -- sg_to_sec4_sg_last(req->src, src_nents, edesc->sec4_sg, 0); -- sec4_sg_index += src_nents; -- } -- -- if (!iv_contig) { -- dma_to_sec4_sg_one(edesc->sec4_sg + sec4_sg_index, -- iv_dma, ivsize, 0); -- sec4_sg_index += 1; -- sg_to_sec4_sg_last(req->dst, dst_nents, -- edesc->sec4_sg + sec4_sg_index, 0); -- } -- -- edesc->sec4_sg_dma = dma_map_single(jrdev, edesc->sec4_sg, -- sec4_sg_bytes, DMA_TO_DEVICE); -- if (dma_mapping_error(jrdev, edesc->sec4_sg_dma)) { -- dev_err(jrdev, "unable to map S/G table\n"); -- return ERR_PTR(-ENOMEM); -- } -- edesc->iv_dma = iv_dma; -- --#ifdef DEBUG -- print_hex_dump(KERN_ERR, -- "ablkcipher sec4_sg@" __stringify(__LINE__) ": ", -- DUMP_PREFIX_ADDRESS, 16, 4, edesc->sec4_sg, -- sec4_sg_bytes, 1); --#endif -- -- *iv_contig_out = iv_contig; -- return edesc; --} -- --static int ablkcipher_givencrypt(struct skcipher_givcrypt_request *creq) --{ -- struct ablkcipher_request *req = &creq->creq; -- struct ablkcipher_edesc *edesc; -- struct crypto_ablkcipher *ablkcipher = crypto_ablkcipher_reqtfm(req); -- struct caam_ctx *ctx = crypto_ablkcipher_ctx(ablkcipher); -- struct device *jrdev = ctx->jrdev; -- bool iv_contig; -- u32 *desc; -- int ret = 0; -- -- /* allocate extended descriptor */ -- edesc = ablkcipher_giv_edesc_alloc(creq, DESC_JOB_IO_LEN * -- CAAM_CMD_SZ, &iv_contig); -- if (IS_ERR(edesc)) -- return PTR_ERR(edesc); -- -- /* Create and submit job descriptor*/ -- init_ablkcipher_giv_job(ctx->sh_desc_givenc, ctx->sh_desc_givenc_dma, -- edesc, req, iv_contig); --#ifdef DEBUG -- print_hex_dump(KERN_ERR, -- "ablkcipher jobdesc@" __stringify(__LINE__) ": ", -- DUMP_PREFIX_ADDRESS, 16, 4, edesc->hw_desc, -- desc_bytes(edesc->hw_desc), 1); --#endif -- desc = edesc->hw_desc; -- ret = caam_jr_enqueue(jrdev, desc, ablkcipher_encrypt_done, req); -- -- if (!ret) { -- ret = -EINPROGRESS; -- } else { -- ablkcipher_unmap(jrdev, edesc, req); -- kfree(edesc); -- } -- -- return ret; --} -- - #define template_aead template_u.aead - #define template_ablkcipher template_u.ablkcipher - struct caam_alg_template { - char name[CRYPTO_MAX_ALG_NAME]; -- char driver_name[CRYPTO_MAX_ALG_NAME]; -- unsigned int blocksize; -- u32 type; -- union { -- struct ablkcipher_alg ablkcipher; -- struct aead_alg aead; -- struct blkcipher_alg blkcipher; -- struct cipher_alg cipher; -- struct compress_alg compress; -- struct rng_alg rng; -- } template_u; -- u32 class1_alg_type; -- u32 class2_alg_type; -- u32 alg_op; --}; -- --static struct caam_alg_template driver_algs[] = { -- /* single-pass ipsec_esp descriptor */ -- { -- .name = "authenc(hmac(md5),ecb(cipher_null))", -- .driver_name = "authenc-hmac-md5-ecb-cipher_null-caam", -- .blocksize = NULL_BLOCK_SIZE, -- .type = CRYPTO_ALG_TYPE_AEAD, -- .template_aead = { -- .setkey = aead_setkey, -- .setauthsize = aead_setauthsize, -- .encrypt = aead_encrypt, -- .decrypt = aead_decrypt, -- .givencrypt = aead_null_givencrypt, -- .geniv = "", -- .ivsize = NULL_IV_SIZE, -- .maxauthsize = MD5_DIGEST_SIZE, -- }, -- .class1_alg_type = 0, -- .class2_alg_type = OP_ALG_ALGSEL_MD5 | OP_ALG_AAI_HMAC_PRECOMP, -- .alg_op = OP_ALG_ALGSEL_MD5 | OP_ALG_AAI_HMAC, -- }, -- { -- .name = "authenc(hmac(sha1),ecb(cipher_null))", -- .driver_name = "authenc-hmac-sha1-ecb-cipher_null-caam", -- .blocksize = NULL_BLOCK_SIZE, -- .type = CRYPTO_ALG_TYPE_AEAD, -- .template_aead = { -- .setkey = aead_setkey, -- .setauthsize = aead_setauthsize, -- .encrypt = aead_encrypt, -- .decrypt = aead_decrypt, -- .givencrypt = aead_null_givencrypt, -- .geniv = "", -- .ivsize = NULL_IV_SIZE, -- .maxauthsize = SHA1_DIGEST_SIZE, -- }, -- .class1_alg_type = 0, -- .class2_alg_type = OP_ALG_ALGSEL_SHA1 | OP_ALG_AAI_HMAC_PRECOMP, -- .alg_op = OP_ALG_ALGSEL_SHA1 | OP_ALG_AAI_HMAC, -- }, -- { -- .name = "authenc(hmac(sha224),ecb(cipher_null))", -- .driver_name = "authenc-hmac-sha224-ecb-cipher_null-caam", -- .blocksize = NULL_BLOCK_SIZE, -- .type = CRYPTO_ALG_TYPE_AEAD, -- .template_aead = { -- .setkey = aead_setkey, -- .setauthsize = aead_setauthsize, -- .encrypt = aead_encrypt, -- .decrypt = aead_decrypt, -- .givencrypt = aead_null_givencrypt, -- .geniv = "", -- .ivsize = NULL_IV_SIZE, -- .maxauthsize = SHA224_DIGEST_SIZE, -- }, -- .class1_alg_type = 0, -- .class2_alg_type = OP_ALG_ALGSEL_SHA224 | -- OP_ALG_AAI_HMAC_PRECOMP, -- .alg_op = OP_ALG_ALGSEL_SHA224 | OP_ALG_AAI_HMAC, -- }, -- { -- .name = "authenc(hmac(sha256),ecb(cipher_null))", -- .driver_name = "authenc-hmac-sha256-ecb-cipher_null-caam", -- .blocksize = NULL_BLOCK_SIZE, -- .type = CRYPTO_ALG_TYPE_AEAD, -- .template_aead = { -- .setkey = aead_setkey, -- .setauthsize = aead_setauthsize, -- .encrypt = aead_encrypt, -- .decrypt = aead_decrypt, -- .givencrypt = aead_null_givencrypt, -- .geniv = "", -- .ivsize = NULL_IV_SIZE, -- .maxauthsize = SHA256_DIGEST_SIZE, -- }, -- .class1_alg_type = 0, -- .class2_alg_type = OP_ALG_ALGSEL_SHA256 | -- OP_ALG_AAI_HMAC_PRECOMP, -- .alg_op = OP_ALG_ALGSEL_SHA256 | OP_ALG_AAI_HMAC, -- }, -- { -- .name = "authenc(hmac(sha384),ecb(cipher_null))", -- .driver_name = "authenc-hmac-sha384-ecb-cipher_null-caam", -- .blocksize = NULL_BLOCK_SIZE, -- .type = CRYPTO_ALG_TYPE_AEAD, -- .template_aead = { -- .setkey = aead_setkey, -- .setauthsize = aead_setauthsize, -- .encrypt = aead_encrypt, -- .decrypt = aead_decrypt, -- .givencrypt = aead_null_givencrypt, -- .geniv = "", -- .ivsize = NULL_IV_SIZE, -- .maxauthsize = SHA384_DIGEST_SIZE, -- }, -- .class1_alg_type = 0, -- .class2_alg_type = OP_ALG_ALGSEL_SHA384 | -- OP_ALG_AAI_HMAC_PRECOMP, -- .alg_op = OP_ALG_ALGSEL_SHA384 | OP_ALG_AAI_HMAC, -- }, -- { -- .name = "authenc(hmac(sha512),ecb(cipher_null))", -- .driver_name = "authenc-hmac-sha512-ecb-cipher_null-caam", -- .blocksize = NULL_BLOCK_SIZE, -- .type = CRYPTO_ALG_TYPE_AEAD, -- .template_aead = { -- .setkey = aead_setkey, -- .setauthsize = aead_setauthsize, -- .encrypt = aead_encrypt, -- .decrypt = aead_decrypt, -- .givencrypt = aead_null_givencrypt, -- .geniv = "", -- .ivsize = NULL_IV_SIZE, -- .maxauthsize = SHA512_DIGEST_SIZE, -- }, -- .class1_alg_type = 0, -- .class2_alg_type = OP_ALG_ALGSEL_SHA512 | -- OP_ALG_AAI_HMAC_PRECOMP, -- .alg_op = OP_ALG_ALGSEL_SHA512 | OP_ALG_AAI_HMAC, -- }, -+ char driver_name[CRYPTO_MAX_ALG_NAME]; -+ unsigned int blocksize; -+ u32 type; -+ union { -+ struct ablkcipher_alg ablkcipher; -+ struct aead_alg aead; -+ struct blkcipher_alg blkcipher; -+ struct cipher_alg cipher; -+ struct compress_alg compress; -+ struct rng_alg rng; -+ } template_u; -+ u32 class1_alg_type; -+ u32 class2_alg_type; -+ u32 alg_op; -+}; -+ -+static struct caam_alg_template driver_algs[] = { -+ /* single-pass ipsec_esp descriptor */ - { - .name = "authenc(hmac(md5),cbc(aes))", - .driver_name = "authenc-hmac-md5-cbc-aes-caam", -@@ -3865,188 +2047,81 @@ - OP_ALG_AAI_HMAC_PRECOMP, - .alg_op = OP_ALG_ALGSEL_SHA512 | OP_ALG_AAI_HMAC, - }, -+ /* ablkcipher descriptor */ - { -- .name = "authenc(hmac(md5),rfc3686(ctr(aes)))", -- .driver_name = "authenc-hmac-md5-rfc3686-ctr-aes-caam", -- .blocksize = 1, -- .type = CRYPTO_ALG_TYPE_AEAD, -- .template_aead = { -- .setkey = aead_setkey, -- .setauthsize = aead_setauthsize, -- .encrypt = aead_encrypt, -- .decrypt = aead_decrypt, -- .givencrypt = aead_givencrypt, -- .geniv = "", -- .ivsize = CTR_RFC3686_IV_SIZE, -- .maxauthsize = MD5_DIGEST_SIZE, -- }, -- .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CTR_MOD128, -- .class2_alg_type = OP_ALG_ALGSEL_MD5 | OP_ALG_AAI_HMAC_PRECOMP, -- .alg_op = OP_ALG_ALGSEL_MD5 | OP_ALG_AAI_HMAC, -- }, -- { -- .name = "authenc(hmac(sha1),rfc3686(ctr(aes)))", -- .driver_name = "authenc-hmac-sha1-rfc3686-ctr-aes-caam", -- .blocksize = 1, -- .type = CRYPTO_ALG_TYPE_AEAD, -- .template_aead = { -- .setkey = aead_setkey, -- .setauthsize = aead_setauthsize, -- .encrypt = aead_encrypt, -- .decrypt = aead_decrypt, -- .givencrypt = aead_givencrypt, -- .geniv = "", -- .ivsize = CTR_RFC3686_IV_SIZE, -- .maxauthsize = SHA1_DIGEST_SIZE, -- }, -- .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CTR_MOD128, -- .class2_alg_type = OP_ALG_ALGSEL_SHA1 | OP_ALG_AAI_HMAC_PRECOMP, -- .alg_op = OP_ALG_ALGSEL_SHA1 | OP_ALG_AAI_HMAC, -- }, -- { -- .name = "authenc(hmac(sha224),rfc3686(ctr(aes)))", -- .driver_name = "authenc-hmac-sha224-rfc3686-ctr-aes-caam", -- .blocksize = 1, -- .type = CRYPTO_ALG_TYPE_AEAD, -- .template_aead = { -- .setkey = aead_setkey, -- .setauthsize = aead_setauthsize, -- .encrypt = aead_encrypt, -- .decrypt = aead_decrypt, -- .givencrypt = aead_givencrypt, -- .geniv = "", -- .ivsize = CTR_RFC3686_IV_SIZE, -- .maxauthsize = SHA224_DIGEST_SIZE, -+ .name = "ecb(des)", -+ .driver_name = "ecb-des-caam", -+ .blocksize = DES_BLOCK_SIZE, -+ .type = CRYPTO_ALG_TYPE_ABLKCIPHER, -+ .template_ablkcipher = { -+ .setkey = ablkcipher_setkey, -+ .encrypt = ablkcipher_encrypt, -+ .decrypt = ablkcipher_decrypt, -+ .geniv = "eseqiv", -+ .min_keysize = DES_KEY_SIZE, -+ .max_keysize = DES_KEY_SIZE, -+ .ivsize = DES_BLOCK_SIZE, - }, -- .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CTR_MOD128, -- .class2_alg_type = OP_ALG_ALGSEL_SHA224 | -- OP_ALG_AAI_HMAC_PRECOMP, -- .alg_op = OP_ALG_ALGSEL_SHA224 | OP_ALG_AAI_HMAC, -+ .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_ECB, - }, - { -- .name = "authenc(hmac(sha256),rfc3686(ctr(aes)))", -- .driver_name = "authenc-hmac-sha256-rfc3686-ctr-aes-caam", -- .blocksize = 1, -- .type = CRYPTO_ALG_TYPE_AEAD, -- .template_aead = { -- .setkey = aead_setkey, -- .setauthsize = aead_setauthsize, -- .encrypt = aead_encrypt, -- .decrypt = aead_decrypt, -- .givencrypt = aead_givencrypt, -- .geniv = "", -- .ivsize = CTR_RFC3686_IV_SIZE, -- .maxauthsize = SHA256_DIGEST_SIZE, -- }, -- .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CTR_MOD128, -- .class2_alg_type = OP_ALG_ALGSEL_SHA256 | -- OP_ALG_AAI_HMAC_PRECOMP, -- .alg_op = OP_ALG_ALGSEL_SHA256 | OP_ALG_AAI_HMAC, -+ .name = "ecb(arc4)", -+ .driver_name = "ecb-arc4-caam", -+ .blocksize = ARC4_BLOCK_SIZE, -+ .type = CRYPTO_ALG_TYPE_ABLKCIPHER, -+ .template_ablkcipher = { -+ .setkey = ablkcipher_setkey, -+ .encrypt = ablkcipher_encrypt, -+ .decrypt = ablkcipher_decrypt, -+ .geniv = "eseqiv", -+ .min_keysize = ARC4_MIN_KEY_SIZE, -+ .max_keysize = ARC4_MAX_KEY_SIZE, -+ .ivsize = ARC4_BLOCK_SIZE, -+ }, -+ .class1_alg_type = OP_ALG_ALGSEL_ARC4 | OP_ALG_AAI_ECB - }, - { -- .name = "authenc(hmac(sha384),rfc3686(ctr(aes)))", -- .driver_name = "authenc-hmac-sha384-rfc3686-ctr-aes-caam", -- .blocksize = 1, -- .type = CRYPTO_ALG_TYPE_AEAD, -- .template_aead = { -- .setkey = aead_setkey, -- .setauthsize = aead_setauthsize, -- .encrypt = aead_encrypt, -- .decrypt = aead_decrypt, -- .givencrypt = aead_givencrypt, -- .geniv = "", -- .ivsize = CTR_RFC3686_IV_SIZE, -- .maxauthsize = SHA384_DIGEST_SIZE, -+ .name = "ecb(aes)", -+ .driver_name = "ecb-aes-caam", -+ .blocksize = AES_BLOCK_SIZE, -+ .type = CRYPTO_ALG_TYPE_ABLKCIPHER, -+ .template_ablkcipher = { -+ .setkey = ablkcipher_setkey, -+ .encrypt = ablkcipher_encrypt, -+ .decrypt = ablkcipher_decrypt, -+ .geniv = "eseqiv", -+ .min_keysize = AES_MIN_KEY_SIZE, -+ .max_keysize = AES_MAX_KEY_SIZE, -+ .ivsize = AES_BLOCK_SIZE, - }, -- .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CTR_MOD128, -- .class2_alg_type = OP_ALG_ALGSEL_SHA384 | -- OP_ALG_AAI_HMAC_PRECOMP, -- .alg_op = OP_ALG_ALGSEL_SHA384 | OP_ALG_AAI_HMAC, -+ .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_ECB, - }, - { -- .name = "authenc(hmac(sha512),rfc3686(ctr(aes)))", -- .driver_name = "authenc-hmac-sha512-rfc3686-ctr-aes-caam", -- .blocksize = 1, -- .type = CRYPTO_ALG_TYPE_AEAD, -- .template_aead = { -- .setkey = aead_setkey, -- .setauthsize = aead_setauthsize, -- .encrypt = aead_encrypt, -- .decrypt = aead_decrypt, -- .givencrypt = aead_givencrypt, -- .geniv = "", -- .ivsize = CTR_RFC3686_IV_SIZE, -- .maxauthsize = SHA512_DIGEST_SIZE, -+ .name = "ctr(aes)", -+ .driver_name = "ctr-aes-caam", -+ .blocksize = AES_BLOCK_SIZE, -+ .type = CRYPTO_ALG_TYPE_ABLKCIPHER, -+ .template_ablkcipher = { -+ .setkey = ablkcipher_setkey, -+ .encrypt = ablkcipher_encrypt, -+ .decrypt = ablkcipher_decrypt, -+ .geniv = "eseqiv", -+ .min_keysize = AES_MIN_KEY_SIZE, -+ .max_keysize = AES_MAX_KEY_SIZE, -+ .ivsize = AES_BLOCK_SIZE, - }, - .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CTR_MOD128, -- .class2_alg_type = OP_ALG_ALGSEL_SHA512 | -- OP_ALG_AAI_HMAC_PRECOMP, -- .alg_op = OP_ALG_ALGSEL_SHA512 | OP_ALG_AAI_HMAC, -- }, -- { -- .name = "rfc4106(gcm(aes))", -- .driver_name = "rfc4106-gcm-aes-caam", -- .blocksize = 1, -- .type = CRYPTO_ALG_TYPE_AEAD, -- .template_aead = { -- .setkey = rfc4106_setkey, -- .setauthsize = rfc4106_setauthsize, -- .encrypt = aead_encrypt, -- .decrypt = aead_decrypt, -- .givencrypt = aead_givencrypt, -- .geniv = "", -- .ivsize = 8, -- .maxauthsize = AES_BLOCK_SIZE, -- }, -- .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_GCM, -- }, -- { -- .name = "rfc4543(gcm(aes))", -- .driver_name = "rfc4543-gcm-aes-caam", -- .blocksize = 1, -- .type = CRYPTO_ALG_TYPE_AEAD, -- .template_aead = { -- .setkey = rfc4543_setkey, -- .setauthsize = rfc4543_setauthsize, -- .encrypt = aead_encrypt, -- .decrypt = aead_decrypt, -- .givencrypt = aead_givencrypt, -- .geniv = "", -- .ivsize = 8, -- .maxauthsize = AES_BLOCK_SIZE, -- }, -- .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_GCM, -- }, -- /* Galois Counter Mode */ -- { -- .name = "gcm(aes)", -- .driver_name = "gcm-aes-caam", -- .blocksize = 1, -- .type = CRYPTO_ALG_TYPE_AEAD, -- .template_aead = { -- .setkey = gcm_setkey, -- .setauthsize = gcm_setauthsize, -- .encrypt = aead_encrypt, -- .decrypt = aead_decrypt, -- .givencrypt = NULL, -- .geniv = "", -- .ivsize = 12, -- .maxauthsize = AES_BLOCK_SIZE, -- }, -- .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_GCM, - }, -- /* ablkcipher descriptor */ - { - .name = "cbc(aes)", - .driver_name = "cbc-aes-caam", - .blocksize = AES_BLOCK_SIZE, -- .type = CRYPTO_ALG_TYPE_GIVCIPHER, -+ .type = CRYPTO_ALG_TYPE_ABLKCIPHER, - .template_ablkcipher = { - .setkey = ablkcipher_setkey, - .encrypt = ablkcipher_encrypt, - .decrypt = ablkcipher_decrypt, -- .givencrypt = ablkcipher_givencrypt, -- .geniv = "", -+ .geniv = "eseqiv", - .min_keysize = AES_MIN_KEY_SIZE, - .max_keysize = AES_MAX_KEY_SIZE, - .ivsize = AES_BLOCK_SIZE, -@@ -4054,16 +2129,31 @@ - .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC, - }, - { -+ .name = "ecb(des3_ede)", -+ .driver_name = "ecb-des3-caam", -+ .blocksize = DES3_EDE_BLOCK_SIZE, -+ .type = CRYPTO_ALG_TYPE_ABLKCIPHER, -+ .template_ablkcipher = { -+ .setkey = ablkcipher_setkey, -+ .encrypt = ablkcipher_encrypt, -+ .decrypt = ablkcipher_decrypt, -+ .geniv = "eseqiv", -+ .min_keysize = DES3_EDE_KEY_SIZE, -+ .max_keysize = DES3_EDE_KEY_SIZE, -+ .ivsize = DES3_EDE_BLOCK_SIZE, -+ }, -+ .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_ECB, -+ }, -+ { - .name = "cbc(des3_ede)", - .driver_name = "cbc-3des-caam", - .blocksize = DES3_EDE_BLOCK_SIZE, -- .type = CRYPTO_ALG_TYPE_GIVCIPHER, -+ .type = CRYPTO_ALG_TYPE_ABLKCIPHER, - .template_ablkcipher = { - .setkey = ablkcipher_setkey, - .encrypt = ablkcipher_encrypt, - .decrypt = ablkcipher_decrypt, -- .givencrypt = ablkcipher_givencrypt, -- .geniv = "", -+ .geniv = "eseqiv", - .min_keysize = DES3_EDE_KEY_SIZE, - .max_keysize = DES3_EDE_KEY_SIZE, - .ivsize = DES3_EDE_BLOCK_SIZE, -@@ -4074,58 +2164,23 @@ - .name = "cbc(des)", - .driver_name = "cbc-des-caam", - .blocksize = DES_BLOCK_SIZE, -- .type = CRYPTO_ALG_TYPE_GIVCIPHER, -+ .type = CRYPTO_ALG_TYPE_ABLKCIPHER, - .template_ablkcipher = { - .setkey = ablkcipher_setkey, - .encrypt = ablkcipher_encrypt, - .decrypt = ablkcipher_decrypt, -- .givencrypt = ablkcipher_givencrypt, -- .geniv = "", -+ .geniv = "eseqiv", - .min_keysize = DES_KEY_SIZE, - .max_keysize = DES_KEY_SIZE, - .ivsize = DES_BLOCK_SIZE, - }, - .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC, -- }, -- { -- .name = "ctr(aes)", -- .driver_name = "ctr-aes-caam", -- .blocksize = 1, -- .type = CRYPTO_ALG_TYPE_ABLKCIPHER, -- .template_ablkcipher = { -- .setkey = ablkcipher_setkey, -- .encrypt = ablkcipher_encrypt, -- .decrypt = ablkcipher_decrypt, -- .geniv = "chainiv", -- .min_keysize = AES_MIN_KEY_SIZE, -- .max_keysize = AES_MAX_KEY_SIZE, -- .ivsize = AES_BLOCK_SIZE, -- }, -- .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CTR_MOD128, -- }, -- { -- .name = "rfc3686(ctr(aes))", -- .driver_name = "rfc3686-ctr-aes-caam", -- .blocksize = 1, -- .type = CRYPTO_ALG_TYPE_GIVCIPHER, -- .template_ablkcipher = { -- .setkey = ablkcipher_setkey, -- .encrypt = ablkcipher_encrypt, -- .decrypt = ablkcipher_decrypt, -- .givencrypt = ablkcipher_givencrypt, -- .geniv = "", -- .min_keysize = AES_MIN_KEY_SIZE + -- CTR_RFC3686_NONCE_SIZE, -- .max_keysize = AES_MAX_KEY_SIZE + -- CTR_RFC3686_NONCE_SIZE, -- .ivsize = CTR_RFC3686_IV_SIZE, -- }, -- .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CTR_MOD128, - } - }; - - struct caam_crypto_alg { - struct list_head entry; -+ struct device *ctrldev; - int class1_alg_type; - int class2_alg_type; - int alg_op; -@@ -4138,12 +2193,14 @@ - struct caam_crypto_alg *caam_alg = - container_of(alg, struct caam_crypto_alg, crypto_alg); - struct caam_ctx *ctx = crypto_tfm_ctx(tfm); -+ struct caam_drv_private *priv = dev_get_drvdata(caam_alg->ctrldev); -+ int tgt_jr = atomic_inc_return(&priv->tfm_count); - -- ctx->jrdev = caam_jr_alloc(); -- if (IS_ERR(ctx->jrdev)) { -- pr_err("Job Ring Device allocation for transform failed\n"); -- return PTR_ERR(ctx->jrdev); -- } -+ /* -+ * distribute tfms across job rings to ensure in-order -+ * crypto request processing per tfm -+ */ -+ ctx->jrdev = priv->algapi_jr[(tgt_jr / 2) % priv->num_jrs_for_algapi]; - - /* copy descriptor header template value */ - ctx->class1_alg_type = OP_TYPE_CLASS1_ALG | caam_alg->class1_alg_type; -@@ -4170,31 +2227,57 @@ - dma_unmap_single(ctx->jrdev, ctx->sh_desc_givenc_dma, - desc_bytes(ctx->sh_desc_givenc), - DMA_TO_DEVICE); -- if (ctx->key_dma && -- !dma_mapping_error(ctx->jrdev, ctx->key_dma)) -- dma_unmap_single(ctx->jrdev, ctx->key_dma, -- ctx->enckeylen + ctx->split_key_pad_len, -- DMA_TO_DEVICE); -- -- caam_jr_free(ctx->jrdev); - } - - static void __exit caam_algapi_exit(void) - { - -+ struct device_node *dev_node; -+ struct platform_device *pdev; -+ struct device *ctrldev; -+ struct caam_drv_private *priv; - struct caam_crypto_alg *t_alg, *n; -+ int i, err; -+ -+ dev_node = of_find_compatible_node(NULL, NULL, "fsl,sec-v4.0"); -+ if (!dev_node) { -+ dev_node = of_find_compatible_node(NULL, NULL, "fsl,sec4.0"); -+ if (!dev_node) -+ return; -+ } -+ -+ pdev = of_find_device_by_node(dev_node); -+ if (!pdev) { -+ of_node_put(dev_node); -+ return; -+ } - -- if (!alg_list.next) -+ ctrldev = &pdev->dev; -+ priv = dev_get_drvdata(ctrldev); -+ -+ if (!priv->alg_list.next) { -+ of_node_put(dev_node); - return; -+ } - -- list_for_each_entry_safe(t_alg, n, &alg_list, entry) { -+ list_for_each_entry_safe(t_alg, n, &priv->alg_list, entry) { - crypto_unregister_alg(&t_alg->crypto_alg); - list_del(&t_alg->entry); - kfree(t_alg); - } -+ -+ for (i = 0; i < priv->total_jobrs; i++) { -+ err = caam_jr_deregister(priv->algapi_jr[i]); -+ if (err < 0) -+ break; -+ } -+ kfree(priv->algapi_jr); -+ -+ of_node_put(dev_node); - } - --static struct caam_crypto_alg *caam_alg_alloc(struct caam_alg_template -+static struct caam_crypto_alg *caam_alg_alloc(struct device *ctrldev, -+ struct caam_alg_template - *template) - { - struct caam_crypto_alg *t_alg; -@@ -4202,7 +2285,7 @@ - - t_alg = kzalloc(sizeof(struct caam_crypto_alg), GFP_KERNEL); - if (!t_alg) { -- pr_err("failed to allocate t_alg\n"); -+ dev_err(ctrldev, "failed to allocate t_alg\n"); - return ERR_PTR(-ENOMEM); - } - -@@ -4218,13 +2301,13 @@ - alg->cra_blocksize = template->blocksize; - alg->cra_alignmask = 0; - alg->cra_ctxsize = sizeof(struct caam_ctx); -- alg->cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_KERN_DRIVER_ONLY | -- template->type; -+ alg->cra_flags = CRYPTO_ALG_ASYNC | template->type; -+ -+#ifdef CRYPTO_ALG_KERN_DRIVER_ONLY -+ alg->cra_flags |= CRYPTO_ALG_KERN_DRIVER_ONLY; -+#endif -+ - switch (template->type) { -- case CRYPTO_ALG_TYPE_GIVCIPHER: -- alg->cra_type = &crypto_givcipher_type; -- alg->cra_ablkcipher = template->template_ablkcipher; -- break; - case CRYPTO_ALG_TYPE_ABLKCIPHER: - alg->cra_type = &crypto_ablkcipher_type; - alg->cra_ablkcipher = template->template_ablkcipher; -@@ -4238,6 +2321,7 @@ - t_alg->class1_alg_type = template->class1_alg_type; - t_alg->class2_alg_type = template->class2_alg_type; - t_alg->alg_op = template->alg_op; -+ t_alg->ctrldev = ctrldev; - - return t_alg; - } -@@ -4246,9 +2330,11 @@ - { - struct device_node *dev_node; - struct platform_device *pdev; -- struct device *ctrldev; -- void *priv; -- int i = 0, err = 0; -+ struct device *ctrldev, **jrdev; -+ struct caam_drv_private *priv; -+ int i = 0, err = 0, md_limit = 0; -+ int des_inst, aes_inst, md_inst; -+ u64 cha_inst; - - dev_node = of_find_compatible_node(NULL, NULL, "fsl,sec-v4.0"); - if (!dev_node) { -@@ -4265,42 +2351,117 @@ - - ctrldev = &pdev->dev; - priv = dev_get_drvdata(ctrldev); -- of_node_put(dev_node); - -- /* -- * If priv is NULL, it's probably because the caam driver wasn't -- * properly initialized (e.g. RNG4 init failed). Thus, bail out here. -- */ -- if (!priv) -- return -ENODEV; -+ INIT_LIST_HEAD(&priv->alg_list); -+ -+ jrdev = kmalloc(sizeof(*jrdev) * priv->total_jobrs, GFP_ATOMIC); -+ if (!jrdev) { -+ of_node_put(dev_node); -+ return -ENOMEM; -+ } - -+ for (i = 0; i < priv->total_jobrs; i++) { -+ err = caam_jr_register(ctrldev, &jrdev[i]); -+ if (err < 0) -+ break; -+ } -+ if (err < 0 && i == 0) { -+ dev_err(ctrldev, "algapi error in job ring registration: %d\n", -+ err); -+ of_node_put(dev_node); -+ kfree(jrdev); -+ return err; -+ } - -- INIT_LIST_HEAD(&alg_list); -+ priv->num_jrs_for_algapi = i; -+ priv->algapi_jr = jrdev; -+ atomic_set(&priv->tfm_count, -1); -+ -+ /* -+ * register crypto algorithms the device supports -+ * first, detect presence of DES, AES, and MD blocks. If MD present, -+ * determine limit of supported digest size -+ */ -+ cha_inst = rd_reg64(&priv->ctrl->perfmon.cha_num); -+ des_inst = (cha_inst & CHA_ID_DES_MASK) >> CHA_ID_DES_SHIFT; -+ aes_inst = (cha_inst & CHA_ID_AES_MASK) >> CHA_ID_AES_SHIFT; -+ md_inst = (cha_inst & CHA_ID_MD_MASK) >> CHA_ID_MD_SHIFT; -+ if (md_inst) { -+ md_limit = SHA512_DIGEST_SIZE; -+ if ((rd_reg64(&priv->ctrl->perfmon.cha_id) & CHA_ID_MD_MASK) -+ == CHA_ID_MD_LP256) /* LP256 limits digest size */ -+ md_limit = SHA256_DIGEST_SIZE; -+ } - -- /* register crypto algorithms the device supports */ - for (i = 0; i < ARRAY_SIZE(driver_algs); i++) { -- /* TODO: check if h/w supports alg */ - struct caam_crypto_alg *t_alg; -+ bool done = false; -+ -+authencesn: -+ /* -+ * All registrable algs in this module require a blockcipher -+ * All aead algs require message digests, so check them for -+ * instantiation and size. -+ */ -+ if (driver_algs[i].type == CRYPTO_ALG_TYPE_AEAD) { -+ /* If no MD instantiated, or MD too small, skip */ -+ if ((!md_inst) || -+ (driver_algs[i].template_aead.maxauthsize > -+ md_limit)) -+ continue; -+ } -+ /* If DES alg, and CHA not instantiated, skip */ -+ if ((driver_algs[i].class1_alg_type & OP_ALG_ALGSEL_3DES) || -+ (driver_algs[i].class1_alg_type & OP_ALG_ALGSEL_DES)) -+ if (!des_inst) -+ continue; -+ /* If AES alg, and CHA not instantiated, skip */ -+ if (driver_algs[i].class1_alg_type & OP_ALG_ALGSEL_AES) -+ if (!aes_inst) -+ continue; - -- t_alg = caam_alg_alloc(&driver_algs[i]); -+ t_alg = caam_alg_alloc(ctrldev, &driver_algs[i]); - if (IS_ERR(t_alg)) { - err = PTR_ERR(t_alg); -- pr_warn("%s alg allocation failed\n", -- driver_algs[i].driver_name); -+ dev_warn(ctrldev, "%s alg allocation failed\n", -+ driver_algs[i].driver_name); - continue; - } - - err = crypto_register_alg(&t_alg->crypto_alg); - if (err) { -- pr_warn("%s alg registration failed\n", -+ dev_warn(ctrldev, "%s alg registration failed\n", - t_alg->crypto_alg.cra_driver_name); - kfree(t_alg); -- } else -- list_add_tail(&t_alg->entry, &alg_list); -+ } else { -+ list_add_tail(&t_alg->entry, &priv->alg_list); -+ dev_info(ctrldev, "%s\n", -+ t_alg->crypto_alg.cra_driver_name); -+ -+ if (driver_algs[i].type == CRYPTO_ALG_TYPE_AEAD && -+ !memcmp(driver_algs[i].name, "authenc", 7) && -+ !done) { -+ char *name; -+ -+ name = driver_algs[i].name; -+ memmove(name + 10, name + 7, strlen(name) - 7); -+ memcpy(name + 7, "esn", 3); -+ -+ name = driver_algs[i].driver_name; -+ memmove(name + 10, name + 7, strlen(name) - 7); -+ memcpy(name + 7, "esn", 3); -+ -+ done = true; -+ goto authencesn; -+ } -+ } - } -- if (!list_empty(&alg_list)) -- pr_info("caam algorithms registered in /proc/crypto\n"); - -+ if (!list_empty(&priv->alg_list)) -+ dev_info(ctrldev, "%s algorithms registered in /proc/crypto\n", -+ (char *)of_get_property(dev_node, "compatible", NULL)); -+ -+ of_node_put(dev_node); - return err; - } - -diff -Nur linux-4.1.3/drivers/crypto/caam/caamhash.c linux-xbian-imx6/drivers/crypto/caam/caamhash.c ---- linux-4.1.3/drivers/crypto/caam/caamhash.c 2015-07-21 19:10:33.000000000 +0200 -+++ linux-xbian-imx6/drivers/crypto/caam/caamhash.c 2015-07-27 23:13:04.209961631 +0200 -@@ -1,7 +1,7 @@ - /* - * caam - Freescale FSL CAAM support for ahash functions of crypto API - * -- * Copyright 2011 Freescale Semiconductor, Inc. -+ * Copyright (C) 2011-2013 Freescale Semiconductor, Inc. - * - * Based on caamalg.c crypto API driver. - * -@@ -62,6 +62,7 @@ - #include "error.h" - #include "sg_sw_sec4.h" - #include "key_gen.h" -+#include - - #define CAAM_CRA_PRIORITY 3000 - -@@ -72,6 +73,8 @@ - #define CAAM_MAX_HASH_DIGEST_SIZE SHA512_DIGEST_SIZE - - /* length of descriptors text */ -+#define DESC_JOB_IO_LEN (CAAM_CMD_SZ * 5 + CAAM_PTR_SZ * 3) -+ - #define DESC_AHASH_BASE (4 * CAAM_CMD_SZ) - #define DESC_AHASH_UPDATE_LEN (6 * CAAM_CMD_SZ) - #define DESC_AHASH_UPDATE_FIRST_LEN (DESC_AHASH_BASE + 4 * CAAM_CMD_SZ) -@@ -89,14 +92,13 @@ - - #ifdef DEBUG - /* for print_hex_dumps with line references */ -+#define xstr(s) str(s) -+#define str(s) #s - #define debug(format, arg...) printk(format, arg) - #else - #define debug(format, arg...) - #endif - -- --static struct list_head hash_list; -- - /* ahash per-session context */ - struct caam_hash_ctx { - struct device *jrdev; -@@ -115,6 +117,7 @@ - u8 key[CAAM_MAX_HASH_KEY_SIZE]; - dma_addr_t key_dma; - int ctx_len; -+ unsigned int key_len; - unsigned int split_key_len; - unsigned int split_key_pad_len; - }; -@@ -137,20 +140,13 @@ - /* Common job descriptor seq in/out ptr routines */ - - /* Map state->caam_ctx, and append seq_out_ptr command that points to it */ --static inline int map_seq_out_ptr_ctx(u32 *desc, struct device *jrdev, -- struct caam_hash_state *state, -- int ctx_len) -+static inline void map_seq_out_ptr_ctx(u32 *desc, struct device *jrdev, -+ struct caam_hash_state *state, -+ int ctx_len) - { - state->ctx_dma = dma_map_single(jrdev, state->caam_ctx, - ctx_len, DMA_FROM_DEVICE); -- if (dma_mapping_error(jrdev, state->ctx_dma)) { -- dev_err(jrdev, "unable to map ctx\n"); -- return -ENOMEM; -- } -- - append_seq_out_ptr(desc, state->ctx_dma, ctx_len, 0); -- -- return 0; - } - - /* Map req->result, and append seq_out_ptr command that points to it */ -@@ -173,6 +169,7 @@ - dma_addr_t buf_dma; - - buf_dma = dma_map_single(jrdev, buf, buflen, DMA_TO_DEVICE); -+ dma_sync_single_for_device(jrdev, buf_dma, buflen, DMA_TO_DEVICE); - dma_to_sec4_sg_one(sec4_sg, buf_dma, buflen, 0); - - return buf_dma; -@@ -208,19 +205,17 @@ - } - - /* Map state->caam_ctx, and add it to link table */ --static inline int ctx_map_to_sec4_sg(u32 *desc, struct device *jrdev, -- struct caam_hash_state *state, int ctx_len, -- struct sec4_sg_entry *sec4_sg, u32 flag) -+static inline void ctx_map_to_sec4_sg(u32 *desc, struct device *jrdev, -+ struct caam_hash_state *state, -+ int ctx_len, -+ struct sec4_sg_entry *sec4_sg, -+ u32 flag) - { - state->ctx_dma = dma_map_single(jrdev, state->caam_ctx, ctx_len, flag); -- if (dma_mapping_error(jrdev, state->ctx_dma)) { -- dev_err(jrdev, "unable to map ctx\n"); -- return -ENOMEM; -- } -- -+ if ((flag == DMA_TO_DEVICE) || (flag == DMA_BIDIRECTIONAL)) -+ dma_sync_single_for_device(jrdev, state->ctx_dma, ctx_len, -+ flag); - dma_to_sec4_sg_one(sec4_sg, state->ctx_dma, ctx_len, 0); -- -- return 0; - } - - /* Common shared descriptor commands */ -@@ -231,6 +226,13 @@ - KEY_DEST_MDHA_SPLIT | KEY_ENC); - } - -+static inline void append_key_axcbc(u32 *desc, struct caam_hash_ctx *ctx) -+{ -+ append_key_as_imm(desc, ctx->key, ctx->key_len, -+ ctx->key_len, CLASS_1 | -+ KEY_DEST_CLASS_REG); -+} -+ - /* Append key if it has been set */ - static inline void init_sh_desc_key_ahash(u32 *desc, struct caam_hash_ctx *ctx) - { -@@ -252,6 +254,25 @@ - append_cmd(desc, SET_OK_NO_PROP_ERRORS | CMD_LOAD); - } - -+static inline void init_sh_desc_key_axcbc(u32 *desc, struct caam_hash_ctx *ctx) -+{ -+ u32 *key_jump_cmd; -+ -+ init_sh_desc(desc, HDR_SHARE_SERIAL); -+ -+ if (ctx->key_len) { -+ key_jump_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL | -+ JUMP_COND_SHRD); -+ -+ append_key_axcbc(desc, ctx); -+ -+ set_jump_tgt_here(desc, key_jump_cmd); -+ } -+ -+ /* Propagate errors from shared to job descriptor */ -+ append_cmd(desc, SET_OK_NO_PROP_ERRORS | CMD_LOAD); -+ -+} - /* - * For ahash read data from seqin following state->caam_ctx, - * and write resulting class2 context to seqout, which may be state->caam_ctx -@@ -271,6 +292,20 @@ - LDST_SRCDST_BYTE_CONTEXT); - } - -+static inline void axcbc_append_load_str(u32 *desc, int digestsize) -+{ -+ /* Calculate remaining bytes to read */ -+ append_math_add(desc, VARSEQINLEN, SEQINLEN, REG0, CAAM_CMD_SZ); -+ -+ /* Read remaining bytes */ -+ append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS1 | FIFOLD_TYPE_LAST1 | -+ FIFOLD_TYPE_MSG | KEY_VLF); -+ -+ /* Store class1 context bytes */ -+ append_seq_store(desc, digestsize, LDST_CLASS_1_CCB | -+ LDST_SRCDST_BYTE_CONTEXT); -+} -+ - /* - * For ahash update, final and finup, import context, read and write to seqout - */ -@@ -293,6 +328,27 @@ - ahash_append_load_str(desc, digestsize); - } - -+/* -+ * For ahash update, final and finup, import context, read and write to seqout -+ */ -+static inline void axcbc_ctx_data_to_out(u32 *desc, u32 op, u32 state, -+ int digestsize, -+ struct caam_hash_ctx *ctx) -+{ -+ init_sh_desc_key_axcbc(desc, ctx); -+ -+ /* Import context from software */ -+ append_cmd(desc, CMD_SEQ_LOAD | LDST_SRCDST_BYTE_CONTEXT | -+ LDST_CLASS_1_CCB | ctx->ctx_len); -+ -+ /* Class 1 operation */ -+ append_operation(desc, op | state | OP_ALG_ENCRYPT); -+ -+ /* -+ * Load from buf and/or src and write to req->result or state->context -+ */ -+ axcbc_append_load_str(desc, digestsize); -+} - /* For ahash firsts and digest, read and write to seqout */ - static inline void ahash_data_to_out(u32 *desc, u32 op, u32 state, - int digestsize, struct caam_hash_ctx *ctx) -@@ -308,6 +364,21 @@ - ahash_append_load_str(desc, digestsize); - } - -+/* For ahash firsts and digest, read and write to seqout */ -+static inline void axcbc_data_to_out(u32 *desc, u32 op, u32 state, -+ int digestsize, struct caam_hash_ctx *ctx) -+{ -+ init_sh_desc_key_axcbc(desc, ctx); -+ -+ /* Class 1 operation */ -+ append_operation(desc, op | state | OP_ALG_ENCRYPT); -+ -+ /* -+ * Load from buf and/or src and write to req->result or state->context -+ */ -+ axcbc_append_load_str(desc, digestsize); -+} -+ - static int ahash_set_sh_desc(struct crypto_ahash *ahash) - { - struct caam_hash_ctx *ctx = crypto_ahash_ctx(ahash); -@@ -342,8 +413,7 @@ - return -ENOMEM; - } - #ifdef DEBUG -- print_hex_dump(KERN_ERR, -- "ahash update shdesc@"__stringify(__LINE__)": ", -+ print_hex_dump(KERN_ERR, "ahash update shdesc@"xstr(__LINE__)": ", - DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 1); - #endif - -@@ -361,10 +431,11 @@ - return -ENOMEM; - } - #ifdef DEBUG -- print_hex_dump(KERN_ERR, -- "ahash update first shdesc@"__stringify(__LINE__)": ", -+ print_hex_dump(KERN_ERR, "ahash update first shdesc@"xstr(__LINE__)": ", - DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 1); - #endif -+ dma_sync_single_for_device(jrdev, ctx->sh_desc_update_first_dma, -+ desc_bytes(desc), DMA_TO_DEVICE); - - /* ahash_final shared descriptor */ - desc = ctx->sh_desc_fin; -@@ -379,10 +450,12 @@ - return -ENOMEM; - } - #ifdef DEBUG -- print_hex_dump(KERN_ERR, "ahash final shdesc@"__stringify(__LINE__)": ", -+ print_hex_dump(KERN_ERR, "ahash final shdesc@"xstr(__LINE__)": ", - DUMP_PREFIX_ADDRESS, 16, 4, desc, - desc_bytes(desc), 1); - #endif -+ dma_sync_single_for_device(jrdev, ctx->sh_desc_fin_dma, -+ desc_bytes(desc), DMA_TO_DEVICE); - - /* ahash_finup shared descriptor */ - desc = ctx->sh_desc_finup; -@@ -397,10 +470,12 @@ - return -ENOMEM; - } - #ifdef DEBUG -- print_hex_dump(KERN_ERR, "ahash finup shdesc@"__stringify(__LINE__)": ", -+ print_hex_dump(KERN_ERR, "ahash finup shdesc@"xstr(__LINE__)": ", - DUMP_PREFIX_ADDRESS, 16, 4, desc, - desc_bytes(desc), 1); - #endif -+ dma_sync_single_for_device(jrdev, ctx->sh_desc_finup_dma, -+ desc_bytes(desc), DMA_TO_DEVICE); - - /* ahash_digest shared descriptor */ - desc = ctx->sh_desc_digest; -@@ -416,15 +491,134 @@ - return -ENOMEM; - } - #ifdef DEBUG -- print_hex_dump(KERN_ERR, -- "ahash digest shdesc@"__stringify(__LINE__)": ", -+ print_hex_dump(KERN_ERR, "ahash digest shdesc@"xstr(__LINE__)": ", - DUMP_PREFIX_ADDRESS, 16, 4, desc, - desc_bytes(desc), 1); - #endif -+ dma_sync_single_for_device(jrdev, ctx->sh_desc_digest_dma, -+ desc_bytes(desc), DMA_TO_DEVICE); - - return 0; - } - -+static int axcbc_set_sh_desc(struct crypto_ahash *ahash) -+{ -+ struct caam_hash_ctx *ctx = crypto_ahash_ctx(ahash); -+ int digestsize = crypto_ahash_digestsize(ahash); -+ struct device *jrdev = ctx->jrdev; -+ u32 have_key = 0; -+ u32 *desc; -+ -+ /* ahash_update shared descriptor */ -+ desc = ctx->sh_desc_update; -+ -+ init_sh_desc(desc, HDR_SHARE_SERIAL); -+ -+ /* Import context from software */ -+ append_cmd(desc, CMD_SEQ_LOAD | LDST_SRCDST_BYTE_CONTEXT | -+ LDST_CLASS_1_CCB | ctx->ctx_len); -+ -+ /* Class 1 operation */ -+ append_operation(desc, ctx->alg_type | OP_ALG_AS_UPDATE | -+ OP_ALG_ENCRYPT); -+ -+ /* Load data and write to result or context */ -+ axcbc_append_load_str(desc, ctx->ctx_len); -+ -+ ctx->sh_desc_update_dma = dma_map_single(jrdev, desc, desc_bytes(desc), -+ DMA_TO_DEVICE); -+ if (dma_mapping_error(jrdev, ctx->sh_desc_update_dma)) { -+ dev_err(jrdev, "unable to map shared descriptor\n"); -+ return -ENOMEM; -+ } -+#ifdef DEBUG -+ print_hex_dump(KERN_ERR, "ahash update shdesc@"xstr(__LINE__)": ", -+ DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 1); -+#endif -+ -+ /* ahash_update_first shared descriptor */ -+ desc = ctx->sh_desc_update_first; -+ -+ axcbc_data_to_out(desc, have_key | ctx->alg_type, OP_ALG_AS_INIT, -+ ctx->ctx_len, ctx); -+ -+ ctx->sh_desc_update_first_dma = dma_map_single(jrdev, desc, -+ desc_bytes(desc), -+ DMA_TO_DEVICE); -+ if (dma_mapping_error(jrdev, ctx->sh_desc_update_first_dma)) { -+ dev_err(jrdev, "unable to map shared descriptor\n"); -+ return -ENOMEM; -+ } -+#ifdef DEBUG -+ print_hex_dump(KERN_ERR, "ahash update first shdesc@"xstr(__LINE__)": ", -+ DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 1); -+#endif -+ dma_sync_single_for_device(jrdev, ctx->sh_desc_update_first_dma, -+ desc_bytes(desc), DMA_TO_DEVICE); -+ -+ /* ahash_final shared descriptor */ -+ desc = ctx->sh_desc_fin; -+ -+ axcbc_ctx_data_to_out(desc, have_key | ctx->alg_type, -+ OP_ALG_AS_FINALIZE, digestsize, ctx); -+ -+ ctx->sh_desc_fin_dma = dma_map_single(jrdev, desc, desc_bytes(desc), -+ DMA_TO_DEVICE); -+ if (dma_mapping_error(jrdev, ctx->sh_desc_fin_dma)) { -+ dev_err(jrdev, "unable to map shared descriptor\n"); -+ return -ENOMEM; -+ } -+#ifdef DEBUG -+ print_hex_dump(KERN_ERR, "ahash final shdesc@"xstr(__LINE__)": ", -+ DUMP_PREFIX_ADDRESS, 16, 4, desc, -+ desc_bytes(desc), 1); -+#endif -+ dma_sync_single_for_device(jrdev, ctx->sh_desc_fin_dma, -+ desc_bytes(desc), DMA_TO_DEVICE); -+ -+ /* ahash_finup shared descriptor */ -+ desc = ctx->sh_desc_finup; -+ -+ axcbc_ctx_data_to_out(desc, have_key | ctx->alg_type, -+ OP_ALG_AS_FINALIZE, digestsize, ctx); -+ -+ ctx->sh_desc_finup_dma = dma_map_single(jrdev, desc, desc_bytes(desc), -+ DMA_TO_DEVICE); -+ if (dma_mapping_error(jrdev, ctx->sh_desc_finup_dma)) { -+ dev_err(jrdev, "unable to map shared descriptor\n"); -+ return -ENOMEM; -+ } -+#ifdef DEBUG -+ print_hex_dump(KERN_ERR, "ahash finup shdesc@"xstr(__LINE__)": ", -+ DUMP_PREFIX_ADDRESS, 16, 4, desc, -+ desc_bytes(desc), 1); -+#endif -+ dma_sync_single_for_device(jrdev, ctx->sh_desc_finup_dma, -+ desc_bytes(desc), DMA_TO_DEVICE); -+ -+ /* ahash_digest shared descriptor */ -+ desc = ctx->sh_desc_digest; -+ -+ axcbc_data_to_out(desc, have_key | ctx->alg_type, OP_ALG_AS_INITFINAL, -+ digestsize, ctx); -+ -+ ctx->sh_desc_digest_dma = dma_map_single(jrdev, desc, -+ desc_bytes(desc), -+ DMA_TO_DEVICE); -+ if (dma_mapping_error(jrdev, ctx->sh_desc_digest_dma)) { -+ dev_err(jrdev, "unable to map shared descriptor\n"); -+ return -ENOMEM; -+ } -+#ifdef DEBUG -+ print_hex_dump(KERN_ERR, "ahash digest shdesc@"xstr(__LINE__)": ", -+ DUMP_PREFIX_ADDRESS, 16, 4, desc, -+ desc_bytes(desc), 1); -+#endif -+ dma_sync_single_for_device(jrdev, ctx->sh_desc_digest_dma, -+ desc_bytes(desc), DMA_TO_DEVICE); -+ -+ return 0; -+} - static int gen_split_hash_key(struct caam_hash_ctx *ctx, const u8 *key_in, - u32 keylen) - { -@@ -458,6 +652,8 @@ - kfree(desc); - return -ENOMEM; - } -+ dma_sync_single_for_device(jrdev, src_dma, *keylen, DMA_TO_DEVICE); -+ - dst_dma = dma_map_single(jrdev, (void *)key_out, digestsize, - DMA_FROM_DEVICE); - if (dma_mapping_error(jrdev, dst_dma)) { -@@ -478,9 +674,9 @@ - LDST_SRCDST_BYTE_CONTEXT); - - #ifdef DEBUG -- print_hex_dump(KERN_ERR, "key_in@"__stringify(__LINE__)": ", -+ print_hex_dump(KERN_ERR, "key_in@"xstr(__LINE__)": ", - DUMP_PREFIX_ADDRESS, 16, 4, key_in, *keylen, 1); -- print_hex_dump(KERN_ERR, "jobdesc@"__stringify(__LINE__)": ", -+ print_hex_dump(KERN_ERR, "jobdesc@"xstr(__LINE__)": ", - DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 1); - #endif - -@@ -493,17 +689,17 @@ - wait_for_completion_interruptible(&result.completion); - ret = result.err; - #ifdef DEBUG -- print_hex_dump(KERN_ERR, -- "digested key@"__stringify(__LINE__)": ", -+ print_hex_dump(KERN_ERR, "digested key@"xstr(__LINE__)": ", - DUMP_PREFIX_ADDRESS, 16, 4, key_in, - digestsize, 1); - #endif - } -+ *keylen = digestsize; -+ - dma_unmap_single(jrdev, src_dma, *keylen, DMA_TO_DEVICE); -+ dma_sync_single_for_cpu(jrdev, dst_dma, digestsize, DMA_FROM_DEVICE); - dma_unmap_single(jrdev, dst_dma, digestsize, DMA_FROM_DEVICE); - -- *keylen = digestsize; -- - kfree(desc); - - return ret; -@@ -545,7 +741,7 @@ - #ifdef DEBUG - printk(KERN_ERR "split_key_len %d split_key_pad_len %d\n", - ctx->split_key_len, ctx->split_key_pad_len); -- print_hex_dump(KERN_ERR, "key in @"__stringify(__LINE__)": ", -+ print_hex_dump(KERN_ERR, "key in @"xstr(__LINE__)": ", - DUMP_PREFIX_ADDRESS, 16, 4, key, keylen, 1); - #endif - -@@ -557,11 +753,14 @@ - DMA_TO_DEVICE); - if (dma_mapping_error(jrdev, ctx->key_dma)) { - dev_err(jrdev, "unable to map key i/o memory\n"); -- ret = -ENOMEM; -- goto map_err; -+ return -ENOMEM; - } -+ -+ dma_sync_single_for_device(jrdev, ctx->key_dma, ctx->split_key_pad_len, -+ DMA_TO_DEVICE); -+ - #ifdef DEBUG -- print_hex_dump(KERN_ERR, "ctx.key@"__stringify(__LINE__)": ", -+ print_hex_dump(KERN_ERR, "ctx.key@"xstr(__LINE__)": ", - DUMP_PREFIX_ADDRESS, 16, 4, ctx->key, - ctx->split_key_pad_len, 1); - #endif -@@ -572,7 +771,6 @@ - DMA_TO_DEVICE); - } - --map_err: - kfree(hashed_key); - return ret; - badkey: -@@ -581,6 +779,25 @@ - return -EINVAL; - } - -+static int axcbc_setkey(struct crypto_ahash *ahash, -+ const u8 *key, unsigned int keylen) -+{ -+ struct caam_hash_ctx *ctx = crypto_ahash_ctx(ahash); -+ int ret = 0; -+ -+ ctx->key_len = keylen; -+ memcpy(ctx->key, key, keylen); -+ -+#ifdef DEBUG -+ print_hex_dump(KERN_ERR, "ctx.key@"xstr(__LINE__)": ", -+ DUMP_PREFIX_ADDRESS, 16, 4, ctx->key, -+ ctx->key_len, 1); -+#endif -+ -+ ret = axcbc_set_sh_desc(ahash); -+ -+ return ret; -+} - /* - * ahash_edesc - s/w-extended ahash descriptor - * @dst_dma: physical mapped address of req->result -@@ -608,8 +825,11 @@ - if (edesc->src_nents) - dma_unmap_sg_chained(dev, req->src, edesc->src_nents, - DMA_TO_DEVICE, edesc->chained); -- if (edesc->dst_dma) -+ if (edesc->dst_dma) { -+ dma_sync_single_for_cpu(dev, edesc->dst_dma, dst_len, -+ DMA_FROM_DEVICE); - dma_unmap_single(dev, edesc->dst_dma, dst_len, DMA_FROM_DEVICE); -+ } - - if (edesc->sec4_sg_bytes) - dma_unmap_single(dev, edesc->sec4_sg_dma, -@@ -624,8 +844,12 @@ - struct caam_hash_ctx *ctx = crypto_ahash_ctx(ahash); - struct caam_hash_state *state = ahash_request_ctx(req); - -- if (state->ctx_dma) -+ if (state->ctx_dma) { -+ if ((flag == DMA_FROM_DEVICE) || (flag == DMA_BIDIRECTIONAL)) -+ dma_sync_single_for_cpu(dev, state->ctx_dma, -+ ctx->ctx_len, flag); - dma_unmap_single(dev, state->ctx_dma, ctx->ctx_len, flag); -+ } - ahash_unmap(dev, edesc, req, dst_len); - } - -@@ -645,18 +869,21 @@ - - edesc = (struct ahash_edesc *)((char *)desc - - offsetof(struct ahash_edesc, hw_desc)); -- if (err) -- caam_jr_strstatus(jrdev, err); -+ if (err) { -+ char tmp[CAAM_ERROR_STR_MAX]; -+ -+ dev_err(jrdev, "%08x: %s\n", err, caam_jr_strstatus(tmp, err)); -+ } - - ahash_unmap(jrdev, edesc, req, digestsize); - kfree(edesc); - - #ifdef DEBUG -- print_hex_dump(KERN_ERR, "ctx@"__stringify(__LINE__)": ", -+ print_hex_dump(KERN_ERR, "ctx@"xstr(__LINE__)": ", - DUMP_PREFIX_ADDRESS, 16, 4, state->caam_ctx, - ctx->ctx_len, 1); - if (req->result) -- print_hex_dump(KERN_ERR, "result@"__stringify(__LINE__)": ", -+ print_hex_dump(KERN_ERR, "result@"xstr(__LINE__)": ", - DUMP_PREFIX_ADDRESS, 16, 4, req->result, - digestsize, 1); - #endif -@@ -680,18 +907,21 @@ - - edesc = (struct ahash_edesc *)((char *)desc - - offsetof(struct ahash_edesc, hw_desc)); -- if (err) -- caam_jr_strstatus(jrdev, err); -+ if (err) { -+ char tmp[CAAM_ERROR_STR_MAX]; -+ -+ dev_err(jrdev, "%08x: %s\n", err, caam_jr_strstatus(tmp, err)); -+ } - - ahash_unmap_ctx(jrdev, edesc, req, ctx->ctx_len, DMA_BIDIRECTIONAL); - kfree(edesc); - - #ifdef DEBUG -- print_hex_dump(KERN_ERR, "ctx@"__stringify(__LINE__)": ", -+ print_hex_dump(KERN_ERR, "ctx@"xstr(__LINE__)": ", - DUMP_PREFIX_ADDRESS, 16, 4, state->caam_ctx, - ctx->ctx_len, 1); - if (req->result) -- print_hex_dump(KERN_ERR, "result@"__stringify(__LINE__)": ", -+ print_hex_dump(KERN_ERR, "result@"xstr(__LINE__)": ", - DUMP_PREFIX_ADDRESS, 16, 4, req->result, - digestsize, 1); - #endif -@@ -715,18 +945,21 @@ - - edesc = (struct ahash_edesc *)((char *)desc - - offsetof(struct ahash_edesc, hw_desc)); -- if (err) -- caam_jr_strstatus(jrdev, err); -+ if (err) { -+ char tmp[CAAM_ERROR_STR_MAX]; - -- ahash_unmap_ctx(jrdev, edesc, req, digestsize, DMA_TO_DEVICE); -+ dev_err(jrdev, "%08x: %s\n", err, caam_jr_strstatus(tmp, err)); -+ } -+ -+ ahash_unmap_ctx(jrdev, edesc, req, digestsize, DMA_FROM_DEVICE); - kfree(edesc); - - #ifdef DEBUG -- print_hex_dump(KERN_ERR, "ctx@"__stringify(__LINE__)": ", -+ print_hex_dump(KERN_ERR, "ctx@"xstr(__LINE__)": ", - DUMP_PREFIX_ADDRESS, 16, 4, state->caam_ctx, - ctx->ctx_len, 1); - if (req->result) -- print_hex_dump(KERN_ERR, "result@"__stringify(__LINE__)": ", -+ print_hex_dump(KERN_ERR, "result@"xstr(__LINE__)": ", - DUMP_PREFIX_ADDRESS, 16, 4, req->result, - digestsize, 1); - #endif -@@ -750,18 +983,21 @@ - - edesc = (struct ahash_edesc *)((char *)desc - - offsetof(struct ahash_edesc, hw_desc)); -- if (err) -- caam_jr_strstatus(jrdev, err); -+ if (err) { -+ char tmp[CAAM_ERROR_STR_MAX]; -+ -+ dev_err(jrdev, "%08x: %s\n", err, caam_jr_strstatus(tmp, err)); -+ } - -- ahash_unmap_ctx(jrdev, edesc, req, ctx->ctx_len, DMA_FROM_DEVICE); -+ ahash_unmap_ctx(jrdev, edesc, req, ctx->ctx_len, DMA_TO_DEVICE); - kfree(edesc); - - #ifdef DEBUG -- print_hex_dump(KERN_ERR, "ctx@"__stringify(__LINE__)": ", -+ print_hex_dump(KERN_ERR, "ctx@"xstr(__LINE__)": ", - DUMP_PREFIX_ADDRESS, 16, 4, state->caam_ctx, - ctx->ctx_len, 1); - if (req->result) -- print_hex_dump(KERN_ERR, "result@"__stringify(__LINE__)": ", -+ print_hex_dump(KERN_ERR, "result@"xstr(__LINE__)": ", - DUMP_PREFIX_ADDRESS, 16, 4, req->result, - digestsize, 1); - #endif -@@ -807,7 +1043,7 @@ - * allocate space for base edesc and hw desc commands, - * link tables - */ -- edesc = kmalloc(sizeof(struct ahash_edesc) + DESC_JOB_IO_LEN + -+ edesc = kzalloc(sizeof(struct ahash_edesc) + DESC_JOB_IO_LEN + - sec4_sg_bytes, GFP_DMA | flags); - if (!edesc) { - dev_err(jrdev, -@@ -820,11 +1056,12 @@ - edesc->sec4_sg_bytes = sec4_sg_bytes; - edesc->sec4_sg = (void *)edesc + sizeof(struct ahash_edesc) + - DESC_JOB_IO_LEN; -+ edesc->sec4_sg_dma = dma_map_single(jrdev, edesc->sec4_sg, -+ sec4_sg_bytes, -+ DMA_TO_DEVICE); - -- ret = ctx_map_to_sec4_sg(desc, jrdev, state, ctx->ctx_len, -- edesc->sec4_sg, DMA_BIDIRECTIONAL); -- if (ret) -- return ret; -+ ctx_map_to_sec4_sg(desc, jrdev, state, ctx->ctx_len, -+ edesc->sec4_sg, DMA_BIDIRECTIONAL); - - state->buf_dma = try_buf_map_to_sec4_sg(jrdev, - edesc->sec4_sg + 1, -@@ -851,21 +1088,16 @@ - init_job_desc_shared(desc, ptr, sh_len, HDR_SHARE_DEFER | - HDR_REVERSE); - -- edesc->sec4_sg_dma = dma_map_single(jrdev, edesc->sec4_sg, -- sec4_sg_bytes, -- DMA_TO_DEVICE); -- if (dma_mapping_error(jrdev, edesc->sec4_sg_dma)) { -- dev_err(jrdev, "unable to map S/G table\n"); -- return -ENOMEM; -- } -- - append_seq_in_ptr(desc, edesc->sec4_sg_dma, ctx->ctx_len + - to_hash, LDST_SGF); - - append_seq_out_ptr(desc, state->ctx_dma, ctx->ctx_len, 0); - -+ dma_sync_single_for_device(jrdev, edesc->sec4_sg_dma, -+ sec4_sg_bytes, DMA_TO_DEVICE); -+ - #ifdef DEBUG -- print_hex_dump(KERN_ERR, "jobdesc@"__stringify(__LINE__)": ", -+ print_hex_dump(KERN_ERR, "jobdesc@"xstr(__LINE__)": ", - DUMP_PREFIX_ADDRESS, 16, 4, desc, - desc_bytes(desc), 1); - #endif -@@ -885,9 +1117,9 @@ - *next_buflen = last_buflen; - } - #ifdef DEBUG -- print_hex_dump(KERN_ERR, "buf@"__stringify(__LINE__)": ", -+ print_hex_dump(KERN_ERR, "buf@"xstr(__LINE__)": ", - DUMP_PREFIX_ADDRESS, 16, 4, buf, *buflen, 1); -- print_hex_dump(KERN_ERR, "next buf@"__stringify(__LINE__)": ", -+ print_hex_dump(KERN_ERR, "next buf@"xstr(__LINE__)": ", - DUMP_PREFIX_ADDRESS, 16, 4, next_buf, - *next_buflen, 1); - #endif -@@ -918,7 +1150,7 @@ - sec4_sg_bytes = (1 + (buflen ? 1 : 0)) * sizeof(struct sec4_sg_entry); - - /* allocate space for base edesc and hw desc commands, link tables */ -- edesc = kmalloc(sizeof(struct ahash_edesc) + DESC_JOB_IO_LEN + -+ edesc = kzalloc(sizeof(struct ahash_edesc) + DESC_JOB_IO_LEN + - sec4_sg_bytes, GFP_DMA | flags); - if (!edesc) { - dev_err(jrdev, "could not allocate extended descriptor\n"); -@@ -932,37 +1164,29 @@ - edesc->sec4_sg_bytes = sec4_sg_bytes; - edesc->sec4_sg = (void *)edesc + sizeof(struct ahash_edesc) + - DESC_JOB_IO_LEN; -+ edesc->sec4_sg_dma = dma_map_single(jrdev, edesc->sec4_sg, -+ sec4_sg_bytes, DMA_TO_DEVICE); - edesc->src_nents = 0; - -- ret = ctx_map_to_sec4_sg(desc, jrdev, state, ctx->ctx_len, -- edesc->sec4_sg, DMA_TO_DEVICE); -- if (ret) -- return ret; -+ ctx_map_to_sec4_sg(desc, jrdev, state, ctx->ctx_len, edesc->sec4_sg, -+ DMA_TO_DEVICE); - - state->buf_dma = try_buf_map_to_sec4_sg(jrdev, edesc->sec4_sg + 1, - buf, state->buf_dma, buflen, - last_buflen); - (edesc->sec4_sg + sec4_sg_bytes - 1)->len |= SEC4_SG_LEN_FIN; - -- edesc->sec4_sg_dma = dma_map_single(jrdev, edesc->sec4_sg, -- sec4_sg_bytes, DMA_TO_DEVICE); -- if (dma_mapping_error(jrdev, edesc->sec4_sg_dma)) { -- dev_err(jrdev, "unable to map S/G table\n"); -- return -ENOMEM; -- } -- - append_seq_in_ptr(desc, edesc->sec4_sg_dma, ctx->ctx_len + buflen, - LDST_SGF); - - edesc->dst_dma = map_seq_out_ptr_result(desc, jrdev, req->result, - digestsize); -- if (dma_mapping_error(jrdev, edesc->dst_dma)) { -- dev_err(jrdev, "unable to map dst\n"); -- return -ENOMEM; -- } -+ -+ dma_sync_single_for_device(jrdev, edesc->sec4_sg_dma, sec4_sg_bytes, -+ DMA_TO_DEVICE); - - #ifdef DEBUG -- print_hex_dump(KERN_ERR, "jobdesc@"__stringify(__LINE__)": ", -+ print_hex_dump(KERN_ERR, "jobdesc@"xstr(__LINE__)": ", - DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 1); - #endif - -@@ -1005,7 +1229,7 @@ - sizeof(struct sec4_sg_entry); - - /* allocate space for base edesc and hw desc commands, link tables */ -- edesc = kmalloc(sizeof(struct ahash_edesc) + DESC_JOB_IO_LEN + -+ edesc = kzalloc(sizeof(struct ahash_edesc) + DESC_JOB_IO_LEN + - sec4_sg_bytes, GFP_DMA | flags); - if (!edesc) { - dev_err(jrdev, "could not allocate extended descriptor\n"); -@@ -1021,11 +1245,11 @@ - edesc->sec4_sg_bytes = sec4_sg_bytes; - edesc->sec4_sg = (void *)edesc + sizeof(struct ahash_edesc) + - DESC_JOB_IO_LEN; -+ edesc->sec4_sg_dma = dma_map_single(jrdev, edesc->sec4_sg, -+ sec4_sg_bytes, DMA_TO_DEVICE); - -- ret = ctx_map_to_sec4_sg(desc, jrdev, state, ctx->ctx_len, -- edesc->sec4_sg, DMA_TO_DEVICE); -- if (ret) -- return ret; -+ ctx_map_to_sec4_sg(desc, jrdev, state, ctx->ctx_len, edesc->sec4_sg, -+ DMA_TO_DEVICE); - - state->buf_dma = try_buf_map_to_sec4_sg(jrdev, edesc->sec4_sg + 1, - buf, state->buf_dma, buflen, -@@ -1034,25 +1258,17 @@ - src_map_to_sec4_sg(jrdev, req->src, src_nents, edesc->sec4_sg + - sec4_sg_src_index, chained); - -- edesc->sec4_sg_dma = dma_map_single(jrdev, edesc->sec4_sg, -- sec4_sg_bytes, DMA_TO_DEVICE); -- if (dma_mapping_error(jrdev, edesc->sec4_sg_dma)) { -- dev_err(jrdev, "unable to map S/G table\n"); -- return -ENOMEM; -- } -- - append_seq_in_ptr(desc, edesc->sec4_sg_dma, ctx->ctx_len + - buflen + req->nbytes, LDST_SGF); - - edesc->dst_dma = map_seq_out_ptr_result(desc, jrdev, req->result, - digestsize); -- if (dma_mapping_error(jrdev, edesc->dst_dma)) { -- dev_err(jrdev, "unable to map dst\n"); -- return -ENOMEM; -- } -+ -+ dma_sync_single_for_device(jrdev, edesc->sec4_sg_dma, sec4_sg_bytes, -+ DMA_TO_DEVICE); - - #ifdef DEBUG -- print_hex_dump(KERN_ERR, "jobdesc@"__stringify(__LINE__)": ", -+ print_hex_dump(KERN_ERR, "jobdesc@"xstr(__LINE__)": ", - DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 1); - #endif - -@@ -1091,7 +1307,7 @@ - sec4_sg_bytes = src_nents * sizeof(struct sec4_sg_entry); - - /* allocate space for base edesc and hw desc commands, link tables */ -- edesc = kmalloc(sizeof(struct ahash_edesc) + sec4_sg_bytes + -+ edesc = kzalloc(sizeof(struct ahash_edesc) + sec4_sg_bytes + - DESC_JOB_IO_LEN, GFP_DMA | flags); - if (!edesc) { - dev_err(jrdev, "could not allocate extended descriptor\n"); -@@ -1099,6 +1315,8 @@ - } - edesc->sec4_sg = (void *)edesc + sizeof(struct ahash_edesc) + - DESC_JOB_IO_LEN; -+ edesc->sec4_sg_dma = dma_map_single(jrdev, edesc->sec4_sg, -+ sec4_sg_bytes, DMA_TO_DEVICE); - edesc->sec4_sg_bytes = sec4_sg_bytes; - edesc->src_nents = src_nents; - edesc->chained = chained; -@@ -1109,12 +1327,6 @@ - - if (src_nents) { - sg_to_sec4_sg_last(req->src, src_nents, edesc->sec4_sg, 0); -- edesc->sec4_sg_dma = dma_map_single(jrdev, edesc->sec4_sg, -- sec4_sg_bytes, DMA_TO_DEVICE); -- if (dma_mapping_error(jrdev, edesc->sec4_sg_dma)) { -- dev_err(jrdev, "unable to map S/G table\n"); -- return -ENOMEM; -- } - src_dma = edesc->sec4_sg_dma; - options = LDST_SGF; - } else { -@@ -1123,15 +1335,14 @@ - } - append_seq_in_ptr(desc, src_dma, req->nbytes, options); - -+ dma_sync_single_for_device(jrdev, edesc->sec4_sg_dma, -+ edesc->sec4_sg_bytes, DMA_TO_DEVICE); -+ - edesc->dst_dma = map_seq_out_ptr_result(desc, jrdev, req->result, - digestsize); -- if (dma_mapping_error(jrdev, edesc->dst_dma)) { -- dev_err(jrdev, "unable to map dst\n"); -- return -ENOMEM; -- } - - #ifdef DEBUG -- print_hex_dump(KERN_ERR, "jobdesc@"__stringify(__LINE__)": ", -+ print_hex_dump(KERN_ERR, "jobdesc@"xstr(__LINE__)": ", - DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 1); - #endif - -@@ -1165,7 +1376,7 @@ - int sh_len; - - /* allocate space for base edesc and hw desc commands, link tables */ -- edesc = kmalloc(sizeof(struct ahash_edesc) + DESC_JOB_IO_LEN, -+ edesc = kzalloc(sizeof(struct ahash_edesc) + DESC_JOB_IO_LEN, - GFP_DMA | flags); - if (!edesc) { - dev_err(jrdev, "could not allocate extended descriptor\n"); -@@ -1178,23 +1389,17 @@ - init_job_desc_shared(desc, ptr, sh_len, HDR_SHARE_DEFER | HDR_REVERSE); - - state->buf_dma = dma_map_single(jrdev, buf, buflen, DMA_TO_DEVICE); -- if (dma_mapping_error(jrdev, state->buf_dma)) { -- dev_err(jrdev, "unable to map src\n"); -- return -ENOMEM; -- } - - append_seq_in_ptr(desc, state->buf_dma, buflen, 0); - - edesc->dst_dma = map_seq_out_ptr_result(desc, jrdev, req->result, - digestsize); -- if (dma_mapping_error(jrdev, edesc->dst_dma)) { -- dev_err(jrdev, "unable to map dst\n"); -- return -ENOMEM; -- } - edesc->src_nents = 0; - -+ dma_sync_single_for_device(jrdev, state->buf_dma, buflen, -+ DMA_TO_DEVICE); - #ifdef DEBUG -- print_hex_dump(KERN_ERR, "jobdesc@"__stringify(__LINE__)": ", -+ print_hex_dump(KERN_ERR, "jobdesc@"xstr(__LINE__)": ", - DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 1); - #endif - -@@ -1245,7 +1450,7 @@ - * allocate space for base edesc and hw desc commands, - * link tables - */ -- edesc = kmalloc(sizeof(struct ahash_edesc) + DESC_JOB_IO_LEN + -+ edesc = kzalloc(sizeof(struct ahash_edesc) + DESC_JOB_IO_LEN + - sec4_sg_bytes, GFP_DMA | flags); - if (!edesc) { - dev_err(jrdev, -@@ -1258,7 +1463,9 @@ - edesc->sec4_sg_bytes = sec4_sg_bytes; - edesc->sec4_sg = (void *)edesc + sizeof(struct ahash_edesc) + - DESC_JOB_IO_LEN; -- edesc->dst_dma = 0; -+ edesc->sec4_sg_dma = dma_map_single(jrdev, edesc->sec4_sg, -+ sec4_sg_bytes, -+ DMA_TO_DEVICE); - - state->buf_dma = buf_map_to_sec4_sg(jrdev, edesc->sec4_sg, - buf, *buflen); -@@ -1276,22 +1483,14 @@ - init_job_desc_shared(desc, ptr, sh_len, HDR_SHARE_DEFER | - HDR_REVERSE); - -- edesc->sec4_sg_dma = dma_map_single(jrdev, edesc->sec4_sg, -- sec4_sg_bytes, -- DMA_TO_DEVICE); -- if (dma_mapping_error(jrdev, edesc->sec4_sg_dma)) { -- dev_err(jrdev, "unable to map S/G table\n"); -- return -ENOMEM; -- } -- - append_seq_in_ptr(desc, edesc->sec4_sg_dma, to_hash, LDST_SGF); - -- ret = map_seq_out_ptr_ctx(desc, jrdev, state, ctx->ctx_len); -- if (ret) -- return ret; -+ map_seq_out_ptr_ctx(desc, jrdev, state, ctx->ctx_len); - -+ dma_sync_single_for_device(jrdev, edesc->sec4_sg_dma, -+ sec4_sg_bytes, DMA_TO_DEVICE); - #ifdef DEBUG -- print_hex_dump(KERN_ERR, "jobdesc@"__stringify(__LINE__)": ", -+ print_hex_dump(KERN_ERR, "jobdesc@"xstr(__LINE__)": ", - DUMP_PREFIX_ADDRESS, 16, 4, desc, - desc_bytes(desc), 1); - #endif -@@ -1314,9 +1513,9 @@ - *next_buflen = 0; - } - #ifdef DEBUG -- print_hex_dump(KERN_ERR, "buf@"__stringify(__LINE__)": ", -+ print_hex_dump(KERN_ERR, "buf@"xstr(__LINE__)": ", - DUMP_PREFIX_ADDRESS, 16, 4, buf, *buflen, 1); -- print_hex_dump(KERN_ERR, "next buf@"__stringify(__LINE__)": ", -+ print_hex_dump(KERN_ERR, "next buf@"xstr(__LINE__)": ", - DUMP_PREFIX_ADDRESS, 16, 4, next_buf, - *next_buflen, 1); - #endif -@@ -1352,7 +1551,7 @@ - sizeof(struct sec4_sg_entry); - - /* allocate space for base edesc and hw desc commands, link tables */ -- edesc = kmalloc(sizeof(struct ahash_edesc) + DESC_JOB_IO_LEN + -+ edesc = kzalloc(sizeof(struct ahash_edesc) + DESC_JOB_IO_LEN + - sec4_sg_bytes, GFP_DMA | flags); - if (!edesc) { - dev_err(jrdev, "could not allocate extended descriptor\n"); -@@ -1368,6 +1567,8 @@ - edesc->sec4_sg_bytes = sec4_sg_bytes; - edesc->sec4_sg = (void *)edesc + sizeof(struct ahash_edesc) + - DESC_JOB_IO_LEN; -+ edesc->sec4_sg_dma = dma_map_single(jrdev, edesc->sec4_sg, -+ sec4_sg_bytes, DMA_TO_DEVICE); - - state->buf_dma = try_buf_map_to_sec4_sg(jrdev, edesc->sec4_sg, buf, - state->buf_dma, buflen, -@@ -1376,25 +1577,17 @@ - src_map_to_sec4_sg(jrdev, req->src, src_nents, edesc->sec4_sg + 1, - chained); - -- edesc->sec4_sg_dma = dma_map_single(jrdev, edesc->sec4_sg, -- sec4_sg_bytes, DMA_TO_DEVICE); -- if (dma_mapping_error(jrdev, edesc->sec4_sg_dma)) { -- dev_err(jrdev, "unable to map S/G table\n"); -- return -ENOMEM; -- } -- - append_seq_in_ptr(desc, edesc->sec4_sg_dma, buflen + - req->nbytes, LDST_SGF); - - edesc->dst_dma = map_seq_out_ptr_result(desc, jrdev, req->result, - digestsize); -- if (dma_mapping_error(jrdev, edesc->dst_dma)) { -- dev_err(jrdev, "unable to map dst\n"); -- return -ENOMEM; -- } -+ -+ dma_sync_single_for_device(jrdev, edesc->sec4_sg_dma, sec4_sg_bytes, -+ DMA_TO_DEVICE); - - #ifdef DEBUG -- print_hex_dump(KERN_ERR, "jobdesc@"__stringify(__LINE__)": ", -+ print_hex_dump(KERN_ERR, "jobdesc@"xstr(__LINE__)": ", - DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 1); - #endif - -@@ -1447,7 +1640,7 @@ - * allocate space for base edesc and hw desc commands, - * link tables - */ -- edesc = kmalloc(sizeof(struct ahash_edesc) + DESC_JOB_IO_LEN + -+ edesc = kzalloc(sizeof(struct ahash_edesc) + DESC_JOB_IO_LEN + - sec4_sg_bytes, GFP_DMA | flags); - if (!edesc) { - dev_err(jrdev, -@@ -1460,19 +1653,13 @@ - edesc->sec4_sg_bytes = sec4_sg_bytes; - edesc->sec4_sg = (void *)edesc + sizeof(struct ahash_edesc) + - DESC_JOB_IO_LEN; -- edesc->dst_dma = 0; -+ edesc->sec4_sg_dma = dma_map_single(jrdev, edesc->sec4_sg, -+ sec4_sg_bytes, -+ DMA_TO_DEVICE); - - if (src_nents) { - sg_to_sec4_sg_last(req->src, src_nents, - edesc->sec4_sg, 0); -- edesc->sec4_sg_dma = dma_map_single(jrdev, -- edesc->sec4_sg, -- sec4_sg_bytes, -- DMA_TO_DEVICE); -- if (dma_mapping_error(jrdev, edesc->sec4_sg_dma)) { -- dev_err(jrdev, "unable to map S/G table\n"); -- return -ENOMEM; -- } - src_dma = edesc->sec4_sg_dma; - options = LDST_SGF; - } else { -@@ -1491,12 +1678,12 @@ - - append_seq_in_ptr(desc, src_dma, to_hash, options); - -- ret = map_seq_out_ptr_ctx(desc, jrdev, state, ctx->ctx_len); -- if (ret) -- return ret; -+ map_seq_out_ptr_ctx(desc, jrdev, state, ctx->ctx_len); - -+ dma_sync_single_for_device(jrdev, edesc->sec4_sg_dma, -+ sec4_sg_bytes, DMA_TO_DEVICE); - #ifdef DEBUG -- print_hex_dump(KERN_ERR, "jobdesc@"__stringify(__LINE__)": ", -+ print_hex_dump(KERN_ERR, "jobdesc@"xstr(__LINE__)": ", - DUMP_PREFIX_ADDRESS, 16, 4, desc, - desc_bytes(desc), 1); - #endif -@@ -1521,7 +1708,7 @@ - req->nbytes, 0); - } - #ifdef DEBUG -- print_hex_dump(KERN_ERR, "next buf@"__stringify(__LINE__)": ", -+ print_hex_dump(KERN_ERR, "next buf@"xstr(__LINE__)": ", - DUMP_PREFIX_ADDRESS, 16, 4, next_buf, - *next_buflen, 1); - #endif -@@ -1735,10 +1922,33 @@ - .alg_type = OP_ALG_ALGSEL_MD5, - .alg_op = OP_ALG_ALGSEL_MD5 | OP_ALG_AAI_HMAC, - }, -+ { -+ .name = "xcbc(aes)", -+ .driver_name = "xcbc-aes-caam", -+ .hmac_name = "xcbc(aes)", -+ .hmac_driver_name = "xcbc-aes-caam", -+ .blocksize = XCBC_MAC_BLOCK_WORDS * 4, -+ .template_ahash = { -+ .init = ahash_init, -+ .update = ahash_update, -+ .final = ahash_final, -+ .finup = ahash_finup, -+ .digest = ahash_digest, -+ .export = ahash_export, -+ .import = ahash_import, -+ .setkey = axcbc_setkey, -+ .halg = { -+ .digestsize = XCBC_MAC_DIGEST_SIZE, -+ }, -+ }, -+ .alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_XCBC_MAC, -+ .alg_op = OP_ALG_ALGSEL_AES, -+ }, - }; - - struct caam_hash_alg { - struct list_head entry; -+ struct device *ctrldev; - int alg_type; - int alg_op; - struct ahash_alg ahash_alg; -@@ -1755,6 +1965,7 @@ - struct caam_hash_alg *caam_hash = - container_of(alg, struct caam_hash_alg, ahash_alg); - struct caam_hash_ctx *ctx = crypto_tfm_ctx(tfm); -+ struct caam_drv_private *priv = dev_get_drvdata(caam_hash->ctrldev); - /* Sizes for MDHA running digests: MD5, SHA1, 224, 256, 384, 512 */ - static const u8 runninglen[] = { HASH_MSG_LEN + MD5_DIGEST_SIZE, - HASH_MSG_LEN + SHA1_DIGEST_SIZE, -@@ -1762,17 +1973,15 @@ - HASH_MSG_LEN + SHA256_DIGEST_SIZE, - HASH_MSG_LEN + 64, - HASH_MSG_LEN + SHA512_DIGEST_SIZE }; -+ int tgt_jr = atomic_inc_return(&priv->tfm_count); - int ret = 0; - - /* -- * Get a Job ring from Job Ring driver to ensure in-order -+ * distribute tfms across job rings to ensure in-order - * crypto request processing per tfm - */ -- ctx->jrdev = caam_jr_alloc(); -- if (IS_ERR(ctx->jrdev)) { -- pr_err("Job Ring Device allocation for transform failed\n"); -- return PTR_ERR(ctx->jrdev); -- } -+ ctx->jrdev = priv->jrdev[tgt_jr % priv->total_jobrs]; -+ - /* copy descriptor header template value */ - ctx->alg_type = OP_TYPE_CLASS2_ALG | caam_hash->alg_type; - ctx->alg_op = OP_TYPE_CLASS2_ALG | caam_hash->alg_op; -@@ -1788,6 +1997,39 @@ - return ret; - } - -+static int caam_axcbc_cra_init(struct crypto_tfm *tfm) -+{ -+ struct crypto_ahash *ahash = __crypto_ahash_cast(tfm); -+ struct crypto_alg *base = tfm->__crt_alg; -+ struct hash_alg_common *halg = -+ container_of(base, struct hash_alg_common, base); -+ struct ahash_alg *alg = -+ container_of(halg, struct ahash_alg, halg); -+ struct caam_hash_alg *caam_hash = -+ container_of(alg, struct caam_hash_alg, ahash_alg); -+ struct caam_hash_ctx *ctx = crypto_tfm_ctx(tfm); -+ struct caam_drv_private *priv = dev_get_drvdata(caam_hash->ctrldev); -+ int tgt_jr = atomic_inc_return(&priv->tfm_count); -+ int ret = 0; -+ -+ /* -+ * distribute tfms across job rings to ensure in-order -+ * crypto request processing per tfm -+ */ -+ ctx->jrdev = priv->jrdev[tgt_jr % priv->total_jobrs]; -+ -+ /* copy descriptor header template value */ -+ ctx->alg_type = OP_TYPE_CLASS1_ALG | caam_hash->alg_type; -+ ctx->alg_op = OP_TYPE_CLASS1_ALG | caam_hash->alg_op; -+ -+ crypto_ahash_set_reqsize(__crypto_ahash_cast(tfm), -+ sizeof(struct caam_hash_state)); -+ -+ ret = axcbc_set_sh_desc(ahash); -+ -+ return ret; -+} -+ - static void caam_hash_cra_exit(struct crypto_tfm *tfm) - { - struct caam_hash_ctx *ctx = crypto_tfm_ctx(tfm); -@@ -1815,35 +2057,57 @@ - !dma_mapping_error(ctx->jrdev, ctx->sh_desc_finup_dma)) - dma_unmap_single(ctx->jrdev, ctx->sh_desc_finup_dma, - desc_bytes(ctx->sh_desc_finup), DMA_TO_DEVICE); -- -- caam_jr_free(ctx->jrdev); - } - - static void __exit caam_algapi_hash_exit(void) - { -+ struct device_node *dev_node; -+ struct platform_device *pdev; -+ struct device *ctrldev; -+ struct caam_drv_private *priv; - struct caam_hash_alg *t_alg, *n; - -- if (!hash_list.next) -+ dev_node = of_find_compatible_node(NULL, NULL, "fsl,sec-v4.0"); -+ if (!dev_node) { -+ dev_node = of_find_compatible_node(NULL, NULL, "fsl,sec4.0"); -+ if (!dev_node) -+ return; -+ } -+ -+ pdev = of_find_device_by_node(dev_node); -+ if (!pdev) { -+ of_node_put(dev_node); - return; -+ } - -- list_for_each_entry_safe(t_alg, n, &hash_list, entry) { -+ ctrldev = &pdev->dev; -+ priv = dev_get_drvdata(ctrldev); -+ -+ if (!priv->hash_list.next) { -+ of_node_put(dev_node); -+ return; -+ } -+ -+ list_for_each_entry_safe(t_alg, n, &priv->hash_list, entry) { - crypto_unregister_ahash(&t_alg->ahash_alg); - list_del(&t_alg->entry); - kfree(t_alg); - } -+ -+ of_node_put(dev_node); - } - - static struct caam_hash_alg * --caam_hash_alloc(struct caam_hash_template *template, -+caam_hash_alloc(struct device *ctrldev, struct caam_hash_template *template, - bool keyed) - { - struct caam_hash_alg *t_alg; - struct ahash_alg *halg; - struct crypto_alg *alg; - -- t_alg = kzalloc(sizeof(struct caam_hash_alg), GFP_KERNEL); -+ t_alg = kzalloc(sizeof(struct caam_hash_alg), GFP_ATOMIC); - if (!t_alg) { -- pr_err("failed to allocate t_alg\n"); -+ dev_err(ctrldev, "failed to allocate t_alg\n"); - return ERR_PTR(-ENOMEM); - } - -@@ -1863,7 +2127,11 @@ - template->driver_name); - } - alg->cra_module = THIS_MODULE; -- alg->cra_init = caam_hash_cra_init; -+ -+ if (strstr(alg->cra_name, "xcbc") > 0) -+ alg->cra_init = caam_axcbc_cra_init; -+ else -+ alg->cra_init = caam_hash_cra_init; - alg->cra_exit = caam_hash_cra_exit; - alg->cra_ctxsize = sizeof(struct caam_hash_ctx); - alg->cra_priority = CAAM_CRA_PRIORITY; -@@ -1874,6 +2142,7 @@ - - t_alg->alg_type = template->alg_type; - t_alg->alg_op = template->alg_op; -+ t_alg->ctrldev = ctrldev; - - return t_alg; - } -@@ -1883,8 +2152,9 @@ - struct device_node *dev_node; - struct platform_device *pdev; - struct device *ctrldev; -- void *priv; -- int i = 0, err = 0; -+ struct caam_drv_private *priv; -+ int i = 0, err = 0, md_limit = 0, md_inst; -+ u64 cha_inst; - - dev_node = of_find_compatible_node(NULL, NULL, "fsl,sec-v4.0"); - if (!dev_node) { -@@ -1898,60 +2168,68 @@ - of_node_put(dev_node); - return -ENODEV; - } -- - ctrldev = &pdev->dev; - priv = dev_get_drvdata(ctrldev); -- of_node_put(dev_node); - -- /* -- * If priv is NULL, it's probably because the caam driver wasn't -- * properly initialized (e.g. RNG4 init failed). Thus, bail out here. -- */ -- if (!priv) -- return -ENODEV; -+ INIT_LIST_HEAD(&priv->hash_list); - -- INIT_LIST_HEAD(&hash_list); -+ atomic_set(&priv->tfm_count, -1); -+ -+ /* register algorithms the device supports */ -+ cha_inst = rd_reg64(&priv->ctrl->perfmon.cha_num); -+ md_inst = (cha_inst & CHA_ID_MD_MASK) >> CHA_ID_MD_SHIFT; -+ if (md_inst) { -+ md_limit = SHA512_DIGEST_SIZE; -+ if ((rd_reg64(&priv->ctrl->perfmon.cha_id) & CHA_ID_MD_MASK) -+ == CHA_ID_MD_LP256) /* LP256 limits digest size */ -+ md_limit = SHA256_DIGEST_SIZE; -+ } - -- /* register crypto algorithms the device supports */ - for (i = 0; i < ARRAY_SIZE(driver_hash); i++) { -- /* TODO: check if h/w supports alg */ - struct caam_hash_alg *t_alg; - -+ /* If no MD instantiated, or MD too small, skip */ -+ if ((!md_inst) || -+ (driver_hash[i].template_ahash.halg.digestsize > -+ md_limit)) -+ continue; -+ - /* register hmac version */ -- t_alg = caam_hash_alloc(&driver_hash[i], true); -+ t_alg = caam_hash_alloc(ctrldev, &driver_hash[i], true); - if (IS_ERR(t_alg)) { - err = PTR_ERR(t_alg); -- pr_warn("%s alg allocation failed\n", -- driver_hash[i].driver_name); -+ dev_warn(ctrldev, "%s alg allocation failed\n", -+ driver_hash[i].driver_name); - continue; - } - - err = crypto_register_ahash(&t_alg->ahash_alg); - if (err) { -- pr_warn("%s alg registration failed\n", -+ dev_warn(ctrldev, "%s alg registration failed\n", - t_alg->ahash_alg.halg.base.cra_driver_name); - kfree(t_alg); - } else -- list_add_tail(&t_alg->entry, &hash_list); -+ list_add_tail(&t_alg->entry, &priv->hash_list); - - /* register unkeyed version */ -- t_alg = caam_hash_alloc(&driver_hash[i], false); -+ t_alg = caam_hash_alloc(ctrldev, &driver_hash[i], false); - if (IS_ERR(t_alg)) { - err = PTR_ERR(t_alg); -- pr_warn("%s alg allocation failed\n", -- driver_hash[i].driver_name); -+ dev_warn(ctrldev, "%s alg allocation failed\n", -+ driver_hash[i].driver_name); - continue; - } - - err = crypto_register_ahash(&t_alg->ahash_alg); - if (err) { -- pr_warn("%s alg registration failed\n", -+ dev_warn(ctrldev, "%s alg registration failed\n", - t_alg->ahash_alg.halg.base.cra_driver_name); - kfree(t_alg); - } else -- list_add_tail(&t_alg->entry, &hash_list); -+ list_add_tail(&t_alg->entry, &priv->hash_list); - } - -+ of_node_put(dev_node); - return err; - } - -diff -Nur linux-4.1.3/drivers/crypto/caam/caam_keyblob.c linux-xbian-imx6/drivers/crypto/caam/caam_keyblob.c ---- linux-4.1.3/drivers/crypto/caam/caam_keyblob.c 1970-01-01 01:00:00.000000000 +0100 -+++ linux-xbian-imx6/drivers/crypto/caam/caam_keyblob.c 2015-07-27 23:13:04.205975852 +0200 -@@ -0,0 +1,687 @@ -+/* -+ * Key blob driver based on CAAM hardware -+ * -+ * Copyright (C) 2015 Freescale Semiconductor, Inc. -+ */ -+ -+#include -+#include -+ -+#include "compat.h" -+#include "regs.h" -+#include "jr.h" -+#include "desc.h" -+#include "intern.h" -+#include "sm.h" -+#include "caam_keyblob.h" -+ -+#define INITIAL_DESCSZ 16 /* size of tmp buffer for descriptor const. */ -+ -+/** -+ * struct kb_device - the metadata of the caam key blob device node -+ * @dev: the actual misc device -+ */ -+struct kb_device { -+ struct miscdevice misc_dev; -+ struct device *jr_dev; -+}; -+ -+/* -+ * Pseudo-synchronous ring access functions for carrying out key -+ * encapsulation and decapsulation -+ */ -+ -+struct sm_key_job_result { -+ int error; -+ struct completion completion; -+}; -+ -+ -+static struct kb_device *kb_dev; -+ -+static struct kb_device *kb_device_create(void); -+static int kb_device_destroy(struct kb_device *kb_dev); -+static int kb_open(struct inode *inode, struct file *file); -+static int kb_release(struct inode *inode, struct file *file); -+static void sm_key_job_done(struct device *dev, u32 *desc, -+ u32 err, void *context); -+static int gen_mem_encap(struct device *jr_dev, void __user *secretbuf, -+ int keylen, void __user *kmodbuf, void __user *outbuf); -+static int gen_mem_decap(struct device *jr_dev, void __user *keyblobbuf, -+ int bloblen, void __user *kmodbuf, void __user *outbuf); -+static long kb_ioctl(struct file *file, unsigned int cmd, unsigned long arg); -+static int caam_keyblob_probe(struct platform_device *pdev); -+static int caam_keyblob_remove(struct platform_device *pdev); -+ -+static int kb_open(struct inode *inode, struct file *file) -+{ -+ struct miscdevice *miscdev = file->private_data; -+ struct kb_device *dev = container_of(miscdev, struct kb_device, misc_dev); -+ struct device *jr_dev; -+ -+ if (!dev->jr_dev) { -+ jr_dev = caam_jr_alloc(); -+ if (IS_ERR(jr_dev)) { -+ pr_err("Job Ring Device allocation for transform failed\n"); -+ return -ENOMEM; -+ } -+ pr_info("Allocate a job ring device\n"); -+ dev->jr_dev = jr_dev; -+ } -+ else { -+ pr_err("Already created a job ring device"); -+ return -EPERM; -+ } -+ -+ return 0; -+} -+ -+static int kb_release(struct inode *inode, struct file *file) -+{ -+ struct miscdevice *miscdev = file->private_data; -+ struct kb_device *dev = container_of(miscdev, struct kb_device, misc_dev); -+ -+ if (dev && dev->jr_dev) { -+ caam_jr_free(dev->jr_dev); -+ pr_info("Free a job ring device\n"); -+ dev->jr_dev = NULL; -+ } -+ return 0; -+} -+ -+static void sm_key_job_done(struct device *dev, u32 *desc, -+ u32 err, void *context) -+{ -+ struct sm_key_job_result *res = context; -+ -+ res->error = err; /* save off the error for postprocessing */ -+ complete(&res->completion); /* mark us complete */ -+} -+ -+/* -+ * Construct a blob encapsulation job descriptor -+ * -+ * This function dynamically constructs a blob encapsulation job descriptor -+ * from the following arguments: -+ * -+ * - desc pointer to a pointer to the descriptor generated by this -+ * function. Caller will be responsible to kfree() this -+ * descriptor after execution. -+ * - keymod Physical pointer to a key modifier, which must reside in a -+ * contiguous piece of memory. Modifier will be assumed to be -+ * 8 bytes long for a blob of type SM_SECMEM, or 16 bytes long -+ * for a blob of type SM_GENMEM (see blobtype argument). -+ * - secretbuf Physical pointer to a secret, normally a black or red key, -+ * possibly residing within an accessible secure memory page, -+ * of the secret to be encapsulated to an output blob. -+ * - outbuf Physical pointer to the destination buffer to receive the -+ * encapsulated output. This buffer will need to be 48 bytes -+ * larger than the input because of the added encapsulation data. -+ * The generated descriptor will account for the increase in size, -+ * but the caller must also account for this increase in the -+ * buffer allocator. -+ * - secretsz Size of input secret, in bytes. This is limited to 65536 -+ * less the size of blob overhead, since the length embeds into -+ * DECO pointer in/out instructions. -+ * - keycolor Determines if the source data is covered (black key) or -+ * plaintext (red key). RED_KEY or BLACK_KEY are defined in -+ * for this purpose. -+ * - blobtype Determine if encapsulated blob should be a secure memory -+ * blob (SM_SECMEM), with partition data embedded with key -+ * material, or a general memory blob (SM_GENMEM). -+ * - auth If BLACK_KEY source is covered via AES-CCM, specify -+ * KEY_COVER_CCM, else uses AES-ECB (KEY_COVER_ECB). -+ * -+ * Upon completion, desc points to a buffer containing a CAAM job -+ * descriptor which encapsulates data into an externally-storable blob -+ * suitable for use across power cycles. -+ * -+ * This is an example of a black key encapsulation job into a general memory -+ * blob. Notice the 16-byte key modifier in the LOAD instruction. Also note -+ * the output 48 bytes longer than the input: -+ * -+ * [00] B0800008 jobhdr: stidx=0 len=8 -+ * [01] 14400010 ld: ccb2-key len=16 offs=0 -+ * [02] 08144891 ptr->@0x08144891 -+ * [03] F800003A seqoutptr: len=58 -+ * [04] 01000000 out_ptr->@0x01000000 -+ * [05] F000000A seqinptr: len=10 -+ * [06] 09745090 in_ptr->@0x09745090 -+ * [07] 870D0004 operation: encap blob reg=memory, black, format=normal -+ * -+ * This is an example of a red key encapsulation job for storing a red key -+ * into a secure memory blob. Note the 8 byte modifier on the 12 byte offset -+ * in the LOAD instruction; this accounts for blob permission storage: -+ * -+ * [00] B0800008 jobhdr: stidx=0 len=8 -+ * [01] 14400C08 ld: ccb2-key len=8 offs=12 -+ * [02] 087D0784 ptr->@0x087d0784 -+ * [03] F8000050 seqoutptr: len=80 -+ * [04] 09251BB2 out_ptr->@0x09251bb2 -+ * [05] F0000020 seqinptr: len=32 -+ * [06] 40000F31 in_ptr->@0x40000f31 -+ * [07] 870D0008 operation: encap blob reg=memory, red, sec_mem, -+ * format=normal -+ * -+ * Note: this function only generates 32-bit pointers at present, and should -+ * be refactored using a scheme that allows both 32 and 64 bit addressing -+ */ -+ -+static int blob_encap_jobdesc(u32 **desc, dma_addr_t keymod, -+ void *secretbuf, dma_addr_t outbuf, -+ u16 secretsz, u8 keycolor, u8 blobtype, u8 auth) -+{ -+ u32 *tdesc, tmpdesc[INITIAL_DESCSZ]; -+ u16 dsize, idx; -+ -+ memset(tmpdesc, 0, INITIAL_DESCSZ * sizeof(u32)); -+ idx = 1; -+ -+ /* -+ * Key modifier works differently for secure/general memory blobs -+ * This accounts for the permission/protection data encapsulated -+ * within the blob if a secure memory blob is requested -+ */ -+ if (blobtype == SM_SECMEM) -+ tmpdesc[idx++] = CMD_LOAD | LDST_CLASS_2_CCB | -+ LDST_SRCDST_BYTE_KEY | -+ ((12 << LDST_OFFSET_SHIFT) & LDST_OFFSET_MASK) -+ | (8 & LDST_LEN_MASK); -+ else /* is general memory blob */ -+ tmpdesc[idx++] = CMD_LOAD | LDST_CLASS_2_CCB | -+ LDST_SRCDST_BYTE_KEY | (16 & LDST_LEN_MASK); -+ -+ tmpdesc[idx++] = (u32)keymod; -+ -+ /* -+ * Encapsulation output must include space for blob key encryption -+ * key and MAC tag -+ */ -+ tmpdesc[idx++] = CMD_SEQ_OUT_PTR | (secretsz + BLOB_OVERHEAD); -+ tmpdesc[idx++] = (u32)outbuf; -+ -+ /* Input data, should be somewhere in secure memory */ -+ tmpdesc[idx++] = CMD_SEQ_IN_PTR | secretsz; -+ tmpdesc[idx++] = (u32)secretbuf; -+ -+ /* Set blob encap, then color */ -+ tmpdesc[idx] = CMD_OPERATION | OP_TYPE_ENCAP_PROTOCOL | OP_PCLID_BLOB; -+ -+ if (blobtype == SM_SECMEM) -+ tmpdesc[idx] |= OP_PCL_BLOB_PTXT_SECMEM; -+ -+ if (auth == KEY_COVER_CCM) -+ tmpdesc[idx] |= OP_PCL_BLOB_EKT; -+ -+ if (keycolor == BLACK_KEY) -+ tmpdesc[idx] |= OP_PCL_BLOB_BLACK; -+ -+ idx++; -+ tmpdesc[0] = CMD_DESC_HDR | HDR_ONE | (idx & HDR_DESCLEN_MASK); -+ dsize = idx * sizeof(u32); -+ -+ tdesc = kmalloc(dsize, GFP_KERNEL | GFP_DMA); -+ if (tdesc == NULL) -+ return 0; -+ -+ memcpy(tdesc, tmpdesc, dsize); -+ *desc = tdesc; -+ return dsize; -+} -+ -+/* -+ * Construct a blob decapsulation job descriptor -+ * -+ * This function dynamically constructs a blob decapsulation job descriptor -+ * from the following arguments: -+ * -+ * - desc pointer to a pointer to the descriptor generated by this -+ * function. Caller will be responsible to kfree() this -+ * descriptor after execution. -+ * - keymod Physical pointer to a key modifier, which must reside in a -+ * contiguous piece of memory. Modifier will be assumed to be -+ * 8 bytes long for a blob of type SM_SECMEM, or 16 bytes long -+ * for a blob of type SM_GENMEM (see blobtype argument). -+ * - blobbuf Physical pointer (into external memory) of the blob to -+ * be decapsulated. Blob must reside in a contiguous memory -+ * segment. -+ * - outbuf Physical pointer of the decapsulated output, possibly into -+ * a location within a secure memory page. Must be contiguous. -+ * - secretsz Size of encapsulated secret in bytes (not the size of the -+ * input blob). -+ * - keycolor Determines if decapsulated content is encrypted (BLACK_KEY) -+ * or left as plaintext (RED_KEY). -+ * - blobtype Determine if encapsulated blob should be a secure memory -+ * blob (SM_SECMEM), with partition data embedded with key -+ * material, or a general memory blob (SM_GENMEM). -+ * - auth If decapsulation path is specified by BLACK_KEY, then if -+ * AES-CCM is requested for key covering use KEY_COVER_CCM, else -+ * use AES-ECB (KEY_COVER_ECB). -+ * -+ * Upon completion, desc points to a buffer containing a CAAM job descriptor -+ * that decapsulates a key blob from external memory into a black (encrypted) -+ * key or red (plaintext) content. -+ * -+ * This is an example of a black key decapsulation job from a general memory -+ * blob. Notice the 16-byte key modifier in the LOAD instruction. -+ * -+ * [00] B0800008 jobhdr: stidx=0 len=8 -+ * [01] 14400010 ld: ccb2-key len=16 offs=0 -+ * [02] 08A63B7F ptr->@0x08a63b7f -+ * [03] F8000010 seqoutptr: len=16 -+ * [04] 01000000 out_ptr->@0x01000000 -+ * [05] F000003A seqinptr: len=58 -+ * [06] 01000010 in_ptr->@0x01000010 -+ * [07] 860D0004 operation: decap blob reg=memory, black, format=normal -+ * -+ * This is an example of a red key decapsulation job for restoring a red key -+ * from a secure memory blob. Note the 8 byte modifier on the 12 byte offset -+ * in the LOAD instruction: -+ * -+ * [00] B0800008 jobhdr: stidx=0 len=8 -+ * [01] 14400C08 ld: ccb2-key len=8 offs=12 -+ * [02] 01000000 ptr->@0x01000000 -+ * [03] F8000020 seqoutptr: len=32 -+ * [04] 400000E6 out_ptr->@0x400000e6 -+ * [05] F0000050 seqinptr: len=80 -+ * [06] 08F0C0EA in_ptr->@0x08f0c0ea -+ * [07] 860D0008 operation: decap blob reg=memory, red, sec_mem, -+ * format=normal -+ * -+ * Note: this function only generates 32-bit pointers at present, and should -+ * be refactored using a scheme that allows both 32 and 64 bit addressing -+ */ -+ -+static int blob_decap_jobdesc(u32 **desc, dma_addr_t keymod, dma_addr_t blobbuf, -+ u8 *outbuf, u16 secretsz, u8 keycolor, -+ u8 blobtype, u8 auth) -+{ -+ u32 *tdesc, tmpdesc[INITIAL_DESCSZ]; -+ u16 dsize, idx; -+ -+ memset(tmpdesc, 0, INITIAL_DESCSZ * sizeof(u32)); -+ idx = 1; -+ -+ /* Load key modifier */ -+ if (blobtype == SM_SECMEM) -+ tmpdesc[idx++] = CMD_LOAD | LDST_CLASS_2_CCB | -+ LDST_SRCDST_BYTE_KEY | -+ ((12 << LDST_OFFSET_SHIFT) & LDST_OFFSET_MASK) -+ | (8 & LDST_LEN_MASK); -+ else /* is general memory blob */ -+ tmpdesc[idx++] = CMD_LOAD | LDST_CLASS_2_CCB | -+ LDST_SRCDST_BYTE_KEY | (16 & LDST_LEN_MASK); -+ -+ tmpdesc[idx++] = (u32)keymod; -+ -+ /* Compensate BKEK + MAC tag over size of encapsulated secret */ -+ tmpdesc[idx++] = CMD_SEQ_IN_PTR | (secretsz + BLOB_OVERHEAD); -+ tmpdesc[idx++] = (u32)blobbuf; -+ tmpdesc[idx++] = CMD_SEQ_OUT_PTR | secretsz; -+ tmpdesc[idx++] = (u32)outbuf; -+ -+ /* Decapsulate from secure memory partition to black blob */ -+ tmpdesc[idx] = CMD_OPERATION | OP_TYPE_DECAP_PROTOCOL | OP_PCLID_BLOB; -+ -+ if (blobtype == SM_SECMEM) -+ tmpdesc[idx] |= OP_PCL_BLOB_PTXT_SECMEM; -+ -+ if (auth == KEY_COVER_CCM) -+ tmpdesc[idx] |= OP_PCL_BLOB_EKT; -+ -+ if (keycolor == BLACK_KEY) -+ tmpdesc[idx] |= OP_PCL_BLOB_BLACK; -+ -+ idx++; -+ tmpdesc[0] = CMD_DESC_HDR | HDR_ONE | (idx & HDR_DESCLEN_MASK); -+ dsize = idx * sizeof(u32); -+ -+ tdesc = kmalloc(dsize, GFP_KERNEL | GFP_DMA); -+ if (tdesc == NULL) -+ return 0; -+ -+ memcpy(tdesc, tmpdesc, dsize); -+ *desc = tdesc; -+ return dsize; -+} -+ -+ -+ -+static int gen_mem_encap(struct device *jr_dev, void __user *secretbuf, -+ int keylen, void __user *kmodbuf, void __user *outbuf) -+{ -+ int retval = 0; -+ u32 dsize; -+ u32 __iomem *encapdesc = NULL; -+ dma_addr_t secret_dma = 0, keymod_dma = 0, outbuf_dma = 0; -+ u8 __iomem *lsecret = NULL, *lkeymod = NULL, *loutbuf = NULL; -+ struct sm_key_job_result testres; -+ -+ /* Build/map/flush the scret */ -+ lsecret = kmalloc(keylen, GFP_KERNEL | GFP_DMA); -+ if (!lsecret) { -+ dev_err(jr_dev, "%s: can't alloc for key\n", __func__); -+ retval = -ENOMEM; -+ goto out; -+ } -+ if (copy_from_user(lsecret, secretbuf, keylen)) { -+ dev_err(jr_dev, "%s: can't Copy for key\n", __func__); -+ retval = -EFAULT; -+ goto out; -+ } -+ secret_dma = dma_map_single(jr_dev, lsecret, keylen, -+ DMA_TO_DEVICE); -+ -+ /* Build/map/flush the key modifier */ -+ lkeymod = kmalloc(GENMEM_KEYMOD_LEN, GFP_KERNEL | GFP_DMA); -+ if (!lkeymod) { -+ dev_err(jr_dev, "%s: can't alloc for keymod\n", __func__); -+ retval = -ENOMEM; -+ goto out; -+ } -+ if (copy_from_user(lkeymod, kmodbuf, GENMEM_KEYMOD_LEN)) { -+ dev_err(jr_dev, "%s: can't Copy for keymod\n", __func__); -+ retval = -EFAULT; -+ goto out; -+ } -+ keymod_dma = dma_map_single(jr_dev, lkeymod, GENMEM_KEYMOD_LEN, -+ DMA_TO_DEVICE); -+ -+ loutbuf = kmalloc(keylen + BLOB_OVERHEAD, GFP_KERNEL | GFP_DMA); -+ if (!lkeymod) { -+ dev_err(jr_dev, "%s: can't alloc for output\n", __func__); -+ retval = -ENOMEM; -+ goto out; -+ } -+ outbuf_dma = dma_map_single(jr_dev, loutbuf, keylen + BLOB_OVERHEAD, -+ DMA_FROM_DEVICE); -+ dsize = blob_encap_jobdesc(&encapdesc, keymod_dma, (void *)secret_dma, outbuf_dma, -+ keylen, RED_KEY, SM_GENMEM, KEY_COVER_ECB); -+ if (!dsize) { -+ dev_err(jr_dev, "can't alloc an encapsulation descriptor\n"); -+ retval = -ENOMEM; -+ goto out; -+ } -+ init_completion(&testres.completion); -+ -+ retval = caam_jr_enqueue(jr_dev, encapdesc, sm_key_job_done, -+ &testres); -+ if (!retval) { -+ wait_for_completion_interruptible(&testres.completion); -+ dev_info(jr_dev, "job ring return %d\n", testres.error); -+ if (!testres.error) { -+ dma_sync_single_for_cpu(jr_dev, outbuf_dma, keylen + BLOB_OVERHEAD, -+ DMA_FROM_DEVICE); -+ -+ if (copy_to_user(outbuf, loutbuf, keylen + BLOB_OVERHEAD)) { -+ retval = -EFAULT; -+ dev_err(jr_dev, "can't copy for output\n"); -+ goto out; -+ } -+ } -+ retval = testres.error; -+ } -+ -+out: -+ if (outbuf_dma) -+ dma_unmap_single(jr_dev, outbuf_dma, keylen + BLOB_OVERHEAD, -+ DMA_FROM_DEVICE); -+ if (keymod_dma) -+ dma_unmap_single(jr_dev, keymod_dma, GENMEM_KEYMOD_LEN, DMA_TO_DEVICE); -+ if (secret_dma) -+ dma_unmap_single(jr_dev, secret_dma, keylen, DMA_TO_DEVICE); -+ kfree(encapdesc); -+ kfree(lkeymod); -+ kfree(lsecret); -+ kfree(loutbuf); -+ -+ return retval; -+} -+ -+static int gen_mem_decap(struct device *jr_dev, void __user *keyblobbuf, -+ int bloblen, void __user *kmodbuf, void __user *outbuf) -+{ -+ int retval = 0; -+ int keylen = bloblen - BLOB_OVERHEAD; -+ u32 dsize; -+ dma_addr_t keyblob_dma = 0, keymod_dma = 0, outbuf_dma = 0; -+ u8 __iomem *lkeyblob = NULL, *lkeymod = NULL, *loutbuf = NULL; -+ struct sm_key_job_result testres; -+ u32 __iomem *decapdesc = NULL; -+ -+ /* Build/map/flush the scret */ -+ lkeyblob = kmalloc(bloblen, GFP_KERNEL | GFP_DMA); -+ if (!lkeyblob) { -+ dev_err(jr_dev, "%s: can't alloc for keylob\n", __func__); -+ retval = -ENOMEM; -+ goto out; -+ } -+ if (copy_from_user(lkeyblob, keyblobbuf, bloblen)) { -+ dev_err(jr_dev, "%s: can't Copy for keyblob\n", __func__); -+ retval = -EFAULT; -+ goto out; -+ } -+ keyblob_dma = dma_map_single(jr_dev, lkeyblob, bloblen, -+ DMA_TO_DEVICE); -+ -+ /* Build/map/flush the key modifier */ -+ lkeymod = kmalloc(GENMEM_KEYMOD_LEN, GFP_KERNEL | GFP_DMA); -+ if (!lkeymod) { -+ dev_err(jr_dev, "%s: can't alloc for keymod\n", __func__); -+ retval = -ENOMEM; -+ goto out; -+ } -+ if (copy_from_user(lkeymod, kmodbuf, GENMEM_KEYMOD_LEN)) { -+ dev_err(jr_dev, "%s: can't Copy for keymod\n", __func__); -+ retval = -EFAULT; -+ goto out; -+ } -+ keymod_dma = dma_map_single(jr_dev, lkeymod, GENMEM_KEYMOD_LEN, -+ DMA_TO_DEVICE); -+ -+ loutbuf = kmalloc(keylen, GFP_KERNEL | GFP_DMA); -+ if (!loutbuf) { -+ dev_err(jr_dev, "%s: can't alloc for outbuf\n", __func__); -+ retval = -ENOMEM; -+ goto out; -+ } -+ outbuf_dma = dma_map_single(jr_dev, loutbuf, keylen, -+ DMA_FROM_DEVICE); -+ -+ /* Build the encapsulation job descriptor */ -+ dsize = blob_decap_jobdesc(&decapdesc, keymod_dma, keyblob_dma, (u8 *)outbuf_dma, -+ keylen, RED_KEY, SM_GENMEM, KEY_COVER_ECB); -+ if (!dsize) { -+ dev_err(jr_dev, "can't alloc a decapsulation descriptor\n"); -+ retval = -ENOMEM; -+ goto out; -+ } -+ -+ init_completion(&testres.completion); -+ -+ retval = caam_jr_enqueue(jr_dev, decapdesc, sm_key_job_done, -+ &testres); -+ if (!retval) { -+ wait_for_completion_interruptible(&testres.completion); -+ dev_info(jr_dev, "job ring return %d\n", testres.error); -+ if (!testres.error) { -+ dma_sync_single_for_cpu(jr_dev, outbuf_dma, keylen, -+ DMA_FROM_DEVICE); -+ -+ if (copy_to_user(outbuf, loutbuf, keylen)) { -+ retval = -EFAULT; -+ goto out; -+ } -+ } -+ retval = testres.error; -+ } -+ -+out: -+ if (outbuf_dma) -+ dma_unmap_single(jr_dev, outbuf_dma, keylen, -+ DMA_FROM_DEVICE); -+ if (keymod_dma) -+ dma_unmap_single(jr_dev, keymod_dma, GENMEM_KEYMOD_LEN, -+ DMA_TO_DEVICE); -+ if (keyblob_dma) -+ dma_unmap_single(jr_dev, keyblob_dma, bloblen, -+ DMA_TO_DEVICE); -+ kfree(decapdesc); -+ kfree(lkeymod); -+ kfree(lkeyblob); -+ kfree(loutbuf); -+ -+ return retval; -+} -+ -+ -+static long kb_ioctl(struct file *file, unsigned int cmd, unsigned long arg) -+{ -+ int retval = 0; -+ struct caam_kb_data kb_data; -+ struct miscdevice *miscdev = file->private_data; -+ struct kb_device *dev = container_of(miscdev, struct kb_device, misc_dev); -+ -+ if (copy_from_user(&kb_data, (void *)arg, sizeof(kb_data))) { -+ retval = -EFAULT; -+ goto err; -+ } -+ -+ if (!kb_data.rawkey || !kb_data.keyblob || -+ (kb_data.rawkey_len + BLOB_OVERHEAD != kb_data.keyblob_len) || -+ (kb_data.keymod_len != GENMEM_KEYMOD_LEN)) { -+ retval = -EINVAL; -+ goto err; -+ } -+ -+ printk(KERN_INFO"%s:rawkey_len %d, keyblob_len %d\n", -+ __func__, kb_data.rawkey_len, kb_data.keyblob_len); -+ -+ switch (cmd) { -+ case CAAM_KB_ENCRYPT: -+ { -+ retval = gen_mem_encap(dev->jr_dev, kb_data.rawkey, kb_data.rawkey_len, -+ kb_data.keymod, kb_data.keyblob); -+ break; -+ } -+ case CAAM_KB_DECRYPT: -+ { -+ retval = gen_mem_decap(dev->jr_dev, kb_data.keyblob, kb_data.keyblob_len, -+ kb_data.keymod, kb_data.rawkey); -+ break; -+ } -+ default: -+ return -ENOTTY; -+ } -+ -+err: -+ return retval; -+} -+ -+static const struct file_operations kb_fops = { -+ .owner = THIS_MODULE, -+ .open = kb_open, -+ .release = kb_release, -+ .unlocked_ioctl = kb_ioctl, -+}; -+ -+static struct kb_device *kb_device_create(void) -+{ -+ struct kb_device *idev; -+ int ret; -+ -+ idev = kzalloc(sizeof(struct kb_device), GFP_KERNEL); -+ if (!idev) -+ return ERR_PTR(-ENOMEM); -+ -+ idev->misc_dev.minor = MISC_DYNAMIC_MINOR; -+ idev->misc_dev.name = "caam_kb"; -+ idev->misc_dev.fops = &kb_fops; -+ idev->misc_dev.parent = NULL; -+ ret = misc_register(&idev->misc_dev); -+ if (ret) { -+ pr_err("ion: failed to register misc device.\n"); -+ return ERR_PTR(ret); -+ } -+ -+ return idev; -+} -+ -+static int kb_device_destroy(struct kb_device *kb_dev) -+{ -+ if ((kb_dev) && (kb_dev->jr_dev)) { -+ caam_jr_free(kb_dev->jr_dev); -+ kb_dev->jr_dev = NULL; -+ } -+ -+ if (kb_dev) -+ misc_deregister(&kb_dev->misc_dev); -+ -+ return 0; -+} -+/* -+ * Probe key blob device -+ */ -+static int caam_keyblob_probe(struct platform_device *pdev) -+{ -+ int err; -+ -+ dev_dbg(&pdev->dev, "%s enter\n", __func__); -+ kb_dev = kb_device_create(); -+ if (IS_ERR_OR_NULL(kb_dev)) { -+ err = PTR_ERR(kb_dev); -+ goto err; -+ } -+ return 0; -+err: -+ return err; -+} -+ -+/* -+ * Remove key blob device -+ */ -+static int caam_keyblob_remove(struct platform_device *pdev) -+{ -+ kb_device_destroy(kb_dev); -+ return 0; -+} -+ -+static struct of_device_id caam_keyblob_match[] = { -+ { -+ .compatible = "fsl,sec-v4.0-keyblob", -+ }, -+ { -+ .compatible = "fsl,sec4.0-keyblob", -+ }, -+ {}, -+}; -+ -+MODULE_DEVICE_TABLE(of, caam_keyblob_match); -+ -+static struct platform_driver caam_keyblob_driver = { -+ .driver = { -+ .name = "caam_keyblob", -+ .owner = THIS_MODULE, -+ .of_match_table = caam_keyblob_match, -+ }, -+ .probe = caam_keyblob_probe, -+ .remove = caam_keyblob_remove, -+}; -+ -+static int __init keyblob_driver_init(void) -+{ -+ return platform_driver_register(&caam_keyblob_driver); -+} -+ -+static void __exit keyblob_driver_exit(void) -+{ -+ platform_driver_unregister(&caam_keyblob_driver); -+} -+ -+module_init(keyblob_driver_init); -+module_exit(keyblob_driver_exit); -+ -+ -+MODULE_LICENSE("Dual BSD/GPL"); -+MODULE_DESCRIPTION("FSL CAAM Secure Memory / Keystore"); -+MODULE_AUTHOR("Freescale Semiconductor - NMSG/MAD"); -diff -Nur linux-4.1.3/drivers/crypto/caam/caam_keyblob.h linux-xbian-imx6/drivers/crypto/caam/caam_keyblob.h ---- linux-4.1.3/drivers/crypto/caam/caam_keyblob.h 1970-01-01 01:00:00.000000000 +0100 -+++ linux-xbian-imx6/drivers/crypto/caam/caam_keyblob.h 2015-07-27 23:13:04.205975852 +0200 -@@ -0,0 +1,45 @@ -+/* -+ * CAAM public-level include definitions for the key blob -+ * -+ * Copyright (C) 2015 Freescale Semiconductor, Inc. -+ */ -+ -+#ifndef CAAM_KEYBLOB_H -+#define CAAM_KEYBLOB_H -+ -+ -+#include -+#include -+ -+struct caam_kb_data { -+ char *rawkey; -+ size_t rawkey_len; -+ char *keyblob; -+ size_t keyblob_len; -+ char *keymod; -+ size_t keymod_len; -+}; -+ -+ -+#define CAAM_KB_MAGIC 'I' -+ -+/** -+ * DOC: CAAM_KB_ENCRYPT - generate a key blob from raw key -+ * -+ * Takes an caam_kb_data struct and returns it with the key blob -+ */ -+#define CAAM_KB_ENCRYPT _IOWR(CAAM_KB_MAGIC, 0, \ -+ struct caam_kb_data) -+ -+/** -+ * DOC: CAAM_KB_DECRYPT - get keys from a key blob -+ * -+ * Takes an caam_kb_data struct and returns it with the raw key. -+ */ -+#define CAAM_KB_DECRYPT _IOWR(CAAM_KB_MAGIC, 1, struct caam_kb_data) -+ -+#ifndef GENMEM_KEYMOD_LEN -+#define GENMEM_KEYMOD_LEN 16 -+#endif -+ -+#endif /* CAAM_KEYBLOB_H */ -diff -Nur linux-4.1.3/drivers/crypto/caam/caamrng.c linux-xbian-imx6/drivers/crypto/caam/caamrng.c ---- linux-4.1.3/drivers/crypto/caam/caamrng.c 2015-07-21 19:10:33.000000000 +0200 -+++ linux-xbian-imx6/drivers/crypto/caam/caamrng.c 2015-07-27 23:13:04.209961631 +0200 -@@ -1,7 +1,7 @@ - /* - * caam - Freescale FSL CAAM support for hw_random - * -- * Copyright 2011 Freescale Semiconductor, Inc. -+ * Copyright (C) 2011-2013 Freescale Semiconductor, Inc. - * - * Based on caamalg.c crypto API driver. - * -@@ -80,9 +80,12 @@ - - static inline void rng_unmap_buf(struct device *jrdev, struct buf_data *bd) - { -- if (bd->addr) -+ if (bd->addr) { -+ dma_sync_single_for_cpu(jrdev, bd->addr, RN_BUF_SIZE, -+ DMA_FROM_DEVICE); - dma_unmap_single(jrdev, bd->addr, RN_BUF_SIZE, - DMA_FROM_DEVICE); -+ } - } - - static inline void rng_unmap_ctx(struct caam_rng_ctx *ctx) -@@ -103,11 +106,18 @@ - bd = (struct buf_data *)((char *)desc - - offsetof(struct buf_data, hw_desc)); - -- if (err) -- caam_jr_strstatus(jrdev, err); -+ if (err) { -+ char tmp[CAAM_ERROR_STR_MAX]; -+ -+ dev_err(jrdev, "%08x: %s\n", err, caam_jr_strstatus(tmp, err)); -+ } - - atomic_set(&bd->empty, BUF_NOT_EMPTY); - complete(&bd->filled); -+ -+ /* Buffer refilled, invalidate cache */ -+ dma_sync_single_for_cpu(jrdev, bd->addr, RN_BUF_SIZE, DMA_FROM_DEVICE); -+ - #ifdef DEBUG - print_hex_dump(KERN_ERR, "rng refreshed buf@: ", - DUMP_PREFIX_ADDRESS, 16, 4, bd->buf, RN_BUF_SIZE, 1); -@@ -185,7 +195,7 @@ - max - copied_idx, false); - } - --static inline int rng_create_sh_desc(struct caam_rng_ctx *ctx) -+static inline void rng_create_sh_desc(struct caam_rng_ctx *ctx) - { - struct device *jrdev = ctx->jrdev; - u32 *desc = ctx->sh_desc; -@@ -203,18 +213,16 @@ - - ctx->sh_desc_dma = dma_map_single(jrdev, desc, desc_bytes(desc), - DMA_TO_DEVICE); -- if (dma_mapping_error(jrdev, ctx->sh_desc_dma)) { -- dev_err(jrdev, "unable to map shared descriptor\n"); -- return -ENOMEM; -- } -+ dma_sync_single_for_device(jrdev, ctx->sh_desc_dma, desc_bytes(desc), -+ DMA_TO_DEVICE); -+ - #ifdef DEBUG - print_hex_dump(KERN_ERR, "rng shdesc@: ", DUMP_PREFIX_ADDRESS, 16, 4, - desc, desc_bytes(desc), 1); - #endif -- return 0; - } - --static inline int rng_create_job_desc(struct caam_rng_ctx *ctx, int buf_id) -+static inline void rng_create_job_desc(struct caam_rng_ctx *ctx, int buf_id) - { - struct device *jrdev = ctx->jrdev; - struct buf_data *bd = &ctx->bufs[buf_id]; -@@ -225,17 +233,12 @@ - HDR_REVERSE); - - bd->addr = dma_map_single(jrdev, bd->buf, RN_BUF_SIZE, DMA_FROM_DEVICE); -- if (dma_mapping_error(jrdev, bd->addr)) { -- dev_err(jrdev, "unable to map dst\n"); -- return -ENOMEM; -- } - - append_seq_out_ptr_intlen(desc, bd->addr, RN_BUF_SIZE, 0); - #ifdef DEBUG - print_hex_dump(KERN_ERR, "rng job desc@: ", DUMP_PREFIX_ADDRESS, 16, 4, - desc, desc_bytes(desc), 1); - #endif -- return 0; - } - - static void caam_cleanup(struct hwrng *rng) -@@ -252,44 +255,67 @@ - rng_unmap_ctx(rng_ctx); - } - --static int caam_init_buf(struct caam_rng_ctx *ctx, int buf_id) -+#ifdef CONFIG_CRYPTO_DEV_FSL_CAAM_RNG_TEST -+static inline void test_len(struct hwrng *rng, size_t len, bool wait) - { -- struct buf_data *bd = &ctx->bufs[buf_id]; -- int err; -+ u8 *buf; -+ int real_len; - -- err = rng_create_job_desc(ctx, buf_id); -- if (err) -- return err; -+ buf = kzalloc(sizeof(u8) * len, GFP_KERNEL); -+ real_len = rng->read(rng, buf, len, wait); -+ if (real_len == 0 && wait) -+ pr_err("WAITING FAILED\n"); -+ pr_info("wanted %d bytes, got %d\n", len, real_len); -+ print_hex_dump(KERN_INFO, "random bytes@: ", DUMP_PREFIX_ADDRESS, -+ 16, 4, buf, real_len, 1); -+ kfree(buf); -+} - -- atomic_set(&bd->empty, BUF_EMPTY); -- submit_job(ctx, buf_id == ctx->current_buf); -- wait_for_completion(&bd->filled); -+static inline void test_mode_once(struct hwrng *rng, bool wait) -+{ -+#define TEST_CHUNK (RN_BUF_SIZE / 4) - -- return 0; -+ test_len(rng, TEST_CHUNK, wait); -+ test_len(rng, RN_BUF_SIZE * 2, wait); -+ test_len(rng, RN_BUF_SIZE * 2 - TEST_CHUNK, wait); - } - --static int caam_init_rng(struct caam_rng_ctx *ctx, struct device *jrdev) -+static inline void test_mode(struct hwrng *rng, bool wait) - { -- int err; -- -- ctx->jrdev = jrdev; -+#define TEST_PASS 1 -+ int i; - -- err = rng_create_sh_desc(ctx); -- if (err) -- return err; -+ for (i = 0; i < TEST_PASS; i++) -+ test_mode_once(rng, wait); -+} - -- ctx->current_buf = 0; -- ctx->cur_buf_idx = 0; -+static void self_test(struct hwrng *rng) -+{ -+ pr_info("testing without waiting\n"); -+ test_mode(rng, false); -+ pr_info("testing with waiting\n"); -+ test_mode(rng, true); -+} -+#endif - -- err = caam_init_buf(ctx, 0); -- if (err) -- return err; -+static void caam_init_buf(struct caam_rng_ctx *ctx, int buf_id) -+{ -+ struct buf_data *bd = &ctx->bufs[buf_id]; - -- err = caam_init_buf(ctx, 1); -- if (err) -- return err; -+ rng_create_job_desc(ctx, buf_id); -+ atomic_set(&bd->empty, BUF_EMPTY); -+ submit_job(ctx, buf_id == ctx->current_buf); -+ wait_for_completion(&bd->filled); -+} - -- return 0; -+static void caam_init_rng(struct caam_rng_ctx *ctx, struct device *jrdev) -+{ -+ ctx->jrdev = jrdev; -+ rng_create_sh_desc(ctx); -+ ctx->current_buf = 0; -+ ctx->cur_buf_idx = 0; -+ caam_init_buf(ctx, 0); -+ caam_init_buf(ctx, 1); - } - - static struct hwrng caam_rng = { -@@ -300,19 +326,15 @@ - - static void __exit caam_rng_exit(void) - { -- caam_jr_free(rng_ctx->jrdev); - hwrng_unregister(&caam_rng); -- kfree(rng_ctx); - } - - static int __init caam_rng_init(void) - { -- struct device *dev; - struct device_node *dev_node; - struct platform_device *pdev; - struct device *ctrldev; -- void *priv; -- int err; -+ struct caam_drv_private *priv; - - dev_node = of_find_compatible_node(NULL, NULL, "fsl,sec-v4.0"); - if (!dev_node) { -@@ -322,35 +344,26 @@ - } - - pdev = of_find_device_by_node(dev_node); -- if (!pdev) { -- of_node_put(dev_node); -+ if (!pdev) - return -ENODEV; -- } - - ctrldev = &pdev->dev; - priv = dev_get_drvdata(ctrldev); - of_node_put(dev_node); - -- /* -- * If priv is NULL, it's probably because the caam driver wasn't -- * properly initialized (e.g. RNG4 init failed). Thus, bail out here. -- */ -- if (!priv) -+ /* Check RNG present in hardware before registration */ -+ if (!(rd_reg64(&priv->ctrl->perfmon.cha_num) & CHA_ID_RNG_MASK)) - return -ENODEV; - -- dev = caam_jr_alloc(); -- if (IS_ERR(dev)) { -- pr_err("Job Ring Device allocation for transform failed\n"); -- return PTR_ERR(dev); -- } -- rng_ctx = kmalloc(sizeof(struct caam_rng_ctx), GFP_DMA); -- if (!rng_ctx) -- return -ENOMEM; -- err = caam_init_rng(rng_ctx, dev); -- if (err) -- return err; -+ rng_ctx = kmalloc(sizeof(struct caam_rng_ctx), GFP_KERNEL | GFP_DMA); -+ -+ caam_init_rng(rng_ctx, priv->jrdev[0]); -+ -+#ifdef CONFIG_CRYPTO_DEV_FSL_CAAM_RNG_TEST -+ self_test(&caam_rng); -+#endif - -- dev_info(dev, "registering rng-caam\n"); -+ dev_info(priv->jrdev[0], "registering rng-caam\n"); - return hwrng_register(&caam_rng); - } - -diff -Nur linux-4.1.3/drivers/crypto/caam/compat.h linux-xbian-imx6/drivers/crypto/caam/compat.h ---- linux-4.1.3/drivers/crypto/caam/compat.h 2015-07-21 19:10:33.000000000 +0200 -+++ linux-xbian-imx6/drivers/crypto/caam/compat.h 2015-07-27 23:13:04.209961631 +0200 -@@ -14,6 +14,8 @@ - #include - #include - #include -+#include -+#include - #include - #include - #include -@@ -23,12 +25,15 @@ - #include - #include - #include -+#include -+ -+#ifdef CONFIG_ARM /* needs the clock control subsystem */ -+#include -+#endif - #include - - #include --#include - #include --#include - #include - #include - #include -diff -Nur linux-4.1.3/drivers/crypto/caam/ctrl.c linux-xbian-imx6/drivers/crypto/caam/ctrl.c ---- linux-4.1.3/drivers/crypto/caam/ctrl.c 2015-07-21 19:10:33.000000000 +0200 -+++ linux-xbian-imx6/drivers/crypto/caam/ctrl.c 2015-07-27 23:13:04.209961631 +0200 -@@ -1,405 +1,260 @@ --/* * CAAM control-plane driver backend -+/* -+ * CAAM control-plane driver backend - * Controller-level driver, kernel property detection, initialization - * -- * Copyright 2008-2012 Freescale Semiconductor, Inc. -+ * Copyright (C) 2008-2013 Freescale Semiconductor, Inc. - */ - --#include --#include --#include -- - #include "compat.h" - #include "regs.h" - #include "intern.h" - #include "jr.h" - #include "desc_constr.h" - #include "error.h" -+#include "ctrl.h" -+#include "sm.h" -+#include - --/* -- * Descriptor to instantiate RNG State Handle 0 in normal mode and -- * load the JDKEK, TDKEK and TDSK registers -- */ --static void build_instantiation_desc(u32 *desc, int handle, int do_sk) --{ -- u32 *jump_cmd, op_flags; -- -- init_job_desc(desc, 0); -- -- op_flags = OP_TYPE_CLASS1_ALG | OP_ALG_ALGSEL_RNG | -- (handle << OP_ALG_AAI_SHIFT) | OP_ALG_AS_INIT; -- -- /* INIT RNG in non-test mode */ -- append_operation(desc, op_flags); -- -- if (!handle && do_sk) { -- /* -- * For SH0, Secure Keys must be generated as well -- */ -+/* Used to capture the array of job rings */ -+struct device **caam_jr_dev; - -- /* wait for done */ -- jump_cmd = append_jump(desc, JUMP_CLASS_CLASS1); -- set_jump_tgt_here(desc, jump_cmd); -+static int caam_remove(struct platform_device *pdev) -+{ -+ struct device *ctrldev; -+ struct caam_drv_private *ctrlpriv; -+ struct caam_drv_private_jr *jrpriv; -+ struct caam_full __iomem *topregs; -+ int ring, ret = 0; - -- /* -- * load 1 to clear written reg: -- * resets the done interrrupt and returns the RNG to idle. -- */ -- append_load_imm_u32(desc, 1, LDST_SRCDST_WORD_CLRW); -+ ctrldev = &pdev->dev; -+ ctrlpriv = dev_get_drvdata(ctrldev); -+ topregs = (struct caam_full __iomem *)ctrlpriv->ctrl; - -- /* Initialize State Handle */ -- append_operation(desc, OP_TYPE_CLASS1_ALG | OP_ALG_ALGSEL_RNG | -- OP_ALG_AAI_RNG4_SK); -+ /* shut down JobRs */ -+ for (ring = 0; ring < ctrlpriv->total_jobrs; ring++) { -+ ret |= caam_jr_shutdown(ctrlpriv->jrdev[ring]); -+ jrpriv = dev_get_drvdata(ctrlpriv->jrdev[ring]); -+ irq_dispose_mapping(jrpriv->irq); - } - -- append_jump(desc, JUMP_CLASS_CLASS1 | JUMP_TYPE_HALT); --} -+ /* Shut down debug views */ -+#ifdef CONFIG_DEBUG_FS -+ debugfs_remove_recursive(ctrlpriv->dfs_root); -+#endif - --/* Descriptor for deinstantiation of State Handle 0 of the RNG block. */ --static void build_deinstantiation_desc(u32 *desc, int handle) --{ -- init_job_desc(desc, 0); -+ /* Unmap controller region */ -+ iounmap(&topregs->ctrl); - -- /* Uninstantiate State Handle 0 */ -- append_operation(desc, OP_TYPE_CLASS1_ALG | OP_ALG_ALGSEL_RNG | -- (handle << OP_ALG_AAI_SHIFT) | OP_ALG_AS_INITFINAL); -+#ifdef CONFIG_ARM -+ /* shut clocks off before finalizing shutdown */ -+ clk_disable(ctrlpriv->caam_ipg); -+ clk_disable(ctrlpriv->caam_mem); -+ clk_disable(ctrlpriv->caam_aclk); -+#endif -+ -+ kfree(ctrlpriv->jrdev); -+ kfree(ctrlpriv); - -- append_jump(desc, JUMP_CLASS_CLASS1 | JUMP_TYPE_HALT); -+ return ret; - } - - /* -- * run_descriptor_deco0 - runs a descriptor on DECO0, under direct control of -- * the software (no JR/QI used). -- * @ctrldev - pointer to device -- * @status - descriptor status, after being run -- * -- * Return: - 0 if no error occurred -- * - -ENODEV if the DECO couldn't be acquired -- * - -EAGAIN if an error occurred while executing the descriptor -+ * Descriptor to instantiate RNG State Handle 0 in normal mode and -+ * load the JDKEK, TDKEK and TDSK registers - */ --static inline int run_descriptor_deco0(struct device *ctrldev, u32 *desc, -- u32 *status) -+static void build_instantiation_desc(u32 *desc) - { -- struct caam_drv_private *ctrlpriv = dev_get_drvdata(ctrldev); -- struct caam_ctrl __iomem *ctrl = ctrlpriv->ctrl; -- struct caam_deco __iomem *deco = ctrlpriv->deco; -- unsigned int timeout = 100000; -- u32 deco_dbg_reg, flags; -- int i; -- -+ u32 *jump_cmd; - -- if (ctrlpriv->virt_en == 1) { -- setbits32(&ctrl->deco_rsr, DECORSR_JR0); -- -- while (!(rd_reg32(&ctrl->deco_rsr) & DECORSR_VALID) && -- --timeout) -- cpu_relax(); -- -- timeout = 100000; -- } -- -- setbits32(&ctrl->deco_rq, DECORR_RQD0ENABLE); -- -- while (!(rd_reg32(&ctrl->deco_rq) & DECORR_DEN0) && -- --timeout) -- cpu_relax(); -+ init_job_desc(desc, 0); - -- if (!timeout) { -- dev_err(ctrldev, "failed to acquire DECO 0\n"); -- clrbits32(&ctrl->deco_rq, DECORR_RQD0ENABLE); -- return -ENODEV; -- } -+ /* INIT RNG in non-test mode */ -+ append_operation(desc, OP_TYPE_CLASS1_ALG | OP_ALG_ALGSEL_RNG | -+ OP_ALG_AS_INIT); - -- for (i = 0; i < desc_len(desc); i++) -- wr_reg32(&deco->descbuf[i], *(desc + i)); -+ /* wait for done */ -+ jump_cmd = append_jump(desc, JUMP_CLASS_CLASS1); -+ set_jump_tgt_here(desc, jump_cmd); - -- flags = DECO_JQCR_WHL; - /* -- * If the descriptor length is longer than 4 words, then the -- * FOUR bit in JRCTRL register must be set. -+ * load 1 to clear written reg: -+ * resets the done interrupt and returns the RNG to idle. - */ -- if (desc_len(desc) >= 4) -- flags |= DECO_JQCR_FOUR; -+ append_load_imm_u32(desc, 1, LDST_SRCDST_WORD_CLRW); - -- /* Instruct the DECO to execute it */ -- wr_reg32(&deco->jr_ctl_hi, flags); -- -- timeout = 10000000; -- do { -- deco_dbg_reg = rd_reg32(&deco->desc_dbg); -- /* -- * If an error occured in the descriptor, then -- * the DECO status field will be set to 0x0D -- */ -- if ((deco_dbg_reg & DESC_DBG_DECO_STAT_MASK) == -- DESC_DBG_DECO_STAT_HOST_ERR) -- break; -- cpu_relax(); -- } while ((deco_dbg_reg & DESC_DBG_DECO_STAT_VALID) && --timeout); -- -- *status = rd_reg32(&deco->op_status_hi) & -- DECO_OP_STATUS_HI_ERR_MASK; -- -- if (ctrlpriv->virt_en == 1) -- clrbits32(&ctrl->deco_rsr, DECORSR_JR0); -- -- /* Mark the DECO as free */ -- clrbits32(&ctrl->deco_rq, DECORR_RQD0ENABLE); -- -- if (!timeout) -- return -EAGAIN; -- -- return 0; - } - --/* -- * instantiate_rng - builds and executes a descriptor on DECO0, -- * which initializes the RNG block. -- * @ctrldev - pointer to device -- * @state_handle_mask - bitmask containing the instantiation status -- * for the RNG4 state handles which exist in -- * the RNG4 block: 1 if it's been instantiated -- * by an external entry, 0 otherwise. -- * @gen_sk - generate data to be loaded into the JDKEK, TDKEK and TDSK; -- * Caution: this can be done only once; if the keys need to be -- * regenerated, a POR is required -- * -- * Return: - 0 if no error occurred -- * - -ENOMEM if there isn't enough memory to allocate the descriptor -- * - -ENODEV if DECO0 couldn't be acquired -- * - -EAGAIN if an error occurred when executing the descriptor -- * f.i. there was a RNG hardware error due to not "good enough" -- * entropy being aquired. -- */ --static int instantiate_rng(struct device *ctrldev, int state_handle_mask, -- int gen_sk) -+static void generate_secure_keys_desc(u32 *desc) - { -- struct caam_drv_private *ctrlpriv = dev_get_drvdata(ctrldev); -- struct caam_ctrl __iomem *ctrl; -- u32 *desc, status, rdsta_val; -- int ret = 0, sh_idx; -- -- ctrl = (struct caam_ctrl __iomem *)ctrlpriv->ctrl; -- desc = kmalloc(CAAM_CMD_SZ * 7, GFP_KERNEL); -- if (!desc) -- return -ENOMEM; -+ /* generate secure keys (non-test) */ -+ append_operation(desc, OP_TYPE_CLASS1_ALG | OP_ALG_ALGSEL_RNG | -+ OP_ALG_RNG4_SK); -+} - -- for (sh_idx = 0; sh_idx < RNG4_MAX_HANDLES; sh_idx++) { -- /* -- * If the corresponding bit is set, this state handle -- * was initialized by somebody else, so it's left alone. -- */ -- if ((1 << sh_idx) & state_handle_mask) -- continue; -+struct instantiate_result { -+ struct completion completion; -+ int err; -+}; - -- /* Create the descriptor for instantiating RNG State Handle */ -- build_instantiation_desc(desc, sh_idx, gen_sk); -+static void rng4_init_done(struct device *dev, u32 *desc, u32 err, -+ void *context) -+{ -+ struct instantiate_result *instantiation = context; - -- /* Try to run it through DECO0 */ -- ret = run_descriptor_deco0(ctrldev, desc, &status); -+ if (err) { -+ char tmp[CAAM_ERROR_STR_MAX]; - -- /* -- * If ret is not 0, or descriptor status is not 0, then -- * something went wrong. No need to try the next state -- * handle (if available), bail out here. -- * Also, if for some reason, the State Handle didn't get -- * instantiated although the descriptor has finished -- * without any error (HW optimizations for later -- * CAAM eras), then try again. -- */ -- rdsta_val = rd_reg32(&ctrl->r4tst[0].rdsta) & RDSTA_IFMASK; -- if (status || !(rdsta_val & (1 << sh_idx))) -- ret = -EAGAIN; -- if (ret) -- break; -- dev_info(ctrldev, "Instantiated RNG4 SH%d\n", sh_idx); -- /* Clear the contents before recreating the descriptor */ -- memset(desc, 0x00, CAAM_CMD_SZ * 7); -+ dev_err(dev, "%08x: %s\n", err, caam_jr_strstatus(tmp, err)); - } - -- kfree(desc); -- -- return ret; -+ instantiation->err = err; -+ complete(&instantiation->completion); - } - --/* -- * deinstantiate_rng - builds and executes a descriptor on DECO0, -- * which deinitializes the RNG block. -- * @ctrldev - pointer to device -- * @state_handle_mask - bitmask containing the instantiation status -- * for the RNG4 state handles which exist in -- * the RNG4 block: 1 if it's been instantiated -- * -- * Return: - 0 if no error occurred -- * - -ENOMEM if there isn't enough memory to allocate the descriptor -- * - -ENODEV if DECO0 couldn't be acquired -- * - -EAGAIN if an error occurred when executing the descriptor -- */ --static int deinstantiate_rng(struct device *ctrldev, int state_handle_mask) -+static int instantiate_rng(struct device *jrdev, u32 keys_generated) - { -- u32 *desc, status; -- int sh_idx, ret = 0; -+ struct instantiate_result instantiation; - -- desc = kmalloc(CAAM_CMD_SZ * 3, GFP_KERNEL); -- if (!desc) -+ dma_addr_t desc_dma; -+ u32 *desc; -+ int ret; -+ -+ desc = kmalloc(CAAM_CMD_SZ * 6, GFP_KERNEL | GFP_DMA); -+ if (!desc) { -+ dev_err(jrdev, "cannot allocate RNG init descriptor memory\n"); - return -ENOMEM; -- -- for (sh_idx = 0; sh_idx < RNG4_MAX_HANDLES; sh_idx++) { -- /* -- * If the corresponding bit is set, then it means the state -- * handle was initialized by us, and thus it needs to be -- * deintialized as well -- */ -- if ((1 << sh_idx) & state_handle_mask) { -- /* -- * Create the descriptor for deinstantating this state -- * handle -- */ -- build_deinstantiation_desc(desc, sh_idx); -- -- /* Try to run it through DECO0 */ -- ret = run_descriptor_deco0(ctrldev, desc, &status); -- -- if (ret || status) { -- dev_err(ctrldev, -- "Failed to deinstantiate RNG4 SH%d\n", -- sh_idx); -- break; -- } -- dev_info(ctrldev, "Deinstantiated RNG4 SH%d\n", sh_idx); -- } - } - -- kfree(desc); -- -- return ret; --} -+ build_instantiation_desc(desc); - --static int caam_remove(struct platform_device *pdev) --{ -- struct device *ctrldev; -- struct caam_drv_private *ctrlpriv; -- struct caam_ctrl __iomem *ctrl; -- int ring, ret = 0; -- -- ctrldev = &pdev->dev; -- ctrlpriv = dev_get_drvdata(ctrldev); -- ctrl = (struct caam_ctrl __iomem *)ctrlpriv->ctrl; -- -- /* Remove platform devices for JobRs */ -- for (ring = 0; ring < ctrlpriv->total_jobrs; ring++) { -- if (ctrlpriv->jrpdev[ring]) -- of_device_unregister(ctrlpriv->jrpdev[ring]); -+ /* If keys have not been generated, add op code to generate key. */ -+ if (!keys_generated) -+ generate_secure_keys_desc(desc); -+ -+ desc_dma = dma_map_single(jrdev, desc, desc_bytes(desc), DMA_TO_DEVICE); -+ dma_sync_single_for_device(jrdev, desc_dma, desc_bytes(desc), -+ DMA_TO_DEVICE); -+ init_completion(&instantiation.completion); -+ ret = caam_jr_enqueue(jrdev, desc, rng4_init_done, &instantiation); -+ if (!ret) { -+ wait_for_completion_interruptible(&instantiation.completion); -+ ret = instantiation.err; -+ if (ret) -+ dev_err(jrdev, "unable to instantiate RNG\n"); - } - -- /* De-initialize RNG state handles initialized by this driver. */ -- if (ctrlpriv->rng4_sh_init) -- deinstantiate_rng(ctrldev, ctrlpriv->rng4_sh_init); -- -- /* Shut down debug views */ --#ifdef CONFIG_DEBUG_FS -- debugfs_remove_recursive(ctrlpriv->dfs_root); --#endif -+ dma_unmap_single(jrdev, desc_dma, desc_bytes(desc), DMA_TO_DEVICE); - -- /* Unmap controller region */ -- iounmap(&ctrl); -+ kfree(desc); - - return ret; - } - - /* -- * kick_trng - sets the various parameters for enabling the initialization -- * of the RNG4 block in CAAM -- * @pdev - pointer to the platform device -- * @ent_delay - Defines the length (in system clocks) of each entropy sample. -+ * By default, the TRNG runs for 200 clocks per sample; -+ * 1600 clocks per sample generates better entropy. - */ --static void kick_trng(struct platform_device *pdev, int ent_delay) -+static void kick_trng(struct platform_device *pdev) - { - struct device *ctrldev = &pdev->dev; - struct caam_drv_private *ctrlpriv = dev_get_drvdata(ctrldev); -- struct caam_ctrl __iomem *ctrl; -+ struct caam_full __iomem *topregs; - struct rng4tst __iomem *r4tst; - u32 val; - -- ctrl = (struct caam_ctrl __iomem *)ctrlpriv->ctrl; -- r4tst = &ctrl->r4tst[0]; -+ topregs = (struct caam_full __iomem *)ctrlpriv->ctrl; -+ r4tst = &topregs->ctrl.r4tst[0]; - -+ val = rd_reg32(&r4tst->rtmctl); - /* put RNG4 into program mode */ - setbits32(&r4tst->rtmctl, RTMCTL_PRGM); -- -- /* -- * Performance-wise, it does not make sense to -- * set the delay to a value that is lower -- * than the last one that worked (i.e. the state handles -- * were instantiated properly. Thus, instead of wasting -- * time trying to set the values controlling the sample -- * frequency, the function simply returns. -- */ -- val = (rd_reg32(&r4tst->rtsdctl) & RTSDCTL_ENT_DLY_MASK) -- >> RTSDCTL_ENT_DLY_SHIFT; -- if (ent_delay <= val) { -- /* put RNG4 into run mode */ -- clrbits32(&r4tst->rtmctl, RTMCTL_PRGM); -- return; -- } -- -+ /* Set clocks per sample to the default, and divider to zero */ - val = rd_reg32(&r4tst->rtsdctl); - val = (val & ~RTSDCTL_ENT_DLY_MASK) | -- (ent_delay << RTSDCTL_ENT_DLY_SHIFT); -+ (RNG4_ENT_CLOCKS_SAMPLE << RTSDCTL_ENT_DLY_SHIFT); - wr_reg32(&r4tst->rtsdctl, val); -- /* min. freq. count, equal to 1/4 of the entropy sample length */ -- wr_reg32(&r4tst->rtfrqmin, ent_delay >> 2); -- /* disable maximum frequency count */ -- wr_reg32(&r4tst->rtfrqmax, RTFRQMAX_DISABLE); -- /* read the control register */ -- val = rd_reg32(&r4tst->rtmctl); -- /* -- * select raw sampling in both entropy shifter -- * and statistical checker -- */ -- setbits32(&val, RTMCTL_SAMP_MODE_RAW_ES_SC); -+ /* min. freq. count */ -+ wr_reg32(&r4tst->rtfrqmin, RNG4_ENT_CLOCKS_SAMPLE / 4); -+ /* max. freq. count */ -+ wr_reg32(&r4tst->rtfrqmax, RNG4_ENT_CLOCKS_SAMPLE * 8); - /* put RNG4 into run mode */ -- clrbits32(&val, RTMCTL_PRGM); -- /* write back the control register */ -- wr_reg32(&r4tst->rtmctl, val); -+ clrbits32(&r4tst->rtmctl, RTMCTL_PRGM); - } - - /** - * caam_get_era() - Return the ERA of the SEC on SoC, based -- * on "sec-era" propery in the DTS. This property is updated by u-boot. -+ * on the SEC_VID register. -+ * Returns the ERA number (1..4) or -ENOTSUPP if the ERA is unknown. -+ * @caam_id - the value of the SEC_VID register - **/ --int caam_get_era(void) -+int caam_get_era(u64 caam_id) - { -- struct device_node *caam_node; -- for_each_compatible_node(caam_node, NULL, "fsl,sec-v4.0") { -- const uint32_t *prop = (uint32_t *)of_get_property(caam_node, -- "fsl,sec-era", -- NULL); -- return prop ? *prop : -ENOTSUPP; -- } -+ struct sec_vid *sec_vid = (struct sec_vid *)&caam_id; -+ static const struct { -+ u16 ip_id; -+ u8 maj_rev; -+ u8 era; -+ } caam_eras[] = { -+ {0x0A10, 1, 1}, -+ {0x0A10, 2, 2}, -+ {0x0A12, 1, 3}, -+ {0x0A14, 1, 3}, -+ {0x0A14, 2, 4}, -+ {0x0A16, 1, 4}, -+ {0x0A11, 1, 4}, -+ {0x0A10, 3, 4}, -+ {0x0A18, 1, 4}, -+ {0x0A11, 2, 5}, -+ {0x0A12, 2, 5}, -+ {0x0A13, 1, 5}, -+ {0x0A1C, 1, 5}, -+ {0x0A12, 4, 6}, -+ {0x0A13, 2, 6}, -+ {0x0A16, 2, 6}, -+ {0x0A18, 2, 6}, -+ {0x0A1A, 1, 6}, -+ {0x0A1C, 2, 6}, -+ {0x0A17, 1, 6} -+ }; -+ int i; -+ -+ for (i = 0; i < ARRAY_SIZE(caam_eras); i++) -+ if (caam_eras[i].ip_id == sec_vid->ip_id && -+ caam_eras[i].maj_rev == sec_vid->maj_rev) -+ return caam_eras[i].era; - - return -ENOTSUPP; - } - EXPORT_SYMBOL(caam_get_era); - -+/* -+ * Return a job ring device. This is available so outside -+ * entities can gain direct access to the job ring. For now, -+ * this function returns the first job ring (at index 0). -+ */ -+struct device *caam_get_jrdev(void) -+{ -+ return caam_jr_dev[0]; -+} -+EXPORT_SYMBOL(caam_get_jrdev); -+ -+ - /* Probe routine for CAAM top (controller) level */ - static int caam_probe(struct platform_device *pdev) - { -- int ret, ring, rspec, gen_sk, ent_delay = RTSDCTL_ENT_DLY_MIN; -+ int ret, ring, rspec; - u64 caam_id; - struct device *dev; - struct device_node *nprop, *np; - struct caam_ctrl __iomem *ctrl; -+ struct caam_full __iomem *topregs; -+ struct snvs_full __iomem *snvsregs; - struct caam_drv_private *ctrlpriv; - #ifdef CONFIG_DEBUG_FS - struct caam_perfmon *perfmon; - #endif -- u32 scfgr, comp_params; -- u32 cha_vid_ls; -- int pg_size; -- int BLOCK_OFFSET = 0; - -- ctrlpriv = devm_kzalloc(&pdev->dev, sizeof(struct caam_drv_private), -- GFP_KERNEL); -+ ctrlpriv = kzalloc(sizeof(struct caam_drv_private), GFP_KERNEL); - if (!ctrlpriv) - return -ENOMEM; - -@@ -415,71 +270,128 @@ - dev_err(dev, "caam: of_iomap() failed\n"); - return -ENOMEM; - } -- /* Finding the page size for using the CTPR_MS register */ -- comp_params = rd_reg32(&ctrl->perfmon.comp_parms_ms); -- pg_size = (comp_params & CTPR_MS_PG_SZ_MASK) >> CTPR_MS_PG_SZ_SHIFT; -- -- /* Allocating the BLOCK_OFFSET based on the supported page size on -- * the platform -- */ -- if (pg_size == 0) -- BLOCK_OFFSET = PG_SIZE_4K; -- else -- BLOCK_OFFSET = PG_SIZE_64K; -- - ctrlpriv->ctrl = (struct caam_ctrl __force *)ctrl; -- ctrlpriv->assure = (struct caam_assurance __force *) -- ((uint8_t *)ctrl + -- BLOCK_OFFSET * ASSURE_BLOCK_NUMBER -- ); -- ctrlpriv->deco = (struct caam_deco __force *) -- ((uint8_t *)ctrl + -- BLOCK_OFFSET * DECO_BLOCK_NUMBER -- ); -+ -+ /* topregs used to derive pointers to CAAM sub-blocks only */ -+ topregs = (struct caam_full __iomem *)ctrl; - - /* Get the IRQ of the controller (for security violations only) */ -- ctrlpriv->secvio_irq = irq_of_parse_and_map(nprop, 0); -+ ctrlpriv->secvio_irq = of_irq_to_resource(nprop, 0, NULL); -+ -+ /* Get SNVS register Page */ -+ np = of_find_compatible_node(NULL, NULL, "fsl,imx6q-caam-snvs"); -+ -+ if (!np) -+ return -ENODEV; -+ -+ snvsregs = of_iomap(np, 0); -+ ctrlpriv->snvs = snvsregs; -+ /* Get CAAM-SM node and of_iomap() and save */ -+ np = of_find_compatible_node(NULL, NULL, "fsl,imx6q-caam-sm"); -+ -+ if (!np) -+ return -ENODEV; -+ -+ ctrlpriv->sm_base = of_iomap(np, 0); -+ ctrlpriv->sm_size = 0x3fff; -+ -+/* -+ * ARM targets tend to have clock control subsystems that can -+ * enable/disable clocking to our device. Turn clocking on to proceed -+ */ -+#ifdef CONFIG_ARM -+ ctrlpriv->caam_ipg = devm_clk_get(&ctrlpriv->pdev->dev, "caam_ipg"); -+ if (IS_ERR(ctrlpriv->caam_ipg)) { -+ ret = PTR_ERR(ctrlpriv->caam_ipg); -+ dev_err(&ctrlpriv->pdev->dev, -+ "can't identify CAAM ipg clk: %d\n", ret); -+ return -ENODEV; -+ } -+ ctrlpriv->caam_mem = devm_clk_get(&ctrlpriv->pdev->dev, "caam_mem"); -+ if (IS_ERR(ctrlpriv->caam_mem)) { -+ ret = PTR_ERR(ctrlpriv->caam_mem); -+ dev_err(&ctrlpriv->pdev->dev, -+ "can't identify CAAM secure mem clk: %d\n", ret); -+ return -ENODEV; -+ } -+ ctrlpriv->caam_aclk = devm_clk_get(&ctrlpriv->pdev->dev, "caam_aclk"); -+ if (IS_ERR(ctrlpriv->caam_aclk)) { -+ ret = PTR_ERR(ctrlpriv->caam_aclk); -+ dev_err(&ctrlpriv->pdev->dev, -+ "can't identify CAAM aclk clk: %d\n", ret); -+ return -ENODEV; -+ } -+ -+ ret = clk_prepare(ctrlpriv->caam_ipg); -+ if (ret < 0) { -+ dev_err(&pdev->dev, "can't prepare CAAM ipg clock: %d\n", ret); -+ return -ENODEV; -+ } -+ ret = clk_prepare(ctrlpriv->caam_mem); -+ if (ret < 0) { -+ dev_err(&pdev->dev, "can't prepare CAAM secure mem clock: %d\n", ret); -+ return -ENODEV; -+ } -+ ret = clk_prepare(ctrlpriv->caam_aclk); -+ if (ret < 0) { -+ dev_err(&pdev->dev, "can't prepare CAAM aclk clock: %d\n", ret); -+ return -ENODEV; -+ } -+ -+ ret = clk_enable(ctrlpriv->caam_ipg); -+ if (ret < 0) { -+ dev_err(&pdev->dev, "can't enable CAAM ipg clock: %d\n", ret); -+ return -ENODEV; -+ } -+ ret = clk_enable(ctrlpriv->caam_mem); -+ if (ret < 0) { -+ dev_err(&pdev->dev, "can't enable CAAM secure mem clock: %d\n", ret); -+ return -ENODEV; -+ } -+ ret = clk_enable(ctrlpriv->caam_aclk); -+ if (ret < 0) { -+ dev_err(&pdev->dev, "can't enable CAAM aclk clock: %d\n", ret); -+ return -ENODEV; -+ } -+ -+ pr_debug("%s caam_ipg clock:%d\n", __func__, -+ (int)clk_get_rate(ctrlpriv->caam_ipg)); -+ pr_debug("%s caam_mem clock:%d\n", __func__, -+ (int)clk_get_rate(ctrlpriv->caam_mem)); -+ pr_debug("%s caam_aclk clock:%d\n", __func__, -+ (int)clk_get_rate(ctrlpriv->caam_aclk)); -+#endif - - /* - * Enable DECO watchdogs and, if this is a PHYS_ADDR_T_64BIT kernel, - * long pointers in master configuration register - */ -- setbits32(&ctrl->mcr, MCFGR_WDENABLE | -+ setbits32(&topregs->ctrl.mcr, MCFGR_WDENABLE | - (sizeof(dma_addr_t) == sizeof(u64) ? MCFGR_LONG_PTR : 0)); - -+#ifdef CONFIG_ARCH_MX6 - /* -- * Read the Compile Time paramters and SCFGR to determine -- * if Virtualization is enabled for this platform -+ * ERRATA: mx6 devices have an issue wherein AXI bus transactions -+ * may not occur in the correct order. This isn't a problem running -+ * single descriptors, but can be if running multiple concurrent -+ * descriptors. Reworking the driver to throttle to single requests -+ * is impractical, thus the workaround is to limit the AXI pipeline -+ * to a depth of 1 (from it's default of 4) to preclude this situation -+ * from occurring. - */ -- scfgr = rd_reg32(&ctrl->scfgr); -- -- ctrlpriv->virt_en = 0; -- if (comp_params & CTPR_MS_VIRT_EN_INCL) { -- /* VIRT_EN_INCL = 1 & VIRT_EN_POR = 1 or -- * VIRT_EN_INCL = 1 & VIRT_EN_POR = 0 & SCFGR_VIRT_EN = 1 -- */ -- if ((comp_params & CTPR_MS_VIRT_EN_POR) || -- (!(comp_params & CTPR_MS_VIRT_EN_POR) && -- (scfgr & SCFGR_VIRT_EN))) -- ctrlpriv->virt_en = 1; -- } else { -- /* VIRT_EN_INCL = 0 && VIRT_EN_POR_VALUE = 1 */ -- if (comp_params & CTPR_MS_VIRT_EN_POR) -- ctrlpriv->virt_en = 1; -- } -- -- if (ctrlpriv->virt_en == 1) -- setbits32(&ctrl->jrstart, JRSTART_JR0_START | -- JRSTART_JR1_START | JRSTART_JR2_START | -- JRSTART_JR3_START); -+ wr_reg32(&topregs->ctrl.mcr, -+ (rd_reg32(&topregs->ctrl.mcr) & ~(MCFGR_AXIPIPE_MASK)) | -+ ((1 << MCFGR_AXIPIPE_SHIFT) & MCFGR_AXIPIPE_MASK)); -+#endif - -+ /* Set DMA masks according to platform ranging */ - if (sizeof(dma_addr_t) == sizeof(u64)) -- if (of_device_is_compatible(nprop, "fsl,sec-v5.0")) -- dma_set_mask_and_coherent(dev, DMA_BIT_MASK(40)); -+ if (of_device_is_compatible(nprop, "fsl,sec-v4.0")) -+ dma_set_mask(dev, DMA_BIT_MASK(40)); - else -- dma_set_mask_and_coherent(dev, DMA_BIT_MASK(36)); -+ dma_set_mask(dev, DMA_BIT_MASK(36)); - else -- dma_set_mask_and_coherent(dev, DMA_BIT_MASK(32)); -+ dma_set_mask(dev, DMA_BIT_MASK(32)); - - /* - * Detect and enable JobRs -@@ -487,51 +399,65 @@ - * for all, then go probe each one. - */ - rspec = 0; -- for_each_available_child_of_node(nprop, np) -- if (of_device_is_compatible(np, "fsl,sec-v4.0-job-ring") || -- of_device_is_compatible(np, "fsl,sec4.0-job-ring")) -+ for_each_compatible_node(np, NULL, "fsl,sec-v4.0-job-ring") -+ rspec++; -+ if (!rspec) { -+ /* for backward compatible with device trees */ -+ for_each_compatible_node(np, NULL, "fsl,sec4.0-job-ring") - rspec++; -+ } - -- ctrlpriv->jrpdev = devm_kzalloc(&pdev->dev, -- sizeof(struct platform_device *) * rspec, -- GFP_KERNEL); -- if (ctrlpriv->jrpdev == NULL) { -- iounmap(&ctrl); -+ ctrlpriv->jrdev = kzalloc(sizeof(struct device *) * rspec, GFP_KERNEL); -+ if (ctrlpriv->jrdev == NULL) { -+ iounmap(&topregs->ctrl); - return -ENOMEM; - } - - ring = 0; - ctrlpriv->total_jobrs = 0; -- for_each_available_child_of_node(nprop, np) -- if (of_device_is_compatible(np, "fsl,sec-v4.0-job-ring") || -- of_device_is_compatible(np, "fsl,sec4.0-job-ring")) { -- ctrlpriv->jrpdev[ring] = -- of_platform_device_create(np, NULL, dev); -- if (!ctrlpriv->jrpdev[ring]) { -- pr_warn("JR%d Platform device creation error\n", -- ring); -- continue; -+ for_each_compatible_node(np, NULL, "fsl,sec-v4.0-job-ring") { -+ ret = caam_jr_probe(pdev, np, ring); -+ if (ret < 0) { -+ /* -+ * Job ring not found, error out. At some -+ * point, we should enhance job ring handling -+ * to allow for non-consecutive job rings to -+ * be found. -+ */ -+ pr_err("fsl,sec-v4.0-job-ring not found "); -+ pr_err("(ring %d)\n", ring); -+ return ret; -+ } -+ ctrlpriv->total_jobrs++; -+ ring++; -+ } -+ -+ if (!ring) { -+ for_each_compatible_node(np, NULL, "fsl,sec4.0-job-ring") { -+ ret = caam_jr_probe(pdev, np, ring); -+ if (ret < 0) { -+ /* -+ * Job ring not found, error out. At some -+ * point, we should enhance job ring handling -+ * to allow for non-consecutive job rings to -+ * be found. -+ */ -+ pr_err("fsl,sec4.0-job-ring not found "); -+ pr_err("(ring %d)\n", ring); -+ return ret; - } -- ctrlpriv->jr[ring] = (struct caam_job_ring __force *) -- ((uint8_t *)ctrl + -- (ring + JR_BLOCK_NUMBER) * -- BLOCK_OFFSET -- ); - ctrlpriv->total_jobrs++; - ring++; -+ } - } - - /* Check to see if QI present. If so, enable */ -- ctrlpriv->qi_present = -- !!(rd_reg32(&ctrl->perfmon.comp_parms_ms) & -- CTPR_MS_QI_MASK); -+ ctrlpriv->qi_present = !!(rd_reg64(&topregs->ctrl.perfmon.comp_parms) & -+ CTPR_QI_MASK); - if (ctrlpriv->qi_present) { -- ctrlpriv->qi = (struct caam_queue_if __force *) -- ((uint8_t *)ctrl + -- BLOCK_OFFSET * QI_BLOCK_NUMBER -- ); -+ ctrlpriv->qi = (struct caam_queue_if __force *)&topregs->qi; - /* This is all that's required to physically enable QI */ -- wr_reg32(&ctrlpriv->qi->qi_control_lo, QICTL_DQEN); -+ wr_reg32(&topregs->qi.qi_control_lo, QICTL_DQEN); - } - - /* If no QI and no rings specified, quit and go home */ -@@ -541,81 +467,53 @@ - return -ENOMEM; - } - -- cha_vid_ls = rd_reg32(&ctrl->perfmon.cha_id_ls); -- - /* -- * If SEC has RNG version >= 4 and RNG state handle has not been -- * already instantiated, do RNG instantiation -+ * RNG4 based SECs (v5+ | >= i.MX6) need special initialization prior -+ * to executing any descriptors. If there's a problem with init, -+ * remove other subsystems and return; internal padding functions -+ * cannot run without an RNG. This procedure assumes a single RNG4 -+ * instance. - */ -- if ((cha_vid_ls & CHA_ID_LS_RNG_MASK) >> CHA_ID_LS_RNG_SHIFT >= 4) { -- ctrlpriv->rng4_sh_init = -- rd_reg32(&ctrl->r4tst[0].rdsta); -+ if ((rd_reg64(&topregs->ctrl.perfmon.cha_id) & CHA_ID_RNG_MASK) -+ == CHA_ID_RNG_4) { -+ struct rng4tst __iomem *r4tst; -+ u32 rdsta, rng_if, rng_skvn; -+ - /* -- * If the secure keys (TDKEK, JDKEK, TDSK), were already -- * generated, signal this to the function that is instantiating -- * the state handles. An error would occur if RNG4 attempts -- * to regenerate these keys before the next POR. -+ * Check to see if the RNG has already been instantiated. -+ * If either the state 0 or 1 instantiated flags are set, -+ * then don't continue on and try to instantiate the RNG -+ * again. - */ -- gen_sk = ctrlpriv->rng4_sh_init & RDSTA_SKVN ? 0 : 1; -- ctrlpriv->rng4_sh_init &= RDSTA_IFMASK; -- do { -- int inst_handles = -- rd_reg32(&ctrl->r4tst[0].rdsta) & -- RDSTA_IFMASK; -- /* -- * If either SH were instantiated by somebody else -- * (e.g. u-boot) then it is assumed that the entropy -- * parameters are properly set and thus the function -- * setting these (kick_trng(...)) is skipped. -- * Also, if a handle was instantiated, do not change -- * the TRNG parameters. -- */ -- if (!(ctrlpriv->rng4_sh_init || inst_handles)) { -- dev_info(dev, -- "Entropy delay = %u\n", -- ent_delay); -- kick_trng(pdev, ent_delay); -- ent_delay += 400; -+ r4tst = &topregs->ctrl.r4tst[0]; -+ rdsta = rd_reg32(&r4tst->rdsta); /* Read RDSTA register */ -+ -+ /* Check IF bit for non-deterministic instantiation */ -+ rng_if = rdsta & RDSTA_IF; -+ -+ /* Check SKVN bit for non-deterministic key generation */ -+ rng_skvn = rdsta & RDSTA_SKVN; -+ if (!rng_if) { -+ kick_trng(pdev); -+ ret = instantiate_rng(ctrlpriv->jrdev[0], rng_skvn); -+ if (ret) { -+ caam_remove(pdev); -+ return -ENODEV; - } -- /* -- * if instantiate_rng(...) fails, the loop will rerun -- * and the kick_trng(...) function will modfiy the -- * upper and lower limits of the entropy sampling -- * interval, leading to a sucessful initialization of -- * the RNG. -- */ -- ret = instantiate_rng(dev, inst_handles, -- gen_sk); -- if (ret == -EAGAIN) -- /* -- * if here, the loop will rerun, -- * so don't hog the CPU -- */ -- cpu_relax(); -- } while ((ret == -EAGAIN) && (ent_delay < RTSDCTL_ENT_DLY_MAX)); -- if (ret) { -- dev_err(dev, "failed to instantiate RNG"); -- caam_remove(pdev); -- return ret; -+ ctrlpriv->rng_inst++; - } -- /* -- * Set handles init'ed by this module as the complement of the -- * already initialized ones -- */ -- ctrlpriv->rng4_sh_init = ~ctrlpriv->rng4_sh_init & RDSTA_IFMASK; -- -- /* Enable RDB bit so that RNG works faster */ -- setbits32(&ctrl->scfgr, SCFGR_RDBENABLE); - } - - /* NOTE: RTIC detection ought to go here, around Si time */ - -- caam_id = (u64)rd_reg32(&ctrl->perfmon.caam_id_ms) << 32 | -- (u64)rd_reg32(&ctrl->perfmon.caam_id_ls); -+ /* Initialize queue allocator lock */ -+ spin_lock_init(&ctrlpriv->jr_alloc_lock); -+ -+ caam_id = rd_reg64(&topregs->ctrl.perfmon.caam_id); - - /* Report "alive" for developer to see */ - dev_info(dev, "device ID = 0x%016llx (Era %d)\n", caam_id, -- caam_get_era()); -+ caam_get_era(caam_id)); - dev_info(dev, "job rings = %d, qi = %d\n", - ctrlpriv->total_jobrs, ctrlpriv->qi_present); - -@@ -627,7 +525,7 @@ - */ - perfmon = (struct caam_perfmon __force *)&ctrl->perfmon; - -- ctrlpriv->dfs_root = debugfs_create_dir(dev_name(dev), NULL); -+ ctrlpriv->dfs_root = debugfs_create_dir("caam", NULL); - ctrlpriv->ctl = debugfs_create_dir("ctl", ctrlpriv->dfs_root); - - /* Controller-level - performance monitor counters */ -@@ -716,6 +614,7 @@ - static struct platform_driver caam_driver = { - .driver = { - .name = "caam", -+ .owner = THIS_MODULE, - .of_match_table = caam_match, - }, - .probe = caam_probe, -diff -Nur linux-4.1.3/drivers/crypto/caam/ctrl.h linux-xbian-imx6/drivers/crypto/caam/ctrl.h ---- linux-4.1.3/drivers/crypto/caam/ctrl.h 2015-07-21 19:10:33.000000000 +0200 -+++ linux-xbian-imx6/drivers/crypto/caam/ctrl.h 2015-07-27 23:13:04.209961631 +0200 -@@ -8,6 +8,6 @@ - #define CTRL_H - - /* Prototypes for backend-level services exposed to APIs */ --int caam_get_era(void); -+int caam_get_era(u64 caam_id); - - #endif /* CTRL_H */ -diff -Nur linux-4.1.3/drivers/crypto/caam/desc_constr.h linux-xbian-imx6/drivers/crypto/caam/desc_constr.h ---- linux-4.1.3/drivers/crypto/caam/desc_constr.h 2015-07-21 19:10:33.000000000 +0200 -+++ linux-xbian-imx6/drivers/crypto/caam/desc_constr.h 2015-07-27 23:13:04.209961631 +0200 -@@ -10,7 +10,6 @@ - #define CAAM_CMD_SZ sizeof(u32) - #define CAAM_PTR_SZ sizeof(dma_addr_t) - #define CAAM_DESC_BYTES_MAX (CAAM_CMD_SZ * MAX_CAAM_DESCSIZE) --#define DESC_JOB_IO_LEN (CAAM_CMD_SZ * 5 + CAAM_PTR_SZ * 3) - - #ifdef DEBUG - #define PRINT_POS do { printk(KERN_DEBUG "%02d: %s\n", desc_len(desc),\ -@@ -111,26 +110,6 @@ - (*desc)++; - } - --#define append_u32 append_cmd -- --static inline void append_u64(u32 *desc, u64 data) --{ -- u32 *offset = desc_end(desc); -- -- *offset = upper_32_bits(data); -- *(++offset) = lower_32_bits(data); -- -- (*desc) += 2; --} -- --/* Write command without affecting header, and return pointer to next word */ --static inline u32 *write_cmd(u32 *desc, u32 command) --{ -- *desc = command; -- -- return desc + 1; --} -- - static inline void append_cmd_ptr(u32 *desc, dma_addr_t ptr, int len, - u32 command) - { -@@ -143,8 +122,7 @@ - unsigned int len, u32 command) - { - append_cmd(desc, command); -- if (!(command & (SQIN_RTO | SQIN_PRE))) -- append_ptr(desc, ptr); -+ append_ptr(desc, ptr); - append_cmd(desc, len); - } - -@@ -155,29 +133,21 @@ - append_data(desc, data, len); - } - --#define APPEND_CMD_RET(cmd, op) \ --static inline u32 *append_##cmd(u32 *desc, u32 options) \ --{ \ -- u32 *cmd = desc_end(desc); \ -- PRINT_POS; \ -- append_cmd(desc, CMD_##op | options); \ -- return cmd; \ -+static inline u32 *append_jump(u32 *desc, u32 options) -+{ -+ u32 *cmd = desc_end(desc); -+ -+ PRINT_POS; -+ append_cmd(desc, CMD_JUMP | options); -+ -+ return cmd; - } --APPEND_CMD_RET(jump, JUMP) --APPEND_CMD_RET(move, MOVE) - - static inline void set_jump_tgt_here(u32 *desc, u32 *jump_cmd) - { - *jump_cmd = *jump_cmd | (desc_len(desc) - (jump_cmd - desc)); - } - --static inline void set_move_tgt_here(u32 *desc, u32 *move_cmd) --{ -- *move_cmd &= ~MOVE_OFFSET_MASK; -- *move_cmd = *move_cmd | ((desc_len(desc) << (MOVE_OFFSET_SHIFT + 2)) & -- MOVE_OFFSET_MASK); --} -- - #define APPEND_CMD(cmd, op) \ - static inline void append_##cmd(u32 *desc, u32 options) \ - { \ -@@ -185,6 +155,7 @@ - append_cmd(desc, CMD_##op | options); \ - } - APPEND_CMD(operation, OPERATION) -+APPEND_CMD(move, MOVE) - - #define APPEND_CMD_LEN(cmd, op) \ - static inline void append_##cmd(u32 *desc, unsigned int len, u32 options) \ -@@ -192,8 +163,6 @@ - PRINT_POS; \ - append_cmd(desc, CMD_##op | len | options); \ - } -- --APPEND_CMD_LEN(seq_load, SEQ_LOAD) - APPEND_CMD_LEN(seq_store, SEQ_STORE) - APPEND_CMD_LEN(seq_fifo_load, SEQ_FIFO_LOAD) - APPEND_CMD_LEN(seq_fifo_store, SEQ_FIFO_STORE) -@@ -207,36 +176,17 @@ - } - APPEND_CMD_PTR(key, KEY) - APPEND_CMD_PTR(load, LOAD) -+APPEND_CMD_PTR(store, STORE) - APPEND_CMD_PTR(fifo_load, FIFO_LOAD) - APPEND_CMD_PTR(fifo_store, FIFO_STORE) - --static inline void append_store(u32 *desc, dma_addr_t ptr, unsigned int len, -- u32 options) --{ -- u32 cmd_src; -- -- cmd_src = options & LDST_SRCDST_MASK; -- -- append_cmd(desc, CMD_STORE | options | len); -- -- /* The following options do not require pointer */ -- if (!(cmd_src == LDST_SRCDST_WORD_DESCBUF_SHARED || -- cmd_src == LDST_SRCDST_WORD_DESCBUF_JOB || -- cmd_src == LDST_SRCDST_WORD_DESCBUF_JOB_WE || -- cmd_src == LDST_SRCDST_WORD_DESCBUF_SHARED_WE)) -- append_ptr(desc, ptr); --} -- - #define APPEND_SEQ_PTR_INTLEN(cmd, op) \ - static inline void append_seq_##cmd##_ptr_intlen(u32 *desc, dma_addr_t ptr, \ - unsigned int len, \ - u32 options) \ - { \ - PRINT_POS; \ -- if (options & (SQIN_RTO | SQIN_PRE)) \ -- append_cmd(desc, CMD_SEQ_##op##_PTR | len | options); \ -- else \ -- append_cmd_ptr(desc, ptr, len, CMD_SEQ_##op##_PTR | options); \ -+ append_cmd_ptr(desc, ptr, len, CMD_SEQ_##op##_PTR | options); \ - } - APPEND_SEQ_PTR_INTLEN(in, IN) - APPEND_SEQ_PTR_INTLEN(out, OUT) -@@ -309,7 +259,7 @@ - */ - #define APPEND_MATH(op, desc, dest, src_0, src_1, len) \ - append_cmd(desc, CMD_MATH | MATH_FUN_##op | MATH_DEST_##dest | \ -- MATH_SRC0_##src_0 | MATH_SRC1_##src_1 | (u32)len); -+ MATH_SRC0_##src_0 | MATH_SRC1_##src_1 | (u32) (len & MATH_LEN_MASK)); - - #define append_math_add(desc, dest, src0, src1, len) \ - APPEND_MATH(ADD, desc, dest, src0, src1, len) -@@ -329,15 +279,13 @@ - APPEND_MATH(LSHIFT, desc, dest, src0, src1, len) - #define append_math_rshift(desc, dest, src0, src1, len) \ - APPEND_MATH(RSHIFT, desc, dest, src0, src1, len) --#define append_math_ldshift(desc, dest, src0, src1, len) \ -- APPEND_MATH(SHLD, desc, dest, src0, src1, len) - - /* Exactly one source is IMM. Data is passed in as u32 value */ - #define APPEND_MATH_IMM_u32(op, desc, dest, src_0, src_1, data) \ - do { \ - APPEND_MATH(op, desc, dest, src_0, src_1, CAAM_CMD_SZ); \ - append_cmd(desc, data); \ --} while (0) -+} while (0); - - #define append_math_add_imm_u32(desc, dest, src0, src1, data) \ - APPEND_MATH_IMM_u32(ADD, desc, dest, src0, src1, data) -@@ -357,34 +305,3 @@ - APPEND_MATH_IMM_u32(LSHIFT, desc, dest, src0, src1, data) - #define append_math_rshift_imm_u32(desc, dest, src0, src1, data) \ - APPEND_MATH_IMM_u32(RSHIFT, desc, dest, src0, src1, data) -- --/* Exactly one source is IMM. Data is passed in as u64 value */ --#define APPEND_MATH_IMM_u64(op, desc, dest, src_0, src_1, data) \ --do { \ -- u32 upper = (data >> 16) >> 16; \ -- APPEND_MATH(op, desc, dest, src_0, src_1, CAAM_CMD_SZ * 2 | \ -- (upper ? 0 : MATH_IFB)); \ -- if (upper) \ -- append_u64(desc, data); \ -- else \ -- append_u32(desc, data); \ --} while (0) -- --#define append_math_add_imm_u64(desc, dest, src0, src1, data) \ -- APPEND_MATH_IMM_u64(ADD, desc, dest, src0, src1, data) --#define append_math_sub_imm_u64(desc, dest, src0, src1, data) \ -- APPEND_MATH_IMM_u64(SUB, desc, dest, src0, src1, data) --#define append_math_add_c_imm_u64(desc, dest, src0, src1, data) \ -- APPEND_MATH_IMM_u64(ADDC, desc, dest, src0, src1, data) --#define append_math_sub_b_imm_u64(desc, dest, src0, src1, data) \ -- APPEND_MATH_IMM_u64(SUBB, desc, dest, src0, src1, data) --#define append_math_and_imm_u64(desc, dest, src0, src1, data) \ -- APPEND_MATH_IMM_u64(AND, desc, dest, src0, src1, data) --#define append_math_or_imm_u64(desc, dest, src0, src1, data) \ -- APPEND_MATH_IMM_u64(OR, desc, dest, src0, src1, data) --#define append_math_xor_imm_u64(desc, dest, src0, src1, data) \ -- APPEND_MATH_IMM_u64(XOR, desc, dest, src0, src1, data) --#define append_math_lshift_imm_u64(desc, dest, src0, src1, data) \ -- APPEND_MATH_IMM_u64(LSHIFT, desc, dest, src0, src1, data) --#define append_math_rshift_imm_u64(desc, dest, src0, src1, data) \ -- APPEND_MATH_IMM_u64(RSHIFT, desc, dest, src0, src1, data) -diff -Nur linux-4.1.3/drivers/crypto/caam/desc.h linux-xbian-imx6/drivers/crypto/caam/desc.h ---- linux-4.1.3/drivers/crypto/caam/desc.h 2015-07-21 19:10:33.000000000 +0200 -+++ linux-xbian-imx6/drivers/crypto/caam/desc.h 2015-07-27 23:13:04.209961631 +0200 -@@ -2,19 +2,35 @@ - * CAAM descriptor composition header - * Definitions to support CAAM descriptor instruction generation - * -- * Copyright 2008-2011 Freescale Semiconductor, Inc. -+ * Copyright (C) 2008-2013 Freescale Semiconductor, Inc. - */ - - #ifndef DESC_H - #define DESC_H - -+/* -+ * 16-byte hardware scatter/gather table -+ * An 8-byte table exists in the hardware spec, but has never been -+ * implemented to date. The 8/16 option is selected at RTL-compile-time. -+ * and this selection is visible in the Compile Time Parameters Register -+ */ -+ -+#define SEC4_SG_LEN_EXT 0x80000000 /* Entry points to table */ -+#define SEC4_SG_LEN_FIN 0x40000000 /* Last ent in table */ -+#define SEC4_SG_BPID_MASK 0x000000ff -+#define SEC4_SG_BPID_SHIFT 16 -+#define SEC4_SG_LEN_MASK 0x3fffffff /* Excludes EXT and FINAL */ -+#define SEC4_SG_OFFS_MASK 0x00001fff -+ - struct sec4_sg_entry { -+#ifdef CONFIG_64BIT - u64 ptr; --#define SEC4_SG_LEN_FIN 0x40000000 --#define SEC4_SG_LEN_EXT 0x80000000 -+#else -+ u32 reserved; -+ u32 ptr; -+#endif - u32 len; -- u8 reserved; -- u8 buf_pool_id; -+ u16 buf_pool_id; - u16 offset; - }; - -@@ -231,12 +247,7 @@ - #define LDST_SRCDST_WORD_PKHA_B_SZ (0x11 << LDST_SRCDST_SHIFT) - #define LDST_SRCDST_WORD_PKHA_N_SZ (0x12 << LDST_SRCDST_SHIFT) - #define LDST_SRCDST_WORD_PKHA_E_SZ (0x13 << LDST_SRCDST_SHIFT) --#define LDST_SRCDST_WORD_CLASS_CTX (0x20 << LDST_SRCDST_SHIFT) - #define LDST_SRCDST_WORD_DESCBUF (0x40 << LDST_SRCDST_SHIFT) --#define LDST_SRCDST_WORD_DESCBUF_JOB (0x41 << LDST_SRCDST_SHIFT) --#define LDST_SRCDST_WORD_DESCBUF_SHARED (0x42 << LDST_SRCDST_SHIFT) --#define LDST_SRCDST_WORD_DESCBUF_JOB_WE (0x45 << LDST_SRCDST_SHIFT) --#define LDST_SRCDST_WORD_DESCBUF_SHARED_WE (0x46 << LDST_SRCDST_SHIFT) - #define LDST_SRCDST_WORD_INFO_FIFO (0x7a << LDST_SRCDST_SHIFT) - - /* Offset in source/destination */ -@@ -321,6 +332,7 @@ - /* Continue - Not the last FIFO store to come */ - #define FIFOST_CONT_SHIFT 23 - #define FIFOST_CONT_MASK (1 << FIFOST_CONT_SHIFT) -+#define FIFOST_CONT_MASK (1 << FIFOST_CONT_SHIFT) - - /* - * Extended Length - use 32-bit extended length that -@@ -370,7 +382,6 @@ - #define FIFOLD_TYPE_LAST2FLUSH1 (0x05 << FIFOLD_TYPE_SHIFT) - #define FIFOLD_TYPE_LASTBOTH (0x06 << FIFOLD_TYPE_SHIFT) - #define FIFOLD_TYPE_LASTBOTHFL (0x07 << FIFOLD_TYPE_SHIFT) --#define FIFOLD_TYPE_NOINFOFIFO (0x0F << FIFOLD_TYPE_SHIFT) - - #define FIFOLDST_LEN_MASK 0xffff - #define FIFOLDST_EXT_LEN_MASK 0xffffffff -@@ -1092,6 +1103,23 @@ - #define OP_PCL_PKPROT_ECC 0x0002 - #define OP_PCL_PKPROT_F2M 0x0001 - -+/* Blob protocol protinfo bits */ -+#define OP_PCL_BLOB_TK 0x0200 -+#define OP_PCL_BLOB_EKT 0x0100 -+ -+#define OP_PCL_BLOB_K2KR_MEM 0x0000 -+#define OP_PCL_BLOB_K2KR_C1KR 0x0010 -+#define OP_PCL_BLOB_K2KR_C2KR 0x0030 -+#define OP_PCL_BLOB_K2KR_AFHAS 0x0050 -+#define OP_PCL_BLOB_K2KR_C2KR_SPLIT 0x0070 -+ -+#define OP_PCL_BLOB_PTXT_SECMEM 0x0008 -+#define OP_PCL_BLOB_BLACK 0x0004 -+ -+#define OP_PCL_BLOB_FMT_NORMAL 0x0000 -+#define OP_PCL_BLOB_FMT_MSTR 0x0002 -+#define OP_PCL_BLOB_FMT_TEST 0x0003 -+ - /* For non-protocol/alg-only op commands */ - #define OP_ALG_TYPE_SHIFT 24 - #define OP_ALG_TYPE_MASK (0x7 << OP_ALG_TYPE_SHIFT) -@@ -1154,15 +1182,8 @@ - - /* randomizer AAI set */ - #define OP_ALG_AAI_RNG (0x00 << OP_ALG_AAI_SHIFT) --#define OP_ALG_AAI_RNG_NZB (0x10 << OP_ALG_AAI_SHIFT) --#define OP_ALG_AAI_RNG_OBP (0x20 << OP_ALG_AAI_SHIFT) -- --/* RNG4 AAI set */ --#define OP_ALG_AAI_RNG4_SH_0 (0x00 << OP_ALG_AAI_SHIFT) --#define OP_ALG_AAI_RNG4_SH_1 (0x01 << OP_ALG_AAI_SHIFT) --#define OP_ALG_AAI_RNG4_PS (0x40 << OP_ALG_AAI_SHIFT) --#define OP_ALG_AAI_RNG4_AI (0x80 << OP_ALG_AAI_SHIFT) --#define OP_ALG_AAI_RNG4_SK (0x100 << OP_ALG_AAI_SHIFT) -+#define OP_ALG_AAI_RNG_NOZERO (0x10 << OP_ALG_AAI_SHIFT) -+#define OP_ALG_AAI_RNG_ODD (0x20 << OP_ALG_AAI_SHIFT) - - /* hmac/smac AAI set */ - #define OP_ALG_AAI_HASH (0x00 << OP_ALG_AAI_SHIFT) -@@ -1184,6 +1205,12 @@ - #define OP_ALG_AAI_GSM (0x10 << OP_ALG_AAI_SHIFT) - #define OP_ALG_AAI_EDGE (0x20 << OP_ALG_AAI_SHIFT) - -+/* RNG4 set */ -+#define OP_ALG_RNG4_SHIFT 4 -+#define OP_ALG_RNG4_MASK (0x1f3 << OP_ALG_RNG4_SHIFT) -+ -+#define OP_ALG_RNG4_SK (0x100 << OP_ALG_RNG4_SHIFT) -+ - #define OP_ALG_AS_SHIFT 2 - #define OP_ALG_AS_MASK (0x3 << OP_ALG_AS_SHIFT) - #define OP_ALG_AS_UPDATE (0 << OP_ALG_AS_SHIFT) -@@ -1300,10 +1327,10 @@ - #define SQOUT_SGF 0x01000000 - - /* Appends to a previous pointer */ --#define SQOUT_PRE SQIN_PRE -+#define SQOUT_PRE 0x00800000 - - /* Restore sequence with pointer/length */ --#define SQOUT_RTO SQIN_RTO -+#define SQOUT_RTO 0x00200000 - - /* Use extended length following pointer */ - #define SQOUT_EXT 0x00400000 -@@ -1365,7 +1392,6 @@ - #define MOVE_DEST_MATH3 (0x07 << MOVE_DEST_SHIFT) - #define MOVE_DEST_CLASS1INFIFO (0x08 << MOVE_DEST_SHIFT) - #define MOVE_DEST_CLASS2INFIFO (0x09 << MOVE_DEST_SHIFT) --#define MOVE_DEST_INFIFO_NOINFO (0x0a << MOVE_DEST_SHIFT) - #define MOVE_DEST_PK_A (0x0c << MOVE_DEST_SHIFT) - #define MOVE_DEST_CLASS1KEY (0x0d << MOVE_DEST_SHIFT) - #define MOVE_DEST_CLASS2KEY (0x0e << MOVE_DEST_SHIFT) -@@ -1418,7 +1444,6 @@ - #define MATH_SRC0_REG2 (0x02 << MATH_SRC0_SHIFT) - #define MATH_SRC0_REG3 (0x03 << MATH_SRC0_SHIFT) - #define MATH_SRC0_IMM (0x04 << MATH_SRC0_SHIFT) --#define MATH_SRC0_DPOVRD (0x07 << MATH_SRC0_SHIFT) - #define MATH_SRC0_SEQINLEN (0x08 << MATH_SRC0_SHIFT) - #define MATH_SRC0_SEQOUTLEN (0x09 << MATH_SRC0_SHIFT) - #define MATH_SRC0_VARSEQINLEN (0x0a << MATH_SRC0_SHIFT) -@@ -1433,7 +1458,6 @@ - #define MATH_SRC1_REG2 (0x02 << MATH_SRC1_SHIFT) - #define MATH_SRC1_REG3 (0x03 << MATH_SRC1_SHIFT) - #define MATH_SRC1_IMM (0x04 << MATH_SRC1_SHIFT) --#define MATH_SRC1_DPOVRD (0x07 << MATH_SRC0_SHIFT) - #define MATH_SRC1_INFIFO (0x0a << MATH_SRC1_SHIFT) - #define MATH_SRC1_OUTFIFO (0x0b << MATH_SRC1_SHIFT) - #define MATH_SRC1_ONE (0x0c << MATH_SRC1_SHIFT) -@@ -1609,13 +1633,28 @@ - #define NFIFOENTRY_PLEN_SHIFT 0 - #define NFIFOENTRY_PLEN_MASK (0xFF << NFIFOENTRY_PLEN_SHIFT) - --/* Append Load Immediate Command */ --#define FD_CMD_APPEND_LOAD_IMMEDIATE 0x80000000 -+/* -+ * PDB internal definitions -+ */ -+ -+/* IPSec ESP CBC Encap/Decap Options */ -+#define PDBOPTS_ESPCBC_ARSNONE 0x00 /* no antireplay window */ -+#define PDBOPTS_ESPCBC_ARS32 0x40 /* 32-entry antireplay window */ -+#define PDBOPTS_ESPCBC_ARS64 0xc0 /* 64-entry antireplay window */ -+#define PDBOPTS_ESPCBC_IVSRC 0x20 /* IV comes from internal random gen */ -+#define PDBOPTS_ESPCBC_ESN 0x10 /* extended sequence included */ -+#define PDBOPTS_ESPCBC_OUTFMT 0x08 /* output only decapsulation (decap) */ -+#define PDBOPTS_ESPCBC_IPHDRSRC 0x08 /* IP header comes from PDB (encap) */ -+#define PDBOPTS_ESPCBC_INCIPHDR 0x04 /* Prepend IP header to output frame */ -+#define PDBOPTS_ESPCBC_IPVSN 0x02 /* process IPv6 header */ -+#define PDBOPTS_ESPCBC_TUNNEL 0x01 /* tunnel mode next-header byte */ -+ -+#define ARC4_BLOCK_SIZE 1 -+#define ARC4_MAX_KEY_SIZE 256 -+#define ARC4_MIN_KEY_SIZE 1 - --/* Set SEQ LIODN equal to the Non-SEQ LIODN for the job */ --#define FD_CMD_SET_SEQ_LIODN_EQUAL_NONSEQ_LIODN 0x40000000 -+#define XCBC_MAC_DIGEST_SIZE 16 -+#define XCBC_MAC_BLOCK_WORDS 16 - --/* Frame Descriptor Command for Replacement Job Descriptor */ --#define FD_CMD_REPLACE_JOB_DESC 0x20000000 - - #endif /* DESC_H */ -diff -Nur linux-4.1.3/drivers/crypto/caam/error.c linux-xbian-imx6/drivers/crypto/caam/error.c ---- linux-4.1.3/drivers/crypto/caam/error.c 2015-07-21 19:10:33.000000000 +0200 -+++ linux-xbian-imx6/drivers/crypto/caam/error.c 2015-07-27 23:13:04.213947410 +0200 -@@ -11,243 +11,264 @@ - #include "jr.h" - #include "error.h" - --static const struct { -- u8 value; -- const char *error_text; --} desc_error_list[] = { -- { 0x00, "No error." }, -- { 0x01, "SGT Length Error. The descriptor is trying to read more data than is contained in the SGT table." }, -- { 0x02, "SGT Null Entry Error." }, -- { 0x03, "Job Ring Control Error. There is a bad value in the Job Ring Control register." }, -- { 0x04, "Invalid Descriptor Command. The Descriptor Command field is invalid." }, -- { 0x05, "Reserved." }, -- { 0x06, "Invalid KEY Command" }, -- { 0x07, "Invalid LOAD Command" }, -- { 0x08, "Invalid STORE Command" }, -- { 0x09, "Invalid OPERATION Command" }, -- { 0x0A, "Invalid FIFO LOAD Command" }, -- { 0x0B, "Invalid FIFO STORE Command" }, -- { 0x0C, "Invalid MOVE/MOVE_LEN Command" }, -- { 0x0D, "Invalid JUMP Command. A nonlocal JUMP Command is invalid because the target is not a Job Header Command, or the jump is from a Trusted Descriptor to a Job Descriptor, or because the target Descriptor contains a Shared Descriptor." }, -- { 0x0E, "Invalid MATH Command" }, -- { 0x0F, "Invalid SIGNATURE Command" }, -- { 0x10, "Invalid Sequence Command. A SEQ IN PTR OR SEQ OUT PTR Command is invalid or a SEQ KEY, SEQ LOAD, SEQ FIFO LOAD, or SEQ FIFO STORE decremented the input or output sequence length below 0. This error may result if a built-in PROTOCOL Command has encountered a malformed PDU." }, -- { 0x11, "Skip data type invalid. The type must be 0xE or 0xF."}, -- { 0x12, "Shared Descriptor Header Error" }, -- { 0x13, "Header Error. Invalid length or parity, or certain other problems." }, -- { 0x14, "Burster Error. Burster has gotten to an illegal state" }, -- { 0x15, "Context Register Length Error. The descriptor is trying to read or write past the end of the Context Register. A SEQ LOAD or SEQ STORE with the VLF bit set was executed with too large a length in the variable length register (VSOL for SEQ STORE or VSIL for SEQ LOAD)." }, -- { 0x16, "DMA Error" }, -- { 0x17, "Reserved." }, -- { 0x1A, "Job failed due to JR reset" }, -- { 0x1B, "Job failed due to Fail Mode" }, -- { 0x1C, "DECO Watchdog timer timeout error" }, -- { 0x1D, "DECO tried to copy a key from another DECO but the other DECO's Key Registers were locked" }, -- { 0x1E, "DECO attempted to copy data from a DECO that had an unmasked Descriptor error" }, -- { 0x1F, "LIODN error. DECO was trying to share from itself or from another DECO but the two Non-SEQ LIODN values didn't match or the 'shared from' DECO's Descriptor required that the SEQ LIODNs be the same and they aren't." }, -- { 0x20, "DECO has completed a reset initiated via the DRR register" }, -- { 0x21, "Nonce error. When using EKT (CCM) key encryption option in the FIFO STORE Command, the Nonce counter reached its maximum value and this encryption mode can no longer be used." }, -- { 0x22, "Meta data is too large (> 511 bytes) for TLS decap (input frame; block ciphers) and IPsec decap (output frame, when doing the next header byte update) and DCRC (output frame)." }, -- { 0x23, "Read Input Frame error" }, -- { 0x24, "JDKEK, TDKEK or TDSK not loaded error" }, -- { 0x80, "DNR (do not run) error" }, -- { 0x81, "undefined protocol command" }, -- { 0x82, "invalid setting in PDB" }, -- { 0x83, "Anti-replay LATE error" }, -- { 0x84, "Anti-replay REPLAY error" }, -- { 0x85, "Sequence number overflow" }, -- { 0x86, "Sigver invalid signature" }, -- { 0x87, "DSA Sign Illegal test descriptor" }, -- { 0x88, "Protocol Format Error - A protocol has seen an error in the format of data received. When running RSA, this means that formatting with random padding was used, and did not follow the form: 0x00, 0x02, 8-to-N bytes of non-zero pad, 0x00, F data." }, -- { 0x89, "Protocol Size Error - A protocol has seen an error in size. When running RSA, pdb size N < (size of F) when no formatting is used; or pdb size N < (F + 11) when formatting is used." }, -- { 0xC1, "Blob Command error: Undefined mode" }, -- { 0xC2, "Blob Command error: Secure Memory Blob mode error" }, -- { 0xC4, "Blob Command error: Black Blob key or input size error" }, -- { 0xC5, "Blob Command error: Invalid key destination" }, -- { 0xC8, "Blob Command error: Trusted/Secure mode error" }, -- { 0xF0, "IPsec TTL or hop limit field either came in as 0, or was decremented to 0" }, -- { 0xF1, "3GPP HFN matches or exceeds the Threshold" }, --}; -- --static const char * const cha_id_list[] = { -- "", -- "AES", -- "DES", -- "ARC4", -- "MDHA", -- "RNG", -- "SNOW f8", -- "Kasumi f8/9", -- "PKHA", -- "CRCA", -- "SNOW f9", -- "ZUCE", -- "ZUCA", --}; -- --static const char * const err_id_list[] = { -- "No error.", -- "Mode error.", -- "Data size error.", -- "Key size error.", -- "PKHA A memory size error.", -- "PKHA B memory size error.", -- "Data arrived out of sequence error.", -- "PKHA divide-by-zero error.", -- "PKHA modulus even error.", -- "DES key parity error.", -- "ICV check failed.", -- "Hardware error.", -- "Unsupported CCM AAD size.", -- "Class 1 CHA is not reset", -- "Invalid CHA combination was selected", -- "Invalid CHA selected.", --}; -- --static const char * const rng_err_id_list[] = { -- "", -- "", -- "", -- "Instantiate", -- "Not instantiated", -- "Test instantiate", -- "Prediction resistance", -- "Prediction resistance and test request", -- "Uninstantiate", -- "Secure key generation", --}; -+#define SPRINTFCAT(str, format, param, max_alloc) \ -+{ \ -+ char *tmp; \ -+ \ -+ tmp = kmalloc(sizeof(format) + max_alloc, GFP_ATOMIC); \ -+ if (likely(tmp)) { \ -+ sprintf(tmp, format, param); \ -+ strcat(str, tmp); \ -+ kfree(tmp); \ -+ } else { \ -+ strcat(str, "kmalloc failure in SPRINTFCAT"); \ -+ } \ -+} - --static void report_ccb_status(struct device *jrdev, const u32 status, -- const char *error) -+static void report_jump_idx(u32 status, char *outstr) - { -- u8 cha_id = (status & JRSTA_CCBERR_CHAID_MASK) >> -- JRSTA_CCBERR_CHAID_SHIFT; -- u8 err_id = status & JRSTA_CCBERR_ERRID_MASK; - u8 idx = (status & JRSTA_DECOERR_INDEX_MASK) >> - JRSTA_DECOERR_INDEX_SHIFT; -- char *idx_str; -- const char *cha_str = "unidentified cha_id value 0x"; -- char cha_err_code[3] = { 0 }; -- const char *err_str = "unidentified err_id value 0x"; -- char err_err_code[3] = { 0 }; - - if (status & JRSTA_DECOERR_JUMP) -- idx_str = "jump tgt desc idx"; -+ strcat(outstr, "jump tgt desc idx "); - else -- idx_str = "desc idx"; -+ strcat(outstr, "desc idx "); - -- if (cha_id < ARRAY_SIZE(cha_id_list)) -- cha_str = cha_id_list[cha_id]; -- else -- snprintf(cha_err_code, sizeof(cha_err_code), "%02x", cha_id); -+ SPRINTFCAT(outstr, "%d: ", idx, sizeof("255")); -+} -+ -+static void report_ccb_status(u32 status, char *outstr) -+{ -+ static const char * const cha_id_list[] = { -+ "", -+ "AES", -+ "DES", -+ "ARC4", -+ "MDHA", -+ "RNG", -+ "SNOW f8", -+ "Kasumi f8/9", -+ "PKHA", -+ "CRCA", -+ "SNOW f9", -+ "ZUCE", -+ "ZUCA", -+ }; -+ static const char * const err_id_list[] = { -+ "No error.", -+ "Mode error.", -+ "Data size error.", -+ "Key size error.", -+ "PKHA A memory size error.", -+ "PKHA B memory size error.", -+ "Data arrived out of sequence error.", -+ "PKHA divide-by-zero error.", -+ "PKHA modulus even error.", -+ "DES key parity error.", -+ "ICV check failed.", -+ "Hardware error.", -+ "Unsupported CCM AAD size.", -+ "Class 1 CHA is not reset", -+ "Invalid CHA combination was selected", -+ "Invalid CHA selected.", -+ }; -+ static const char * const rng_err_id_list[] = { -+ "", -+ "", -+ "", -+ "Instantiate", -+ "Not instantiated", -+ "Test instantiate", -+ "Prediction resistance", -+ "Prediction resistance and test request", -+ "Uninstantiate", -+ "Secure key generation", -+ }; -+ u8 cha_id = (status & JRSTA_CCBERR_CHAID_MASK) >> -+ JRSTA_CCBERR_CHAID_SHIFT; -+ u8 err_id = status & JRSTA_CCBERR_ERRID_MASK; -+ -+ report_jump_idx(status, outstr); -+ -+ if (cha_id < ARRAY_SIZE(cha_id_list)) { -+ SPRINTFCAT(outstr, "%s: ", cha_id_list[cha_id], -+ strlen(cha_id_list[cha_id])); -+ } else { -+ SPRINTFCAT(outstr, "unidentified cha_id value 0x%02x: ", -+ cha_id, sizeof("ff")); -+ } - - if ((cha_id << JRSTA_CCBERR_CHAID_SHIFT) == JRSTA_CCBERR_CHAID_RNG && - err_id < ARRAY_SIZE(rng_err_id_list) && - strlen(rng_err_id_list[err_id])) { - /* RNG-only error */ -- err_str = rng_err_id_list[err_id]; -- } else if (err_id < ARRAY_SIZE(err_id_list)) -- err_str = err_id_list[err_id]; -- else -- snprintf(err_err_code, sizeof(err_err_code), "%02x", err_id); -- -- /* -- * CCB ICV check failures are part of normal operation life; -- * we leave the upper layers to do what they want with them. -- */ -- if (err_id != JRSTA_CCBERR_ERRID_ICVCHK) -- dev_err(jrdev, "%08x: %s: %s %d: %s%s: %s%s\n", -- status, error, idx_str, idx, -- cha_str, cha_err_code, -- err_str, err_err_code); -+ SPRINTFCAT(outstr, "%s", rng_err_id_list[err_id], -+ strlen(rng_err_id_list[err_id])); -+ } else if (err_id < ARRAY_SIZE(err_id_list)) { -+ SPRINTFCAT(outstr, "%s", err_id_list[err_id], -+ strlen(err_id_list[err_id])); -+ } else { -+ SPRINTFCAT(outstr, "unidentified err_id value 0x%02x", -+ err_id, sizeof("ff")); -+ } - } - --static void report_jump_status(struct device *jrdev, const u32 status, -- const char *error) -+static void report_jump_status(u32 status, char *outstr) - { -- dev_err(jrdev, "%08x: %s: %s() not implemented\n", -- status, error, __func__); -+ SPRINTFCAT(outstr, "%s() not implemented", __func__, sizeof(__func__)); - } - --static void report_deco_status(struct device *jrdev, const u32 status, -- const char *error) -+static void report_deco_status(u32 status, char *outstr) - { -- u8 err_id = status & JRSTA_DECOERR_ERROR_MASK; -- u8 idx = (status & JRSTA_DECOERR_INDEX_MASK) >> -- JRSTA_DECOERR_INDEX_SHIFT; -- char *idx_str; -- const char *err_str = "unidentified error value 0x"; -- char err_err_code[3] = { 0 }; -+ static const struct { -+ u8 value; -+ char *error_text; -+ } desc_error_list[] = { -+ { 0x00, "No error." }, -+ { 0x01, "SGT Length Error. The descriptor is trying to read " -+ "more data than is contained in the SGT table." }, -+ { 0x02, "SGT Null Entry Error." }, -+ { 0x03, "Job Ring Control Error. There is a bad value in the " -+ "Job Ring Control register." }, -+ { 0x04, "Invalid Descriptor Command. The Descriptor Command " -+ "field is invalid." }, -+ { 0x05, "Reserved." }, -+ { 0x06, "Invalid KEY Command" }, -+ { 0x07, "Invalid LOAD Command" }, -+ { 0x08, "Invalid STORE Command" }, -+ { 0x09, "Invalid OPERATION Command" }, -+ { 0x0A, "Invalid FIFO LOAD Command" }, -+ { 0x0B, "Invalid FIFO STORE Command" }, -+ { 0x0C, "Invalid MOVE/MOVE_LEN Command" }, -+ { 0x0D, "Invalid JUMP Command. A nonlocal JUMP Command is " -+ "invalid because the target is not a Job Header " -+ "Command, or the jump is from a Trusted Descriptor to " -+ "a Job Descriptor, or because the target Descriptor " -+ "contains a Shared Descriptor." }, -+ { 0x0E, "Invalid MATH Command" }, -+ { 0x0F, "Invalid SIGNATURE Command" }, -+ { 0x10, "Invalid Sequence Command. A SEQ IN PTR OR SEQ OUT PTR " -+ "Command is invalid or a SEQ KEY, SEQ LOAD, SEQ FIFO " -+ "LOAD, or SEQ FIFO STORE decremented the input or " -+ "output sequence length below 0. This error may result " -+ "if a built-in PROTOCOL Command has encountered a " -+ "malformed PDU." }, -+ { 0x11, "Skip data type invalid. The type must be 0xE or 0xF."}, -+ { 0x12, "Shared Descriptor Header Error" }, -+ { 0x13, "Header Error. Invalid length or parity, or certain " -+ "other problems." }, -+ { 0x14, "Burster Error. Burster has gotten to an illegal " -+ "state" }, -+ { 0x15, "Context Register Length Error. The descriptor is " -+ "trying to read or write past the end of the Context " -+ "Register. A SEQ LOAD or SEQ STORE with the VLF bit " -+ "set was executed with too large a length in the " -+ "variable length register (VSOL for SEQ STORE or VSIL " -+ "for SEQ LOAD)." }, -+ { 0x16, "DMA Error" }, -+ { 0x17, "Reserved." }, -+ { 0x1A, "Job failed due to JR reset" }, -+ { 0x1B, "Job failed due to Fail Mode" }, -+ { 0x1C, "DECO Watchdog timer timeout error" }, -+ { 0x1D, "DECO tried to copy a key from another DECO but the " -+ "other DECO's Key Registers were locked" }, -+ { 0x1E, "DECO attempted to copy data from a DECO that had an " -+ "unmasked Descriptor error" }, -+ { 0x1F, "LIODN error. DECO was trying to share from itself or " -+ "from another DECO but the two Non-SEQ LIODN values " -+ "didn't match or the 'shared from' DECO's Descriptor " -+ "required that the SEQ LIODNs be the same and they " -+ "aren't." }, -+ { 0x20, "DECO has completed a reset initiated via the DRR " -+ "register" }, -+ { 0x21, "Nonce error. When using EKT (CCM) key encryption " -+ "option in the FIFO STORE Command, the Nonce counter " -+ "reached its maximum value and this encryption mode " -+ "can no longer be used." }, -+ { 0x22, "Meta data is too large (> 511 bytes) for TLS decap " -+ "(input frame; block ciphers) and IPsec decap (output " -+ "frame, when doing the next header byte update) and " -+ "DCRC (output frame)." }, -+ { 0x23, "Read Input Frame error" }, -+ { 0x24, "JDKEK, TDKEK or TDSK not loaded error" }, -+ { 0x80, "DNR (do not run) error" }, -+ { 0x81, "undefined protocol command" }, -+ { 0x82, "invalid setting in PDB" }, -+ { 0x83, "Anti-replay LATE error" }, -+ { 0x84, "Anti-replay REPLAY error" }, -+ { 0x85, "Sequence number overflow" }, -+ { 0x86, "Sigver invalid signature" }, -+ { 0x87, "DSA Sign Illegal test descriptor" }, -+ { 0x88, "Protocol Format Error - A protocol has seen an error " -+ "in the format of data received. When running RSA, " -+ "this means that formatting with random padding was " -+ "used, and did not follow the form: 0x00, 0x02, 8-to-N " -+ "bytes of non-zero pad, 0x00, F data." }, -+ { 0x89, "Protocol Size Error - A protocol has seen an error in " -+ "size. When running RSA, pdb size N < (size of F) when " -+ "no formatting is used; or pdb size N < (F + 11) when " -+ "formatting is used." }, -+ { 0xC1, "Blob Command error: Undefined mode" }, -+ { 0xC2, "Blob Command error: Secure Memory Blob mode error" }, -+ { 0xC4, "Blob Command error: Black Blob key or input size " -+ "error" }, -+ { 0xC5, "Blob Command error: Invalid key destination" }, -+ { 0xC8, "Blob Command error: Trusted/Secure mode error" }, -+ { 0xF0, "IPsec TTL or hop limit field either came in as 0, " -+ "or was decremented to 0" }, -+ { 0xF1, "3GPP HFN matches or exceeds the Threshold" }, -+ }; -+ u8 desc_error = status & JRSTA_DECOERR_ERROR_MASK; - int i; - -- if (status & JRSTA_DECOERR_JUMP) -- idx_str = "jump tgt desc idx"; -- else -- idx_str = "desc idx"; -+ report_jump_idx(status, outstr); - - for (i = 0; i < ARRAY_SIZE(desc_error_list); i++) -- if (desc_error_list[i].value == err_id) -+ if (desc_error_list[i].value == desc_error) - break; - -- if (i != ARRAY_SIZE(desc_error_list) && desc_error_list[i].error_text) -- err_str = desc_error_list[i].error_text; -- else -- snprintf(err_err_code, sizeof(err_err_code), "%02x", err_id); -- -- dev_err(jrdev, "%08x: %s: %s %d: %s%s\n", -- status, error, idx_str, idx, err_str, err_err_code); -+ if (i != ARRAY_SIZE(desc_error_list) && desc_error_list[i].error_text) { -+ SPRINTFCAT(outstr, "%s", desc_error_list[i].error_text, -+ strlen(desc_error_list[i].error_text)); -+ } else { -+ SPRINTFCAT(outstr, "unidentified error value 0x%02x", -+ desc_error, sizeof("ff")); -+ } - } - --static void report_jr_status(struct device *jrdev, const u32 status, -- const char *error) -+static void report_jr_status(u32 status, char *outstr) - { -- dev_err(jrdev, "%08x: %s: %s() not implemented\n", -- status, error, __func__); -+ SPRINTFCAT(outstr, "%s() not implemented", __func__, sizeof(__func__)); - } - --static void report_cond_code_status(struct device *jrdev, const u32 status, -- const char *error) -+static void report_cond_code_status(u32 status, char *outstr) - { -- dev_err(jrdev, "%08x: %s: %s() not implemented\n", -- status, error, __func__); -+ SPRINTFCAT(outstr, "%s() not implemented", __func__, sizeof(__func__)); - } - --void caam_jr_strstatus(struct device *jrdev, u32 status) -+char *caam_jr_strstatus(char *outstr, u32 status) - { - static const struct stat_src { -- void (*report_ssed)(struct device *jrdev, const u32 status, -- const char *error); -- const char *error; -- } status_src[16] = { -+ void (*report_ssed)(u32 status, char *outstr); -+ char *error; -+ } status_src[] = { - { NULL, "No error" }, - { NULL, NULL }, - { report_ccb_status, "CCB" }, - { report_jump_status, "Jump" }, - { report_deco_status, "DECO" }, -- { NULL, "Queue Manager Interface" }, -+ { NULL, NULL }, - { report_jr_status, "Job Ring" }, - { report_cond_code_status, "Condition Code" }, -- { NULL, NULL }, -- { NULL, NULL }, -- { NULL, NULL }, -- { NULL, NULL }, -- { NULL, NULL }, -- { NULL, NULL }, -- { NULL, NULL }, -- { NULL, NULL }, - }; - u32 ssrc = status >> JRSTA_SSRC_SHIFT; -- const char *error = status_src[ssrc].error; - -- /* -- * If there is an error handling function, call it to report the error. -- * Otherwise print the error source name. -- */ -+ sprintf(outstr, "%s: ", status_src[ssrc].error); -+ - if (status_src[ssrc].report_ssed) -- status_src[ssrc].report_ssed(jrdev, status, error); -- else if (error) -- dev_err(jrdev, "%d: %s\n", ssrc, error); -- else -- dev_err(jrdev, "%d: unknown error source\n", ssrc); -+ status_src[ssrc].report_ssed(status, outstr); -+ -+ return outstr; - } - EXPORT_SYMBOL(caam_jr_strstatus); -diff -Nur linux-4.1.3/drivers/crypto/caam/error.h linux-xbian-imx6/drivers/crypto/caam/error.h ---- linux-4.1.3/drivers/crypto/caam/error.h 2015-07-21 19:10:33.000000000 +0200 -+++ linux-xbian-imx6/drivers/crypto/caam/error.h 2015-07-27 23:13:04.213947410 +0200 -@@ -7,5 +7,5 @@ - #ifndef CAAM_ERROR_H - #define CAAM_ERROR_H - #define CAAM_ERROR_STR_MAX 302 --void caam_jr_strstatus(struct device *jrdev, u32 status); -+extern char *caam_jr_strstatus(char *outstr, u32 status); - #endif /* CAAM_ERROR_H */ -diff -Nur linux-4.1.3/drivers/crypto/caam/intern.h linux-xbian-imx6/drivers/crypto/caam/intern.h ---- linux-4.1.3/drivers/crypto/caam/intern.h 2015-07-21 19:10:33.000000000 +0200 -+++ linux-xbian-imx6/drivers/crypto/caam/intern.h 2015-07-27 23:13:04.213947410 +0200 -@@ -2,13 +2,19 @@ - * CAAM/SEC 4.x driver backend - * Private/internal definitions between modules - * -- * Copyright 2008-2011 Freescale Semiconductor, Inc. -+ * Copyright (C) 2008-2013 Freescale Semiconductor, Inc. - * - */ - - #ifndef INTERN_H - #define INTERN_H - -+#define JOBR_UNASSIGNED 0 -+#define JOBR_ASSIGNED 1 -+ -+/* Default clock/sample settings for an RNG4 entropy source */ -+#define RNG4_ENT_CLOCKS_SAMPLE 1600 -+ - /* Currently comes from Kconfig param as a ^2 (driver-required) */ - #define JOBR_DEPTH (1 << CONFIG_CRYPTO_DEV_FSL_CAAM_RINGSIZE) - -@@ -37,15 +43,13 @@ - - /* Private sub-storage for a single JobR */ - struct caam_drv_private_jr { -- struct list_head list_node; /* Job Ring device list */ -- struct device *dev; -+ struct device *parentdev; /* points back to controller dev */ -+ struct platform_device *jr_pdev;/* points to platform device for JR */ - int ridx; - struct caam_job_ring __iomem *rregs; /* JobR's register space */ - struct tasklet_struct irqtask; - int irq; /* One per queue */ -- -- /* Number of scatterlist crypt transforms active on the JobR */ -- atomic_t tfm_count ____cacheline_aligned; -+ int assign; /* busy/free */ - - /* Job ring info */ - int ringsize; /* Size of rings (assume input = output) */ -@@ -66,15 +70,20 @@ - struct caam_drv_private { - - struct device *dev; -- struct platform_device **jrpdev; /* Alloc'ed array per sub-device */ -+ struct device *smdev; -+ struct device *secviodev; -+ struct device **jrdev; /* Alloc'ed array per sub-device */ -+ spinlock_t jr_alloc_lock; - struct platform_device *pdev; - - /* Physical-presence section */ -- struct caam_ctrl __iomem *ctrl; /* controller region */ -- struct caam_deco __iomem *deco; /* DECO/CCB views */ -- struct caam_assurance __iomem *assure; -- struct caam_queue_if __iomem *qi; /* QI control region */ -- struct caam_job_ring __iomem *jr[4]; /* JobR's register space */ -+ struct caam_ctrl *ctrl; /* controller region */ -+ struct caam_deco **deco; /* DECO/CCB views */ -+ struct caam_assurance *ac; -+ struct caam_queue_if *qi; /* QI control region */ -+ struct snvs_full __iomem *snvs; /* SNVS HP+LP register space */ -+ dma_addr_t __iomem *sm_base; /* Secure memory storage base */ -+ u32 sm_size; - - /* - * Detected geometry block. Filled in from device tree if powerpc, -@@ -83,14 +92,22 @@ - u8 total_jobrs; /* Total Job Rings in device */ - u8 qi_present; /* Nonzero if QI present in device */ - int secvio_irq; /* Security violation interrupt number */ -- int virt_en; /* Virtualization enabled in CAAM */ -- --#define RNG4_MAX_HANDLES 2 -- /* RNG4 block */ -- u32 rng4_sh_init; /* This bitmap shows which of the State -- Handles of the RNG4 block are initialized -- by this driver */ -+ int rng_inst; /* Total instantiated RNGs */ - -+ /* which jr allocated to scatterlist crypto */ -+ atomic_t tfm_count ____cacheline_aligned; -+ int num_jrs_for_algapi; -+ struct device **algapi_jr; -+ /* list of registered crypto algorithms (mk generic context handle?) */ -+ struct list_head alg_list; -+ /* list of registered hash algorithms (mk generic context handle?) */ -+ struct list_head hash_list; -+ -+#ifdef CONFIG_ARM -+ struct clk *caam_ipg; -+ struct clk *caam_mem; -+ struct clk *caam_aclk; -+#endif - /* - * debugfs entries for developer view into driver/device - * variables at runtime. -diff -Nur linux-4.1.3/drivers/crypto/caam/jr.c linux-xbian-imx6/drivers/crypto/caam/jr.c ---- linux-4.1.3/drivers/crypto/caam/jr.c 2015-07-21 19:10:33.000000000 +0200 -+++ linux-xbian-imx6/drivers/crypto/caam/jr.c 2015-07-27 23:13:04.213947410 +0200 -@@ -2,125 +2,15 @@ - * CAAM/SEC 4.x transport/backend driver - * JobR backend functionality - * -- * Copyright 2008-2012 Freescale Semiconductor, Inc. -+ * Copyright (C) 2008-2013 Freescale Semiconductor, Inc. - */ - --#include --#include -- - #include "compat.h" - #include "regs.h" - #include "jr.h" - #include "desc.h" - #include "intern.h" - --struct jr_driver_data { -- /* List of Physical JobR's with the Driver */ -- struct list_head jr_list; -- spinlock_t jr_alloc_lock; /* jr_list lock */ --} ____cacheline_aligned; -- --static struct jr_driver_data driver_data; -- --static int caam_reset_hw_jr(struct device *dev) --{ -- struct caam_drv_private_jr *jrp = dev_get_drvdata(dev); -- unsigned int timeout = 100000; -- -- /* -- * mask interrupts since we are going to poll -- * for reset completion status -- */ -- setbits32(&jrp->rregs->rconfig_lo, JRCFG_IMSK); -- -- /* initiate flush (required prior to reset) */ -- wr_reg32(&jrp->rregs->jrcommand, JRCR_RESET); -- while (((rd_reg32(&jrp->rregs->jrintstatus) & JRINT_ERR_HALT_MASK) == -- JRINT_ERR_HALT_INPROGRESS) && --timeout) -- cpu_relax(); -- -- if ((rd_reg32(&jrp->rregs->jrintstatus) & JRINT_ERR_HALT_MASK) != -- JRINT_ERR_HALT_COMPLETE || timeout == 0) { -- dev_err(dev, "failed to flush job ring %d\n", jrp->ridx); -- return -EIO; -- } -- -- /* initiate reset */ -- timeout = 100000; -- wr_reg32(&jrp->rregs->jrcommand, JRCR_RESET); -- while ((rd_reg32(&jrp->rregs->jrcommand) & JRCR_RESET) && --timeout) -- cpu_relax(); -- -- if (timeout == 0) { -- dev_err(dev, "failed to reset job ring %d\n", jrp->ridx); -- return -EIO; -- } -- -- /* unmask interrupts */ -- clrbits32(&jrp->rregs->rconfig_lo, JRCFG_IMSK); -- -- return 0; --} -- --/* -- * Shutdown JobR independent of platform property code -- */ --int caam_jr_shutdown(struct device *dev) --{ -- struct caam_drv_private_jr *jrp = dev_get_drvdata(dev); -- dma_addr_t inpbusaddr, outbusaddr; -- int ret; -- -- ret = caam_reset_hw_jr(dev); -- -- tasklet_kill(&jrp->irqtask); -- -- /* Release interrupt */ -- free_irq(jrp->irq, dev); -- -- /* Free rings */ -- inpbusaddr = rd_reg64(&jrp->rregs->inpring_base); -- outbusaddr = rd_reg64(&jrp->rregs->outring_base); -- dma_free_coherent(dev, sizeof(dma_addr_t) * JOBR_DEPTH, -- jrp->inpring, inpbusaddr); -- dma_free_coherent(dev, sizeof(struct jr_outentry) * JOBR_DEPTH, -- jrp->outring, outbusaddr); -- kfree(jrp->entinfo); -- -- return ret; --} -- --static int caam_jr_remove(struct platform_device *pdev) --{ -- int ret; -- struct device *jrdev; -- struct caam_drv_private_jr *jrpriv; -- -- jrdev = &pdev->dev; -- jrpriv = dev_get_drvdata(jrdev); -- -- /* -- * Return EBUSY if job ring already allocated. -- */ -- if (atomic_read(&jrpriv->tfm_count)) { -- dev_err(jrdev, "Device is busy\n"); -- return -EBUSY; -- } -- -- /* Remove the node from Physical JobR list maintained by driver */ -- spin_lock(&driver_data.jr_alloc_lock); -- list_del(&jrpriv->list_node); -- spin_unlock(&driver_data.jr_alloc_lock); -- -- /* Release ring */ -- ret = caam_jr_shutdown(jrdev); -- if (ret) -- dev_err(jrdev, "Failed to shut down job ring\n"); -- irq_dispose_mapping(jrpriv->irq); -- -- return ret; --} -- - /* Main per-ring interrupt handler */ - static irqreturn_t caam_jr_interrupt(int irq, void *st_dev) - { -@@ -168,6 +58,9 @@ - void (*usercall)(struct device *dev, u32 *desc, u32 status, void *arg); - u32 *userdesc, userstatus; - void *userarg; -+ dma_addr_t outbusaddr; -+ -+ outbusaddr = rd_reg64(&jrp->rregs->outring_base); - - while (rd_reg32(&jrp->rregs->outring_used)) { - -@@ -177,10 +70,15 @@ - - sw_idx = tail = jrp->tail; - hw_idx = jrp->out_ring_read_index; -+ dma_sync_single_for_cpu(dev, outbusaddr, -+ sizeof(struct jr_outentry) * JOBR_DEPTH, -+ DMA_FROM_DEVICE); - - for (i = 0; CIRC_CNT(head, tail + i, JOBR_DEPTH) >= 1; i++) { - sw_idx = (tail + i) & (JOBR_DEPTH - 1); - -+ smp_read_barrier_depends(); -+ - if (jrp->outring[hw_idx].desc == - jrp->entinfo[sw_idx].desc_addr_dma) - break; /* found */ -@@ -202,6 +100,8 @@ - userdesc = jrp->entinfo[sw_idx].desc_addr_virt; - userstatus = jrp->outring[hw_idx].jrstatus; - -+ smp_mb(); -+ - /* set done */ - wr_reg32(&jrp->rregs->outring_rmvd, 1); - -@@ -216,6 +116,7 @@ - if (sw_idx == tail) { - do { - tail = (tail + 1) & (JOBR_DEPTH - 1); -+ smp_read_barrier_depends(); - } while (CIRC_CNT(head, tail, JOBR_DEPTH) >= 1 && - jrp->entinfo[tail].desc_addr_dma == 0); - -@@ -233,57 +134,70 @@ - } - - /** -- * caam_jr_alloc() - Alloc a job ring for someone to use as needed. -- * -- * returns : pointer to the newly allocated physical -- * JobR dev can be written to if successful. -+ * caam_jr_register() - Alloc a ring for someone to use as needed. Returns -+ * an ordinal of the rings allocated, else returns -ENODEV if no rings -+ * are available. -+ * @ctrldev: points to the controller level dev (parent) that -+ * owns rings available for use. -+ * @dev: points to where a pointer to the newly allocated queue's -+ * dev can be written to if successful. - **/ --struct device *caam_jr_alloc(void) -+int caam_jr_register(struct device *ctrldev, struct device **rdev) - { -- struct caam_drv_private_jr *jrpriv, *min_jrpriv = NULL; -- struct device *dev = NULL; -- int min_tfm_cnt = INT_MAX; -- int tfm_cnt; -- -- spin_lock(&driver_data.jr_alloc_lock); -- -- if (list_empty(&driver_data.jr_list)) { -- spin_unlock(&driver_data.jr_alloc_lock); -- return ERR_PTR(-ENODEV); -- } -- -- list_for_each_entry(jrpriv, &driver_data.jr_list, list_node) { -- tfm_cnt = atomic_read(&jrpriv->tfm_count); -- if (tfm_cnt < min_tfm_cnt) { -- min_tfm_cnt = tfm_cnt; -- min_jrpriv = jrpriv; -+ struct caam_drv_private *ctrlpriv = dev_get_drvdata(ctrldev); -+ struct caam_drv_private_jr *jrpriv = NULL; -+ int ring; -+ -+ /* Lock, if free ring - assign, unlock */ -+ spin_lock(&ctrlpriv->jr_alloc_lock); -+ for (ring = 0; ring < ctrlpriv->total_jobrs; ring++) { -+ jrpriv = dev_get_drvdata(ctrlpriv->jrdev[ring]); -+ if (jrpriv->assign == JOBR_UNASSIGNED) { -+ jrpriv->assign = JOBR_ASSIGNED; -+ *rdev = ctrlpriv->jrdev[ring]; -+ spin_unlock(&ctrlpriv->jr_alloc_lock); -+ return ring; - } -- if (!min_tfm_cnt) -- break; - } - -- if (min_jrpriv) { -- atomic_inc(&min_jrpriv->tfm_count); -- dev = min_jrpriv->dev; -- } -- spin_unlock(&driver_data.jr_alloc_lock); -+ /* If assigned, write dev where caller needs it */ -+ spin_unlock(&ctrlpriv->jr_alloc_lock); -+ *rdev = NULL; - -- return dev; -+ return -ENODEV; - } --EXPORT_SYMBOL(caam_jr_alloc); -+EXPORT_SYMBOL(caam_jr_register); - - /** -- * caam_jr_free() - Free the Job Ring -- * @rdev - points to the dev that identifies the Job ring to -- * be released. -+ * caam_jr_deregister() - Deregister an API and release the queue. -+ * Returns 0 if OK, -EBUSY if queue still contains pending entries -+ * or unprocessed results at the time of the call -+ * @dev - points to the dev that identifies the queue to -+ * be released. - **/ --void caam_jr_free(struct device *rdev) -+int caam_jr_deregister(struct device *rdev) - { - struct caam_drv_private_jr *jrpriv = dev_get_drvdata(rdev); -+ struct caam_drv_private *ctrlpriv; -+ -+ /* Get the owning controller's private space */ -+ ctrlpriv = dev_get_drvdata(jrpriv->parentdev); -+ -+ /* -+ * Make sure ring empty before release -+ */ -+ if (rd_reg32(&jrpriv->rregs->outring_used) || -+ (rd_reg32(&jrpriv->rregs->inpring_avail) != JOBR_DEPTH)) -+ return -EBUSY; - -- atomic_dec(&jrpriv->tfm_count); -+ /* Release ring */ -+ spin_lock(&ctrlpriv->jr_alloc_lock); -+ jrpriv->assign = JOBR_UNASSIGNED; -+ spin_unlock(&ctrlpriv->jr_alloc_lock); -+ -+ return 0; - } --EXPORT_SYMBOL(caam_jr_free); -+EXPORT_SYMBOL(caam_jr_deregister); - - /** - * caam_jr_enqueue() - Enqueue a job descriptor head. Returns 0 if OK, -@@ -321,7 +235,7 @@ - struct caam_drv_private_jr *jrp = dev_get_drvdata(dev); - struct caam_jrentry_info *head_entry; - int head, tail, desc_size; -- dma_addr_t desc_dma; -+ dma_addr_t desc_dma, inpbusaddr; - - desc_size = (*desc & HDR_JD_LENGTH_MASK) * sizeof(u32); - desc_dma = dma_map_single(dev, desc, desc_size, DMA_TO_DEVICE); -@@ -330,6 +244,13 @@ - return -EIO; - } - -+ dma_sync_single_for_device(dev, desc_dma, desc_size, DMA_TO_DEVICE); -+ -+ inpbusaddr = rd_reg64(&jrp->rregs->inpring_base); -+ dma_sync_single_for_device(dev, inpbusaddr, -+ sizeof(dma_addr_t) * JOBR_DEPTH, -+ DMA_TO_DEVICE); -+ - spin_lock_bh(&jrp->inplock); - - head = jrp->head; -@@ -351,12 +272,18 @@ - - jrp->inpring[jrp->inp_ring_write_index] = desc_dma; - -+ dma_sync_single_for_device(dev, inpbusaddr, -+ sizeof(dma_addr_t) * JOBR_DEPTH, -+ DMA_TO_DEVICE); -+ - smp_wmb(); - - jrp->inp_ring_write_index = (jrp->inp_ring_write_index + 1) & - (JOBR_DEPTH - 1); - jrp->head = (head + 1) & (JOBR_DEPTH - 1); - -+ wmb(); -+ - wr_reg32(&jrp->rregs->inpring_jobadd, 1); - - spin_unlock_bh(&jrp->inplock); -@@ -365,6 +292,46 @@ - } - EXPORT_SYMBOL(caam_jr_enqueue); - -+static int caam_reset_hw_jr(struct device *dev) -+{ -+ struct caam_drv_private_jr *jrp = dev_get_drvdata(dev); -+ unsigned int timeout = 100000; -+ -+ /* -+ * mask interrupts since we are going to poll -+ * for reset completion status -+ */ -+ setbits32(&jrp->rregs->rconfig_lo, JRCFG_IMSK); -+ -+ /* initiate flush (required prior to reset) */ -+ wr_reg32(&jrp->rregs->jrcommand, JRCR_RESET); -+ while (((rd_reg32(&jrp->rregs->jrintstatus) & JRINT_ERR_HALT_MASK) == -+ JRINT_ERR_HALT_INPROGRESS) && --timeout) -+ cpu_relax(); -+ -+ if ((rd_reg32(&jrp->rregs->jrintstatus) & JRINT_ERR_HALT_MASK) != -+ JRINT_ERR_HALT_COMPLETE || timeout == 0) { -+ dev_err(dev, "failed to flush job ring %d\n", jrp->ridx); -+ return -EIO; -+ } -+ -+ /* initiate reset */ -+ timeout = 100000; -+ wr_reg32(&jrp->rregs->jrcommand, JRCR_RESET); -+ while ((rd_reg32(&jrp->rregs->jrcommand) & JRCR_RESET) && --timeout) -+ cpu_relax(); -+ -+ if (timeout == 0) { -+ dev_err(dev, "failed to reset job ring %d\n", jrp->ridx); -+ return -EIO; -+ } -+ -+ /* unmask interrupts */ -+ clrbits32(&jrp->rregs->rconfig_lo, JRCFG_IMSK); -+ -+ return 0; -+} -+ - /* - * Init JobR independent of platform property detection - */ -@@ -380,32 +347,34 @@ - - /* Connect job ring interrupt handler. */ - error = request_irq(jrp->irq, caam_jr_interrupt, IRQF_SHARED, -- dev_name(dev), dev); -+ "caam-jobr", dev); - if (error) { - dev_err(dev, "can't connect JobR %d interrupt (%d)\n", - jrp->ridx, jrp->irq); -- goto out_kill_deq; -+ irq_dispose_mapping(jrp->irq); -+ jrp->irq = 0; -+ return -EINVAL; - } - - error = caam_reset_hw_jr(dev); - if (error) -- goto out_free_irq; -+ return error; - -- error = -ENOMEM; - jrp->inpring = dma_alloc_coherent(dev, sizeof(dma_addr_t) * JOBR_DEPTH, - &inpbusaddr, GFP_KERNEL); -- if (!jrp->inpring) -- goto out_free_irq; - - jrp->outring = dma_alloc_coherent(dev, sizeof(struct jr_outentry) * - JOBR_DEPTH, &outbusaddr, GFP_KERNEL); -- if (!jrp->outring) -- goto out_free_inpring; - - jrp->entinfo = kzalloc(sizeof(struct caam_jrentry_info) * JOBR_DEPTH, - GFP_KERNEL); -- if (!jrp->entinfo) -- goto out_free_outring; -+ -+ if ((jrp->inpring == NULL) || (jrp->outring == NULL) || -+ (jrp->entinfo == NULL)) { -+ dev_err(dev, "can't allocate job rings for %d\n", -+ jrp->ridx); -+ return -ENOMEM; -+ } - - for (i = 0; i < JOBR_DEPTH; i++) - jrp->entinfo[i].desc_addr_dma = !0; -@@ -431,120 +400,123 @@ - (JOBR_INTC_COUNT_THLD << JRCFG_ICDCT_SHIFT) | - (JOBR_INTC_TIME_THLD << JRCFG_ICTT_SHIFT)); - -+ jrp->assign = JOBR_UNASSIGNED; - return 0; -+} - --out_free_outring: -- dma_free_coherent(dev, sizeof(struct jr_outentry) * JOBR_DEPTH, -- jrp->outring, outbusaddr); --out_free_inpring: -+/* -+ * Shutdown JobR independent of platform property code -+ */ -+int caam_jr_shutdown(struct device *dev) -+{ -+ struct caam_drv_private_jr *jrp = dev_get_drvdata(dev); -+ dma_addr_t inpbusaddr, outbusaddr; -+ int ret; -+ -+ ret = caam_reset_hw_jr(dev); -+ -+ tasklet_kill(&jrp->irqtask); -+ -+ /* Release interrupt */ -+ free_irq(jrp->irq, dev); -+ -+ /* Free rings */ -+ inpbusaddr = rd_reg64(&jrp->rregs->inpring_base); -+ outbusaddr = rd_reg64(&jrp->rregs->outring_base); - dma_free_coherent(dev, sizeof(dma_addr_t) * JOBR_DEPTH, - jrp->inpring, inpbusaddr); -- dev_err(dev, "can't allocate job rings for %d\n", jrp->ridx); --out_free_irq: -- free_irq(jrp->irq, dev); --out_kill_deq: -- tasklet_kill(&jrp->irqtask); -- return error; --} -+ dma_free_coherent(dev, sizeof(struct jr_outentry) * JOBR_DEPTH, -+ jrp->outring, outbusaddr); -+ kfree(jrp->entinfo); -+ of_device_unregister(jrp->jr_pdev); - -+ return ret; -+} - - /* -- * Probe routine for each detected JobR subsystem. -+ * Probe routine for each detected JobR subsystem. It assumes that -+ * property detection was picked up externally. - */ --static int caam_jr_probe(struct platform_device *pdev) -+int caam_jr_probe(struct platform_device *pdev, struct device_node *np, -+ int ring) - { -- struct device *jrdev; -- struct device_node *nprop; -- struct caam_job_ring __iomem *ctrl; -+ struct device *ctrldev, *jrdev; -+ struct platform_device *jr_pdev; -+ struct caam_drv_private *ctrlpriv; - struct caam_drv_private_jr *jrpriv; -- static int total_jobrs; -+ const __be32 *jroffset_addr; -+ u32 jroffset; - int error; - -- jrdev = &pdev->dev; -- jrpriv = devm_kmalloc(jrdev, sizeof(struct caam_drv_private_jr), -- GFP_KERNEL); -- if (!jrpriv) -+ ctrldev = &pdev->dev; -+ ctrlpriv = dev_get_drvdata(ctrldev); -+ -+ jrpriv = kmalloc(sizeof(struct caam_drv_private_jr), -+ GFP_KERNEL); -+ if (jrpriv == NULL) { -+ dev_err(ctrldev, "can't alloc private mem for job ring %d\n", -+ ring); - return -ENOMEM; -+ } -+ jrpriv->parentdev = ctrldev; /* point back to parent */ -+ jrpriv->ridx = ring; /* save ring identity relative to detection */ - -- dev_set_drvdata(jrdev, jrpriv); -+ /* -+ * Derive a pointer to the detected JobRs regs -+ * Driver has already iomapped the entire space, we just -+ * need to add in the offset to this JobR. Don't know if I -+ * like this long-term, but it'll run -+ */ -+ jroffset_addr = of_get_property(np, "reg", NULL); - -- /* save ring identity relative to detection */ -- jrpriv->ridx = total_jobrs++; -+ if (jroffset_addr == NULL) { -+ kfree(jrpriv); -+ return -EINVAL; -+ } - -- nprop = pdev->dev.of_node; -- /* Get configuration properties from device tree */ -- /* First, get register page */ -- ctrl = of_iomap(nprop, 0); -- if (!ctrl) { -- dev_err(jrdev, "of_iomap() failed\n"); -- return -ENOMEM; -+ /* -+ * Fix the endianness of this value read from the device -+ * tree if running on ARM. -+ */ -+ jroffset = be32_to_cpup(jroffset_addr); -+ -+ jrpriv->rregs = (struct caam_job_ring __iomem *)((void *)ctrlpriv->ctrl -+ + jroffset); -+ -+ /* Build a local dev for each detected queue */ -+ jr_pdev = of_platform_device_create(np, NULL, ctrldev); -+ if (jr_pdev == NULL) { -+ kfree(jrpriv); -+ return -EINVAL; - } - -- jrpriv->rregs = (struct caam_job_ring __force *)ctrl; -+ jrpriv->jr_pdev = jr_pdev; -+ jrdev = &jr_pdev->dev; -+ dev_set_drvdata(jrdev, jrpriv); -+ ctrlpriv->jrdev[ring] = jrdev; - - if (sizeof(dma_addr_t) == sizeof(u64)) -- if (of_device_is_compatible(nprop, "fsl,sec-v5.0-job-ring")) -- dma_set_mask_and_coherent(jrdev, DMA_BIT_MASK(40)); -+ if (of_device_is_compatible(np, "fsl,sec-v4.0-job-ring")) -+ dma_set_mask(jrdev, DMA_BIT_MASK(40)); - else -- dma_set_mask_and_coherent(jrdev, DMA_BIT_MASK(36)); -+ dma_set_mask(jrdev, DMA_BIT_MASK(36)); - else -- dma_set_mask_and_coherent(jrdev, DMA_BIT_MASK(32)); -+ dma_set_mask(jrdev, DMA_BIT_MASK(32)); - - /* Identify the interrupt */ -- jrpriv->irq = irq_of_parse_and_map(nprop, 0); -+ jrpriv->irq = of_irq_to_resource(np, 0, NULL); -+ if (jrpriv->irq <= 0) { -+ kfree(jrpriv); -+ return -EINVAL; -+ } - - /* Now do the platform independent part */ - error = caam_jr_init(jrdev); /* now turn on hardware */ - if (error) { -- irq_dispose_mapping(jrpriv->irq); -+ of_device_unregister(jr_pdev); -+ kfree(jrpriv); - return error; - } - -- jrpriv->dev = jrdev; -- spin_lock(&driver_data.jr_alloc_lock); -- list_add_tail(&jrpriv->list_node, &driver_data.jr_list); -- spin_unlock(&driver_data.jr_alloc_lock); -- -- atomic_set(&jrpriv->tfm_count, 0); -- -- return 0; --} -- --static struct of_device_id caam_jr_match[] = { -- { -- .compatible = "fsl,sec-v4.0-job-ring", -- }, -- { -- .compatible = "fsl,sec4.0-job-ring", -- }, -- {}, --}; --MODULE_DEVICE_TABLE(of, caam_jr_match); -- --static struct platform_driver caam_jr_driver = { -- .driver = { -- .name = "caam_jr", -- .of_match_table = caam_jr_match, -- }, -- .probe = caam_jr_probe, -- .remove = caam_jr_remove, --}; -- --static int __init jr_driver_init(void) --{ -- spin_lock_init(&driver_data.jr_alloc_lock); -- INIT_LIST_HEAD(&driver_data.jr_list); -- return platform_driver_register(&caam_jr_driver); --} -- --static void __exit jr_driver_exit(void) --{ -- platform_driver_unregister(&caam_jr_driver); -+ return error; - } -- --module_init(jr_driver_init); --module_exit(jr_driver_exit); -- --MODULE_LICENSE("GPL"); --MODULE_DESCRIPTION("FSL CAAM JR request backend"); --MODULE_AUTHOR("Freescale Semiconductor - NMG/STC"); -diff -Nur linux-4.1.3/drivers/crypto/caam/jr.h linux-xbian-imx6/drivers/crypto/caam/jr.h ---- linux-4.1.3/drivers/crypto/caam/jr.h 2015-07-21 19:10:33.000000000 +0200 -+++ linux-xbian-imx6/drivers/crypto/caam/jr.h 2015-07-27 23:13:04.213947410 +0200 -@@ -1,18 +1,22 @@ - /* - * CAAM public-level include definitions for the JobR backend - * -- * Copyright 2008-2011 Freescale Semiconductor, Inc. -+ * Copyright (C) 2008-2013 Freescale Semiconductor, Inc. - */ - - #ifndef JR_H - #define JR_H - - /* Prototypes for backend-level services exposed to APIs */ --struct device *caam_jr_alloc(void); --void caam_jr_free(struct device *rdev); -+int caam_jr_register(struct device *ctrldev, struct device **rdev); -+int caam_jr_deregister(struct device *rdev); - int caam_jr_enqueue(struct device *dev, u32 *desc, - void (*cbk)(struct device *dev, u32 *desc, u32 status, - void *areq), - void *areq); - -+extern int caam_jr_probe(struct platform_device *pdev, struct device_node *np, -+ int ring); -+extern int caam_jr_shutdown(struct device *dev); -+extern struct device *caam_get_jrdev(void); - #endif /* JR_H */ -diff -Nur linux-4.1.3/drivers/crypto/caam/Kconfig linux-xbian-imx6/drivers/crypto/caam/Kconfig ---- linux-4.1.3/drivers/crypto/caam/Kconfig 2015-07-21 19:10:33.000000000 +0200 -+++ linux-xbian-imx6/drivers/crypto/caam/Kconfig 2015-07-27 23:13:04.205975852 +0200 -@@ -1,32 +1,19 @@ - config CRYPTO_DEV_FSL_CAAM - tristate "Freescale CAAM-Multicore driver backend" -- depends on FSL_SOC -+ depends on FSL_SOC || ARCH_MXC - help - Enables the driver module for Freescale's Cryptographic Accelerator - and Assurance Module (CAAM), also known as the SEC version 4 (SEC4). -- This module creates job ring devices, and configures h/w -+ This module adds a job ring operation interface, and configures h/w - to operate as a DPAA component automatically, depending - on h/w feature availability. - - To compile this driver as a module, choose M here: the module - will be called caam. - --config CRYPTO_DEV_FSL_CAAM_JR -- tristate "Freescale CAAM Job Ring driver backend" -- depends on CRYPTO_DEV_FSL_CAAM -- default y -- help -- Enables the driver module for Job Rings which are part of -- Freescale's Cryptographic Accelerator -- and Assurance Module (CAAM). This module adds a job ring operation -- interface. -- -- To compile this driver as a module, choose M here: the module -- will be called caam_jr. -- - config CRYPTO_DEV_FSL_CAAM_RINGSIZE - int "Job Ring size" -- depends on CRYPTO_DEV_FSL_CAAM_JR -+ depends on CRYPTO_DEV_FSL_CAAM - range 2 9 - default "9" - help -@@ -44,7 +31,7 @@ - - config CRYPTO_DEV_FSL_CAAM_INTC - bool "Job Ring interrupt coalescing" -- depends on CRYPTO_DEV_FSL_CAAM_JR -+ depends on CRYPTO_DEV_FSL_CAAM - default n - help - Enable the Job Ring's interrupt coalescing feature. -@@ -75,7 +62,7 @@ - - config CRYPTO_DEV_FSL_CAAM_CRYPTO_API - tristate "Register algorithm implementations with the Crypto API" -- depends on CRYPTO_DEV_FSL_CAAM && CRYPTO_DEV_FSL_CAAM_JR -+ depends on CRYPTO_DEV_FSL_CAAM - default y - select CRYPTO_ALGAPI - select CRYPTO_AUTHENC -@@ -89,7 +76,7 @@ - - config CRYPTO_DEV_FSL_CAAM_AHASH_API - tristate "Register hash algorithm implementations with Crypto API" -- depends on CRYPTO_DEV_FSL_CAAM && CRYPTO_DEV_FSL_CAAM_JR -+ depends on CRYPTO_DEV_FSL_CAAM - default y - select CRYPTO_HASH - help -@@ -101,7 +88,7 @@ - - config CRYPTO_DEV_FSL_CAAM_RNG_API - tristate "Register caam device for hwrng API" -- depends on CRYPTO_DEV_FSL_CAAM && CRYPTO_DEV_FSL_CAAM_JR -+ depends on CRYPTO_DEV_FSL_CAAM - default y - select CRYPTO_RNG - select HW_RANDOM -@@ -112,6 +99,54 @@ - To compile this as a module, choose M here: the module - will be called caamrng. - -+config CRYPTO_DEV_FSL_CAAM_RNG_TEST -+ boolean "Test caam rng" -+ depends on CRYPTO_DEV_FSL_CAAM_RNG_API -+ default n -+ help -+ Selecting this will enable self-test for caam rng. -+ -+config CRYPTO_DEV_FSL_CAAM_SM -+ tristate "CAAM Secure Memory / Keystore API (EXPERIMENTAL)" -+ default n -+ help -+ Enables use of a prototype kernel-level Keystore API with CAAM -+ Secure Memory for insertion/extraction of bus-protected secrets. -+ -+config CRYPTO_DEV_FSL_CAAM_SM_SLOTSIZE -+ int "Size of each keystore slot in Secure Memory" -+ depends on CRYPTO_DEV_FSL_CAAM_SM -+ range 5 9 -+ default 7 -+ help -+ Select size of allocation units to divide Secure Memory pages into -+ (the size of a "slot" as referenced inside the API code). -+ Established as powers of two. -+ Examples: -+ 5 => 32 bytes -+ 6 => 64 bytes -+ 7 => 128 bytes -+ 8 => 256 bytes -+ 9 => 512 bytes -+ -+config CRYPTO_DEV_FSL_CAAM_SM_TEST -+ tristate "CAAM Secure Memory - Keystore Test/Example (EXPERIMENTAL)" -+ depends on CRYPTO_DEV_FSL_CAAM_SM -+ default n -+ help -+ Example thread to exercise the Keystore API and to verify that -+ stored and recovered secrets can be used for general purpose -+ encryption/decryption. -+ -+config CRYPTO_DEV_FSL_CAAM_SECVIO -+ tristate "CAAM/SNVS Security Violation Handler (EXPERIMENTAL)" -+ depends on CRYPTO_DEV_FSL_CAAM -+ default n -+ help -+ Enables installation of an interrupt handler with registrable -+ handler functions which can be specified to act on the consequences -+ of a security violation. -+ - config CRYPTO_DEV_FSL_CAAM_DEBUG - bool "Enable debug output in CAAM driver" - depends on CRYPTO_DEV_FSL_CAAM -@@ -119,3 +154,19 @@ - help - Selecting this will enable printing of various debug - information in the CAAM driver. -+ -+config CRYPTO_DEV_FSL_CAAM_KEYBLOB -+ tristate "Freescale CAAM memory keyblob driver backend" -+ depends on CRYPTO_DEV_FSL_CAAM -+ depends on CRYPTO_DEV_FSL_CAAM_JR -+ default y -+ help -+ Enables the driver module for Key Blob which are part of -+ Freescale's Cryptographic Accelerator -+ and Assurance Module (CAAM). This module adds a key blob operation -+ interface. -+ -+ To compile this driver as a module, choose M here: the module -+ will be called caam_keyblob. -+ -+ -diff -Nur linux-4.1.3/drivers/crypto/caam/key_gen.c linux-xbian-imx6/drivers/crypto/caam/key_gen.c ---- linux-4.1.3/drivers/crypto/caam/key_gen.c 2015-07-21 19:10:33.000000000 +0200 -+++ linux-xbian-imx6/drivers/crypto/caam/key_gen.c 2015-07-27 23:13:04.213947410 +0200 -@@ -1,7 +1,7 @@ - /* - * CAAM/SEC 4.x functions for handling key-generation jobs - * -- * Copyright 2008-2011 Freescale Semiconductor, Inc. -+ * Copyright (C) 2008-2013 Freescale Semiconductor, Inc. - * - */ - #include "compat.h" -@@ -19,8 +19,11 @@ - dev_err(dev, "%s %d: err 0x%x\n", __func__, __LINE__, err); - #endif - -- if (err) -- caam_jr_strstatus(dev, err); -+ if (err) { -+ char tmp[CAAM_ERROR_STR_MAX]; -+ -+ dev_err(dev, "%08x: %s\n", err, caam_jr_strstatus(tmp, err)); -+ } - - res->err = err; - -@@ -48,29 +51,24 @@ - u32 *desc; - struct split_key_result result; - dma_addr_t dma_addr_in, dma_addr_out; -- int ret = -ENOMEM; -+ int ret = 0; - - desc = kmalloc(CAAM_CMD_SZ * 6 + CAAM_PTR_SZ * 2, GFP_KERNEL | GFP_DMA); - if (!desc) { - dev_err(jrdev, "unable to allocate key input memory\n"); -- return ret; -+ return -ENOMEM; - } - -+ init_job_desc(desc, 0); -+ - dma_addr_in = dma_map_single(jrdev, (void *)key_in, keylen, - DMA_TO_DEVICE); - if (dma_mapping_error(jrdev, dma_addr_in)) { - dev_err(jrdev, "unable to map key input memory\n"); -- goto out_free; -+ kfree(desc); -+ return -ENOMEM; - } -- -- dma_addr_out = dma_map_single(jrdev, key_out, split_key_pad_len, -- DMA_FROM_DEVICE); -- if (dma_mapping_error(jrdev, dma_addr_out)) { -- dev_err(jrdev, "unable to map key output memory\n"); -- goto out_unmap_in; -- } -- -- init_job_desc(desc, 0); -+ dma_sync_single_for_device(jrdev, dma_addr_in, keylen, DMA_TO_DEVICE); - append_key(desc, dma_addr_in, keylen, CLASS_2 | KEY_DEST_CLASS_REG); - - /* Sets MDHA up into an HMAC-INIT */ -@@ -91,9 +89,9 @@ - LDST_CLASS_2_CCB | FIFOST_TYPE_SPLIT_KEK); - - #ifdef DEBUG -- print_hex_dump(KERN_ERR, "ctx.key@"__stringify(__LINE__)": ", -+ print_hex_dump(KERN_ERR, "ctx.key@"xstr(__LINE__)": ", - DUMP_PREFIX_ADDRESS, 16, 4, key_in, keylen, 1); -- print_hex_dump(KERN_ERR, "jobdesc@"__stringify(__LINE__)": ", -+ print_hex_dump(KERN_ERR, "jobdesc@"xstr(__LINE__)": ", - DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 1); - #endif - -@@ -106,12 +104,13 @@ - wait_for_completion_interruptible(&result.completion); - ret = result.err; - #ifdef DEBUG -- print_hex_dump(KERN_ERR, "ctx.key@"__stringify(__LINE__)": ", -+ print_hex_dump(KERN_ERR, "ctx.key@"xstr(__LINE__)": ", - DUMP_PREFIX_ADDRESS, 16, 4, key_out, - split_key_pad_len, 1); - #endif - } -- -+ dma_sync_single_for_cpu(jrdev, dma_addr_out, split_key_pad_len, -+ DMA_FROM_DEVICE); - dma_unmap_single(jrdev, dma_addr_out, split_key_pad_len, - DMA_FROM_DEVICE); - out_unmap_in: -diff -Nur linux-4.1.3/drivers/crypto/caam/Makefile linux-xbian-imx6/drivers/crypto/caam/Makefile ---- linux-4.1.3/drivers/crypto/caam/Makefile 2015-07-21 19:10:33.000000000 +0200 -+++ linux-xbian-imx6/drivers/crypto/caam/Makefile 2015-07-27 23:13:04.205975852 +0200 -@@ -1,15 +1,14 @@ - # - # Makefile for the CAAM backend and dependent components - # --ifeq ($(CONFIG_CRYPTO_DEV_FSL_CAAM_DEBUG), y) -- EXTRA_CFLAGS := -DDEBUG --endif - - obj-$(CONFIG_CRYPTO_DEV_FSL_CAAM) += caam.o --obj-$(CONFIG_CRYPTO_DEV_FSL_CAAM_JR) += caam_jr.o - obj-$(CONFIG_CRYPTO_DEV_FSL_CAAM_CRYPTO_API) += caamalg.o - obj-$(CONFIG_CRYPTO_DEV_FSL_CAAM_AHASH_API) += caamhash.o - obj-$(CONFIG_CRYPTO_DEV_FSL_CAAM_RNG_API) += caamrng.o -+obj-$(CONFIG_CRYPTO_DEV_FSL_CAAM_SM) += sm_store.o -+obj-$(CONFIG_CRYPTO_DEV_FSL_CAAM_SM_TEST) += sm_test.o -+obj-$(CONFIG_CRYPTO_DEV_FSL_CAAM_SECVIO) += secvio.o -+obj-$(CONFIG_CRYPTO_DEV_FSL_CAAM_KEYBLOB) += caam_keyblob.o - --caam-objs := ctrl.o --caam_jr-objs := jr.o key_gen.o error.o -+caam-objs := ctrl.o jr.o error.o key_gen.o -diff -Nur linux-4.1.3/drivers/crypto/caam/pdb.h linux-xbian-imx6/drivers/crypto/caam/pdb.h ---- linux-4.1.3/drivers/crypto/caam/pdb.h 2015-07-21 19:10:33.000000000 +0200 -+++ linux-xbian-imx6/drivers/crypto/caam/pdb.h 2015-07-27 23:13:04.213947410 +0200 -@@ -44,7 +44,6 @@ - #define PDBOPTS_ESP_IPHDRSRC 0x08 /* IP header comes from PDB (encap) */ - #define PDBOPTS_ESP_INCIPHDR 0x04 /* Prepend IP header to output frame */ - #define PDBOPTS_ESP_IPVSN 0x02 /* process IPv6 header */ --#define PDBOPTS_ESP_AOFL 0x04 /* adjust out frame len (decap, SEC>=5.3)*/ - #define PDBOPTS_ESP_TUNNEL 0x01 /* tunnel mode next-header byte */ - #define PDBOPTS_ESP_IPV6 0x02 /* ip header version is V6 */ - #define PDBOPTS_ESP_DIFFSERV 0x40 /* copy TOS/TC from inner iphdr */ -diff -Nur linux-4.1.3/drivers/crypto/caam/regs.h linux-xbian-imx6/drivers/crypto/caam/regs.h ---- linux-4.1.3/drivers/crypto/caam/regs.h 2015-07-21 19:10:33.000000000 +0200 -+++ linux-xbian-imx6/drivers/crypto/caam/regs.h 2015-07-27 23:13:04.213947410 +0200 -@@ -1,7 +1,7 @@ - /* - * CAAM hardware register-level view - * -- * Copyright 2008-2011 Freescale Semiconductor, Inc. -+ * Copyright (C) 2008-2013 Freescale Semiconductor, Inc. - */ - - #ifndef REGS_H -@@ -74,17 +74,22 @@ - #endif - #else - #ifdef __LITTLE_ENDIAN --#define wr_reg32(reg, data) __raw_writel(data, reg) --#define rd_reg32(reg) __raw_readl(reg) -+#define wr_reg32(reg, data) writel(data, reg) -+#define rd_reg32(reg) readl(reg) - #ifdef CONFIG_64BIT --#define wr_reg64(reg, data) __raw_writeq(data, reg) --#define rd_reg64(reg) __raw_readq(reg) -+#define wr_reg64(reg, data) writeq(data, reg) -+#define rd_reg64(reg) readq(reg) - #endif - #endif - #endif - -+#ifdef CONFIG_ARM -+/* These are common macros for Power, put here for ARMs */ -+#define setbits32(_addr, _v) writel((readl(_addr) | (_v)), (_addr)) -+#define clrbits32(_addr, _v) writel((readl(_addr) & ~(_v)), (_addr)) -+#endif -+ - #ifndef CONFIG_64BIT --#ifdef __BIG_ENDIAN - static inline void wr_reg64(u64 __iomem *reg, u64 data) - { - wr_reg32((u32 __iomem *)reg, (data & 0xffffffff00000000ull) >> 32); -@@ -96,21 +101,6 @@ - return (((u64)rd_reg32((u32 __iomem *)reg)) << 32) | - ((u64)rd_reg32((u32 __iomem *)reg + 1)); - } --#else --#ifdef __LITTLE_ENDIAN --static inline void wr_reg64(u64 __iomem *reg, u64 data) --{ -- wr_reg32((u32 __iomem *)reg + 1, (data & 0xffffffff00000000ull) >> 32); -- wr_reg32((u32 __iomem *)reg, data & 0x00000000ffffffffull); --} -- --static inline u64 rd_reg64(u64 __iomem *reg) --{ -- return (((u64)rd_reg32((u32 __iomem *)reg + 1)) << 32) | -- ((u64)rd_reg32((u32 __iomem *)reg)); --} --#endif --#endif - #endif - - /* -@@ -123,6 +113,98 @@ - } __packed; - - /* -+ * CHA version ID / instantiation bitfields -+ * Defined for use within cha_id in perfmon -+ * Note that the same shift/mask selectors can be used to pull out number -+ * of instantiated blocks within cha_num in perfmon, the locations are -+ * the same. -+ */ -+ -+/* Job Ring */ -+#define CHA_ID_JR_SHIFT 60 -+#define CHA_ID_JR_MASK (0xfull << CHA_ID_JR_SHIFT) -+ -+/* DEscriptor COntroller */ -+#define CHA_ID_DECO_SHIFT 56 -+#define CHA_ID_DECO_MASK (0xfull << CHA_ID_DECO_SHIFT) -+#define CHA_NUM_DECONUM_SHIFT 56 /* legacy definition */ -+#define CHA_NUM_DECONUM_MASK (0xfull << CHA_NUM_DECONUM_SHIFT) -+ -+/* ZUC-Authentication */ -+#define CHA_ID_ZA_SHIFT 44 -+#define CHA_ID_ZA_MASK (0xfull << CHA_ID_ZA_SHIFT) -+ -+/* ZUC-Encryption */ -+#define CHA_ID_ZE_SHIFT 40 -+#define CHA_ID_ZE_MASK (0xfull << CHA_ID_ZE_SHIFT) -+ -+/* SNOW f9 */ -+#define CHA_ID_SNW9_SHIFT 36 -+#define CHA_ID_SNW9_MASK (0xfull << CHA_ID_SNW9_SHIFT) -+ -+/* CRC */ -+#define CHA_ID_CRC_SHIFT 32 -+#define CHA_ID_CRC_MASK (0xfull << CHA_ID_CRC_SHIFT) -+ -+/* Public Key */ -+#define CHA_ID_PK_SHIFT 28 -+#define CHA_ID_PK_MASK (0xfull << CHA_ID_PK_SHIFT) -+ -+/* Kasumi */ -+#define CHA_ID_KAS_SHIFT 24 -+#define CHA_ID_KAS_MASK (0xfull << CHA_ID_KAS_SHIFT) -+ -+/* SNOW f8 */ -+#define CHA_ID_SNW8_SHIFT 20 -+#define CHA_ID_SNW8_MASK (0xfull << CHA_ID_SNW8_SHIFT) -+ -+/* -+ * Random Generator -+ * RNG4 = FIPS-verification-compliant, requires init kickstart for use -+ */ -+#define CHA_ID_RNG_SHIFT 16 -+#define CHA_ID_RNG_MASK (0xfull << CHA_ID_RNG_SHIFT) -+#define CHA_ID_RNG_A (0x1ull << CHA_ID_RNG_SHIFT) -+#define CHA_ID_RNG_B (0x2ull << CHA_ID_RNG_SHIFT) -+#define CHA_ID_RNG_C (0x3ull << CHA_ID_RNG_SHIFT) -+#define CHA_ID_RNG_4 (0x4ull << CHA_ID_RNG_SHIFT) -+ -+/* -+ * Message Digest -+ * LP256 = Low Power (MD5/SHA1/SHA224/SHA256 + HMAC) -+ * LP512 = Low Power (LP256 + SHA384/SHA512) -+ * HP = High Power (LP512 + SMAC) -+ */ -+#define CHA_ID_MD_SHIFT 12 -+#define CHA_ID_MD_MASK (0xfull << CHA_ID_MD_SHIFT) -+#define CHA_ID_MD_LP256 (0x0ull << CHA_ID_MD_SHIFT) -+#define CHA_ID_MD_LP512 (0x1ull << CHA_ID_MD_SHIFT) -+#define CHA_ID_MD_HP (0x2ull << CHA_ID_MD_SHIFT) -+ -+/* ARC4 Streamcipher */ -+#define CHA_ID_ARC4_SHIFT 8 -+#define CHA_ID_ARC4_MASK (0xfull << CHA_ID_ARC4_SHIFT) -+#define CHA_ID_ARC4_LP (0x0ull << CHA_ID_ARC4_SHIFT) -+#define CHA_ID_ARC4_HP (0x1ull << CHA_ID_ARC4_SHIFT) -+ -+/* DES Blockcipher Accelerator */ -+#define CHA_ID_DES_SHIFT 4 -+#define CHA_ID_DES_MASK (0xfull << CHA_ID_DES_SHIFT) -+ -+/* -+ * AES Blockcipher + Combo Mode Accelerator -+ * LP = Low Power (includes ECB/CBC/CFB128/OFB/CTR/CCM/CMAC/XCBC-MAC) -+ * HP = High Power (LP + CBCXCBC/CTRXCBC/XTS/GCM) -+ * DIFFPWR = ORed in if differential-power-analysis resistance implemented -+ */ -+#define CHA_ID_AES_SHIFT 0 -+#define CHA_ID_AES_MASK (0xfull << CHA_ID_AES_SHIFT) -+#define CHA_ID_AES_LP (0x3ull << CHA_ID_AES_SHIFT) -+#define CHA_ID_AES_HP (0x4ull << CHA_ID_AES_SHIFT) -+#define CHA_ID_AES_DIFFPWR (0x1ull << CHA_ID_AES_SHIFT) -+ -+ -+/* - * caam_perfmon - Performance Monitor/Secure Memory Status/ - * CAAM Global Status/Component Version IDs - * -@@ -130,45 +212,8 @@ - */ - - /* Number of DECOs */ --#define CHA_NUM_MS_DECONUM_SHIFT 24 --#define CHA_NUM_MS_DECONUM_MASK (0xfull << CHA_NUM_MS_DECONUM_SHIFT) -- --/* CHA Version IDs */ --#define CHA_ID_LS_AES_SHIFT 0 --#define CHA_ID_LS_AES_MASK (0xfull << CHA_ID_LS_AES_SHIFT) -- --#define CHA_ID_LS_DES_SHIFT 4 --#define CHA_ID_LS_DES_MASK (0xfull << CHA_ID_LS_DES_SHIFT) -- --#define CHA_ID_LS_ARC4_SHIFT 8 --#define CHA_ID_LS_ARC4_MASK (0xfull << CHA_ID_LS_ARC4_SHIFT) -- --#define CHA_ID_LS_MD_SHIFT 12 --#define CHA_ID_LS_MD_MASK (0xfull << CHA_ID_LS_MD_SHIFT) -- --#define CHA_ID_LS_RNG_SHIFT 16 --#define CHA_ID_LS_RNG_MASK (0xfull << CHA_ID_LS_RNG_SHIFT) -- --#define CHA_ID_LS_SNW8_SHIFT 20 --#define CHA_ID_LS_SNW8_MASK (0xfull << CHA_ID_LS_SNW8_SHIFT) -- --#define CHA_ID_LS_KAS_SHIFT 24 --#define CHA_ID_LS_KAS_MASK (0xfull << CHA_ID_LS_KAS_SHIFT) -- --#define CHA_ID_LS_PK_SHIFT 28 --#define CHA_ID_LS_PK_MASK (0xfull << CHA_ID_LS_PK_SHIFT) -- --#define CHA_ID_MS_CRC_SHIFT 0 --#define CHA_ID_MS_CRC_MASK (0xfull << CHA_ID_MS_CRC_SHIFT) -- --#define CHA_ID_MS_SNW9_SHIFT 4 --#define CHA_ID_MS_SNW9_MASK (0xfull << CHA_ID_MS_SNW9_SHIFT) -- --#define CHA_ID_MS_DECO_SHIFT 24 --#define CHA_ID_MS_DECO_MASK (0xfull << CHA_ID_MS_DECO_SHIFT) -- --#define CHA_ID_MS_JR_SHIFT 28 --#define CHA_ID_MS_JR_MASK (0xfull << CHA_ID_MS_JR_SHIFT) -+#define CHA_NUM_DECONUM_SHIFT 56 -+#define CHA_NUM_DECONUM_MASK (0xfull << CHA_NUM_DECONUM_SHIFT) - - struct sec_vid { - u16 ip_id; -@@ -176,6 +221,10 @@ - u8 min_rev; - }; - -+#define SEC_VID_IPID_SHIFT 16 -+#define SEC_VID_MAJ_SHIFT 8 -+#define SEC_VID_MAJ_MASK 0xFF00 -+ - struct caam_perfmon { - /* Performance Monitor Registers f00-f9f */ - u64 req_dequeued; /* PC_REQ_DEQ - Dequeued Requests */ -@@ -188,36 +237,89 @@ - u64 rsvd[13]; - - /* CAAM Hardware Instantiation Parameters fa0-fbf */ -- u32 cha_rev_ms; /* CRNR - CHA Rev No. Most significant half*/ -- u32 cha_rev_ls; /* CRNR - CHA Rev No. Least significant half*/ --#define CTPR_MS_QI_SHIFT 25 --#define CTPR_MS_QI_MASK (0x1ull << CTPR_MS_QI_SHIFT) --#define CTPR_MS_VIRT_EN_INCL 0x00000001 --#define CTPR_MS_VIRT_EN_POR 0x00000002 --#define CTPR_MS_PG_SZ_MASK 0x10 --#define CTPR_MS_PG_SZ_SHIFT 4 -- u32 comp_parms_ms; /* CTPR - Compile Parameters Register */ -- u32 comp_parms_ls; /* CTPR - Compile Parameters Register */ -- u64 rsvd1[2]; -+ u64 cha_rev; /* CRNR - CHA Revision Number */ -+#define CTPR_QI_SHIFT 57 -+#define CTPR_QI_MASK (0x1ull << CTPR_QI_SHIFT) -+ u64 comp_parms; /* CTPR - Compile Parameters Register */ -+ -+ /* Secure Memory State Visibility */ -+ u32 rsvd1; -+ u32 smstatus; /* Secure memory status */ -+ u32 rsvd2; -+ u32 smpartown; /* Secure memory partition owner */ - - /* CAAM Global Status fc0-fdf */ - u64 faultaddr; /* FAR - Fault Address */ - u32 faultliodn; /* FALR - Fault Address LIODN */ - u32 faultdetail; /* FADR - Fault Addr Detail */ -- u32 rsvd2; -+ u32 rsvd3; - u32 status; /* CSTA - CAAM Status */ -- u64 rsvd3; -+ u32 smpart; /* Secure Memory Partition Parameters */ -+ u32 smvid; /* Secure Memory Version ID */ - - /* Component Instantiation Parameters fe0-fff */ - u32 rtic_id; /* RVID - RTIC Version ID */ - u32 ccb_id; /* CCBVID - CCB Version ID */ -- u32 cha_id_ms; /* CHAVID - CHA Version ID Most Significant*/ -- u32 cha_id_ls; /* CHAVID - CHA Version ID Least Significant*/ -- u32 cha_num_ms; /* CHANUM - CHA Number Most Significant */ -- u32 cha_num_ls; /* CHANUM - CHA Number Least Significant*/ -- u32 caam_id_ms; /* CAAMVID - CAAM Version ID MS */ -- u32 caam_id_ls; /* CAAMVID - CAAM Version ID LS */ --}; -+ u64 cha_id; /* CHAVID - CHA Version ID */ -+ u64 cha_num; /* CHANUM - CHA Number */ -+ u64 caam_id; /* CAAMVID - CAAM Version ID */ -+}; -+ -+#define SMSTATUS_PART_SHIFT 28 -+#define SMSTATUS_PART_MASK (0xf << SMSTATUS_PART_SHIFT) -+#define SMSTATUS_PAGE_SHIFT 16 -+#define SMSTATUS_PAGE_MASK (0x7ff << SMSTATUS_PAGE_SHIFT) -+#define SMSTATUS_MID_SHIFT 8 -+#define SMSTATUS_MID_MASK (0x3f << SMSTATUS_MID_SHIFT) -+#define SMSTATUS_ACCERR_SHIFT 4 -+#define SMSTATUS_ACCERR_MASK (0xf << SMSTATUS_ACCERR_SHIFT) -+#define SMSTATUS_ACCERR_NONE 0 -+#define SMSTATUS_ACCERR_ALLOC 1 /* Page not allocated */ -+#define SMSTATUS_ACCESS_ID 2 /* Not granted by ID */ -+#define SMSTATUS_ACCESS_WRITE 3 /* Writes not allowed */ -+#define SMSTATUS_ACCESS_READ 4 /* Reads not allowed */ -+#define SMSTATUS_ACCESS_NONKEY 6 /* Non-key reads not allowed */ -+#define SMSTATUS_ACCESS_BLOB 9 /* Blob access not allowed */ -+#define SMSTATUS_ACCESS_DESCB 10 /* Descriptor Blob access spans pages */ -+#define SMSTATUS_ACCESS_NON_SM 11 /* Outside Secure Memory range */ -+#define SMSTATUS_ACCESS_XPAGE 12 /* Access crosses pages */ -+#define SMSTATUS_ACCESS_INITPG 13 /* Page still initializing */ -+#define SMSTATUS_STATE_SHIFT 0 -+#define SMSTATUS_STATE_MASK (0xf << SMSTATUS_STATE_SHIFT) -+#define SMSTATUS_STATE_RESET 0 -+#define SMSTATUS_STATE_INIT 1 -+#define SMSTATUS_STATE_NORMAL 2 -+#define SMSTATUS_STATE_FAIL 3 -+ -+/* up to 15 rings, 2 bits shifted by ring number */ -+#define SMPARTOWN_RING_SHIFT 2 -+#define SMPARTOWN_RING_MASK 3 -+#define SMPARTOWN_AVAILABLE 0 -+#define SMPARTOWN_NOEXIST 1 -+#define SMPARTOWN_UNAVAILABLE 2 -+#define SMPARTOWN_OURS 3 -+ -+/* Maximum number of pages possible */ -+#define SMPART_MAX_NUMPG_SHIFT 16 -+#define SMPART_MAX_NUMPG_MASK (0x3f << SMPART_MAX_NUMPG_SHIFT) -+ -+/* Maximum partition number */ -+#define SMPART_MAX_PNUM_SHIFT 12 -+#define SMPART_MAX_PNUM_MASK (0xf << SMPART_MAX_PNUM_SHIFT) -+ -+/* Highest possible page number */ -+#define SMPART_MAX_PG_SHIFT 0 -+#define SMPART_MAX_PG_MASK (0x3f << SMPART_MAX_PG_SHIFT) -+ -+/* Max size of a page */ -+#define SMVID_PG_SIZE_SHIFT 16 -+#define SMVID_PG_SIZE_MASK (0x7 << SMVID_PG_SIZE_SHIFT) -+ -+/* Major/Minor Version ID */ -+#define SMVID_MAJ_VERS_SHIFT 8 -+#define SMVID_MAJ_VERS (0xf << SMVID_MAJ_VERS_SHIFT) -+#define SMVID_MIN_VERS_SHIFT 0 -+#define SMVID_MIN_VERS (0xf << SMVID_MIN_VERS_SHIFT) - - /* LIODN programming for DMA configuration */ - #define MSTRID_LOCK_LIODN 0x80000000 -@@ -270,17 +372,7 @@ - - /* RNG4 TRNG test registers */ - struct rng4tst { --#define RTMCTL_PRGM 0x00010000 /* 1 -> program mode, 0 -> run mode */ --#define RTMCTL_SAMP_MODE_VON_NEUMANN_ES_SC 0 /* use von Neumann data in -- both entropy shifter and -- statistical checker */ --#define RTMCTL_SAMP_MODE_RAW_ES_SC 1 /* use raw data in both -- entropy shifter and -- statistical checker */ --#define RTMCTL_SAMP_MODE_VON_NEUMANN_ES_RAW_SC 2 /* use von Neumann data in -- entropy shifter, raw data -- in statistical checker */ --#define RTMCTL_SAMP_MODE_INVALID 3 /* invalid combination */ -+#define RTMCTL_PRGM 0x00010000 /* 1 -> program mode, 0 -> run mode */ - u32 rtmctl; /* misc. control register */ - u32 rtscmisc; /* statistical check misc. register */ - u32 rtpkrrng; /* poker range register */ -@@ -290,26 +382,22 @@ - }; - #define RTSDCTL_ENT_DLY_SHIFT 16 - #define RTSDCTL_ENT_DLY_MASK (0xffff << RTSDCTL_ENT_DLY_SHIFT) --#define RTSDCTL_ENT_DLY_MIN 3200 --#define RTSDCTL_ENT_DLY_MAX 12800 - u32 rtsdctl; /* seed control register */ - union { - u32 rtsblim; /* PRGM=1: sparse bit limit register */ - u32 rttotsam; /* PRGM=0: total samples register */ - }; - u32 rtfrqmin; /* frequency count min. limit register */ --#define RTFRQMAX_DISABLE (1 << 20) - union { - u32 rtfrqmax; /* PRGM=1: freq. count max. limit register */ - u32 rtfrqcnt; /* PRGM=0: freq. count register */ - }; - u32 rsvd1[40]; --#define RDSTA_SKVT 0x80000000 --#define RDSTA_SKVN 0x40000000 --#define RDSTA_IF0 0x00000001 --#define RDSTA_IF1 0x00000002 --#define RDSTA_IFMASK (RDSTA_IF1 | RDSTA_IF0) -- u32 rdsta; -+#define RDSTA_IF 0x00000003 /* state handle instantiated flags 0 and 1 */ -+#define RDSTA_SKVN 0x40000000 /* Secure Key Valid Non-Test mode */ -+#define RDSTA_SKVT 0x80000000 /* Secure Key Valid Test. non-test mode */ -+#define RDSTA_TF 0x00000300 /* State handle instantiated Test-mode */ -+ u32 rdsta; /* DRNG status register */ - u32 rsvd2[15]; - }; - -@@ -340,12 +428,9 @@ - /* Bus Access Configuration Section 010-11f */ - /* Read/Writable */ - struct masterid jr_mid[4]; /* JRxLIODNR - JobR LIODN setup */ -- u32 rsvd3[11]; -- u32 jrstart; /* JRSTART - Job Ring Start Register */ -+ u32 rsvd3[12]; - struct masterid rtic_mid[4]; /* RTICxLIODNR - RTIC LIODN setup */ -- u32 rsvd4[5]; -- u32 deco_rsr; /* DECORSR - Deco Request Source */ -- u32 rsvd11; -+ u32 rsvd4[7]; - u32 deco_rq; /* DECORR - DECO Request */ - struct partid deco_mid[5]; /* DECOxLIODNR - 1 per DECO */ - u32 rsvd5[22]; -@@ -386,11 +471,6 @@ - #define MCFGR_DMA_RESET 0x10000000 - #define MCFGR_LONG_PTR 0x00010000 /* Use >32-bit desc addressing */ - #define SCFGR_RDBENABLE 0x00000400 --#define SCFGR_VIRT_EN 0x00008000 --#define DECORR_RQD0ENABLE 0x00000001 /* Enable DECO0 for direct access */ --#define DECORSR_JR0 0x00000001 /* JR to supply TZ, SDID, ICID */ --#define DECORSR_VALID 0x80000000 --#define DECORR_DEN0 0x00010000 /* DECO0 available for access*/ - - /* AXI read cache control */ - #define MCFGR_ARCACHE_SHIFT 12 -@@ -407,12 +487,6 @@ - #define MCFGR_AXIPRI 0x00000008 /* Assert AXI priority sideband */ - #define MCFGR_BURST_64 0x00000001 /* Max burst size */ - --/* JRSTART register offsets */ --#define JRSTART_JR0_START 0x00000001 /* Start Job ring 0 */ --#define JRSTART_JR1_START 0x00000002 /* Start Job ring 1 */ --#define JRSTART_JR2_START 0x00000004 /* Start Job ring 2 */ --#define JRSTART_JR3_START 0x00000008 /* Start Job ring 3 */ -- - /* - * caam_job_ring - direct job ring setup - * 1-4 possible per instantiation, base + 1000/2000/3000/4000 -@@ -455,7 +529,18 @@ - u32 rsvd11; - u32 jrcommand; /* JRCRx - JobR command */ - -- u32 rsvd12[932]; -+ u32 rsvd12[33]; -+ -+ /* Secure Memory Configuration - if you have it */ -+ u32 sm_cmd; /* SMCJRx - Secure memory command */ -+ u32 rsvd13; -+ u32 sm_status; /* SMCSJRx - Secure memory status */ -+ u32 rsvd14; -+ u32 sm_perm; /* SMAPJRx - Secure memory access perms */ -+ u32 sm_group2; /* SMAP2JRx - Secure memory access group 2 */ -+ u32 sm_group1; /* SMAP1JRx - Secure memory access group 1 */ -+ -+ u32 rsvd15[891]; - - /* Performance Monitor f00-fff */ - struct caam_perfmon perfmon; -@@ -578,6 +663,62 @@ - - #define JRCR_RESET 0x01 - -+/* secure memory command */ -+#define SMC_PAGE_SHIFT 16 -+#define SMC_PAGE_MASK (0xffff << SMC_PAGE_SHIFT) -+#define SMC_PART_SHIFT 8 -+#define SMC_PART_MASK (0x0f << SMC_PART_SHIFT) -+#define SMC_CMD_SHIFT 0 -+#define SMC_CMD_MASK (0x0f << SMC_CMD_SHIFT) -+ -+#define SMC_CMD_ALLOC_PAGE 0x01 /* allocate page to this partition */ -+#define SMC_CMD_DEALLOC_PAGE 0x02 /* deallocate page from partition */ -+#define SMC_CMD_DEALLOC_PART 0x03 /* deallocate partition */ -+#define SMC_CMD_PAGE_INQUIRY 0x05 /* find partition associate with page */ -+ -+/* secure memory (command) status */ -+#define SMCS_PAGE_SHIFT 16 -+#define SMCS_PAGE_MASK (0x0fff << SMCS_PAGE_SHIFT) -+#define SMCS_CMDERR_SHIFT 14 -+#define SMCS_CMDERR_MASK (3 << SMCS_CMDERR_SHIFT) -+#define SMCS_ALCERR_SHIFT 12 -+#define SMCS_ALCERR_MASK (3 << SMCS_ALCERR_SHIFT) -+#define SMCS_PGOWN_SHIFT 6 -+#define SMCS_PGWON_MASK (3 << SMCS_PGOWN_SHIFT) -+#define SMCS_PART_SHIFT 0 -+#define SMCS_PART_MASK (0xf << SMCS_PART_SHIFT) -+ -+#define SMCS_CMDERR_NONE 0 -+#define SMCS_CMDERR_INCOMP 1 /* Command not yet complete */ -+#define SMCS_CMDERR_SECFAIL 2 /* Security failure occurred */ -+#define SMCS_CMDERR_OVERFLOW 3 /* Command overflow */ -+ -+#define SMCS_ALCERR_NONE 0 -+#define SMCS_ALCERR_PSPERR 1 /* Partion marked PSP (dealloc only) */ -+#define SMCS_ALCERR_PAGEAVAIL 2 /* Page not available */ -+#define SMCS_ALCERR_PARTOWN 3 /* Partition ownership error */ -+ -+#define SMCS_PGOWN_AVAIL 0 /* Page is available */ -+#define SMCS_PGOWN_NOEXIST 1 /* Page initializing or nonexistent */ -+#define SMCS_PGOWN_NOOWN 2 /* Page owned by another processor */ -+#define SMCS_PGOWN_OWNED 3 /* Page belongs to this processor */ -+ -+/* secure memory access permissions */ -+#define SMCS_PERM_KEYMOD_SHIFT 16 -+#define SMCA_PERM_KEYMOD_MASK (0xff << SMCS_PERM_KEYMOD_SHIFT) -+#define SMCA_PERM_CSP_ZERO 0x8000 /* Zero when deallocated or released */ -+#define SMCA_PERM_PSP_LOCK 0x4000 /* Part./pages can't be deallocated */ -+#define SMCA_PERM_PERM_LOCK 0x2000 /* Lock permissions */ -+#define SMCA_PERM_GRP_LOCK 0x1000 /* Lock access groups */ -+#define SMCA_PERM_RINGID_SHIFT 10 -+#define SMCA_PERM_RINGID_MASK (3 << SMCA_PERM_RINGID_SHIFT) -+#define SMCA_PERM_G2_BLOB 0x0080 /* Group 2 blob import/export */ -+#define SMCA_PERM_G2_WRITE 0x0020 /* Group 2 write */ -+#define SMCA_PERM_G2_READ 0x0010 /* Group 2 read */ -+#define SMCA_PERM_G1_BLOB 0x0008 /* Group 1... */ -+#define SMCA_PERM_G1_WRITE 0x0002 -+#define SMCA_PERM_G1_READ 0x0001 -+ - /* - * caam_assurance - Assurance Controller View - * base + 0x6000 padded out to 0x1000 -@@ -746,7 +887,6 @@ - u32 jr_ctl_hi; /* CxJRR - JobR Control Register @800 */ - u32 jr_ctl_lo; - u64 jr_descaddr; /* CxDADR - JobR Descriptor Address */ --#define DECO_OP_STATUS_HI_ERR_MASK 0xF00000FF - u32 op_status_hi; /* DxOPSTA - DECO Operation Status */ - u32 op_status_lo; - u32 rsvd24[2]; -@@ -760,21 +900,36 @@ - struct deco_sg_table sctr_tbl[4]; /* DxSTR - Scatter Tables */ - u32 rsvd29[48]; - u32 descbuf[64]; /* DxDESB - Descriptor buffer */ -- u32 rscvd30[193]; --#define DESC_DBG_DECO_STAT_HOST_ERR 0x00D00000 --#define DESC_DBG_DECO_STAT_VALID 0x80000000 --#define DESC_DBG_DECO_STAT_MASK 0x00F00000 -- u32 desc_dbg; /* DxDDR - DECO Debug Register */ -- u32 rsvd31[126]; --}; -- --#define DECO_JQCR_WHL 0x20000000 --#define DECO_JQCR_FOUR 0x10000000 -- --#define JR_BLOCK_NUMBER 1 --#define ASSURE_BLOCK_NUMBER 6 --#define QI_BLOCK_NUMBER 7 --#define DECO_BLOCK_NUMBER 8 --#define PG_SIZE_4K 0x1000 --#define PG_SIZE_64K 0x10000 -+ u32 rsvd30[320]; -+}; -+ -+/* -+ * Current top-level view of memory map is: -+ * -+ * 0x0000 - 0x0fff - CAAM Top-Level Control -+ * 0x1000 - 0x1fff - Job Ring 0 -+ * 0x2000 - 0x2fff - Job Ring 1 -+ * 0x3000 - 0x3fff - Job Ring 2 -+ * 0x4000 - 0x4fff - Job Ring 3 -+ * 0x5000 - 0x5fff - (unused) -+ * 0x6000 - 0x6fff - Assurance Controller -+ * 0x7000 - 0x7fff - Queue Interface -+ * 0x8000 - 0x8fff - DECO-CCB 0 -+ * 0x9000 - 0x9fff - DECO-CCB 1 -+ * 0xa000 - 0xafff - DECO-CCB 2 -+ * 0xb000 - 0xbfff - DECO-CCB 3 -+ * 0xc000 - 0xcfff - DECO-CCB 4 -+ * -+ * caam_full describes the full register view of CAAM if useful, -+ * although many configurations may choose to implement parts of -+ * the register map separately, in differing privilege regions -+ */ -+struct caam_full { -+ struct caam_ctrl __iomem ctrl; -+ struct caam_job_ring jr[4]; -+ u64 rsvd[512]; -+ struct caam_assurance assure; -+ struct caam_queue_if qi; -+}; -+ - #endif /* REGS_H */ -diff -Nur linux-4.1.3/drivers/crypto/caam/secvio.c linux-xbian-imx6/drivers/crypto/caam/secvio.c ---- linux-4.1.3/drivers/crypto/caam/secvio.c 1970-01-01 01:00:00.000000000 +0100 -+++ linux-xbian-imx6/drivers/crypto/caam/secvio.c 2015-07-27 23:13:04.213947410 +0200 -@@ -0,0 +1,290 @@ -+ -+/* -+ * SNVS Security Violation Handler -+ * Copyright (C) 2012-2015 Freescale Semiconductor, Inc., All Rights Reserved -+ */ -+ -+#include "compat.h" -+#include "intern.h" -+#include "secvio.h" -+#include "regs.h" -+ -+/* -+ * These names are associated with each violation handler. -+ * The source names were taken from MX6, and are based on recommendations -+ * for most common SoCs. -+ */ -+static const u8 *violation_src_name[] = { -+ "CAAM Internal Security Violation", -+ "JTAG Alarm", -+ "Watchdog", -+ "(reserved)", -+ "External Boot", -+ "External Tamper Detect", -+}; -+ -+/* These names help describe security monitor state for the console */ -+static const u8 *snvs_ssm_state_name[] = { -+ "init", -+ "hard fail", -+ "(undef:2)", -+ "soft fail", -+ "(undef:4)", -+ "(undef:5)", -+ "(undef:6)", -+ "(undef:7)", -+ "transition", -+ "check", -+ "(undef:10)", -+ "non-secure", -+ "(undef:12)", -+ "trusted", -+ "(undef:14)", -+ "secure", -+}; -+ -+/* Top-level security violation interrupt */ -+static irqreturn_t snvs_secvio_interrupt(int irq, void *snvsdev) -+{ -+ struct device *dev = snvsdev; -+ struct snvs_secvio_drv_private *svpriv = dev_get_drvdata(dev); -+ -+ /* Check the HP secvio status register */ -+ svpriv->irqcause = rd_reg32(&svpriv->svregs->hp.secvio_status) & -+ HP_SECVIOST_SECVIOMASK; -+ -+ if (!svpriv->irqcause) -+ return IRQ_NONE; -+ -+ /* Now ACK cause */ -+ setbits32(&svpriv->svregs->hp.secvio_status, svpriv->irqcause); -+ -+ /* And run deferred service */ -+ preempt_disable(); -+ tasklet_schedule(&svpriv->irqtask[smp_processor_id()]); -+ preempt_enable(); -+ -+ return IRQ_HANDLED; -+} -+ -+/* Deferred service handler. Tasklet arg is simply the SNVS dev */ -+static void snvs_secvio_dispatch(unsigned long indev) -+{ -+ struct device *dev = (struct device *)indev; -+ struct snvs_secvio_drv_private *svpriv = dev_get_drvdata(dev); -+ unsigned long flags; -+ int i; -+ -+ -+ /* Look through stored causes, call each handler if exists */ -+ for (i = 0; i < MAX_SECVIO_SOURCES; i++) -+ if (svpriv->irqcause & (1 << i)) { -+ spin_lock_irqsave(&svpriv->svlock, flags); -+ svpriv->intsrc[i].handler(dev, i, -+ svpriv->intsrc[i].ext); -+ spin_unlock_irqrestore(&svpriv->svlock, flags); -+ }; -+ -+ /* Re-enable now-serviced interrupts */ -+ setbits32(&svpriv->svregs->hp.secvio_intcfg, svpriv->irqcause); -+} -+ -+/* -+ * Default cause handler, used in lieu of an application-defined handler. -+ * All it does at this time is print a console message. It could force a halt. -+ */ -+static void snvs_secvio_default(struct device *dev, u32 cause, void *ext) -+{ -+ struct snvs_secvio_drv_private *svpriv = dev_get_drvdata(dev); -+ -+ dev_err(dev, "Unhandled Security Violation Interrupt %d = %s\n", -+ cause, svpriv->intsrc[cause].intname); -+} -+ -+/* -+ * Install an application-defined handler for a specified cause -+ * Arguments: -+ * - dev points to SNVS-owning device -+ * - cause interrupt source cause -+ * - handler application-defined handler, gets called with dev -+ * source cause, and locally-defined handler argument -+ * - cause_description points to a string to override the default cause -+ * name, this can be used as an alternate for error -+ * messages and such. If left NULL, the default -+ * description string is used. -+ * - ext pointer to any extra data needed by the handler. -+ */ -+int snvs_secvio_install_handler(struct device *dev, enum secvio_cause cause, -+ void (*handler)(struct device *dev, u32 cause, -+ void *ext), -+ u8 *cause_description, void *ext) -+{ -+ unsigned long flags; -+ struct snvs_secvio_drv_private *svpriv; -+ -+ svpriv = dev_get_drvdata(dev); -+ -+ if ((handler == NULL) || (cause > SECVIO_CAUSE_SOURCE_5)) -+ return -EINVAL; -+ -+ spin_lock_irqsave(&svpriv->svlock, flags); -+ svpriv->intsrc[cause].handler = handler; -+ if (cause_description != NULL) -+ svpriv->intsrc[cause].intname = cause_description; -+ if (ext != NULL) -+ svpriv->intsrc[cause].ext = ext; -+ spin_unlock_irqrestore(&svpriv->svlock, flags); -+ -+ return 0; -+} -+EXPORT_SYMBOL(snvs_secvio_install_handler); -+ -+/* -+ * Remove an application-defined handler for a specified cause (and, by -+ * implication, restore the "default". -+ * Arguments: -+ * - dev points to SNVS-owning device -+ * - cause interrupt source cause -+ */ -+int snvs_secvio_remove_handler(struct device *dev, enum secvio_cause cause) -+{ -+ unsigned long flags; -+ struct snvs_secvio_drv_private *svpriv; -+ -+ svpriv = dev_get_drvdata(dev); -+ -+ if (cause > SECVIO_CAUSE_SOURCE_5) -+ return -EINVAL; -+ -+ spin_lock_irqsave(&svpriv->svlock, flags); -+ svpriv->intsrc[cause].intname = violation_src_name[cause]; -+ svpriv->intsrc[cause].handler = snvs_secvio_default; -+ svpriv->intsrc[cause].ext = NULL; -+ spin_unlock_irqrestore(&svpriv->svlock, flags); -+ return 0; -+} -+EXPORT_SYMBOL(snvs_secvio_remove_handler); -+ -+static int snvs_secvio_remove(struct platform_device *pdev) -+{ -+ struct device *svdev; -+ struct snvs_secvio_drv_private *svpriv; -+ int i; -+ -+ svdev = &pdev->dev; -+ svpriv = dev_get_drvdata(svdev); -+ -+ /* Set all sources to nonfatal */ -+ wr_reg32(&svpriv->svregs->hp.secvio_intcfg, 0); -+ -+ /* Remove tasklets and release interrupt */ -+ for_each_possible_cpu(i) -+ tasklet_kill(&svpriv->irqtask[i]); -+ -+ free_irq(svpriv->irq, svdev); -+ iounmap(svpriv->svregs); -+ kfree(svpriv); -+ -+ return 0; -+} -+ -+static int snvs_secvio_probe(struct platform_device *pdev) -+{ -+ struct device *svdev; -+ struct snvs_secvio_drv_private *svpriv; -+ struct device_node *np, *npirq; -+ struct snvs_full __iomem *snvsregs; -+ int i, error; -+ u32 hpstate; -+ -+ svpriv = kzalloc(sizeof(struct snvs_secvio_drv_private), GFP_KERNEL); -+ if (!svpriv) -+ return -ENOMEM; -+ -+ svdev = &pdev->dev; -+ dev_set_drvdata(svdev, svpriv); -+ svpriv->pdev = pdev; -+ np = pdev->dev.of_node; -+ -+ npirq = of_find_compatible_node(NULL, NULL, "fsl,imx6q-caam-secvio"); -+ if (!npirq) { -+ dev_err(svdev, "can't identify secvio interrupt\n"); -+ kfree(svpriv); -+ return -EINVAL; -+ } -+ svpriv->irq = irq_of_parse_and_map(npirq, 0); -+ if (svpriv->irq <= 0) { -+ kfree(svpriv); -+ return -EINVAL; -+ } -+ -+ snvsregs = of_iomap(np, 0); -+ if (!snvsregs) { -+ dev_err(svdev, "register mapping failed\n"); -+ return -ENOMEM; -+ } -+ svpriv->svregs = (struct snvs_full __force *)snvsregs; -+ -+ /* Device data set up. Now init interrupt source descriptions */ -+ for (i = 0; i < MAX_SECVIO_SOURCES; i++) { -+ svpriv->intsrc[i].intname = violation_src_name[i]; -+ svpriv->intsrc[i].handler = snvs_secvio_default; -+ } -+ /* Connect main handler */ -+ for_each_possible_cpu(i) -+ tasklet_init(&svpriv->irqtask[i], snvs_secvio_dispatch, -+ (unsigned long)svdev); -+ -+ error = request_irq(svpriv->irq, snvs_secvio_interrupt, -+ IRQF_SHARED, "snvs-secvio", svdev); -+ if (error) { -+ dev_err(svdev, "can't connect secvio interrupt\n"); -+ irq_dispose_mapping(svpriv->irq); -+ svpriv->irq = 0; -+ iounmap(svpriv->svregs); -+ kfree(svpriv); -+ return -EINVAL; -+ } -+ -+ /* -+ * Configure all sources as fatal violations except LP section, -+ * source #5 (typically used as an external tamper detect), and -+ * source #3 (typically unused). Whenever the transition to -+ * secure mode has occurred, these will now be "fatal" violations -+ */ -+ wr_reg32(&svpriv->svregs->hp.secvio_intcfg, -+ HP_SECVIO_INTEN_SRC4 | HP_SECVIO_INTEN_SRC2 | -+ HP_SECVIO_INTEN_SRC1 | HP_SECVIO_INTEN_SRC0); -+ -+ hpstate = (rd_reg32(&svpriv->svregs->hp.status) & -+ HP_STATUS_SSM_ST_MASK) >> HP_STATUS_SSM_ST_SHIFT; -+ dev_info(svdev, "violation handlers armed - %s state\n", -+ snvs_ssm_state_name[hpstate]); -+ -+ return 0; -+} -+ -+static struct of_device_id snvs_secvio_match[] = { -+ { -+ .compatible = "fsl,imx6q-caam-snvs", -+ }, -+ {}, -+}; -+MODULE_DEVICE_TABLE(of, snvs_secvio_match); -+ -+static struct platform_driver snvs_secvio_driver = { -+ .driver = { -+ .name = "snvs-secvio", -+ .owner = THIS_MODULE, -+ .of_match_table = snvs_secvio_match, -+ }, -+ .probe = snvs_secvio_probe, -+ .remove = snvs_secvio_remove, -+}; -+ -+module_platform_driver(snvs_secvio_driver); -+ -+MODULE_LICENSE("Dual BSD/GPL"); -+MODULE_DESCRIPTION("FSL SNVS Security Violation Handler"); -+MODULE_AUTHOR("Freescale Semiconductor - MCU"); -+ -diff -Nur linux-4.1.3/drivers/crypto/caam/secvio.h linux-xbian-imx6/drivers/crypto/caam/secvio.h ---- linux-4.1.3/drivers/crypto/caam/secvio.h 1970-01-01 01:00:00.000000000 +0100 -+++ linux-xbian-imx6/drivers/crypto/caam/secvio.h 2015-07-27 23:13:04.213947410 +0200 -@@ -0,0 +1,66 @@ -+ -+/* -+ * CAAM Security Violation Handler -+ * Copyright (C) 2012-2014 Freescale Semiconductor, Inc., All Rights Reserved -+ */ -+ -+#ifndef SECVIO_H -+#define SECVIO_H -+ -+#include "snvsregs.h" -+ -+ -+/* -+ * Defines the published interfaces to install/remove application-specified -+ * handlers for catching violations -+ */ -+ -+#define MAX_SECVIO_SOURCES 6 -+ -+/* these are the untranslated causes */ -+enum secvio_cause { -+ SECVIO_CAUSE_SOURCE_0, -+ SECVIO_CAUSE_SOURCE_1, -+ SECVIO_CAUSE_SOURCE_2, -+ SECVIO_CAUSE_SOURCE_3, -+ SECVIO_CAUSE_SOURCE_4, -+ SECVIO_CAUSE_SOURCE_5 -+}; -+ -+/* These are common "recommended" cause definitions for most devices */ -+#define SECVIO_CAUSE_CAAM_VIOLATION SECVIO_CAUSE_SOURCE_0 -+#define SECVIO_CAUSE_JTAG_ALARM SECVIO_CAUSE_SOURCE_1 -+#define SECVIO_CAUSE_WATCHDOG SECVIO_CAUSE_SOURCE_2 -+#define SECVIO_CAUSE_EXTERNAL_BOOT SECVIO_CAUSE_SOURCE_4 -+#define SECVIO_CAUSE_TAMPER_DETECT SECVIO_CAUSE_SOURCE_5 -+ -+int snvs_secvio_install_handler(struct device *dev, enum secvio_cause cause, -+ void (*handler)(struct device *dev, u32 cause, -+ void *ext), -+ u8 *cause_description, void *ext); -+int snvs_secvio_remove_handler(struct device *dev, enum secvio_cause cause); -+ -+/* -+ * Private data definitions for the secvio "driver" -+ */ -+ -+struct secvio_int_src { -+ const u8 *intname; /* Points to a descriptive name for source */ -+ void *ext; /* Extended data to pass to the handler */ -+ void (*handler)(struct device *dev, u32 cause, void *ext); -+}; -+ -+struct snvs_secvio_drv_private { -+ struct platform_device *pdev; -+ spinlock_t svlock ____cacheline_aligned; -+ struct tasklet_struct irqtask[NR_CPUS]; -+ struct snvs_full __iomem *svregs; /* both HP and LP domains */ -+ int irq; -+ u32 irqcause; /* stashed cause of violation interrupt */ -+ -+ /* Registered handlers for each violation */ -+ struct secvio_int_src intsrc[MAX_SECVIO_SOURCES]; -+ -+}; -+ -+#endif /* SECVIO_H */ -diff -Nur linux-4.1.3/drivers/crypto/caam/sg_sw_sec4.h linux-xbian-imx6/drivers/crypto/caam/sg_sw_sec4.h ---- linux-4.1.3/drivers/crypto/caam/sg_sw_sec4.h 2015-07-21 19:10:33.000000000 +0200 -+++ linux-xbian-imx6/drivers/crypto/caam/sg_sw_sec4.h 2015-07-27 23:13:04.213947410 +0200 -@@ -1,7 +1,7 @@ - /* - * CAAM/SEC 4.x functions for using scatterlists in caam driver - * -- * Copyright 2008-2011 Freescale Semiconductor, Inc. -+ * Copyright (C) 2008-2013 Freescale Semiconductor, Inc. - * - */ - -@@ -91,13 +91,22 @@ - { - if (unlikely(chained)) { - int i; -+ struct scatterlist *tsg = sg; -+ -+ /* We use a local copy of the sg pointer to avoid moving the -+ * head of the list pointed to by sg as we wall the list. -+ */ - for (i = 0; i < nents; i++) { -- dma_map_sg(dev, sg, 1, dir); -- sg = sg_next(sg); -+ dma_map_sg(dev, tsg, 1, dir); -+ tsg = sg_next(tsg); - } - } else { - dma_map_sg(dev, sg, nents, dir); - } -+ -+ if ((dir == DMA_TO_DEVICE) || (dir == DMA_BIDIRECTIONAL)) -+ dma_sync_sg_for_device(dev, sg, nents, dir); -+ - return nents; - } - -@@ -105,6 +114,9 @@ - unsigned int nents, enum dma_data_direction dir, - bool chained) - { -+ if ((dir == DMA_FROM_DEVICE) || (dir == DMA_BIDIRECTIONAL)) -+ dma_sync_sg_for_cpu(dev, sg, nents, dir); -+ - if (unlikely(chained)) { - int i; - for (i = 0; i < nents; i++) { -@@ -116,3 +128,41 @@ - } - return nents; - } -+ -+/* Copy from len bytes of sg to dest, starting from beginning */ -+static inline void sg_copy(u8 *dest, struct scatterlist *sg, unsigned int len) -+{ -+ struct scatterlist *current_sg = sg; -+ int cpy_index = 0, next_cpy_index = current_sg->length; -+ -+ while (next_cpy_index < len) { -+ memcpy(dest + cpy_index, (u8 *) sg_virt(current_sg), -+ current_sg->length); -+ current_sg = sg_next(current_sg); -+ cpy_index = next_cpy_index; -+ next_cpy_index += current_sg->length; -+ } -+ if (cpy_index < len) -+ memcpy(dest + cpy_index, (u8 *) sg_virt(current_sg), -+ len - cpy_index); -+} -+ -+/* Copy sg data, from to_skip to end, to dest */ -+static inline void sg_copy_part(u8 *dest, struct scatterlist *sg, -+ int to_skip, unsigned int end) -+{ -+ struct scatterlist *current_sg = sg; -+ int sg_index, cpy_index; -+ -+ sg_index = current_sg->length; -+ while (sg_index <= to_skip) { -+ current_sg = sg_next(current_sg); -+ sg_index += current_sg->length; -+ } -+ cpy_index = sg_index - to_skip; -+ memcpy(dest, (u8 *) sg_virt(current_sg) + -+ current_sg->length - cpy_index, cpy_index); -+ current_sg = sg_next(current_sg); -+ if (end - sg_index) -+ sg_copy(dest + cpy_index, current_sg, end - sg_index); -+} -diff -Nur linux-4.1.3/drivers/crypto/caam/sm.h linux-xbian-imx6/drivers/crypto/caam/sm.h ---- linux-4.1.3/drivers/crypto/caam/sm.h 1970-01-01 01:00:00.000000000 +0100 -+++ linux-xbian-imx6/drivers/crypto/caam/sm.h 2015-07-27 23:13:04.213947410 +0200 -@@ -0,0 +1,88 @@ -+ -+/* -+ * CAAM Secure Memory/Keywrap API Definitions -+ * Copyright (C) 2008-2013 Freescale Semiconductor, Inc. -+ */ -+ -+#ifndef SM_H -+#define SM_H -+ -+ -+/* Storage access permissions */ -+#define SM_PERM_READ 0x01 -+#define SM_PERM_WRITE 0x02 -+#define SM_PERM_BLOB 0x03 -+ -+ -+/* Keystore maintenance functions */ -+void sm_init_keystore(struct device *dev); -+u32 sm_detect_keystore_units(struct device *dev); -+int sm_establish_keystore(struct device *dev, u32 unit); -+void sm_release_keystore(struct device *dev, u32 unit); -+void caam_sm_shutdown(struct platform_device *pdev); -+int caam_sm_example_init(struct platform_device *pdev); -+ -+/* Keystore accessor functions */ -+extern int sm_keystore_slot_alloc(struct device *dev, u32 unit, u32 size, -+ u32 *slot); -+extern int sm_keystore_slot_dealloc(struct device *dev, u32 unit, u32 slot); -+extern int sm_keystore_slot_load(struct device *dev, u32 unit, u32 slot, -+ const u8 *key_data, u32 key_length); -+extern int sm_keystore_slot_read(struct device *dev, u32 unit, u32 slot, -+ u32 key_length, u8 *key_data); -+extern int sm_keystore_slot_encapsulate(struct device *dev, u32 unit, -+ u32 inslot, u32 outslot, u16 secretlen, -+ u8 *keymod, u16 keymodlen); -+extern int sm_keystore_slot_decapsulate(struct device *dev, u32 unit, -+ u32 inslot, u32 outslot, u16 secretlen, -+ u8 *keymod, u16 keymodlen); -+ -+/* Data structure to hold per-slot information */ -+struct keystore_data_slot_info { -+ u8 allocated; /* Track slot assignments */ -+ u32 key_length; /* Size of the key */ -+}; -+ -+/* Data structure to hold keystore information */ -+struct keystore_data { -+ void *base_address; /* Base of the Secure Partition */ -+ u32 slot_count; /* Number of slots in the keystore */ -+ struct keystore_data_slot_info *slot; /* Per-slot information */ -+}; -+ -+/* store the detected attributes of a secure memory page */ -+struct sm_page_descriptor { -+ u16 phys_pagenum; /* may be discontiguous */ -+ u16 own_part; /* Owning partition */ -+ void *pg_base; /* Calculated virtual address */ -+ struct keystore_data *ksdata; -+}; -+ -+struct caam_drv_private_sm { -+ struct device *parentdev; /* this ends up as the controller */ -+ struct device *smringdev; /* ring that owns this instance */ -+ spinlock_t kslock ____cacheline_aligned; -+ -+ /* Default parameters for geometry */ -+ u32 max_pages; /* maximum pages this instance can support */ -+ u32 top_partition; /* highest partition number in this instance */ -+ u32 top_page; /* highest page number in this instance */ -+ u32 page_size; /* page size */ -+ u32 slot_size; /* selected size of each storage block */ -+ -+ /* Partition/Page Allocation Map */ -+ u32 localpages; /* Number of pages we can access */ -+ struct sm_page_descriptor *pagedesc; /* Allocated per-page */ -+ -+ /* Installed handlers for keystore access */ -+ int (*data_init)(struct device *dev, u32 unit); -+ void (*data_cleanup)(struct device *dev, u32 unit); -+ int (*slot_alloc)(struct device *dev, u32 unit, u32 size, u32 *slot); -+ int (*slot_dealloc)(struct device *dev, u32 unit, u32 slot); -+ void *(*slot_get_address)(struct device *dev, u32 unit, u32 handle); -+ u32 (*slot_get_base)(struct device *dev, u32 unit, u32 handle); -+ u32 (*slot_get_offset)(struct device *dev, u32 unit, u32 handle); -+ u32 (*slot_get_slot_size)(struct device *dev, u32 unit, u32 handle); -+}; -+ -+#endif /* SM_H */ -diff -Nur linux-4.1.3/drivers/crypto/caam/sm_store.c linux-xbian-imx6/drivers/crypto/caam/sm_store.c ---- linux-4.1.3/drivers/crypto/caam/sm_store.c 1970-01-01 01:00:00.000000000 +0100 -+++ linux-xbian-imx6/drivers/crypto/caam/sm_store.c 2015-07-27 23:13:04.213947410 +0200 -@@ -0,0 +1,896 @@ -+ -+/* -+ * CAAM Secure Memory Storage Interface -+ * Copyright (C) 2008-2013 Freescale Semiconductor, Inc. -+ * -+ * Loosely based on the SHW Keystore API for SCC/SCC2 -+ * Experimental implementation and NOT intended for upstream use. Expect -+ * this interface to be amended significantly in the future once it becomes -+ * integrated into live applications. -+ * -+ * Known issues: -+ * -+ * - Executes one instance of an secure memory "driver". This is tied to the -+ * fact that job rings can't run as standalone instances in the present -+ * configuration. -+ * -+ * - It does not expose a userspace interface. The value of a userspace -+ * interface for access to secrets is a point for further architectural -+ * discussion. -+ * -+ * - Partition/permission management is not part of this interface. It -+ * depends on some level of "knowledge" agreed upon between bootloader, -+ * provisioning applications, and OS-hosted software (which uses this -+ * driver). -+ * -+ * - No means of identifying the location or purpose of secrets managed by -+ * this interface exists; "slot location" and format of a given secret -+ * needs to be agreed upon between bootloader, provisioner, and OS-hosted -+ * application. -+ */ -+ -+#include "compat.h" -+#include "regs.h" -+#include "jr.h" -+#include "desc.h" -+#include "intern.h" -+#include "error.h" -+#include "sm.h" -+ -+#ifdef SM_DEBUG_CONT -+void sm_show_page(struct device *dev, struct sm_page_descriptor *pgdesc) -+{ -+ struct caam_drv_private_sm *smpriv = dev_get_drvdata(dev); -+ u32 i, *smdata; -+ -+ dev_info(dev, "physical page %d content at 0x%08x\n", -+ pgdesc->phys_pagenum, pgdesc->pg_base); -+ smdata = pgdesc->pg_base; -+ for (i = 0; i < (smpriv->page_size / sizeof(u32)); i += 4) -+ dev_info(dev, "[0x%08x] 0x%08x 0x%08x 0x%08x 0x%08x\n", -+ (u32)&smdata[i], smdata[i], smdata[i+1], smdata[i+2], -+ smdata[i+3]); -+} -+#endif -+ -+/* -+ * Construct a secure memory blob encapsulation job descriptor -+ * -+ * - desc pointer to hold new (to be allocated) pointer to the generated -+ * descriptor for later use. Calling thread can kfree the -+ * descriptor after execution. -+ * - keymod Physical pointer to key modifier (contiguous piece). -+ * - keymodsz Size of key modifier in bytes (should normally be 8). -+ * - secretbuf Physical pointer (within an accessible secure memory page) -+ * of the secret to be encapsulated. -+ * - outbuf Physical pointer (within an accessible secure memory page) -+ * of the encapsulated output. This will be larger than the -+ * input secret because of the added encapsulation data. -+ * - secretsz Size of input secret, in bytes. -+ * - auth If nonzero, use AES-CCM for encapsulation, else use ECB -+ * -+ * Note: this uses 32-bit pointers at present -+ */ -+#define INITIAL_DESCSZ 16 /* size of tmp buffer for descriptor const. */ -+static int blob_encap_desc(u32 **desc, dma_addr_t keymod, u16 keymodsz, -+ dma_addr_t secretbuf, dma_addr_t outbuf, -+ u16 secretsz, bool auth) -+{ -+ u32 *tdesc, tmpdesc[INITIAL_DESCSZ]; -+ u16 dsize, idx; -+ -+ memset(tmpdesc, 0, INITIAL_DESCSZ * sizeof(u32)); -+ idx = 1; -+ -+ /* Load key modifier */ -+ tmpdesc[idx++] = CMD_LOAD | LDST_CLASS_2_CCB | LDST_SRCDST_BYTE_KEY | -+ ((12 << LDST_OFFSET_SHIFT) & LDST_OFFSET_MASK) | -+ (keymodsz & LDST_LEN_MASK); -+ -+ tmpdesc[idx++] = (u32)keymod; -+ -+ /* Encapsulate to secure memory */ -+ tmpdesc[idx++] = CMD_SEQ_IN_PTR | secretsz; -+ tmpdesc[idx++] = (u32)secretbuf; -+ -+ /* Add space for BKEK and MAC tag */ -+ tmpdesc[idx++] = CMD_SEQ_IN_PTR | (secretsz + (32 + 16)); -+ -+ tmpdesc[idx++] = (u32)outbuf; -+ tmpdesc[idx] = CMD_OPERATION | OP_TYPE_ENCAP_PROTOCOL | OP_PCLID_BLOB | -+ OP_PCL_BLOB_PTXT_SECMEM; -+ if (auth) -+ tmpdesc[idx] |= OP_PCL_BLOB_EKT; -+ -+ idx++; -+ tmpdesc[0] = CMD_DESC_HDR | HDR_ONE | (idx & HDR_DESCLEN_MASK); -+ dsize = idx * sizeof(u32); -+ -+ tdesc = kmalloc(dsize, GFP_KERNEL | GFP_DMA); -+ if (tdesc == NULL) -+ return 0; -+ -+ memcpy(tdesc, tmpdesc, dsize); -+ *desc = tdesc; -+ return dsize; -+} -+ -+/* -+ * Construct a secure memory blob decapsulation job descriptor -+ * -+ * - desc pointer to hold new (to be allocated) pointer to the generated -+ * descriptor for later use. Calling thread can kfree the -+ * descriptor after execution. -+ * - keymod Physical pointer to key modifier (contiguous piece). -+ * - keymodsz Size of key modifier in bytes (should normally be 16). -+ * - blobbuf Physical pointer (within an accessible secure memory page) -+ * of the blob to be decapsulated. -+ * - outbuf Physical pointer (within an accessible secure memory page) -+ * of the decapsulated output. -+ * - secretsz Size of input blob, in bytes. -+ * - auth If nonzero, assume AES-CCM for decapsulation, else use ECB -+ * -+ * Note: this uses 32-bit pointers at present -+ */ -+static int blob_decap_desc(u32 **desc, dma_addr_t keymod, u16 keymodsz, -+ dma_addr_t blobbuf, dma_addr_t outbuf, -+ u16 blobsz, bool auth) -+{ -+ u32 *tdesc, tmpdesc[INITIAL_DESCSZ]; -+ u16 dsize, idx; -+ -+ memset(tmpdesc, 0, INITIAL_DESCSZ * sizeof(u32)); -+ idx = 1; -+ -+ /* Load key modifier */ -+ tmpdesc[idx++] = CMD_LOAD | LDST_CLASS_2_CCB | LDST_SRCDST_BYTE_KEY | -+ ((12 << LDST_OFFSET_SHIFT) & LDST_OFFSET_MASK) | -+ (keymodsz & LDST_LEN_MASK); -+ -+ tmpdesc[idx++] = (u32)keymod; -+ -+ /* Compensate BKEK + MAC tag */ -+ tmpdesc[idx++] = CMD_SEQ_IN_PTR | (blobsz + 32 + 16); -+ -+ tmpdesc[idx++] = (u32)blobbuf; -+ tmpdesc[idx++] = CMD_SEQ_OUT_PTR | blobsz; -+ tmpdesc[idx++] = (u32)outbuf; -+ -+ /* Decapsulate from secure memory partition to black blob */ -+ tmpdesc[idx] = CMD_OPERATION | OP_TYPE_DECAP_PROTOCOL | OP_PCLID_BLOB | -+ OP_PCL_BLOB_PTXT_SECMEM | OP_PCL_BLOB_BLACK; -+ if (auth) -+ tmpdesc[idx] |= OP_PCL_BLOB_EKT; -+ -+ idx++; -+ tmpdesc[0] = CMD_DESC_HDR | HDR_ONE | (idx & HDR_DESCLEN_MASK); -+ dsize = idx * sizeof(u32); -+ -+ tdesc = kmalloc(dsize, GFP_KERNEL | GFP_DMA); -+ if (tdesc == NULL) -+ return 0; -+ -+ memcpy(tdesc, tmpdesc, dsize); -+ *desc = tdesc; -+ return dsize; -+} -+ -+/* -+ * Pseudo-synchronous ring access functions for carrying out key -+ * encapsulation and decapsulation -+ */ -+ -+struct sm_key_job_result { -+ int error; -+ struct completion completion; -+}; -+ -+void sm_key_job_done(struct device *dev, u32 *desc, u32 err, void *context) -+{ -+ struct sm_key_job_result *res = context; -+ -+ res->error = err; /* save off the error for postprocessing */ -+ complete(&res->completion); /* mark us complete */ -+} -+ -+static int sm_key_job(struct device *ksdev, u32 *jobdesc) -+{ -+ struct sm_key_job_result testres; -+ struct caam_drv_private_sm *kspriv; -+ int rtn = 0; -+ -+ kspriv = dev_get_drvdata(ksdev); -+ -+ init_completion(&testres.completion); -+ -+ rtn = caam_jr_enqueue(kspriv->smringdev, jobdesc, sm_key_job_done, -+ &testres); -+ if (!rtn) { -+ wait_for_completion_interruptible(&testres.completion); -+ rtn = testres.error; -+ } -+ return rtn; -+} -+ -+/* -+ * Following section establishes the default methods for keystore access -+ * They are NOT intended for use external to this module -+ * -+ * In the present version, these are the only means for the higher-level -+ * interface to deal with the mechanics of accessing the phyiscal keystore -+ */ -+ -+ -+int slot_alloc(struct device *dev, u32 unit, u32 size, u32 *slot) -+{ -+ struct caam_drv_private_sm *smpriv = dev_get_drvdata(dev); -+ struct keystore_data *ksdata = smpriv->pagedesc[unit].ksdata; -+ u32 i; -+#ifdef SM_DEBUG -+ dev_info(dev, "slot_alloc(): requesting slot for %d bytes\n", size); -+#endif -+ -+ if (size > smpriv->slot_size) -+ return -EKEYREJECTED; -+ -+ for (i = 0; i < ksdata->slot_count; i++) { -+ if (ksdata->slot[i].allocated == 0) { -+ ksdata->slot[i].allocated = 1; -+ (*slot) = i; -+#ifdef SM_DEBUG -+ dev_info(dev, "slot_alloc(): new slot %d allocated\n", -+ *slot); -+#endif -+ return 0; -+ } -+ } -+ -+ return -ENOSPC; -+} -+EXPORT_SYMBOL(slot_alloc); -+ -+int slot_dealloc(struct device *dev, u32 unit, u32 slot) -+{ -+ struct caam_drv_private_sm *smpriv = dev_get_drvdata(dev); -+ struct keystore_data *ksdata = smpriv->pagedesc[unit].ksdata; -+ u8 __iomem *slotdata; -+ -+#ifdef SM_DEBUG -+ dev_info(dev, "slot_dealloc(): releasing slot %d\n", slot); -+#endif -+ if (slot >= ksdata->slot_count) -+ return -EINVAL; -+ slotdata = ksdata->base_address + slot * smpriv->slot_size; -+ -+ if (ksdata->slot[slot].allocated == 1) { -+ /* Forcibly overwrite the data from the keystore */ -+ memset(ksdata->base_address + slot * smpriv->slot_size, 0, -+ smpriv->slot_size); -+ -+ ksdata->slot[slot].allocated = 0; -+#ifdef SM_DEBUG -+ dev_info(dev, "slot_dealloc(): slot %d released\n", slot); -+#endif -+ return 0; -+ } -+ -+ return -EINVAL; -+} -+EXPORT_SYMBOL(slot_dealloc); -+ -+void *slot_get_address(struct device *dev, u32 unit, u32 slot) -+{ -+ struct caam_drv_private_sm *smpriv = dev_get_drvdata(dev); -+ struct keystore_data *ksdata = smpriv->pagedesc[unit].ksdata; -+ -+ if (slot >= ksdata->slot_count) -+ return NULL; -+ -+#ifdef SM_DEBUG -+ dev_info(dev, "slot_get_address(): slot %d is 0x%08x\n", slot, -+ (u32)ksdata->base_address + slot * smpriv->slot_size); -+#endif -+ -+ return ksdata->base_address + slot * smpriv->slot_size; -+} -+ -+u32 slot_get_base(struct device *dev, u32 unit, u32 slot) -+{ -+ struct caam_drv_private_sm *smpriv = dev_get_drvdata(dev); -+ struct keystore_data *ksdata = smpriv->pagedesc[unit].ksdata; -+ -+ /* -+ * There could potentially be more than one secure partition object -+ * associated with this keystore. For now, there is just one. -+ */ -+ -+ (void)slot; -+ -+#ifdef SM_DEBUG -+ dev_info(dev, "slot_get_base(): slot %d = 0x%08x\n", -+ slot, (u32)ksdata->base_address); -+#endif -+ -+ return (u32)(ksdata->base_address); -+} -+ -+u32 slot_get_offset(struct device *dev, u32 unit, u32 slot) -+{ -+ struct caam_drv_private_sm *smpriv = dev_get_drvdata(dev); -+ struct keystore_data *ksdata = smpriv->pagedesc[unit].ksdata; -+ -+ if (slot >= ksdata->slot_count) -+ return -EINVAL; -+ -+#ifdef SM_DEBUG -+ dev_info(dev, "slot_get_offset(): slot %d = %d\n", slot, -+ slot * smpriv->slot_size); -+#endif -+ -+ return slot * smpriv->slot_size; -+} -+ -+u32 slot_get_slot_size(struct device *dev, u32 unit, u32 slot) -+{ -+ struct caam_drv_private_sm *smpriv = dev_get_drvdata(dev); -+ -+ -+#ifdef SM_DEBUG -+ dev_info(dev, "slot_get_slot_size(): slot %d = %d\n", slot, -+ smpriv->slot_size); -+#endif -+ /* All slots are the same size in the default implementation */ -+ return smpriv->slot_size; -+} -+ -+ -+ -+int kso_init_data(struct device *dev, u32 unit) -+{ -+ struct caam_drv_private_sm *smpriv = dev_get_drvdata(dev); -+ int retval = -EINVAL; -+ struct keystore_data *keystore_data = NULL; -+ u32 slot_count; -+ u32 keystore_data_size; -+ -+ /* -+ * Calculate the required size of the keystore data structure, based -+ * on the number of keys that can fit in the partition. -+ */ -+ slot_count = smpriv->page_size / smpriv->slot_size; -+#ifdef SM_DEBUG -+ dev_info(dev, "kso_init_data: %d slots initializing\n", slot_count); -+#endif -+ -+ keystore_data_size = sizeof(struct keystore_data) + -+ slot_count * -+ sizeof(struct keystore_data_slot_info); -+ -+ keystore_data = kzalloc(keystore_data_size, GFP_KERNEL); -+ -+ if (keystore_data == NULL) { -+ retval = -ENOSPC; -+ goto out; -+ } -+ -+#ifdef SM_DEBUG -+ dev_info(dev, "kso_init_data: keystore data size = %d\n", -+ keystore_data_size); -+#endif -+ -+ /* -+ * Place the slot information structure directly after the keystore data -+ * structure. -+ */ -+ keystore_data->slot = (struct keystore_data_slot_info *) -+ (keystore_data + 1); -+ keystore_data->slot_count = slot_count; -+ -+ smpriv->pagedesc[unit].ksdata = keystore_data; -+ smpriv->pagedesc[unit].ksdata->base_address = -+ smpriv->pagedesc[unit].pg_base; -+ -+ retval = 0; -+ -+out: -+ if (retval != 0) -+ if (keystore_data != NULL) -+ kfree(keystore_data); -+ -+ -+ return retval; -+} -+ -+void kso_cleanup_data(struct device *dev, u32 unit) -+{ -+ struct caam_drv_private_sm *smpriv = dev_get_drvdata(dev); -+ struct keystore_data *keystore_data = NULL; -+ -+ if (smpriv->pagedesc[unit].ksdata != NULL) -+ keystore_data = smpriv->pagedesc[unit].ksdata; -+ -+ /* Release the allocated keystore management data */ -+ kfree(smpriv->pagedesc[unit].ksdata); -+ -+ return; -+} -+ -+ -+ -+/* -+ * Keystore management section -+ */ -+ -+void sm_init_keystore(struct device *dev) -+{ -+ struct caam_drv_private_sm *smpriv = dev_get_drvdata(dev); -+ -+ smpriv->data_init = kso_init_data; -+ smpriv->data_cleanup = kso_cleanup_data; -+ smpriv->slot_alloc = slot_alloc; -+ smpriv->slot_dealloc = slot_dealloc; -+ smpriv->slot_get_address = slot_get_address; -+ smpriv->slot_get_base = slot_get_base; -+ smpriv->slot_get_offset = slot_get_offset; -+ smpriv->slot_get_slot_size = slot_get_slot_size; -+#ifdef SM_DEBUG -+ dev_info(dev, "sm_init_keystore(): handlers installed\n"); -+#endif -+} -+EXPORT_SYMBOL(sm_init_keystore); -+ -+/* Return available pages/units */ -+u32 sm_detect_keystore_units(struct device *dev) -+{ -+ struct caam_drv_private_sm *smpriv = dev_get_drvdata(dev); -+ -+ return smpriv->localpages; -+} -+EXPORT_SYMBOL(sm_detect_keystore_units); -+ -+/* -+ * Do any keystore specific initializations -+ */ -+int sm_establish_keystore(struct device *dev, u32 unit) -+{ -+ struct caam_drv_private_sm *smpriv = dev_get_drvdata(dev); -+ -+#ifdef SM_DEBUG -+ dev_info(dev, "sm_establish_keystore(): unit %d initializing\n", unit); -+#endif -+ -+ if (smpriv->data_init == NULL) -+ return -EINVAL; -+ -+ /* Call the data_init function for any user setup */ -+ return smpriv->data_init(dev, unit); -+} -+EXPORT_SYMBOL(sm_establish_keystore); -+ -+void sm_release_keystore(struct device *dev, u32 unit) -+{ -+ struct caam_drv_private_sm *smpriv = dev_get_drvdata(dev); -+ -+#ifdef SM_DEBUG -+ dev_info(dev, "sm_establish_keystore(): unit %d releasing\n", unit); -+#endif -+ if ((smpriv != NULL) && (smpriv->data_cleanup != NULL)) -+ smpriv->data_cleanup(dev, unit); -+ -+ return; -+} -+EXPORT_SYMBOL(sm_release_keystore); -+ -+/* -+ * Subsequent interfacce (sm_keystore_*) forms the accessor interfacce to -+ * the keystore -+ */ -+int sm_keystore_slot_alloc(struct device *dev, u32 unit, u32 size, u32 *slot) -+{ -+ struct caam_drv_private_sm *smpriv = dev_get_drvdata(dev); -+ int retval = -EINVAL; -+ -+ spin_lock(&smpriv->kslock); -+ -+ if ((smpriv->slot_alloc == NULL) || -+ (smpriv->pagedesc[unit].ksdata == NULL)) -+ goto out; -+ -+ retval = smpriv->slot_alloc(dev, unit, size, slot); -+ -+out: -+ spin_unlock(&smpriv->kslock); -+ return retval; -+} -+EXPORT_SYMBOL(sm_keystore_slot_alloc); -+ -+int sm_keystore_slot_dealloc(struct device *dev, u32 unit, u32 slot) -+{ -+ struct caam_drv_private_sm *smpriv = dev_get_drvdata(dev); -+ int retval = -EINVAL; -+ -+ spin_lock(&smpriv->kslock); -+ -+ if ((smpriv->slot_alloc == NULL) || -+ (smpriv->pagedesc[unit].ksdata == NULL)) -+ goto out; -+ -+ retval = smpriv->slot_dealloc(dev, unit, slot); -+out: -+ spin_unlock(&smpriv->kslock); -+ return retval; -+} -+EXPORT_SYMBOL(sm_keystore_slot_dealloc); -+ -+int sm_keystore_slot_load(struct device *dev, u32 unit, u32 slot, -+ const u8 *key_data, u32 key_length) -+{ -+ struct caam_drv_private_sm *smpriv = dev_get_drvdata(dev); -+ int retval = -EINVAL; -+ u32 slot_size; -+ u32 i; -+ u8 __iomem *slot_location; -+ -+ spin_lock(&smpriv->kslock); -+ -+ slot_size = smpriv->slot_get_slot_size(dev, unit, slot); -+ -+ if (key_length > slot_size) { -+ retval = -EFBIG; -+ goto out; -+ } -+ -+ slot_location = smpriv->slot_get_address(dev, unit, slot); -+ -+ for (i = 0; i < key_length; i++) -+ slot_location[i] = key_data[i]; -+ -+ retval = 0; -+ -+out: -+ spin_unlock(&smpriv->kslock); -+ return retval; -+} -+EXPORT_SYMBOL(sm_keystore_slot_load); -+ -+int sm_keystore_slot_read(struct device *dev, u32 unit, u32 slot, -+ u32 key_length, u8 *key_data) -+{ -+ struct caam_drv_private_sm *smpriv = dev_get_drvdata(dev); -+ int retval = -EINVAL; -+ u8 __iomem *slot_addr; -+ u32 slot_size; -+ -+ spin_lock(&smpriv->kslock); -+ -+ slot_addr = smpriv->slot_get_address(dev, unit, slot); -+ slot_size = smpriv->slot_get_slot_size(dev, unit, slot); -+ -+ if (key_length > slot_size) { -+ retval = -EKEYREJECTED; -+ goto out; -+ } -+ -+ memcpy(key_data, slot_addr, key_length); -+ retval = 0; -+ -+out: -+ spin_unlock(&smpriv->kslock); -+ return retval; -+} -+EXPORT_SYMBOL(sm_keystore_slot_read); -+ -+int sm_keystore_slot_encapsulate(struct device *dev, u32 unit, u32 inslot, -+ u32 outslot, u16 secretlen, u8 *keymod, -+ u16 keymodlen) -+{ -+ struct caam_drv_private_sm *smpriv = dev_get_drvdata(dev); -+ int retval = 0; -+ u32 slot_length, dsize, jstat; -+ u32 __iomem *encapdesc = NULL; -+ u8 __iomem *lkeymod, *inpslotaddr, *outslotaddr; -+ dma_addr_t keymod_dma; -+ -+ /* Ensure that the full blob will fit in the key slot */ -+ slot_length = smpriv->slot_get_slot_size(dev, unit, outslot); -+ if ((secretlen + 48) > slot_length) -+ goto out; -+ -+ /* Get the base addresses of both keystore slots */ -+ inpslotaddr = (u8 *)smpriv->slot_get_address(dev, unit, inslot); -+ outslotaddr = (u8 *)smpriv->slot_get_address(dev, unit, outslot); -+ -+ /* Build the key modifier */ -+ lkeymod = kmalloc(keymodlen, GFP_KERNEL | GFP_DMA); -+ memcpy(lkeymod, keymod, keymodlen); -+ keymod_dma = dma_map_single(dev, lkeymod, keymodlen, DMA_TO_DEVICE); -+ dma_sync_single_for_device(dev, keymod_dma, keymodlen, DMA_TO_DEVICE); -+ -+ /* Build the encapsulation job descriptor */ -+ dsize = blob_encap_desc(&encapdesc, keymod_dma, keymodlen, -+ __pa(inpslotaddr), __pa(outslotaddr), -+ secretlen, 0); -+ if (!dsize) { -+ dev_err(dev, "can't alloc an encap descriptor\n"); -+ retval = -ENOMEM; -+ goto out; -+ } -+ jstat = sm_key_job(dev, encapdesc); -+ -+ dma_unmap_single(dev, keymod_dma, keymodlen, DMA_TO_DEVICE); -+ kfree(encapdesc); -+ -+out: -+ return retval; -+ -+} -+EXPORT_SYMBOL(sm_keystore_slot_encapsulate); -+ -+int sm_keystore_slot_decapsulate(struct device *dev, u32 unit, u32 inslot, -+ u32 outslot, u16 secretlen, u8 *keymod, -+ u16 keymodlen) -+{ -+ struct caam_drv_private_sm *smpriv = dev_get_drvdata(dev); -+ int retval = 0; -+ u32 slot_length, dsize, jstat; -+ u32 __iomem *decapdesc = NULL; -+ u8 __iomem *lkeymod, *inpslotaddr, *outslotaddr; -+ dma_addr_t keymod_dma; -+ -+ /* Ensure that the decap data will fit in the key slot */ -+ slot_length = smpriv->slot_get_slot_size(dev, unit, outslot); -+ if (secretlen > slot_length) -+ goto out; -+ -+ /* Get the base addresses of both keystore slots */ -+ inpslotaddr = (u8 *)smpriv->slot_get_address(dev, unit, inslot); -+ outslotaddr = (u8 *)smpriv->slot_get_address(dev, unit, outslot); -+ -+ /* Build the key modifier */ -+ lkeymod = kmalloc(keymodlen, GFP_KERNEL | GFP_DMA); -+ memcpy(lkeymod, keymod, keymodlen); -+ keymod_dma = dma_map_single(dev, lkeymod, keymodlen, DMA_TO_DEVICE); -+ dma_sync_single_for_device(dev, keymod_dma, keymodlen, DMA_TO_DEVICE); -+ -+ /* Build the decapsulation job descriptor */ -+ dsize = blob_decap_desc(&decapdesc, keymod_dma, keymodlen, -+ __pa(inpslotaddr), __pa(outslotaddr), -+ secretlen, 0); -+ if (!dsize) { -+ dev_err(dev, "can't alloc a decap descriptor\n"); -+ retval = -ENOMEM; -+ goto out; -+ } -+ jstat = sm_key_job(dev, decapdesc); -+ -+ dma_unmap_single(dev, keymod_dma, keymodlen, DMA_TO_DEVICE); -+ kfree(decapdesc); -+ -+out: -+ return retval; -+ -+} -+EXPORT_SYMBOL(sm_keystore_slot_decapsulate); -+ -+ -+/* -+ * Initialization/shutdown subsystem -+ * Assumes statically-invoked startup/shutdown from the controller driver -+ * for the present time, to be reworked when a device tree becomes -+ * available. This code will not modularize in present form. -+ * -+ * Also, simply uses ring 0 for execution at the present -+ */ -+ -+int caam_sm_startup(struct platform_device *pdev) -+{ -+ struct device *ctrldev, *smdev; -+ struct caam_drv_private *ctrlpriv; -+ struct caam_drv_private_sm *smpriv; -+ struct caam_drv_private_jr *jrpriv; /* need this for reg page */ -+ struct platform_device *sm_pdev; -+ struct sm_page_descriptor *lpagedesc; -+ u32 page, pgstat, lpagect, detectedpage; -+ -+ struct device_node *np; -+ ctrldev = &pdev->dev; -+ ctrlpriv = dev_get_drvdata(ctrldev); -+ -+ /* -+ * Set up the private block for secure memory -+ * Only one instance is possible -+ */ -+ smpriv = kzalloc(sizeof(struct caam_drv_private_sm), GFP_KERNEL); -+ if (smpriv == NULL) { -+ dev_err(ctrldev, "can't alloc private mem for secure memory\n"); -+ return -ENOMEM; -+ } -+ smpriv->parentdev = ctrldev; /* copy of parent dev is handy */ -+ -+ /* Create the dev */ -+#ifdef CONFIG_OF -+ np = of_find_compatible_node(NULL, NULL, "fsl,imx6q-caam-sm"); -+ sm_pdev = of_platform_device_create(np, "caam_sm", ctrldev); -+#else -+ sm_pdev = platform_device_register_data(ctrldev, "caam_sm", 0, -+ smpriv, -+ sizeof(struct caam_drv_private_sm)); -+#endif -+ if (sm_pdev == NULL) { -+ kfree(smpriv); -+ return -EINVAL; -+ } -+ smdev = &sm_pdev->dev; -+ dev_set_drvdata(smdev, smpriv); -+ ctrlpriv->smdev = smdev; -+ -+ /* -+ * Collect configuration limit data for reference -+ * This batch comes from the partition data/vid registers in perfmon -+ */ -+ smpriv->max_pages = ((rd_reg32(&ctrlpriv->ctrl->perfmon.smpart) -+ & SMPART_MAX_NUMPG_MASK) >> -+ SMPART_MAX_NUMPG_SHIFT) + 1; -+ smpriv->top_partition = ((rd_reg32(&ctrlpriv->ctrl->perfmon.smpart) -+ & SMPART_MAX_PNUM_MASK) >> -+ SMPART_MAX_PNUM_SHIFT) + 1; -+ smpriv->top_page = ((rd_reg32(&ctrlpriv->ctrl->perfmon.smpart) -+ & SMPART_MAX_PG_MASK) >> SMPART_MAX_PG_SHIFT) + 1; -+ smpriv->page_size = 1024 << ((rd_reg32(&ctrlpriv->ctrl->perfmon.smvid) -+ & SMVID_PG_SIZE_MASK) >> SMVID_PG_SIZE_SHIFT); -+ smpriv->slot_size = 1 << CONFIG_CRYPTO_DEV_FSL_CAAM_SM_SLOTSIZE; -+ -+#ifdef SM_DEBUG -+ dev_info(smdev, "max pages = %d, top partition = %d\n", -+ smpriv->max_pages, smpriv->top_partition); -+ dev_info(smdev, "top page = %d, page size = %d (total = %d)\n", -+ smpriv->top_page, smpriv->page_size, -+ smpriv->top_page * smpriv->page_size); -+ dev_info(smdev, "selected slot size = %d\n", smpriv->slot_size); -+#endif -+ -+ /* -+ * Now probe for partitions/pages to which we have access. Note that -+ * these have likely been set up by a bootloader or platform -+ * provisioning application, so we have to assume that we "inherit" -+ * a configuration and work within the constraints of what it might be. -+ * -+ * Assume use of the zeroth ring in the present iteration (until -+ * we can divorce the controller and ring drivers, and then assign -+ * an SM instance to any ring instance). -+ */ -+ smpriv->smringdev = ctrlpriv->jrdev[0]; -+ jrpriv = dev_get_drvdata(smpriv->smringdev); -+ lpagect = 0; -+ lpagedesc = kzalloc(sizeof(struct sm_page_descriptor) -+ * smpriv->max_pages, GFP_KERNEL); -+ if (lpagedesc == NULL) { -+ kfree(smpriv); -+ return -ENOMEM; -+ } -+ -+ for (page = 0; page < smpriv->max_pages; page++) { -+ wr_reg32(&jrpriv->rregs->sm_cmd, -+ ((page << SMC_PAGE_SHIFT) & SMC_PAGE_MASK) | -+ (SMC_CMD_PAGE_INQUIRY & SMC_CMD_MASK)); -+ pgstat = rd_reg32(&jrpriv->rregs->sm_status); -+ if (((pgstat & SMCS_PGWON_MASK) >> SMCS_PGOWN_SHIFT) -+ == SMCS_PGOWN_OWNED) { /* our page? */ -+ lpagedesc[page].phys_pagenum = -+ (pgstat & SMCS_PAGE_MASK) >> SMCS_PAGE_SHIFT; -+ lpagedesc[page].own_part = -+ (pgstat & SMCS_PART_SHIFT) >> SMCS_PART_MASK; -+ lpagedesc[page].pg_base = ctrlpriv->sm_base + -+ ((smpriv->page_size * page) / sizeof(u32)); -+ lpagect++; -+#ifdef SM_DEBUG -+ dev_info(smdev, -+ "physical page %d, owning partition = %d\n", -+ lpagedesc[page].phys_pagenum, -+ lpagedesc[page].own_part); -+#endif -+ } -+ } -+ -+ smpriv->pagedesc = kzalloc(sizeof(struct sm_page_descriptor) * lpagect, -+ GFP_KERNEL); -+ if (smpriv->pagedesc == NULL) { -+ kfree(lpagedesc); -+ kfree(smpriv); -+ return -ENOMEM; -+ } -+ smpriv->localpages = lpagect; -+ -+ detectedpage = 0; -+ for (page = 0; page < smpriv->max_pages; page++) { -+ if (lpagedesc[page].pg_base != NULL) { /* e.g. live entry */ -+ memcpy(&smpriv->pagedesc[detectedpage], -+ &lpagedesc[page], -+ sizeof(struct sm_page_descriptor)); -+#ifdef SM_DEBUG_CONT -+ sm_show_page(smdev, &smpriv->pagedesc[detectedpage]); -+#endif -+ detectedpage++; -+ } -+ } -+ -+ kfree(lpagedesc); -+ -+ sm_init_keystore(smdev); -+ -+ return 0; -+} -+ -+void caam_sm_shutdown(struct platform_device *pdev) -+{ -+ struct device *ctrldev, *smdev; -+ struct caam_drv_private *priv; -+ struct caam_drv_private_sm *smpriv; -+ -+ ctrldev = &pdev->dev; -+ priv = dev_get_drvdata(ctrldev); -+ smdev = priv->smdev; -+ smpriv = dev_get_drvdata(smdev); -+ -+ kfree(smpriv->pagedesc); -+ kfree(smpriv); -+} -+EXPORT_SYMBOL(caam_sm_shutdown); -+#ifdef CONFIG_OF -+static void __exit caam_sm_exit(void) -+{ -+ struct device_node *dev_node; -+ struct platform_device *pdev; -+ -+ dev_node = of_find_compatible_node(NULL, NULL, "fsl,sec-v4.0"); -+ if (!dev_node) { -+ dev_node = of_find_compatible_node(NULL, NULL, "fsl,sec4.0"); -+ if (!dev_node) -+ return; -+ } -+ -+ pdev = of_find_device_by_node(dev_node); -+ if (!pdev) -+ return; -+ -+ of_node_put(dev_node); -+ -+ caam_sm_shutdown(pdev); -+ -+ return; -+} -+ -+static int __init caam_sm_init(void) -+{ -+ struct device_node *dev_node; -+ struct platform_device *pdev; -+ -+ /* -+ * Do of_find_compatible_node() then of_find_device_by_node() -+ * once a functional device tree is available -+ */ -+ dev_node = of_find_compatible_node(NULL, NULL, "fsl,sec-v4.0"); -+ if (!dev_node) { -+ dev_node = of_find_compatible_node(NULL, NULL, "fsl,sec4.0"); -+ if (!dev_node) -+ return -ENODEV; -+ } -+ -+ pdev = of_find_device_by_node(dev_node); -+ if (!pdev) -+ return -ENODEV; -+ -+ of_node_get(dev_node); -+ -+ caam_sm_startup(pdev); -+ -+ return 0; -+} -+ -+module_init(caam_sm_init); -+module_exit(caam_sm_exit); -+ -+MODULE_LICENSE("Dual BSD/GPL"); -+MODULE_DESCRIPTION("FSL CAAM Secure Memory / Keystore"); -+MODULE_AUTHOR("Freescale Semiconductor - NMSG/MAD"); -+#endif -diff -Nur linux-4.1.3/drivers/crypto/caam/sm_test.c linux-xbian-imx6/drivers/crypto/caam/sm_test.c ---- linux-4.1.3/drivers/crypto/caam/sm_test.c 1970-01-01 01:00:00.000000000 +0100 -+++ linux-xbian-imx6/drivers/crypto/caam/sm_test.c 2015-07-27 23:13:04.213947410 +0200 -@@ -0,0 +1,844 @@ -+/* -+ * Secure Memory / Keystore Exemplification Module -+ * Copyright (C) 2013 Freescale Semiconductor, Inc. All Rights Reserved -+ * -+ * Serves as a functional example, and as a self-contained unit test for -+ * the functionality contained in sm_store.c. -+ * -+ * The example function, caam_sm_example_init(), runs a thread that: -+ * -+ * - initializes a set of fixed keys -+ * - stores one copy in clear buffers -+ * - stores them again in secure memory -+ * - extracts stored keys back out for use -+ * - intializes 3 data buffers for a test: -+ * (1) containing cleartext -+ * (2) to hold ciphertext encrypted with an extracted black key -+ * (3) to hold extracted cleartext decrypted with an equivalent clear key -+ * -+ * The function then builds simple job descriptors that reference the key -+ * material and buffers as initialized, and executes an encryption job -+ * with a black key, and a decryption job using a the same key held in the -+ * clear. The output of the decryption job is compared to the original -+ * cleartext; if they don't compare correctly, one can assume a key problem -+ * exists, where the function will exit with an error. -+ * -+ * This module can use a substantial amount of refactoring, which may occur -+ * after the API gets some mileage. Furthermore, expect this module to -+ * eventually disappear once the API is integrated into "real" software. -+ */ -+ -+#include "compat.h" -+#include "intern.h" -+#include "desc.h" -+#include "error.h" -+#include "jr.h" -+#include "sm.h" -+ -+static u8 skeymod[] = { -+ 0x0f, 0x0e, 0x0d, 0x0c, 0x0b, 0x0a, 0x09, 0x08, -+ 0x07, 0x06, 0x05, 0x04, 0x03, 0x02, 0x01, 0x00 -+}; -+static u8 symkey[] = { -+ 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, -+ 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, -+ 0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17, -+ 0x18, 0x19, 0x1a, 0x1b, 0x1c, 0x1d, 0x1e, 0x1f -+}; -+ -+static u8 symdata[] = { -+ 0x00, 0x01, 0x02, 0x03, 0x04, 0x0f, 0x06, 0x07, -+ 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, -+ 0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17, -+ 0x18, 0x19, 0x1a, 0x1b, 0x1c, 0x1d, 0x1e, 0x1f, -+ 0x20, 0x21, 0x22, 0x23, 0x24, 0x25, 0x26, 0x27, -+ 0x28, 0x29, 0x2a, 0x2b, 0x2c, 0x2d, 0x2e, 0x2f, -+ 0x30, 0x31, 0x32, 0x33, 0x34, 0x35, 0x36, 0x37, -+ 0x38, 0x39, 0x3a, 0x3b, 0x3c, 0x3d, 0x3e, 0x3f, -+ 0x40, 0x41, 0x42, 0x43, 0x44, 0x45, 0x46, 0x47, -+ 0x48, 0x49, 0x4a, 0x4b, 0x4c, 0x4d, 0x4e, 0x4f, -+ 0x50, 0x51, 0x52, 0x53, 0x54, 0x55, 0x56, 0x57, -+ 0x58, 0x59, 0x5a, 0x5b, 0x5c, 0x5d, 0x5e, 0x5f, -+ 0x60, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, -+ 0x68, 0x69, 0x6a, 0x6b, 0x6c, 0x6d, 0x6e, 0x6f, -+ 0x70, 0x71, 0x72, 0x73, 0x74, 0x75, 0x76, 0x77, -+ 0x78, 0x79, 0x7a, 0x7b, 0x7c, 0x7d, 0x7e, 0x7f, -+ 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, -+ 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, -+ 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, -+ 0x98, 0x99, 0x9a, 0x9b, 0x9c, 0x9d, 0x9e, 0x9f, -+ 0xa0, 0xa1, 0xa2, 0xa3, 0xa4, 0xa5, 0xa6, 0xa7, -+ 0xa8, 0xa9, 0xaa, 0xab, 0xac, 0xad, 0xae, 0xaf, -+ 0xb0, 0xb1, 0xb2, 0xb3, 0xb4, 0xb5, 0xb6, 0xb7, -+ 0xb8, 0xb9, 0xba, 0xbb, 0xbc, 0xbd, 0xbe, 0xbf, -+ 0xc0, 0xc1, 0xc2, 0xc3, 0xc4, 0xc5, 0xc6, 0xc7, -+ 0xc8, 0xc9, 0xca, 0xcb, 0xcc, 0xcd, 0xce, 0xcf, -+ 0xd0, 0xd1, 0xd2, 0xd3, 0xd4, 0xd5, 0xd6, 0xd7, -+ 0xd8, 0xd9, 0xda, 0xdb, 0xdc, 0xdd, 0xde, 0xdf, -+ 0xe0, 0xe1, 0xe2, 0xe3, 0xe4, 0xe5, 0xe6, 0xe7, -+ 0xe8, 0xe9, 0xea, 0xeb, 0xec, 0xed, 0xee, 0xef, -+ 0xf0, 0xf1, 0xf2, 0xf3, 0xf4, 0xf5, 0xf6, 0xf7, -+ 0xf8, 0xf9, 0xfa, 0xfb, 0xfc, 0xfd, 0xfe, 0xff -+}; -+ -+static int mk_job_desc(u32 *desc, dma_addr_t key, u16 keysz, dma_addr_t indata, -+ dma_addr_t outdata, u16 sz, u32 cipherdir, u32 keymode) -+{ -+ desc[1] = CMD_KEY | CLASS_1 | (keysz & KEY_LENGTH_MASK) | keymode; -+ desc[2] = (u32)key; -+ desc[3] = CMD_OPERATION | OP_TYPE_CLASS1_ALG | OP_ALG_AAI_ECB | -+ cipherdir; -+ desc[4] = CMD_FIFO_LOAD | FIFOLD_CLASS_CLASS1 | -+ FIFOLD_TYPE_MSG | FIFOLD_TYPE_LAST1 | sz; -+ desc[5] = (u32)indata; -+ desc[6] = CMD_FIFO_STORE | FIFOST_TYPE_MESSAGE_DATA | sz; -+ desc[7] = (u32)outdata; -+ -+ desc[0] = CMD_DESC_HDR | HDR_ONE | (8 & HDR_DESCLEN_MASK); -+ return 8 * sizeof(u32); -+} -+ -+struct exec_test_result { -+ int error; -+ struct completion completion; -+}; -+ -+void exec_test_done(struct device *dev, u32 *desc, u32 err, void *context) -+{ -+ struct exec_test_result *res = context; -+ -+ if (err) { -+ char tmp[CAAM_ERROR_STR_MAX]; -+ dev_err(dev, "%08x: %s\n", err, caam_jr_strstatus(tmp, err)); -+ } -+ -+ res->error = err; -+ complete(&res->completion); -+} -+ -+static int exec_test_job(struct device *ksdev, u32 *jobdesc) -+{ -+ struct exec_test_result testres; -+ struct caam_drv_private_sm *kspriv; -+ int rtn = 0; -+ -+ kspriv = dev_get_drvdata(ksdev); -+ -+ init_completion(&testres.completion); -+ -+ rtn = caam_jr_enqueue(kspriv->smringdev, jobdesc, exec_test_done, -+ &testres); -+ if (!rtn) { -+ wait_for_completion_interruptible(&testres.completion); -+ rtn = testres.error; -+ } -+ return rtn; -+} -+ -+ -+int caam_sm_example_init(struct platform_device *pdev) -+{ -+ struct device *ctrldev, *ksdev; -+ struct caam_drv_private *ctrlpriv; -+ struct caam_drv_private_sm *kspriv; -+ u32 unit, units, jdescsz; -+ int stat, jstat, rtnval = 0; -+ u8 __iomem *syminp, *symint, *symout = NULL; -+ dma_addr_t syminp_dma, symint_dma, symout_dma; -+ u8 __iomem *black_key_des, *black_key_aes128; -+ u8 __iomem *black_key_aes256; -+ dma_addr_t black_key_des_dma, black_key_aes128_dma; -+ dma_addr_t black_key_aes256_dma; -+ u8 __iomem *clear_key_des, *clear_key_aes128, *clear_key_aes256; -+ dma_addr_t clear_key_des_dma, clear_key_aes128_dma; -+ dma_addr_t clear_key_aes256_dma; -+ u32 __iomem *jdesc; -+ u32 keyslot_des, keyslot_aes128, keyslot_aes256 = 0; -+ -+ jdesc = NULL; -+ black_key_des = black_key_aes128 = black_key_aes256 = NULL; -+ clear_key_des = clear_key_aes128 = clear_key_aes256 = NULL; -+ -+ /* We can lose this cruft once we can get a pdev by name */ -+ ctrldev = &pdev->dev; -+ ctrlpriv = dev_get_drvdata(ctrldev); -+ ksdev = ctrlpriv->smdev; -+ kspriv = dev_get_drvdata(ksdev); -+ if (kspriv == NULL) -+ return -ENODEV; -+ -+ /* Now that we have the dev for the single SM instance, connect */ -+#ifdef SM_TEST_DETAIL -+ dev_info(ksdev, "caam_sm_test_init() running\n"); -+#endif -+ /* Probe to see what keystores are available to us */ -+ units = sm_detect_keystore_units(ksdev); -+ if (!units) -+ dev_err(ksdev, "caam_sm_test: no keystore units available\n"); -+ -+ /* -+ * MX6 bootloader stores some stuff in unit 0, so let's -+ * use 1 or above -+ */ -+ if (units < 2) { -+ dev_err(ksdev, "caam_sm_test: insufficient keystore units\n"); -+ return -ENODEV; -+ } -+ unit = 1; -+ -+#ifdef SM_TEST_DETAIL -+ dev_info(ksdev, "caam_sm_test: %d keystore units available\n", units); -+#endif -+ -+ /* Initialize/Establish Keystore */ -+ sm_establish_keystore(ksdev, unit); /* Initalize store in #1 */ -+ -+ /* -+ * Top of main test thread -+ */ -+ -+ /* Allocate test data blocks (input, intermediate, output) */ -+ syminp = kmalloc(256, GFP_KERNEL | GFP_DMA); -+ symint = kmalloc(256, GFP_KERNEL | GFP_DMA); -+ symout = kmalloc(256, GFP_KERNEL | GFP_DMA); -+ if ((syminp == NULL) || (symint == NULL) || (symout == NULL)) { -+ rtnval = -ENOMEM; -+ dev_err(ksdev, "caam_sm_test: can't get test data buffers\n"); -+ goto freemem; -+ } -+ -+ /* Allocate storage for 3 black keys: encapsulated 8, 16, 32 */ -+ black_key_des = kmalloc(16, GFP_KERNEL | GFP_DMA); /* padded to 16... */ -+ black_key_aes128 = kmalloc(16, GFP_KERNEL | GFP_DMA); -+ black_key_aes256 = kmalloc(16, GFP_KERNEL | GFP_DMA); -+ if ((black_key_des == NULL) || (black_key_aes128 == NULL) || -+ (black_key_aes256 == NULL)) { -+ rtnval = -ENOMEM; -+ dev_err(ksdev, "caam_sm_test: can't black key buffers\n"); -+ goto freemem; -+ } -+ -+ clear_key_des = kmalloc(8, GFP_KERNEL | GFP_DMA); -+ clear_key_aes128 = kmalloc(16, GFP_KERNEL | GFP_DMA); -+ clear_key_aes256 = kmalloc(32, GFP_KERNEL | GFP_DMA); -+ if ((clear_key_des == NULL) || (clear_key_aes128 == NULL) || -+ (clear_key_aes256 == NULL)) { -+ rtnval = -ENOMEM; -+ dev_err(ksdev, "caam_sm_test: can't get clear key buffers\n"); -+ goto freemem; -+ } -+ -+ /* Allocate storage for job descriptor */ -+ jdesc = kmalloc(8 * sizeof(u32), GFP_KERNEL | GFP_DMA); -+ if (jdesc == NULL) { -+ rtnval = -ENOMEM; -+ dev_err(ksdev, "caam_sm_test: can't get descriptor buffers\n"); -+ goto freemem; -+ } -+ -+#ifdef SM_TEST_DETAIL -+ dev_info(ksdev, "caam_sm_test: all buffers allocated\n"); -+#endif -+ -+ /* Load up input data block, clear outputs */ -+ memcpy(syminp, symdata, 256); -+ memset(symint, 0, 256); -+ memset(symout, 0, 256); -+#ifdef SM_TEST_DETAIL -+ dev_info(ksdev, "0x%02x 0x%02x 0x%02x 0x%02x " \ -+ "0x%02x 0x%02x 0x%02x 0x%02x\n", -+ syminp[0], syminp[1], syminp[2], syminp[3], -+ syminp[4], syminp[5], syminp[6], syminp[7]); -+ dev_info(ksdev, "0x%02x 0x%02x 0x%02x 0x%02x " \ -+ "0x%02x 0x%02x 0x%02x 0x%02x\n", -+ symint[0], symint[1], symint[2], symint[3], -+ symint[4], symint[5], symint[6], symint[7]); -+ dev_info(ksdev, "0x%02x 0x%02x 0x%02x 0x%02x " \ -+ "0x%02x 0x%02x 0x%02x 0x%02x\n", -+ symout[0], symout[1], symout[2], symout[3], -+ symout[4], symout[5], symout[6], symout[7]); -+ -+ dev_info(ksdev, "caam_sm_test: data buffers initialized\n"); -+#endif -+ -+ /* Load up clear keys */ -+ memcpy(clear_key_des, symkey, 8); -+ memcpy(clear_key_aes128, symkey, 16); -+ memcpy(clear_key_aes256, symkey, 32); -+ -+#ifdef SM_TEST_DETAIL -+ dev_info(ksdev, "caam_sm_test: all clear keys loaded\n"); -+#endif -+ -+ /* -+ * Place clear keys in keystore. -+ * All the interesting stuff happens here. -+ */ -+ /* 8 bit DES key */ -+ stat = sm_keystore_slot_alloc(ksdev, unit, 8, &keyslot_des); -+ if (stat) -+ goto freemem; -+#ifdef SM_TEST_DETAIL -+ dev_info(ksdev, "caam_sm_test: 8 byte key slot in %d\n", keyslot_des); -+#endif -+ stat = sm_keystore_slot_load(ksdev, unit, keyslot_des, clear_key_des, -+ 8); -+ if (stat) { -+#ifdef SM_TEST_DETAIL -+ dev_info(ksdev, "caam_sm_test: can't load 8 byte key in %d\n", -+ keyslot_des); -+#endif -+ sm_keystore_slot_dealloc(ksdev, unit, keyslot_des); -+ goto freemem; -+ } -+ -+ /* 16 bit AES key */ -+ stat = sm_keystore_slot_alloc(ksdev, unit, 16, &keyslot_aes128); -+ if (stat) { -+ sm_keystore_slot_dealloc(ksdev, unit, keyslot_des); -+ goto freemem; -+ } -+#ifdef SM_TEST_DETAIL -+ dev_info(ksdev, "caam_sm_test: 16 byte key slot in %d\n", -+ keyslot_aes128); -+#endif -+ stat = sm_keystore_slot_load(ksdev, unit, keyslot_aes128, -+ clear_key_aes128, 16); -+ if (stat) { -+#ifdef SM_TEST_DETAIL -+ dev_info(ksdev, "caam_sm_test: can't load 16 byte key in %d\n", -+ keyslot_aes128); -+#endif -+ sm_keystore_slot_dealloc(ksdev, unit, keyslot_aes128); -+ sm_keystore_slot_dealloc(ksdev, unit, keyslot_des); -+ goto freemem; -+ } -+ -+ /* 32 bit AES key */ -+ stat = sm_keystore_slot_alloc(ksdev, unit, 32, &keyslot_aes256); -+ if (stat) { -+ sm_keystore_slot_dealloc(ksdev, unit, keyslot_aes128); -+ sm_keystore_slot_dealloc(ksdev, unit, keyslot_des); -+ goto freemem; -+ } -+#ifdef SM_TEST_DETAIL -+ dev_info(ksdev, "caam_sm_test: 32 byte key slot in %d\n", -+ keyslot_aes256); -+#endif -+ stat = sm_keystore_slot_load(ksdev, unit, keyslot_aes256, -+ clear_key_aes256, 32); -+ if (stat) { -+#ifdef SM_TEST_DETAIL -+ dev_info(ksdev, "caam_sm_test: can't load 32 byte key in %d\n", -+ keyslot_aes128); -+#endif -+ sm_keystore_slot_dealloc(ksdev, unit, keyslot_aes256); -+ sm_keystore_slot_dealloc(ksdev, unit, keyslot_aes128); -+ sm_keystore_slot_dealloc(ksdev, unit, keyslot_des); -+ goto freemem; -+ } -+ -+ /* Encapsulate all keys as SM blobs */ -+ stat = sm_keystore_slot_encapsulate(ksdev, unit, keyslot_des, -+ keyslot_des, 8, skeymod, 8); -+ if (stat) { -+ dev_info(ksdev, "caam_sm_test: can't encapsulate DES key\n"); -+ goto freekeys; -+ } -+ -+ stat = sm_keystore_slot_encapsulate(ksdev, unit, keyslot_aes128, -+ keyslot_aes128, 16, skeymod, 8); -+ if (stat) { -+ dev_info(ksdev, "caam_sm_test: can't encapsulate AES128 key\n"); -+ goto freekeys; -+ } -+ -+ stat = sm_keystore_slot_encapsulate(ksdev, unit, keyslot_aes256, -+ keyslot_aes256, 32, skeymod, 8); -+ if (stat) { -+ dev_info(ksdev, "caam_sm_test: can't encapsulate AES256 key\n"); -+ goto freekeys; -+ } -+ -+ /* Now decapsulate as black key blobs */ -+ stat = sm_keystore_slot_decapsulate(ksdev, unit, keyslot_des, -+ keyslot_des, 8, skeymod, 8); -+ if (stat) { -+ dev_info(ksdev, "caam_sm_test: can't decapsulate DES key\n"); -+ goto freekeys; -+ } -+ -+ stat = sm_keystore_slot_decapsulate(ksdev, unit, keyslot_aes128, -+ keyslot_aes128, 16, skeymod, 8); -+ if (stat) { -+ dev_info(ksdev, "caam_sm_test: can't decapsulate AES128 key\n"); -+ goto freekeys; -+ } -+ -+ stat = sm_keystore_slot_decapsulate(ksdev, unit, keyslot_aes256, -+ keyslot_aes256, 32, skeymod, 8); -+ if (stat) { -+ dev_info(ksdev, "caam_sm_test: can't decapsulate AES128 key\n"); -+ goto freekeys; -+ } -+ -+ /* Extract 8/16/32 byte black keys */ -+ sm_keystore_slot_read(ksdev, unit, keyslot_des, 8, black_key_des); -+ sm_keystore_slot_read(ksdev, unit, keyslot_aes128, 16, -+ black_key_aes128); -+ sm_keystore_slot_read(ksdev, unit, keyslot_aes256, 32, -+ black_key_aes256); -+ -+#ifdef SM_TEST_DETAIL -+ dev_info(ksdev, "caam_sm_test: all black keys extracted\n"); -+#endif -+ -+ /* DES encrypt using 8 byte black key */ -+ black_key_des_dma = dma_map_single(ksdev, black_key_des, 8, -+ DMA_TO_DEVICE); -+ dma_sync_single_for_device(ksdev, black_key_des_dma, 8, DMA_TO_DEVICE); -+ syminp_dma = dma_map_single(ksdev, syminp, 256, DMA_TO_DEVICE); -+ dma_sync_single_for_device(ksdev, syminp_dma, 256, DMA_TO_DEVICE); -+ symint_dma = dma_map_single(ksdev, symint, 256, DMA_FROM_DEVICE); -+ -+ jdescsz = mk_job_desc(jdesc, black_key_des_dma, 8, syminp_dma, -+ symint_dma, 256, -+ OP_ALG_ENCRYPT | OP_ALG_ALGSEL_DES, 0); -+ -+#ifdef SM_TEST_DETAIL -+ dev_info(ksdev, "jobdesc:\n"); -+ dev_info(ksdev, "0x%08x\n", jdesc[0]); -+ dev_info(ksdev, "0x%08x\n", jdesc[1]); -+ dev_info(ksdev, "0x%08x\n", jdesc[2]); -+ dev_info(ksdev, "0x%08x\n", jdesc[3]); -+ dev_info(ksdev, "0x%08x\n", jdesc[4]); -+ dev_info(ksdev, "0x%08x\n", jdesc[5]); -+ dev_info(ksdev, "0x%08x\n", jdesc[6]); -+ dev_info(ksdev, "0x%08x\n", jdesc[7]); -+#endif -+ -+ jstat = exec_test_job(ksdev, jdesc); -+ -+ dma_sync_single_for_cpu(ksdev, symint_dma, 256, DMA_FROM_DEVICE); -+ dma_unmap_single(ksdev, symint_dma, 256, DMA_FROM_DEVICE); -+ dma_unmap_single(ksdev, syminp_dma, 256, DMA_TO_DEVICE); -+ dma_unmap_single(ksdev, black_key_des_dma, 8, DMA_TO_DEVICE); -+ -+#ifdef SM_TEST_DETAIL -+ dev_info(ksdev, "input block:\n"); -+ dev_info(ksdev, "0x%02x 0x%02x 0x%02x 0x%02x " \ -+ "0x%02x 0x%02x 0x%02x 0x%02x\n", -+ syminp[0], syminp[1], syminp[2], syminp[3], -+ syminp[4], syminp[5], syminp[6], syminp[7]); -+ dev_info(ksdev, "0x%02x 0x%02x 0x%02x 0x%02x " \ -+ "0x%02x 0x%02x 0x%02x 0x%02x\n", -+ syminp[8], syminp[9], syminp[10], syminp[11], -+ syminp[12], syminp[13], syminp[14], syminp[15]); -+ dev_info(ksdev, "intermediate block:\n"); -+ dev_info(ksdev, "0x%02x 0x%02x 0x%02x 0x%02x " \ -+ "0x%02x 0x%02x 0x%02x 0x%02x\n", -+ symint[0], symint[1], symint[2], symint[3], -+ symint[4], symint[5], symint[6], symint[7]); -+ dev_info(ksdev, "0x%02x 0x%02x 0x%02x 0x%02x " \ -+ "0x%02x 0x%02x 0x%02x 0x%02x\n", -+ symint[8], symint[9], symint[10], symint[11], -+ symint[12], symint[13], symint[14], symint[15]); -+ dev_info(ksdev, "caam_sm_test: encrypt cycle with 8 byte key\n"); -+#endif -+ -+ /* DES decrypt using 8 byte clear key */ -+ clear_key_des_dma = dma_map_single(ksdev, clear_key_des, 8, -+ DMA_TO_DEVICE); -+ dma_sync_single_for_device(ksdev, clear_key_des_dma, 8, DMA_TO_DEVICE); -+ symint_dma = dma_map_single(ksdev, symint, 256, DMA_TO_DEVICE); -+ dma_sync_single_for_device(ksdev, symint_dma, 256, DMA_TO_DEVICE); -+ symout_dma = dma_map_single(ksdev, symout, 256, DMA_FROM_DEVICE); -+ -+ jdescsz = mk_job_desc(jdesc, clear_key_des_dma, 8, symint_dma, -+ symout_dma, 256, -+ OP_ALG_DECRYPT | OP_ALG_ALGSEL_DES, 0); -+ -+#ifdef SM_TEST_DETAIL -+ dev_info(ksdev, "jobdesc:\n"); -+ dev_info(ksdev, "0x%08x\n", jdesc[0]); -+ dev_info(ksdev, "0x%08x\n", jdesc[1]); -+ dev_info(ksdev, "0x%08x\n", jdesc[2]); -+ dev_info(ksdev, "0x%08x\n", jdesc[3]); -+ dev_info(ksdev, "0x%08x\n", jdesc[4]); -+ dev_info(ksdev, "0x%08x\n", jdesc[5]); -+ dev_info(ksdev, "0x%08x\n", jdesc[6]); -+ dev_info(ksdev, "0x%08x\n", jdesc[7]); -+#endif -+ -+ jstat = exec_test_job(ksdev, jdesc); -+ -+ dma_sync_single_for_cpu(ksdev, symout_dma, 256, DMA_FROM_DEVICE); -+ dma_unmap_single(ksdev, symout_dma, 256, DMA_FROM_DEVICE); -+ dma_unmap_single(ksdev, symint_dma, 256, DMA_TO_DEVICE); -+ dma_unmap_single(ksdev, clear_key_des_dma, 8, DMA_TO_DEVICE); -+ -+#ifdef SM_TEST_DETAIL -+ dev_info(ksdev, "intermediate block:\n"); -+ dev_info(ksdev, "0x%02x 0x%02x 0x%02x 0x%02x " \ -+ "0x%02x 0x%02x 0x%02x 0x%02x\n", -+ symint[0], symint[1], symint[2], symint[3], -+ symint[4], symint[5], symint[6], symint[7]); -+ dev_info(ksdev, "0x%02x 0x%02x 0x%02x 0x%02x " \ -+ "0x%02x 0x%02x 0x%02x 0x%02x\n", -+ symint[8], symint[9], symint[10], symint[11], -+ symint[12], symint[13], symint[14], symint[15]); -+ dev_info(ksdev, "decrypted block:\n"); -+ dev_info(ksdev, "0x%02x 0x%02x 0x%02x 0x%02x " \ -+ "0x%02x 0x%02x 0x%02x 0x%02x\n", -+ symout[0], symout[1], symout[2], symout[3], -+ symout[4], symout[5], symout[6], symout[7]); -+ dev_info(ksdev, "0x%02x 0x%02x 0x%02x 0x%02x " \ -+ "0x%02x 0x%02x 0x%02x 0x%02x\n", -+ symout[8], symout[9], symout[10], symout[11], -+ symout[12], symout[13], symout[14], symout[15]); -+ dev_info(ksdev, "caam_sm_test: decrypt cycle with 8 byte key\n"); -+#endif -+ -+ /* Check result */ -+ if (memcmp(symout, syminp, 256)) { -+ dev_info(ksdev, "caam_sm_test: 8-byte key test mismatch\n"); -+ rtnval = -1; -+ goto freekeys; -+ } else -+ dev_info(ksdev, "caam_sm_test: 8-byte key test match OK\n"); -+ -+ /* AES-128 encrypt using 16 byte black key */ -+ black_key_aes128_dma = dma_map_single(ksdev, black_key_aes128, 16, -+ DMA_TO_DEVICE); -+ dma_sync_single_for_device(ksdev, black_key_aes128_dma, 16, -+ DMA_TO_DEVICE); -+ syminp_dma = dma_map_single(ksdev, syminp, 256, DMA_TO_DEVICE); -+ dma_sync_single_for_device(ksdev, syminp_dma, 256, DMA_TO_DEVICE); -+ symint_dma = dma_map_single(ksdev, symint, 256, DMA_FROM_DEVICE); -+ -+ jdescsz = mk_job_desc(jdesc, black_key_aes128_dma, 16, syminp_dma, -+ symint_dma, 256, -+ OP_ALG_ENCRYPT | OP_ALG_ALGSEL_AES, 0); -+ -+#ifdef SM_TEST_DETAIL -+ dev_info(ksdev, "jobdesc:\n"); -+ dev_info(ksdev, "0x%08x\n", jdesc[0]); -+ dev_info(ksdev, "0x%08x\n", jdesc[1]); -+ dev_info(ksdev, "0x%08x\n", jdesc[2]); -+ dev_info(ksdev, "0x%08x\n", jdesc[3]); -+ dev_info(ksdev, "0x%08x\n", jdesc[4]); -+ dev_info(ksdev, "0x%08x\n", jdesc[5]); -+ dev_info(ksdev, "0x%08x\n", jdesc[6]); -+ dev_info(ksdev, "0x%08x\n", jdesc[7]); -+#endif -+ -+ jstat = exec_test_job(ksdev, jdesc); -+ -+ dma_sync_single_for_cpu(ksdev, symint_dma, 256, DMA_FROM_DEVICE); -+ dma_unmap_single(ksdev, symint_dma, 256, DMA_FROM_DEVICE); -+ dma_unmap_single(ksdev, syminp_dma, 256, DMA_TO_DEVICE); -+ dma_unmap_single(ksdev, black_key_aes128_dma, 16, DMA_TO_DEVICE); -+ -+#ifdef SM_TEST_DETAIL -+ dev_info(ksdev, "input block:\n"); -+ dev_info(ksdev, "0x%02x 0x%02x 0x%02x 0x%02x " \ -+ "0x%02x 0x%02x 0x%02x 0x%02x\n", -+ syminp[0], syminp[1], syminp[2], syminp[3], -+ syminp[4], syminp[5], syminp[6], syminp[7]); -+ dev_info(ksdev, "0x%02x 0x%02x 0x%02x 0x%02x " \ -+ "0x%02x 0x%02x 0x%02x 0x%02x\n", -+ syminp[8], syminp[9], syminp[10], syminp[11], -+ syminp[12], syminp[13], syminp[14], syminp[15]); -+ dev_info(ksdev, "intermediate block:\n"); -+ dev_info(ksdev, "0x%02x 0x%02x 0x%02x 0x%02x " \ -+ "0x%02x 0x%02x 0x%02x 0x%02x\n", -+ symint[0], symint[1], symint[2], symint[3], -+ symint[4], symint[5], symint[6], symint[7]); -+ dev_info(ksdev, "0x%02x 0x%02x 0x%02x 0x%02x " \ -+ "0x%02x 0x%02x 0x%02x 0x%02x\n", -+ symint[8], symint[9], symint[10], symint[11], -+ symint[12], symint[13], symint[14], symint[15]); -+ dev_info(ksdev, "caam_sm_test: encrypt cycle with 16 byte key\n"); -+#endif -+ -+ /* AES-128 decrypt using 16 byte clear key */ -+ clear_key_aes128_dma = dma_map_single(ksdev, clear_key_aes128, 16, -+ DMA_TO_DEVICE); -+ dma_sync_single_for_device(ksdev, clear_key_aes128_dma, 16, -+ DMA_TO_DEVICE); -+ symint_dma = dma_map_single(ksdev, symint, 256, DMA_TO_DEVICE); -+ dma_sync_single_for_device(ksdev, symint_dma, 256, DMA_TO_DEVICE); -+ symout_dma = dma_map_single(ksdev, symout, 256, DMA_FROM_DEVICE); -+ -+ jdescsz = mk_job_desc(jdesc, clear_key_aes128_dma, 16, symint_dma, -+ symout_dma, 256, -+ OP_ALG_DECRYPT | OP_ALG_ALGSEL_AES, 0); -+ -+#ifdef SM_TEST_DETAIL -+ dev_info(ksdev, "jobdesc:\n"); -+ dev_info(ksdev, "0x%08x\n", jdesc[0]); -+ dev_info(ksdev, "0x%08x\n", jdesc[1]); -+ dev_info(ksdev, "0x%08x\n", jdesc[2]); -+ dev_info(ksdev, "0x%08x\n", jdesc[3]); -+ dev_info(ksdev, "0x%08x\n", jdesc[4]); -+ dev_info(ksdev, "0x%08x\n", jdesc[5]); -+ dev_info(ksdev, "0x%08x\n", jdesc[6]); -+ dev_info(ksdev, "0x%08x\n", jdesc[7]); -+#endif -+ jstat = exec_test_job(ksdev, jdesc); -+ -+ dma_sync_single_for_cpu(ksdev, symout_dma, 256, DMA_FROM_DEVICE); -+ dma_unmap_single(ksdev, symout_dma, 256, DMA_FROM_DEVICE); -+ dma_unmap_single(ksdev, symint_dma, 256, DMA_TO_DEVICE); -+ dma_unmap_single(ksdev, clear_key_aes128_dma, 16, DMA_TO_DEVICE); -+ -+#ifdef SM_TEST_DETAIL -+ dev_info(ksdev, "intermediate block:\n"); -+ dev_info(ksdev, "0x%02x 0x%02x 0x%02x 0x%02x " \ -+ "0x%02x 0x%02x 0x%02x 0x%02x\n", -+ symint[0], symint[1], symint[2], symint[3], -+ symint[4], symint[5], symint[6], symint[7]); -+ dev_info(ksdev, "0x%02x 0x%02x 0x%02x 0x%02x " \ -+ "0x%02x 0x%02x 0x%02x 0x%02x\n", -+ symint[8], symint[9], symint[10], symint[11], -+ symint[12], symint[13], symint[14], symint[15]); -+ dev_info(ksdev, "decrypted block:\n"); -+ dev_info(ksdev, "0x%02x 0x%02x 0x%02x 0x%02x " \ -+ "0x%02x 0x%02x 0x%02x 0x%02x\n", -+ symout[0], symout[1], symout[2], symout[3], -+ symout[4], symout[5], symout[6], symout[7]); -+ dev_info(ksdev, "0x%02x 0x%02x 0x%02x 0x%02x " \ -+ "0x%02x 0x%02x 0x%02x 0x%02x\n", -+ symout[8], symout[9], symout[10], symout[11], -+ symout[12], symout[13], symout[14], symout[15]); -+ dev_info(ksdev, "caam_sm_test: decrypt cycle with 16 byte key\n"); -+#endif -+ -+ /* Check result */ -+ if (memcmp(symout, syminp, 256)) { -+ dev_info(ksdev, "caam_sm_test: 16-byte key test mismatch\n"); -+ rtnval = -1; -+ goto freekeys; -+ } else -+ dev_info(ksdev, "caam_sm_test: 16-byte key test match OK\n"); -+ -+ /* AES-256 encrypt using 32 byte black key */ -+ black_key_aes256_dma = dma_map_single(ksdev, black_key_aes256, 32, -+ DMA_TO_DEVICE); -+ dma_sync_single_for_device(ksdev, black_key_aes256_dma, 32, -+ DMA_TO_DEVICE); -+ syminp_dma = dma_map_single(ksdev, syminp, 256, DMA_TO_DEVICE); -+ dma_sync_single_for_device(ksdev, syminp_dma, 256, DMA_TO_DEVICE); -+ symint_dma = dma_map_single(ksdev, symint, 256, DMA_FROM_DEVICE); -+ -+ jdescsz = mk_job_desc(jdesc, black_key_aes256_dma, 32, syminp_dma, -+ symint_dma, 256, -+ OP_ALG_ENCRYPT | OP_ALG_ALGSEL_AES, 0); -+ -+#ifdef SM_TEST_DETAIL -+ dev_info(ksdev, "jobdesc:\n"); -+ dev_info(ksdev, "0x%08x\n", jdesc[0]); -+ dev_info(ksdev, "0x%08x\n", jdesc[1]); -+ dev_info(ksdev, "0x%08x\n", jdesc[2]); -+ dev_info(ksdev, "0x%08x\n", jdesc[3]); -+ dev_info(ksdev, "0x%08x\n", jdesc[4]); -+ dev_info(ksdev, "0x%08x\n", jdesc[5]); -+ dev_info(ksdev, "0x%08x\n", jdesc[6]); -+ dev_info(ksdev, "0x%08x\n", jdesc[7]); -+#endif -+ -+ jstat = exec_test_job(ksdev, jdesc); -+ -+ dma_sync_single_for_cpu(ksdev, symint_dma, 256, DMA_FROM_DEVICE); -+ dma_unmap_single(ksdev, symint_dma, 256, DMA_FROM_DEVICE); -+ dma_unmap_single(ksdev, syminp_dma, 256, DMA_TO_DEVICE); -+ dma_unmap_single(ksdev, black_key_aes256_dma, 32, DMA_TO_DEVICE); -+ -+#ifdef SM_TEST_DETAIL -+ dev_info(ksdev, "input block:\n"); -+ dev_info(ksdev, "0x%02x 0x%02x 0x%02x 0x%02x " \ -+ "0x%02x 0x%02x 0x%02x 0x%02x\n", -+ syminp[0], syminp[1], syminp[2], syminp[3], -+ syminp[4], syminp[5], syminp[6], syminp[7]); -+ dev_info(ksdev, "0x%02x 0x%02x 0x%02x 0x%02x " \ -+ "0x%02x 0x%02x 0x%02x 0x%02x\n", -+ syminp[8], syminp[9], syminp[10], syminp[11], -+ syminp[12], syminp[13], syminp[14], syminp[15]); -+ dev_info(ksdev, "intermediate block:\n"); -+ dev_info(ksdev, "0x%02x 0x%02x 0x%02x 0x%02x " \ -+ "0x%02x 0x%02x 0x%02x 0x%02x\n", -+ symint[0], symint[1], symint[2], symint[3], -+ symint[4], symint[5], symint[6], symint[7]); -+ dev_info(ksdev, "0x%02x 0x%02x 0x%02x 0x%02x " \ -+ "0x%02x 0x%02x 0x%02x 0x%02x\n", -+ symint[8], symint[9], symint[10], symint[11], -+ symint[12], symint[13], symint[14], symint[15]); -+ dev_info(ksdev, "caam_sm_test: encrypt cycle with 32 byte key\n"); -+#endif -+ -+ /* AES-256 decrypt using 32-byte black key */ -+ clear_key_aes256_dma = dma_map_single(ksdev, clear_key_aes256, 32, -+ DMA_TO_DEVICE); -+ dma_sync_single_for_device(ksdev, clear_key_aes256_dma, 32, -+ DMA_TO_DEVICE); -+ symint_dma = dma_map_single(ksdev, symint, 256, DMA_TO_DEVICE); -+ dma_sync_single_for_device(ksdev, symint_dma, 256, DMA_TO_DEVICE); -+ symout_dma = dma_map_single(ksdev, symout, 256, DMA_FROM_DEVICE); -+ -+ jdescsz = mk_job_desc(jdesc, clear_key_aes256_dma, 32, symint_dma, -+ symout_dma, 256, -+ OP_ALG_DECRYPT | OP_ALG_ALGSEL_AES, 0); -+ -+#ifdef SM_TEST_DETAIL -+ dev_info(ksdev, "jobdesc:\n"); -+ dev_info(ksdev, "0x%08x\n", jdesc[0]); -+ dev_info(ksdev, "0x%08x\n", jdesc[1]); -+ dev_info(ksdev, "0x%08x\n", jdesc[2]); -+ dev_info(ksdev, "0x%08x\n", jdesc[3]); -+ dev_info(ksdev, "0x%08x\n", jdesc[4]); -+ dev_info(ksdev, "0x%08x\n", jdesc[5]); -+ dev_info(ksdev, "0x%08x\n", jdesc[6]); -+ dev_info(ksdev, "0x%08x\n", jdesc[7]); -+#endif -+ -+ jstat = exec_test_job(ksdev, jdesc); -+ -+ dma_sync_single_for_cpu(ksdev, symout_dma, 256, DMA_FROM_DEVICE); -+ dma_unmap_single(ksdev, symout_dma, 256, DMA_FROM_DEVICE); -+ dma_unmap_single(ksdev, symint_dma, 256, DMA_TO_DEVICE); -+ dma_unmap_single(ksdev, clear_key_aes256_dma, 32, DMA_TO_DEVICE); -+ -+#ifdef SM_TEST_DETAIL -+ dev_info(ksdev, "intermediate block:\n"); -+ dev_info(ksdev, "0x%02x 0x%02x 0x%02x 0x%02x " \ -+ "0x%02x 0x%02x 0x%02x 0x%02x\n", -+ symint[0], symint[1], symint[2], symint[3], -+ symint[4], symint[5], symint[6], symint[7]); -+ dev_info(ksdev, "0x%02x 0x%02x 0x%02x 0x%02x " \ -+ "0x%02x 0x%02x 0x%02x 0x%02x\n", -+ symint[8], symint[9], symint[10], symint[11], -+ symint[12], symint[13], symint[14], symint[15]); -+ dev_info(ksdev, "decrypted block:\n"); -+ dev_info(ksdev, "0x%02x 0x%02x 0x%02x 0x%02x " \ -+ "0x%02x 0x%02x 0x%02x 0x%02x\n", -+ symout[0], symout[1], symout[2], symout[3], -+ symout[4], symout[5], symout[6], symout[7]); -+ dev_info(ksdev, "0x%02x 0x%02x 0x%02x 0x%02x " \ -+ "0x%02x 0x%02x 0x%02x 0x%02x\n", -+ symout[8], symout[9], symout[10], symout[11], -+ symout[12], symout[13], symout[14], symout[15]); -+ dev_info(ksdev, "caam_sm_test: decrypt cycle with 32 byte key\n"); -+#endif -+ -+ /* Check result */ -+ if (memcmp(symout, syminp, 256)) { -+ dev_info(ksdev, "caam_sm_test: 32-byte key test mismatch\n"); -+ rtnval = -1; -+ goto freekeys; -+ } else -+ dev_info(ksdev, "caam_sm_test: 32-byte key test match OK\n"); -+ -+ -+ /* Remove 8/16/32 byte keys from keystore */ -+freekeys: -+ stat = sm_keystore_slot_dealloc(ksdev, unit, keyslot_des); -+ if (stat) -+ dev_info(ksdev, "caam_sm_test: can't release slot %d\n", -+ keyslot_des); -+ -+ stat = sm_keystore_slot_dealloc(ksdev, unit, keyslot_aes128); -+ if (stat) -+ dev_info(ksdev, "caam_sm_test: can't release slot %d\n", -+ keyslot_aes128); -+ -+ stat = sm_keystore_slot_dealloc(ksdev, unit, keyslot_aes256); -+ if (stat) -+ dev_info(ksdev, "caam_sm_test: can't release slot %d\n", -+ keyslot_aes256); -+ -+ -+ /* Free resources */ -+freemem: -+#ifdef SM_TEST_DETAIL -+ dev_info(ksdev, "caam_sm_test: cleaning up\n"); -+#endif -+ kfree(syminp); -+ kfree(symint); -+ kfree(symout); -+ kfree(clear_key_des); -+ kfree(clear_key_aes128); -+ kfree(clear_key_aes256); -+ kfree(black_key_des); -+ kfree(black_key_aes128); -+ kfree(black_key_aes256); -+ kfree(jdesc); -+ -+ /* Disconnect from keystore and leave */ -+ sm_release_keystore(ksdev, unit); -+ -+ return rtnval; -+} -+EXPORT_SYMBOL(caam_sm_example_init); -+ -+void caam_sm_example_shutdown(void) -+{ -+ /* unused in present version */ -+ struct device_node *dev_node; -+ struct platform_device *pdev; -+ -+ /* -+ * Do of_find_compatible_node() then of_find_device_by_node() -+ * once a functional device tree is available -+ */ -+ dev_node = of_find_compatible_node(NULL, NULL, "fsl,sec-v4.0"); -+ if (!dev_node) { -+ dev_node = of_find_compatible_node(NULL, NULL, "fsl,sec4.0"); -+ if (!dev_node) -+ return; -+ } -+ -+ pdev = of_find_device_by_node(dev_node); -+ if (!pdev) -+ return; -+ -+ of_node_get(dev_node); -+ -+} -+ -+static int __init caam_sm_test_init(void) -+{ -+ struct device_node *dev_node; -+ struct platform_device *pdev; -+ -+ /* -+ * Do of_find_compatible_node() then of_find_device_by_node() -+ * once a functional device tree is available -+ */ -+ dev_node = of_find_compatible_node(NULL, NULL, "fsl,sec-v4.0"); -+ if (!dev_node) { -+ dev_node = of_find_compatible_node(NULL, NULL, "fsl,sec4.0"); -+ if (!dev_node) -+ return -ENODEV; -+ } -+ -+ pdev = of_find_device_by_node(dev_node); -+ if (!pdev) -+ return -ENODEV; -+ -+ of_node_put(dev_node); -+ -+ caam_sm_example_init(pdev); -+ -+ return 0; -+} -+ -+ -+/* Module-based initialization needs to wait for dev tree */ -+#ifdef CONFIG_OF -+module_init(caam_sm_test_init); -+module_exit(caam_sm_example_shutdown); -+ -+MODULE_LICENSE("Dual BSD/GPL"); -+MODULE_DESCRIPTION("FSL CAAM Keystore Usage Example"); -+MODULE_AUTHOR("Freescale Semiconductor - NMSG/MAD"); -+#endif -diff -Nur linux-4.1.3/drivers/crypto/caam/snvsregs.h linux-xbian-imx6/drivers/crypto/caam/snvsregs.h ---- linux-4.1.3/drivers/crypto/caam/snvsregs.h 1970-01-01 01:00:00.000000000 +0100 -+++ linux-xbian-imx6/drivers/crypto/caam/snvsregs.h 2015-07-27 23:13:04.213947410 +0200 -@@ -0,0 +1,237 @@ -+/* -+ * SNVS hardware register-level view -+ * -+ * Copyright (C) 2012-2014 Freescale Semiconductor, Inc., All Rights Reserved -+ */ -+ -+#ifndef SNVSREGS_H -+#define SNVSREGS_H -+ -+#include -+#include -+ -+/* -+ * SNVS High Power Domain -+ * Includes security violations, HA counter, RTC, alarm -+ */ -+struct snvs_hp { -+ u32 lock; /* HPLR - HP Lock */ -+ u32 cmd; /* HPCOMR - HP Command */ -+ u32 ctl; /* HPCR - HP Control */ -+ u32 secvio_intcfg; /* HPSICR - Security Violation Int Config */ -+ u32 secvio_ctl; /* HPSVCR - Security Violation Control */ -+ u32 status; /* HPSR - HP Status */ -+ u32 secvio_status; /* HPSVSR - Security Violation Status */ -+ u32 ha_counteriv; /* High Assurance Counter IV */ -+ u32 ha_counter; /* High Assurance Counter */ -+ u32 rtc_msb; /* Real Time Clock/Counter MSB */ -+ u32 rtc_lsb; /* Real Time Counter LSB */ -+ u32 time_alarm_msb; /* Time Alarm MSB */ -+ u32 time_alarm_lsb; /* Time Alarm LSB */ -+}; -+ -+#define HP_LOCK_HAC_LCK 0x00040000 -+#define HP_LOCK_HPSICR_LCK 0x00020000 -+#define HP_LOCK_HPSVCR_LCK 0x00010000 -+#define HP_LOCK_MKEYSEL_LCK 0x00000200 -+#define HP_LOCK_TAMPCFG_LCK 0x00000100 -+#define HP_LOCK_TAMPFLT_LCK 0x00000080 -+#define HP_LOCK_SECVIO_LCK 0x00000040 -+#define HP_LOCK_GENP_LCK 0x00000020 -+#define HP_LOCK_MONOCTR_LCK 0x00000010 -+#define HP_LOCK_CALIB_LCK 0x00000008 -+#define HP_LOCK_SRTC_LCK 0x00000004 -+#define HP_LOCK_ZMK_RD_LCK 0x00000002 -+#define HP_LOCK_ZMK_WT_LCK 0x00000001 -+ -+#define HP_CMD_NONPRIV_AXS 0x80000000 -+#define HP_CMD_HAC_STOP 0x00080000 -+#define HP_CMD_HAC_CLEAR 0x00040000 -+#define HP_CMD_HAC_LOAD 0x00020000 -+#define HP_CMD_HAC_CFG_EN 0x00010000 -+#define HP_CMD_SNVS_MSTR_KEY 0x00002000 -+#define HP_CMD_PROG_ZMK 0x00001000 -+#define HP_CMD_SW_LPSV 0x00000400 -+#define HP_CMD_SW_FSV 0x00000200 -+#define HP_CMD_SW_SV 0x00000100 -+#define HP_CMD_LP_SWR_DIS 0x00000020 -+#define HP_CMD_LP_SWR 0x00000010 -+#define HP_CMD_SSM_SFNS_DIS 0x00000004 -+#define HP_CMD_SSM_ST_DIS 0x00000002 -+#define HP_CMD_SMM_ST 0x00000001 -+ -+#define HP_CTL_TIME_SYNC 0x00010000 -+#define HP_CTL_CAL_VAL_SHIFT 10 -+#define HP_CTL_CAL_VAL_MASK (0x1f << HP_CTL_CALIB_SHIFT) -+#define HP_CTL_CALIB_EN 0x00000100 -+#define HP_CTL_PI_FREQ_SHIFT 4 -+#define HP_CTL_PI_FREQ_MASK (0xf << HP_CTL_PI_FREQ_SHIFT) -+#define HP_CTL_PI_EN 0x00000008 -+#define HP_CTL_TIMEALARM_EN 0x00000002 -+#define HP_CTL_RTC_EN 0x00000001 -+ -+#define HP_SECVIO_INTEN_EN 0x10000000 -+#define HP_SECVIO_INTEN_SRC5 0x00000020 -+#define HP_SECVIO_INTEN_SRC4 0x00000010 -+#define HP_SECVIO_INTEN_SRC3 0x00000008 -+#define HP_SECVIO_INTEN_SRC2 0x00000004 -+#define HP_SECVIO_INTEN_SRC1 0x00000002 -+#define HP_SECVIO_INTEN_SRC0 0x00000001 -+#define HP_SECVIO_INTEN_ALL 0x8000003f -+ -+#define HP_SECVIO_ICTL_CFG_SHIFT 30 -+#define HP_SECVIO_ICTL_CFG_MASK (0x3 << HP_SECVIO_ICTL_CFG_SHIFT) -+#define HP_SECVIO_ICTL_CFG5_SHIFT 5 -+#define HP_SECVIO_ICTL_CFG5_MASK (0x3 << HP_SECVIO_ICTL_CFG5_SHIFT) -+#define HP_SECVIO_ICTL_CFG_DISABLE 0 -+#define HP_SECVIO_ICTL_CFG_NONFATAL 1 -+#define HP_SECVIO_ICTL_CFG_FATAL 2 -+#define HP_SECVIO_ICTL_CFG4_FATAL 0x00000010 -+#define HP_SECVIO_ICTL_CFG3_FATAL 0x00000008 -+#define HP_SECVIO_ICTL_CFG2_FATAL 0x00000004 -+#define HP_SECVIO_ICTL_CFG1_FATAL 0x00000002 -+#define HP_SECVIO_ICTL_CFG0_FATAL 0x00000001 -+ -+#define HP_STATUS_ZMK_ZERO 0x80000000 -+#define HP_STATUS_OTPMK_ZERO 0x08000000 -+#define HP_STATUS_OTPMK_SYN_SHIFT 16 -+#define HP_STATUS_OTPMK_SYN_MASK (0x1ff << HP_STATUS_OTPMK_SYN_SHIFT) -+#define HP_STATUS_SSM_ST_SHIFT 8 -+#define HP_STATUS_SSM_ST_MASK (0xf << HP_STATUS_SSM_ST_SHIFT) -+#define HP_STATUS_SSM_ST_INIT 0 -+#define HP_STATUS_SSM_ST_HARDFAIL 1 -+#define HP_STATUS_SSM_ST_SOFTFAIL 3 -+#define HP_STATUS_SSM_ST_INITINT 8 -+#define HP_STATUS_SSM_ST_CHECK 9 -+#define HP_STATUS_SSM_ST_NONSECURE 11 -+#define HP_STATUS_SSM_ST_TRUSTED 13 -+#define HP_STATUS_SSM_ST_SECURE 15 -+ -+#define HP_SECVIOST_ZMK_ECC_FAIL 0x08000000 /* write to clear */ -+#define HP_SECVIOST_ZMK_SYN_SHIFT 16 -+#define HP_SECVIOST_ZMK_SYN_MASK (0x1ff << HP_SECVIOST_ZMK_SYN_SHIFT) -+#define HP_SECVIOST_SECVIO5 0x00000020 -+#define HP_SECVIOST_SECVIO4 0x00000010 -+#define HP_SECVIOST_SECVIO3 0x00000008 -+#define HP_SECVIOST_SECVIO2 0x00000004 -+#define HP_SECVIOST_SECVIO1 0x00000002 -+#define HP_SECVIOST_SECVIO0 0x00000001 -+#define HP_SECVIOST_SECVIOMASK 0x0000003f -+ -+/* -+ * SNVS Low Power Domain -+ * Includes glitch detector, SRTC, alarm, monotonic counter, ZMK -+ */ -+struct snvs_lp { -+ u32 lock; -+ u32 ctl; -+ u32 mstr_key_ctl; /* Master Key Control */ -+ u32 secvio_ctl; /* Security Violation Control */ -+ u32 tamper_filt_cfg; /* Tamper Glitch Filters Configuration */ -+ u32 tamper_det_cfg; /* Tamper Detectors Configuration */ -+ u32 status; -+ u32 srtc_msb; /* Secure Real Time Clock/Counter MSB */ -+ u32 srtc_lsb; /* Secure Real Time Clock/Counter LSB */ -+ u32 time_alarm; /* Time Alarm */ -+ u32 smc_msb; /* Secure Monotonic Counter MSB */ -+ u32 smc_lsb; /* Secure Monotonic Counter LSB */ -+ u32 pwr_glitch_det; /* Power Glitch Detector */ -+ u32 gen_purpose; -+ u32 zmk[8]; /* Zeroizable Master Key */ -+}; -+ -+#define LP_LOCK_MKEYSEL_LCK 0x00000200 -+#define LP_LOCK_TAMPDET_LCK 0x00000100 -+#define LP_LOCK_TAMPFLT_LCK 0x00000080 -+#define LP_LOCK_SECVIO_LCK 0x00000040 -+#define LP_LOCK_GENP_LCK 0x00000020 -+#define LP_LOCK_MONOCTR_LCK 0x00000010 -+#define LP_LOCK_CALIB_LCK 0x00000008 -+#define LP_LOCK_SRTC_LCK 0x00000004 -+#define LP_LOCK_ZMK_RD_LCK 0x00000002 -+#define LP_LOCK_ZMK_WT_LCK 0x00000001 -+ -+#define LP_CTL_CAL_VAL_SHIFT 10 -+#define LP_CTL_CAL_VAL_MASK (0x1f << LP_CTL_CAL_VAL_SHIFT) -+#define LP_CTL_CALIB_EN 0x00000100 -+#define LP_CTL_SRTC_INVAL_EN 0x00000010 -+#define LP_CTL_WAKE_INT_EN 0x00000008 -+#define LP_CTL_MONOCTR_EN 0x00000004 -+#define LP_CTL_TIMEALARM_EN 0x00000002 -+#define LP_CTL_SRTC_EN 0x00000001 -+ -+#define LP_MKEYCTL_ZMKECC_SHIFT 8 -+#define LP_MKEYCTL_ZMKECC_MASK (0xff << LP_MKEYCTL_ZMKECC_SHIFT) -+#define LP_MKEYCTL_ZMKECC_EN 0x00000010 -+#define LP_MKEYCTL_ZMKECC_VAL 0x00000008 -+#define LP_MKEYCTL_ZMKECC_PROG 0x00000004 -+#define LP_MKEYCTL_MKSEL_SHIFT 0 -+#define LP_MKEYCTL_MKSEL_MASK (3 << LP_MKEYCTL_MKSEL_SHIFT) -+#define LP_MKEYCTL_MK_OTP 0 -+#define LP_MKEYCTL_MK_ZMK 2 -+#define LP_MKEYCTL_MK_COMB 3 -+ -+#define LP_SECVIO_CTL_SRC5 0x20 -+#define LP_SECVIO_CTL_SRC4 0x10 -+#define LP_SECVIO_CTL_SRC3 0x08 -+#define LP_SECVIO_CTL_SRC2 0x04 -+#define LP_SECVIO_CTL_SRC1 0x02 -+#define LP_SECVIO_CTL_SRC0 0x01 -+ -+#define LP_TAMPFILT_EXT2_EN 0x80000000 -+#define LP_TAMPFILT_EXT2_SHIFT 24 -+#define LP_TAMPFILT_EXT2_MASK (0x1f << LP_TAMPFILT_EXT2_SHIFT) -+#define LP_TAMPFILT_EXT1_EN 0x00800000 -+#define LP_TAMPFILT_EXT1_SHIFT 16 -+#define LP_TAMPFILT_EXT1_MASK (0x1f << LP_TAMPFILT_EXT1_SHIFT) -+#define LP_TAMPFILT_WM_EN 0x00000080 -+#define LP_TAMPFILT_WM_SHIFT 0 -+#define LP_TAMPFILT_WM_MASK (0x1f << LP_TAMPFILT_WM_SHIFT) -+ -+#define LP_TAMPDET_OSC_BPS 0x10000000 -+#define LP_TAMPDET_VRC_SHIFT 24 -+#define LP_TAMPDET_VRC_MASK (3 << LP_TAMPFILT_VRC_SHIFT) -+#define LP_TAMPDET_HTDC_SHIFT 20 -+#define LP_TAMPDET_HTDC_MASK (3 << LP_TAMPFILT_HTDC_SHIFT) -+#define LP_TAMPDET_LTDC_SHIFT 16 -+#define LP_TAMPDET_LTDC_MASK (3 << LP_TAMPFILT_LTDC_SHIFT) -+#define LP_TAMPDET_POR_OBS 0x00008000 -+#define LP_TAMPDET_PFD_OBS 0x00004000 -+#define LP_TAMPDET_ET2_EN 0x00000400 -+#define LP_TAMPDET_ET1_EN 0x00000200 -+#define LP_TAMPDET_WMT2_EN 0x00000100 -+#define LP_TAMPDET_WMT1_EN 0x00000080 -+#define LP_TAMPDET_VT_EN 0x00000040 -+#define LP_TAMPDET_TT_EN 0x00000020 -+#define LP_TAMPDET_CT_EN 0x00000010 -+#define LP_TAMPDET_MCR_EN 0x00000004 -+#define LP_TAMPDET_SRTCR_EN 0x00000002 -+ -+#define LP_STATUS_SECURE -+#define LP_STATUS_NONSECURE -+#define LP_STATUS_SCANEXIT 0x00100000 /* all write 1 clear here on */ -+#define LP_STATUS_EXT_SECVIO 0x00010000 -+#define LP_STATUS_ET2 0x00000400 -+#define LP_STATUS_ET1 0x00000200 -+#define LP_STATUS_WMT2 0x00000100 -+#define LP_STATUS_WMT1 0x00000080 -+#define LP_STATUS_VTD 0x00000040 -+#define LP_STATUS_TTD 0x00000020 -+#define LP_STATUS_CTD 0x00000010 -+#define LP_STATUS_PGD 0x00000008 -+#define LP_STATUS_MCR 0x00000004 -+#define LP_STATUS_SRTCR 0x00000002 -+#define LP_STATUS_LPTA 0x00000001 -+ -+/* Full SNVS register page, including version/options */ -+struct snvs_full { -+ struct snvs_hp hp; -+ struct snvs_lp lp; -+ u32 rsvd[731]; /* deadspace 0x08c-0xbf7 */ -+ -+ /* Version / Revision / Option ID space - end of register page */ -+ u32 vid; /* 0xbf8 HP Version ID (VID 1) */ -+ u32 opt_rev; /* 0xbfc HP Options / Revision (VID 2) */ -+}; -+ -+#endif /* SNVSREGS_H */ -diff -Nur linux-4.1.3/drivers/dma/imx-sdma.c linux-xbian-imx6/drivers/dma/imx-sdma.c ---- linux-4.1.3/drivers/dma/imx-sdma.c 2015-07-21 19:10:33.000000000 +0200 -+++ linux-xbian-imx6/drivers/dma/imx-sdma.c 2015-07-27 23:13:04.285691435 +0200 -@@ -29,6 +29,7 @@ - #include - #include - #include -+#include - #include - #include - #include -@@ -232,6 +233,14 @@ - - struct sdma_engine; - -+enum sdma_mode { -+ SDMA_MODE_INVALID = 0, -+ SDMA_MODE_LOOP, -+ SDMA_MODE_NORMAL, -+ SDMA_MODE_P2P, -+ SDMA_MODE_NO_BD, -+}; -+ - /** - * struct sdma_channel - housekeeping for a SDMA channel - * -@@ -244,6 +253,7 @@ - * @word_size peripheral access size - * @buf_tail ID of the buffer that was processed - * @num_bd max NUM_BD. number of descriptors currently handling -+ * @bd_iram flag indicating the memory location of buffer descriptor - */ - struct sdma_channel { - struct sdma_engine *sdma; -@@ -258,12 +268,16 @@ - unsigned int period_len; - struct sdma_buffer_descriptor *bd; - dma_addr_t bd_phys; -+ bool bd_iram; - unsigned int pc_from_device, pc_to_device; -- unsigned long flags; -- dma_addr_t per_address; -+ unsigned int device_to_device; -+ unsigned int other_script; -+ enum sdma_mode mode; -+ dma_addr_t per_address, per_address2; - unsigned long event_mask[2]; - unsigned long watermark_level; - u32 shp_addr, per_addr; -+ u32 data_addr1, data_addr2; - struct dma_chan chan; - spinlock_t lock; - struct dma_async_tx_descriptor desc; -@@ -271,11 +285,8 @@ - unsigned int chn_count; - unsigned int chn_real_count; - struct tasklet_struct tasklet; -- struct imx_dma_data data; - }; - --#define IMX_DMA_SG_LOOP BIT(0) -- - #define MAX_DMA_CHANNELS 32 - #define MXC_SDMA_DEFAULT_PRIORITY 1 - #define MXC_SDMA_MIN_PRIORITY 1 -@@ -327,6 +338,7 @@ - spinlock_t channel_0_lock; - u32 script_number; - struct sdma_script_start_addrs *script_addrs; -+ struct gen_pool *iram_pool; - const struct sdma_driver_data *drvdata; - }; - -@@ -546,12 +558,14 @@ - dma_addr_t buf_phys; - int ret; - unsigned long flags; -+ bool use_iram = true; - -- buf_virt = dma_alloc_coherent(NULL, -- size, -- &buf_phys, GFP_KERNEL); -+ buf_virt = gen_pool_dma_alloc(sdma->iram_pool, size, &buf_phys); - if (!buf_virt) { -- return -ENOMEM; -+ use_iram = false; -+ buf_virt = dma_alloc_coherent(NULL, size, &buf_phys, GFP_KERNEL); -+ if (!buf_virt) -+ return -ENOMEM; - } - - spin_lock_irqsave(&sdma->channel_0_lock, flags); -@@ -568,7 +582,10 @@ - - spin_unlock_irqrestore(&sdma->channel_0_lock, flags); - -- dma_free_coherent(NULL, size, buf_virt, buf_phys); -+ if (use_iram) -+ gen_pool_free(sdma->iram_pool, (unsigned long)buf_virt, size); -+ else -+ dma_free_coherent(NULL, size, buf_virt, buf_phys); - - return ret; - } -@@ -654,14 +671,31 @@ - sdmac->desc.callback(sdmac->desc.callback_param); - } - -+static void sdma_handle_other_intr(struct sdma_channel *sdmac) -+{ -+ if (sdmac->desc.callback) -+ sdmac->desc.callback(sdmac->desc.callback_param); -+} -+ - static void sdma_tasklet(unsigned long data) - { - struct sdma_channel *sdmac = (struct sdma_channel *) data; -+ struct sdma_engine *sdma = sdmac->sdma; - -- if (sdmac->flags & IMX_DMA_SG_LOOP) -+ switch (sdmac->mode) { -+ case SDMA_MODE_LOOP: - sdma_handle_channel_loop(sdmac); -- else -+ break; -+ case SDMA_MODE_NORMAL: - mxc_sdma_handle_channel_normal(sdmac); -+ break; -+ case SDMA_MODE_NO_BD: -+ sdma_handle_other_intr(sdmac); -+ break; -+ default: -+ dev_err(sdma->dev, "invalid SDMA MODE!\n"); -+ break; -+ } - } - - static irqreturn_t sdma_int_handler(int irq, void *dev_id) -@@ -678,7 +712,7 @@ - int channel = fls(stat) - 1; - struct sdma_channel *sdmac = &sdma->channel[channel]; - -- if (sdmac->flags & IMX_DMA_SG_LOOP) -+ if (sdmac->mode & SDMA_MODE_LOOP) - sdma_update_channel_loop(sdmac); - - tasklet_schedule(&sdmac->tasklet); -@@ -702,9 +736,12 @@ - * two peripherals or memory-to-memory transfers - */ - int per_2_per = 0, emi_2_emi = 0; -+ int other = 0; - - sdmac->pc_from_device = 0; - sdmac->pc_to_device = 0; -+ sdmac->device_to_device = 0; -+ sdmac->other_script = 0; - - switch (peripheral_type) { - case IMX_DMATYPE_MEMORY: -@@ -733,7 +770,6 @@ - case IMX_DMATYPE_CSPI: - case IMX_DMATYPE_EXT: - case IMX_DMATYPE_SSI: -- case IMX_DMATYPE_SAI: - per_2_emi = sdma->script_addrs->app_2_mcu_addr; - emi_2_per = sdma->script_addrs->mcu_2_app_addr; - break; -@@ -751,11 +787,6 @@ - emi_2_per = sdma->script_addrs->mcu_2_shp_addr; - break; - case IMX_DMATYPE_ASRC: -- per_2_emi = sdma->script_addrs->asrc_2_mcu_addr; -- emi_2_per = sdma->script_addrs->asrc_2_mcu_addr; -- per_2_per = sdma->script_addrs->per_2_per_addr; -- break; -- case IMX_DMATYPE_ASRC_SP: - per_2_emi = sdma->script_addrs->shp_2_mcu_addr; - emi_2_per = sdma->script_addrs->mcu_2_shp_addr; - per_2_per = sdma->script_addrs->per_2_per_addr; -@@ -774,12 +805,17 @@ - case IMX_DMATYPE_IPU_MEMORY: - emi_2_per = sdma->script_addrs->ext_mem_2_ipu_addr; - break; -+ case IMX_DMATYPE_HDMI: -+ other = sdma->script_addrs->hdmi_dma_addr; -+ break; - default: - break; - } - - sdmac->pc_from_device = per_2_emi; - sdmac->pc_to_device = emi_2_per; -+ sdmac->device_to_device = per_2_per; -+ sdmac->other_script = other; - } - - static int sdma_load_context(struct sdma_channel *sdmac) -@@ -792,11 +828,14 @@ - int ret; - unsigned long flags; - -- if (sdmac->direction == DMA_DEV_TO_MEM) { -+ if (sdmac->direction == DMA_DEV_TO_MEM) - load_address = sdmac->pc_from_device; -- } else { -+ else if (sdmac->direction == DMA_DEV_TO_DEV) -+ load_address = sdmac->device_to_device; -+ else if (sdmac->direction == DMA_MEM_TO_DEV) - load_address = sdmac->pc_to_device; -- } -+ else -+ load_address = sdmac->other_script; - - if (load_address < 0) - return load_address; -@@ -816,11 +855,16 @@ - /* Send by context the event mask,base address for peripheral - * and watermark level - */ -- context->gReg[0] = sdmac->event_mask[1]; -- context->gReg[1] = sdmac->event_mask[0]; -- context->gReg[2] = sdmac->per_addr; -- context->gReg[6] = sdmac->shp_addr; -- context->gReg[7] = sdmac->watermark_level; -+ if (sdmac->peripheral_type == IMX_DMATYPE_HDMI) { -+ context->gReg[4] = sdmac->data_addr1; -+ context->gReg[6] = sdmac->data_addr2; -+ } else { -+ context->gReg[0] = sdmac->event_mask[1]; -+ context->gReg[1] = sdmac->event_mask[0]; -+ context->gReg[2] = sdmac->per_addr; -+ context->gReg[6] = sdmac->shp_addr; -+ context->gReg[7] = sdmac->watermark_level; -+ } - - bd0->mode.command = C0_SETDM; - bd0->mode.status = BD_DONE | BD_INTR | BD_WRAP | BD_EXTD; -@@ -854,6 +898,7 @@ - static int sdma_config_channel(struct dma_chan *chan) - { - struct sdma_channel *sdmac = to_sdma_chan(chan); -+ struct imx_dma_data *data = sdmac->chan.private; - int ret; - - sdma_disable_channel(chan); -@@ -862,12 +907,19 @@ - sdmac->event_mask[1] = 0; - sdmac->shp_addr = 0; - sdmac->per_addr = 0; -+ sdmac->data_addr1 = 0; -+ sdmac->data_addr2 = 0; - -- if (sdmac->event_id0) { -+ if (sdmac->event_id0 >= 0) { - if (sdmac->event_id0 >= sdmac->sdma->drvdata->num_events) - return -EINVAL; - sdma_event_enable(sdmac, sdmac->event_id0); - } -+ if (sdmac->event_id1) { -+ if (sdmac->event_id1 >= sdmac->sdma->drvdata->num_events) -+ return -EINVAL; -+ sdma_event_enable(sdmac, sdmac->event_id1); -+ } - - switch (sdmac->peripheral_type) { - case IMX_DMATYPE_DSP: -@@ -887,19 +939,75 @@ - (sdmac->peripheral_type != IMX_DMATYPE_DSP)) { - /* Handle multiple event channels differently */ - if (sdmac->event_id1) { -- sdmac->event_mask[1] = BIT(sdmac->event_id1 % 32); -- if (sdmac->event_id1 > 31) -- __set_bit(31, &sdmac->watermark_level); -- sdmac->event_mask[0] = BIT(sdmac->event_id0 % 32); -- if (sdmac->event_id0 > 31) -- __set_bit(30, &sdmac->watermark_level); -+ if (sdmac->event_id0 > 31) { -+ sdmac->event_mask[0] |= 0; -+ __set_bit(28, &sdmac->watermark_level); -+ sdmac->event_mask[1] |= -+ BIT(sdmac->event_id0 % 32); -+ } else { -+ sdmac->event_mask[1] |= 0; -+ sdmac->event_mask[0] |= -+ BIT(sdmac->event_id0 % 32); -+ } -+ if (sdmac->event_id1 > 31) { -+ sdmac->event_mask[0] |= 0; -+ __set_bit(29, &sdmac->watermark_level); -+ sdmac->event_mask[1] |= -+ BIT(sdmac->event_id1 % 32); -+ } else { -+ sdmac->event_mask[1] |= 0; -+ sdmac->event_mask[0] |= -+ BIT(sdmac->event_id1 % 32); -+ } -+ /* BIT 11: -+ * 1 : Source on SPBA -+ * 0 : Source on AIPS -+ */ -+ __set_bit(11, &sdmac->watermark_level); -+ /* BIT 12: -+ * 1 : Destination on SPBA -+ * 0 : Destination on AIPS -+ */ -+ __set_bit(12, &sdmac->watermark_level); -+ __set_bit(31, &sdmac->watermark_level); -+ /* BIT 31: -+ * 1 : Amount of samples to be transferred is -+ * unknown and script will keep on transferring -+ * samples as long as both events are detected -+ * and script must be manually stopped by the -+ * application. -+ * 0 : The amount of samples to be is equal to -+ * the count field of mode word -+ * */ -+ __set_bit(25, &sdmac->watermark_level); -+ __clear_bit(24, &sdmac->watermark_level); - } else { -- __set_bit(sdmac->event_id0, sdmac->event_mask); -+ if (sdmac->event_id0 > 31) { -+ sdmac->event_mask[0] = 0; -+ sdmac->event_mask[1] |= -+ BIT(sdmac->event_id0 % 32); -+ } else { -+ sdmac->event_mask[0] |= -+ BIT(sdmac->event_id0 % 32); -+ sdmac->event_mask[1] = 0; -+ } - } - /* Watermark Level */ - sdmac->watermark_level |= sdmac->watermark_level; - /* Address */ -- sdmac->shp_addr = sdmac->per_address; -+ if (sdmac->direction == DMA_DEV_TO_DEV) { -+ sdmac->shp_addr = sdmac->per_address2; -+ sdmac->per_addr = sdmac->per_address; -+ } else if (sdmac->direction == DMA_TRANS_NONE) { -+ if (sdmac->peripheral_type != IMX_DMATYPE_HDMI || -+ !data->data_addr1 || !data->data_addr2) -+ return -EINVAL; -+ sdmac->data_addr1 = *(u32 *)data->data_addr1; -+ sdmac->data_addr2 = *(u32 *)data->data_addr2; -+ sdmac->watermark_level = 0; -+ } else { -+ sdmac->shp_addr = sdmac->per_address; -+ } - } else { - sdmac->watermark_level = 0; /* FIXME: M3_BASE_ADDRESS */ - } -@@ -931,13 +1039,19 @@ - int channel = sdmac->channel; - int ret = -EBUSY; - -- sdmac->bd = dma_zalloc_coherent(NULL, PAGE_SIZE, &sdmac->bd_phys, -- GFP_KERNEL); -+ sdmac->bd_iram = true; -+ sdmac->bd = gen_pool_dma_alloc(sdma->iram_pool, PAGE_SIZE, &sdmac->bd_phys); - if (!sdmac->bd) { -- ret = -ENOMEM; -- goto out; -+ sdmac->bd_iram = false; -+ sdmac->bd = dma_alloc_coherent(NULL, PAGE_SIZE, &sdmac->bd_phys, GFP_KERNEL); -+ if (!sdmac->bd) { -+ ret = -ENOMEM; -+ goto out; -+ } - } - -+ memset(sdmac->bd, 0, PAGE_SIZE); -+ - sdma->channel_control[channel].base_bd_ptr = sdmac->bd_phys; - sdma->channel_control[channel].current_bd_ptr = sdmac->bd_phys; - -@@ -987,6 +1101,7 @@ - - sdmac->peripheral_type = data->peripheral_type; - sdmac->event_id0 = data->dma_request; -+ sdmac->event_id1 = data->dma_request2; - - clk_enable(sdmac->sdma->clk_ipg); - clk_enable(sdmac->sdma->clk_ahb); -@@ -1004,6 +1119,9 @@ - /* txd.flags will be overwritten in prep funcs */ - sdmac->desc.flags = DMA_CTRL_ACK; - -+ /* Set SDMA channel mode to unvalid to avoid misconfig */ -+ sdmac->mode = SDMA_MODE_INVALID; -+ - return 0; - } - -@@ -1014,7 +1132,7 @@ - - sdma_disable_channel(chan); - -- if (sdmac->event_id0) -+ if (sdmac->event_id0 >= 0) - sdma_event_disable(sdmac, sdmac->event_id0); - if (sdmac->event_id1) - sdma_event_disable(sdmac, sdmac->event_id1); -@@ -1024,7 +1142,10 @@ - - sdma_set_channel_priority(sdmac, 0); - -- dma_free_coherent(NULL, PAGE_SIZE, sdmac->bd, sdmac->bd_phys); -+ if (sdmac->bd_iram) -+ gen_pool_free(sdma->iram_pool, (unsigned long)sdmac->bd, PAGE_SIZE); -+ else -+ dma_free_coherent(NULL, PAGE_SIZE, sdmac->bd, sdmac->bd_phys); - - clk_disable(sdma->clk_ipg); - clk_disable(sdma->clk_ahb); -@@ -1045,7 +1166,7 @@ - return NULL; - sdmac->status = DMA_IN_PROGRESS; - -- sdmac->flags = 0; -+ sdmac->mode = SDMA_MODE_NORMAL; - - sdmac->buf_tail = 0; - -@@ -1134,13 +1255,13 @@ - static struct dma_async_tx_descriptor *sdma_prep_dma_cyclic( - struct dma_chan *chan, dma_addr_t dma_addr, size_t buf_len, - size_t period_len, enum dma_transfer_direction direction, -- unsigned long flags) -+ unsigned long flags, void *context) - { - struct sdma_channel *sdmac = to_sdma_chan(chan); - struct sdma_engine *sdma = sdmac->sdma; -- int num_periods = buf_len / period_len; - int channel = sdmac->channel; - int ret, i = 0, buf = 0; -+ int num_periods; - - dev_dbg(sdma->dev, "%s channel: %d\n", __func__, channel); - -@@ -1152,12 +1273,33 @@ - sdmac->buf_tail = 0; - sdmac->period_len = period_len; - -- sdmac->flags |= IMX_DMA_SG_LOOP; - sdmac->direction = direction; -+ -+ switch (sdmac->direction) { -+ case DMA_DEV_TO_DEV: -+ sdmac->mode = SDMA_MODE_P2P; -+ break; -+ case DMA_TRANS_NONE: -+ sdmac->mode = SDMA_MODE_NO_BD; -+ break; -+ case DMA_MEM_TO_DEV: -+ case DMA_DEV_TO_MEM: -+ sdmac->mode = SDMA_MODE_LOOP; -+ break; -+ default: -+ dev_err(sdma->dev, "invalid SDMA direction %d\n", direction); -+ return NULL; -+ } -+ - ret = sdma_load_context(sdmac); - if (ret) - goto err_out; - -+ if (period_len) -+ num_periods = buf_len / period_len; -+ else -+ return &sdmac->desc; -+ - if (num_periods > NUM_BD) { - dev_err(sdma->dev, "SDMA channel %d: maximum number of sg exceeded: %d > %d\n", - channel, num_periods, NUM_BD); -@@ -1216,7 +1358,16 @@ - { - struct sdma_channel *sdmac = to_sdma_chan(chan); - -- if (dmaengine_cfg->direction == DMA_DEV_TO_MEM) { -+ if (dmaengine_cfg->direction == DMA_DEV_TO_DEV) { -+ sdmac->per_address = dmaengine_cfg->src_addr; -+ sdmac->per_address2 = dmaengine_cfg->dst_addr; -+ sdmac->watermark_level = 0; -+ sdmac->watermark_level |= -+ dmaengine_cfg->src_maxburst; -+ sdmac->watermark_level |= -+ dmaengine_cfg->dst_maxburst << 16; -+ sdmac->word_size = dmaengine_cfg->dst_addr_width; -+ } else if (dmaengine_cfg->direction == DMA_DEV_TO_MEM) { - sdmac->per_address = dmaengine_cfg->src_addr; - sdmac->watermark_level = dmaengine_cfg->src_maxburst * - dmaengine_cfg->src_addr_width; -@@ -1238,7 +1389,7 @@ - struct sdma_channel *sdmac = to_sdma_chan(chan); - u32 residue; - -- if (sdmac->flags & IMX_DMA_SG_LOOP) -+ if (sdmac->mode & SDMA_MODE_LOOP) - residue = (sdmac->num_bd - sdmac->buf_tail) * sdmac->period_len; - else - residue = sdmac->chn_count - sdmac->chn_real_count; -@@ -1286,8 +1437,7 @@ - unsigned short *ram_code; - - if (!fw) { -- dev_info(sdma->dev, "external firmware not found, using ROM firmware\n"); -- /* In this case we just use the ROM firmware. */ -+ dev_err(sdma->dev, "firmware not found\n"); - return; - } - -@@ -1302,7 +1452,10 @@ - goto err_firmware; - switch (header->version_major) { - case 1: -- sdma->script_number = SDMA_SCRIPT_ADDRS_ARRAY_SIZE_V1; -+ if (header->version_minor > 0) -+ sdma->script_number = SDMA_SCRIPT_ADDRS_ARRAY_SIZE_V2; -+ else -+ sdma->script_number = SDMA_SCRIPT_ADDRS_ARRAY_SIZE_V1; - break; - case 2: - sdma->script_number = SDMA_SCRIPT_ADDRS_ARRAY_SIZE_V2; -@@ -1337,7 +1490,7 @@ - release_firmware(fw); - } - --static int sdma_get_firmware(struct sdma_engine *sdma, -+static int __init sdma_get_firmware(struct sdma_engine *sdma, - const char *fw_name) - { - int ret; -@@ -1349,9 +1502,9 @@ - return ret; - } - --static int sdma_init(struct sdma_engine *sdma) -+static int __init sdma_init(struct sdma_engine *sdma) - { -- int i, ret; -+ int i, ret, ccbsize; - dma_addr_t ccb_phys; - - clk_enable(sdma->clk_ipg); -@@ -1360,14 +1513,17 @@ - /* Be sure SDMA has not started yet */ - writel_relaxed(0, sdma->regs + SDMA_H_C0PTR); - -- sdma->channel_control = dma_alloc_coherent(NULL, -- MAX_DMA_CHANNELS * sizeof (struct sdma_channel_control) + -- sizeof(struct sdma_context_data), -- &ccb_phys, GFP_KERNEL); -+ ccbsize = MAX_DMA_CHANNELS * sizeof (struct sdma_channel_control) -+ + sizeof(struct sdma_context_data); - -+ sdma->channel_control = gen_pool_dma_alloc(sdma->iram_pool, ccbsize, &ccb_phys); - if (!sdma->channel_control) { -- ret = -ENOMEM; -- goto err_dma_alloc; -+ sdma->channel_control = dma_alloc_coherent(NULL, ccbsize, -+ &ccb_phys, GFP_KERNEL); -+ if (!sdma->channel_control) { -+ ret = -ENOMEM; -+ goto err_dma_alloc; -+ } - } - - sdma->context = (void *)sdma->channel_control + -@@ -1419,14 +1575,12 @@ - - static bool sdma_filter_fn(struct dma_chan *chan, void *fn_param) - { -- struct sdma_channel *sdmac = to_sdma_chan(chan); - struct imx_dma_data *data = fn_param; - - if (!imx_dma_is_general_purpose(chan)) - return false; - -- sdmac->data = *data; -- chan->private = &sdmac->data; -+ chan->private = data; - - return true; - } -@@ -1444,11 +1598,12 @@ - data.dma_request = dma_spec->args[0]; - data.peripheral_type = dma_spec->args[1]; - data.priority = dma_spec->args[2]; -+ data.dma_request2 = 0; - - return dma_request_channel(mask, sdma_filter_fn, &data); - } - --static int sdma_probe(struct platform_device *pdev) -+static int __init sdma_probe(struct platform_device *pdev) - { - const struct of_device_id *of_id = - of_match_device(sdma_dt_ids, &pdev->dev); -@@ -1547,6 +1702,11 @@ - &sdma->dma_device.channels); - } - -+ if (np) -+ sdma->iram_pool = of_get_named_gen_pool(np, "iram", 0); -+ if (!sdma->iram_pool) -+ dev_warn(&pdev->dev, "no iram assigned, using external mem\n"); -+ - ret = sdma_init(sdma); - if (ret) - goto err_init; -@@ -1583,7 +1743,7 @@ - sdma->dma_device.device_free_chan_resources = sdma_free_chan_resources; - sdma->dma_device.device_tx_status = sdma_tx_status; - sdma->dma_device.device_prep_slave_sg = sdma_prep_slave_sg; -- sdma->dma_device.device_prep_dma_cyclic = sdma_prep_dma_cyclic; -+ sdma->dma_device.device_prep_dma_cyclic = (void*)sdma_prep_dma_cyclic; - sdma->dma_device.device_config = sdma_config; - sdma->dma_device.device_terminate_all = sdma_disable_channel; - sdma->dma_device.src_addr_widths = BIT(DMA_SLAVE_BUSWIDTH_4_BYTES); -@@ -1594,8 +1754,6 @@ - sdma->dma_device.dev->dma_parms = &sdma->dma_parms; - dma_set_max_seg_size(sdma->dma_device.dev, 65535); - -- platform_set_drvdata(pdev, sdma); -- - ret = dma_async_device_register(&sdma->dma_device); - if (ret) { - dev_err(&pdev->dev, "unable to register\n"); -@@ -1647,10 +1805,13 @@ - }, - .id_table = sdma_devtypes, - .remove = sdma_remove, -- .probe = sdma_probe, - }; - --module_platform_driver(sdma_driver); -+static int __init sdma_module_init(void) -+{ -+ return platform_driver_probe(&sdma_driver, sdma_probe); -+} -+module_init(sdma_module_init); - - MODULE_AUTHOR("Sascha Hauer, Pengutronix "); - MODULE_DESCRIPTION("i.MX SDMA driver"); -diff -Nur linux-4.1.3/drivers/gpu/drm/Kconfig linux-xbian-imx6/drivers/gpu/drm/Kconfig ---- linux-4.1.3/drivers/gpu/drm/Kconfig 2015-07-21 19:10:33.000000000 +0200 -+++ linux-xbian-imx6/drivers/gpu/drm/Kconfig 2015-07-27 23:13:04.353449682 +0200 -@@ -217,3 +217,5 @@ - source "drivers/gpu/drm/amd/amdkfd/Kconfig" - - source "drivers/gpu/drm/imx/Kconfig" -+ -+source "drivers/gpu/drm/vivante/Kconfig" -diff -Nur linux-4.1.3/drivers/gpu/drm/Makefile linux-xbian-imx6/drivers/gpu/drm/Makefile ---- linux-4.1.3/drivers/gpu/drm/Makefile 2015-07-21 19:10:33.000000000 +0200 -+++ linux-xbian-imx6/drivers/gpu/drm/Makefile 2015-07-27 23:13:04.353449682 +0200 -@@ -67,6 +67,7 @@ - obj-$(CONFIG_DRM_TEGRA) += tegra/ - obj-$(CONFIG_DRM_STI) += sti/ - obj-$(CONFIG_DRM_IMX) += imx/ -+obj-$(CONFIG_DRM_VIVANTE) += vivante/ - obj-y += i2c/ - obj-y += panel/ - obj-y += bridge/ -diff -Nur linux-4.1.3/drivers/gpu/drm/vivante/Kconfig linux-xbian-imx6/drivers/gpu/drm/vivante/Kconfig ---- linux-4.1.3/drivers/gpu/drm/vivante/Kconfig 1970-01-01 01:00:00.000000000 +0100 -+++ linux-xbian-imx6/drivers/gpu/drm/vivante/Kconfig 2015-07-27 23:13:04.744056042 +0200 -@@ -0,0 +1,6 @@ -+config DRM_VIVANTE -+ tristate "Vivante GCCore" -+ depends on DRM -+ help -+ Choose this option if you have a Vivante graphics card. -+ If M is selected, the module will be called vivante. -diff -Nur linux-4.1.3/drivers/gpu/drm/vivante/Makefile linux-xbian-imx6/drivers/gpu/drm/vivante/Makefile ---- linux-4.1.3/drivers/gpu/drm/vivante/Makefile 1970-01-01 01:00:00.000000000 +0100 -+++ linux-xbian-imx6/drivers/gpu/drm/vivante/Makefile 2015-07-27 23:13:04.744056042 +0200 -@@ -0,0 +1,29 @@ -+############################################################################## -+# -+# Copyright (C) 2005 - 2013 by Vivante Corp. -+# -+# This program is free software; you can redistribute it and/or modify -+# it under the terms of the GNU General Public License as published by -+# the Free Software Foundation; either version 2 of the license, or -+# (at your option) any later version. -+# -+# This program is distributed in the hope that it will be useful, -+# but WITHOUT ANY WARRANTY; without even the implied warranty of -+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -+# GNU General Public License for more details. -+# -+# You should have received a copy of the GNU General Public License -+# along with this program; if not write to the Free Software -+# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA. -+# -+############################################################################## -+ -+ -+# -+# Makefile for the drm device driver. This driver provides support for the -+# Direct Rendering Infrastructure (DRI) in XFree86 4.1.0 and higher. -+ -+ccflags-y := -Iinclude/drm -+vivante-y := vivante_drv.o -+ -+obj-$(CONFIG_DRM_VIVANTE) += vivante.o -diff -Nur linux-4.1.3/drivers/gpu/drm/vivante/vivante_drv.c linux-xbian-imx6/drivers/gpu/drm/vivante/vivante_drv.c ---- linux-4.1.3/drivers/gpu/drm/vivante/vivante_drv.c 1970-01-01 01:00:00.000000000 +0100 -+++ linux-xbian-imx6/drivers/gpu/drm/vivante/vivante_drv.c 2015-07-27 23:13:04.744056042 +0200 -@@ -0,0 +1,112 @@ -+/**************************************************************************** -+* -+* Copyright (C) 2005 - 2013 by Vivante Corp. -+* -+* This program is free software; you can redistribute it and/or modify -+* it under the terms of the GNU General Public License as published by -+* the Free Software Foundation; either version 2 of the license, or -+* (at your option) any later version. -+* -+* This program is distributed in the hope that it will be useful, -+* but WITHOUT ANY WARRANTY; without even the implied warranty of -+* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -+* GNU General Public License for more details. -+* -+* You should have received a copy of the GNU General Public License -+* along with this program; if not write to the Free Software -+* Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA. -+* -+*****************************************************************************/ -+ -+ -+/* vivante_drv.c -- vivante driver -*- linux-c -*- -+ * -+ * -+ * Permission is hereby granted, free of charge, to any person obtaining a -+ * copy of this software and associated documentation files (the "Software"), -+ * to deal in the Software without restriction, including without limitation -+ * the rights to use, copy, modify, merge, publish, distribute, sublicense, -+ * and/or sell copies of the Software, and to permit persons to whom the -+ * Software is furnished to do so, subject to the following conditions: -+ * -+ * The above copyright notice and this permission notice (including the next -+ * paragraph) shall be included in all copies or substantial portions of the -+ * Software. -+ * -+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL -+ * PRECISION INSIGHT AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM, DAMAGES OR -+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, -+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER -+ * DEALINGS IN THE SOFTWARE. -+ * -+ * Authors: -+ * Rickard E. (Rik) Faith -+ * Daryll Strauss -+ * Gareth Hughes -+ */ -+ -+#include -+#include -+ -+#include "drmP.h" -+#include "vivante_drv.h" -+ -+#include "drm_pciids.h" -+ -+static char platformdevicename[] = "Vivante GCCore"; -+static struct platform_device *pplatformdev; -+ -+static const struct file_operations viv_driver_fops = { -+ .owner = THIS_MODULE, -+ .open = drm_open, -+ .release = drm_release, -+ .unlocked_ioctl = drm_ioctl, -+ .mmap = drm_legacy_mmap, -+ .poll = drm_poll, -+ .llseek = noop_llseek, -+}; -+ -+static struct drm_driver driver = { -+// .driver_features = DRIVER_RENDER, -+ .fops = &viv_driver_fops, -+#if LINUX_VERSION_CODE >= KERNEL_VERSION(3, 18, 0) -+ .set_busid = drm_platform_set_busid, -+#endif -+ .name = DRIVER_NAME, -+ .desc = DRIVER_DESC, -+ .date = DRIVER_DATE, -+ .major = DRIVER_MAJOR, -+ .minor = DRIVER_MINOR, -+ .patchlevel = DRIVER_PATCHLEVEL, -+}; -+ -+static int __init vivante_init(void) -+{ -+ int retcode; -+ -+ pplatformdev = platform_device_register_simple(platformdevicename, -+ -1, NULL, 0); -+ if (pplatformdev == NULL) -+ printk(KERN_ERR"Platform device is null\n"); -+ -+ retcode = drm_platform_init(&driver, pplatformdev); -+ -+ return retcode; -+} -+ -+static void __exit vivante_exit(void) -+{ -+ if (pplatformdev) { -+ platform_device_unregister(pplatformdev); -+ pplatformdev = NULL; -+ } -+} -+ -+module_init(vivante_init); -+module_exit(vivante_exit); -+ -+MODULE_AUTHOR(DRIVER_AUTHOR); -+MODULE_DESCRIPTION(DRIVER_DESC); -+MODULE_LICENSE("GPL and additional rights"); -diff -Nur linux-4.1.3/drivers/gpu/drm/vivante/vivante_drv.h linux-xbian-imx6/drivers/gpu/drm/vivante/vivante_drv.h ---- linux-4.1.3/drivers/gpu/drm/vivante/vivante_drv.h 1970-01-01 01:00:00.000000000 +0100 -+++ linux-xbian-imx6/drivers/gpu/drm/vivante/vivante_drv.h 2015-07-27 23:13:04.744056042 +0200 -@@ -0,0 +1,69 @@ -+/**************************************************************************** -+* -+* Copyright (C) 2005 - 2013 by Vivante Corp. -+* -+* This program is free software; you can redistribute it and/or modify -+* it under the terms of the GNU General Public License as published by -+* the Free Software Foundation; either version 2 of the license, or -+* (at your option) any later version. -+* -+* This program is distributed in the hope that it will be useful, -+* but WITHOUT ANY WARRANTY; without even the implied warranty of -+* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -+* GNU General Public License for more details. -+* -+* You should have received a copy of the GNU General Public License -+* along with this program; if not write to the Free Software -+* Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA. -+* -+*****************************************************************************/ -+ -+ -+/* vivante_drv.h -- Vivante DRM template customization -*- linux-c -*- -+ * Created: Wed Feb 14 12:32:32 2012 by John Zhao -+ */ -+/* -+ * -+ * Permission is hereby granted, free of charge, to any person obtaining a -+ * copy of this software and associated documentation files (the "Software"), -+ * to deal in the Software without restriction, including without limitation -+ * the rights to use, copy, modify, merge, publish, distribute, sublicense, -+ * and/or sell copies of the Software, and to permit persons to whom the -+ * Software is furnished to do so, subject to the following conditions: -+ * -+ * The above copyright notice and this permission notice (including the next -+ * paragraph) shall be included in all copies or substantial portions of the -+ * Software. -+ * -+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL -+ * VA LINUX SYSTEMS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM, DAMAGES OR -+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, -+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR -+ * OTHER DEALINGS IN THE SOFTWARE. -+ * -+ * Authors: -+ * Gareth Hughes -+ */ -+ -+#ifndef __VIVANTE_DRV_H__ -+#define __VIVANTE_DRV_H__ -+ -+/* General customization: -+ */ -+ -+#include -+#include -+ -+#define DRIVER_AUTHOR "Vivante Inc." -+ -+#define DRIVER_NAME "vivante" -+#define DRIVER_DESC "Vivante GCCore" -+#define DRIVER_DATE "20120216" -+ -+#define DRIVER_MAJOR 1 -+#define DRIVER_MINOR 0 -+#define DRIVER_PATCHLEVEL 0 -+ -+#endif -diff -Nur linux-4.1.3/drivers/Kconfig linux-xbian-imx6/drivers/Kconfig ---- linux-4.1.3/drivers/Kconfig 2015-07-21 19:10:33.000000000 +0200 -+++ linux-xbian-imx6/drivers/Kconfig 2015-07-27 23:13:03.699781896 +0200 -@@ -182,4 +182,6 @@ - - source "drivers/android/Kconfig" - -+source "drivers/mxc/Kconfig" -+ - endmenu -diff -Nur linux-4.1.3/drivers/Makefile linux-xbian-imx6/drivers/Makefile ---- linux-4.1.3/drivers/Makefile 2015-07-21 19:10:33.000000000 +0200 -+++ linux-xbian-imx6/drivers/Makefile 2015-07-27 23:13:03.699781896 +0200 -@@ -165,3 +165,4 @@ - obj-$(CONFIG_THUNDERBOLT) += thunderbolt/ - obj-$(CONFIG_CORESIGHT) += hwtracing/coresight/ - obj-$(CONFIG_ANDROID) += android/ -+obj-y += mxc/ -diff -Nur linux-4.1.3/drivers/mfd/Kconfig linux-xbian-imx6/drivers/mfd/Kconfig ---- linux-4.1.3/drivers/mfd/Kconfig 2015-07-21 19:10:33.000000000 +0200 -+++ linux-xbian-imx6/drivers/mfd/Kconfig 2015-07-27 23:13:06.023491162 +0200 -@@ -207,6 +207,13 @@ - Additional drivers must be enabled in order to use the specific - features of the device. - -+config MFD_MXC_HDMI -+ tristate "Freescale HDMI Core" -+ select MFD_CORE -+ help -+ This is the core driver for the Freescale i.MX6 on-chip HDMI. -+ This MFD driver connects with the video and audio drivers for HDMI. -+ - config MFD_DLN2 - tristate "Diolan DLN2 support" - select MFD_CORE -diff -Nur linux-4.1.3/drivers/mfd/Makefile linux-xbian-imx6/drivers/mfd/Makefile ---- linux-4.1.3/drivers/mfd/Makefile 2015-07-21 19:10:33.000000000 +0200 -+++ linux-xbian-imx6/drivers/mfd/Makefile 2015-07-27 23:13:06.023491162 +0200 -@@ -185,3 +185,4 @@ - intel-soc-pmic-objs := intel_soc_pmic_core.o intel_soc_pmic_crc.o - obj-$(CONFIG_INTEL_SOC_PMIC) += intel-soc-pmic.o - obj-$(CONFIG_MFD_MT6397) += mt6397-core.o -+obj-$(CONFIG_MFD_MXC_HDMI) += mxc-hdmi-core.o -diff -Nur linux-4.1.3/drivers/mfd/mxc-hdmi-core.c linux-xbian-imx6/drivers/mfd/mxc-hdmi-core.c ---- linux-4.1.3/drivers/mfd/mxc-hdmi-core.c 1970-01-01 01:00:00.000000000 +0100 -+++ linux-xbian-imx6/drivers/mfd/mxc-hdmi-core.c 2015-07-27 23:13:06.051391618 +0200 -@@ -0,0 +1,723 @@ -+/* -+ * Copyright (C) 2011-2014 Freescale Semiconductor, Inc. -+ * -+ * This program is free software; you can redistribute it and/or modify -+ * it under the terms of the GNU General Public License as published by -+ * the Free Software Foundation; either version 2 of the License, or -+ * (at your option) any later version. -+ * -+ * This program is distributed in the hope that it will be useful, -+ * but WITHOUT ANY WARRANTY; without even the implied warranty of -+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -+ * GNU General Public License for more details. -+ * -+ * You should have received a copy of the GNU General Public License -+ * along with this program; if not, write to the Free Software -+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA -+ * -+ */ -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+ -+#include -+#include -+#include -+ -+#include