一、PCIe调试
1.1 编译内核
1.1 配置设备树
pcie设备节点定义在arch/arm64/boot/dts/rockchip/rk3399.dtsi；
pcie0: pcie@f8000000 {
compatible = "rockchip,rk3399-pcie";
reg = <0x0 0xf8000000 0x0 0x2000000>,
<0x0 0xfd000000 0x0 0x1000000>;
reg-names = "axi-base", "apb-base";
device_type = "pci";
#address-cells = <3>;
#size-cells = <2>;
#interrupt-cells = <1>;
aspm-no-l0s;
bus-range = <0x0 0x1f>;
clocks = <&cru ACLK_PCIE>, <&cru ACLK_PERF_PCIE>,
<&cru PCLK_PCIE>, <&cru SCLK_PCIE_PM>;
clock-names = "aclk", "aclk-perf",
"hclk", "pm";
interrupts = <GIC_SPI 49 IRQ_TYPE_LEVEL_HIGH 0>,
<GIC_SPI 50 IRQ_TYPE_LEVEL_HIGH 0>,
<GIC_SPI 51 IRQ_TYPE_LEVEL_HIGH 0>;
interrupt-names = "sys", "legacy", "client";
interrupt-map-mask = <0 0 0 7>;
interrupt-map = <0 0 0 1 &pcie0_intc 0>,
<0 0 0 2 &pcie0_intc 1>,
<0 0 0 3 &pcie0_intc 2>,
<0 0 0 4 &pcie0_intc 3>;
max-link-speed = <1>;
msi-map = <0x0 &its 0x0 0x1000>;
phys = <&pcie_phy 0>, <&pcie_phy 1>,
<&pcie_phy 2>, <&pcie_phy 3>;
phy-names = "pcie-phy-0", "pcie-phy-1",
"pcie-phy-2", "pcie-phy-3";
ranges = <0x82000000 0x0 0xfa000000 0x0 0xfa000000 0x0 0x1e00000>,
<0x81000000 0x0 0xfbe00000 0x0 0xfbe00000 0x0 0x100000>;
resets = <&cru SRST_PCIE_CORE>, <&cru SRST_PCIE_MGMT>,
<&cru SRST_PCIE_MGMT_STICKY>, <&cru SRST_PCIE_PIPE>,
<&cru SRST_PCIE_PM>, <&cru SRST_P_PCIE>,
<&cru SRST_A_PCIE>;
reset-names = "core", "mgmt", "mgmt-sticky", "pipe",
"pm", "pclk", "aclk";
status = "disabled";
pcie0_intc: interrupt-controller {
interrupt-controller;
#address-cells = <0>;
#interrupt-cells = <1>;
};
};
pcie_phy设备节点：
grf: syscon@ff770000 {
compatible = "rockchip,rk3399-grf", "syscon", "simple-mfd";
reg = <0x0 0xff770000 0x0 0x10000>;
#address-cells = <1>;
#size-cells = <1>;
......
pcie_phy: pcie-phy {
compatible = "rockchip,rk3399-pcie-phy";
clocks = <&cru SCLK_PCIEPHY_REF>;
clock-names = "refclk";
#phy-cells = <1>;
resets = <&cru SRST_PCIEPHY>;
reset-names = "phy";
status = "disabled";
};
};
引脚配置节点pcie：
pinctrl: pinctrl {
compatible = "rockchip,rk3399-pinctrl";
rockchip,grf = <&grf>;
rockchip,pmu = <&pmugrf>;
#address-cells = <2>;
#size-cells = <2>;
ranges;
......
pcie {
pcie_clkreqn_cpm: pci-clkreqn-cpm {
rockchip,pins =
<2 RK_PD2 RK_FUNC_GPIO &pcfg_pull_none>;
};
pcie_clkreqnb_cpm: pci-clkreqnb-cpm {
rockchip,pins =
<4 RK_PD0 RK_FUNC_GPIO &pcfg_pull_none>;
};
};
};
如果我们希望pcie使能，则需要在arch/arm64/boot/dts/rockchip/rk3399-evb.dts中为以下节点新增属性：
&pcie_phy {
assigned-clock-parents = <&cru SCLK_PCIEPHY_REF100M>;
assigned-clock-rates = <100000000>;
assigned-clocks = <&cru SCLK_PCIEPHY_REF>;
status = "okay";
};
&pcie0 {
ep-gpios = <&gpio2 RK_PA4 GPIO_ACTIVE_HIGH>;
max-link-speed = <2>;
num-lanes = <2>;
vpcie0v9-supply = <&vcca0v9_s3>;
vpcie1v8-supply = <&vcca1v8_s3>;
status = "okay";
};
/ {
.....
vcca0v9_s3: vcca0v9-s3 {
compatible = "regulator-fixed";
regulator-min-microvolt = <900000>;
regulator-max-microvolt = <900000>;
regulator-name = "vcca0v9_s3";
vin-supply = <&vcc1v8_s3>;
};
/* As above, actually supplied by vcc3v3_sys */
vcca1v8_s3: vcca1v8-s3 {
compatible = "regulator-fixed";
regulator-min-microvolt = <1800000>;
regulator-max-microvolt = <1800000>;
regulator-name = "vcca1v8_s3";
vin-supply = <&vcc1v8_s3>;
};
};
1.1.2 配置内核
linux需要确保如下配置打开，⽅可正确的使⽤PCIe相关功能;
Device Drivers --->
[*] PCI support ---> # CONFIG_PCI
[ ] PCI Express Port Bus support (NEW) # CONFIG_PCIEPORTBUS
[*] PCI Express ASPM control (NEW)
Default ASPM policy (BIOS default) --->
[ ] PCI Express Precision Time Measurement support (NEW)
-*- Message Signaled Interrupts (MSI and MSI-X)
[*] Enable PCI quirk workarounds (NEW)
[ ] PCI Debugging (NEW)
< > PCI Stub driver (NEW)
[ ] PCI IOV support (NEW)
[ ] PCI PRI support (NEW)
[ ] PCI PASID support (NEW)
PCI Express hierarchy optimization setting (Default) --->
[*] VGA Arbitration (NEW)
(16) Maximum number of GPUs (NEW)
[ ] Support for PCI Hotplug (NEW) ----
PCI controller drivers --->
<*> Rockchip PCIe host controller
PCI Endpoint --->
PCI switch controller drivers --->
< > CXL (Compute Express Link) Devices Support (NEW) ----
PHY Subsystem --->
<*> Rockchip NANENG COMBO PHY Driver # CONFIG_PHY_ROCKCHIP_NANENG_COMBO_PHY
<*> Rockchip PCIe PHY Driver # CONFIG_PHY_ROCKCHIP_PCIE # RK3399配置
<*> Rockchip Snps PCIe3 PHY Driver # CONFIG_PHY_ROCKCHIP_SNPS_PCIE3 # RK3568配置
使能NVMe设备(建⽴在PCIe接口的SSD)，PCIe转接AHCI设备（SATA），PCIe转接USB设备(XHCI) 均已在默认config中打开，烦请确认。其他转接设备例如以太⽹卡，WiFi等请⾃⾏确认相关config配置；
Device Drivers --->
NVME Support --->
<*> NVM Express block device # CONFIG_BLK_DEV_NVME
<*> Serial ATA and Parallel ATA drivers (libata) ---> # CONFIG_ATA
[*] SATA Port Multiplier support # CONFIG_SATA_PMP
<*> AHCI SATA support # CONFIG_SATA_AHCI
<*> Platform AHCI SATA support # CONFIG_SATA_AHCI_PLATFORM
[*] ATA SFF support (for legacy IDE and PATA) # CONFIG_ATA_SFF
USB support
xHCI HCD (USB 3.0) support # CONFIG_USB_XHCI_HCD
<*> Support for additional Renesas xHCI controller with firmware # USB_XHCI_PCI_RENESAS
1.1.3 烧录内核
1.2 查看PCI信息
列出所有连接PCI接口的外部设备；
root@rk3399:/# lspci
00:00.0 PCI bridge: Fuzhou Rockchip Electronics Co., Ltd RK3399 PCI Express Root Port
01:00.0 Non-Volatile memory controller: Yangtze Memory Technologies Co.,Ltd Device 0021 (rev 01)
这里我是用的是长江存储致钛TiPlus5000（NVMe M.2）SSD固态硬盘。
列出所有连接PCI接口的外部设备（内容更详细）;
root@rk3399:/# lspci -v
00:00.0 PCI bridge: Fuzhou Rockchip Electronics Co., Ltd RK3399 PCI Express Root Port (prog-if 00 [Normal decode])
Flags: bus master, fast devsel, latency 0, IRQ 80
Bus: primary=00, secondary=01, subordinate=01, sec-latency=0
I/O behind bridge: 00000000-00000fff [size=4K]
Memory behind bridge: fa000000-fa0fffff [size=1M]
Prefetchable memory behind bridge: 00000000-000fffff [size=1M]
Capabilities: [80] Power Management version 3
Capabilities: [90] MSI: Enable+ Count=1/1 Maskable+ 64bit+
Capabilities: [b0] MSI-X: Enable- Count=1 Masked-
Capabilities: [c0] Express Root Port (Slot+), MSI 00
Capabilities: [100] Advanced Error Reporting
Capabilities: [274] Transaction Processing Hints
Kernel driver in use: pcieport
lspci: Unable to load libkmod resources: error -12
01:00.0 Non-Volatile memory controller: Yangtze Memory Technologies Co.,Ltd Device 0021 (rev 01) (prog-if 02 [NVM Express])
Subsystem: Yangtze Memory Technologies Co.,Ltd Device 0021
Flags: bus master, fast devsel, latency 0, IRQ 79, NUMA node 0
Memory at fa000000 (64-bit, non-prefetchable) [size=16K]
Capabilities: [40] Power Management version 3
Capabilities: [50] MSI: Enable- Count=1/32 Maskable+ 64bit+
Capabilities: [70] Express Endpoint, MSI 1f
Capabilities: [b0] MSI-X: Enable+ Count=9 Masked-
Capabilities: [100] Advanced Error Reporting
Capabilities: [148] Device Serial Number 00-00-00-00-00-00-00-00
Capabilities: [158] Alternative Routing-ID Interpretation (ARI)
Capabilities: [168] Secondary PCI Express
Capabilities: [1d4] Latency Tolerance Reporting
Capabilities: [1dc] L1 PM Substates
Capabilities: [1ec] Vendor Specific Information: ID=0002 Rev=4 Len=100 <?>
Capabilities: [2ec] Vendor Specific Information: ID=0001 Rev=1 Len=038 <?>
Kernel driver in use: nvme
1.3 挂载SSD固态硬盘
使用fdisk -l查看磁盘分区情况：
root@rk3399:/# fdisk -l
Disk /dev/mmcblk0: 14.57 GiB, 15634268160 bytes, 30535680 sectors
Units: sectors of 1 * 512 = 512 bytes
Sector size (logical/physical): 512 bytes / 512 bytes
I/O size (minimum/optimal): 512 bytes / 512 bytes
Disklabel type: gpt
Disk identifier: 69DAD710-2CE4-4E3C-B16C-21A1D49ABED3
Device Start End Sectors Size Type
/dev/mmcblk0p1 64 8063 8000 3.9M Microsoft basic data
/dev/mmcblk0p2 16384 24575 8192 4M Microsoft basic data
/dev/mmcblk0p3 24576 32767 8192 4M Microsoft basic data
/dev/mmcblk0p4 32768 262143 229376 112M Microsoft basic data
/dev/mmcblk0p5 262144 30535646 30273503 14.4G Microsoft basic data
Disk /dev/mmcblk0boot0: 4 MiB, 4194304 bytes, 8192 sectors
Units: sectors of 1 * 512 = 512 bytes
Sector size (logical/physical): 512 bytes / 512 bytes
I/O size (minimum/optimal): 512 bytes / 512 bytes
Disk /dev/mmcblk0boot1: 4 MiB, 4194304 bytes, 8192 sectors
Units: sectors of 1 * 512 = 512 bytes
Sector size (logical/physical): 512 bytes / 512 bytes
I/O size (minimum/optimal): 512 bytes / 512 bytes
Disk /dev/nvme0n1: 953.89 GiB, 1024209543168 bytes, 2000409264 sectors
Disk model: ZHITAI TiPlus5000 1TB
Units: sectors of 1 * 512 = 512 bytes
Sector size (logical/physical): 512 bytes / 512 bytes
I/O size (minimum/optimal): 512 bytes / 512 bytes
从上面的内容可以看到第一块磁盘Disk /dev/mmcblk0大小为14.57GB，总容量为1908480*16*512=15,634,268,160字节。Disklabel type: gpt表示分区表为gpt。这实际上就是我们开发板自带的16GB eMMC。
最后一块磁盘Disk /dev/nvme0n1大小为953.89 GiB，这个实际上就是我们使用的SSD固态硬盘了。
1.3.1 格式化
lsblk命令可以列出磁盘分区信息，并显示每个分区的文件系统类型。列出所有分区信息，并显示文件系统类型：
root@rk3399:/data# lsblk -f
NAME FSTYPE LABEL UUID FSAVAIL FSUSE% MOUNTPOINT
mmcblk0
|-mmcblk0p1
|-mmcblk0p2
|-mmcblk0p3
|-mmcblk0p4
`-mmcblk0p5 ext4 816ef390-7441-461e-b5df-547b6e69bd80 9.8G 26% /
mmcblk0boot0
mmcblk0boot1
nvme0n1 ext4 5bb76a82-1d64-432f-9fb9-4d718dae76c1 890.1G 0% /data
运行格式化命令，将磁盘格式化为ext4文件系统：
root@rk3399:/# mkfs.ext4 /dev/nvme0n1
mke2fs 1.45.5 (07-Jan-2020)
Discarding device blocks: done
Creating filesystem with 250051158 4k blocks and 62513152 inodes
Filesystem UUID: 5bb76a82-1d64-432f-9fb9-4d718dae76c1
Superblock backups stored on blocks:
32768, 98304, 163840, 229376, 294912, 819200, 884736, 1605632, 2654208,
4096000, 7962624, 11239424, 20480000, 23887872, 71663616, 78675968,
102400000, 214990848
Allocating group tables: done
Writing inode tables: done
Creating journal (262144 blocks): done
Writing superblocks and filesystem accounting information: done
1.3.2 挂载
运行下面命令进行挂载：
root@rk3399:/# mount /dev/nvme0n1 /data
然后再通过df -hT命令查看整体磁盘挂载情况，这时可以看到硬盘已成功挂载
root@rk3399:/# mount /dev/nvme0n1 /data
root@rk3399:/# df -hT
Filesystem Type Size Used Avail Use% Mounted on
/dev/root ext4 15G 3.8G 9.8G 28% /
devtmpfs devtmpfs 1.9G 0 1.9G 0% /dev
tmpfs tmpfs 1.9G 0 1.9G 0% /dev/shm
tmpfs tmpfs 386M 1.9M 384M 1% /run
tmpfs tmpfs 5.0M 4.0K 5.0M 1% /run/lock
tmpfs tmpfs 1.9G 0 1.9G 0% /sys/fs/cgroup
tmpfs tmpfs 386M 12K 386M 1% /run/user/0
tmpfs tmpfs 386M 8.0K 386M 1% /run/user/1000
/dev/nvme0n1 ext4 938G 28K 891G 1% /data
这里我们可以看到根目录挂载在/dev/root设备节点上，文件系统格式是ext4，/dev/root其实就是/dev/mmcblk0p5，也就是eMMC设备的rootfs分区。
列出所有ext4类型的文件系统信息，并显示文件系统类型；
root@rk3399:/data# mount -t ext4
/dev/mmcblk0p5 on / type ext4 (rw,relatime)
/dev/nvme0n1 on /data type ext4 (rw,relatime)
由于每次手动挂载都很麻烦，因此可以采用以下方法实现它们的自动挂载或卸载。
修改 /etc/fstab文件，添加：
/dev/nvme0n1 /data ext4 defaults 0 0
其中：
- 第1列是设备名或者卷标；
- 第2列是挂载点（也就是你想挂载到哪里）；
- 第3列是所要挂载设备的文件系统或者文件系统类型；
- 第4列是挂载选项，通常使用
defaults就可以； - 第5列设置是否使用
dump备份，置0为不备份，置1，2为备份，但2的备份重要性比1小； - 第6列设置是否开机的时候使用
fsck检验所挂载的磁盘，置0为不检验，置1，2为检验，但置2盘比置1的盘晚检验。；
参考文章
[1] 令人惊艳的NanoPC-T4(RK3399)作为工作站的初始配置和相关应用