dpdk源码-vfio(zym)

主要想找到从nvme设备 到 vfio驱动的代码路径

[spdk/lib/nvme/nvme_pcie.c]

nvme_pcie_ctrlr_scan(struct spdk_nvme_probe_ctx *probe_ctx,bool direct_connect) (nvme_tcp,nvme_rdma等都有自己的scan,probe等函数，统一被封装起来,如下)

1 int nvme_transport_ctrlr_scan(struct spdk_nvme_probe_ctx *probe_ctx,
2      bool direct_connect)
3 {
4  NVME_TRANSPORT_CALL(probe_ctx->trid.trtype, ctrlr_scan, (probe_ctx, direct_connect));
5 }

1.nvme_pcie_ctrl_scan:

 1 int
 2 nvme_pcie_ctrlr_scan(struct spdk_nvme_probe_ctx *probe_ctx,
 3              bool direct_connect)
 4 {
 5     .......//若probe_ctx中有指定nvme pci id，has_pci_addr=true
 6     if (enum_ctx.has_pci_addr == false) {
 7         return spdk_pci_enumerate(spdk_pci_nvme_get_driver(),//若没有指定
 8                       pcie_nvme_enum_cb, &enum_ctx);
 9     } else {
10         return spdk_pci_device_attach(spdk_pci_nvme_get_driver(),//指定了话直接attach
11                           pcie_nvme_enum_cb, &enum_ctx, &enum_ctx.pci_addr);
12     }
13 

spdk_pci_enumerate:

 1 int
 2 spdk_pci_enumerate(struct spdk_pci_driver *driver,
 3            spdk_pci_enum_cb enum_cb,
 4            void *enum_ctx)
 5 {
 6     struct spdk_pci_device *dev;
 7     int rc;
 8 
 9     cleanup_pci_devices();
10 
11     pthread_mutex_lock(&g_pci_mutex);
12     TAILQ_FOREACH(dev, &g_pci_devices, internal.tailq) { //遍历g_pci_devices
13         if (dev->internal.attached ||
14             dev->internal.driver != driver ||
15             dev->internal.pending_removal) {
16             continue;
17         }
18 
19         rc = enum_cb(enum_ctx, dev);// 从g_pci_devices找到我们要的driver，那么调用回调函数进行后续的ctrl_construct工作
20         if (rc == 0) {
21             dev->internal.attached = true;//构建成功，attach设置为true
22         } else if (rc < 0) {
23             pthread_mutex_unlock(&g_pci_mutex);
24             return -1;
25         }
26     }
27     pthread_mutex_unlock(&g_pci_mutex);
28 
29     if (!driver->is_registered) {//若改driver还没注册，那么register到rte_pci_bus.driver_list
30         driver->is_registered = true;
31         rte_pci_register(&driver->driver);
32     }
33 //上面不是找到了吗，下面这些是干啥？为啥要scan和probe？？上面只是放到rte_pci_bus.driver_list了，后面还需做什么？
34     driver->cb_fn = enum_cb;
35     driver->cb_arg = enum_ctx;
36 
37     if (rte_bus_scan() != 0 || rte_bus_probe() != 0) {//若bus scan和probe都不成功
38         driver->cb_arg = NULL;
39         driver->cb_fn = NULL;
40         return -1;
41     }
42 
43     driver->cb_arg = NULL;
44     driver->cb_fn = NULL;
45 
46     cleanup_pci_devices();
47     return 0;
48 }

回调函数pcie_nvme_enum_cb：

主要是执行nvme_ctrlr_probe(&trid, enum_ctx->probe_ctx, pci_dev) 从而构建相应type的ctrl：nvme_transport_ctrlr_construct(trid, &opts, devhandle)，这些不放在这里介绍，会在spdk的源码解读里面分析。

rte_pci_register:

1 /* register a driver */
2 void
3 rte_pci_register(struct rte_pci_driver *driver)
4 {
5     TAILQ_INSERT_TAIL(&rte_pci_bus.driver_list, driver, next);
6     driver->bus = &rte_pci_bus;
7 }//注意这个struct rte_pci_bus rte_pci_bus = { .bus = {  .scan = rte_pci_scan,  .probe = rte_pci_probe,  .find_device = pci_find_device,  .plug = pci_plug,  .unplug = pci_unplug,  .parse = pci_parse,  .dma_map = pci_dma_map,  .dma_unmap = pci_dma_unmap,  .get_iommu_class = rte_pci_get_iommu_class,  .dev_iterate = rte_pci_dev_iterate,  .hot_unplug_handler = pci_hot_unplug_handler,  .sigbus_handler = pci_sigbus_handler, }, .device_list = TAILQ_HEAD_INITIALIZER(rte_pci_bus.device_list), .driver_list = TAILQ_HEAD_INITIALIZER(rte_pci_bus.driver_list),};

rte_bus_scan()和rte_bus_probe() 对所有bus进行scan和probe，若是pci_bus,执行的是上面注册的

.scan = rte_pci_scan, .probe = rte_pci_probe,

回到 1 中的attach函数：

spdk_pci_device_attach：跟原来看的有些不一样，之前看的版本直接调用的是rte_pci_scan

 1 int
 2 spdk_pci_device_attach(struct spdk_pci_driver *driver,
 3                spdk_pci_enum_cb enum_cb,
 4                void *enum_ctx, struct spdk_pci_addr *pci_address)
 5 {
 6     struct spdk_pci_device *dev;
 7     int rc;
 8     char bdf[32];
 9 
10     spdk_pci_addr_fmt(bdf, sizeof(bdf), pci_address);
11 
12     cleanup_pci_devices();
13 
14     TAILQ_FOREACH(dev, &g_pci_devices, internal.tailq) {
15         if (spdk_pci_addr_compare(&dev->addr, pci_address) == 0) {
16             break;
17         }
18     }
19 
20     if (dev != NULL && dev->internal.driver == driver) {
21         pthread_mutex_lock(&g_pci_mutex);
22         if (dev->internal.attached || dev->internal.pending_removal) {
23             pthread_mutex_unlock(&g_pci_mutex);
24             return -1;
25         }
26 
27         rc = enum_cb(enum_ctx, dev);//执行construct ctrl等
28         if (rc == 0) {
29             dev->internal.attached = true;
30         }
31         pthread_mutex_unlock(&g_pci_mutex);
32         return rc;
33     }
34 
35     if (!driver->is_registered) {
36         driver->is_registered = true;
37         rte_pci_register(&driver->driver);//register到rte_pci_bus.driver_list
38     }
39 
40     driver->cb_fn = enum_cb;
41     driver->cb_arg = enum_ctx;
42 
43 #if RTE_VERSION >= RTE_VERSION_NUM(18, 11, 0, 0)
44     int i = 0;
45 
46     do {
47         rc = rte_eal_hotplug_add("pci", bdf, "");
48     } while (rc == -ENOMSG && ++i <= DPDK_HOTPLUG_RETRY_COUNT);
49 
50     if (i > 1 && rc == -EEXIST) {
51         /* Even though the previous request timed out, the device
52          * was attached successfully.
53          */
54         rc = 0;
55     }
56 #else
57     rc = rte_eal_dev_attach(bdf, "");//这个函数在旧版本的dpdk中
58 #endif
59 
60     driver->cb_arg = NULL;
61     driver->cb_fn = NULL;
62 
63     cleanup_pci_devices();
64     return rc == 0 ? 0 : -1;
65 }

rte_eal_hotplug_add:

--------rte_dev_probe(devargs)

 1 int
 2 rte_dev_probe(const char *devargs)
 3 {
 4     struct eal_dev_mp_req req;
 5     struct rte_device *dev;
 6     int ret;
 7 
 8     memset(&req, 0, sizeof(req));
 9     req.t = EAL_DEV_REQ_TYPE_ATTACH;
10     strlcpy(req.devargs, devargs, EAL_DEV_MP_DEV_ARGS_MAX_LEN);
11 
12     if (rte_eal_process_type() != RTE_PROC_PRIMARY) {//不是primary进程
13         /**
14          * If in secondary process, just send IPC request to
15          * primary process.
16          */
17         ret = eal_dev_hotplug_request_to_primary(&req);//那么给primary进程发attch的请求
18         if (ret != 0) {
19             RTE_LOG(ERR, EAL,
20                 "Failed to send hotplug request to primary
");
21             return -ENOMSG;
22         }
23         if (req.result != 0)
24             RTE_LOG(ERR, EAL,
25                 "Failed to hotplug add device
");
26         return req.result;
27     }
28 
29     /* attach a shared device from primary start from here: */
31     /* primary attach the new device itself. */
32     ret = local_dev_probe(devargs, &dev);//其中会执行da->bus->scan()
33 
34     if (ret != 0) {
35         RTE_LOG(ERR, EAL,
36             "Failed to attach device on primary process
");
37 
38         /**
39          * it is possible that secondary process failed to attached a
40          * device that primary process have during initialization,
41          * so for -EEXIST case, we still need to sync with secondary
42          * process.
43          */
44         if (ret != -EEXIST)
45             return ret;
46     }
47 
48     /* primary send attach sync request to secondary. */
49     ret = eal_dev_hotplug_request_to_secondary(&req);//给secondary进程发消息同步attach情况
50 
51     /* if any communication error, we need to rollback. */
52     if (ret != 0) {
53         RTE_LOG(ERR, EAL,
54             "Failed to send hotplug add request to secondary
");
55         ret = -ENOMSG;
56         goto rollback;
57     }
59     /**
60      * if any secondary failed to attach, we need to consider if rollback
61      * is necessary.
62      */
63     if (req.result != 0) {
64         RTE_LOG(ERR, EAL,
65             "Failed to attach device on secondary process
");
66         ret = req.result;
67 
68         /* for -EEXIST, we don't need to rollback. */
69         if (ret == -EEXIST)
70             return ret;
71         goto rollback;
72     }
73 
74     return 0;
75 
76 rollback:
77     req.t = EAL_DEV_REQ_TYPE_ATTACH_ROLLBACK;
78 
79     /* primary send rollback request to secondary. */
80     if (eal_dev_hotplug_request_to_secondary(&req) != 0)
81         RTE_LOG(WARNING, EAL,
82             "Failed to rollback device attach on secondary."
83             "Devices in secondary may not sync with primary
");
84 
85     /* primary rollback itself. */
86     if (local_dev_remove(dev) != 0)
87         RTE_LOG(WARNING, EAL,
88             "Failed to rollback device attach on primary."
89             "Devices in secondary may not sync with primary
");
90 
91     return ret;
92 }

local_dev_probe:

最终也是调用rte_pci_scan和pci_probe_all_drivers

 1 /* probe device at local process. */
 2 int
 3 local_dev_probe(const char *devargs, struct rte_device **new_dev)
 4 {
 5     struct rte_device *dev;
 6     struct rte_devargs *da;
 7     int ret;
 8 
 9     *new_dev = NULL;
10     da = calloc(1, sizeof(*da));
11     if (da == NULL)
12         return -ENOMEM;
13 
14     ret = rte_devargs_parse(da, devargs);
15     if (ret)
16         goto err_devarg;
17 
18     if (da->bus->plug == NULL) {
19         RTE_LOG(ERR, EAL, "Function plug not supported by bus (%s)
",
20             da->bus->name);
21         ret = -ENOTSUP;
22         goto err_devarg;
23     }
24 
25     ret = rte_devargs_insert(&da);
26     if (ret)
27         goto err_devarg;
28 
29     /* the rte_devargs will be referenced in the matching rte_device */
30     ret = da->bus->scan();//pci的话执行register函数中注册的rte_pci_scan
31     if (ret)
32         goto err_devarg;
33 
34     dev = da->bus->find_device(NULL, cmp_dev_name, da->name);
35     if (dev == NULL) {
36         RTE_LOG(ERR, EAL, "Cannot find device (%s)
",
37             da->name);
38         ret = -ENODEV;
39         goto err_devarg;
40     }
41     /* Since there is a matching device, it is now its responsibility
42      * to manage the devargs we've just inserted. From this point
43      * those devargs shouldn't be removed manually anymore.
44      */
45 
46     ret = dev->bus->plug(dev);//rte_pci_plug中执行的是pci_probe_all_drivers(RTE_DEV_TO_PCI(dev))
47     if (ret > 0)
48         ret = -ENOTSUP;
49 
50     if (ret && !rte_dev_is_probed(dev)) { /* if hasn't ever succeeded */
51         RTE_LOG(ERR, EAL, "Driver cannot attach the device (%s)
",
52             dev->name);
53         return ret;
54     }
55 
56     *new_dev = dev;
57     return ret;
58 
59 err_devarg:
60     if (rte_devargs_remove(da) != 0) {
61         free(da->args);
62         free(da);
63     }
64     return ret;
65 }

所以 1 中 nvme_pcie_ctrl_scan 不管是spdk_pci_enumerate还是spdk_pci_device_attach，核心的流程都是一样的：

执行回调函数pcie_nvme_enum_cb；

rte_pci_scan

rte_pci_probe/pci_probe_all_drivers (rte_pci_probe也是调用pci_probe_all_drivers)

所以后面我们看 rte_pci_bus.scan（rte_pci_scan）和rte_pci_bus.probe（rte_pci_probe）的实现。

rte_pci_scan:

 1 /*
 2  * Scan the content of the PCI bus, and the devices in the devices
 3  * list
 4  */
 5 int
 6 rte_pci_scan(void)
 7 {
 8     struct dirent *e;
 9     DIR *dir;
10     char dirname[PATH_MAX];
11     struct rte_pci_addr addr;
12     
13         ........
14     dir = opendir(rte_pci_get_sysfs_path()); ///sys/bus/pci/devices
15     
16         ........
17     while ((e = readdir(dir)) != NULL) {
18         if (e->d_name[0] == '.')
19             continue;
20 
21         if (parse_pci_addr_format(e->d_name, sizeof(e->d_name), &addr) != 0)
22             continue;
23 
24         snprintf(dirname, sizeof(dirname), "%s/%s",
25                 rte_pci_get_sysfs_path(), e->d_name);
26 
27         if (pci_scan_one(dirname, &addr) < 0)//扫描/sys/bus/pci/devices下面每个目录
28             goto error;
29     }
30     closedir(dir);
31     return 0;
32 ..........
33 }

ls /sys/bus/pci/devices/ 0000:00:00.0 0000:00:04.4 0000:00:1c.6 0000:03:00.0 0000:ff:0d.1 0000:ff:12.2 0000:ff:14.5 0000:ff:17.6 0000:00:01.0 0000:00:04.5 0000:00:1c.7 0000:05:00.0 0000:ff:0f.0 0000:ff:12.4 0000:ff:14.6 0000:ff:17.7

  1 /* Scan one pci sysfs entry, and fill the devices list from it. */
  2 static int
  3 pci_scan_one(const char *dirname, const struct rte_pci_addr *addr)
  4 {
  5     char filename[PATH_MAX];
  6     unsigned long tmp;
  7     struct rte_pci_device *dev;
  8     char driver[PATH_MAX];
  9     int ret;
 10 
 11     dev = malloc(sizeof(*dev));
 12     if (dev == NULL)
 13         return -1;
 14 
 15     memset(dev, 0, sizeof(*dev));
 16     dev->device.bus = &rte_pci_bus.bus;
 17     dev->addr = *addr;
 18 
 19     /* get vendor id */
 20     snprintf(filename, sizeof(filename), "%s/vendor", dirname);
 21     if (eal_parse_sysfs_value(filename, &tmp) < 0) {
 22         free(dev);
 23         return -1;
 24     }
 25     dev->id.vendor_id = (uint16_t)tmp;
 26 
 27     /* get device id */
 28     snprintf(filename, sizeof(filename), "%s/device", dirname);
 29     if (eal_parse_sysfs_value(filename, &tmp) < 0) {
 30         free(dev);
 31         return -1;
 32     }
 33     dev->id.device_id = (uint16_t)tmp;
 34 
 35     /* get subsystem_vendor id */
 36     snprintf(filename, sizeof(filename), "%s/subsystem_vendor",
 37          dirname);
 38     if (eal_parse_sysfs_value(filename, &tmp) < 0) {
 39         free(dev);
 40         return -1;
 41     }
 42     dev->id.subsystem_vendor_id = (uint16_t)tmp;
 43 
 44     /* get subsystem_device id */
 45     snprintf(filename, sizeof(filename), "%s/subsystem_device",
 46          dirname);
 47     if (eal_parse_sysfs_value(filename, &tmp) < 0) {
 48         free(dev);
 49         return -1;
 50     }
 51     dev->id.subsystem_device_id = (uint16_t)tmp;
 52 
 53     /* get class_id */
 54     snprintf(filename, sizeof(filename), "%s/class",
 55          dirname);
 56     if (eal_parse_sysfs_value(filename, &tmp) < 0) {
 57         free(dev);
 58         return -1;
 59     }
 60     /* the least 24 bits are valid: class, subclass, program interface */
 61     dev->id.class_id = (uint32_t)tmp & RTE_CLASS_ANY_ID;
 62 
 63     /* get max_vfs */
 64     dev->max_vfs = 0;
 65     snprintf(filename, sizeof(filename), "%s/max_vfs", dirname);
 66     if (!access(filename, F_OK) &&
 67         eal_parse_sysfs_value(filename, &tmp) == 0)
 68         dev->max_vfs = (uint16_t)tmp;
 69     else {
 70         /* for non igb_uio driver, need kernel version >= 3.8 */
 71         snprintf(filename, sizeof(filename),
 72              "%s/sriov_numvfs", dirname);
 73         if (!access(filename, F_OK) &&
 74             eal_parse_sysfs_value(filename, &tmp) == 0)
 75             dev->max_vfs = (uint16_t)tmp;
 76     }
 77 
 78     /* get numa node, default to 0 if not present */
 79     snprintf(filename, sizeof(filename), "%s/numa_node",
 80          dirname);
 81 
 82     if (access(filename, F_OK) != -1) {
 83         if (eal_parse_sysfs_value(filename, &tmp) == 0)
 84             dev->device.numa_node = tmp;
 85         else
 86             dev->device.numa_node = -1;
 87     } else {
 88         dev->device.numa_node = 0;
 89     }
 90 
 91     pci_name_set(dev);
 92 
 93     /* parse resources */
 94     snprintf(filename, sizeof(filename), "%s/resource", dirname);
 95     if (pci_parse_sysfs_resource(filename, dev) < 0) {
 96         RTE_LOG(ERR, EAL, "%s(): cannot parse resource
", __func__);
 97         free(dev);
 98         return -1;
 99     }
100 
101     /* parse driver */
102     snprintf(filename, sizeof(filename), "%s/driver", dirname);///sys/bus/pci/drivers
103     ret = pci_get_kernel_driver_by_path(filename, driver, sizeof(driver));
104     if (ret < 0) {
105         RTE_LOG(ERR, EAL, "Fail to get kernel driver
");
106         free(dev);
107         return -1;
108     }
109 
110     if (!ret) {
111         if (!strcmp(driver, "vfio-pci"))
112             dev->kdrv = RTE_KDRV_VFIO;
113         else if (!strcmp(driver, "igb_uio"))
114             dev->kdrv = RTE_KDRV_IGB_UIO;
115         else if (!strcmp(driver, "uio_pci_generic"))
116             dev->kdrv = RTE_KDRV_UIO_GENERIC;
117         else
118             dev->kdrv = RTE_KDRV_UNKNOWN;
119     } else
120         dev->kdrv = RTE_KDRV_NONE;
121 
122     /* device is valid, add in list (sorted) */
123     if (TAILQ_EMPTY(&rte_pci_bus.device_list)) {
124         rte_pci_add_device(dev);
125     } else {
126         struct rte_pci_device *dev2;
127         int ret;
128 
129         TAILQ_FOREACH(dev2, &rte_pci_bus.device_list, next) {
130             ret = rte_pci_addr_cmp(&dev->addr, &dev2->addr);
131             if (ret > 0)
132                 continue;
133 
134             if (ret < 0) {
135                 rte_pci_insert_device(dev2, dev);
136             } else { /* already registered */
137                 if (!rte_dev_is_probed(&dev2->device)) {
138                     dev2->kdrv = dev->kdrv;
139                     dev2->max_vfs = dev->max_vfs;
140                     pci_name_set(dev2);
141                     memmove(dev2->mem_resource,
142                         dev->mem_resource,
143                         sizeof(dev->mem_resource));
144                 } else {
145                     /**
146                      * If device is plugged and driver is
147                      * probed already, (This happens when
148                      * we call rte_dev_probe which will
149                      * scan all device on the bus) we don't
150                      * need to do anything here unless...
151                      **/
152                     if (dev2->kdrv != dev->kdrv ||
153                         dev2->max_vfs != dev->max_vfs)
154                         /*
155                          * This should not happens.
156                          * But it is still possible if
157                          * we unbind a device from
158                          * vfio or uio before hotplug
159                          * remove and rebind it with
160                          * a different configure.
161                          * So we just print out the
162                          * error as an alarm.
163                          */
164                         RTE_LOG(ERR, EAL, "Unexpected device scan at %s!
",
165                             filename);
166                 }
167                 free(dev);
168             }
169             return 0;
170         }
171 
172         rte_pci_add_device(dev);
173     }
174 
175     return 0;
176 }

rte_pci_probe:

------FOREACH_DEVICE_ON_PCIBUS(dev) pci_probe_all_drivers(dev)

------------FOREACH_DRIVER_ON_PCIBUS(dr) rte_pci_probe_one_driver(dr, dev)

  1 /*
  2  * If vendor/device ID match, call the probe() function of the
  3  * driver.
  4  */
  5 static int
  6 rte_pci_probe_one_driver(struct rte_pci_driver *dr,
  7              struct rte_pci_device *dev)
  8 {
  9     int ret;
 10     bool already_probed;
 11     struct rte_pci_addr *loc;
 12 
 13     if ((dr == NULL) || (dev == NULL))
 14         return -EINVAL;
 15 
 16     loc = &dev->addr;
 17 
 18     /* The device is not blacklisted; Check if driver supports it */
 19     if (!rte_pci_match(dr, dev))
 20         /* Match of device and driver failed */
 21         return 1;
 22 
 23     RTE_LOG(INFO, EAL, "PCI device "PCI_PRI_FMT" on NUMA socket %i
",
 24             loc->domain, loc->bus, loc->devid, loc->function,
 25             dev->device.numa_node);
 26 
 27     /* no initialization when blacklisted, return without error */
 28     if (dev->device.devargs != NULL &&
 29         dev->device.devargs->policy ==
 30             RTE_DEV_BLACKLISTED) {
 31         RTE_LOG(INFO, EAL, "  Device is blacklisted, not"
 32             " initializing
");
 33         return 1;
 34     }
 35 
 36     if (dev->device.numa_node < 0) {
 37         RTE_LOG(WARNING, EAL, "  Invalid NUMA socket, default to 0
");
 38         dev->device.numa_node = 0;
 39     }
 40 
 41     already_probed = rte_dev_is_probed(&dev->device);
 42     if (already_probed && !(dr->drv_flags & RTE_PCI_DRV_PROBE_AGAIN)) {
 43         RTE_LOG(DEBUG, EAL, "Device %s is already probed
",
 44                 dev->device.name);
 45         return -EEXIST;
 46     }
 47 
 48     RTE_LOG(INFO, EAL, "  probe driver: %x:%x %s
", dev->id.vendor_id,
 49         dev->id.device_id, dr->driver.name);
 50 
 51     /*
 52      * reference driver structure
 53      * This needs to be before rte_pci_map_device(), as it enables to use
 54      * driver flags for adjusting configuration.
 55      */
 56     if (!already_probed) {
 57         enum rte_iova_mode dev_iova_mode;
 58         enum rte_iova_mode iova_mode;
 59 
 60         dev_iova_mode = pci_device_iova_mode(dr, dev);//从设备绑定的驱动判断dev_iova_mode,例如若是UIO驱动，则这里是PA
 61         iova_mode = rte_eal_iova_mode();//dpdk初始化过程判断的，首先会去判断设备的驱动，如果驱动还没注册，那么里面是根据机器上是否有iommu等判断。而spdk中nvme     驱动是在dpdk初始化之后才注册进去的，所以即使是用uio驱动，dpdk给出的iova_mode也是VA，所以这里有点小问题！ 62         if (dev_iova_mode != RTE_IOVA_DC &&
 63             dev_iova_mode != iova_mode) {
 64             RTE_LOG(ERR, EAL, "  Expecting '%s' IOVA mode but current mode is '%s', not initializing
",
 65                 dev_iova_mode == RTE_IOVA_PA ? "PA" : "VA",
 66                 iova_mode == RTE_IOVA_PA ? "PA" : "VA");
 67             return -EINVAL;
 68         }
 69 
 70         dev->driver = dr;
 71     }
 72 
 73     if (!already_probed && (dr->drv_flags & RTE_PCI_DRV_NEED_MAPPING)) {
 74         /* map resources for devices that use igb_uio */？？？注释有问题??
 75         ret = rte_pci_map_device(dev);
 76         if (ret != 0) {
 77             dev->driver = NULL;
 78             return ret;
 79         }
 80     }
 81 
 82     /* call the driver probe() function */
 83     ret = dr->probe(dr, dev);  //在哪里？？？？？
 84     if (already_probed)
 85         return ret; /* no rollback if already succeeded earlier */
 86     if (ret) {
 87         dev->driver = NULL;
 88         if ((dr->drv_flags & RTE_PCI_DRV_NEED_MAPPING) &&
 89             /* Don't unmap if device is unsupported and
 90              * driver needs mapped resources.
 91              */
 92             !(ret > 0 &&
 93                 (dr->drv_flags & RTE_PCI_DRV_KEEP_MAPPED_RES)))
 94             rte_pci_unmap_device(dev);
 95     } else {
 96         dev->device.driver = &dr->driver;
 97     }
 98 
 99     return ret;
100 }

rte_pci_map_device 从这遍开始和vfio，uio等相关

 1 /* Map pci device */
 2 int
 3 rte_pci_map_device(struct rte_pci_device *dev)
 4 {
 5     int ret = -1;
 6 
 7     /* try mapping the NIC resources using VFIO if it exists */
 8     switch (dev->kdrv) {
 9     case RTE_KDRV_VFIO:
10 #ifdef VFIO_PRESENT
11         if (pci_vfio_is_enabled())
12             ret = pci_vfio_map_resource(dev);
13 #endif
14         break;
15     case RTE_KDRV_IGB_UIO:
16     case RTE_KDRV_UIO_GENERIC:
17         if (rte_eal_using_phys_addrs()) {
18             /* map resources for devices that use uio */
19             ret = pci_uio_map_resource(dev);
20         }
21         break;
22     default:
23         RTE_LOG(DEBUG, EAL,
24             "  Not managed by a supported kernel driver, skipped
");
25         ret = 1;
26         break;
27     }
28 
29     return ret;
30 }

pci_vfio_map_resource:

---1---pci_vfio_map_resource_primary 若设primary进程

------pci_vfio_map_resource_secondary

-------2---------rte_vfio_setup_device

------------3------------vfio_mem_event_callback

这三个函数要重点看。。。。

原文链接：https://www.cnblogs.com/yi-mu-xi/p/12441357.html 原文作者：yimuxi

LinuxC/C++服务器开发/架构师 面试题、学习资料、教学视频和学习路线图（资料包括C/C++，Linux，Nginx，ZeroMQ，MySQL，Redis、MongoDB，ZK，流媒体，CDN，P2P，K8S，Docker，TCP/IP，协程，DPDK等），或点击这里加qun免费领取,关注我持续更新哦! !

​