stubdom: PCI passthrough support via PV-PCI

Signed-off-by: Samuel Thibault <[email protected]>
royger · Jul 2, 2008 · de4e084 · de4e084
1 parent dac6f79
commit de4e084
Show file tree

Hide file tree

Showing 11 changed files with 638 additions and 7 deletions.
diff --git a/extras/mini-os/include/pcifront.h b/extras/mini-os/include/pcifront.h
@@ -0,0 +1,15 @@
+#include <types.h>
+#include <xen/io/pciif.h>
+struct pcifront_dev;
+struct pcifront_dev *init_pcifront(char *nodename);
+void pcifront_scan(struct pcifront_dev *dev, void (*fun)(unsigned int domain, unsigned int bus, unsigned slot, unsigned int fun));
+void pcifront_op(struct pcifront_dev *dev, struct xen_pci_op *op);
+void shutdown_pcifront(struct pcifront_dev *dev);
+int pcifront_conf_read(struct pcifront_dev *dev,
+                       unsigned int dom,
+                       unsigned int bus, unsigned int slot, unsigned long fun,
+                       unsigned int off, unsigned int size, unsigned int *val);
+int pcifront_conf_write(struct pcifront_dev *dev,
+                        unsigned int dom,
+                        unsigned int bus, unsigned int slot, unsigned long fun,
+                        unsigned int off, unsigned int size, unsigned int val);
diff --git a/extras/mini-os/include/posix/sys/mman.h b/extras/mini-os/include/posix/sys/mman.h
@@ -9,6 +9,9 @@
 #define MAP_PRIVATE	0x02
 #define MAP_ANON	0x20
 
+/* Pages are always resident anyway */
+#define MAP_LOCKED	0x0
+
 #define MAP_FAILED	((void*)0)
 
 void *mmap(void *start, size_t length, int prot, int flags, int fd, off_t offset);

diff --git a/extras/mini-os/kernel.c b/extras/mini-os/kernel.c
@@ -40,6 +40,7 @@
 #include <netfront.h>
 #include <blkfront.h>
 #include <fbfront.h>
+#include <pcifront.h>
 #include <fs.h>
 #include <xmalloc.h>
 #include <fcntl.h>
@@ -431,6 +432,27 @@ static void kbdfront_thread(void *p)
     }
 }
 
+static struct pcifront_dev *pci_dev;
+
+static void pcifront_thread(void *p)
+{
+    void print(unsigned int domain, unsigned int bus, unsigned int slot, unsigned int fun)
+    {
+        unsigned int vendor, device, rev, class;
+
+        pcifront_conf_read(pci_dev, domain, bus, slot, fun, 0x00, 2, &vendor);
+        pcifront_conf_read(pci_dev, domain, bus, slot, fun, 0x02, 2, &device);
+        pcifront_conf_read(pci_dev, domain, bus, slot, fun, 0x08, 1, &rev);
+        pcifront_conf_read(pci_dev, domain, bus, slot, fun, 0x0a, 2, &class);
+
+        printk("%04x:%02x:%02x.%02x %04x: %04x:%04x (rev %02x)\n", domain, bus, slot, fun, class, vendor, device, rev);
+    }
+
+    pci_dev = init_pcifront(NULL);
+    printk("PCI devices:\n");
+    pcifront_scan(pci_dev, print);
+}
+
 static void fs_thread(void *p)
 {
     init_fs_frontend();
@@ -446,6 +468,7 @@ __attribute__((weak)) int app_main(start_info_t *si)
     create_thread("blkfront", blkfront_thread, si);
     create_thread("fbfront", fbfront_thread, si);
     create_thread("kbdfront", kbdfront_thread, si);
+    create_thread("pcifront", pcifront_thread, si);
     create_thread("fs-frontend", fs_thread, si);
     return 0;
 }
@@ -524,6 +547,9 @@ void stop_kernel(void)
     if (kbd_dev)
         shutdown_kbdfront(kbd_dev);
 
+    if (pci_dev)
+        shutdown_pcifront(pci_dev);
+
     /* TODO: fs import */
 
     local_irq_disable();

diff --git a/extras/mini-os/pcifront.c b/extras/mini-os/pcifront.c
@@ -0,0 +1,278 @@
+/* Minimal PCI driver for Mini-OS. 
+ * Copyright (c) 2007-2008 Samuel Thibault.
+ * Based on blkfront.c.
+ */
+
+#include <os.h>
+#include <xenbus.h>
+#include <events.h>
+#include <errno.h>
+#include <gnttab.h>
+#include <xmalloc.h>
+#include <wait.h>
+#include <pcifront.h>
+
+#define PCI_DEVFN(slot, func) ((((slot) & 0x1f) << 3) | ((func) & 0x07))
+
+DECLARE_WAIT_QUEUE_HEAD(pcifront_queue);
+
+struct pcifront_dev {
+    domid_t dom;
+
+    struct xen_pci_sharedinfo *info;
+    grant_ref_t info_ref;
+    evtchn_port_t evtchn;
+
+    char *nodename;
+    char *backend;
+
+    xenbus_event_queue events;
+};
+
+void pcifront_handler(evtchn_port_t port, struct pt_regs *regs, void *data)
+{
+    wake_up(&pcifront_queue);
+}
+
+static void free_pcifront(struct pcifront_dev *dev)
+{
+    mask_evtchn(dev->evtchn);
+
+    free(dev->backend);
+
+    gnttab_end_access(dev->info_ref);
+    free_page(dev->info);
+
+    unbind_evtchn(dev->evtchn);
+
+    free(dev->nodename);
+    free(dev);
+}
+
+struct pcifront_dev *init_pcifront(char *nodename)
+{
+    xenbus_transaction_t xbt;
+    char* err;
+    char* message=NULL;
+    int retry=0;
+    char* msg;
+
+    struct pcifront_dev *dev;
+
+    if (!nodename)
+        nodename = "device/pci/0";
+
+    char path[strlen(nodename) + 1 + 10 + 1];
+
+    printk("******************* PCIFRONT for %s **********\n\n\n", nodename);
+
+    dev = malloc(sizeof(*dev));
+    memset(dev, 0, sizeof(*dev));
+    dev->nodename = strdup(nodename);
+
+    snprintf(path, sizeof(path), "%s/backend-id", nodename);
+    dev->dom = xenbus_read_integer(path); 
+    evtchn_alloc_unbound(dev->dom, pcifront_handler, dev, &dev->evtchn);
+
+    dev->info = (struct xen_pci_sharedinfo*) alloc_page();
+    memset(dev->info,0,PAGE_SIZE);
+
+    dev->info_ref = gnttab_grant_access(dev->dom,virt_to_mfn(dev->info),0);
+
+    dev->events = NULL;
+
+again:
+    err = xenbus_transaction_start(&xbt);
+    if (err) {
+        printk("starting transaction\n");
+    }
+
+    err = xenbus_printf(xbt, nodename, "pci-op-ref","%u",
+                dev->info_ref);
+    if (err) {
+        message = "writing pci-op-ref";
+        goto abort_transaction;
+    }
+    err = xenbus_printf(xbt, nodename,
+                "event-channel", "%u", dev->evtchn);
+    if (err) {
+        message = "writing event-channel";
+        goto abort_transaction;
+    }
+    err = xenbus_printf(xbt, nodename,
+                "magic", XEN_PCI_MAGIC);
+    if (err) {
+        message = "writing magic";
+        goto abort_transaction;
+    }
+
+    err = xenbus_printf(xbt, nodename, "state", "%u",
+            3); /* initialised */
+
+
+    err = xenbus_transaction_end(xbt, 0, &retry);
+    if (retry) {
+            goto again;
+        printk("completing transaction\n");
+    }
+
+    goto done;
+
+abort_transaction:
+    xenbus_transaction_end(xbt, 1, &retry);
+    goto error;
+
+done:
+
+    snprintf(path, sizeof(path), "%s/backend", nodename);
+    msg = xenbus_read(XBT_NIL, path, &dev->backend);
+    if (msg) {
+        printk("Error %s when reading the backend path %s\n", msg, path);
+        goto error;
+    }
+
+    printk("backend at %s\n", dev->backend);
+
+    {
+        char path[strlen(dev->backend) + 1 + 5 + 1];
+        snprintf(path, sizeof(path), "%s/state", dev->backend);
+
+        xenbus_watch_path_token(XBT_NIL, path, path, &dev->events);
+
+        xenbus_wait_for_value(path, "4", &dev->events);
+
+        xenbus_printf(xbt, nodename, "state", "%u", 4); /* connected */
+    }
+    unmask_evtchn(dev->evtchn);
+
+    printk("**************************\n");
+
+    return dev;
+
+error:
+    free_pcifront(dev);
+    return NULL;
+}
+
+void pcifront_scan(struct pcifront_dev *dev, void (*func)(unsigned int domain, unsigned int bus, unsigned slot, unsigned int fun))
+{
+    char path[strlen(dev->backend) + 1 + 5 + 10 + 1];
+    int i, n;
+    char *s, *msg;
+    unsigned int domain, bus, slot, fun;
+
+    snprintf(path, sizeof(path), "%s/num_devs", dev->backend);
+    n = xenbus_read_integer(path);
+
+    for (i = 0; i < n; i++) {
+        snprintf(path, sizeof(path), "%s/vdev-%d", dev->backend, i);
+        msg = xenbus_read(XBT_NIL, path, &s);
+        if (msg) {
+            printk("Error %s when reading the PCI root name at %s\n", path);
+            continue;
+        }
+
+        if (sscanf(s, "%x:%x:%x.%x", &domain, &bus, &slot, &fun) != 4) {
+            printk("\"%s\" does not look like a PCI device address\n", s);
+            free(s);
+            continue;
+        }
+        free(s);
+
+        func(domain, bus, slot, fun);
+    }
+}
+
+void shutdown_pcifront(struct pcifront_dev *dev)
+{
+    char* err;
+    char *nodename = dev->nodename;
+
+    char path[strlen(dev->backend) + 1 + 5 + 1];
+
+    printk("close pci: backend at %s\n",dev->backend);
+
+    snprintf(path, sizeof(path), "%s/state", dev->backend);
+    err = xenbus_printf(XBT_NIL, nodename, "state", "%u", 5); /* closing */
+    xenbus_wait_for_value(path, "5", &dev->events);
+
+    err = xenbus_printf(XBT_NIL, nodename, "state", "%u", 6);
+    xenbus_wait_for_value(path, "6", &dev->events);
+
+    err = xenbus_printf(XBT_NIL, nodename, "state", "%u", 1);
+    xenbus_wait_for_value(path, "2", &dev->events);
+
+    xenbus_unwatch_path(XBT_NIL, path);
+
+    snprintf(path, sizeof(path), "%s/info-ref", nodename);
+    xenbus_rm(XBT_NIL, path);
+    snprintf(path, sizeof(path), "%s/event-channel", nodename);
+    xenbus_rm(XBT_NIL, path);
+
+    free_pcifront(dev);
+}
+
+
+void pcifront_op(struct pcifront_dev *dev, struct xen_pci_op *op)
+{
+    dev->info->op = *op;
+    /* Make sure info is written before the flag */
+    wmb();
+    set_bit(_XEN_PCIF_active, &dev->info->flags);
+    notify_remote_via_evtchn(dev->evtchn);
+
+    wait_event(pcifront_queue, !test_bit(_XEN_PCIF_active, &dev->info->flags));
+
+    /* Make sure flag is read before info */
+    rmb();
+    *op = dev->info->op;
+}
+
+int pcifront_conf_read(struct pcifront_dev *dev,
+                       unsigned int dom,
+                       unsigned int bus, unsigned int slot, unsigned long fun,
+                       unsigned int off, unsigned int size, unsigned int *val)
+{
+    struct xen_pci_op op;
+
+    memset(&op, 0, sizeof(op));
+
+    op.cmd = XEN_PCI_OP_conf_read;
+    op.domain = dom;
+    op.bus = bus;
+    op.devfn = PCI_DEVFN(slot, fun);
+    op.offset = off;
+    op.size = size;
+
+    pcifront_op(dev, &op);
+
+    if (op.err)
+        return op.err;
+
+    *val = op.value;
+
+    return 0;
+}
+
+int pcifront_conf_write(struct pcifront_dev *dev,
+                        unsigned int dom,
+                        unsigned int bus, unsigned int slot, unsigned long fun,
+                        unsigned int off, unsigned int size, unsigned int val)
+{
+    struct xen_pci_op op;
+
+    memset(&op, 0, sizeof(op));
+
+    op.cmd = XEN_PCI_OP_conf_write;
+    op.domain = dom;
+    op.bus = bus;
+    op.devfn = PCI_DEVFN(slot, fun);
+    op.offset = off;
+    op.size = size;
+
+    op.value = val;
+
+    pcifront_op(dev, &op);
+
+    return op.err;
+}
diff --git a/stubdom/Makefile b/stubdom/Makefile
@@ -131,16 +131,20 @@ $(ZLIB_STAMPFILE): zlib-$(ZLIB_VERSION).tar.gz $(NEWLIB_STAMPFILE)
 pciutils-$(LIBPCI_VERSION).tar.bz2:
 	$(WGET) http://www.kernel.org/pub/software/utils/pciutils/pciutils-$(LIBPCI_VERSION).tar.bz2
 
+pciutils-$(LIBPCI_VERSION): pciutils-$(LIBPCI_VERSION).tar.bz2
+	tar xjf $<
+	patch -d $@ -p1 < pciutils.patch
+	touch $@
+
 LIBPCI_STAMPFILE=$(CROSS_ROOT)/$(GNU_TARGET_ARCH)-xen-elf/lib/libpci.a
 .PHONY: cross-libpci
 cross-libpci: $(LIBPCI_STAMPFILE)
-$(LIBPCI_STAMPFILE): pciutils-$(LIBPCI_VERSION).tar.bz2 $(NEWLIB_STAMPFILE) $(ZLIB_STAMPFILE)
-	tar xjf $<
+$(LIBPCI_STAMPFILE): pciutils-$(LIBPCI_VERSION) $(NEWLIB_STAMPFILE) $(ZLIB_STAMPFILE)
 	( cd pciutils-$(LIBPCI_VERSION) && \
 	  cp ../libpci.config.h lib/config.h && \
 	  echo '#define PCILIB_VERSION "$(LIBPCI_VERSION)"' >> lib/config.h && \
 	  cp ../libpci.config.mak lib/config.mk && \
-	  $(MAKE) CC="$(GNU_TARGET_ARCH)-xen-elf-gcc $(TARGET_CFLAGS)" lib/libpci.a && \
+	  $(MAKE) CC="$(GNU_TARGET_ARCH)-xen-elf-gcc $(TARGET_CFLAGS) -I$(realpath $(MINI_OS)/include)" lib/libpci.a && \
 	  $(INSTALL_DATA) lib/libpci.a $(CROSS_PREFIX)/$(GNU_TARGET_ARCH)-xen-elf/lib/ && \
 	  $(INSTALL_DIR) $(CROSS_PREFIX)/$(GNU_TARGET_ARCH)-xen-elf/include/pci && \
 	  $(INSTALL_DATA) lib/{config,header,pci,types}.h $(CROSS_PREFIX)/$(GNU_TARGET_ARCH)-xen-elf/include/pci/ \

diff --git a/stubdom/libpci.config.h b/stubdom/libpci.config.h
@@ -1,4 +1,4 @@
-#define PCI_OS_STUBDOM
+#define PCI_OS_MINIOS
 #define PCI_HAVE_STDINT_H
 #define PCI_PATH_IDS_DIR "."
 #define PCI_COMPRESSED_IDS

diff --git a/stubdom/libpci.config.mak b/stubdom/libpci.config.mak
@@ -1,2 +1,7 @@
 LIBZ=-lz
 LDLIBS+=$(LIBZ)
+PCI_OS_MINIOS=1
+PCI_HAVE_STDINT_H=1
+PCI_PATH_IDS_DIR=.
+PCI_COMPRESSED_IDS=1
+PCI_IDS=pci.ids.gz