ref: 82c08a8a4469c4b4ba515d74addc7aac747a17d9
dir: /sys/src/9/pc/usbxhci.c/
#include "u.h" #include "../port/lib.h" #include "mem.h" #include "dat.h" #include "fns.h" #include "io.h" #include "../port/error.h" #include "../port/usb.h" enum { /* Capability Registers */ CAPLENGTH = 0x00/4, // 1 HCIVERSION = 0x02/4, // 2 HCSPARAMS1 = 0x04/4, HCSPARAMS2 = 0x08/4, HCSPARAMS3 = 0x0C/4, HCCPARAMS = 0x10/4, AC64 = 1<<0, BNC = 1<<1, CSZ = 1<<2, PPC = 1<<3, PIND = 1<<4, LHRC = 1<<5, LTC = 1<<6, NSS = 1<<7, DBOFF = 0x14/4, RTSOFF = 0x18/4, /* Operational Registers */ USBCMD = 0x00/4, /* USB Command Register */ RUNSTOP = 1<<0, /* Run/Stop - RW */ HCRST = 1<<1, /* Host Controller Reset - RW */ INTE = 1<<2, /* Interrupter Enable - RW */ HSEE = 1<<3, /* Host System Error Enable - RW */ LHCRST = 1<<7, /* Light Host Controller Reset - RO/RW */ CSS = 1<<8, /* Controller Save State - RW */ CRS = 1<<9, /* Controller Restore State - RW */ EWE = 1<<10, /* Enable Wrap Event - RW */ EU3S = 1<<11, /* Enable U3 MFINDEX Stop - RW */ USBSTS = 0x04/4, /* USB Status Register */ HCH = 1<<0, /* HCHalted - RO */ HSE = 1<<2, /* Host System Error - RW1C */ EINT = 1<<3, /* Event Interrupt - RW1C */ PCD = 1<<4, /* Port Change Detect - RW1C */ SSS = 1<<8, /* Save State Status - RO */ RSS = 1<<9, /* Restore State Status - RO */ SRE = 1<<10, /* Save/Restore Error - RW1C */ CNR = 1<<11, /* Controller Not Ready - RO */ HCE = 1<<12, /* Host Controller Error - RO */ PAGESIZE = 0x08/4, /* Page Size - RO */ DNCTRL = 0x14/4, /* Device Notification Control Register - RW */ CRCR = 0x18/4, /* Command Ring Control Register - RW */ RCS = 1<<0, /* Ring Cycle State - RW */ CS = 1<<1, /* Command Stop - RW1S */ CA = 1<<2, /* Command Abort - RW1S */ CRR = 1<<3, /* Command Ring Running - RO */ DCBAAP = 0x30/4, // 8 CONFIG = 0x38/4, /* Configure Register (MaxSlotEn[7:0]) */ /* Port Register Set */ PORTSC = 0x00/4, /* Port tatus and Control Register */ CCS = 1<<0, /* Current Connect Status - ROS */ PED = 1<<1, /* Port Enable/Disabled - RW1CS */ OCA = 1<<3, /* Over-current Active - RO */ PR = 1<<4, /* Port Reset - RW1S */ PLS = 15<<5, /* Port Link State - RWS */ PP = 1<<9, /* Port Power - RWS */ PS = 15<<10, /* Port Speed - ROS */ PIC = 3<<14, /* Port Indicator Control - RWS */ LWS = 1<<16, /* Port Link Write Strobe - RW */ CSC = 1<<17, /* Connect Status Change - RW1CS */ PEC = 1<<18, /* Port Enabled/Disabled Change - RW1CS */ WRC = 1<<19, /* Warm Port Reset Change - RW1CS */ OCC = 1<<20, /* Over-current Change - RW1CS */ PRC = 1<<21, /* Port Reset Change - RW1CS */ PLC = 1<<22, /* Port Link State Change - RW1CS */ CEC = 1<<23, /* Port Config Error Change - RW1CS */ CAS = 1<<24, /* Cold Attach Status - RO */ WCE = 1<<25, /* Wake on Connect Enable - RWS */ WDE = 1<<26, /* Wake on Disconnect Enable - RWS */ WOE = 1<<27, /* Wake on Over-current Enable - RWS */ DR = 1<<30, /* Device Removable - RO */ WPR = 1<<31, /* Warm Port Reset - RW1S */ PORTPMSC = 0x04/4, PORTLI = 0x08/4, /* Host Controller Runtime Register */ MFINDEX = 0x0000/4, /* Microframe Index */ IR0 = 0x0020/4, /* Interrupt Register Set 0 */ /* Interrupter Registers */ IMAN = 0x00/4, /* Interrupter Management */ IMOD = 0x04/4, /* Interrupter Moderation */ ERSTSZ = 0x08/4, /* Event Ring Segment Table Size */ ERSTBA = 0x10/4, /* Event Ring Segment Table Base Address */ ERDP = 0x18/4, /* Event Ring Dequeue Pointer */ /* TRB flags */ TR_ENT = 1<<1, TR_ISP = 1<<2, TR_NS = 1<<3, TR_CH = 1<<4, TR_IOC = 1<<5, TR_IDT = 1<<6, TR_BEI = 1<<9, /* TRB types */ TR_RESERVED = 0<<10, TR_NORMAL = 1<<10, TR_SETUPSTAGE = 2<<10, TR_DATASTAGE = 3<<10, TR_STATUSSTAGE = 4<<10, TR_ISOCH = 5<<10, TR_LINK = 6<<10, TR_EVENTDATA = 7<<10, TR_NOOP = 8<<10, CR_ENABLESLOT = 9<<10, CR_DISABLESLOT = 10<<10, CR_ADDRESSDEV = 11<<10, CR_CONFIGEP = 12<<10, CR_EVALCTX = 13<<10, CR_RESETEP = 14<<10, CR_STOPEP = 15<<10, CR_SETTRDQP = 16<<10, CR_RESETDEV = 17<<10, CR_FORCECMD = 18<<10, CR_NEGBW = 19<<10, CR_SETLAT = 20<<10, CR_GETPORTBW = 21<<10, CR_FORCEHDR = 22<<10, CR_NOOP = 23<<10, ER_TRANSFER = 32<<10, ER_CMDCOMPL = 33<<10, ER_PORTSC = 34<<10, ER_BWREQ = 35<<10, ER_DOORBELL = 36<<10, ER_HCE = 37<<10, ER_DEVNOTE = 38<<10, ER_MFINDEXWRAP = 39<<10, }; typedef struct Ctlr Ctlr; typedef struct Wait Wait; typedef struct Ring Ring; typedef struct Slot Slot; typedef struct Epio Epio; typedef struct Port Port; struct Wait { Wait *next; Ring *ring; u32int *td; u32int er[4]; Rendez *z; }; struct Ring { int id; Slot *slot; u32int *base; u32int mask; u32int shift; u32int rp; u32int wp; u32int *ctx; u32int *doorbell; int stopped; Wait *pending; Lock; }; struct Slot { int id; Ctlr *ctlr; Udev *dev; u32int *ibase; u32int *obase; /* endpoint rings */ int nep; Ring epr[32]; }; struct Port { char spec[4]; int proto; u32int *reg; }; struct Ctlr { Pcidev *pcidev; u32int *mmio; u32int *opr; /* operational registers */ u32int *rts; /* runtime registers */ u32int *dba; /* doorbell array */ u64int *dcba; /* device context base array */ u64int *sba; /* scratchpad buffer array */ void *sbp; /* scratchpad buffer pages */ u32int *erst[1]; /* event ring segment table */ Ring er[1]; /* event ring segment */ Ring cr[1]; /* command ring segment */ QLock cmdlock; u32int µframe; QLock slotlock; Slot **slot; /* slots by slot id */ Port *port; u32int hccparams; int csz; int pagesize; int nscratch; int nintrs; int nslots; void (*setrptr)(u32int*, u64int); Rendez recover; void *active; uintptr base; }; struct Epio { QLock; Ring *ring; Block *b; /* iso */ u32int frame; u32int period; u32int incr; u32int tdsz; int nleft; }; static char Ebadlen[] = "bad usb request length"; static char Enotconfig[] = "usb endpoint not configured"; static char Erecover[] = "xhci controller needs reset"; static void setrptr32(u32int *reg, u64int pa) { coherence(); reg[0] = pa; reg[1] = pa>>32; } static void setrptr64(u32int *reg, u64int pa) { coherence(); *((u64int*)reg) = pa; } static u32int µframe(Ctlr *ctlr) { u32int µ; do { µ = (ctlr->rts[MFINDEX] & (1<<14)-1) | (ctlr->µframe & ~((1<<14)-1)); } while((int)(µ - ctlr->µframe) < 0); return µ; } static void freering(Ring *r) { if(r == nil) return; free(r->base); memset(r, 0, sizeof(*r)); } static Ring* initring(Ring *r, int shift) { r->id = 0; r->ctx = nil; r->slot = nil; r->doorbell = nil; r->pending = nil; r->stopped = 0; r->shift = shift; r->mask = (1<<shift)-1; r->rp = r->wp = 0; r->base = mallocalign(4*4<<shift, 64, 0, 64*1024); if(r->base == nil){ freering(r); error(Enomem); } return r; } static void flushring(Ring *r) { Rendez *z; Wait *w; while((w = r->pending) != nil){ r->pending = w->next; w->next = nil; if((z = w->z) != nil){ w->z = nil; wakeup(z); } } } static u64int resetring(Ring *r) { u64int pa; ilock(r); flushring(r); r->rp = r->wp; pa = PADDR(&r->base[4*(r->wp & r->mask)]) | ((~r->wp>>r->shift) & 1); iunlock(r); return pa; } static u32int* xecp(Ctlr *ctlr, uchar id, u32int *p) { u32int x, *e; e = &ctlr->mmio[ctlr->pcidev->mem[0].size/4]; if(p == nil){ p = ctlr->mmio; x = ctlr->hccparams>>16; } else { assert(p < e); x = (*p>>8) & 255; } while(x != 0){ p += x; if(p >= e) break; x = *p; if((x & 255) == id) return p; x >>= 8; x &= 255; } return nil; } static void handoff(Ctlr *ctlr) { u32int *r; int i; if((r = xecp(ctlr, 1, nil)) == nil) return; r[0] |= 1<<24; /* request ownership */ for(i = 0; (r[0] & (1<<16)) != 0 && i<100; i++) tsleep(&up->sleep, return0, nil, 10); r[1] = 0; /* disable SMI interrupts */ r[0] &= ~(1<<16); /* in case of timeout */ } static void shutdown(Hci *hp) { Ctlr *ctlr = hp->aux; int i; ctlr->opr[USBCMD] = 0; for(i=0; (ctlr->opr[USBSTS] & HCH) == 0 && i < 10; i++) delay(10); intrdisable(ctlr->pcidev->intl, hp->interrupt, hp, ctlr->pcidev->tbdf, hp->type); pciclrbme(ctlr->pcidev); } static void release(Ctlr *ctlr) { int i; freering(ctlr->cr); for(i=0; i<nelem(ctlr->er); i++){ freering(&ctlr->er[i]); free(ctlr->erst[i]); ctlr->erst[i] = nil; } free(ctlr->port), ctlr->port = nil; free(ctlr->slot), ctlr->slot = nil; free(ctlr->dcba), ctlr->dcba = nil; free(ctlr->sba), ctlr->sba = nil; free(ctlr->sbp), ctlr->sbp = nil; } static void recover(void *arg); static void init(Hci *hp) { Ctlr *ctlr; Port *pp; u32int *x; uchar *p; int i, j; ctlr = hp->aux; if(ctlr->mmio[CAPLENGTH] == -1) error("controller vanished"); ctlr->opr = &ctlr->mmio[(ctlr->mmio[CAPLENGTH]&0xFF)/4]; ctlr->dba = &ctlr->mmio[ctlr->mmio[DBOFF]/4]; ctlr->rts = &ctlr->mmio[ctlr->mmio[RTSOFF]/4]; ctlr->hccparams = ctlr->mmio[HCCPARAMS]; handoff(ctlr); for(i=0; (ctlr->opr[USBSTS] & CNR) != 0 && i<100; i++) tsleep(&up->sleep, return0, nil, 10); ctlr->opr[USBCMD] = HCRST; for(i=0; (ctlr->opr[USBSTS] & (CNR|HCH)) != HCH && i<100; i++) tsleep(&up->sleep, return0, nil, 10); pcisetbme(ctlr->pcidev); pcisetpms(ctlr->pcidev, 0); intrenable(ctlr->pcidev->intl, hp->interrupt, hp, ctlr->pcidev->tbdf, hp->type); if(waserror()){ shutdown(hp); release(ctlr); nexterror(); } ctlr->csz = (ctlr->hccparams & CSZ) != 0; if(ctlr->hccparams & AC64) ctlr->setrptr = setrptr64; else ctlr->setrptr = setrptr32; ctlr->pagesize = ctlr->opr[PAGESIZE]<<12; ctlr->nscratch = (ctlr->mmio[HCSPARAMS2] >> 27) & 0x1F; ctlr->nintrs = (ctlr->mmio[HCSPARAMS1] >> 8) & 0x7FF; ctlr->nslots = (ctlr->mmio[HCSPARAMS1] >> 0) & 0xFF; hp->highspeed = 1; hp->superspeed = 0; hp->nports = (ctlr->mmio[HCSPARAMS1] >> 24) & 0xFF; ctlr->port = malloc(hp->nports * sizeof(Port)); if(ctlr->port == nil) error(Enomem); for(i=0; i<hp->nports; i++) ctlr->port[i].reg = &ctlr->opr[0x400/4 + i*4]; x = nil; while((x = xecp(ctlr, 2, x)) != nil){ i = x[2]&255; j = (x[2]>>8)&255; while(j--){ if(i < 1 || i > hp->nports) break; pp = &ctlr->port[i-1]; pp->proto = x[0]>>16; memmove(pp->spec, &x[1], 4); if(memcmp(pp->spec, "USB ", 4) == 0 && pp->proto >= 0x0300) hp->superspeed |= 1<<(i-1); i++; } } ctlr->slot = malloc((1+ctlr->nslots)*sizeof(ctlr->slot[0])); ctlr->dcba = mallocalign((1+ctlr->nslots)*8, 64, 0, ctlr->pagesize); if(ctlr->slot == nil || ctlr->dcba == nil) error(Enomem); if(ctlr->nscratch != 0){ ctlr->sba = mallocalign(ctlr->nscratch*8, 64, 0, ctlr->pagesize); ctlr->sbp = mallocalign(ctlr->nscratch*ctlr->pagesize, ctlr->pagesize, 0, 0); if(ctlr->sba == nil || ctlr->sbp == nil) error(Enomem); for(i=0, p = ctlr->sbp; i<ctlr->nscratch; i++, p += ctlr->pagesize){ memset(p, 0, ctlr->pagesize); ctlr->sba[i] = PADDR(p); } ctlr->dcba[0] = PADDR(ctlr->sba); } else { ctlr->dcba[0] = 0; } for(i=1; i<=ctlr->nslots; i++) ctlr->dcba[i] = 0; ctlr->opr[CONFIG] = ctlr->nslots; /* MaxSlotsEn */ ctlr->setrptr(&ctlr->opr[DCBAAP], PADDR(ctlr->dcba)); initring(ctlr->cr, 8); /* 256 entries */ ctlr->cr->id = 0; ctlr->cr->doorbell = &ctlr->dba[0]; ctlr->setrptr(&ctlr->opr[CRCR], resetring(ctlr->cr)); for(i=0; i<ctlr->nintrs; i++){ u32int *irs = &ctlr->rts[IR0 + i*8]; if(i >= nelem(ctlr->er)){ irs[ERSTSZ] = 0; /* disable ring */ continue; } /* allocate and link into event ring segment table */ initring(&ctlr->er[i], 8); /* 256 entries */ ctlr->erst[i] = mallocalign(16, 64, 0, 0); if(ctlr->erst[i] == nil) error(Enomem); *((u64int*)ctlr->erst[i]) = PADDR(ctlr->er[i].base); ctlr->erst[i][2] = ctlr->er[i].mask+1; ctlr->erst[i][3] = 0; irs[ERSTSZ] = 1; /* just one segment */ ctlr->setrptr(&irs[ERDP], PADDR(ctlr->er[i].base)); ctlr->setrptr(&irs[ERSTBA], PADDR(ctlr->erst[i])); irs[IMAN] = 3; irs[IMOD] = 0; } poperror(); ctlr->µframe = 0; ctlr->opr[USBCMD] = RUNSTOP|INTE|HSEE|EWE; for(i=0; (ctlr->opr[USBSTS] & (CNR|HCH)) != 0 && i<100; i++) tsleep(&up->sleep, return0, nil, 10); kproc("xhcirecover", recover, hp); } static int needrecover(void *arg) { Ctlr *ctlr = arg; return ctlr->er->stopped || (ctlr->opr[USBSTS] & (HCH|HCE|HSE)) != 0; } static void recover(void *arg) { Hci *hp = arg; Ctlr *ctlr = hp->aux; while(waserror()) ; while(!needrecover(ctlr)) tsleep(&ctlr->recover, needrecover, ctlr, 1000); shutdown(hp); /* * flush all transactions and wait until all devices have * been detached by usbd. */ for(;;){ int i, j, active; ilock(ctlr->cr); ctlr->cr->stopped = 1; flushring(ctlr->cr); iunlock(ctlr->cr); active = 0; qlock(&ctlr->slotlock); for(i=1; i<=ctlr->nslots; i++){ Slot *slot = ctlr->slot[i]; if(slot == nil) continue; active++; for(j=0; j < slot->nep; j++){ Ring *ring = &slot->epr[j]; if(ring->base == nil) continue; ilock(ring); ring->stopped = 1; flushring(ring); iunlock(ring); } } qunlock(&ctlr->slotlock); if(active == 0) break; tsleep(&up->sleep, return0, nil, 100); } qlock(&ctlr->slotlock); qlock(&ctlr->cmdlock); release(ctlr); if(waserror()) { print("xhci recovery failed: %s\n", up->errstr); } else { init(hp); poperror(); } qunlock(&ctlr->cmdlock); qunlock(&ctlr->slotlock); pexit("", 1); } static void dump(Hci *) { } static void queuetd(Ring *r, u32int c, u32int s, u64int p, Wait *w) { u32int *td, x; x = r->wp++; if((x & r->mask) == r->mask){ td = r->base + 4*(x & r->mask); *(u64int*)td = PADDR(r->base); td[2] = 0; td[3] = ((~x>>r->shift)&1) | (1<<1) | TR_LINK; x = r->wp++; } td = r->base + 4*(x & r->mask); if(w != nil){ w->er[0] = w->er[1] = w->er[2] = w->er[3] = 0; w->ring = r; w->td = td; w->z = &up->sleep; ilock(r); w->next = r->pending; r->pending = w; iunlock(r); } coherence(); *(u64int*)td = p; td[2] = s; td[3] = ((~x>>r->shift)&1) | c; } static char *ccerrtab[] = { [2] "Data Buffer Error", [3] "Babble Detected Error", [4] "USB Transaction Error", [5] "TRB Error", [6] "Stall Error", [7] "Resume Error", [8] "Bandwidth Error", [9] "No Slots Available", [10] "Invalid Stream Type", [11] "Slot Not Enabled", [12] "Endpoint Not Enabled", [13] "Short Packet", [14] "Ring Underrun", [15] "Ring Overrun", [16] "VF Event Ring Full", [17] "Parameter Error", [18] "Bandwidth Overrun Error", [19] "Context State Error", [20] "No Ping Response", [21] "Event Ring Full", [22] "Incompatible Device", [23] "Missed Service Error", [24] "Command Ring Stopped", [25] "Command Aborted", [26] "Stopped", [27] "Stoppe - Length Invalid", [29] "Max Exit Latency Too Large", [31] "Isoch Buffer Overrun", [32] "Event Lost Error", [33] "Undefined Error", [34] "Invalid Stream ID", [35] "Secondary Bandwidth Error", [36] "Split Transaction Error", }; static char* ccerrstr(u32int cc) { char *s; if(cc == 1 || cc == 13) return nil; if(cc < nelem(ccerrtab) && ccerrtab[cc] != nil) s = ccerrtab[cc]; else s = "???"; return s; } static int waitdone(void *a) { return ((Wait*)a)->z == nil; } static char* ctlrcmd(Ctlr *ctlr, u32int c, u32int s, u64int p, u32int *er); static char* waittd(Ctlr *ctlr, Wait *w, int tmout) { Ring *r = w->ring; coherence(); *r->doorbell = r->id; while(waserror()){ if(!r->stopped) { if(r == ctlr->cr) ctlr->opr[CRCR] |= CA; else ctlrcmd(ctlr, CR_STOPEP | (r->id<<16) | (r->slot->id<<24), 0, 0, nil); r->stopped = 1; } tmout = 0; } if(tmout > 0){ tsleep(&up->sleep, waitdone, w, tmout); if(!waitdone(w)) error("timed out"); } else { while(!waitdone(w)) sleep(&up->sleep, waitdone, w); } poperror(); return ccerrstr(w->er[2]>>24); } static char* ctlrcmd(Ctlr *ctlr, u32int c, u32int s, u64int p, u32int *er) { Wait w[1]; char *err; qlock(&ctlr->cmdlock); if(needrecover(ctlr)){ qunlock(&ctlr->cmdlock); return Erecover; } ctlr->cr->stopped = 0; queuetd(ctlr->cr, c, s, p, w); err = waittd(ctlr, w, 5000); qunlock(&ctlr->cmdlock); if(er != nil) memmove(er, w->er, 4*4); return err; } static void completering(Ring *r, u32int *er) { Wait *w, **wp; u32int *td, x; u64int pa; pa = (*(u64int*)er) & ~15ULL; ilock(r); for(x = r->rp; (int)(r->wp - x) > 0;){ td = &r->base[4*(x++ & r->mask)]; if((u64int)PADDR(td) == pa){ r->rp = x; break; } } wp = &r->pending; while(w = *wp){ if((u64int)PADDR(w->td) == pa){ Rendez *z = w->z; memmove(w->er, er, 4*4); *wp = w->next; w->next = nil; if(z != nil){ w->z = nil; wakeup(z); } break; } else { wp = &w->next; } } iunlock(r); } static void interrupt(Ureg*, void *arg) { Hci *hp = arg; Ctlr *ctlr = hp->aux; Ring *ring = ctlr->er; Slot *slot; u32int *irs, *td, x; if(ring->base == nil) return; irs = &ctlr->rts[IR0]; x = irs[IMAN]; if(x & 1) irs[IMAN] = x & 3; for(x = ring->rp;; x=++ring->rp){ td = ring->base + 4*(x & ring->mask); if((((x>>ring->shift)^td[3])&1) == 0) break; switch(td[3] & 0xFC00){ case ER_CMDCOMPL: completering(ctlr->cr, td); break; case ER_TRANSFER: x = td[3]>>24; if(x == 0 || x > ctlr->nslots) break; slot = ctlr->slot[x]; if(slot == nil) break; completering(&slot->epr[(td[3]>>16)-1&31], td); break; case ER_MFINDEXWRAP: ctlr->µframe = (ctlr->rts[MFINDEX] & (1<<14)-1) | (ctlr->µframe+(1<<14) & ~((1<<14)-1)); break; case ER_HCE: iprint("xhci: host controller error: %ux %ux %ux %ux\n", td[0], td[1], td[2], td[3]); ctlr->er->stopped = 1; wakeup(&ctlr->recover); return; case ER_PORTSC: break; case ER_BWREQ: case ER_DOORBELL: case ER_DEVNOTE: default: iprint("xhci: event %ud: %ux %ux %ux %ux\n", x, td[0], td[1], td[2], td[3]); } } ctlr->setrptr(&irs[ERDP], PADDR(td) | (1<<3)); } static void freeslot(void *arg) { Slot *slot; if(arg == nil) return; slot = arg; if(slot->id != 0){ Ctlr *ctlr = slot->ctlr; qlock(&ctlr->slotlock); if(ctlr->slot != nil && ctlr->slot[slot->id] == slot){ ctlrcmd(ctlr, CR_DISABLESLOT | (slot->id<<24), 0, 0, nil); ctlr->dcba[slot->id] = 0; ctlr->slot[slot->id] = nil; } qunlock(&ctlr->slotlock); } freering(&slot->epr[0]); free(slot->ibase); free(slot->obase); free(slot); } static Slot* allocslot(Ctlr *ctlr, Udev *dev) { u32int r[4]; Slot *slot; char *err; slot = malloc(sizeof(Slot)); if(slot == nil) error(Enomem); slot->ctlr = ctlr; slot->dev = dev; slot->nep = 0; slot->id = 0; qlock(&ctlr->slotlock); if(waserror()){ qunlock(&ctlr->slotlock); freeslot(slot); nexterror(); } if(ctlr->slot == nil) error(Erecover); slot->ibase = mallocalign(32*33 << ctlr->csz, 64, 0, ctlr->pagesize); slot->obase = mallocalign(32*32 << ctlr->csz, 64, 0, ctlr->pagesize); if(slot->ibase == nil || slot->obase == nil) error(Enomem); if((err = ctlrcmd(ctlr, CR_ENABLESLOT, 0, 0, r)) != nil) error(err); slot->id = r[3]>>24; if(slot->id <= 0 || slot->id > ctlr->nslots || ctlr->slot[slot->id] != nil){ slot->id = 0; error("bad slot id from controller"); } poperror(); ctlr->dcba[slot->id] = PADDR(slot->obase); ctlr->slot[slot->id] = slot; qunlock(&ctlr->slotlock); dev->aux = slot; dev->free = freeslot; return slot; } static void setdebug(Hci *, int) { } static void epclose(Ep *ep) { Ctlr *ctlr; Slot *slot; Ring *ring; Epio *io; if(ep->dev->isroot) return; io = ep->aux; if(io == nil) return; ep->aux = nil; ctlr = ep->hp->aux; slot = ep->dev->aux; if(ep->nb > 0 && (io[OREAD].ring != nil || io[OWRITE].ring != nil)){ u32int *w; /* input control context */ w = slot->ibase; memset(w, 0, 32<<ctlr->csz); w[1] = 1; if((ring = io[OREAD].ring) != nil){ w[0] |= 1 << ring->id; if(ring->id == slot->nep) slot->nep--; ctlrcmd(ctlr, CR_STOPEP | (ring->id<<16) | (slot->id<<24), 0, 0, nil); } if((ring = io[OWRITE].ring) != nil){ w[0] |= 1 << ring->id; if(ring->id == slot->nep) slot->nep--; ctlrcmd(ctlr, CR_STOPEP | (ring->id<<16) | (slot->id<<24), 0, 0, nil); } /* (input) slot context */ w += 8<<ctlr->csz; w[0] = (w[0] & ~(0x1F<<27)) | slot->nep<<27; /* (input) ep context */ w += ep->nb*2*8<<ctlr->csz; memset(w, 0, 2*32<<ctlr->csz); ctlrcmd(ctlr, CR_CONFIGEP | (slot->id<<24), 0, PADDR(slot->ibase), nil); freering(io[OREAD].ring); freering(io[OWRITE].ring); } freeb(io[OREAD].b); freeb(io[OWRITE].b); free(io); } static void initepctx(u32int *w, Ring *r, Ep *ep) { int ival; if(ep->dev->speed == Lowspeed || ep->dev->speed == Fullspeed){ for(ival=3; ival < 11 && (1<<ival) < ep->pollival; ival++) ; } else { for(ival=0; ival < 15 && (1<<ival) < ep->pollival; ival++) ; } w[0] = ival<<16; w[1] = ((ep->ttype-Tctl)|(r->id&1)<<2)<<3 | (ep->ntds-1)<<8 | ep->maxpkt<<16; if(ep->ttype != Tiso) w[1] |= 3<<1; *((u64int*)&w[2]) = PADDR(r->base) | 1; w[4] = ep->maxpkt; if(ep->ttype == Tintr || ep->ttype == Tiso) w[4] |= (ep->maxpkt*ep->ntds)<<16; } static void initisoio(Epio *io, Ep *ep) { if(io->ring == nil) return; io->frame = 0; io->period = ep->pollival<<3*(ep->dev->speed == Fullspeed); io->incr = (ep->hz*io->period<<8)/8000; io->tdsz = (io->incr+255>>8)*ep->samplesz; io->b = allocb((io->ring->mask+1)*io->tdsz); } static void initep(Ep *ep) { Epio *io; Ctlr *ctlr; Slot *slot; Ring *ring; u32int *w; char *err; io = ep->aux; ctlr = ep->hp->aux; slot = ep->dev->aux; io[OREAD].ring = io[OWRITE].ring = nil; if(ep->nb == 0){ io[OWRITE].ring = &slot->epr[0]; return; } /* (input) control context */ w = slot->ibase; memset(w, 0, 32<<ctlr->csz); w[1] = 1; if(waserror()){ freering(io[OWRITE].ring), io[OWRITE].ring = nil; freering(io[OREAD].ring), io[OREAD].ring = nil; nexterror(); } if(ep->mode != OREAD){ ring = initring(io[OWRITE].ring = &slot->epr[ep->nb*2-1], 8); ring->id = ep->nb*2; if(ring->id > slot->nep) slot->nep = ring->id; ring->slot = slot; ring->doorbell = &ctlr->dba[slot->id]; ring->ctx = &slot->obase[ring->id*8<<ctlr->csz]; w[1] |= 1 << ring->id; } if(ep->mode != OWRITE){ ring = initring(io[OREAD].ring = &slot->epr[ep->nb*2], 8); ring->id = ep->nb*2+1; if(ring->id > slot->nep) slot->nep = ring->id; ring->slot = slot; ring->doorbell = &ctlr->dba[slot->id]; ring->ctx = &slot->obase[ring->id*8<<ctlr->csz]; w[1] |= 1 << ring->id; } /* (input) slot context */ w += 8<<ctlr->csz; w[0] = (w[0] & ~(0x1F<<27)) | slot->nep<<27; /* (input) ep context */ w += ep->nb*2*8<<ctlr->csz; memset(w, 0, 2*32<<ctlr->csz); if(io[OWRITE].ring != nil) initepctx(w, io[OWRITE].ring, ep); w += 8<<ctlr->csz; if(io[OREAD].ring != nil) initepctx(w, io[OREAD].ring, ep); if((err = ctlrcmd(ctlr, CR_CONFIGEP | (slot->id<<24), 0, PADDR(slot->ibase), nil)) != nil){ error(err); } if(ep->ttype == Tiso){ initisoio(io+OWRITE, ep); initisoio(io+OREAD, ep); } poperror(); } static int speedid(int speed) { switch(speed){ case Fullspeed: return 1; case Lowspeed: return 2; case Highspeed: return 3; case Superspeed: return 4; } return 0; } static void epopen(Ep *ep) { Ctlr *ctlr = ep->hp->aux; Slot *slot, *hub; Ring *ring; Epio *io; Udev *dev; char *err; u32int *w; int i; if(ep->dev->isroot) return; if(needrecover(ctlr)) error(Erecover); io = malloc(sizeof(Epio)*2); if(io == nil) error(Enomem); ep->aux = io; if(waserror()){ epclose(ep); nexterror(); } dev = ep->dev; slot = dev->aux; if(slot != nil && slot->dev == dev){ initep(ep); poperror(); return; } /* first open has to be control endpoint */ if(ep->nb != 0) error(Egreg); slot = allocslot(ctlr, dev); /* allocate control ep 0 ring */ ring = initring(io[OWRITE].ring = &slot->epr[0], 4); ring->id = 1; slot->nep = 1; ring->slot = slot; ring->doorbell = &ctlr->dba[slot->id]; ring->ctx = &slot->obase[8]; /* (input) control context */ w = slot->ibase; memset(w, 0, 3*32<<ctlr->csz); w[1] = 3; /* A0, A1 */ /* (input) slot context */ w += 8<<ctlr->csz; w[2] = w[3] = 0; w[0] = dev->routestr | speedid(dev->speed)<<20 | (dev->speed == Highspeed && dev->ishub != 0 ||dev->speed < Highspeed && dev->ishub == 0)<<25 | (dev->ishub != 0)<<26 | slot->nep<<27; w[1] = dev->rootport<<16; /* find the parent hub that this device is conected to */ qlock(&ctlr->slotlock); for(i=1; i<=ctlr->nslots; i++){ hub = ctlr->slot[i]; if(hub == nil || hub->dev == nil || hub->dev->aux != hub) continue; if(hub == slot || hub->dev == dev) continue; if(!hub->dev->ishub) continue; if(hub->dev->addr != dev->hub) continue; if(hub->dev->rootport != dev->rootport) continue; if(dev->speed < Highspeed && hub->dev->speed == Highspeed) w[2] = hub->id | dev->port<<8; break; } qunlock(&ctlr->slotlock); /* (input) ep context 0 */ w += 8<<ctlr->csz; initepctx(w, io[OWRITE].ring, ep); if((err = ctlrcmd(ctlr, CR_ADDRESSDEV | (slot->id<<24), 0, PADDR(slot->ibase), nil)) != nil){ error(err); } /* (output) slot context */ w = slot->obase; ep->dev->addr = w[3] & 0xFF; poperror(); } static long isoread(Ep *, uchar *, long) { error(Egreg); return 0; } static long isowrite(Ep *ep, uchar *p, long n) { uchar *s, *d; Ctlr *ctlr; Epio *io; u32int i, µ; long m; s = p; io = (Epio*)ep->aux + OWRITE; qlock(io); if(waserror()){ qunlock(io); nexterror(); } µ = io->period; ctlr = ep->hp->aux; if(needrecover(ctlr)) error(Erecover); for(i = io->frame;; i++){ for(;;){ m = (int)(io->ring->wp - io->ring->rp); if(m <= 0) i = (80 + µframe(ctlr))/µ; if(m < io->ring->mask) break; *io->ring->doorbell = io->ring->id; tsleep(&up->sleep, return0, nil, 5); } m = ((io->incr + (i*io->incr&255))>>8)*ep->samplesz; d = io->b->rp + (i&io->ring->mask)*io->tdsz; m -= io->nleft, d += io->nleft; if(n < m){ memmove(d, p, n); p += n; io->nleft += n; break; } memmove(d, p, m); p += m, n -= m; m += io->nleft, d -= io->nleft; io->nleft = 0; queuetd(io->ring, TR_ISOCH | (i*µ/8 & 0x7ff)<<20 | TR_IOC, m, PADDR(d), nil); } io->frame = i; while(io->ring->rp != io->ring->wp){ int d = (int)(i*µ - µframe(ctlr))/8; d -= ep->sampledelay*1000 / ep->hz; if(d < 5) break; *io->ring->doorbell = io->ring->id; tsleep(&up->sleep, return0, nil, d); } qunlock(io); poperror(); return p - s; } static char* unstall(Ring *r) { char *err; switch(r->ctx[0]&7){ case 2: /* halted */ case 4: /* error */ r->stopped = 1; err = ctlrcmd(r->slot->ctlr, CR_RESETEP | (r->id<<16) | (r->slot->id<<24), 0, 0, nil); if(err != nil) return err; } if(r->stopped){ err = ctlrcmd(r->slot->ctlr, CR_SETTRDQP | (r->id<<16) | (r->slot->id<<24), 0, resetring(r), nil); if(err != nil) return err; r->stopped = 0; } if(r->wp - r->rp >= r->mask) return "Ring Full"; return nil; } static long epread(Ep *ep, void *va, long n) { Epio *io; uchar *p; char *err; Wait w[1]; if(ep->dev->isroot) error(Egreg); p = va; if(ep->ttype == Tctl){ io = (Epio*)ep->aux + OREAD; qlock(io); if(io->b == nil || BLEN(io->b) == 0){ qunlock(io); return 0; } if(n > BLEN(io->b)) n = BLEN(io->b); memmove(p, io->b->rp, n); io->b->rp += n; qunlock(io); return n; } else if(ep->ttype == Tiso) return isoread(ep, p, n); if((uintptr)p <= KZERO){ Block *b; b = allocb(n); if(waserror()){ freeb(b); nexterror(); } n = epread(ep, b->rp, n); memmove(p, b->rp, n); freeb(b); poperror(); return n; } io = (Epio*)ep->aux + OREAD; qlock(io); if(waserror()){ qunlock(io); nexterror(); } if((err = unstall(io->ring)) != nil) error(err); queuetd(io->ring, TR_NORMAL | TR_IOC, n, PADDR(p), w); if((err = waittd((Ctlr*)ep->hp->aux, w, ep->tmout)) != nil) error(err); qunlock(io); poperror(); n -= (w->er[2] & 0xFFFFFF); if(n < 0) n = 0; return n; } static long epwrite(Ep *ep, void *va, long n) { Wait w[3]; Epio *io; uchar *p; char *err; if(ep->dev->isroot) error(Egreg); p = va; if(ep->ttype == Tctl){ int dir, len; Ring *ring; if(n < 8) error(Eshort); if(p[0] == 0x00 && p[1] == 0x05) return n; io = (Epio*)ep->aux + OREAD; ring = io[OWRITE-OREAD].ring; qlock(io); if(waserror()){ ilock(ring); ring->pending = nil; iunlock(ring); qunlock(io); nexterror(); } if(io->b != nil){ freeb(io->b); io->b = nil; } len = GET2(&p[6]); dir = (p[0] & Rd2h) != 0; if(len > 0){ io->b = allocb(len); if(dir == 0){ /* out */ assert(len >= n-8); memmove(io->b->wp, p+8, n-8); } else { memset(io->b->wp, 0, len); io->b->wp += len; } } if((err = unstall(ring)) != nil) error(err); if((ring->ctx[1]>>16) != ep->maxpkt){ Slot *slot = ring->slot; Ctlr *ctlr = slot->ctlr; u32int *w = slot->ibase; w[0] = 0; w[1] = 1<<ring->id; w += (ring->id+1)*8<<ctlr->csz; initepctx(w, ring, ep); if((err = ctlrcmd(ctlr, CR_EVALCTX | (slot->id<<24), 0, PADDR(slot->ibase), nil)) != nil) error(err); } queuetd(ring, TR_SETUPSTAGE | (len > 0 ? 2+dir : 0)<<16 | TR_IDT | TR_IOC, 8, p[0] | p[1]<<8 | GET2(&p[2])<<16 | (u64int)(GET2(&p[4]) | len<<16)<<32, &w[0]); if(len > 0) queuetd(ring, TR_DATASTAGE | dir<<16 | TR_IOC, len, PADDR(io->b->rp), &w[1]); queuetd(ring, TR_STATUSSTAGE | (len == 0 || !dir)<<16 | TR_IOC, 0, 0, &w[2]); if((err = waittd((Ctlr*)ep->hp->aux, &w[0], ep->tmout)) != nil) error(err); if(len > 0){ if((err = waittd((Ctlr*)ep->hp->aux, &w[1], ep->tmout)) != nil) error(err); if(dir != 0){ io->b->wp -= (w[1].er[2] & 0xFFFFFF); if(io->b->wp < io->b->rp) io->b->wp = io->b->rp; } } if((err = waittd((Ctlr*)ep->hp->aux, &w[2], ep->tmout)) != nil) error(err); qunlock(io); poperror(); return n; } else if(ep->ttype == Tiso) return isowrite(ep, p, n); if((uintptr)p <= KZERO){ Block *b; b = allocb(n); if(waserror()){ freeb(b); nexterror(); } memmove(b->wp, p, n); n = epwrite(ep, b->wp, n); freeb(b); poperror(); return n; } io = (Epio*)ep->aux + OWRITE; qlock(io); if(waserror()){ qunlock(io); nexterror(); } if((err = unstall(io->ring)) != nil) error(err); queuetd(io->ring, TR_NORMAL | TR_IOC, n, PADDR(p), w); if((err = waittd((Ctlr*)ep->hp->aux, w, ep->tmout)) != nil) error(err); qunlock(io); poperror(); return n; } static char* seprintep(char *s, char*, Ep*) { return s; } static int portstatus(Hci *hp, int port) { Ctlr *ctlr = hp->aux; u32int psc, ps; if(ctlr->port == nil || needrecover(ctlr)) return 0; ps = 0; psc = ctlr->port[port-1].reg[PORTSC]; if(psc & CCS) ps |= HPpresent; if(psc & PED) ps |= HPenable; if(psc & OCA) ps |= HPovercurrent; if(psc & PR) ps |= HPreset; if((hp->superspeed & (1<<(port-1))) != 0){ ps |= psc & (PLS|PP); if(psc & CSC) ps |= 1<<0+16; if(psc & OCC) ps |= 1<<3+16; if(psc & PRC) ps |= 1<<4+16; if(psc & WRC) ps |= 1<<5+16; if(psc & PLC) ps |= 1<<6+16; if(psc & CEC) ps |= 1<<7+16; } else { if((ps & HPreset) == 0){ switch((psc>>10)&15){ case 1: /* full speed */ break; case 2: ps |= HPslow; break; case 3: ps |= HPhigh; break; } } if(psc & PP) ps |= HPpower; if(psc & CSC) ps |= HPstatuschg; if(psc & PRC) ps |= HPchange; } return ps; } static int portenable(Hci*, int, int) { return 0; } static int portreset(Hci *hp, int port, int on) { Ctlr *ctlr = hp->aux; if(ctlr->port == nil || needrecover(ctlr)) return 0; if(on){ ctlr->port[port-1].reg[PORTSC] |= PR; tsleep(&up->sleep, return0, nil, 200); } return 0; } static Ctlr *ctlrs[Nhcis]; static void scanpci(void) { static int already = 0; int i; uintptr io; Ctlr *ctlr; Pcidev *p; u32int *mmio; if(already) return; already = 1; p = nil; while ((p = pcimatch(p, 0, 0)) != nil) { /* * Find XHCI controllers (Programming Interface = 0x30). */ if(p->ccrb != Pcibcserial || p->ccru != Pciscusb || p->ccrp != 0x30) continue; io = p->mem[0].bar & ~0x0f; if(io == 0) continue; print("usbxhci: %#x %#x: port %#p size %#x irq %d\n", p->vid, p->did, io, p->mem[0].size, p->intl); mmio = vmap(io, p->mem[0].size); if(mmio == nil){ print("usbxhci: cannot map registers\n"); continue; } ctlr = malloc(sizeof(Ctlr)); if(ctlr == nil){ print("usbxhci: no memory\n"); vunmap(mmio, p->mem[0].size); continue; } ctlr->base = io; ctlr->active = nil; ctlr->pcidev = p; ctlr->mmio = mmio; for(i = 0; i < nelem(ctlrs); i++) if(ctlrs[i] == nil){ ctlrs[i] = ctlr; break; } if(i >= nelem(ctlrs)) print("xhci: bug: more than %d controllers\n", nelem(ctlrs)); } } static int reset(Hci *hp) { Ctlr *ctlr; int i; if(getconf("*nousbxhci")) return -1; scanpci(); /* * Any adapter matches if no hp->port is supplied, * otherwise the ports must match. */ for(i = 0; i < nelem(ctlrs) && ctlrs[i] != nil; i++){ ctlr = ctlrs[i]; if(ctlr->active == nil) if(hp->port == 0 || hp->port == ctlr->base){ ctlr->active = hp; goto Found; } } return -1; Found: hp->aux = ctlr; hp->port = ctlr->base; hp->irq = ctlr->pcidev->intl; hp->tbdf = ctlr->pcidev->tbdf; hp->init = init; hp->dump = dump; hp->interrupt = interrupt; hp->epopen = epopen; hp->epclose = epclose; hp->epread = epread; hp->epwrite = epwrite; hp->seprintep = seprintep; hp->portenable = portenable; hp->portreset = portreset; hp->portstatus = portstatus; hp->shutdown = shutdown; hp->debug = setdebug; hp->type = "xhci"; return 0; } void usbxhcilink(void) { addhcitype("xhci", reset); }