ref: 0fd4ad8016cecf7210e64e9878d2cd6861a72fbf
dir: /sys/src/9/teg2/cache-l2-pl310.c/
/*
* PL310 level 2 cache (non-architectural bag on the side)
*
* guaranteed to work incorrectly with default settings; must set Sharovr.
*
* clean & invalidate (wbinv) is buggy, so we work around erratum 588369
* by disabling write-back and cache line-fill before, and restoring after.
*/
#include "u.h"
#include "../port/lib.h"
#include "mem.h"
#include "dat.h"
#include "fns.h"
#include "io.h"
#include "../port/error.h"
#include "arm.h"
#define NWAYS(l2p) ((l2p)->auxctl & Assoc16way? 16: 8)
#define L2P ((L2pl310 *)soc.l2cache)
enum {
L2size = 1024 * 1024, /* according to the tegra 2 manual */
Wayszgran = 16 * KiB, /* granularity of way sizes */
};
typedef struct L2pl310 L2pl310;
typedef struct Pl310op Pl310op;
struct Pl310op {
ulong pa;
ulong _pad;
ulong indexway;
ulong way;
};
struct L2pl310 {
ulong id;
ulong type;
uchar _pad0[0x100 - 0x8];
ulong ctl;
ulong auxctl;
uchar _pad1[0x730 - 0x108]; /* boring regs */
ulong sync;
uchar _pad2[0x740 - 0x734];
ulong r3p0sync; /* workaround for r3p0 bug */
uchar _pad3[0x770 - 0x744];
Pl310op inv; /* inv.indexway doesn't exist */
uchar _pad4[0x7b0 - 0x780];
Pl310op clean;
uchar _pad5[0x7f0 - 0x7c0];
Pl310op cleaninv;
uchar _pad6[0xc00 - 0x7d0];
ulong filtstart;
ulong filtend;
uchar _pad6[0xf40 - 0xc08];
ulong debug;
/* ... */
};
enum {
/* ctl bits */
L2enable = 1,
/* auxctl bits */
Ipref = 1<<29, /* prefetch enables */
Dpref = 1<<28,
Mbo = 1<<25,
Sharovr = 1<<22, /* shared attribute override (i.e., work right!) */
Parity = 1<<21,
Waycfgshift= 17,
Waycfgmask = (1<<3) - 1,
Assoc16way = 1<<16,
/*
* optim'n to 0 cache lines; must be enabled in a9(?!).
* set CpAClwr0line on all cpus 1st.
*/
Fullline0= 1<<0,
/* debug bits */
Wt = 1<<1, /* write-through, not write-back */
Nolinefill= 1<<0,
Basecfg = Wt | Nolinefill,
};
static Lock l2lock;
static int disallowed; /* by user: *l2off= in plan9.ini */
static int l2ison;
static int bg_op_running;
static ulong waysmask;
static Cacheimpl l2cacheimpl;
static void
awaitbgop(void)
{
while (bg_op_running)
;
}
static void
getlock(void)
{
awaitbgop(); /* wait at normal PL first */
ilock(&l2lock);
awaitbgop(); /* wait under lock */
}
static void
l2pl310sync(void)
{
L2P->sync = 0;
coherence();
}
/* call this first to set sets/ways configuration */
void
l2pl310init(void)
{
int waysz, nways;
ulong new;
L2pl310 *l2p = L2P;
static int configed;
if (getconf("*l2off") != nil) {
// iprint("l2 cache (pl310) disabled\n");
disallowed = 1;
return;
}
if (l2ison || configed)
return;
l2cache = &l2cacheimpl;
cachedwb();
/*
* default config is:
* l2: ext unified, 8 ways 512 sets 32 bytes/line => 128KB
* but the tegra 2 manual says there's 1MB available.
* ways or way-size may be fixed by hardware; the only way to tell
* is to try to change the setting and read it back.
*/
l2pl310sync();
l2cache->inv();
/* figure out number of ways */
l2pl310sync();
nways = NWAYS(l2p);
if (!(l2p->auxctl & Assoc16way)) {
l2p->auxctl |= Assoc16way;
coherence();
l2pl310sync();
nways = NWAYS(l2p);
// iprint("\nl2: was set for 8 ways, asked for 16, got %d\n", nways);
}
waysmask = MASK(nways);
/* figure out way size (and thus number of sets) */
waysz = L2size / nways;
new = l2p->auxctl & ~(Waycfgmask << Waycfgshift) |
(log2(waysz / Wayszgran) + 1) << Waycfgshift;
l2p->auxctl = new;
coherence();
l2pl310sync();
l2cache->inv();
// iprint("\nl2: configed %d ways, %d sets (way size %d)\n", nways,
// waysz / CACHELINESZ, waysz);
if (l2p->auxctl != new)
iprint("l2 config %#8.8lux didn't stick; is now %#8.8lux\n",
new, l2p->auxctl);
configed++;
}
void
l2pl310info(Memcache *cp)
{
int pow2;
ulong waysz;
L2pl310 *l2p = L2P;
memset(cp, 0, sizeof *cp);
if (!l2ison)
return;
l2pl310init();
assert((l2p->id >> 24) == 'A');
cp->level = 2;
cp->type = Unified;
cp->external = Extcache;
cp->setsways = Cara | Cawa | Cawt | Cawb;
cp->l1ip = 3<<14; /* PIPT */
cp->setsh = cp->waysh = 0; /* bag on the side */
cp->linelen = CACHELINESZ;
cp->log2linelen = log2(CACHELINESZ);
cp->nways = NWAYS(l2p);
pow2 = ((l2p->auxctl >> Waycfgshift) & Waycfgmask) - 1;
if (pow2 < 0)
pow2 = 0;
waysz = (1 << pow2) * Wayszgran;
cp->nsets = waysz / CACHELINESZ;
}
void
l2pl310on(void)
{
ulong ctl;
L2pl310 *l2p = L2P;
if (getconf("*l2off") != nil) {
// iprint("l2 cache (pl310) disabled\n");
disallowed = 1;
return;
}
if (l2ison)
return;
l2pl310init();
l2cache->inv();
/*
* drain l1. can't turn it off (which would make locks not work)
* because doing so makes references below to the l2 registers wedge
* the system.
*/
cacheuwbinv();
cacheiinv();
/*
* this is only called once, on cpu0 at startup,
* so we don't need locks here.
* must do all configuration before enabling l2 cache.
*/
l2p->filtend = 0;
coherence();
l2p->filtstart = 0; /* no enable bit */
l2p->debug = 0; /* write-back, line fills allowed */
coherence();
ctl = l2p->auxctl;
/* don't change number of sets & ways, but reset all else. */
ctl &= Waycfgmask << Waycfgshift | Assoc16way;
ctl |= Sharovr; /* actually work correctly for a change */
ctl |= Mbo | Ipref | Dpref | Parity | Fullline0;
l2p->auxctl = ctl;
coherence();
l2p->ctl |= L2enable;
coherence();
l2ison = 1;
// iprint("l2 cache (pl310) now on\n");
}
void
l2pl310off(void)
{
if (!l2ison)
return;
l2cache->wbinv();
getlock();
L2P->ctl &= ~L2enable;
coherence();
l2ison = 0;
iunlock(&l2lock);
}
static void
applyrange(ulong *reg, void *ava, int len)
{
uintptr va, endva;
if (disallowed || !l2ison)
return;
if (len < 0)
panic("l2cache*se called with negative length");
endva = (uintptr)ava + len;
for (va = (uintptr)ava & ~(CACHELINESZ-1); va < endva;
va += CACHELINESZ)
*reg = PADDR(va);
l2pl310sync();
}
void
l2pl310invse(void *va, int bytes)
{
uintptr start, end;
L2pl310 *l2p = L2P;
/*
* if start & end addresses are not on cache-line boundaries,
* flush first & last cachelines before invalidating.
*/
start = (uintptr)va;
end = start + bytes;
getlock();
if (start % CACHELINESZ != 0) {
// iprint("l2pl310invse: unaligned start %#p from %#p\n", start,
// getcallerpc(&va));
applyrange(&l2p->clean.pa, va, 1);
}
if (end % CACHELINESZ != 0) {
// iprint("l2pl310invse: unaligned end %#p from %#p\n", end,
// getcallerpc(&va));
applyrange(&l2p->clean.pa, (char *)va + bytes, 1);
}
applyrange(&l2p->inv.pa, va, bytes);
iunlock(&l2lock);
}
void
l2pl310wbse(void *va, int bytes)
{
getlock();
applyrange(&L2P->clean.pa, va, bytes);
iunlock(&l2lock);
}
/*
* assume that ldrex/strex (thus locks) won't work when Wt in is effect,
* so don't manipulate locks between setting and clearing Wt.
*/
void
l2pl310wbinvse(void *va, int bytes)
{
int odb;
L2pl310 *l2p = L2P;
if (!l2ison)
return;
getlock();
applyrange(&l2p->clean.pa, va, bytes); /* paranoia */
odb = l2p->debug;
l2p->debug |= Wt | Nolinefill; /* erratum workaround */
coherence();
applyrange(&l2p->cleaninv.pa, va, bytes);
l2p->debug = odb;
iunlock(&l2lock);
}
/*
* we want to wait for completion at normal PL.
* if waiting is interrupted, interrupt code that calls
* these ops could deadlock on a uniprocessor, so we only
* give up l2lock before waiting on multiprocessors.
* in this port, only cpu 0 gets interrupts other than local timer ones.
*/
void
l2pl310inv(void)
{
L2pl310 *l2p = L2P;
if (disallowed)
return;
getlock();
bg_op_running = 1;
l2p->inv.way = waysmask;
coherence();
if (conf.nmach > 1)
iunlock(&l2lock);
while (l2p->inv.way & waysmask)
;
if (conf.nmach > 1)
ilock(&l2lock);
l2pl310sync();
bg_op_running = 0;
iunlock(&l2lock);
}
/*
* maximum time seen is 2542µs, typical is 625µs.
*/
void
l2pl310wb(void)
{
L2pl310 *l2p = L2P;
if (disallowed || !l2ison)
return;
getlock();
bg_op_running = 1;
l2p->clean.way = waysmask;
coherence();
if (conf.nmach > 1)
iunlock(&l2lock);
while (l2p->clean.way & waysmask)
;
if (conf.nmach > 1)
ilock(&l2lock);
l2pl310sync();
bg_op_running = 0;
iunlock(&l2lock);
}
void
l2pl310wbinv(void)
{
int odb;
L2pl310 *l2p = L2P;
if (disallowed || !l2ison)
return;
l2pl310wb(); /* paranoia */
getlock();
bg_op_running = 1;
odb = l2p->debug;
l2p->debug |= Wt | Nolinefill; /* erratum workaround */
coherence();
l2p->cleaninv.way = waysmask;
coherence();
if (conf.nmach > 1)
iunlock(&l2lock);
while (l2p->cleaninv.way & waysmask)
;
if (conf.nmach > 1)
ilock(&l2lock);
l2pl310sync();
l2p->debug = odb;
bg_op_running = 0;
iunlock(&l2lock);
}
static Cacheimpl l2cacheimpl = {
.info = l2pl310info,
.on = l2pl310on,
.off = l2pl310off,
.inv = l2pl310inv,
.wb = l2pl310wb,
.wbinv = l2pl310wbinv,
.invse = l2pl310invse,
.wbse = l2pl310wbse,
.wbinvse= l2pl310wbinvse,
};