shithub: riscv

Info  •  Files  •  Log  •  Branches

pc64: allow using the FPU in syscall and pagefault handlers

The aim is to take advantage of SSE instructions such as AES-NI
in the kernel by lazily saving and restoring FPU state across
system calls and pagefaults. (everything can can do I/O)

This is accomplished by the functions fpusave() and fpurestore().

fpusave() remembers the current state and disables the FPU if it
was active by setting the TS flag. In case the FPU gets used,
the current state gets saved and a new PFPU.fpslot is allocated
by mathemu().

fpurestore() restores the previous FPU state, reenabling the FPU
if fpusave() disabled it.

In the most common case, when userspace is not using the FPU,
then fpusave()/fpurestore() just toggle the FPpush bit in
up->fpstate.

When the FPU was active, but we do not use the FPU, then nothing
needs to be saved or restored. We just switched the TS flag on
and off agaian.

Note, this is done for the amd64 kernel only.

--- a/sys/src/9/pc64/dat.h

+++ b/sys/src/9/pc64/dat.h

@@ -65,12 +65,6 @@

 	uchar	ign[96];		/* reserved, ignored */

 };

-struct PFPU

-{

-	int	fpstate;

-	FPsave	*fpsave;

-};

-

 enum

 {

 	/* this is a state */

@@ -78,8 +72,27 @@

 	FPactive=	1,

 	FPinactive=	2,

-	/* the following is a bit that can be or'd into the state */

-	FPillegal=	0x100,

+	/*

+	 * the following are bits that can be or'd into the state.

+	 *

+	 * this is biased so that FPinit, FPactive and FPinactive

+	 * without any flags refer to user fp state in fpslot[0].

+	 */

+	FPillegal=	1<<8,	/* fp forbidden in note handler */

+	FPpush=		2<<8,	/* trap on use and initialize new fpslot */

+	FPnouser=	4<<8,	/* fpslot[0] is kernel regs */

+	FPkernel=	8<<8,	/* fp use in kernel (user in fpslot[0] when !FPnouser) */

+

+	FPindexs=	16,

+	FPindex1=	1<<FPindexs,

+	FPindexm=	3<<FPindexs,

+};

+

+struct PFPU

+{

+	int	fpstate;

+	FPsave	*fpsave;	/* fpslot[fpstate>>FPindexs] */

+	FPsave	*fpslot[(FPindexm+1)>>FPindexs];

 };

 struct Confmem

--- a/sys/src/9/pc64/fns.h

+++ b/sys/src/9/pc64/fns.h

@@ -41,6 +41,8 @@

 void	fpssesave(FPsave*);

 void	fpx87restore(FPsave*);

 void	fpx87save(FPsave*);

+int	fpusave(void);

+void	fpurestore(int);

 u64int	getcr0(void);

 u64int	getcr2(void);

 u64int	getcr3(void);

--- a/sys/src/9/pc64/main.c

+++ b/sys/src/9/pc64/main.c

@@ -473,13 +473,13 @@

  *  math coprocessor error

  */

 static void

-matherror(Ureg*, void*)

+matherror(Ureg *, void*)

 {

 	/*

 	 * Save FPU state to check out the error.

 	 */

 	fpsave(up->fpsave);

-	up->fpstate = FPinactive;

+	up->fpstate = FPinactive | (up->fpstate & (FPnouser|FPkernel|FPindexm));

 	mathnote(up->fpsave->fsw, up->fpsave->rip);

 }

@@ -490,7 +490,7 @@

 simderror(Ureg *ureg, void*)

 {

 	fpsave(up->fpsave);

-	up->fpstate = FPinactive;

+	up->fpstate = FPinactive | (up->fpstate & (FPnouser|FPkernel|FPindexm));

 	mathnote(up->fpsave->mxcsr & 0x3f, ureg->pc);

 }

@@ -519,6 +519,7 @@

 mathemu(Ureg *ureg, void*)

 {

 	ulong status, control;

+	int index;

 	if(up->fpstate & FPillegal){

 		/* someone did floating point in a note handler */

@@ -525,12 +526,30 @@

 		postnote(up, 1, "sys: floating point in note handler", NDebug);

 		return;

 	}

-	switch(up->fpstate){

+	switch(up->fpstate & ~(FPnouser|FPkernel|FPindexm)){

+	case FPactive	| FPpush:

+		_clts();

+		fpsave(up->fpsave);

+	case FPinactive	| FPpush:

+		up->fpstate += FPindex1;

+	case FPinit	| FPpush:

 	case FPinit:

 		fpinit();

-		while(up->fpsave == nil)

-			up->fpsave = mallocalign(sizeof(FPsave), FPalign, 0, 0);

-		up->fpstate = FPactive;

+		index = up->fpstate >> FPindexs;

+		if(index < 0 || index > FPindexm)

+			panic("fpslot index overflow: %d", index);

+		if(userureg(ureg)){

+			if(index != 0)

+				panic("fpslot index %d != 0 for user", index);

+		} else {

+			if(index == 0)

+				up->fpstate |= FPnouser;

+			up->fpstate |= FPkernel;

+		}

+		while(up->fpslot[index] == nil)

+			up->fpslot[index] = mallocalign(sizeof(FPsave), FPalign, 0, 0);

+		up->fpsave = up->fpslot[index];

+		up->fpstate = FPactive | (up->fpstate & (FPnouser|FPkernel|FPindexm));

 		break;

 	case FPinactive:

 		/*

@@ -547,7 +566,7 @@

 			break;

 		}

 		fprestore(up->fpsave);

-		up->fpstate = FPactive;

+		up->fpstate = FPactive | (up->fpstate & (FPnouser|FPkernel|FPindexm));

 		break;

 	case FPactive:

 		panic("math emu pid %ld %s pc %#p", 

@@ -596,17 +615,21 @@

 	/* save floating point state */

 	s = splhi();

 	switch(up->fpstate & ~FPillegal){

+	case FPactive	| FPpush:

+		_clts();

 	case FPactive:

 		fpsave(up->fpsave);

-		up->fpstate = FPinactive;

+		up->fpstate = FPinactive | (up->fpstate & FPpush);

+	case FPactive	| FPkernel:

+	case FPinactive	| FPkernel:

+	case FPinactive	| FPpush:

 	case FPinactive:

-		while(p->fpsave == nil)

-			p->fpsave = mallocalign(sizeof(FPsave), FPalign, 0, 0);

-		memmove(p->fpsave, up->fpsave, sizeof(FPsave));

+		while(p->fpslot[0] == nil)

+			p->fpslot[0] = mallocalign(sizeof(FPsave), FPalign, 0, 0);

+		memmove(p->fpsave = p->fpslot[0], up->fpslot[0], sizeof(FPsave));

 		p->fpstate = FPinactive;

 	}

 	splx(s);

-

 }

 void

@@ -644,24 +667,26 @@

 	p->kentry -= t;

 	p->pcycles += t;

-	if(p->fpstate == FPactive){

+	switch(p->fpstate & ~(FPnouser|FPkernel|FPindexm)){

+	case FPactive	| FPpush:

+		_clts();

+	case FPactive:

 		if(p->state == Moribund){

-			_clts();

 			_fnclex();

 			_stts();

+			break;

 		}

-		else{

-			/*

-			 * Fpsave() stores without handling pending

-			 * unmasked exeptions. Postnote() can't be called

-			 * here as sleep() already has up->rlock, so

-			 * the handling of pending exceptions is delayed

-			 * until the process runs again and generates an

-			 * emulation fault to activate the FPU.

-			 */

-			fpsave(p->fpsave);

-		}

-		p->fpstate = FPinactive;

+		/*

+		 * Fpsave() stores without handling pending

+		 * unmasked exeptions. Postnote() can't be called

+		 * here as sleep() already has up->rlock, so

+		 * the handling of pending exceptions is delayed

+		 * until the process runs again and generates an

+		 * emulation fault to activate the FPU.

+		 */

+		fpsave(p->fpsave);

+		p->fpstate = FPinactive | (p->fpstate & (FPpush|FPnouser|FPkernel|FPindexm));

+		break;

 	}

 	/*

@@ -676,4 +701,33 @@

 	 * especially on VMware, but it turns out not to matter.

 	 */

 	mmuflushtlb();

+}

+

+/*

+ * Fpusave and fpurestore lazily save and restore FPU state across

+ * system calls and the pagefault handler so that we can take

+ * advantage of SSE instructions such as AES-NI in the kernel.

+ */

+int

+fpusave(void)

+{

+	int ostate = up->fpstate;

+	if((up->fpstate & ~(FPnouser|FPkernel|FPindexm)) == FPactive)

+		_stts();

+	up->fpstate = FPpush | (up->fpstate & ~FPillegal);

+	return ostate;

+}

+void

+fpurestore(int ostate)

+{

+	if((up->fpstate & ~(FPnouser|FPkernel|FPindexm)) == FPactive)

+		_stts();

+	if((ostate & FPindexm) == (up->fpstate & FPindexm)){

+		if((ostate & ~(FPnouser|FPkernel|FPindexm)) == FPactive)

+			_clts();

+	} else {

+		up->fpsave = up->fpslot[ostate>>FPindexs];

+		ostate = FPinactive | (ostate & (FPillegal|FPpush|FPnouser|FPkernel|FPindexm));

+	}

+	up->fpstate = ostate;

 }

--- a/sys/src/9/pc64/trap.c

+++ b/sys/src/9/pc64/trap.c

@@ -649,11 +649,12 @@

 }

 extern void checkpages(void);

+

 static void

 faultamd64(Ureg* ureg, void*)

 {

 	uintptr addr;

-	int read, user, n, insyscall;

+	int read, user, n, insyscall, f;

 	char buf[ERRMAX];

 	addr = getcr2();

@@ -670,6 +671,14 @@

 	insyscall = up->insyscall;

 	up->insyscall = 1;

+	f = fpusave();

+	if(!user && waserror()){

+		int s = splhi();

+		fpurestore(f);

+		up->insyscall = insyscall;

+		splx(s);

+		nexterror();

+	}

 	n = fault(addr, read);

 	if(n < 0){

 		if(!user){

@@ -681,6 +690,9 @@

 			read ? "read" : "write", addr);

 		postnote(up, 1, buf, NDebug);

 	}

+	if(!user) poperror();

+	splhi();

+	fpurestore(f);

 	up->insyscall = insyscall;

 }

@@ -698,7 +710,7 @@

 	char *e;

 	uintptr	sp;

 	long long ret;

-	int	i, s;

+	int	i, s, f;

 	ulong scallnr;

 	vlong startns, stopns;

@@ -715,11 +727,12 @@

 	sp = ureg->sp;

 	scallnr = ureg->bp;	/* RARG */

 	up->scallnr = scallnr;

-

+	f = fpusave();

 	spllo();

+

+	ret = -1;

 	startns = 0;

 	up->nerrlab = 0;

-	ret = -1;

 	if(!waserror()){

 		if(sp<(USTKTOP-BY2PG) || sp>(USTKTOP-sizeof(Sargs)-BY2WD))

 			validaddr(sp, sizeof(Sargs)+BY2WD, 0);

@@ -778,12 +791,13 @@

 		splx(s);

 	}

+	splhi();

+	fpurestore(f);

 	up->insyscall = 0;

 	up->psstate = 0;

 	if(scallnr == NOTED){

 		noted(ureg, *((ulong*)up->s.args));

-

 		/*

 		 * normally, syscall() returns to forkret()

 		 * not restoring general registers when going

@@ -796,10 +810,10 @@

 	}

 	if(scallnr!=RFORK && (up->procctl || up->nnote)){

-		splhi();

 		notify(ureg);

 		((void**)&ureg)[-1] = (void*)noteret;	/* loads RARG */

 	}

+

 	/* if we delayed sched because we held a lock, sched now */

 	if(up->delaysched)

 		sched();

@@ -813,7 +827,7 @@

 int

 notify(Ureg* ureg)

 {

-	int l, s;

+	int l;

 	uintptr sp;

 	Note *n;

@@ -821,14 +835,7 @@

 		procctl();

 	if(up->nnote == 0)

 		return 0;

-

-	if(up->fpstate == FPactive){

-		fpsave(up->fpsave);

-		up->fpstate = FPinactive;

-	}

-	up->fpstate |= FPillegal;

-

-	s = spllo();

+	spllo();

 	qlock(&up->debug);

 	up->notepending = 0;

 	n = &up->note[0];

@@ -887,11 +894,14 @@

 	up->nnote--;

 	memmove(&up->lastnote, &up->note[0], sizeof(Note));

 	memmove(&up->note[0], &up->note[1], up->nnote*sizeof(Note));

-

 	qunlock(&up->debug);

-	splx(s);

+	splhi();

+	if(up->fpstate == FPactive){

+		fpsave(up->fpsave);

+		up->fpstate = FPinactive;

+	}

+	up->fpstate |= FPillegal;

 	return 1;

-

 }

 /*

@@ -903,6 +913,8 @@

 	Ureg *nureg;

 	uintptr oureg, sp;

+	up->fpstate &= ~FPillegal;

+	spllo();

 	qlock(&up->debug);

 	if(arg0!=NRSTR && !up->notified) {

 		qunlock(&up->debug);

@@ -912,8 +924,6 @@

 	up->notified = 0;

 	nureg = up->ureg;	/* pointer to user returned Ureg struct */

-

-	up->fpstate &= ~FPillegal;

 	/* sanity clause */

 	oureg = (uintptr)nureg;