ref: 8e50556f14d6fc1271c606dce05f1229bed7ae93
parent: 7a8e875668d92020548d07b5ad30f7b83b61d1fc
author: cinap_lenrek <[email protected]>
date: Tue Nov 20 19:37:35 EST 2018
upas/fs: rework the refcounting, use 64bit qid path, add checks add function to check the refcounts for Mailbox and Message on a fid use the full 64 bit of the qid.path, so we can use the full 32 bit for the id instead of only maintaining refcount for the top message, msgincref() now adds a reference to all its parent messages including self and top message. then we can check in recursive delmessage() that all the parts have a zero refcount. remove the Fid.mtop field, it was never used. make sure deletion and flag changes only affect the top message. cachefree(): only look for top message in lru. sub-parts are never added to the cache. use the nparts field when reading sub-part of existing message, so that we parse the index right in case the number of parts somehow changed. messages marked as Deleted but still in inbox should be written to the index.
--- a/sys/src/cmd/upas/fs/cache.c
+++ b/sys/src/cmd/upas/fs/cache.c
@@ -31,17 +31,29 @@
addlru(mb, m);
}
-static void
-cachefree0(Mailbox *mb, Message *m, int force)
+void
+cachefree(Mailbox *mb, Message *m, int force)
{
- long sz, i;
- Message *s;
+ long i;
+ Message *s, **ll;
if(!force && mb->fetch == nil)
return;
+ if(Topmsg(mb, m)){
+ for(ll = &mb->lru; *ll != nil; ll = &((*ll)->lru)){
+ if(*ll == m){
+ mb->nlru--;
+ *ll = m->lru;
+ m->lru = nil;
+ break;
+ }
+ }
+ if(mb->decache)
+ mb->decache(mb, m);
+ mb->cached -= m->csize;
+ }
for(s = m->part; s; s = s->next)
cachefree(mb, s, force);
- dprint("cachefree: %D %p, %p\n", m->fileid, m, m->start);
if(m->mallocd){
free(m->start);
m->mallocd = 0;
@@ -58,7 +70,6 @@
free(m->references[i]);
m->references[i] = 0;
}
- sz = m->csize;
m->csize = 0;
m->start = 0;
m->end = 0;
@@ -69,33 +80,13 @@
m->bend = 0;
m->mheader = 0;
m->mhend = 0;
- if(mb->decache)
- mb->decache(mb, m);
m->decoded = 0;
m->converted = 0;
m->badchars = 0;
m->cstate &= ~(Cheader|Cbody);
- if(Topmsg(mb, m))
- mb->cached -= sz;
}
void
-cachefree(Mailbox *mb, Message *m, int force)
-{
- Message **ll;
-
- for(ll = &mb->lru; *ll != nil; ll = &((*ll)->lru)){
- if(*ll == m){
- mb->nlru--;
- *ll = m->lru;
- m->lru = nil;
- break;
- }
- }
- cachefree0(mb, m, force);
-}
-
-void
putcache(Mailbox *mb, Message *m)
{
int n;
@@ -277,7 +268,8 @@
}
if(y == 0)
return 0;
- dprint("middlecache %d [%D] %lud %lud\n", m->id, m->fileid, (ulong)(m->end - m->start), m->size);
+ dprint("middlecache %lud [%D] %lud %lud\n",
+ m->id, m->fileid, (ulong)(m->end - m->start), m->size);
return cachebody(mb, m);
}
@@ -292,7 +284,7 @@
return 0;
if(!Topmsg(mb, m))
return middlecache(mb, m);
- dprint("cacheheaders %d %D\n", m->id, m->fileid);
+ dprint("cacheheaders %lud %D\n", m->id, m->fileid);
if(m->size < 10000)
r = fetch(mb, m, 0, m->size);
else for(r = 0; (o = m->end - m->start) < m->size; ){
@@ -327,7 +319,7 @@
m->deleted = Dup; /* no dups allowed */
}else
mtreeadd(mb, m);
- dprint("%d %#A\n", m->id, m->digest);
+ dprint("%lud %#A\n", m->id, m->digest);
}
int
@@ -337,10 +329,10 @@
while(!Topmsg(mb, m))
m = m->whole;
- if(!mb->fetch || m->cstate&Cbody)
+ if(mb->fetch == nil || m->cstate&Cbody)
return 0;
o = m->end - m->start;
- dprint("cachebody %d [%D] %lud %lud %s", m->id, m->fileid, o, m->size, cstate(m));
+ dprint("cachebody %lud [%D] %lud %lud %s", m->id, m->fileid, o, m->size, cstate(m));
if(o < m->size)
if(fetch(mb, m, o, m->size - o) < 0)
return -1;
@@ -389,7 +381,7 @@
{
if(m->deleted || !m->inmbox)
return -1;
- msgincref(m);
+ msgincref(mb, m);
cacheidx(mb, m);
if((m->cstate & Cidx) == 0){
logmsg(m, "%s: can't cache: %s: %r", mb->path, m->name);
--- a/sys/src/cmd/upas/fs/dat.h
+++ b/sys/src/cmd/upas/fs/dat.h
@@ -74,7 +74,7 @@
typedef struct Message Message;
struct Message {
- int id;
+ ulong id;
int refs;
int subname;
char name[12];
@@ -152,7 +152,7 @@
struct Mailbox {
int refs;
Mailbox *next;
- int id;
+ ulong id;
int flags;
char rmflags;
char dolock; /* lock when syncing? */
@@ -209,7 +209,6 @@
void putcache(Mailbox*, Message*); /* asymmetricial */
void cachefree(Mailbox*, Message*, int);
-Message* gettopmsg(Mailbox*, Message*);
char* syncmbox(Mailbox*, int);
void* emalloc(ulong);
void* erealloc(void*, ulong);
@@ -223,13 +222,12 @@
void eprint(char*, ...);
void iprint(char *, ...);
-int newid(void);
char* newmbox(char*, char*, int, Mailbox**);
void freembox(char*);
char* removembox(char*, int);
void syncallmboxes(void);
void logmsg(Message*, char*, ...);
-void msgincref(Message*);
+void msgincref(Mailbox*, Message*);
void msgdecref(Mailbox*, Message*);
void mboxincref(Mailbox*);
void mboxdecref(Mailbox*);
@@ -306,7 +304,7 @@
Qmboxctl,
};
-#define PATH(id, f) ((((id) & 0xfffff)<<10) | (f))
+#define PATH(id, f) (((uvlong)(id)<<10) | (f))
#define FILE(p) ((p) & 0x3ff)
/* hash table to aid in name lookup, all files have an entry */
@@ -314,16 +312,16 @@
struct Hash {
Hash *next;
char *name;
- ulong ppath;
+ uvlong ppath;
Qid qid;
Mailbox *mb;
Message *m;
};
-uint hash(char*);
-Hash *hlook(ulong, char*);
-void henter(ulong, char*, Qid, Message*, Mailbox*);
-void hfree(ulong, char*);
+ulong hash(char*);
+Hash *hlook(uvlong, char*);
+void henter(uvlong, char*, Qid, Message*, Mailbox*);
+void hfree(uvlong, char*);
char *intern(char*);
void idxfree(Idx*);
--- a/sys/src/cmd/upas/fs/fs.c
+++ b/sys/src/cmd/upas/fs/fs.c
@@ -14,7 +14,6 @@
Fid *next;
Mailbox *mb;
Message *m;
- Message *mtop; /* top level message */
long foff; /* offset/DIRLEN of finger */
Message *fptr; /* pointer to message at off */
@@ -132,8 +131,6 @@
void
sanemsg(Message *m)
{
- assert(m->refs < 100);
- assert(m->next != m);
if(m->end < m->start)
abort();
if(m->ballocd && (m->start <= m->body && m->end >= m->body))
@@ -769,13 +766,10 @@
return nil;
nf->busy = 1;
nf->open = 0;
- nf->mb = f->mb;
- if(nf->mb)
+ if(nf->mb = f->mb)
mboxincref(nf->mb);
if(nf->m = f->m)
- msgincref(gettopmsg(nf->mb, nf->m));
- if(nf->mtop = f->mtop)
- msgincref(nf->mtop);
+ msgincref(nf->mb, nf->m);
nf->qid = f->qid;
return nf;
}
@@ -798,9 +792,10 @@
{
char *rv, *p;
int t, t1;
- Mailbox *mb;
Hash *h;
+ if(f->qid.type != QTDIR)
+ return Enotdir;
t = FILE(f->qid.path);
rv = Enotexist;
@@ -815,26 +810,16 @@
}else
h = hlook(f->qid.path, name);
if(h != nil){
+ if(h->mb)
+ mboxincref(h->mb);
+ if(h->m)
+ msgincref(h->mb, h->m);
if(f->m)
- msgdecref(f->mb, gettopmsg(f->mb, f->m));
- if(f->mb && f->mb != h->mb)
+ msgdecref(f->mb, f->m);
+ if(f->mb)
mboxdecref(f->mb);
- f->mb = h->mb;
f->m = h->m;
- if(f->m)
- msgincref(gettopmsg(f->mb, f->m));
- switch(t){
- case Qtop:
- if(f->mb)
- mboxincref(f->mb);
- break;
- case Qmbox:
- if(f->m){
- msgincref(f->m);
- f->mtop = f->m;
- }
- break;
- }
+ f->mb = h->mb;
f->qid = h->qid;
if(t1 < Qmax)
f->qid.path = PATH(f->m->id, t1); /* sleezy speedup */
@@ -842,18 +827,9 @@
}else if((p = strchr(name, '.')) != nil && *name != '.'){
*p = 0;
goto retry;
- }
-
- if(rv == nil)
- return rv;
-
- if(strcmp(name, ".") == 0)
- return nil;
-
- if(f->qid.type != QTDIR)
- return Enotdir;
-
- if(strcmp(name, "..") == 0){
+ } else if(strcmp(name, ".") == 0){
+ rv = nil;
+ } else if(strcmp(name, "..") == 0){
switch(t){
case Qtop:
f->qid.path = PATH(0, Qtop);
@@ -864,9 +840,8 @@
f->qid.path = PATH(0, Qtop);
f->qid.type = QTDIR;
f->qid.vers = 0;
- mb = f->mb;
+ mboxdecref(f->mb);
f->mb = nil;
- mboxdecref(mb);
break;
case Qdir:
if(Topmsg(f->mb, f->m)){
@@ -873,11 +848,11 @@
f->qid.path = PATH(f->mb->id, Qmbox);
f->qid.type = QTDIR;
f->qid.vers = f->mb->vers;
- msgdecref(f->mb, f->mtop);
msgdecref(f->mb, f->m);
- f->m = f->mtop = nil;
+ f->m = nil;
} else {
- /* refs don't change; still the same message */
+ msgincref(f->mb, f->m->whole);
+ msgdecref(f->mb, f->m);
f->m = f->m->whole;
f->qid.path = PATH(f->m->id, Qdir);
f->qid.type = QTDIR;
@@ -987,6 +962,8 @@
pos += m;
for(mb = mbl; mb != nil; mb = mb->next){
+ assert(mb->refs > 0);
+
mkstat(&d, mb, nil, Qmbox);
m = convD2M(&d, &buf[n], blen - n);
if(off <= pos){
@@ -1008,6 +985,8 @@
long pos;
Message *msg;
+ assert(f->mb->refs > 0);
+
if(off == 0)
syncmbox(f->mb, 1);
@@ -1198,7 +1177,6 @@
{
char *argvbuf[1024], **argv, file[Pathlen], *err, *v0;
int i, t, argc, flags;
- Message *m;
t = FILE(f->qid.path);
rhdr.count = thdr.count;
@@ -1288,21 +1266,19 @@
}
return Ebadctl;
case Qmboxctl:
- if(f->mb && f->mb->ctl){
- argc = tokenize(thdr.data, argv, nelem(argvbuf));
- if(argc == 0)
- return Ebadctl;
- return f->mb->ctl(f->mb, argc, argv);
- }
- break;
+ if(f->mb->ctl == nil)
+ break;
+ argc = tokenize(thdr.data, argv, nelem(argvbuf));
+ if(argc == 0)
+ return Ebadctl;
+ return f->mb->ctl(f->mb, argc, argv);
case Qflags:
/*
* modifying flags on subparts is a little strange.
*/
- if(!f->mb || !f->m)
+ if(!Topmsg(f->mb, f->m))
break;
- m = gettopmsg(f->mb, f->m);
- return modflags(f->mb, m, thdr.data);
+ return modflags(f->mb, f->m, thdr.data);
}
return Eperm;
}
@@ -1310,21 +1286,17 @@
char*
rclunk(Fid *f)
{
- Mailbox *mb;
-
f->busy = 1;
/* coherence(); */
f->fid = -1;
f->open = 0;
- mb = f->mb;
- if(f->mtop)
- msgdecref(mb, f->mtop);
- if(f->m)
- msgdecref(mb, gettopmsg(mb, f->m));
- f->m = f->mtop = nil;
- if(mb){
+ if(f->m != nil){
+ msgdecref(f->mb, f->m);
+ f->m = nil;
+ }
+ if(f->mb != nil){
+ mboxdecref(f->mb);
f->mb = nil;
- mboxdecref(mb);
}
f->busy = 0;
return 0;
@@ -1333,7 +1305,7 @@
char *
rremove(Fid *f)
{
- if(f->m != nil && f->m->deleted == 0)
+ if(f->mb != nil && f->m != nil && Topmsg(f->mb, f->m) && f->m->deleted == 0)
f->m->deleted = Deleted;
return rclunk(f);
}
@@ -1357,6 +1329,28 @@
return Eperm;
}
+static Fid*
+checkfid(Fid *f)
+{
+ switch(FILE(f->qid.path)){
+ case Qtop:
+ case Qctl:
+ assert(f->mb == nil);
+ assert(f->m == nil);
+ break;
+ case Qmbox:
+ case Qmboxctl:
+ assert(f->mb != nil && f->mb->refs > 0);
+ assert(f->m == nil);
+ break;
+ default:
+ assert(f->mb != nil && f->mb->refs > 0);
+ assert(f->m != nil && f->m->refs > 0);
+ break;
+ }
+ return f;
+}
+
Fid*
newfid(int fid)
{
@@ -1365,7 +1359,7 @@
ff = 0;
for(f = fids; f; f = f->next)
if(f->fid == fid)
- return f;
+ return checkfid(f);
else if(!ff && !f->busy)
ff = f;
if(ff){
@@ -1475,20 +1469,6 @@
}
}
-int
-newid(void)
-{
- int rv;
- static int id;
- static Lock idlock;
-
- lock(&idlock);
- rv = ++id;
- unlock(&idlock);
-
- return rv;
-}
-
void
error(char *s)
{
@@ -1600,10 +1580,10 @@
return to - buf;
}
-uint
+ulong
hash(char *s)
{
- uint c, h;
+ ulong c, h;
h = 0;
while(c = *s++)
@@ -1613,9 +1593,9 @@
}
Hash*
-hlook(ulong ppath, char *name)
+hlook(uvlong ppath, char *name)
{
- int h;
+ ulong h;
Hash *hp;
h = (hash(name)+ppath) % nelem(htab);
@@ -1626,9 +1606,9 @@
}
void
-henter(ulong ppath, char *name, Qid qid, Message *m, Mailbox *mb)
+henter(uvlong ppath, char *name, Qid qid, Message *m, Mailbox *mb)
{
- int h;
+ ulong h;
Hash *hp, **l;
h = (hash(name)+ppath) % nelem(htab);
@@ -1650,9 +1630,9 @@
}
void
-hfree(ulong ppath, char *name)
+hfree(uvlong ppath, char *name)
{
- int h;
+ ulong h;
Hash *hp, **l;
h = (hash(name)+ppath) % nelem(htab);
--- a/sys/src/cmd/upas/fs/idx.c
+++ b/sys/src/cmd/upas/fs/idx.c
@@ -366,18 +366,20 @@
/*
* read in mutable information.
* currently this is only flags
+ * and nparts.
*/
redux++;
if(level == 0)
m->deleted &= ~Dmark;
- if(m->nparts)
+ n = m->nparts;
+ m->nparts = strtoul(f[21], 0, 0);
if(rdidx(b, mb, m, m->nparts, level + 1) == -1)
goto dead;
ll = &m->next;
- idprint("%d seen before %d... %.2ux", level, m->id, m->cstate);
+ idprint("%d seen before %lud... %.2ux", level, m->id, m->cstate);
flags = m->flags;
m->flags |= strtoul(f[1], 0, 16);
- if(flags != m->flags)
+ if(flags != m->flags || n != m->nparts)
m->cstate |= Cidxstale;
m->cstate |= Cidx;
idprint("→%.2ux\n", m->cstate);
@@ -385,7 +387,7 @@
continue;
}
m = newmessage(parent);
- idprint("%d new %d %#A\n", level, m->id, digest);
+ idprint("%d new %lud %#A\n", level, m->id, digest);
m->digest = digest;
m->flags = strtoul(f[1], 0, 16);
m->fileid = rdfileid(f[2], level);
--- a/sys/src/cmd/upas/fs/mbox.c
+++ b/sys/src/cmd/upas/fs/mbox.c
@@ -191,6 +191,20 @@
head[i].len = strlen(head[i].type);
}
+static ulong
+newid(void)
+{
+ ulong rv;
+ static ulong id;
+ static Lock idlock;
+
+ lock(&idlock);
+ rv = ++id;
+ unlock(&idlock);
+
+ return rv;
+}
+
char*
newmbox(char *path, char *name, int flags, Mailbox **r)
{
@@ -271,50 +285,42 @@
{
Mailbox **l, *mb;
- for(l=&mbl; *l != nil; l=&(*l)->next)
- if(strcmp(name, (*l)->name) == 0){
- mb = *l;
+ for(l = &mbl; (mb = *l) != nil; l = &mb->next)
+ if(strcmp(name, mb->name) == 0){
*l = mb->next;
+ mb->next = nil;
+ hfree(PATH(0, Qtop), mb->name);
+ if(mb->ctl)
+ hfree(PATH(mb->id, Qmbox), "ctl");
mboxdecref(mb);
break;
}
- hfree(PATH(0, Qtop), name);
}
-void
-syncallmboxes(void)
-{
- char *err;
- Mailbox *m;
-
- for(m = mbl; m != nil; m = m->next)
- if(err = syncmbox(m, 0))
- eprint("syncmbox: %s\n", err);
-}
-
-
char*
removembox(char *name, int flags)
{
- int found;
- Mailbox **l, *mb;
+ Mailbox *mb;
- found = 0;
- for(l=&mbl; *l != nil; l=&(*l)->next)
- if(strcmp(name, (*l)->path) == 0){
- mb = *l;
- *l = mb->next;
+ for(mb = mbl; mb != nil; mb = mb->next)
+ if(strcmp(name, mb->path) == 0){
mb->flags |= ORCLOSE;
mb->rmflags = flags;
- mboxdecref(mb);
- found = 1;
- break;
+ freembox(mb->name);
+ return 0;
}
- hfree(PATH(0, Qtop), name);
+ return "maibox not found";
+}
- if(found == 0)
- return "maibox not found";
- return 0;
+void
+syncallmboxes(void)
+{
+ Mailbox *mb;
+ char *err;
+
+ for(mb = mbl; mb != nil; mb = mb->next)
+ if(err = syncmbox(mb, 0))
+ eprint("syncmbox: %s\n", err);
}
/*
@@ -347,7 +353,7 @@
return r;
}
-Message*
+static Message*
gettopmsg(Mailbox *mb, Message *m)
{
while(!Topmsg(mb, m))
@@ -355,7 +361,7 @@
return m;
}
-void
+static void
datesec(Mailbox *mb, Message *m)
{
vlong v;
@@ -1011,16 +1017,20 @@
{
Message **l;
+ assert(m->refs == 0);
+ while(m->part)
+ delmessage(mb, m->part);
+
mb->vers++;
msgfreed++;
- if(m->whole != m){
+ if(m != m->whole){
/* unchain from parent */
for(l = &m->whole->part; *l && *l != m; l = &(*l)->next)
;
if(*l != nil)
*l = m->next;
-
+ m->next = nil;
/* clear out of name lookup hash table */
if(m->whole->whole == m->whole)
hfree(PATH(mb->id, Qmbox), m->name);
@@ -1027,15 +1037,12 @@
else
hfree(PATH(m->whole->id, Qdir), m->name);
hfree(PATH(m->id, Qdir), "xxx"); /* sleezy speedup */
- }
- if(Topmsg(mb, m)){
- if(m != mb->root)
+
+ if(Topmsg(mb, m))
mtreedelete(mb, m);
cachefree(mb, m, 1);
+ idxfree(m);
}
- idxfree(m);
- while(m->part)
- delmessage(mb, m->part);
free(m->unixfrom);
free(m->unixheader);
free(m->date822);
@@ -1093,7 +1100,7 @@
if(argc%2)
return "bad flags";
- for(mb = mbl; mb; mb = mb->next)
+ for(mb = mbl; mb != nil; mb = mb->next)
if(strcmp(*argv, mb->name) == 0)
break;
if(mb == nil)
@@ -1114,21 +1121,33 @@
}
void
-msgincref(Message *m)
+msgincref(Mailbox *mb, Message *m)
{
- m->refs++;
+ assert(mb->refs > 0);
+ for(;; m = m->whole){
+ assert(m->refs >= 0);
+ m->refs++;
+ if(Topmsg(mb, m))
+ break;
+ }
}
void
msgdecref(Mailbox *mb, Message *m)
{
- assert(m->refs > 0);
- m->refs--;
- if(m->refs == 0){
- if(m->deleted)
- syncmbox(mb, 1);
- else
- putcache(mb, m);
+ assert(mb->refs >= 0);
+ for(;; m = m->whole){
+ assert(m->refs > 0);
+ m->refs--;
+ if(Topmsg(mb, m)){
+ if(m->refs == 0){
+ if(m->deleted)
+ syncmbox(mb, 1);
+ else
+ putcache(mb, m);
+ }
+ break;
+ }
}
}
@@ -1156,21 +1175,18 @@
mboxdecref(Mailbox *mb)
{
assert(mb->refs > 0);
- mb->refs--;
- if(mb->refs == 0){
- syncmbox(mb, 1);
- delmessage(mb, mb->root);
- if(mb->ctl)
- hfree(PATH(mb->id, Qmbox), "ctl");
- if(mb->close)
- mb->close(mb);
- if(mb->flags & ORCLOSE && mb->remove)
- if(mb->remove(mb, mb->rmflags))
- rmidx(mb->path, mb->rmflags);
- free(mb->mtree);
- free(mb->d);
- free(mb);
- }
+ if(--mb->refs)
+ return;
+ syncmbox(mb, 1);
+ delmessage(mb, mb->root);
+ if(mb->close)
+ mb->close(mb);
+ if(mb->flags & ORCLOSE && mb->remove)
+ if(mb->remove(mb, mb->rmflags))
+ rmidx(mb->path, mb->rmflags);
+ free(mb->mtree);
+ free(mb->d);
+ free(mb);
}
--- a/sys/src/cmd/upas/fs/mkfile
+++ b/sys/src/cmd/upas/fs/mkfile
@@ -1,7 +1,7 @@
</$objtype/mkfile
<../mkupas
-TARG= fs\
+TARG=fs
OFILES=\
cache.$O\