/*  $Revision: 1.20 $
**
**  Article-related routines.
*/
#include <stdio.h>
#include <sys/types.h>
#include "configdata.h"
#include "clibrary.h"
#include "nnrpd.h"

/*
 * OVERSCREAM - to make the overview database screaming fast, and because
 * I scream in terror about the previous implementation.
 * See http://www.xs4all.nl/~johnpc/inn/ for more information on this patch.
 */

#ifdef OVERSCREAM
# include <sys/types.h>
# include <sys/mman.h>
#endif /* OVERSCREAM */


/*
**  Data structures for use in ARTICLE/HEAD/BODY/STAT common code.
*/
typedef enum _SENDTYPE {
    STarticle,
    SThead,
    STbody,
    STstat
} SENDTYPE;

typedef struct _SENDDATA {
    SENDTYPE	Type;
    int		ReplyCode;
    STRING	Item;
} SENDDATA;


/*
**  Information about the schema of the news overview files.
*/
typedef struct _ARTOVERFIELD {
    char	*Header;
    int		Length;
    BOOL	HasHeader;
    BOOL	NeedsHeader;
} ARTOVERFIELD;


STATIC char		ARTnotingroup[] = NNTP_NOTINGROUP;
STATIC char		ARTnoartingroup[] = NNTP_NOARTINGRP;
STATIC char		ARTnocurrart[] = NNTP_NOCURRART;
STATIC QIOSTATE		*ARTqp;
STATIC ARTOVERFIELD	*ARTfields;
STATIC int		ARTfieldsize;
STATIC int		ARTfirstfullfield = 0;
STATIC SENDDATA		SENDbody = {
    STbody,	NNTP_BODY_FOLLOWS_VAL,		"body"
};
STATIC SENDDATA		SENDarticle = {
    STarticle,	NNTP_ARTICLE_FOLLOWS_VAL,	"article"
};
STATIC SENDDATA		SENDstat = {
    STstat,	NNTP_NOTHING_FOLLOWS_VAL,	"status"
};
STATIC SENDDATA		SENDhead = {
    SThead,	NNTP_HEAD_FOLLOWS_VAL,		"head"
};


/*
**  Overview state information.
*/
#ifdef OVERSCREAM

STATIC caddr_t		OVERshm = (caddr_t) NULL; /* location of mmap	*/
STATIC size_t		OVERsize;		/* size of mmap		*/
STATIC size_t		OVERmsize;		/* real size of mmap	*/
STATIC int		OVERfd;			/* fd of file 		*/
STATIC ARTNUM		OVERfirst, OVERlast;	/* first/last entries	*/
STATIC int		OVERopens;		/* Number of opens done	*/
STATIC char*		OVERcache;		/* cached position	*/
STATIC ARTNUM		OVERprev;		/* previous found art	*/
#define LINSEARCH	5			/* linear search range  */
#define MIDSKEW		0.1			/* 10% bias toward middle */

STATIC int		mmapsuck;		/* did we syslog already */
#define YOUR_MMAP_SUCKS if ( ! mmapsuck++ ) \
    syslog(L_NOTICE, "Your mmap() implementation sucks.")

#else /* !OVERSCREAM */

STATIC QIOSTATE		*OVERqp;		/* Open overview file	*/
STATIC char		*OVERline;		/* Current line		*/
STATIC ARTNUM		OVERarticle;		/* Current article	*/
STATIC int		OVERopens;		/* Number of opens done	*/

#endif



/*
**  Read the overview schema.
*/
void
ARTreadschema()
{
    static char			SCHEMA[] = _PATH_SCHEMA;
    register FILE		*F;
    register char		*p;
    register ARTOVERFIELD	*fp;
    register int		i;
    char			buff[SMBUF];

    /* Open file, count lines. */
    if ((F = fopen(SCHEMA, "r")) == NULL)
	return;
    for (i = 0; fgets(buff, sizeof buff, F) != NULL; i++)
	continue;
    (void)fseek(F, (OFFSET_T)0, SEEK_SET);
    ARTfields = NEW(ARTOVERFIELD, i + 1);

    /* Parse each field. */
    for (fp = ARTfields; fgets(buff, sizeof buff, F) != NULL; ) {
	/* Ignore blank and comment lines. */
	if ((p = strchr(buff, '\n')) != NULL)
	    *p = '\0';
	if ((p = strchr(buff, COMMENT_CHAR)) != NULL)
	    *p = '\0';
	if (buff[0] == '\0')
	    continue;
	if ((p = strchr(buff, ':')) != NULL) {
	    *p++ = '\0';
	    fp->NeedsHeader = EQ(p, "full");
	    if (ARTfirstfullfield == 0)
	      ARTfirstfullfield = fp - ARTfields + 1;
	}
	else
	    fp->NeedsHeader = FALSE;
        fp->HasHeader = FALSE;
	fp->Header = COPY(buff);
	fp->Length = strlen(buff);
	fp++;
    }
    ARTfieldsize = fp - ARTfields;
    (void)fclose(F);
}


/*
**  If we have an article open, close it.
*/
void
ARTclose()
{
    if (ARTqp) {
	QIOclose(ARTqp);
	ARTqp = NULL;
    }
}


/*
**  Get the Message-ID from a file.
*/
STATIC void
ARTgetmsgid(qp, id)
    register QIOSTATE	*qp;
    char		*id;
{
    register char	*p;
    register char	*q;

    for (*id = '\0'; (p = QIOread(qp)) != NULL && *p != '\0'; ) {
	if (*p != 'M' && *p != 'm')
	    continue;
	if ((q = strchr(p, ' ')) == NULL)
	    continue;
	*q++ = '\0';
	if (caseEQ(p, "Message-ID:")) {
	    (void)strcpy(id, q);
	    break;
	}
    }
    (void)QIOrewind(qp);
}


/*
**  If the article name is valid, open it and stuff in the ID.
*/
STATIC BOOL
ARTopen(name, id)
    char		*name;
    char		*id;
{
    static ARTNUM	save_artnum;
    static char		save_artid[BIG_BUFFER];
    struct stat		Sb;

    /* Re-use article if it's the same one. */
    if (ARTqp != NULL) {
	if (save_artnum == atol(name) && QIOrewind(ARTqp) != -1) {
	    if (id)
		(void)strcpy(id, save_artid);
	    return TRUE;
	}
	QIOclose(ARTqp);
    }

    /* Open it, make sure it's a regular file. */
    if ((ARTqp = QIOopen(name, QIO_BUFFER)) == NULL)
	return FALSE;
    if (fstat(QIOfileno(ARTqp), &Sb) < 0 || !S_ISREG(Sb.st_mode)) {
	QIOclose(ARTqp);
	ARTqp = NULL;
	return FALSE;
    }
    CloseOnExec(QIOfileno(ARTqp), TRUE);

    save_artnum = atol(name);
    ARTgetmsgid(ARTqp, save_artid);
    (void)strcpy(id, save_artid);
    return TRUE;
}


/*
**  Open the article for a given Message-ID.
*/
STATIC QIOSTATE *
ARTopenbyid(msg_id, ap)
    char	*msg_id;
    ARTNUM	*ap;
{
    QIOSTATE	*qp;
    char	*p;
    char	*q;

    *ap = 0;
    if ((p = HISgetent(msg_id, FALSE)) == NULL)
	return NULL;
    if ((qp = QIOopen(p, QIO_BUFFER)) == NULL)
	return NULL;
    CloseOnExec(QIOfileno(qp), TRUE);
    p += strlen(_PATH_SPOOL) + 1;
    if ((q = strrchr(p, '/')) != NULL)
	*q++ = '\0';
    if (GRPlast[0] && EQ(p, GRPlast))
	*ap = atol(q);
    return qp;
}


/*
**  Send a (part of) a file to stdout, doing newline and dot conversion.
*/
STATIC void
ARTsend(qp, what)
    register QIOSTATE	*qp;
    SENDTYPE		what;
{
    register char	*p;

    ARTcount++;
    GRParticles++;

    /*
     * it should never take 5 minutes to send an article line
     */
    alarm(DEFAULT_TIMEOUT);

    /* Get the headers. */
    for ( ; ; ) {
	p = QIOread(qp);
	if (p == NULL) {
	    if (QIOtoolong(qp))
		continue;
	    break;
	}
	if (*p == '\0')
	    break;
	if (what == STbody)
	    continue;
	alarm(DEFAULT_TIMEOUT);	/* restart timer */
	Printf("%s%s\r\n", *p == '.' ? "." : "", p);
    }

    if (what == SThead) {
	Printf(".\r\n");
	alarm(0);		/* cancel timer */
	return;
    }

    if (what == STarticle)
	Printf("\r\n");
    for ( ; ; ) {
	p = QIOread(qp);
	if (p == NULL) {
	    if (QIOtoolong(qp))
		continue;
	    break;
	}
	alarm(DEFAULT_TIMEOUT);	/* restart timer */
	Printf("%s%s\r\n", *p == '.' ? "." : "", p);
    }
    Printf(".\r\n");
    alarm(0);			/* cancel timer */

#if defined (DONT_LIKE_PULLERS)
    /* If the client has pulled in more than 100 articles, then there's a
       good chance it's a sucking feed. So we slow down a bit the feeding
       of articles after that point. */
    {
        static int count ;

        if (what == STarticle) {
            if (++count > 100) {
                (void) fflush(stdout) ;
                sleep (1) ;               /* slow down sucking readers */
            }
        }
    }
#endif

}


/*
**  Find an article number in the article array via a binary search;
**  return -1 if not found.  Cache last hit to make linear lookups
**  faster.
*/
STATIC int
ARTfind(i)
    register ARTNUM	i;
{
    register ARTNUM	*bottom;
    register ARTNUM	*middle;
    register ARTNUM	*top;

    if (ARTsize == 0)
	return -1;

    top = &ARTnumbers[ARTsize - 1];
    if (ARTcache && ++ARTcache <= top && *ARTcache <= i) {
	if (*ARTcache == i)
	    return ARTcache - ARTnumbers;
	bottom = ARTcache;
    }
    else {
	ARTcache = NULL;
	bottom = ARTnumbers;
    }

    for ( ; ; ) {
	if (i < *bottom || i > *top)
	    break;

	middle = bottom + (top - bottom) / 2;
	if (i == *middle) {
	    /* Found it; update cache. */
	    ARTcache = middle;
	    return middle - ARTnumbers;
	}

	if (i > *middle)
	    bottom = middle + 1;
	else
	    top = middle;
    }
    return -1;
}


/*
**  Ask the innd server for the article.  Only called from CMDfetch,
**  and only if history file is buffered.  Common case:  "oops, cancel
**  that article I just posted."
*/
STATIC QIOSTATE *
ARTfromboss(what, id)
    SENDDATA		*what;
    char		*id;
{
    FILE		*FromServer;
    FILE		*ToServer;
    QIOSTATE		*qp;
    char		buff[NNTP_STRLEN + 2];
    char		*name;
    char		*p;
    BOOL		more;

    /* If we can, open the connection. */
    if (NNTPlocalopen(&FromServer, &ToServer, (char *)NULL) < 0)
	return NULL;

    /* Send the query to the server. */
    qp = NULL;
    (void)fprintf(ToServer, "XPATH %s\r\n", id);
    (void)fflush(ToServer);
    if (ferror(ToServer))
	goto QuitClose;

    /* Get the reply; article exist? */
    if (fgets(buff, sizeof buff, FromServer) == NULL
     || atoi(buff) != NNTP_NOTHING_FOLLOWS_VAL)
	goto QuitClose;

    /* Yes.  Be quick if just doing a stat. */
    if (what == &SENDstat) {
	qp = QIOopen("/dev/null", 0);
	goto QuitClose;
    }

    /* Clean up response. */
    if ((p = strchr(buff, '\r')) != NULL)
	*p = '\0';
    if ((p = strchr(buff, '\n')) != NULL)
	*p = '\0';

    /* Loop over all filenames until we can open one. */
    for (name = buff; *name; name = p + 1) {
	/* Snip off next name, turn dots to slashes. */
	for (p = name; ISWHITE(*p); p++)
	    continue;
	for (name = p; *p && *p != ' '; p++)
	    if (*p == '.')
		*p = '/';
	more = *p == ' ';
	if (more)
	    *p = '\0';
	if ((qp = QIOopen(name, QIO_BUFFER)) != NULL || !more)
	    break;
    }

    /* Send quit, read server's reply, close up and return. */
  QuitClose:
    (void)fprintf(ToServer, "quit\r\n");
    (void)fclose(ToServer);
    (void)fgets(buff, sizeof buff, FromServer);
    (void)fclose(FromServer);
    return qp;
}


/*
**  Fetch part or all of an article and send it to the client.
*/
FUNCTYPE
CMDfetch(ac, av)
    int			ac;
    char		*av[];
{
    char		buff[SMBUF];
    char		idbuff[BIG_BUFFER];
    SENDDATA		*what;
    register QIOSTATE	*qp;
    register BOOL	ok;
    ARTNUM		art;

    /* Find what to send; get permissions. */
    ok = PERMcanread;
    switch (*av[0]) {
    default:
	what = &SENDbody;
	break;
    case 'a': case 'A':
	what = &SENDarticle;
	break;
    case 's': case 'S':
	what = &SENDstat;
	break;
    case 'h': case 'H':
	what = &SENDhead;
	/* Poster might do a "head" command to verify the article. */
	ok = PERMcanread || PERMcanpost;
	break;
    }

    if (!ok) {
	Reply("%s\r\n", NOACCESS);
	return;
    }

    /* Requesting by Message-ID? */
    if (ac == 2 && av[1][0] == '<') {
	if ((qp = ARTopenbyid(av[1], &art)) == NULL) {
	    Reply("%d No such article\r\n", NNTP_DONTHAVEIT_VAL);
	    return;
	}
	if (!PERMartok(qp)) {
	    QIOclose(qp);
	    Reply("%s\r\n", NOACCESS);
	    return;
	}
	Reply("%d %ld %s %s\r\n", what->ReplyCode, art, av[1], what->Item);
	if (what->Type != STstat)
	    ARTsend(qp, what->Type);
	QIOclose(qp);
	return;
    }

    /* Trying to read. */
    if (GRPcount == 0) {
	Reply("%s\r\n", ARTnotingroup);
	return;
    }

    /* Default is to get current article, or specified article. */
    if (ac == 1) {
	if (ARTindex < 0 || ARTindex >= ARTsize) {
	    Reply("%s\r\n", ARTnocurrart);
	    return;
	}
	(void)sprintf(buff, "%ld", ARTnumbers[ARTindex]);
    }
    else {
	if (strspn(av[1], "0123456789") != strlen(av[1])) {
	    Reply("%s\r\n", ARTnoartingroup);
	    return;
	}
	(void)strcpy(buff, av[1]);
    }

    /* Move forward until we can find one. */
    while (!ARTopen(buff, idbuff)) {
	if (ac > 1 || ++ARTindex >= ARTsize) {
	    Reply("%s\r\n", ARTnoartingroup);
	    return;
	}
	(void)sprintf(buff, "%ld", ARTnumbers[ARTindex]);
    }

    /* fjc -- 1.5.1corr -- 19970806
	If we cause Reply() more than 2048 bytes, it will overrun.
       idbuff is a message ID which came out of the history file.
       I really doubt it can be more than 255 bytes without some
       major history file corruption, but we'll force truncate it
       at 512 bytes.
       Note: Decent compilers will optimize out the following ?:
    */
    idbuff[(BIG_BUFFER) > 512 ? 512 : (BIG_BUFFER-1)] = '\0';
    Reply("%d %s %s %s\r\n", what->ReplyCode, buff, idbuff, what->Item);
    if (what->Type != STstat)
	ARTsend(ARTqp, what->Type);
    if (ac > 1)
	ARTindex = ARTfind((ARTNUM)atol(buff));
}


/*
**  Go to the next or last (really previous) article in the group.
*/
FUNCTYPE
CMDnextlast(ac, av)
    int		ac;
    char	*av[];
{
    char	buff[SPOOLNAMEBUFF];
    char	idbuff[SMBUF];
    int		save;
    BOOL	next;
    int		delta;
    int		errcode;
    STRING	message;

    if (!PERMcanread) {
	Reply("%s\r\n", NOACCESS);
	return;
    }
    if (GRPcount == 0) {
	Reply("%s\r\n", ARTnotingroup);
	return;
    }
    if (ARTindex < 0 || ARTindex >= ARTsize) {
	Reply("%s\r\n", ARTnocurrart);
	return;
    }

    next = (av[0][0] == 'n' || av[0][0] == 'N');
    if (next) {
	delta = 1;
	errcode = NNTP_NONEXT_VAL;
	message = "next";
    }
    else {
	delta = -1;
	errcode = NNTP_NOPREV_VAL;
	message = "previous";
    }

    save = ARTindex;
    ARTindex += delta;
    if (ARTindex < 0 || ARTindex >= ARTsize) {
	Reply("%d No %s to retrieve.\r\n", errcode, message);
	ARTindex = save;
	return;
    }

    (void)sprintf(buff, "%ld", ARTnumbers[ARTindex]);
    while (!ARTopen(buff, idbuff)) {
	ARTindex += delta;
	if (ARTindex < 0 || ARTindex >= ARTsize) {
	    Reply("%d No %s article to retrieve.\r\n", errcode, message);
	    ARTindex = save;
	    return;
	}
	(void)sprintf(buff, "%ld", ARTnumbers[ARTindex]);
    }

    Reply("%d %s %s Article retrieved; request text separately.\r\n",
	   NNTP_NOTHING_FOLLOWS_VAL, buff, idbuff);

    if (ac > 1)
	ARTindex = ARTfind((ARTNUM)atol(buff));
}


/*
**  Return the header from the specified file, or NULL if not found.
**  We can estimate the Lines header, if that's what's wanted.
*/
STATIC char *
GetHeader(qp, header, IsLines)
    register QIOSTATE	*qp;
    register char	*header;
    BOOL		IsLines;
{
    static char		buff[40];
    register char	*p;
    register char	*q;
    struct stat		Sb;

    for ( ; ; ) {
	if ((p = QIOread(qp)) == NULL) {
	    if (QIOtoolong(qp))
		continue;
	    break;
	}
	if (*p == '\0')
	    /* End of headers. */
	    break;
	if (ISWHITE(*p) || (q = strchr(p, ':')) == NULL)
	    /* Continuation or bogus (shouldn't happen) line; ignore. */
	    continue;
	*q = '\0';
	if (caseEQ(header, p))
	    return *++q ? q + 1 : NULL;
    }

    if (IsLines && fstat(QIOfileno(qp), &Sb) >= 0) {
	/* Lines estimation taken from Tor Lillqvist <tml@tik.vtt.fi>'s
	 * posting <TML.92Jul10031233@hemuli.tik.vtt.fi> in
	 * news.sysadmin. */
	(void)sprintf(buff, "%d",
	    (int)(6.4e-8 * Sb.st_size * Sb.st_size + 0.023 * Sb.st_size - 12));
	return buff;
    }
    return NULL;
}


STATIC BOOL
CMDgetrange(ac, av, rp)
    int			ac;
    char		*av[];
    register ARTRANGE	*rp;
{
    register char	*p;

    if (GRPcount == 0) {
	Reply("%s\r\n", ARTnotingroup);
	return FALSE;
    }

    if (ac == 1) {
	/* No argument, do only current article. */
	if (ARTindex < 0 || ARTindex >= ARTsize) {
	    Reply("%s\r\n", ARTnocurrart);
	    return FALSE;
	}
	rp->High = rp->Low = ARTnumbers[ARTindex];
	return TRUE;
    }

    /* Got just a single number? */
    if ((p = strchr(av[1], '-')) == NULL) {
	rp->Low = rp->High = atol(av[1]);
	return TRUE;
    }

    /* Parse range. */
    *p++ = '\0';
    rp->Low = atol(av[1]);
    if (ARTsize) {
	if (*p == '\0' || (rp->High = atol(p)) < rp->Low)
	    /* "XHDR 234-0 header" gives everything to the end. */
	    rp->High = ARTnumbers[ARTsize - 1];
	else if (rp->High > ARTnumbers[ARTsize - 1])
	    rp->High = ARTnumbers[ARTsize - 1];
	if (rp->Low < ARTnumbers[0])
	    rp->Low = ARTnumbers[0];
    }
    else
	/* No articles; make sure loops don't run. */
	rp->High = rp->Low ? rp->Low - 1 : 0;
    return TRUE;
}


/*
**  Return a field from the overview line or NULL on error.  Return a copy
**  since we might be re-using the line later.
*/
STATIC char *
OVERGetHeader(p, field)
    register char	*p;
    int			field;
{
    static char		*buff;
    static int		buffsize;
    register int	i;
    ARTOVERFIELD	*fp;
    char		*next;
#ifdef OVERSCREAM
    char*		eol = strchr(p, '\n');
#endif


    fp = &ARTfields[field - 1];

    if (fp->NeedsHeader)            /* we're going to need an exact match */
      field = ARTfirstfullfield;
 
    /* Skip leading headers. */
    for (; --field >= 0 && *p && *p != '\n'; p++)
#ifdef OVERSCREAM
	if ((p = memchr(p, '\t', OVERsize - (p - OVERshm))) == NULL ||
	    p > eol )
#else
	if ((p = strchr(p, '\t')) == NULL)
#endif
	    return NULL;
    if (*p == '\0')
	return NULL;

    if (fp->HasHeader)
	p += fp->Length + 2;

    if (fp->NeedsHeader) {		/* find an exact match */
	 while (strncmp(fp->Header, p, fp->Length) != 0) {
	      if ((p = strchr(p, '\t')) == NULL) 
		return NULL;
	      p++;
	 }
	 p += fp->Length + 2;
    }
  
    /* Figure out length; get space. */

#ifdef OVERSCREAM
    if ((next = memchr(p, '\t', OVERsize - (p - OVERshm))) != NULL &&
	  p < eol )
	i = next - p;
    else 
	i = eol - p;

#else /* !OVERSCREAM */

    if ((next = strchr(p, '\t')) != NULL)
	i = next - p;
    else
	i = strlen(p);
#endif

    if (buffsize == 0) {
	buffsize = i;
	buff = NEW(char, buffsize + 1);
    }
    else if (buffsize < i) {
	buffsize = i;
	RENEW(buff, char, buffsize + 1);
    }

    (void)strncpy(buff, p, i);
    buff[i] = '\0';
    return buff;
}

#ifdef OVERSCREAM

/*
 * helper function, search backwards in memory
 */

STATIC char*
memrchr(p, c, l)
    register char* p;
    register char c;
    register int l;
{
    for (; l--; --p)
	if ( *p == c )
	    return(p);
    return(NULL);
}

/*
 * mmap an OVERVIEW file.
 */

STATIC BOOL
OVERopen()
{
    char		name[SPOOLNAMEBUFF];
    struct stat 	sb;
    char*		p;
    static int		pagesize = 0;

    /* return true if already mapped */
    if ( OVERshm ) {
	return TRUE;
    }
    /* return false if already failed */
    if ( OVERopens++ ) {
	return FALSE;
    }
    /* get memory pagesize if we don't have it already */
    if ( ! pagesize && !
#ifdef _SC_PAGE_SIZE
	(pagesize = sysconf(_SC_PAGE_SIZE))
#else
# ifdef _SC_PAGESIZE
	(pagesize = sysconf(_SC_PAGESIZE))
# else
	(pagesize = getpagesize())
# endif
#endif
	) {
	syslog(L_NOTICE, "%s: Can't getpagesize", ClientHost);
	return FALSE;
    }
    /* mmap the file */
    (void)sprintf(name, "%s/%s/%s", _PATH_OVERVIEWDIR, GRPlast, _PATH_OVERVIEW);
    if ( (OVERfd = open(name, O_RDONLY)) < 0 ) {
	/* no overview file */
	syslog(L_NOTICE, "%s can't open %s: %m", ClientHost, name);
	return FALSE;
    }
    if ( fstat(OVERfd, &sb) == -1 ) {
	syslog(L_NOTICE, "%s can't stat %s: %m", ClientHost, name);
	(void)close(OVERfd);
	return FALSE;
    }
    if ( (OVERsize = sb.st_size) <= 1 ) {
	syslog(L_NOTICE, "%s: %s is too small", ClientHost, name);
	(void)close(OVERfd);
	return FALSE;
    }
    OVERmsize = (OVERsize + pagesize - 1) & ~(pagesize - 1);
    if ( (OVERshm = mmap(NULL, OVERmsize, PROT_READ, MAP_SHARED, OVERfd, 0))
	 == (caddr_t) -1 )
    {
	syslog(L_NOTICE, "%s can't mmap %s: %m", ClientHost, name);
	(void)close(OVERfd);
	OVERshm = NULL;
	return FALSE;
    }
    /* get first entry */
    if ( (OVERfirst = atol((char*) OVERshm)) == 0 ) {
	syslog(L_NOTICE, "%s: %s: bad format", ClientHost, name);
	(void)munmap(OVERshm, OVERmsize);
	(void)close(OVERfd);
	OVERshm = NULL;
	return FALSE;
    }

    /* get last entry */
    if ( *(OVERshm + OVERsize - 1) != '\n' ) {
	/*
	 * If you get here, then your mmap() implementation sucks.
	 * Go complain with your OS vendor, that their mmap() can't
	 * do mmap()ing of growing files properly.
	 * We try to find a decent record near the end, for the poor
	 * sobs without proper mmap. There are a lot of other places
	 * in the code with hacks for bad mmap(). Mainly because I'm
	 * one of the poor sobs :(
	 */
	YOUR_MMAP_SUCKS;
    }
    do {
	/*
         * Try to find any newline. If there isn't any, the entire file
	 * is crap. Normally this finds the newline right at the end.
	 */
	p = memrchr(OVERshm + OVERsize - 1, '\n', OVERsize - 1);
	if ( p == NULL ) {
	    /* overview file only contains garbage. */
	    (void)munmap(OVERshm, OVERmsize);
	    (void)close(OVERfd);
	    OVERshm = NULL;
	    return FALSE;
	}
	OVERsize = p - OVERshm + 1;
	if ( (p = memrchr((char*) OVERshm + OVERsize - 2, '\n',
			    OVERsize - 2)) == NULL )
	{
	    /* Apparently only 1 (usable) line */
	    OVERlast = OVERfirst;
	    OVERcache = NULL;
	    return TRUE;
	}
	OVERlast = atol(p+1);
    }
    while ( OVERlast == 0 && --OVERsize );

    if ( !OVERsize ) {
	(void)munmap(OVERshm, OVERmsize);
	(void)close(OVERfd);
	OVERshm = NULL;
	return FALSE;
    }

    OVERcache = NULL;
    return TRUE;
}

/*
 * Close an overview file, if any.
 */

void
OVERclose()
{
    if ( OVERshm ) {
	if ( munmap(OVERshm, OVERmsize) == -1 ) {
	    syslog(L_NOTICE, "%s can't munmap: %m", ClientHost);
	}
	(void)close(OVERfd);
	OVERshm = NULL;
    }
    OVERopens = 0;
}

/*
 * find an overview article using binary search in the overview file.
 * Returns a pointer to the actual line in the overview file (so it's
 * !!NOT!! null terminated, and can't be written to!!), or NULL on failure.
 */

STATIC char*
OVERfind(artnum)
    ARTNUM	artnum;
{
    char*	bottom;
    char*	top;
    ARTNUM	bottomnr;
    ARTNUM	topnr;
    char*	pos;
    ARTNUM	nr;
    int		i;

    /* default startpos */
    bottom = OVERshm;
    bottomnr = OVERfirst;
    top = OVERshm + OVERsize - 1;
    topnr = OVERlast;

    if ( OVERcache ) {
	/*
	 * for speedy sequential access. OVERcache, if non-NULL, points to
	 * the "next" entry. OVERprev is the previous article number found.
	 * Also check for sucking mmap() implementations.
	 */
	if ( *OVERcache == '\0' ) {
	    YOUR_MMAP_SUCKS;
	    OVERcache = memchr(OVERcache, '\n',
				OVERsize - (OVERshm - OVERcache));
	    if ( OVERcache == NULL || OVERcache == OVERshm + OVERsize - 1 ) {
		OVERcache = NULL;
		return NULL;
	    }
	    OVERcache++;
	}
	nr = atol(OVERcache);
	if ( nr < OVERfirst || nr > OVERlast ) {
	    /* boo */
	    OVERcache = NULL;
	    return NULL;
	}
	if ( nr == artnum ) {
	    pos = OVERcache;
	    goto bingo; /* calculate next OVERcache + return. (EW! a goto! :) */
	}
	else if ( artnum > nr ) {
	    /* treat cache as first binary search */
	    bottom = OVERcache;
	    bottomnr = nr;
	}
	else {
	    /* cache is first top */
	    top = OVERcache - 1;
	    topnr = nr - 1;
	    if ( artnum > OVERprev ) {
		/*
		 * optimization: we're searching for something that isn't
		 * in the database, but we want to keep the cache clean.
		 * this occurs when we think an article is there, but it
		 * really isn't, eg. because NOSCANDIR is on, or simply
		 * because the overview database leaks.
		 */
		return(NULL);
	    }
	}
    }

    /* sanity check */
    if ( artnum < bottomnr || artnum > topnr ) {
	OVERcache = NULL;
	return NULL;
    }

    for (;;) {
	/*
	 * This is the binary search loop, there are about a zillion
	 * exits so I found it neater to code it in an endless loop :)
	 * It simply continues until it is either found or it isn't...
	 *
	 * Note that we don't do a real binary search, but we guess
	 * a position using the fact that the overview database usually
	 * contains a reasonably linear range of articles, without any
	 * big leaps, but we skew it a bit towards the middle to prevent
	 * slow convergence in boundary cases (see also below).
	 *
	 * We switch to linear searching when we're "close",
	 * because on short ranges, linear searches are about as fast
	 * (or faster) anyway. LINSEARCH is currently guessed at 5,
	 * because on average it takes 2.5 searches using a linear search,
	 * where it usually takes 3 "straight" binary searches.
	 *
	 * Unfortunately, we can't be sure we get into linear search when
	 * we're close, because the database may have large holes.
	 */
	/* test if it's near the bottom */
	if ( artnum < bottomnr + LINSEARCH ) {
	    i = 0;
	    while ( artnum > bottomnr && i++ < LINSEARCH ) {
		/* search next line */
		bottom = memchr(bottom, '\n', OVERsize - (bottom - OVERshm));
		if ( bottom == NULL || bottom == top + 1 ) {
		    /* reached end of file */
		    OVERcache = NULL;
		    return NULL;
		}
		if ( *++bottom == 0 ) {
		    YOUR_MMAP_SUCKS;
		    continue;
		}
		bottomnr = atol(bottom);
		if ( bottomnr < OVERfirst || bottomnr > OVERlast ) {
		    OVERcache = NULL;
		    return NULL;
		}
	    }
	    if ( artnum == bottomnr ) {
		pos = bottom;
		goto bingo; /* calculate next OVERcache + return. */
	    }
	    else {
		/* didn't find it, but we came close. still cache position */
		OVERcache = bottom;
		OVERprev = artnum;
		return NULL;
	    }
	    /*NOTREACHED*/
	}
	/* test if it's near the top */
	if ( artnum > topnr - LINSEARCH ) {
	    /*
	     * topnr is frequently guessed, so we must first determine it
	     * correctly. The fun part about searching backwards is that
	     * the next position (OVERcache) follows easily...
	     */
	    i = 0;
	    do {
		OVERcache = (top == OVERshm + OVERsize - 1) ? NULL : top + 1;
		if ( (top = memrchr(--top, '\n', top - OVERshm))
		    == NULL || top + 1 == bottom )
		{
		    /* search hit bottom */
		    OVERcache = NULL;
		    return NULL;
		}
		if ( *(top + 1) == 0 ) {
		    YOUR_MMAP_SUCKS;
		    /* make sure we continue */
		    topnr = artnum + 1;
		    continue;
		}
		topnr = atol(top + 1);
		if ( topnr < OVERfirst || topnr > OVERlast ) {
		    OVERcache = NULL;
		    return NULL;
		}
	    }
	    while ( artnum < topnr && i++ < LINSEARCH );
	    if ( artnum == topnr ) {
		/* bingo. This time we know OVERcache already */
		OVERprev = artnum;
		return(top + 1);
	    }
	    else {
		/* not found, but close. cache position */
		OVERprev = artnum;
		return NULL;
	    }
	    /*NOTREACHED*/
	}

	/*
	 * now for the real binary search:
	 * Estimate the position of artnum, but with a small offset towards
	 * the middle, for better convergence in case the set of articles
	 * is non-linear (you get a straight binary search if MIDSKEW is 1.0).
	 * MIDSKEW is currently determined using a big thumb, occultism,
	 * astrology, cat /dev/uri-geller and some common sense (but not much)
	 * MIDSKEW == 0.0 makes the search take only 1 iteration in case
	 * the overview database is a monotonous array of lines with equal
	 * length, but can make for really lousy searches in anything not like
	 * the above, which, in the real world, is practically always.
	 * MIDSKEW == 1.0 gives you a true binary search without any guessing
	 * whatsoever.
	 * I thought 10% would be good enough. Only riggid testing can
	 * determine the optimal value, and then it still depends on a lot
	 * of settings, like expire times, user newsgroups preference,
	 * presence of cancelbots or cancelwars, frequency of expireover
	 * runs... need I say more? :)
	 */
	if ( topnr <= bottomnr ) {
	    /* Safety net. This REALLY should never happen. */
	    syslog(L_NOTICE,
		   "%s: ASSERTION FAILED: %d < %d looking for %d in %s",
		   ClientHost, topnr, bottomnr, artnum, GRPlast);
	}
	pos = bottom + (int) ((double) (top - bottom) * (MIDSKEW * 0.5) +
			      (top - bottom) * (1.0 - MIDSKEW) *
			      (artnum - bottomnr) / (topnr - bottomnr));
	/* search forward for newline */
	if ( (pos = memchr(pos, '\n', OVERsize - (pos - OVERshm))) == NULL ) {
	    /* this shouldn't happen */
	    OVERcache = NULL;
	    return NULL;
	}
	if ( pos == top ) {
	    /* hmm... */
	    if ( (pos = memrchr(--pos, '\n', pos - OVERshm))
		 == NULL || pos == bottom - 1 )
	    {
		/*
		 * This is what happens when there's a large hole and we're
		 * looking for something inside the hole (which isn't there).
		 * still record the position in this case...
		 */
		OVERcache = (top == OVERshm + OVERsize - 1) ? NULL : top + 1;
		OVERprev = artnum;
		return NULL;
	    }
	}
	/* see where we are */
	if ( *++pos == 0 ) {
	    YOUR_MMAP_SUCKS;
	    pos = memchr(pos, '\n', OVERsize - (pos - OVERshm));
	    if ( pos == NULL || pos == OVERshm + OVERsize - 1 || pos == top ) {
		OVERcache = NULL;
		return NULL;
	    }
	    pos++;
	}
	nr = atol(pos);
	if ( nr < OVERfirst || nr > OVERlast ) {
	    OVERcache = NULL;
	    return NULL;
	}
	if ( nr == artnum ) {
	    /* bingo. Set cache to next entry */
bingo:
	    OVERcache = memchr(pos, '\n', OVERsize - (pos - OVERshm));
	    if ( OVERcache == OVERshm + OVERsize - 1 )
		OVERcache = NULL;
	    else if ( OVERcache )
		OVERcache++;
	    OVERprev = artnum;
	    return (pos);
	}
	if ( nr < artnum ) {
	    /* found a new bottom */
	    bottom = pos;
	    bottomnr = nr;
	}
	else /* nr > artnum */ {
	    /*
	     * found a new top. Setting topnr to nr-1 is not entirely
	     * correct, but who cares. (In fact we do care, but adjust
	     * later :)
	     */
	    top = pos - 1;
	    topnr = nr - 1;
	}
    }
    /*NOTREACHED*/
}

#else /* !OVERSCREAM */

/*
**  Open an OVERVIEW file.
*/
STATIC BOOL
OVERopen()
{
    char	name[SPOOLNAMEBUFF];

    /* Already open? */
    if (OVERqp != NULL)
	/* Don't rewind -- we are probably going forward via repeated
	 * NNTP commands. */
	return TRUE;

    /* Failed here before? */
    if (OVERopens++)
	return FALSE;

    OVERline = NULL;
    OVERarticle = 0;
    (void)sprintf(name, "%s/%s/%s", _PATH_OVERVIEWDIR, GRPlast, _PATH_OVERVIEW);
    OVERqp = QIOopen(name, QIO_BUFFER);
    return OVERqp != NULL;
}


/*
**  Close the OVERVIEW file.
*/
void
OVERclose()
{
    if (OVERqp != NULL) {
	QIOclose(OVERqp);
	OVERqp = NULL;
	OVERopens = 0; /* this is a bug */
    }
}


/*
**  Return the overview data for an article or NULL on failure.
**  Assumes that what we return is never modified.
*/
STATIC char *
OVERfind(artnum)
    ARTNUM	artnum;
{
    if (OVERqp == NULL)
	return NULL;

    if (OVERarticle > artnum) {
	(void)QIOrewind(OVERqp);
	OVERarticle = 0;
	OVERline = NULL;
    }

    for ( ; OVERarticle < artnum; OVERarticle = atol(OVERline))
	while ((OVERline = QIOread(OVERqp)) == NULL) {
	    if (QIOtoolong(OVERqp))
		continue;
	    /* Don't close file; we may rewind. */
	    return NULL;
	}

    return OVERarticle == artnum ? OVERline : NULL;
}

#endif


/*
**  Read an article and create an overview line without the trailing
**  newline.  Returns pointer to static space or NULL on error.
*/
STATIC char *
OVERgen(name)
    char			*name;
{
    static ARTOVERFIELD		*Headers;
    static char			*buff;
    static int			buffsize;
    register ARTOVERFIELD	*fp;
    register ARTOVERFIELD	*hp;
    register QIOSTATE		*qp;
    register char		*colon;
    register char		*line;
    register char		*p;
    register int		i;
    register int		size;
    register int		ov_size;
    register long		lines;
    struct stat			Sb;
    long			t;
    char			value[10];

    /* Open article. */
    if ((qp = QIOopen(name, QIO_BUFFER)) == NULL)
	return NULL;
    if ((p = strrchr(name, '/')) != NULL)
	name = p + 1;

    /* Set up place to store headers. */
    if (Headers == NULL) {
	Headers = NEW(ARTOVERFIELD, ARTfieldsize);
	for (fp = ARTfields, hp = Headers, i = ARTfieldsize; --i >= 0; hp++, fp++) {
	    hp->Length = 0;
	    hp->NeedsHeader = fp->NeedsHeader;
       }
    }
    for (hp = Headers, i = ARTfieldsize; --i >= 0; hp++)
	hp->HasHeader = FALSE;

    for ( ; ; ) {
	/* Read next line. */
	if ((line = QIOread(qp)) == NULL) {
	    if (QIOtoolong(qp))
		continue;
	    /* Error or EOF (in headers!?); shouldn't happen. */
	    QIOclose(qp);
	    return NULL;
	}

	/* End of headers? */
	if (*line == '\0')
	    break;

	/* Is it a continuation line? */
	if (ISWHITE(*line) && (hp - Headers) < ARTfieldsize) {
	    /* Skip whitespace but one. */
	    for (p = line; *p && ISWHITE(*p); p++)
		continue;
	    --p;
	    /* Now append it. */
	    hp->Length += strlen(p);
	    RENEW(hp->Header, char, hp->Length + 1);
	    (void)strcat(hp->Header, p);
	    for (p = hp->Header; *p; p++)
		if (*p == '\t' || *p == '\n')
		    *p = ' ';
	    continue;
	}

	/* See if we want this header. */
	fp = ARTfields;
	for (hp = Headers, i = ARTfieldsize; --i >= 0; hp++, fp++) {
	    colon = &line[fp->Length];
	    if (*colon != ':')
		continue;
	    *colon = '\0';
	    if (!caseEQ(line, fp->Header)) {
		*colon = ':';
		continue;
	    }
	    *colon = ':';
	    if (fp->NeedsHeader)
		p = line;
	    else
		/* Skip colon and whitespace, store value. */
		for (p = colon; *++p && ISWHITE(*p); )
		    continue;
	    size = strlen(p);
	    if (hp->Length == 0) {
		hp->Length = size;
		hp->Header = NEW(char, hp->Length + 1);
	    }
	    else if (hp->Length < size) {
		hp->Length = size;
		RENEW(hp->Header, char, hp->Length + 1);
	    }
	    (void)strcpy(hp->Header, p);
	    for (p = hp->Header; *p; p++)
		if (*p == '\t' || *p == '\n')
		    *p = ' ';
	    hp->HasHeader = TRUE;
	    break;
	}
    }

    /* Read body of article, just to get lines. */
    for (lines = 0; ; lines++)
	if ((p = QIOread(qp)) == NULL) {
	    if (QIOtoolong(qp))
		continue;
	    if (QIOerror(qp)) {
		QIOclose(qp);
		return NULL;
	    }
	    break;
	}

    /* Calculate total size, fix hardwired headers. */
    ov_size = strlen(name) + ARTfieldsize + 2;
    for (hp = Headers, fp = ARTfields, i = ARTfieldsize; --i >= 0; hp++, fp++) {
	if (caseEQ(fp->Header, "Bytes") || caseEQ(fp->Header, "Lines")) {
	    if (fp->Header[0] == 'B' || fp->Header[0] == 'b')
		t = fstat(QIOfileno(qp), &Sb) >= 0 ? (long)Sb.st_size : 0L;
	    else
		t = lines;

	    (void)sprintf(value, "%ld", t);
	    size = strlen(value);
	    if (hp->Length == 0) {
		 hp->Length = size;
		hp->Header = NEW(char, hp->Length + 1);
	    }
	    else if (hp->Length < size) {
		hp->Length = size;
		RENEW(hp->Header, char, hp->Length + 1);
	    }
	    (void)strcpy(hp->Header, value);
	    hp->HasHeader = TRUE;
	}
	if (hp->HasHeader)
	    ov_size += strlen(hp->Header);
    }

    /* Get space. */
    if (buffsize == 0) {
	buffsize = ov_size;
	buff = NEW(char, buffsize + 1);
    }
    else if (buffsize < ov_size) {
	buffsize = ov_size;
	RENEW(buff, char, buffsize + 1);
    }

    /* Glue all the fields together. */
    p = buff + strlen(strcpy(buff, name));
    for (hp = Headers, i = ARTfieldsize; --i >= 0; hp++) {
	 if (hp->NeedsHeader && !hp->HasHeader)
	   continue;
	*p++ = '\t';
	if (hp->HasHeader)
	    p += strlen(strcpy(p, hp->Header));
    }
    *p = '\0';

    QIOclose(qp);
    return buff;
}


/*
**  XHDR, a common extension.  Retrieve specified header from a
**  Message-ID or article range.
*/
FUNCTYPE
CMDxhdr(ac, av)
    int			ac;
    char		*av[];
{
    register QIOSTATE	*qp;
    register ARTNUM	i;
    register char	*p;
    int			Overview;
    BOOL		IsLines;
    ARTRANGE		range;
    char		buff[SPOOLNAMEBUFF];
    ARTNUM		art;

    if (!PERMcanread) {
	Reply("%s\r\n", NOACCESS);
	return;
    }
    IsLines = caseEQ(av[1], "lines");

    /* Message-ID specified? */
    if (ac == 3 && av[2][0] == '<') {
	if ((qp = ARTopenbyid(av[2], &art)) == NULL) {
	    Reply("%d No such article\r\n", NNTP_DONTHAVEIT_VAL);
	    return;
	}
	Reply("%d %ld %s header of article %s.\r\n",
	   NNTP_HEAD_FOLLOWS_VAL, art, av[1], av[2]);
	p = GetHeader(qp, av[1], IsLines);
	Printf("%s %s\r\n", av[2], p ? p : "(none)");
	QIOclose(qp);
	Printf(".\r\n");
	return;
    }

    /* Range specified. */
    if (!CMDgetrange(ac - 1, av + 1, &range))
	return;

    /* Is this a header in our overview? */
    for (Overview = 0, i = 0; i < ARTfieldsize; i++)
	if (caseEQ(ARTfields[i].Header, av[1])) {
	    if (OVERopen())
		Overview = i + 1;
	    break;
	}

    Reply("%d %s fields follow\r\n", NNTP_HEAD_FOLLOWS_VAL, av[1]);
    for (i = range.Low; i <= range.High; i++) {
	alarm(0);			/* stop timer */
	if (ARTfind(i) < 0)
	    continue;

	/*
	 * It should never take this long to send a single line
	 * of overview information.
	 */
	alarm(DEFAULT_TIMEOUT);		/* start timer */

	/* Get it from the overview? */
	if (Overview && (p = OVERfind(i)) != NULL) {
	    p = OVERGetHeader(p, Overview);
	    Printf("%ld %s\r\n", i, p && *p ? p : "(none)");
	    continue;
	}

	(void)sprintf(buff, "%ld", i);
	if ((qp = QIOopen(buff, QIO_BUFFER)) == NULL)
	    continue;
	p = GetHeader(qp, av[1], IsLines);
	Printf("%ld %s\r\n", i, p ? p : "(none)");
	QIOclose(qp);
    }
    Printf(".\r\n");
    alarm (0) ;
}


/*
**  XOVER another extension.  Dump parts of the overview database.
*/
FUNCTYPE
CMDxover(ac, av)
    int			ac;
    char		*av[];
{
    register char	*p;
    register ARTNUM	i;
    register BOOL	Opened;
    ARTRANGE		range;
    char		buff[SPOOLNAMEBUFF];

    if (!PERMcanread) {
	Printf("%s\r\n", NOACCESS);
	return;
    }

    /* Trying to read. */
    if (GRPcount == 0) {
	Reply("%s\r\n", ARTnotingroup);
	return;
    }

    /* Parse range. */
    if (!CMDgetrange(ac, av, &range))
	return;

    Reply("%d data follows\r\n", NNTP_OVERVIEW_FOLLOWS_VAL);
    for (Opened = OVERopen(), i = range.Low; i <= range.High; i++) {
	if (ARTfind(i) < 0)
	    continue;

	/*OVERVIEWcount++;*/
	if (Opened && (p = OVERfind(i)) != NULL) {
#ifdef OVERSCREAM
	    char* eol = memchr(p, '\n', OVERsize - (p - OVERshm));
	    if ( eol == NULL )
		continue; /* this should NEVER NEVER EVER NEVER EVER happen */
	    fwrite(p, 1, eol - p, stdout);
	    fwrite("\r\n", 1, 2, stdout);
#else
	    Printf("%s\r\n", p);
#endif
	    continue;
	}

	(void)sprintf(buff, "%ld", i);
	/*OVERGENcount++;*/
	if ((p = OVERgen(buff)) != NULL)
	    Printf("%s\r\n", p);
    }
    Printf(".\r\n");
}


/*
**  XPAT, an uncommon extension.  Print only headers that match the pattern.
*/
/* ARGSUSED */
FUNCTYPE
CMDxpat(ac, av)
    int			ac;
    char		*av[];
{
    register char	*p;
    register QIOSTATE	*qp;
    register ARTNUM	i;
    ARTRANGE		range;
    char		*header;
    char		*pattern;
    char		*text;
    int			Overview;
    char		buff[SPOOLNAMEBUFF];
    ARTNUM		art;

    if (!PERMcanread) {
	Printf("%s\r\n", NOACCESS);
	return;
    }

    header = av[1];

    /* Message-ID specified? */
    if (av[2][0] == '<') {
	p = av[2];
	qp = ARTopenbyid(p, &art);
	if (qp == NULL) {
	    Printf("%d No such article.\r\n", NNTP_DONTHAVEIT_VAL);
	    return;
	}

	Printf("%d %s matches follow.\r\n", NNTP_HEAD_FOLLOWS_VAL, header);
	pattern = Glom(&av[3]);
	if ((text = GetHeader(qp, header, FALSE)) != NULL
	 && wildmat(text, pattern))
	    Printf("%s %s\r\n", p, text);

	QIOclose(qp);
	Printf(".\r\n");
	DISPOSE(pattern);
	return;
    }

    /* Range specified. */
    if (!CMDgetrange(ac - 1, av + 1, &range))
	return;

    /* In overview? */
    for (Overview = 0, i = 0; i < ARTfieldsize; i++)
	if (caseEQ(ARTfields[i].Header, av[1])) {
	    if (OVERopen())
		Overview = i + 1;
	    break;
	}

    Printf("%d %s matches follow.\r\n", NNTP_HEAD_FOLLOWS_VAL, header);
    for (pattern = Glom(&av[3]), i = range.Low; i <= range.High; i++) {
	if (ARTfind(i) < 0)
	    continue;

	/* Get it from the Overview? */
	if (Overview
	 && (p = OVERfind(i)) != NULL
	 && (p = OVERGetHeader(p, Overview)) != NULL) {
	    if (wildmat(p, pattern))
		Printf("%ld %s\r\n", i, p);
	    continue;
	}

	(void)sprintf(buff, "%ld", i);
	if ((qp = QIOopen(buff, QIO_BUFFER)) == NULL)
	    continue;
	if ((p = GetHeader(qp, av[1], FALSE)) == NULL)
	    p = "(none)";
	if (wildmat(p, pattern))
	    Printf("%ld %s\r\n", i, p);
	QIOclose(qp);
    }

    Printf(".\r\n");
    DISPOSE(pattern);
}
