Here's the performance improvement by caching the 1st OPTIONS return  
from a URI:

[jbj@wellfleet rpmio]$ ./tfts --options http://rpm5.org/files/popt
===== (1/32) dirs/files in:
         http://rpm5.org/files/popt/
    fts:        1      0.000001 MB     18.327713 secs
[jbj@wellfleet rpmio]$ ./tfts --nooptions http://rpm5.org/files/popt
===== (1/32) dirs/files in:
         http://rpm5.org/files/popt/
    fts:        1      0.000001 MB     11.301737 secs

Lots of piggy to be sliced up still. PROPFIND caching is next.

73 de Jeff

On Feb 8, 2008, at 10:08 AM, Jeff Johnson wrote:

>   RPM Package Manager, CVS Repository
>   http://rpm5.org/cvs/
>    
> ______________________________________________________________________ 
> ______
>
>   Server: rpm5.org                         Name:   Jeff Johnson
>   Root:   /v/rpm/cvs                       Email:  jbj@rpm5.org
>   Module: rpm                              Date:   08-Feb-2008  
> 16:08:15
>   Branch: HEAD                             Handle: 2008020815081400
>
>   Modified files:
>     rpm                     CHANGES
>     rpm/rpmio               argv.c librpmio.vers rpmdav.c rpmsw.c  
> rpmsw.h
>                             rpmurl.h tfts.c
>
>   Log:
>     - tweak up tfts debugging spew, add a Fts(3) stopwatch, append  
> pesky /.
>     - rpmsw: add rpmswPrint to display stopwtch results.
>     - rpmdav: fix: check the pesky trailing '/' on collections  
> correctly.
>     - rpmdav: add _dav_noption OPTIONS disabler, cache 1st result  
> from URI.
>
>   Summary:
>     Revision    Changes     Path
>     1.2151      +4  -0      rpm/CHANGES
>     1.13        +1  -1      rpm/rpmio/argv.c
>     2.35        +2  -0      rpm/rpmio/librpmio.vers
>     2.57        +9  -1      rpm/rpmio/rpmdav.c
>     2.15        +12 -0      rpm/rpmio/rpmsw.c
>     2.9         +9  -0      rpm/rpmio/rpmsw.h
>     1.23        +2  -0      rpm/rpmio/rpmurl.h
>     2.15        +41 -7      rpm/rpmio/tfts.c
>    
> ______________________________________________________________________ 
> ______
>
>   patch -p0 <<'@@ .'
>   Index: rpm/CHANGES
>    
> ====================================================================== 
> ======
>   $ cvs diff -u -r1.2150 -r1.2151 CHANGES
>   --- rpm/CHANGES	8 Feb 2008 13:14:34 -0000	1.2150
>   +++ rpm/CHANGES	8 Feb 2008 15:08:14 -0000	1.2151
>   @@ -1,4 +1,8 @@
>    5.0.0 -> 5.1a1:
>   +    - jbj: tweak up tfts debugging spew, add a Fts(3) stopwatch,  
> append pesky /.
>   +    - jbj: rpmsw: add rpmswPrint to display stopwtch results.
>   +    - jbj: rpmdav: fix: check the pesky trailing '/' on  
> collections correctly.
>   +    - jbj: rpmdav: add _dav_noption OPTIONS disabler, cache 1st  
> result from URI.
>        - proyvind: trim multiple slashes from buildroot for check- 
> files script (from Nanar/Olivier Thauvin)
>        - proyvind: make rpmdb cpp compliant (from spturtle/ 
> Christiaan Welvaart)
>        - jbj: "+bing": glob'd directory roots need trailing '/' for  
> DAV URI's.
>   @@ .
>   patch -p0 <<'@@ .'
>   Index: rpm/rpmio/argv.c
>    
> ====================================================================== 
> ======
>   $ cvs diff -u -r1.12 -r1.13 argv.c
>   --- rpm/rpmio/argv.c	4 Feb 2008 18:33:30 -0000	1.12
>   +++ rpm/rpmio/argv.c	8 Feb 2008 15:08:14 -0000	1.13
>   @@ -22,7 +22,7 @@
>
>        if (argv)
>        for (av = argv; *av; av++)
>   -	fprintf(fp, "%s\n", *av);
>   +	fprintf(fp, "\t%s\n", *av);
>
>    }
>
>   @@ .
>   patch -p0 <<'@@ .'
>   Index: rpm/rpmio/librpmio.vers
>    
> ====================================================================== 
> ======
>   $ cvs diff -u -r2.34 -r2.35 librpmio.vers
>   --- rpm/rpmio/librpmio.vers	2 Feb 2008 01:46:05 -0000	2.34
>   +++ rpm/rpmio/librpmio.vers	8 Feb 2008 15:08:14 -0000	2.35
>   @@ -41,6 +41,7 @@
>        davClose;
>        davClosedir;
>        _dav_debug;
>   +    _dav_nooptions;
>        davLstat;
>        davmagicdir;
>        davMkdir;
>   @@ -310,6 +311,7 @@
>        rpmswExit;
>        rpmswInit;
>        rpmswNow;
>   +    rpmswPrint;
>        rpmswSub;
>        rpmUndefineMacro;
>        rpmxarFree;
>   @@ .
>   patch -p0 <<'@@ .'
>   Index: rpm/rpmio/rpmdav.c
>    
> ====================================================================== 
> ======
>   $ cvs diff -u -r2.56 -r2.57 rpmdav.c
>   --- rpm/rpmio/rpmdav.c	25 Jan 2008 16:39:26 -0000	2.56
>   +++ rpm/rpmio/rpmdav.c	8 Feb 2008 15:08:14 -0000	2.57
>   @@ -67,6 +67,10 @@
>    #include "ugid.h"
>    #include "debug.h"
>
>   +/* Should http OPTIONS be run only once? */
>   +/* XXX TODO: determine precisely when OPTIONS need to be run. */
>   +int _dav_nooptions = 0;
>   +
>    /*@access DIR @*/
>    /*@access FD_t @*/
>    /*@access urlinfo @*/
>   @@ -337,6 +341,9 @@
>        if (!(u->urltype == URL_IS_HTTP || u->urltype == URL_IS_HTTPS))
>    	return 0;
>
>   +    if (_dav_nooptions && u->allow & RPMURL_SERVER_OPTIONSDONE)
>   +	return 0;
>   +
>        /* HACK: where should server capabilities be read? */
>        (void) urlPath(u->url, &path);
>        /* HACK: perhaps capture Allow: tag, look for PUT permitted. */
>   @@ -344,6 +351,7 @@
>        rc = ne_options(u->sess, path, u->capabilities);
>        switch (rc) {
>        case NE_OK:
>   +	u->allow |= RPMURL_SERVER_OPTIONSDONE;
>        {	ne_server_capabilities *cap = u->capabilities;
>    	if (cap->dav_class1)
>    	    u->allow |= RPMURL_SERVER_HASDAVCLASS1;
>   @@ -1821,7 +1829,7 @@
>
>        /* HACK: glob does not pass dirs with trailing '/' */
>        nb = strlen(path)+1;
>   -    if (path[nb-1] != '/') {
>   +    if (path[nb-2] != '/') {
>    	char * npath = alloca(nb+1);
>    	*npath = '\0';
>    	(void) stpcpy( stpcpy(npath, path), "/");
>   @@ .
>   patch -p0 <<'@@ .'
>   Index: rpm/rpmio/rpmsw.c
>    
> ====================================================================== 
> ======
>   $ cvs diff -u -r2.14 -r2.15 rpmsw.c
>   --- rpm/rpmio/rpmsw.c	15 Nov 2007 17:27:59 -0000	2.14
>   +++ rpm/rpmio/rpmsw.c	8 Feb 2008 15:08:14 -0000	2.15
>   @@ -295,3 +295,15 @@
>        }
>        return usecs;
>    }
>   +
>   +void rpmswPrint(const char * name, /*@null@*/ rpmop op)
>   +        /*@globals fileSystem @*/
>   +        /*@modifies fileSystem @*/
>   +{
>   +    static unsigned int scale = (1000 * 1000);
>   +    if (op != NULL && op->count > 0)
>   +        fprintf(stderr, "   %s %8d %6lu.%06lu MB %6lu.%06lu secs 
> \n",
>   +                name, op->count,
>   +                (unsigned long)op->bytes/scale, (unsigned long) 
> op->bytes%scale,
>   +                op->usecs/scale, op->usecs%scale);
>   +}
>   @@ .
>   patch -p0 <<'@@ .'
>   Index: rpm/rpmio/rpmsw.h
>    
> ====================================================================== 
> ======
>   $ cvs diff -u -r2.8 -r2.9 rpmsw.h
>   --- rpm/rpmio/rpmsw.h	10 Oct 2004 01:24:16 -0000	2.8
>   +++ rpm/rpmio/rpmsw.h	8 Feb 2008 15:08:14 -0000	2.9
>   @@ -109,6 +109,15 @@
>    rpmtime_t rpmswSub(rpmop to, rpmop from)
>    	/*@modifies to @*/;
>
>   +/** \ingroup rpmio
>   + * Print operation statistics.
>   + * @param name			operation name
>   + * @param op			operation statistics
>   + */
>   +void rpmswPrint(const char * name, /*@null@*/ rpmop op)
>   +        /*@globals fileSystem @*/
>   +        /*@modifies fileSystem @*/;
>   +
>    #ifdef __cplusplus
>    }
>    #endif
>   @@ .
>   patch -p0 <<'@@ .'
>   Index: rpm/rpmio/rpmurl.h
>    
> ====================================================================== 
> ======
>   $ cvs diff -u -r1.22 -r1.23 rpmurl.h
>   --- rpm/rpmio/rpmurl.h	25 May 2007 17:36:37 -0000	1.22
>   +++ rpm/rpmio/rpmurl.h	8 Feb 2008 15:08:14 -0000	1.23
>   @@ -81,6 +81,8 @@
>    #define	RPMURL_SERVER_HASDAVCLASS2	( 1 << 2)
>    #define	RPMURL_SERVER_HASDAVEXEC	( 1 << 3)
>
>   +#define	RPMURL_SERVER_OPTIONSDONE	( 1 << 8)
>   +
>    #define	RPMURL_SERVER_HASDAV	(RPMURL_SERVER_HASDAVCLASS1| 
> RPMURL_SERVER_HASDAVCLASS2|RPMURL_SERVER_HASDAVEXEC)
>        int magic;
>    };
>   @@ .
>   patch -p0 <<'@@ .'
>   Index: rpm/rpmio/tfts.c
>    
> ====================================================================== 
> ======
>   $ cvs diff -u -r2.14 -r2.15 tfts.c
>   --- rpm/rpmio/tfts.c	8 Feb 2008 02:08:40 -0000	2.14
>   +++ rpm/rpmio/tfts.c	8 Feb 2008 15:08:14 -0000	2.15
>   @@ -7,6 +7,7 @@
>    #include <rpmmg.h>
>    #include <mire.h>
>    #include <argv.h>
>   +#include <rpmsw.h>
>    #include <popt.h>
>
>    #include "debug.h"
>   @@ -26,9 +27,10 @@
>    static int mgNMatches = 0;
>    static int mgNFails = 0;
>
>   -/*@unchecked@*/
>    static int _fts_debug = 0;
>
>   +extern int _dav_nooptions;
>   +
>    static int ndirs = 0;
>    static int nfiles = 0;
>
>   @@ -62,7 +64,7 @@
>    {
>        int xx;
>
>   -    if (_fts_debug)
>   +    if (rpmIsDebug())
>    	fprintf(stderr, "FTS_%s\t%*s %s\n", ftsInfoStr(fts->fts_info),
>    		indent * (fts->fts_level < 0 ? 0 : fts->fts_level), "",
>    		fts->fts_name);
>   @@ -70,16 +72,24 @@
>        switch (fts->fts_info) {
>        case FTS_D:		/* preorder directory */
>    	ndirs++;
>   +	if (rpmIsVerbose() && !rpmIsDebug()) {
>   +	    size_t nb = strlen(fts->fts_path);
>   +	    /* Add trailing '/' if not present. */
>   +	    fprintf(stderr, "%s%s\n", fts->fts_path,
>   +		(fts->fts_path[nb-1] != '/' ? "/" : ""));
>   +	}
>    	break;
>        case FTS_DP:	/* postorder directory */
>    	break;
>        case FTS_F:		/* regular file */
>    	nfiles++;
>   +	if (rpmIsVerbose() && !rpmIsDebug())
>   +	    fprintf(stderr, "%s\n", fts->fts_path);
>    	if (mire) {
>    	    mireNExecs++;
>   -	    xx = mireRegexec(mire, fts->fts_accpath);
>   +	    xx = mireRegexec(mire, fts->fts_path);
>    	    if (xx == 0) {
>   -		fprintf(stdout, " mire: %s\n", fts->fts_accpath);
>   +		fprintf(stdout, " mire: %s\n", fts->fts_path);
>    		mireNMatches++;
>    	    } else {
>    		mireNFails++;
>   @@ -89,7 +99,7 @@
>    	    const char * s = rpmmgFile(mg, fts->fts_accpath);
>    	    mgNFiles++;
>    	    if (s != NULL) {
>   -		fprintf(stdout, "magic: %s: %s\n", fts->fts_accpath, s);
>   +		fprintf(stdout, "magic: %s: %s\n", fts->fts_path, s);
>    		mgNMatches++;
>    	    } else {
>    		mgNFails++;
>   @@ -120,18 +130,22 @@
>
>    static int ftsWalk(ARGV_t av)
>    {
>   +    rpmop op = memset(alloca(sizeof(*op)), 0, sizeof(*op));
>        FTS * ftsp;
>        FTSENT * fts;
>        int xx;
>
>        ndirs = nfiles = 0;
>   +    xx = rpmswEnter(op, 0);
>        ftsp = Fts_open((char *const *)av, ftsOpts, NULL);
>        while((fts = Fts_read(ftsp)) != NULL)
>    	xx = ftsPrint(ftsp, fts);
>        xx = Fts_close(ftsp);
>   +    xx = rpmswExit(op, ndirs);
>
>    fprintf(stderr, "===== (%d/%d) dirs/files in:\n", ndirs, nfiles);
>        argvPrint(NULL, av, NULL);
>   +    rpmswPrint("fts:", op);
>
>        return 0;
>    }
>   @@ -157,6 +171,11 @@
>     { "whiteout", '\0', POPT_BIT_SET,	&ftsOpts, FTS_WHITEOUT,
>    	N_("return whiteout information"), NULL },
>
>   + { "options", '\0', POPT_ARG_VAL|POPT_ARGFLAG_DOC_HIDDEN,  
> &_dav_nooptions, 0,
>   +	N_("always send http OPTIONS"), NULL},
>   + { "nooptions", '\0', POPT_ARG_VAL|POPT_ARGFLAG_DOC_HIDDEN,  
> &_dav_nooptions, -1,
>   +	N_("use cached http OPTIONS"), NULL},
>   +
>     { "avdebug", '\0', POPT_ARG_VAL|POPT_ARGFLAG_DOC_HIDDEN,  
> &_av_debug, -1,
>    	N_("debug protocol data stream"), NULL},
>     { "davdebug", '\0', POPT_ARG_VAL|POPT_ARGFLAG_DOC_HIDDEN,  
> &_dav_debug, -1,
>   @@ -185,8 +204,10 @@
>        poptContext optCon = poptGetContext(argv[0], argc, argv,  
> optionsTable, 0);
>        ARGV_t av = NULL;
>        int ac = 0;
>   +    ARGV_t dav;
>        int rc;
>        int xx;
>   +    int i;
>
>        while ((rc = poptGetNextOpt(optCon)) > 0) {
>    	const char * optArg = poptGetOptArg(optCon);
>   @@ -213,13 +234,24 @@
>    _mire_debug = 1;
>        }
>
>   -    av = poptGetArgs(optCon);
>   -    ac = argvCount(av);
>   +    dav = poptGetArgs(optCon);
>   +    ac = argvCount(dav);
>        if (ac < 1) {
>    	poptPrintUsage(optCon, stderr, 0);
>    	goto exit;
>        }
>
>   +    /* XXX Add trailing '/' to http:// URI's */
>   +    for (i = 0; i < ac; i++) {
>   +	const char * dn = dav[i];
>   +	size_t nb = strlen(dn);
>   +	const char *nav[2];
>   +	nav[0] = rpmExpand(dn, (dn[nb-1] != '/' ? "/" : NULL), NULL);
>   +	nav[1] = NULL;
>   +	argvAppend(&av, nav);
>   +	nav[0] = _free(nav[0]);
>   +    }
>   +
>        if (mirePattern) {
>    	mire = mireNew(mireMode, mireTag);
>    	if ((xx = mireRegcomp(mire, mirePattern)) != 0)
>   @@ -236,6 +268,8 @@
>        mg = rpmmgFree(mg);
>        mire = mireFree(mire);
>
>   +    av = argvFree(av);
>   +
>    /*@i@*/ urlFreeCache();
>
>        optCon = poptFreeContext(optCon);
>   @@ .
> ______________________________________________________________________
> RPM Package Manager                                    http://rpm5.org
> CVS Sources Repository                                rpm-cvs@rpm5.org