RPM Package Manager, CVS Repository
http://rpm5.org/cvs/
____________________________________________________________________________
Server: rpm5.org Name: Anders F. Björklund
Root: /v/rpm/cvs Email: afb@rpm5.org
Module: xar Date: 02-Oct-2007 23:46:42
Branch: HEAD Handle: 2007100222464101
Modified files:
xar ChangeLog
xar/include xar.h.in
xar/lib archive.c io.c io.h
Log:
new xar_stream streaming API (from upstream)
Summary:
Revision Changes Path
1.4 +2 -0 xar/ChangeLog
1.5 +18 -0 xar/include/xar.h.in
1.6 +23 -0 xar/lib/archive.c
1.5 +262 -57 xar/lib/io.c
1.3 +17 -0 xar/lib/io.h
____________________________________________________________________________
patch -p0 <<'@@ .'
Index: xar/ChangeLog
============================================================================
$ cvs diff -u -r1.3 -r1.4 ChangeLog
--- xar/ChangeLog 29 Sep 2007 12:29:43 -0000 1.3
+++ xar/ChangeLog 2 Oct 2007 21:46:41 -0000 1.4
@@ -1,4 +1,6 @@
devel
+ 2007-10-02 Rob Braun bbraun@synack.net
+ * include/xar.h.in lib/io.c lib/io.h lib/archive.c: Add a "streaming" API that allows for file data to be extracted incrementally, like the zlib decompression API, instead of requiring the entire file to fit in memory like xar_extract_tobuffer(). This is the inital commit of the API and may still change. This change is based on a patch from Charles Srstka.
2007-09-26 Rob Braun bbraun@synack.net
* lib/io.c: Consolidate the lseek handling code into one function.
Contributed by Charles Srstka as part of Issue 2.
@@ .
patch -p0 <<'@@ .'
Index: xar/include/xar.h.in
============================================================================
$ cvs diff -u -r1.4 -r1.5 xar.h.in
--- xar/include/xar.h.in 29 Sep 2007 12:29:43 -0000 1.4
+++ xar/include/xar.h.in 2 Oct 2007 21:46:41 -0000 1.5
@@ -71,6 +71,16 @@
typedef const struct __xar_subdoc_t *xar_subdoc_t;
typedef const struct __xar_signature_t *xar_signature_t;
+typedef struct {
+ char *next_out;
+ unsigned int avail_out;
+
+ unsigned long long total_in;
+ unsigned long long total_out;
+
+ void *state;
+} xar_stream;
+
typedef int32_t (*err_handler)(int32_t severit, int32_t instance, xar_errctx_t ctx, void *usrctx);
/* the signed_data must be allocated durring the callback and will be released by the xar lib after the callback */
typedef int32_t (*xar_signer_callback)(xar_signature_t sig, void *context, uint8_t *data, uint32_t length, uint8_t **signed_data, uint32_t *signed_len);
@@ -78,6 +88,11 @@
#define READ 0
#define WRITE 1
+/* xar stream return codes */
+#define XAR_STREAM_OK 0
+#define XAR_STREAM_END 1
+#define XAR_STREAM_ERR -1
+
/* Valid xar options & values */
#define XAR_OPT_OWNERSHIP "ownership" /* setting owner/group behavior */
#define XAR_OPT_VAL_SYMBOLIC "symbolic" /* set owner/group based on names */
@@ -134,6 +149,9 @@
int32_t xar_extract_tofile(xar_t x, xar_file_t f, const char *path);
int32_t xar_extract_tobuffer(xar_t x, xar_file_t f, char **buffer);
int32_t xar_extract_tobuffersz(xar_t x, xar_file_t f, char **buffer, size_t *size);
+int32_t xar_extract_tostream_init(xar_t x, xar_file_t f, xar_stream *stream);
+int32_t xar_extract_tostream(xar_stream *stream);
+int32_t xar_extract_tostream_end(xar_stream *stream);
int32_t xar_verify(xar_t x, xar_file_t f);
@@ .
patch -p0 <<'@@ .'
Index: xar/lib/archive.c
============================================================================
$ cvs diff -u -r1.5 -r1.6 archive.c
--- xar/lib/archive.c 21 Sep 2007 12:00:58 -0000 1.5
+++ xar/lib/archive.c 2 Oct 2007 21:46:42 -0000 1.6
@@ -1189,6 +1189,29 @@
return xar_arcmod_extract(x,f,NULL,*buffer,*size);
}
+int32_t xar_extract_tostream_init(xar_t x, xar_file_t f, xar_stream *stream) {
+ xar_prop_t tmpp;
+
+ if( !xar_check_prop(x, "data") )
+ return XAR_STREAM_OK;
+
+ tmpp = xar_prop_pfirst(f);
+ if( tmpp )
+ tmpp = xar_prop_find(tmpp, "data");
+ if( !tmpp )
+ return XAR_STREAM_OK;
+
+ return xar_attrcopy_from_heap_to_stream_init(x, f, tmpp, stream);
+}
+
+int32_t xar_extract_tostream(xar_stream *stream) {
+ return xar_attrcopy_from_heap_to_stream(stream);
+}
+
+int32_t xar_extract_tostream_end(xar_stream *stream) {
+ return xar_attrcopy_from_heap_to_stream_end(stream);
+}
+
/* xar_extract
* x: archive to extract from
* path: path to file to extract
@@ .
patch -p0 <<'@@ .'
Index: xar/lib/io.c
============================================================================
$ cvs diff -u -r1.4 -r1.5 io.c
--- xar/lib/io.c 29 Sep 2007 12:29:43 -0000 1.4
+++ xar/lib/io.c 2 Oct 2007 21:46:42 -0000 1.5
@@ -108,6 +108,62 @@
}
};
+
+static size_t get_rsize(xar_t x) {
+ size_t bsize;
+ const char *opt = NULL;
+
+ opt = xar_opt_get(x, "rsize");
+ if( !opt ) {
+ bsize = 4096;
+ } else {
+ bsize = strtol(opt, NULL, 0);
+ if( ((bsize == LONG_MAX) || (bsize == LONG_MIN)) && (errno == ERANGE) ) {
+ bsize = 4096;
+ }
+ }
+
+ return bsize;
+}
+
+static off_t get_offset(xar_t x, xar_file_t f, xar_prop_t p) {
+ off_t seekoff;
+ xar_prop_t tmpp;
+ const char *opt = NULL;
+
+ tmpp = xar_prop_pget(p, "offset");
+ if( tmpp )
+ opt = xar_prop_getvalue(tmpp);
+
+ seekoff = strtoll(opt, NULL, 0);
+
+ if( ((seekoff == LLONG_MAX) || (seekoff == LLONG_MIN)) && (errno == ERANGE) ) {
+ return -1;
+ }
+
+ return seekoff;
+}
+
+static int64_t get_length(xar_prop_t p) {
+ const char *opt = NULL;
+ int64_t fsize = 0;
+ xar_prop_t tmpp;
+
+ tmpp = xar_prop_pget(p, "length");
+ if( tmpp )
+ opt = xar_prop_getvalue(tmpp);
+ if( !opt ) {
+ return 0;
+ } else {
+ fsize = strtoll(opt, NULL, 10);
+ if( ((fsize == LLONG_MAX) || (fsize == LLONG_MIN)) && (errno == ERANGE) ) {
+ return -1;
+ }
+ }
+
+ return fsize;
+}
+
static void xar_io_seek(xar_t x, xar_file_t f, off_t seekoff) {
int r;
@@ -164,15 +220,7 @@
memset(modulecontext, 0, sizeof(void*)*modulecount);
- opt = xar_opt_get(x, XAR_OPT_RSIZE);
- if( !opt ) {
- bsize = 4096;
- } else {
- bsize = strtol(opt, NULL, 0);
- if( ((bsize == LONG_MAX) || (bsize == LONG_MIN)) && (errno == ERANGE) ) {
- bsize = 4096;
- }
- }
+ bsize = get_rsize(x);
r = 1;
while(r != 0) {
@@ -336,15 +384,7 @@
memset(modulecontext, 0, sizeof(void*)*modulecount);
- opt = xar_opt_get(x, "rsize");
- if( !opt ) {
- def_bsize = 4096;
- } else {
- def_bsize = strtol(opt, NULL, 0);
- if( ((def_bsize == LONG_MAX) || (def_bsize == LONG_MIN)) && (errno == ERANGE) ) {
- def_bsize = 4096;
- }
- }
+ bsize = get_rsize(x);
opt = NULL;
tmpp = xar_prop_pget(p, "offset");
@@ -363,18 +403,11 @@
seekoff += XAR(x)->toc_count + sizeof(xar_header_t);
xar_io_seek(x, f, seekoff);
- opt = NULL;
- tmpp = xar_prop_pget(p, "length");
- if( tmpp )
- opt = xar_prop_getvalue(tmpp);
- if( !opt ) {
+ fsize = get_length(p);
+ if( fsize == 0 )
return 0;
- } else {
- fsize = strtoll(opt, NULL, 10);
- if( ((fsize == LLONG_MAX) || (fsize == LLONG_MIN)) && (errno == ERANGE) ) {
- return -1;
- }
- }
+ if( fsize < 0 )
+ return -1;
bsize = def_bsize;
inbuf = malloc(bsize);
@@ -460,41 +493,20 @@
char *tmpstr = NULL;
xar_prop_t tmpp;
- opt = xar_opt_get(xsource, "rsize");
- if( !opt ) {
- bsize = 4096;
- } else {
- bsize = strtol(opt, NULL, 0);
- if( ((bsize == LONG_MAX) || (bsize == LONG_MIN)) && (errno == ERANGE) ) {
- bsize = 4096;
- }
- }
-
- tmpp = xar_prop_pget(p, "offset");
- if( tmpp )
- opt = xar_prop_getvalue(tmpp);
+ bsize = get_rsize(xsource);
- seekoff = strtoll(opt, NULL, 0);
-
- if( ((seekoff == LLONG_MAX) || (seekoff == LLONG_MIN)) && (errno == ERANGE) ) {
+ seekoff = get_offset(xsource, fsource, p);
+ if( seekoff < 0 )
return -1;
- }
seekoff += XAR(xsource)->toc_count + sizeof(xar_header_t);
xar_io_seek(xsource, fsource, seekoff);
- opt = NULL;
- tmpp = xar_prop_pget(p, "length");
- if( tmpp )
- opt = xar_prop_getvalue(tmpp);
- if( !opt ) {
+ fsize = get_length(p);
+ if( fsize == 0 )
return 0;
- } else {
- fsize = strtoll(opt, NULL, 10);
- if( ((fsize == LLONG_MAX) || (fsize == LLONG_MIN)) && (errno == ERANGE) ) {
- return -1;
- }
- }
+ if( fsize < 0 )
+ return -1;
inbuf = malloc(bsize);
if( !inbuf ) {
@@ -549,6 +561,199 @@
return 0;
}
+
+static int32_t flush_stream(xar_stream *stream) {
+ xar_stream_state_t *state = (xar_stream_state_t *)(stream->state);
+
+ if( state->pending_buf && stream->avail_out ) {
+ size_t len = state->pending_buf_size;
+
+ if( stream->avail_out < len ) {
+ len = stream->avail_out;
+ }
+
+ memcpy(stream->next_out, state->pending_buf, len);
+ stream->next_out += len;
+ stream->avail_out -= len;
+ stream->total_out += len;
+
+ if( state->pending_buf_size == len ) {
+ state->pending_buf_size = 0;
+
+ free(state->pending_buf);
+ state->pending_buf = NULL;
+ } else if( state->pending_buf_size > len ) {
+ state->pending_buf_size -= len;
+ memcpy(state->pending_buf, state->pending_buf + len, state->pending_buf_size);
+ }
+ }
+
+ return XAR_STREAM_OK;
+}
+
+static int32_t write_to_stream(void *inbuf, size_t inlen, xar_stream *stream) {
+ xar_stream_state_t *state = (xar_stream_state_t *)stream->state;
+ size_t len = inlen;
+
+ if( stream->avail_out < len ) {
+ len = stream->avail_out;
+ }
+
+ memcpy(stream->next_out, inbuf, len);
+ stream->next_out += len;
+ stream->avail_out -= len;
+ stream->total_out += len;
+
+ if( inlen > len ) {
+ state->pending_buf_size = inlen - len;
+ state->pending_buf = malloc(state->pending_buf_size);
+
+ memcpy(state->pending_buf, inbuf + len, state->pending_buf_size);
+ }
+
+ return XAR_STREAM_OK;
+}
+
+int32_t xar_attrcopy_from_heap_to_stream_init(xar_t x, xar_file_t f, xar_prop_t p, xar_stream *stream) {
+ xar_stream_state_t *state;
+ off_t seekoff;
+
+ seekoff = get_offset(x, f, p);
+ if( seekoff < 0 )
+ return XAR_STREAM_ERR;
+
+ state = calloc(1, sizeof(xar_stream_state_t));
+ if( !state ) {
+ return XAR_STREAM_ERR;
+ }
+ stream->state = (void*)state;
+ state->bsize = get_rsize(x);
+
+ state->modulecount = (sizeof(xar_datamods)/sizeof(struct datamod));
+ state->modulecontext = calloc(1, sizeof(void*)*state->modulecount);
+ if( !state->modulecontext ) {
+ free(state);
+ return XAR_STREAM_ERR;
+ }
+
+ seekoff += XAR(x)->toc_count + sizeof(xar_header_t);
+ xar_io_seek(x, f, seekoff);
+
+ stream->total_in = 0;
+ stream->total_out = 0;
+
+ state->fsize = get_length(p);
+
+ if(state->fsize == 0) {
+ return XAR_STREAM_OK;
+ } else if(state->fsize == -1) {
+ free(state->modulecontext);
+ free(state);
+ return XAR_STREAM_ERR;
+ }
+
+ state->pending_buf = NULL;
+ state->pending_buf_size = 0;
+ state->x = x;
+ state->f = f;
+ state->p = p;
+
+ return XAR_STREAM_OK;
+}
+
+int32_t xar_attrcopy_from_heap_to_stream(xar_stream *stream) {
+ xar_stream_state_t *state = stream->state;
+
+ int r, i;
+ size_t bsize;
+ void *inbuf;
+
+ if( state->pending_buf_size ) {
+ return flush_stream(stream);
+ }
+
+ bsize = state->bsize;
+ inbuf = malloc(bsize);
+ if( !inbuf ) {
+ return XAR_STREAM_ERR;
+ }
+
+ /* Size has been reached */
+ if( state->fsize == stream->total_in ) {
+ free(inbuf);
+ return XAR_STREAM_END;
+ }
+ if( (state->fsize - stream->total_in) < bsize )
+ bsize = state->fsize - stream->total_in;
+ r = read(XAR(state->x)->fd, inbuf, bsize);
+ if( r == 0 ) {
+ free(inbuf);
+ return XAR_STREAM_END;
+ }
+ if( (r < 0) && (errno == EINTR) ) {
+ free(inbuf);
+ return XAR_STREAM_OK;
+ }
+ if( r < 0 ) {
+ free(inbuf);
+ return XAR_STREAM_ERR;
+ }
+
+ XAR(state->x)->heap_offset += r;
+ stream->total_in += r;
+ bsize = r;
+
+ /* filter the data through the in modules */
+ for( i = 0; i < state->modulecount; i++) {
+ if( xar_datamods[i].fh_in ) {
+ int32_t ret;
+ ret = xar_datamods[i].fh_in(state->x, state->f, state->p, &inbuf, &bsize, &(state->modulecontext[i]));
+ if( ret < 0 )
+ return XAR_STREAM_ERR;
+ }
+ }
+
+ /* filter the data through the out modules */
+ for( i = 0; i < state->modulecount; i++) {
+ if( xar_datamods[i].fh_out ) {
+ int32_t ret;
+ ret = xar_datamods[i].fh_out(state->x, state->f, state->p, inbuf, bsize, &(state->modulecontext[i]));
+ if( ret < 0 )
+ return XAR_STREAM_ERR;
+ }
+ }
+
+ write_to_stream(inbuf, bsize, stream);
+
+ free(inbuf);
+
+ return XAR_STREAM_OK;
+}
+
+int32_t xar_attrcopy_from_heap_to_stream_end(xar_stream *stream) {
+ xar_stream_state_t *state = (xar_stream_state_t *)stream->state;
+ int i;
+
+ /* finish up anything that still needs doing */
+ for( i = 0; i < state->modulecount; i++) {
+ if( xar_datamods[i].fh_done ) {
+ int32_t ret;
+ ret = xar_datamods[i].fh_done(state->x, state->f, state->p, &(state->modulecontext[i]));
+ if( ret < 0 )
+ return ret;
+ }
+ }
+
+ if( state->pending_buf ) {
+ free(state->pending_buf);
+ }
+
+ free(state->modulecontext);
+ free(state);
+
+ return XAR_STREAM_OK;
+}
+
/* xar_heap_to_archive
* x: archive to operate on
* Returns 0 on success, -1 on error
@@ .
patch -p0 <<'@@ .'
Index: xar/lib/io.h
============================================================================
$ cvs diff -u -r1.2 -r1.3 io.h
--- xar/lib/io.h 19 Sep 2007 22:14:01 -0000 1.2
+++ xar/lib/io.h 2 Oct 2007 21:46:42 -0000 1.3
@@ -58,9 +58,26 @@
toheap_done th_done;
};
+typedef struct xar_stream_state {
+ char *pending_buf;
+ size_t pending_buf_size;
+
+ void **modulecontext;
+ int modulecount;
+ size_t bsize;
+ int64_t fsize;
+ xar_t x;
+ xar_file_t f;
+ xar_prop_t p;
+} xar_stream_state_t;
+
int32_t xar_attrcopy_to_heap(xar_t x, xar_file_t f, xar_prop_t p, read_callback rcb, void *context);
int32_t xar_attrcopy_from_heap(xar_t x, xar_file_t f, xar_prop_t p, write_callback wcb, void *context);
int32_t xar_attrcopy_from_heap_to_heap(xar_t xsource, xar_file_t fsource, xar_prop_t p, xar_t xdest, xar_file_t fdest);
+int32_t xar_attrcopy_from_heap_to_stream_init(xar_t x, xar_file_t f, xar_prop_t p, xar_stream *stream);
+int32_t xar_attrcopy_from_heap_to_stream(xar_stream *stream);
+int32_t xar_attrcopy_from_heap_to_stream_end(xar_stream *stream);
+
int32_t xar_heap_to_archive(xar_t x);
#endif /* _XAR_IO_H_ */
@@ .
Received on Tue Oct 2 23:46:42 2007