From 88c8d8913f9db8148e8c8a1bc02e3106ed9aa442 Mon Sep 17 00:00:00 2001 From: Dustin Lang Date: Fri, 16 Mar 2018 13:24:39 -0700 Subject: [PATCH 1/3] hpslit: more seeking around for -C close-files mode; not working --- util/hpsplit.c | 21 ++++++++++++++++++--- 1 file changed, 18 insertions(+), 3 deletions(-) diff --git a/util/hpsplit.c b/util/hpsplit.c index 1760b0d0d..02788e794 100644 --- a/util/hpsplit.c +++ b/util/hpsplit.c @@ -445,6 +445,9 @@ int main(int argc, char *argv[]) { flipped = FALSE; j=0; while (1) { + //anbool resuming = FALSE; + off_t seek_to = 0; + if (hps) { if (j >= il_size(hps)) break; @@ -462,11 +465,12 @@ int main(int argc, char *argv[]) { // MEMLEAK the output filename. You'll live. if (ringindex) { int ringhp = healpix_xy_to_ring(hp, nside); - printf("Ring-indexed healpix: %i\n", ringhp); + printf("Ring-indexed healpix: %i (xy index: %i)\n", ringhp, hp); asprintf_safe(&outfn, outfnpat, ringhp); } else { asprintf_safe(&outfn, outfnpat, hp); } + logmsg("Opening output file \"%s\"...\n", outfn); if (closefiles) { off_t offset = resume_offsets[hp]; @@ -476,8 +480,11 @@ int main(int argc, char *argv[]) { else { // else seek to where we left off... out = fitstable_open_for_appending(outfn); - logmsg("Seeking to %lu\n", (long)offset); - fseeko(outtables[hp]->fid, offset, SEEK_SET); + //logmsg("Seeking to %lu\n", (long)offset); + //fseeko(out->fid, offset, SEEK_SET); + fseeko(out->fid, 0, SEEK_SET); + seek_to = offset; + //resuming = TRUE; } } else { out = fitstable_open_for_writing(outfn); @@ -513,11 +520,18 @@ int main(int argc, char *argv[]) { //printf("Output table:\n"); //fitstable_print_columns(out); + //if (!resuming) { if (fitstable_write_primary_header(out) || fitstable_write_header(out)) { ERROR("Failed to write output file headers for \"%s\"", outfn); exit(-1); } + //} + if (seek_to) { + logmsg("Seeking to %lu\n", (long)seek_to); + fseeko(out->fid, seek_to, SEEK_SET); + } + outtables[hp] = out; } @@ -603,6 +617,7 @@ int main(int argc, char *argv[]) { exit(-1); } logmsg("Closed output file for healpix %i (saved file position %lu)\n", ii, (long)offset); + outtables[ii] = NULL; } else { // the "fitstable_fix_header" call (via // fitsfile_fix_header) adds padding to the file to bring it up to From 36ff28d271f86d80701fdae94511507f95e77378 Mon Sep 17 00:00:00 2001 From: Dustin Lang Date: Fri, 16 Mar 2018 18:09:13 -0700 Subject: [PATCH 2/3] hpslit: try closing and re-opening just the FILE*, saving offset --- util/fitstable.c | 2 ++ util/hpsplit.c | 72 ++++++++++++++++++++++++++---------------------- 2 files changed, 41 insertions(+), 33 deletions(-) diff --git a/util/fitstable.c b/util/fitstable.c index 9b6a80652..9738daac6 100644 --- a/util/fitstable.c +++ b/util/fitstable.c @@ -1541,6 +1541,8 @@ int fitstable_fix_header(fitstable_t* t) { if (in_memory(t)) return 0; + //printf("fitstable_fix_header: ext %i, table offset was %lu, fn %s\n", + //t->extension, (long)t->table_offset, t->fn); if (fitsfile_fix_header(t->fid, t->header, &t->table_offset, &t->end_table_offset, t->extension, t->fn)) { diff --git a/util/hpsplit.c b/util/hpsplit.c index 02788e794..fd0a478b2 100644 --- a/util/hpsplit.c +++ b/util/hpsplit.c @@ -446,7 +446,7 @@ int main(int argc, char *argv[]) { j=0; while (1) { //anbool resuming = FALSE; - off_t seek_to = 0; + //off_t seek_to = 0; if (hps) { if (j >= il_size(hps)) @@ -472,23 +472,7 @@ int main(int argc, char *argv[]) { } logmsg("Opening output file \"%s\"...\n", outfn); - if (closefiles) { - off_t offset = resume_offsets[hp]; - if (offset == 0) - // overwrite if first time - out = fitstable_open_for_writing(outfn); - else { - // else seek to where we left off... - out = fitstable_open_for_appending(outfn); - //logmsg("Seeking to %lu\n", (long)offset); - //fseeko(out->fid, offset, SEEK_SET); - fseeko(out->fid, 0, SEEK_SET); - seek_to = offset; - //resuming = TRUE; - } - } else { - out = fitstable_open_for_writing(outfn); - } + out = fitstable_open_for_writing(outfn); if (!out) { ERROR("Failed to open output table \"%s\"", outfn); exit(-1); @@ -517,20 +501,13 @@ int main(int argc, char *argv[]) { i32type, "backref_index", TRUE); } - //printf("Output table:\n"); - //fitstable_print_columns(out); - - //if (!resuming) { if (fitstable_write_primary_header(out) || fitstable_write_header(out)) { ERROR("Failed to write output file headers for \"%s\"", outfn); exit(-1); } - //} - if (seek_to) { - logmsg("Seeking to %lu\n", (long)seek_to); - fseeko(out->fid, seek_to, SEEK_SET); - } + printf("extension: %i\n", out->extension); + //out->extension = 1; outtables[hp] = out; } @@ -554,6 +531,28 @@ int main(int argc, char *argv[]) { rdata = rowdata; } + if (closefiles && (outtables[hp]->fid == NULL)) { + char* outfn; + /* + // HACK -- copy-n-pasted this snippet -- compute output filename + if (ringindex) { + int ringhp = healpix_xy_to_ring(hp, nside); + printf("Ring-indexed healpix: %i (xy index: %i)\n", ringhp, hp); + asprintf_safe(&outfn, outfnpat, ringhp); + } else { + asprintf_safe(&outfn, outfnpat, hp); + } + */ + outfn = outtables[hp]->fn; + logmsg("Re-opening healpix %i file %s at offset %lu\n", + hp, outfn, (long)resume_offsets[hp]); + //outtables[hp]->fid = fopen(outfn, "a+"); + //outtables[hp]->fid = fopen(outfn, "a+b"); + //outtables[hp]->fid = fopen(outfn, "w+b"); + outtables[hp]->fid = fopen(outfn, "r+b"); + fseeko(outtables[hp]->fid, resume_offsets[hp], SEEK_SET); + } + if (anycols) { /* if (fitstable_write_struct_noflip(outtables[hp], rdata)) { @@ -606,18 +605,23 @@ int main(int argc, char *argv[]) { if (!outtables[ii]) continue; + if (closefiles && (outtables[ii]->fid == NULL)) + continue; + off_t offset = ftello(outtables[ii]->fid); if (closefiles) { resume_offsets[ii] = offset; - if (fitstable_fix_header(outtables[ii]) || - fitstable_fix_primary_header(outtables[ii]) || - fitstable_close(outtables[ii])) { - ERROR("Failed to close output table for healpix %i", ii); + logmsg("Closing healpix %i (saving offset %lu)\n", ii, (long)offset); + if (fitstable_fix_header(outtables[ii])) { + ERROR("Failed to fix header for healpix %i after reading input file \"%s\"", ii, originfn); exit(-1); } - logmsg("Closed output file for healpix %i (saved file position %lu)\n", ii, (long)offset); - outtables[ii] = NULL; + if (fclose(outtables[ii]->fid)) { + SYSERROR("Failed to close file %s\n", outtables[ii]->fn); + exit(-1); + } + outtables[ii]->fid = NULL; } else { // the "fitstable_fix_header" call (via // fitsfile_fix_header) adds padding to the file to bring it up to @@ -640,6 +644,8 @@ int main(int argc, char *argv[]) { for (i=0; ifid == NULL)) + continue; if (fitstable_fix_header(outtables[i]) || fitstable_fix_primary_header(outtables[i]) || fitstable_close(outtables[i])) { From b1148c094ac223d6e02c7d14ca048f3ee0243d6f Mon Sep 17 00:00:00 2001 From: Dustin Lang Date: Fri, 16 Mar 2018 18:20:45 -0700 Subject: [PATCH 3/3] hpsplit: tidy up --- util/fitstable.c | 8 +++++--- util/hpsplit.c | 49 ++++++++++++++---------------------------------- 2 files changed, 19 insertions(+), 38 deletions(-) diff --git a/util/fitstable.c b/util/fitstable.c index 9738daac6..77efbe9e1 100644 --- a/util/fitstable.c +++ b/util/fitstable.c @@ -1291,9 +1291,11 @@ int fitstable_close(fitstable_t* tab) { int rtn = 0; if (!tab) return 0; if (is_writing(tab)) { - if (fclose(tab->fid)) { - SYSERROR("Failed to close output file %s", tab->fn); - rtn = -1; + if (tab->fid) { + if (fclose(tab->fid)) { + SYSERROR("Failed to close output file %s", tab->fn); + rtn = -1; + } } } if (tab->anq) { diff --git a/util/hpsplit.c b/util/hpsplit.c index fd0a478b2..c0ae5f253 100644 --- a/util/hpsplit.c +++ b/util/hpsplit.c @@ -445,9 +445,6 @@ int main(int argc, char *argv[]) { flipped = FALSE; j=0; while (1) { - //anbool resuming = FALSE; - //off_t seek_to = 0; - if (hps) { if (j >= il_size(hps)) break; @@ -465,7 +462,7 @@ int main(int argc, char *argv[]) { // MEMLEAK the output filename. You'll live. if (ringindex) { int ringhp = healpix_xy_to_ring(hp, nside); - printf("Ring-indexed healpix: %i (xy index: %i)\n", ringhp, hp); + logverb("Ring-indexed healpix: %i (xy index: %i)\n", ringhp,hp); asprintf_safe(&outfn, outfnpat, ringhp); } else { asprintf_safe(&outfn, outfnpat, hp); @@ -506,8 +503,6 @@ int main(int argc, char *argv[]) { ERROR("Failed to write output file headers for \"%s\"", outfn); exit(-1); } - printf("extension: %i\n", out->extension); - //out->extension = 1; outtables[hp] = out; } @@ -532,40 +527,20 @@ int main(int argc, char *argv[]) { } if (closefiles && (outtables[hp]->fid == NULL)) { - char* outfn; - /* - // HACK -- copy-n-pasted this snippet -- compute output filename - if (ringindex) { - int ringhp = healpix_xy_to_ring(hp, nside); - printf("Ring-indexed healpix: %i (xy index: %i)\n", ringhp, hp); - asprintf_safe(&outfn, outfnpat, ringhp); - } else { - asprintf_safe(&outfn, outfnpat, hp); - } - */ - outfn = outtables[hp]->fn; - logmsg("Re-opening healpix %i file %s at offset %lu\n", - hp, outfn, (long)resume_offsets[hp]); - //outtables[hp]->fid = fopen(outfn, "a+"); - //outtables[hp]->fid = fopen(outfn, "a+b"); - //outtables[hp]->fid = fopen(outfn, "w+b"); + char* outfn = outtables[hp]->fn; + logverb("Re-opening healpix %i file %s at offset %lu\n", + hp, outfn, (long)resume_offsets[hp]); outtables[hp]->fid = fopen(outfn, "r+b"); fseeko(outtables[hp]->fid, resume_offsets[hp], SEEK_SET); } if (anycols) { - /* - if (fitstable_write_struct_noflip(outtables[hp], rdata)) { - ERROR("Failed to copy a row of data from input table \"%s\" to output healpix %i", infn, hp); - } - */ if (!flipped) { // if we're writing to multiple output // healpixes, only flip once! flipped = TRUE; fitstable_endian_flip_row_data(intable2, rdata); } - //fitstable_endian_flip_row_data(outtables[hp], rdata); if (fitstable_write_struct(outtables[hp], rdata)) { ERROR("Failed to copy a row of data from input table \"%s\" to output healpix %i", infn, hp); } @@ -604,15 +579,13 @@ int main(int argc, char *argv[]) { for (ii=0; iifid == NULL)) continue; off_t offset = ftello(outtables[ii]->fid); - if (closefiles) { resume_offsets[ii] = offset; - logmsg("Closing healpix %i (saving offset %lu)\n", ii, (long)offset); + logverb("Closing healpix %i (saving offset %lu)\n", ii, (long)offset); if (fitstable_fix_header(outtables[ii])) { ERROR("Failed to fix header for healpix %i after reading input file \"%s\"", ii, originfn); exit(-1); @@ -624,8 +597,9 @@ int main(int argc, char *argv[]) { outtables[ii]->fid = NULL; } else { // the "fitstable_fix_header" call (via - // fitsfile_fix_header) adds padding to the file to bring it up to - // a FITS block size, so we ftell and fseek afterward. + // fitsfile_fix_header) adds padding to the file to + // bring it up to a FITS block size, so we ftell and + // fseek afterward. if (fitstable_fix_header(outtables[ii])) { ERROR("Failed to fix header for healpix %i after reading input file \"%s\"", ii, originfn); exit(-1); @@ -644,8 +618,13 @@ int main(int argc, char *argv[]) { for (i=0; ifid == NULL)) + if (closefiles && (outtables[i]->fid == NULL)) { + if (fitstable_close(outtables[i])) { + ERROR("Failed to close output table for healpix %i", i); + exit(-1); + } continue; + } if (fitstable_fix_header(outtables[i]) || fitstable_fix_primary_header(outtables[i]) || fitstable_close(outtables[i])) {