Skip to content

Commit

Permalink
Merge branch 'master' of git.liquidinstruments.com:mercury-open/liqui…
Browse files Browse the repository at this point in the history 
…dreader
  • Loading branch information
Ben Nizette committed Jul 27, 2017
2 parents 045c427 + 5ecd003 commit 2cbf3ae
Show file tree
Hide file tree
Showing 5 changed files with 86 additions and 54 deletions.
22 changes: 16 additions & 6 deletions liconvert.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -51,6 +51,7 @@ int main(int argc, char** argv) {
csv, mat
} kind = csv;
bool use_stdin = false;
bool stdin_already_used = false;

while (*++argv)
if (**argv == '-') { // Process a flag
Expand All @@ -59,18 +60,24 @@ int main(int argc, char** argv) {
} else if (!strcmp(*argv, "--mat")) {
kind = mat;
} else if (!strcmp(*argv, "--stdin")) {
if (stdin_already_used) {
printf("Cannot process stdin twice\n");
return EXIT_FAILURE;
}
use_stdin = true;
} else {
if (!strcmp(*argv, "--help"))
printf("Unrecognized option \"%s\"\n\n", *argv);
printf("Unrecognized option \"%s\"\n", *argv);
help();
}
} else { // Convert a file
FILE* infile;
if (use_stdin)
if (use_stdin) {
infile = stdin;
else
stdin_already_used = true;
} else {
infile = fopen(*argv, "rb");
}

if (!infile) {
fprintf(stderr, "Could not open \"%s\"\n", *argv);
Expand All @@ -96,18 +103,21 @@ int main(int argc, char** argv) {
li_status result;
switch (kind) {
case csv:
result = li_to_csv(infile, outfile);
result = li_to_csv(infile, outfile, NULL, NULL);
break;
case mat:
result = li_to_mat(infile, outfile);
result = li_to_mat(infile, outfile, NULL, NULL);
break;
}
if (result)
printf("%s error converting \"%s\"\n", li_status_string(result), *argv);
fclose(outfile);
cleanup:
free(outname);
fclose(infile);
if (!use_stdin)
fclose(infile);
else
use_stdin = false;
}
return EXIT_SUCCESS;
}
28 changes: 19 additions & 9 deletions litocsv.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,10 @@
#define CONTINUE_SMALL_AFTER(CLEANUP) { if (result != LI_SUCCESS) { { CLEANUP; } if (result == LI_SMALL_SRC) { continue; } else { LI_ON_ERROR; goto cleanup; } } }
#define REQUIRE_FORMAT(X) do { if (!( X )) { result = LI_BAD_FORMAT; LI_ON_ERROR; goto cleanup; } } while(false)

li_status li_to_csv(FILE* input, FILE* output)
li_status li_to_csv(FILE* input,
FILE* output,
void (*callback)(void* user_ptr, uint64_t bytes_read, uint64_t bytes_written),
void* user_ptr)
{
// Todo: reduce duplication with li_to_mat

Expand Down Expand Up @@ -71,6 +74,8 @@ li_status li_to_csv(FILE* input, FILE* output)
// Read up to read_buffer_size bytes from the file, which is at least
// as many as suggested, and set n to the bytes actually read
n = fread(li_array_begin(li_byte)(&buffer), 1, li_array_size(li_byte)(&buffer), input);
if (callback)
callback(user_ptr, n, 0);
// Give n bytes to the reader
result = li_put(r, li_array_begin(li_byte)(&buffer), (size_t)n);
REQUIRE_SUCCESS;
Expand Down Expand Up @@ -130,47 +135,52 @@ li_status li_to_csv(FILE* input, FILE* output)
LI_TRUST(li_string_resize(&csvHeader, (size_t) bytes - 1, 'x'));
LI_TRUST(li_get(r, LI_HDR_STRING_UTF8V, 0, csvHeader, (size_t) bytes));

fprintf(output, "%s", csvHeader);
n = fprintf(output, "%s", csvHeader);
if (callback)
callback(user_ptr, 0, n);
}

if (csvHeader && csvFmt) {
// Try to get all the records for the next time
n = 0; // Accumulate bytes written
while ((result = li_get(r, LI_RECORD_F64V, 0, li_array_begin(double)(&doubles), li_array_size(double)(&doubles) * sizeof(double))) == LI_SUCCESS) {
// Compute the relative time
double t = startOffset + timeStep * rows++;
if (li_array_empty(Replacement)(&replacements)) {
// There's no format string so print time followed by
// everything
fprintf(output, "%.10e", t);
n += fprintf(output, "%.10e", t);
LI_FOR(double, p, &doubles)
fprintf(output, ", %.16e", *p);
n += fprintf(output, ", %.16e", *p);
} else {
// We have parsed the format string

Replacement* p = li_array_begin(Replacement)(&replacements);
for (;;) {
switch (*p->identifier) {
case 't':
fprintf(output, p->format, t);
n += fprintf(output, p->format, t);
break;
case 'n':
fprintf(output, "%lu", rows-1);
n += fprintf(output, "%lu", rows-1);
break;
case 'c':
fprintf(output, p->format, doubles.begin[p->index]);
n += fprintf(output, p->format, doubles.begin[p->index]);
break;
}
++p;
if (p == li_array_end(Replacement)(&replacements)) {
break;
} else {
fprintf(output, ", ");
n += fprintf(output, ", ");
}
}
}
// CR+LF line end
fprintf(output, "\r\n");
n += fprintf(output, "\r\n");
}
if (callback)
callback(user_ptr, 0, n);
if (result != LI_SMALL_SRC) {
// We left the loop because of a serious error
goto cleanup;
Expand Down
11 changes: 8 additions & 3 deletions litocsv.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,10 +19,15 @@ extern "C" {

// Read a Liquid Instruments binary log file file and write a Comma
// Separated Value (CSV) file. Files must be open for binary reading and
// writing respectively

li_status li_to_csv(FILE* input, FILE* output);
// writing respectively. Optionally provide a callback that will report
// whenever bytes are read from input or written to output. user_ptr is
// passed unchanged to the callback.

li_status li_to_csv(FILE* input,
FILE* output,
void (*callback)(void* user_ptr, uint64_t bytes_read, uint64_t bytes_written),
void* user_ptr);

#ifdef __cplusplus
}
#endif
Expand Down
74 changes: 39 additions & 35 deletions litomat.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -35,21 +35,23 @@ typedef struct {

void pTF_ctor(pTF* self) {
assert(self);
// The files returned by tempfile should be automatically deleted on close,
// and automatically closed on process exit.
self->fp = tmpfile();
}

void pTF_dtor(pTF* self) {
assert(self);
// Manually close them anyway to reclaim disk and scarce file descriptors
fclose(self->fp);
if (self && self->fp)
fclose(self->fp);
}

li_array_define(pTF);

li_status li_to_mat(FILE* input, FILE* output) {

li_status li_to_mat(FILE* input,
FILE* output,
void (*callback)(void* user_ptr, uint64_t bytes_read, uint64_t bytes_written),
void* user_ptr)
{

// Todo: reduce duplication with li_to_mat

li_status result = LI_SUCCESS;
Expand Down Expand Up @@ -97,6 +99,8 @@ li_status li_to_mat(FILE* input, FILE* output) {
// Read up to read_buffer_size bytes from the file, which is at least
// as many as suggested, and set n to the bytes actually read
n = fread(li_array_begin(li_byte)(&buffer), 1, li_array_size(li_byte)(&buffer), input);
if (callback)
callback(user_ptr, n, 0);
// Give n bytes to the reader
result = li_put(r, li_array_begin(li_byte)(&buffer), (size_t) n);
REQUIRE_SUCCESS;
Expand Down Expand Up @@ -184,6 +188,7 @@ li_status li_to_mat(FILE* input, FILE* output) {
#endif
fwrite(&d, sizeof(double), 1, (iter++)->fp);
assert(m == 1);
// We don't report I/O with the temporary files to callback
}
}
if (result != LI_SMALL_SRC) // We left the loop because of an error
Expand All @@ -193,13 +198,15 @@ li_status li_to_mat(FILE* input, FILE* output) {
REQUIRE_FORMAT(rows);
// We finished the file and read at least one row
result = LI_SUCCESS;

// to report incremental progress in file we use ftell
long old_offset = 0;
long new_offset = 0;

mh = mat_header_new(csvHeader);
fwrite(mh, sizeof(mat_header), 1, output);
REQUIRE_ALLOC(mh);



fwrite(mh, sizeof(mat_header), 1, output);

long token = mat_element_open(output, miMATRIX);
mat_array_write_flags(output, mxSTRUCT_CLASS);
mat_array_write_dims2(output, 1, 1);
Expand All @@ -210,7 +217,6 @@ li_status li_to_mat(FILE* input, FILE* output) {
char fields[][FIELD_NAME_LENGTH] = {
"comment",
"data",
"time",
"legend",
"version",
"timestamp"
Expand All @@ -222,9 +228,15 @@ li_status li_to_mat(FILE* input, FILE* output) {
{
mat_matrix_write_utf8(output, csvHeader);
}


if (callback) {
new_offset = ftell(output);
callback(user_ptr, 0, new_offset - old_offset);
old_offset = new_offset;
}

size_t columns = li_array_size(pTF)(&files);

// Moku.data
{
long token2 = mat_element_open(output, miMATRIX);
Expand All @@ -249,25 +261,11 @@ li_status li_to_mat(FILE* input, FILE* output) {
// Close early to reduce maximum footprint on disk
fclose(p->fp);
p->fp = 0;
}
mat_element_close(output, token3);
}
mat_element_close(output, token2);
}

// Moku.time
{
long token2 = mat_element_open(output, miMATRIX);
mat_array_write_flags(output, mxDOUBLE_CLASS);
mat_array_write_dims2(output, (int32_t) rows, 1);
mat_array_write_name(output, "");
{
long token3 = mat_element_open(output, miDOUBLE);
for (int j = 0; j != rows; ++j) {
// compute rather than increment to maximise precision
double t = startOffset + timeStep * j;
// append the column to the output one value at a time
fwrite(&t, sizeof(t), 1, output);
if (callback) {
new_offset = ftell(output);
callback(user_ptr, 0, new_offset - old_offset);
old_offset = new_offset;
}
}
mat_element_close(output, token3);
}
Expand Down Expand Up @@ -317,16 +315,22 @@ li_status li_to_mat(FILE* input, FILE* output) {
// Moku.timestamp
{
time_t t = time(NULL);
const int N = 64;
# define N 64
char buf[N];
strftime(buf, N, "%Y-%m-%d T %H:%M:%S %z", localtime(&t));
mat_matrix_write_utf8(output, buf);
}

mat_element_close(output, token); // close the mat_struct


if (callback) {
new_offset = ftell(output);
callback(user_ptr, 0, new_offset - old_offset);
old_offset = new_offset;
}

cleanup:

mat_header_delete(mh);
li_array_dtor(pTF)(&files);
li_array_dtor(Replacement)(&replacements);
Expand Down
5 changes: 4 additions & 1 deletion litomat.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,10 @@ extern "C" {
// MAT-file. Files must be open for binary reading and writing
// respectively.

li_status li_to_mat(FILE* input, FILE* output);
li_status li_to_mat(FILE* input,
FILE* output,
void (*callback)(void* user_ptr, uint64_t bytes_read, uint64_t bytes_written),
void* user_ptr);


#ifdef __cplusplus
Expand Down

0 comments on commit 2cbf3ae

Please sign in to comment.