From dbf96bfc68cf405f7be63609b8ded60513ba7f40 Mon Sep 17 00:00:00 2001 From: Chris Shucksmith Date: Fri, 15 Nov 2024 16:40:26 +0000 Subject: [PATCH 1/3] switch from hand formatting to clang-format --- .clang-format | 20 + .github/workflows/make.yml | 3 + native/Makefile | 4 +- native/buffer.c | 37 +- native/buffer.h | 4 +- native/fuzzmain.c | 99 +++-- native/libchronicle.c | 808 +++++++++++++++++++----------------- native/libchronicle.h | 90 ++-- native/mock_k.h | 68 +-- native/serdes_k.h | 9 +- native/shm_example_reader.c | 28 +- native/shm_example_writer.c | 22 +- native/shmmain.c | 33 +- native/test/test_buffer.c | 45 +- native/test/test_queue.c | 95 ++--- native/test/test_wire.c | 115 +++-- native/test/testdata.h | 55 ++- native/wire.c | 454 ++++++++++---------- native/wire.h | 73 ++-- 19 files changed, 1089 insertions(+), 973 deletions(-) create mode 100644 .clang-format diff --git a/.clang-format b/.clang-format new file mode 100644 index 0000000..8a6642c --- /dev/null +++ b/.clang-format @@ -0,0 +1,20 @@ +BasedOnStyle: LLVM +IndentWidth: 4 +TabWidth: 4 +UseTab: Never +AlignConsecutiveDeclarations: + Enabled: true + AcrossEmptyLines: true + AcrossComments: true +AlignArrayOfStructures: Right +AlignConsecutiveMacros: + Enabled: true + AcrossEmptyLines: true +BraceWrapping: + AfterFunction: false + AfterStruct: false + AfterEnum: false + AfterControlStatement: false + BeforeElse: true + BeforeWhile: true +ColumnLimit: 120 diff --git a/.github/workflows/make.yml b/.github/workflows/make.yml index f8d7c8a..4265274 100644 --- a/.github/workflows/make.yml +++ b/.github/workflows/make.yml @@ -17,6 +17,9 @@ jobs: - name: Build shell: bash run: make -C native + - name: Build + shell: bash + run: make -C native check-format - name: Test shell: bash run: make -C native test diff --git a/native/Makefile b/native/Makefile index 03d510f..1cd2de3 100644 --- a/native/Makefile +++ b/native/Makefile @@ -71,8 +71,10 @@ fuzz: $(ODIR)/fuzzmain mkdir -p test/fuzz_output test/fuzz_queue AFL_SKIP_CPUFREQ=1 afl-fuzz -i test/fuzz_input -o test/fuzz_output $(ODIR)/fuzzmain test/fuzz_queue - +check-format: $(DEPS) $(wildcard test/test*.c) + clang-format --dry-run --Werror $^ -.PHONY: clean grind coverage fuzz test install +.PHONY: clean grind coverage fuzz test install check-format test: $(tests_ok) diff --git a/native/buffer.c b/native/buffer.c index bfd80df..08e189a 100644 --- a/native/buffer.c +++ b/native/buffer.c @@ -15,16 +15,15 @@ #ifndef FILE_LIBBUFFER_SEEN #define FILE_LIBBUFFER_SEEN - -#include #include +#include // Outputs non-printable characters as octal, which allows the resulting // string to be a valid C-string constant. -void printbuf(char* c, int n) { +void printbuf(char *c, int n) { printf("unsigned char* buf=\""); for (int i = 0; i < n; i++) { - switch (c[i]) { + switch (c[i]) { case '\n': printf("\\n"); @@ -44,14 +43,13 @@ void printbuf(char* c, int n) { } else { printf("%c", c[i]); } - break; - } + break; + } } printf("\"\n"); } - -char* formatbuf(char* buf, int sz) { +char *formatbuf(char *buf, int sz) { // nicely format a buffer to hex/ascii with a length offset. each 16 bytes // or part thereof of the input take 76 bytes include newline character. caller // must free() the buffer. @@ -62,24 +60,25 @@ char* formatbuf(char* buf, int sz) { // 00000030 69 63 65 90 00 00 28 41 ice···(A // // ----8--- -------------------------48--------------------- --------17-------\n - const char* hexd = "0123456789abcdef"; - int lines = (sz + (16-1)) / 16; // sz/16 rounded up! - if (lines == 0) lines++; - int osz = lines * 76 + 1; // +trailing null - char* r = malloc(osz); + const char *hexd = "0123456789abcdef"; + int lines = (sz + (16 - 1)) / 16; // sz/16 rounded up! + if (lines == 0) + lines++; + int osz = lines * 76 + 1; // +trailing null + char *r = malloc(osz); for (int i = 0; i < lines; i++) { // this provides our trailing null! - sprintf(r+i*76, "%08x %48s %17s\n", i*16, "", ""); - for (int j = i*16; j < sz && j < (i+1)*16; j++) { + sprintf(r + i * 76, "%08x %48s %17s\n", i * 16, "", ""); + for (int j = i * 16; j < sz && j < (i + 1) * 16; j++) { int c = j % 16; int e = c > 7 ? 1 : 0; - r[i*76+9 +c*3+e] = hexd[(buf[j] >> 4) & 0x0F]; // upper nibble - r[i*76+10+c*3+e] = hexd[buf[j] & 0x0F]; // lower nibble + r[i * 76 + 9 + c * 3 + e] = hexd[(buf[j] >> 4) & 0x0F]; // upper nibble + r[i * 76 + 10 + c * 3 + e] = hexd[buf[j] & 0x0F]; // lower nibble // printable ascii? if (buf[j] >= 32 && buf[j] < 127) { - r[i*76+9+49+c+e] = buf[j]; + r[i * 76 + 9 + 49 + c + e] = buf[j]; } else { - r[i*76+9+49+c+e] = '.'; + r[i * 76 + 9 + 49 + c + e] = '.'; } } } diff --git a/native/buffer.h b/native/buffer.h index fdf1a67..2de8665 100644 --- a/native/buffer.h +++ b/native/buffer.h @@ -15,7 +15,7 @@ #ifndef FILE_LIBBUFFER_SEEN #define FILE_LIBBUFFER_SEEN -void printbuf(char* c, int n); -char* formatbuf(char* buf, int sz); +void printbuf(char *c, int n); +char *formatbuf(char *buf, int sz); #endif diff --git a/native/fuzzmain.c b/native/fuzzmain.c index 3bf00e9..7b62f5a 100644 --- a/native/fuzzmain.c +++ b/native/fuzzmain.c @@ -12,31 +12,30 @@ // See the License for the specific language governing permissions and // limitations under the License. - #define _GNU_SOURCE #define __STDC_FORMAT_MACROS #include #define KXVER 3 +#include "k.h" +#include "libchronicle.h" +#include "mock_k.h" +#include "serdes_k.h" +#include +#include +#include +#include #include #include -#include #include -#include #include -#include -#include -#include "k.h" -#include "mock_k.h" -#include "serdes_k.h" -#include "libchronicle.h" // This is a stand-alone tool for replaying a queue for use with fuzzer int print_data = 0; int print_meta = 0; -int print_msg(void* ctx, uint64_t index, void* msg) { +int print_msg(void *ctx, uint64_t index, void *msg) { K x = (K)msg; if (print_data) { printf("[%" PRIu64 "] KC ", index); @@ -53,7 +52,9 @@ uint32_t xorshift128(uint32_t state[static 4]) { uint32_t s, t = state[3]; t ^= t << 11; t ^= t >> 8; - state[3] = state[2]; state[2] = state[1]; state[1] = s = state[0]; + state[3] = state[2]; + state[2] = state[1]; + state[1] = s = state[0]; t ^= s; t ^= s >> 19; state[0] = t; @@ -68,13 +69,13 @@ uint32_t xorshift128(uint32_t state[static 4]) { // Reported but unresolved https://www.mail-archive.com/kde-bugs-dist@kde.org/msg209613.html static inline uint64_t rdtsc(void) { uint32_t a, d; - __asm__ __volatile__("rdtscp" : "=a" (a), "=d" (d)); - return (((uint64_t) d << 32) | a); + __asm__ __volatile__("rdtscp" : "=a"(a), "=d"(d)); + return (((uint64_t)d << 32) | a); } -int c_mkstemp(char* pattern) { +int c_mkstemp(char *pattern) { uint32_t xor_state[4]; xor_state[0] = xor_state[1] = xor_state[2] = xor_state[3] = (uint32_t)rdtsc(); - char* rep = strstr(pattern, "XXXXXX"); + char *rep = strstr(pattern, "XXXXXX"); if (rep) { for (int i = 0; i < 6; i++) { uint8_t incre = xorshift128(xor_state) & 0x1F; @@ -93,11 +94,11 @@ int c_mkstemp(char* pattern) { int main(const int argc, char **argv) { int c; opterr = 0; - int verboseflag = 0; - FILE * fuzzfid = NULL; + int verboseflag = 0; + FILE *fuzzfid = NULL; while ((c = getopt(argc, argv, "dmv:")) != -1) - switch (c) { + switch (c) { case 'd': print_data = 1; break; @@ -113,7 +114,7 @@ int main(const int argc, char **argv) { default: fprintf(stderr, "%c??", c); exit(3); - } + } if (optind + 2 > argc) { printf("Missing mandatory argument.\n Expected: %s [-d] [-m] [-v] QUEUE FUZZFILE\n", argv[0]); @@ -128,68 +129,70 @@ int main(const int argc, char **argv) { printf(" the second `bytes` appends that many random bytes as a new entry to the queue\n"); printf(" As the script is played, the random number generator seed, recieved index and byte count\n"); printf(" are written to a log. This is then re-opened to verify the data in the queue matches.\n"); - + exit(1); } // mandatory arguments - char* dir = argv[optind]; - char* fuzz = argv[optind+1]; + char *dir = argv[optind]; + char *fuzz = argv[optind + 1]; fuzzfid = stdin; if (strcmp(fuzz, "-") == 0) { fprintf(stderr, "fuzzing from stdin\n"); } else { fuzzfid = fopen(fuzz, "r"); - if (fuzzfid == NULL) { + if (fuzzfid == NULL) { fprintf(stderr, "Unable to open %s\n", fuzz); exit(4); } } - queue_t* queue = chronicle_init(dir); + queue_t *queue = chronicle_init(dir); chronicle_set_decoder(queue, &parse_kx, &free_kx); chronicle_set_encoder(queue, &sizeof_kx, &append_kx); chronicle_set_version(queue, 5); chronicle_set_roll_scheme(queue, "FAST_HOURLY"); chronicle_set_create(queue, 1); - if (chronicle_open(queue) != 0) exit(-1); - - tailer_t* tailer = chronicle_tailer(queue, print_msg, NULL, 0); + if (chronicle_open(queue) != 0) + exit(-1); + + tailer_t *tailer = chronicle_tailer(queue, print_msg, NULL, 0); chronicle_peek_queue(queue); if (verboseflag) { chronicle_debug(); } - - size_t linecap = 0; - ssize_t linelen = 0; - char *msgp = NULL; - char *parsep = NULL; - int line = 0; + size_t linecap = 0; + ssize_t linelen = 0; + char *msgp = NULL; + char *parsep = NULL; + + int line = 0; long long bytes; long long time; - uint64_t clock = 0; - uint64_t index = 0; + uint64_t clock = 0; + uint64_t index = 0; - uint32_t xor_state[4]; + uint32_t xor_state[4]; - char* tmpfile = strdup("/tmp/shmmain.XXXXXX"); - int tmpfid = c_mkstemp(tmpfile); + char *tmpfile = strdup("/tmp/shmmain.XXXXXX"); + int tmpfid = c_mkstemp(tmpfile); printf("logging fuzz expectations to %s fid %d msg %s\n", tmpfile, tmpfid, strerror(errno)); - FILE* tmp = fdopen(tmpfid, "w+"); + FILE *tmp = fdopen(tmpfid, "w+"); printf(" tmp is %p %s\n", tmp, strerror(errno)); while ((linelen = getline(&msgp, &linecap, fuzzfid)) > 0) { line++; parsep = msgp; time = strtoll(parsep, &parsep, 0); - if (*parsep == ' ') parsep++; + if (*parsep == ' ') + parsep++; bytes = strtoll(parsep, &parsep, 0); printf(" FUZ: %lld millis, %lld bytes\n", time, bytes); clock += time; - xor_state[0] = xor_state[1] = xor_state[2] = xor_state[3] = (uint32_t)line+1; + xor_state[0] = xor_state[1] = xor_state[2] = xor_state[3] = (uint32_t)line + 1; K x = ktn(KC, bytes); for (long long b = 0; b < bytes; b++) { kG(x)[b] = (uint8_t)xorshift128(xor_state); @@ -199,11 +202,14 @@ int main(const int argc, char **argv) { r0(x); } // we're done parsing input, now replay checking using temp file - //fclose(tmp); - //close(tmpfid); + // fclose(tmp); + // close(tmpfid); fflush(tmp); - if (fseek(tmp, 0L, SEEK_SET) != 0) { printf("abort not fseek"); abort(); }; + if (fseek(tmp, 0L, SEEK_SET) != 0) { + printf("abort not fseek"); + abort(); + }; int rline = 0; int r; @@ -216,7 +222,7 @@ int main(const int argc, char **argv) { // TODO: verify!! } else if (rline != line) { - printf ("error, %s r=%d at line %d!\n\n", tmpfile, r, rline); + printf("error, %s r=%d at line %d!\n\n", tmpfile, r, rline); } rline++; } while (r != EOF); @@ -225,7 +231,8 @@ int main(const int argc, char **argv) { // unlink(tmpfile); free(tmpfile); - if (msgp) free(msgp); + if (msgp) + free(msgp); printf("parse finished\n"); exit(rline == line ? 0 : 5); // exit with fail diff --git a/native/libchronicle.c b/native/libchronicle.c index f378cc2..0fe7e25 100644 --- a/native/libchronicle.c +++ b/native/libchronicle.c @@ -17,21 +17,21 @@ #define __STDC_FORMAT_MACROS #include +#include +#include +#include #include #include -#include -#include #include -#include -#include #include #include #include +#include #include -#include +#include -#include "wire.h" #include "buffer.h" +#include "wire.h" /** * Implementation notes @@ -80,24 +80,23 @@ // MetaDataKeys `header`index2index`index`roll // error handling -const char* cerr_msg; +const char *cerr_msg; // error handling - trigger -int chronicle_err(const char* msg) { +int chronicle_err(const char *msg) { printf("chronicle error: '%s\n", msg); errno = -1; cerr_msg = msg; return -1; } -void* chronicle_perr(const char* msg) { +void *chronicle_perr(const char *msg) { chronicle_err(msg); - return (void*)NULL; + return (void *)NULL; } // error handling - retrieve -const char* chronicle_strerror() { - return cerr_msg; -} +const char *chronicle_strerror() { return cerr_msg; } -const char* tailer_state_messages[] = {"AWAITING_ENTRY", "BUSY", "AWAITING_QUEUEFILE", "E_STAT", "E_MMAP", "PEEK?", "EXTEND_FAIL", "COLLECTED"}; +const char *tailer_state_messages[] = {"AWAITING_ENTRY", "BUSY", "AWAITING_QUEUEFILE", "E_STAT", + "E_MMAP", "PEEK?", "EXTEND_FAIL", "COLLECTED"}; // structures @@ -108,165 +107,160 @@ typedef struct { } dirlist_fields_t; struct tailer { - uint64_t dispatch_after; // for resume support - tailstate_t state; - cdispatch_f dispatcher; - void* dispatch_ctx; + uint64_t dispatch_after; // for resume support + tailstate_t state; + cdispatch_f dispatcher; + void *dispatch_ctx; // to support the 'collect' operation to wait and return next item, ignoring callback - collected_t* collect; + collected_t *collect; - int mmap_protection; // PROT_READ etc. + int mmap_protection; // PROT_READ etc. // currently open queue file - uint64_t qf_cycle_open; - char* qf_fn; - struct stat qf_statbuf; - int qf_fd; + uint64_t qf_cycle_open; + char *qf_fn; + struct stat qf_statbuf; + int qf_fd; - uint64_t qf_tip; // byte position of the next header, or zero if unknown - uint64_t qf_index; // seqnum of the header pointed to by qf_tip + uint64_t qf_tip; // byte position of the next header, or zero if unknown + uint64_t qf_index; // seqnum of the header pointed to by qf_tip // currently mapped region: buffer, offset (from 0 in file), size - unsigned char* qf_buf; - uint64_t qf_mmapoff; - uint64_t qf_mmapsz; + unsigned char *qf_buf; + uint64_t qf_mmapoff; + uint64_t qf_mmapsz; - struct queue* queue; + struct queue *queue; - struct tailer* next; - struct tailer* prev; + struct tailer *next; + struct tailer *prev; }; struct queue { - char* dirname; - uint blocksize; - uint8_t version; - uint8_t create; + char *dirname; + uint blocksize; + uint8_t version; + uint8_t create; // directory-listing.cq4t - char* dirlist_name; - int dirlist_fd; - struct stat dirlist_statbuf; - unsigned char* dirlist; // mmap base - dirlist_fields_t dirlist_fields; + char *dirlist_name; + int dirlist_fd; + struct stat dirlist_statbuf; + unsigned char *dirlist; // mmap base + dirlist_fields_t dirlist_fields; - char* queuefile_pattern; - glob_t queuefile_glob; // last glob of data files, refreshed by poll on modcount + char *queuefile_pattern; + glob_t queuefile_glob; // last glob of data files, refreshed by poll on modcount // values observed from directory-listing poll - uint64_t highest_cycle; - uint64_t lowest_cycle; - uint64_t modcount; + uint64_t highest_cycle; + uint64_t lowest_cycle; + uint64_t modcount; // roll config populated from (any) queuefile header or set on creation - int roll_length; - int roll_epoch; - char* roll_format; - char* roll_name; - char* roll_strftime; - int index_count; - int index_spacing; + int roll_length; + int roll_epoch; + char *roll_format; + char *roll_name; + char *roll_strftime; + int index_count; + int index_spacing; - int cycle_shift; - uint64_t seqnum_mask; + int cycle_shift; + uint64_t seqnum_mask; - cparse_f parser; - cparsefree_f parser_free; - csizeof_f append_sizeof; - cappend_f append_write; + cparse_f parser; + cparsefree_f parser_free; + csizeof_f append_sizeof; + cappend_f append_write; - tailer_t* tailers; + tailer_t *tailers; // the appender is a shared tailer, polled by append[], with writing logic // and no callback to user code for events - tailer_t* appender; + tailer_t *appender; - struct queue* next; + struct queue *next; }; +typedef enum { QB_AWAITING_ENTRY, QB_BUSY, QB_REACHED_EOF, QB_NEED_EXTEND, QB_NULL_ITEM, QB_COLLECTED } parseqb_state_t; -typedef enum {QB_AWAITING_ENTRY, QB_BUSY, QB_REACHED_EOF, QB_NEED_EXTEND, QB_NULL_ITEM, QB_COLLECTED} parseqb_state_t; - -typedef parseqb_state_t (*datacallback_f)(unsigned char*,int,uint64_t,void* userdata); - +typedef parseqb_state_t (*datacallback_f)(unsigned char *, int, uint64_t, void *userdata); // paramaters that control behavior, not exposed for modification -uint32_t patch_cycles = 3; -long int qf_disk_sz = 83754496L; +uint32_t patch_cycles = 3; +long int qf_disk_sz = 83754496L; // globals -int debug = 0; -uint32_t pid_header = 0; -queue_t* queue_head = NULL; - +int debug = 0; +uint32_t pid_header = 0; +queue_t *queue_head = NULL; // forward declarations that are not part of the public api -void parse_dirlist(queue_t*); -void parse_queuefile_meta(unsigned char*, int, queue_t*); -void parse_queuefile_data(unsigned char*, int, queue_t*, tailer_t*, uint64_t); -int queuefile_init(char*, queue_t*); -int directory_listing_reopen(queue_t*, int, int); -int directory_listing_init(queue_t*, uint64_t cycle); +void parse_dirlist(queue_t *); +void parse_queuefile_meta(unsigned char *, int, queue_t *); +void parse_queuefile_data(unsigned char *, int, queue_t *, tailer_t *, uint64_t); +int queuefile_init(char *, queue_t *); +int directory_listing_reopen(queue_t *, int, int); +int directory_listing_init(queue_t *, uint64_t cycle); -long chronicle_clock_ms(queue_t*); -uint64_t chronicle_cycle_from_ms(queue_t*, long); -int chronicle_peek_queue_tailer(queue_t*, tailer_t*); +long chronicle_clock_ms(queue_t *); +uint64_t chronicle_cycle_from_ms(queue_t *, long); +int chronicle_peek_queue_tailer(queue_t *, tailer_t *); // compare and swap, 32 bits, addressed by a 64bit pointer static inline uint32_t lock_cmpxchgl(unsigned char *mem, uint32_t newval, uint32_t oldval) { - __typeof (*mem) ret; - __asm __volatile ("lock; cmpxchgl %2, %1" - : "=a" (ret), "=m" (*mem) - : "r" (newval), "m" (*mem), "0" (oldval)); - return (uint32_t) ret; + __typeof(*mem) ret; + __asm __volatile("lock; cmpxchgl %2, %1" : "=a"(ret), "=m"(*mem) : "r"(newval), "m"(*mem), "0"(oldval)); + return (uint32_t)ret; } -static inline uint32_t lock_xadd(unsigned char* mem, uint32_t val) { +static inline uint32_t lock_xadd(unsigned char *mem, uint32_t val) { __asm__ volatile("lock; xaddl %0, %1" - : "+r" (val), "+m" (*mem) // input+output - : // No input-only - : "memory" - ); + : "+r"(val), "+m"(*mem) // input+output + : // No input-only + : "memory"); return (uint32_t)val; } -char* chronicle_get_cycle_fn(queue_t* queue, int cycle) { +char *chronicle_get_cycle_fn(queue_t *queue, int cycle) { // TODO: replace with https://ideone.com/7BADb as gmtime_r leaks // time_t aka long. seconds since midnight 1970 - time_t rawtime = (time_t) cycle * (queue->roll_length / 1000); + time_t rawtime = (time_t)cycle * (queue->roll_length / 1000); struct tm info; gmtime_r(&rawtime, &info); // format datetime component using prebuilt pattern - char* strftime_buf = strdup(queue->roll_format); - strftime(strftime_buf, strlen(strftime_buf)+1, queue->roll_strftime, &info); + char *strftime_buf = strdup(queue->roll_format); + strftime(strftime_buf, strlen(strftime_buf) + 1, queue->roll_strftime, &info); // join with dirname and file suffix - char* fnbuf; - int bufsz = asprintf(&fnbuf, "%s/%s.cq4", queue->dirname, strftime_buf); + char *fnbuf; + int bufsz = asprintf(&fnbuf, "%s/%s.cq4", queue->dirname, strftime_buf); free(strftime_buf); - if (bufsz < 0) return NULL; + if (bufsz < 0) + return NULL; return fnbuf; } -void queue_double_blocksize(queue_t* queue) { +void queue_double_blocksize(queue_t *queue) { uint new_blocksize = queue->blocksize << 1; printf("shmipc: doubling blocksize from %x to %x\n", queue->blocksize, new_blocksize); queue->blocksize = new_blocksize; } - -queue_t* chronicle_init(char* dir) { - char* debug_env = getenv("SHMIPC_DEBUG"); +queue_t *chronicle_init(char *dir) { + char *debug_env = getenv("SHMIPC_DEBUG"); debug = (debug_env == NULL) ? 0 : strcmp(debug_env, "1") == 0; - char* wiretrace_env = getenv("SHMIPC_WIRETRACE"); + char *wiretrace_env = getenv("SHMIPC_WIRETRACE"); wire_trace = (wiretrace_env == NULL) ? 0 : strcmp(wiretrace_env, "1") == 0; pid_header = (getpid() & HD_MASK_LENGTH); // allocate struct, we'll link if all checks pass - queue_t* queue = malloc(sizeof(queue_t)); - if (queue == NULL) return chronicle_perr("m fail"); + queue_t *queue = malloc(sizeof(queue_t)); + if (queue == NULL) + return chronicle_perr("m fail"); bzero(queue, sizeof(queue_t)); // wire up default 'text' parsers for data segments @@ -276,7 +270,7 @@ queue_t* chronicle_init(char* dir) { // unsafe to use ref here in case caller doesn't keep in scope, so dup queue->dirname = strdup(dir); - queue->blocksize = 1024*1024; // must be a power of two (single 1 bit) + queue->blocksize = 1024 * 1024; // must be a power of two (single 1 bit) queue->roll_epoch = -1; // Good to use @@ -286,9 +280,9 @@ queue_t* chronicle_init(char* dir) { return queue; } -int chronicle_readable(char* dirname, char* suffix) { - char* dirlist_name; - int dirlist_fd; +int chronicle_readable(char *dirname, char *suffix) { + char *dirlist_name; + int dirlist_fd; // probe v4 directory-listing.cq4t asprintf(&dirlist_name, "%s/%s", dirname, suffix); @@ -301,20 +295,25 @@ int chronicle_readable(char* dirname, char* suffix) { return 0; } -int chronicle_version_detect(queue_t* queue) { - if (chronicle_readable(queue->dirname, "directory-listing.cq4t")) return 4; - if (chronicle_readable(queue->dirname, "metadata.cq4t")) return 5; +int chronicle_version_detect(queue_t *queue) { + if (chronicle_readable(queue->dirname, "directory-listing.cq4t")) + return 4; + if (chronicle_readable(queue->dirname, "metadata.cq4t")) + return 5; return 0; } -int chronicle_open(queue_t* queue) { +int chronicle_open(queue_t *queue) { - if (debug) printf("shmipc: opening dir %s\n", queue->dirname); + if (debug) + printf("shmipc: opening dir %s\n", queue->dirname); // Is this a directory struct stat statbuf; - if (stat(queue->dirname, &statbuf) != 0) return chronicle_err("dir stat fail"); - if (!S_ISDIR(statbuf.st_mode)) return chronicle_err("dir is not a directory"); + if (stat(queue->dirname, &statbuf) != 0) + return chronicle_err("dir stat fail"); + if (!S_ISDIR(statbuf.st_mode)) + return chronicle_err("dir is not a directory"); // autodetect version int auto_version = chronicle_version_detect(queue); @@ -329,16 +328,20 @@ int chronicle_open(queue_t* queue) { glob(queue->queuefile_pattern, GLOB_ERR, NULL, g); if (debug) { printf("shmipc: glob %zu queue files found\n", g->gl_pathc); - for (int i = 0; i < g->gl_pathc;i++) { + for (int i = 0; i < g->gl_pathc; i++) { printf(" %s\n", g->gl_pathv[i]); } } if (auto_version == 0) { - if (queue->create == 0) return chronicle_err("queue should exist (no permission to create), but version detect failed"); - if (queue->version == 0) return chronicle_err("queue create requires chronicle_set_version()"); - if (g->gl_pathc != 0) return chronicle_err("queue create requires empty destination directory"); - if (queue->roll_name == NULL) return chronicle_err("queue create requires chronicle_set_roll_scheme()"); + if (queue->create == 0) + return chronicle_err("queue should exist (no permission to create), but version detect failed"); + if (queue->version == 0) + return chronicle_err("queue create requires chronicle_set_version()"); + if (g->gl_pathc != 0) + return chronicle_err("queue create requires empty destination directory"); + if (queue->roll_name == NULL) + return chronicle_err("queue create requires chronicle_set_roll_scheme()"); } else if (queue->version != 0 && queue->version != auto_version) { return chronicle_err("queue version detected does not match expected set via. chronicle_set_version()"); @@ -347,16 +350,19 @@ int chronicle_open(queue_t* queue) { } // populate dirlist - asprintf(&queue->dirlist_name, queue->version == 4 ? "%s/directory-listing.cq4t" : "%s/metadata.cq4t", queue->dirname); + asprintf(&queue->dirlist_name, queue->version == 4 ? "%s/directory-listing.cq4t" : "%s/metadata.cq4t", + queue->dirname); if (queue->create && auto_version == 0) { uint64_t cycle = chronicle_cycle_from_ms(queue, chronicle_clock_ms(queue)); - int rc = directory_listing_init(queue, cycle); - if(rc != 0) return rc; + int rc = directory_listing_init(queue, cycle); + if (rc != 0) + return rc; } // parse the directory listing and sanity-check all required fields exist int rc = directory_listing_reopen(queue, O_RDONLY, PROT_READ); - if(rc != 0) return rc; + if (rc != 0) + return rc; if (queue->version == 4) { // For v4, we need to read a 'queue' header from any one of the datafiles to get the @@ -364,15 +370,16 @@ int chronicle_open(queue_t* queue) { // yet, so this needs to use the directory listing. if (g->gl_pathc < 1) { - return chronicle_err("V4 and no queue files found so cannot initialise. Call chronicle_set_create(queue, 1) to allow queue creation"); + return chronicle_err("V4 and no queue files found so cannot initialise. Call chronicle_set_create(queue, " + "1) to allow queue creation"); } - int queuefile_fd; - struct stat queuefile_statbuf; - uint64_t queuefile_extent; - unsigned char* queuefile_buf; + int queuefile_fd; + struct stat queuefile_statbuf; + uint64_t queuefile_extent; + unsigned char *queuefile_buf; - char* fn = queue->queuefile_glob.gl_pathv[0]; + char *fn = queue->queuefile_glob.gl_pathv[0]; // find length of queuefile and mmap if ((queuefile_fd = open(fn, O_RDONLY)) < 0) { return chronicle_err("qfi open"); @@ -387,7 +394,8 @@ int chronicle_open(queue_t* queue) { return chronicle_err("qfi mmap fail"); // we don't need a data-parser at this stage as only need values from the header - if (debug) printf("shmipc: parsing queuefile %s 0..%" PRIu64 "\n", fn, queuefile_extent); + if (debug) + printf("shmipc: parsing queuefile %s 0..%" PRIu64 "\n", fn, queuefile_extent); parse_queuefile_meta(queuefile_buf, queuefile_extent, queue); // close queuefile @@ -397,15 +405,18 @@ int chronicle_open(queue_t* queue) { // check we loaded roll settings from queuefile or metadata // Note these might be the settings we wrote during create - if (queue->roll_format == 0) return chronicle_err("qfi roll_format fail"); - if (queue->roll_length == 0) return chronicle_err("qfi roll_length fail"); - if (queue->roll_epoch == -1) return chronicle_err("qfi roll_epoch fail"); + if (queue->roll_format == 0) + return chronicle_err("qfi roll_format fail"); + if (queue->roll_length == 0) + return chronicle_err("qfi roll_length fail"); + if (queue->roll_epoch == -1) + return chronicle_err("qfi roll_epoch fail"); if (chronicle_set_roll_dateformat(queue, queue->roll_format) != 0) { printf("roll format detected %s\n", queue->roll_format); return chronicle_err("detected roll_format is not recognised"); } - //if (queue->index_count == 0) return chronicle_err("qfi index_count fail"); - //if (queue->index_spacing == 0) return chronicle_err("qfi index_spacing fail"); + // if (queue->index_count == 0) return chronicle_err("qfi index_count fail"); + // if (queue->index_spacing == 0) return chronicle_err("qfi index_spacing fail"); queue->cycle_shift = 32; queue->seqnum_mask = 0x00000000FFFFFFFF; @@ -415,75 +426,80 @@ int chronicle_open(queue_t* queue) { // avoids a tailer registration before we have a minimum cycle chronicle_peek_queue(queue); - if (debug) printf("shmipc: chronicle_open() OK\n"); + if (debug) + printf("shmipc: chronicle_open() OK\n"); return 0; } void chronicle_set_decoder(queue_t *queue, cparse_f parser, cparsefree_f parser_free) { - if (debug & !parser) printf("chronicle: setting NULL parser"); + if (debug & !parser) + printf("chronicle: setting NULL parser"); queue->parser = parser; queue->parser_free = parser_free; } void chronicle_set_encoder(queue_t *queue, csizeof_f append_sizeof, cappend_f append_write) { - if (debug & !append_sizeof) printf("chronicle: setting NULL append_sizeof"); - if (debug & !append_write) printf("chronicle: setting NULL append_write"); + if (debug & !append_sizeof) + printf("chronicle: setting NULL append_sizeof"); + if (debug & !append_write) + printf("chronicle: setting NULL append_write"); queue->append_sizeof = append_sizeof; queue->append_write = append_write; } struct ROLL_SCHEME chronicle_roll_schemes[] = { // in use by cq5 - {"FIVE_MINUTELY", "yyyyMMdd-HHmm'V'", 5*60, 2<<10, 256}, - {"TEN_MINUTELY", "yyyyMMdd-HHmm'X'", 10*60, 2<<10, 256}, - {"TWENTY_MINUTELY", "yyyyMMdd-HHmm'XX'", 20*60, 2<<10, 256}, - {"HALF_HOURLY", "yyyyMMdd-HHmm'H'", 30*60, 2<<10, 256}, - {"FAST_HOURLY", "yyyyMMdd-HH'F'", 60*60, 4<<10, 256}, - {"TWO_HOURLY", "yyyyMMdd-HH'II'", 2*60*60, 4<<10, 256}, - {"FOUR_HOURLY", "yyyyMMdd-HH'IV'", 4*60*60, 4<<10, 256}, - {"SIX_HOURLY", "yyyyMMdd-HH'VI'", 6*60*60, 4<<10, 256}, - {"FAST_DAILY", "yyyyMMdd'F'", 24*60*60, 4<<10, 256}, + { "FIVE_MINUTELY", "yyyyMMdd-HHmm'V'", 5 * 60, 2 << 10, 256}, + { "TEN_MINUTELY", "yyyyMMdd-HHmm'X'", 10 * 60, 2 << 10, 256}, + { "TWENTY_MINUTELY", "yyyyMMdd-HHmm'XX'", 20 * 60, 2 << 10, 256}, + { "HALF_HOURLY", "yyyyMMdd-HHmm'H'", 30 * 60, 2 << 10, 256}, + { "FAST_HOURLY", "yyyyMMdd-HH'F'", 60 * 60, 4 << 10, 256}, + { "TWO_HOURLY", "yyyyMMdd-HH'II'", 2 * 60 * 60, 4 << 10, 256}, + { "FOUR_HOURLY", "yyyyMMdd-HH'IV'", 4 * 60 * 60, 4 << 10, 256}, + { "SIX_HOURLY", "yyyyMMdd-HH'VI'", 6 * 60 * 60, 4 << 10, 256}, + { "FAST_DAILY", "yyyyMMdd'F'", 24 * 60 * 60, 4 << 10, 256}, // used historically by cq4 - {"MINUTELY", "yyyyMMdd-HHmm", 60, 2<<10, 16}, - {"HOURLY", "yyyyMMdd-HH", 60*60, 4<<10, 16}, - {"DAILY", "yyyyMMdd", 24*60*60, 8<<10, 64}, + { "MINUTELY", "yyyyMMdd-HHmm", 60, 2 << 10, 16}, + { "HOURLY", "yyyyMMdd-HH", 60 * 60, 4 << 10, 16}, + { "DAILY", "yyyyMMdd", 24 * 60 * 60, 8 << 10, 64}, // minimal rolls with resulting large queue files - {"LARGE_HOURLY", "yyyyMMdd-HH'L'", 60*60, 8<<10, 64}, - {"LARGE_DAILY", "yyyyMMdd'L'", 24*60*60, 32<<10, 128}, - {"XLARGE_DAILY", "yyyyMMdd'X'", 24*60*60, 32<<10, 256}, - {"HUGE_DAILY", "yyyyMMdd'H'", 24*60*60, 32<<10, 1024}, + { "LARGE_HOURLY", "yyyyMMdd-HH'L'", 60 * 60, 8 << 10, 64}, + { "LARGE_DAILY", "yyyyMMdd'L'", 24 * 60 * 60, 32 << 10, 128}, + { "XLARGE_DAILY", "yyyyMMdd'X'", 24 * 60 * 60, 32 << 10, 256}, + { "HUGE_DAILY", "yyyyMMdd'H'", 24 * 60 * 60, 32 << 10, 1024}, // for tests and benchmarking with nearly no indexing - {"SMALL_DAILY", "yyyyMMdd'S'", 24*60*60, 8<<10, 8}, - {"LARGE_HOURLY_SPARSE", "yyyyMMdd-HH'LS'", 60*60, 4<<10, 1024}, - {"LARGE_HOURLY_XSPARSE", "yyyyMMdd-HH'LX'", 60*60, 2<<10, 1<<20}, - {"HUGE_DAILY_XSPARSE", "yyyyMMdd'HX'", 24*60*60, 16<<10, 1<<20}, + { "SMALL_DAILY", "yyyyMMdd'S'", 24 * 60 * 60, 8 << 10, 8}, + { "LARGE_HOURLY_SPARSE", "yyyyMMdd-HH'LS'", 60 * 60, 4 << 10, 1024}, + {"LARGE_HOURLY_XSPARSE", "yyyyMMdd-HH'LX'", 60 * 60, 2 << 10, 1 << 20}, + { "HUGE_DAILY_XSPARSE", "yyyyMMdd'HX'", 24 * 60 * 60, 16 << 10, 1 << 20}, // for tests to create smaller queue files - {"TEST_SECONDLY", "yyyyMMdd-HHmmss'T'", 1, 32<<10, 4}, - {"TEST4_SECONDLY", "yyyyMMdd-HHmmss'T4'", 1, 32, 4}, - {"TEST_HOURLY", "yyyyMMdd-HH'T'", 60*60, 16, 4}, - {"TEST_DAILY", "yyyyMMdd'T1'", 24*60*60, 8, 1}, - {"TEST2_DAILY", "yyyyMMdd'T2'", 24*60*60, 16, 2}, - {"TEST4_DAILY", "yyyyMMdd'T4'", 24*60*60, 32, 4}, - {"TEST8_DAILY", "yyyyMMdd'T8'", 24*60*60, 128, 8}, + { "TEST_SECONDLY", "yyyyMMdd-HHmmss'T'", 1, 32 << 10, 4}, + { "TEST4_SECONDLY", "yyyyMMdd-HHmmss'T4'", 1, 32, 4}, + { "TEST_HOURLY", "yyyyMMdd-HH'T'", 60 * 60, 16, 4}, + { "TEST_DAILY", "yyyyMMdd'T1'", 24 * 60 * 60, 8, 1}, + { "TEST2_DAILY", "yyyyMMdd'T2'", 24 * 60 * 60, 16, 2}, + { "TEST4_DAILY", "yyyyMMdd'T4'", 24 * 60 * 60, 32, 4}, + { "TEST8_DAILY", "yyyyMMdd'T8'", 24 * 60 * 60, 128, 8}, }; -void chronicle_apply_roll_scheme(queue_t* queue, struct ROLL_SCHEME x) { - if (debug) printf("chronicle: chronicle_set_roll_scheme applying %s\n", x.name); +void chronicle_apply_roll_scheme(queue_t *queue, struct ROLL_SCHEME x) { + if (debug) + printf("chronicle: chronicle_set_roll_scheme applying %s\n", x.name); free(queue->roll_name); free(queue->roll_format); free(queue->roll_strftime); - queue->roll_name = strdup(x.name); + queue->roll_name = strdup(x.name); queue->roll_format = strdup(x.formatstr); queue->roll_length = x.roll_length_secs * 1000; // remove appostrophe from java's format string and build // the equivelent strftime string together - char* p = strdup(queue->roll_format); - int px = 0; - char* f = queue->roll_format; - int fi = 0; - int inquote = 0; + char *p = strdup(queue->roll_format); + int px = 0; + char *f = queue->roll_format; + int fi = 0; + int inquote = 0; while (fi < strlen(queue->roll_format)) { if (debug) { printf(" rs parser fi=%d px=%d inquote=%d buffer='%s'\n", fi, px, inquote, p); @@ -523,7 +539,8 @@ void chronicle_apply_roll_scheme(queue_t* queue, struct ROLL_SCHEME x) { p[px++] = 'S'; fi += 2; } else { - printf("chronicle: parser conversion of %s exploded at fi=%d px=%d inquote=%d buffer='%s'\n", queue->roll_format, fi, px, inquote, p); + printf("chronicle: parser conversion of %s exploded at fi=%d px=%d inquote=%d buffer='%s'\n", + queue->roll_format, fi, px, inquote, p); return; } } @@ -534,8 +551,8 @@ void chronicle_apply_roll_scheme(queue_t* queue, struct ROLL_SCHEME x) { queue->roll_strftime = p; } -int chronicle_set_roll_scheme(queue_t* queue, char* scheme) { - for (int i = 0; i < sizeof(chronicle_roll_schemes)/sizeof(chronicle_roll_schemes[0]); i++) { +int chronicle_set_roll_scheme(queue_t *queue, char *scheme) { + for (int i = 0; i < sizeof(chronicle_roll_schemes) / sizeof(chronicle_roll_schemes[0]); i++) { struct ROLL_SCHEME x = chronicle_roll_schemes[i]; if (strcmp(scheme, x.name) == 0) { chronicle_apply_roll_scheme(queue, x); @@ -544,16 +561,12 @@ int chronicle_set_roll_scheme(queue_t* queue, char* scheme) { return 0; } -char* chronicle_get_roll_scheme(queue_t* queue) { - return queue->roll_name; -} +char *chronicle_get_roll_scheme(queue_t *queue) { return queue->roll_name; } -char* chronicle_get_roll_format(queue_t* queue) { - return queue->roll_format; -} +char *chronicle_get_roll_format(queue_t *queue) { return queue->roll_format; } -int chronicle_set_roll_dateformat(queue_t* queue, char* dateformat) { - for (int i = 0; i < sizeof(chronicle_roll_schemes)/sizeof(chronicle_roll_schemes[0]); i++) { +int chronicle_set_roll_dateformat(queue_t *queue, char *dateformat) { + for (int i = 0; i < sizeof(chronicle_roll_schemes) / sizeof(chronicle_roll_schemes[0]); i++) { struct ROLL_SCHEME x = chronicle_roll_schemes[i]; if (strcmp(dateformat, x.formatstr) == 0) { chronicle_apply_roll_scheme(queue, x); @@ -563,11 +576,9 @@ int chronicle_set_roll_dateformat(queue_t* queue, char* dateformat) { return -1; } -void chronicle_set_create(queue_t* queue, int create) { - queue->create = create; -} +void chronicle_set_create(queue_t *queue, int create) { queue->create = create; } -void chronicle_set_version(queue_t* queue, int version) { +void chronicle_set_version(queue_t *queue, int version) { if (version == 4) { queue->version = 4; } else if (version == 5) { @@ -575,21 +586,17 @@ void chronicle_set_version(queue_t* queue, int version) { } } -int chronicle_get_version(queue_t* queue) { - return queue->version; -} +int chronicle_get_version(queue_t *queue) { return queue->version; } -long chronicle_clock_ms(queue_t* queue) { +long chronicle_clock_ms(queue_t *queue) { // switch to custom clock etc. // default case struct timeval tv; gettimeofday(&tv, NULL); - return (tv.tv_sec) * 1000 + (tv.tv_usec) / 1000 ; + return (tv.tv_sec) * 1000 + (tv.tv_usec) / 1000; } -uint64_t chronicle_cycle_from_ms(queue_t* queue, long ms) { - return (ms - queue->roll_epoch) / queue->roll_length; -} +uint64_t chronicle_cycle_from_ms(queue_t *queue, long ms) { return (ms - queue->roll_epoch) / queue->roll_length; } // return codes // 0 awaiting at &base @@ -600,42 +607,52 @@ uint64_t chronicle_cycle_from_ms(queue_t* queue, long ms) { // (7 collected value - from parse_data) // if any entries are read the values at basep and indexp are updated // if parse_data returns non-zero, we pause parsing after the current item -// typedef enum {QB_AWAITING_ENTRY, QB_BUSY, QB_REACHED_EOF, QB_NEED_EXTEND, QB_NULL_ITEM, QB_COLLECTED} parseqb_state_t; - -parseqb_state_t parse_queue_block(queue_t *queue, unsigned char** basep, uint64_t *indexp, unsigned char* extent, wirecallbacks_t* hcbs, datacallback_f parse_data, void* userdata) { - uint32_t header; - int sz; - unsigned char* base = *basep; - uint64_t index = *indexp; +// typedef enum {QB_AWAITING_ENTRY, QB_BUSY, QB_REACHED_EOF, QB_NEED_EXTEND, QB_NULL_ITEM, QB_COLLECTED} +// parseqb_state_t; + +parseqb_state_t parse_queue_block(queue_t *queue, unsigned char **basep, uint64_t *indexp, unsigned char *extent, + wirecallbacks_t *hcbs, datacallback_f parse_data, void *userdata) { + uint32_t header; + int sz; + unsigned char *base = *basep; + uint64_t index = *indexp; parseqb_state_t pd = QB_AWAITING_ENTRY; while (pd == QB_AWAITING_ENTRY) { - if (base+4 >= extent) return 3; + if (base + 4 >= extent) + return 3; memcpy(&header, base, sizeof(header)); // relax, fn optimised away // no speculative fetches before the header is read - asm volatile ("mfence" ::: "memory"); + asm volatile("mfence" ::: "memory"); if (header == HD_UNALLOCATED) { - if (debug) printf(" %" PRIu64 " @%p unallocated\n", index, base); + if (debug) + printf(" %" PRIu64 " @%p unallocated\n", index, base); return QB_AWAITING_ENTRY; } else if ((header & HD_MASK_META) == HD_WORKING) { - if (debug) printf(" @%p locked for writing by pid %d\n", base, header & HD_MASK_LENGTH); + if (debug) + printf(" @%p locked for writing by pid %d\n", base, header & HD_MASK_LENGTH); return QB_BUSY; } else if ((header & HD_MASK_META) == HD_METADATA) { sz = (header & HD_MASK_LENGTH); - if (debug) printf(" @%p metadata size %x\n", base, sz); - if (base+4+sz >= extent) return QB_NEED_EXTEND; - wire_parse(base+4, sz, hcbs); + if (debug) + printf(" @%p metadata size %x\n", base, sz); + if (base + 4 + sz >= extent) + return QB_NEED_EXTEND; + wire_parse(base + 4, sz, hcbs); // EventName header // switch to header parser } else if ((header & HD_MASK_META) == HD_EOF) { - if (debug) printf(" @%p EOF\n", base); + if (debug) + printf(" @%p EOF\n", base); return QB_REACHED_EOF; } else { sz = (header & HD_MASK_LENGTH); - if (debug) printf(" %" PRIu64 " @%p data size %x\n", index, base, sz); + if (debug) + printf(" %" PRIu64 " @%p data size %x\n", index, base, sz); if (parse_data) { - if (base+4+sz >= extent) return QB_NEED_EXTEND; - pd = parse_data(base+4, sz, index, userdata); + if (base + 4 + sz >= extent) + return QB_NEED_EXTEND; + pd = parse_data(base + 4, sz, index, userdata); } else { // bail at first data message return QB_NULL_ITEM; @@ -651,22 +668,23 @@ parseqb_state_t parse_queue_block(queue_t *queue, unsigned char** basep, uint64_ } // parse data callback dispatching to wire.h parser -parseqb_state_t parse_wire_data(unsigned char* base, int lim, uint64_t index, void* cbs) { - wire_parse(base, lim, (wirecallbacks_t*)cbs); +parseqb_state_t parse_wire_data(unsigned char *base, int lim, uint64_t index, void *cbs) { + wire_parse(base, lim, (wirecallbacks_t *)cbs); return QB_AWAITING_ENTRY; } - // return AWAITING_ENTRY to continue dispaching, COLLECTED to signal collected item -parseqb_state_t parse_data_cb(unsigned char* base, int lim, uint64_t index, void* userdata) { - tailer_t* tailer = (tailer_t*)userdata; - if (debug) printbuf((char*)base, lim); +parseqb_state_t parse_data_cb(unsigned char *base, int lim, uint64_t index, void *userdata) { + tailer_t *tailer = (tailer_t *)userdata; + if (debug) + printbuf((char *)base, lim); // prep args and fire callback if (index > tailer->dispatch_after) { COBJ msg = tailer->queue->parser(base, lim); if (msg == NULL) { - if (debug) printf("chronicle: caution at index %" PRIu64 " parse function returned NULL, skipping\n", index); + if (debug) + printf("chronicle: caution at index %" PRIu64 " parse function returned NULL, skipping\n", index); return QB_AWAITING_ENTRY; } // if asked to return inline, we skip dispatcher callback, user @@ -687,11 +705,10 @@ parseqb_state_t parse_data_cb(unsigned char* base, int lim, uint64_t index, void return QB_AWAITING_ENTRY; } - -void handle_dirlist_ptr(char* buf, int sz, unsigned char *dptr, wirecallbacks_t* cbs) { +void handle_dirlist_ptr(char *buf, int sz, unsigned char *dptr, wirecallbacks_t *cbs) { // we are preserving *pointers* within the shared directory data page // we keep the underlying mmap for life of queue - queue_t* queue = (queue_t*)cbs->userdata; + queue_t *queue = (queue_t *)cbs->userdata; if (strncmp(buf, "listing.highestCycle", sz) == 0) { queue->dirlist_fields.highest_cycle = dptr; } else if (strncmp(buf, "listing.lowestCycle", sz) == 0) { @@ -701,55 +718,61 @@ void handle_dirlist_ptr(char* buf, int sz, unsigned char *dptr, wirecallbacks_t* } } -void handle_dirlist_uint64(char* buf, int sz, uint64_t data, wirecallbacks_t* cbs){ - queue_t* queue = (queue_t*)cbs->userdata; +void handle_dirlist_uint64(char *buf, int sz, uint64_t data, wirecallbacks_t *cbs) { + queue_t *queue = (queue_t *)cbs->userdata; if (strncmp(buf, "length", sz) == 0) { - if (debug) printf(" v5 roll_length set to %" PRIu64 "\n", data); + if (debug) + printf(" v5 roll_length set to %" PRIu64 "\n", data); queue->roll_length = data; } else if (strncmp(buf, "epoch", sz) == 0) { - if (debug) printf(" v5 roll_epoch set to %" PRIu64 "\n", data); + if (debug) + printf(" v5 roll_epoch set to %" PRIu64 "\n", data); queue->roll_epoch = data; } } -void handle_dirlist_text(char* buf, int sz, char* data, int dsz, wirecallbacks_t* cbs) { - queue_t* queue = (queue_t*)cbs->userdata; +void handle_dirlist_text(char *buf, int sz, char *data, int dsz, wirecallbacks_t *cbs) { + queue_t *queue = (queue_t *)cbs->userdata; if (strncmp(buf, "format", sz) == 0) { - if (debug) printf(" v5 roll_format set to %.*s\n", dsz, data); + if (debug) + printf(" v5 roll_format set to %.*s\n", dsz, data); free(queue->roll_format); queue->roll_format = strndup(data, dsz); } } -void handle_qf_uint64(char* buf, int sz, uint64_t data, wirecallbacks_t* cbs){ - queue_t* queue = (queue_t*)cbs->userdata; +void handle_qf_uint64(char *buf, int sz, uint64_t data, wirecallbacks_t *cbs) { + queue_t *queue = (queue_t *)cbs->userdata; if (strncmp(buf, "length", sz) == 0) { - if (debug) printf(" v4 roll_length set to %" PRIu64 "\n", data); + if (debug) + printf(" v4 roll_length set to %" PRIu64 "\n", data); queue->roll_length = data; } else if (strncmp(buf, "indexCount", sz) == 0) { queue->index_count = data; } else if (strncmp(buf, "indexSpacing", sz) == 0) { queue->index_spacing = data; } else if (strncmp(buf, "epoch", sz) == 0) { - if (debug) printf(" v4 roll_epoch set to %" PRIu64 "\n", data); + if (debug) + printf(" v4 roll_epoch set to %" PRIu64 "\n", data); queue->roll_epoch = data; } } -void handle_qf_text(char* buf, int sz, char* data, int dsz, wirecallbacks_t* cbs) { - queue_t* queue = (queue_t*)cbs->userdata; +void handle_qf_text(char *buf, int sz, char *data, int dsz, wirecallbacks_t *cbs) { + queue_t *queue = (queue_t *)cbs->userdata; if (strncmp(buf, "format", sz) == 0) { - if (debug) printf(" v4 roll_format qf set to %.*s\n", dsz, data); + if (debug) + printf(" v4 roll_format qf set to %.*s\n", dsz, data); free(queue->roll_format); queue->roll_format = strndup(data, dsz); } } -void parse_dirlist(queue_t* queue) { +void parse_dirlist(queue_t *queue) { // dirlist mmap is the size of the fstat - int lim = queue->dirlist_statbuf.st_size; - unsigned char* base = queue->dirlist; - uint64_t index = 0; + int lim = queue->dirlist_statbuf.st_size; + unsigned char *base = queue->dirlist; + uint64_t index = 0; // used to dump out the test data for test_wire.c // printbuf((char*)base, lim); @@ -763,18 +786,18 @@ void parse_dirlist(queue_t* queue) { hcbs.field_uint64 = &handle_dirlist_uint64; hcbs.field_char = &handle_dirlist_text; hcbs.userdata = queue; - parse_queue_block(queue, &base, &index, base+lim, &hcbs, &parse_wire_data, &cbs); + parse_queue_block(queue, &base, &index, base + lim, &hcbs, &parse_wire_data, &cbs); } -void parse_queuefile_meta(unsigned char* base, int limit, queue_t* queue) { - uint64_t index = 0; +void parse_queuefile_meta(unsigned char *base, int limit, queue_t *queue) { + uint64_t index = 0; wirecallbacks_t hcbs; bzero(&hcbs, sizeof(hcbs)); hcbs.field_uint64 = &handle_qf_uint64; hcbs.field_char = &handle_qf_text; hcbs.userdata = queue; - parse_queue_block(queue, &base, &index, base+limit, &hcbs, NULL, NULL); + parse_queue_block(queue, &base, &index, base + limit, &hcbs, NULL, NULL); } void chronicle_peek() { @@ -785,13 +808,14 @@ void chronicle_peek() { } } -void peek_queue_modcount(queue_t* queue) { +void peek_queue_modcount(queue_t *queue) { // poll shared directory for modcount uint64_t modcount; memcpy(&modcount, queue->dirlist_fields.modcount, sizeof(modcount)); if (queue->modcount != modcount) { - printf("shmipc: %s modcount changed from %" PRIu64 " to %" PRIu64 "\n", queue->dirname, queue->modcount, modcount); + printf("shmipc: %s modcount changed from %" PRIu64 " to %" PRIu64 "\n", queue->dirname, queue->modcount, + modcount); // slowpath poll memcpy(&queue->modcount, queue->dirlist_fields.modcount, sizeof(modcount)); memcpy(&queue->lowest_cycle, queue->dirlist_fields.lowest_cycle, sizeof(modcount)); @@ -799,7 +823,7 @@ void peek_queue_modcount(queue_t* queue) { } } -void poke_queue_modcount(queue_t* queue) { +void poke_queue_modcount(queue_t *queue) { // push modifications to lowestCycle, highestCycle to directory-listing mmap // and atomically increment the modcount uint64_t modcount; @@ -810,7 +834,8 @@ void poke_queue_modcount(queue_t* queue) { } void chronicle_peek_queue(queue_t *queue) { - if (debug) printf("peeking at %s\n", queue->dirname); + if (debug) + printf("peeking at %s\n", queue->dirname); peek_queue_modcount(queue); tailer_t *tailer = queue->tailers; @@ -846,16 +871,20 @@ tailstate_t chronicle_peek_queue_tailer_r(queue_t *queue, tailer_t *tailer) { tailer->qf_fn = chronicle_get_cycle_fn(queue, cycle); tailer->qf_tip = 0; - printf("shmipc: opening cycle %" PRIu64 " filename %s (highest_cycle %" PRIu64 ")\n", cycle, tailer->qf_fn, queue->highest_cycle); + printf("shmipc: opening cycle %" PRIu64 " filename %s (highest_cycle %" PRIu64 ")\n", cycle, tailer->qf_fn, + queue->highest_cycle); int fopen_flags = O_RDONLY; - if (tailer->mmap_protection != PROT_READ) fopen_flags = O_RDWR; + if (tailer->mmap_protection != PROT_READ) + fopen_flags = O_RDWR; if ((tailer->qf_fd = open(tailer->qf_fn, fopen_flags)) < 0) { printf("shmipc: awaiting queuefile for %s open errno=%d %s\n", tailer->qf_fn, errno, strerror(errno)); // if our cycle < highCycle, permitted to skip a missing file rather than wait if (cycle < queue->highest_cycle) { uint64_t skip_to_index = (cycle + 1) << queue->cycle_shift; - printf("shmipc: skipping queuefile (cycle < highest_cycle), bumping next_index from %" PRIu64 " to %" PRIu64 "\n", tailer->qf_index, skip_to_index); + printf("shmipc: skipping queuefile (cycle < highest_cycle), bumping next_index from %" PRIu64 + " to %" PRIu64 "\n", + tailer->qf_index, skip_to_index); tailer->qf_index = skip_to_index; continue; } @@ -864,7 +893,8 @@ tailstate_t chronicle_peek_queue_tailer_r(queue_t *queue, tailer_t *tailer) { tailer->qf_cycle_open = cycle; // renew the stat - if (fstat(tailer->qf_fd, &tailer->qf_statbuf) < 0) return 3; + if (fstat(tailer->qf_fd, &tailer->qf_statbuf) < 0) + return 3; } // assert: we have open fid @@ -880,23 +910,27 @@ tailstate_t chronicle_peek_queue_tailer_r(queue_t *queue, tailer_t *tailer) { // assign mmap limit and offset from tip and blocksize // note: blocksize may have changed, unroll this to a constant with care - uint64_t blocksize_mask = ~(queue->blocksize-1); + uint64_t blocksize_mask = ~(queue->blocksize - 1); uint64_t mmapoff = tailer->qf_tip & blocksize_mask; // renew stat if we would otherwise map less than 2* blocksize // TODO: write needs to extend file here! - if (tailer->qf_statbuf.st_size - mmapoff < 2*queue->blocksize) { - if (debug) printf("shmmain: approaching file size limit, less than two blocks remain\n"); + if (tailer->qf_statbuf.st_size - mmapoff < 2 * queue->blocksize) { + if (debug) + printf("shmmain: approaching file size limit, less than two blocks remain\n"); if (fstat(tailer->qf_fd, &tailer->qf_statbuf) < 0) return TS_E_STAT; // signal to extend queuefile iff we are an appending tailer - if (tailer->qf_statbuf.st_size - mmapoff < 2*queue->blocksize && tailer->mmap_protection != PROT_READ) { + if (tailer->qf_statbuf.st_size - mmapoff < 2 * queue->blocksize && tailer->mmap_protection != PROT_READ) { return TS_EXTEND_FAIL; } } - int limit = tailer->qf_statbuf.st_size - mmapoff > 2*queue->blocksize ? 2*queue->blocksize : tailer->qf_statbuf.st_size - mmapoff; - if (debug) printf("shmipc: tip %" PRIu64 " -> mmapoff %" PRIu64 " size 0x%x blocksize_mask 0x%" PRIx64 "\n", tailer->qf_tip, mmapoff, limit, blocksize_mask); + int limit = tailer->qf_statbuf.st_size - mmapoff > 2 * queue->blocksize ? 2 * queue->blocksize + : tailer->qf_statbuf.st_size - mmapoff; + if (debug) + printf("shmipc: tip %" PRIu64 " -> mmapoff %" PRIu64 " size 0x%x blocksize_mask 0x%" PRIx64 "\n", + tailer->qf_tip, mmapoff, limit, blocksize_mask); // only re-mmap if desired window has changed since last scan if (tailer->qf_buf == NULL || mmapoff != tailer->qf_mmapoff || limit != tailer->qf_mmapsz) { @@ -907,18 +941,21 @@ tailstate_t chronicle_peek_queue_tailer_r(queue_t *queue, tailer_t *tailer) { tailer->qf_mmapsz = limit; tailer->qf_mmapoff = mmapoff; - if ((tailer->qf_buf = mmap(0, tailer->qf_mmapsz, tailer->mmap_protection, MAP_SHARED, tailer->qf_fd, tailer->qf_mmapoff)) == MAP_FAILED) { - printf("shmipc: mmap failed %s %" PRIx64 " size %" PRIx64 " error=%s\n", tailer->qf_fn, tailer->qf_mmapoff, tailer->qf_mmapsz, strerror(errno)); + if ((tailer->qf_buf = mmap(0, tailer->qf_mmapsz, tailer->mmap_protection, MAP_SHARED, tailer->qf_fd, + tailer->qf_mmapoff)) == MAP_FAILED) { + printf("shmipc: mmap failed %s %" PRIx64 " size %" PRIx64 " error=%s\n", tailer->qf_fn, + tailer->qf_mmapoff, tailer->qf_mmapsz, strerror(errno)); tailer->qf_buf = NULL; return TS_E_MMAP; } - printf("shmipc: mmap offset %" PRIx64 " size %" PRIx64 " base=%p extent=%p\n", tailer->qf_mmapoff, tailer->qf_mmapsz, tailer->qf_buf, tailer->qf_buf+tailer->qf_mmapsz); + printf("shmipc: mmap offset %" PRIx64 " size %" PRIx64 " base=%p extent=%p\n", tailer->qf_mmapoff, + tailer->qf_mmapsz, tailer->qf_buf, tailer->qf_buf + tailer->qf_mmapsz); } - unsigned char* basep = (tailer->qf_tip - tailer->qf_mmapoff) + tailer->qf_buf; // basep within mmap - unsigned char* basep_old = basep; - unsigned char* extent = tailer->qf_buf+tailer->qf_mmapsz; - uint64_t index = tailer->qf_index; + unsigned char *basep = (tailer->qf_tip - tailer->qf_mmapoff) + tailer->qf_buf; // basep within mmap + unsigned char *basep_old = basep; + unsigned char *extent = tailer->qf_buf + tailer->qf_mmapsz; + uint64_t index = tailer->qf_index; // 0 awaiting at &base (pass) // 1 we hit working (pass) @@ -928,7 +965,7 @@ tailstate_t chronicle_peek_queue_tailer_r(queue_t *queue, tailer_t *tailer) { // 7 collected item // if any entries are read the values at basep and indexp are updated parseqb_state_t s = parse_queue_block(queue, &basep, &index, extent, &hcbs, parse_data_cb, tailer); - //printf("shmipc: block parser result %d, shm %p to %p\n", s, basep_old, basep); + // printf("shmipc: block parser result %d, shm %p to %p\n", s, basep_old, basep); if (s == QB_NEED_EXTEND && basep == basep_old) { queue_double_blocksize(queue); @@ -936,19 +973,26 @@ tailstate_t chronicle_peek_queue_tailer_r(queue_t *queue, tailer_t *tailer) { if (basep != basep_old) { // commit result of parsing to the tailer, adjusting for the window - uint64_t new_tip = basep-tailer->qf_buf + tailer->qf_mmapoff; - if (debug) printf("shmipc: parser moved shm %p to %p, file %" PRIu64 " -> %" PRIu64 ", index %" PRIu64 " to %" PRIu64 "\n", basep_old, basep, tailer->qf_tip, new_tip, tailer->qf_index, index); + uint64_t new_tip = basep - tailer->qf_buf + tailer->qf_mmapoff; + if (debug) + printf("shmipc: parser moved shm %p to %p, file %" PRIu64 " -> %" PRIu64 ", index %" PRIu64 + " to %" PRIu64 "\n", + basep_old, basep, tailer->qf_tip, new_tip, tailer->qf_index, index); tailer->qf_tip = new_tip; tailer->qf_index = index; } - if (s == QB_BUSY) return TS_BUSY; - if (s == QB_COLLECTED) return TS_COLLECTED; + if (s == QB_BUSY) + return TS_BUSY; + if (s == QB_COLLECTED) + return TS_COLLECTED; - if (s == QB_AWAITING_ENTRY) { // awaiting at end of queuefile - if (cycle < queue->highest_cycle-patch_cycles) { // allowed to fast-forward + if (s == QB_AWAITING_ENTRY) { // awaiting at end of queuefile + if (cycle < queue->highest_cycle - patch_cycles) { // allowed to fast-forward uint64_t skip_to_index = (cycle + 1) << queue->cycle_shift; - printf("shmipc: missing EOF for queuefile (cycle < highest_cycle-patch_cycles), bumping next_index from %" PRIu64 " to %" PRIu64 "\n", tailer->qf_index, skip_to_index); + printf("shmipc: missing EOF for queuefile (cycle < highest_cycle-patch_cycles), bumping next_index " + "from %" PRIu64 " to %" PRIu64 "\n", + tailer->qf_index, skip_to_index); tailer->qf_index = skip_to_index; continue; } @@ -958,15 +1002,14 @@ tailstate_t chronicle_peek_queue_tailer_r(queue_t *queue, tailer_t *tailer) { if (s == QB_REACHED_EOF) { // we've read an EOF marker, so the next expected index is cycle++, seqnum=0 uint64_t eof_cycle = ((tailer->qf_index >> queue->cycle_shift) + 1) << queue->cycle_shift; - printf("shmipc: hit EOF marker, setting next_index from %" PRIu64 " to %" PRIu64 "\n", tailer->qf_index, eof_cycle); + printf("shmipc: hit EOF marker, setting next_index from %" PRIu64 " to %" PRIu64 "\n", tailer->qf_index, + eof_cycle); tailer->qf_index = eof_cycle; } } } -int chronicle_peek_tailer(tailer_t *tailer) { - return chronicle_peek_queue_tailer(tailer->queue, tailer); -} +int chronicle_peek_tailer(tailer_t *tailer) { return chronicle_peek_queue_tailer(tailer->queue, tailer); } int chronicle_peek_queue_tailer(queue_t *queue, tailer_t *tailer) { return tailer->state = chronicle_peek_queue_tailer_r(queue, tailer); @@ -977,27 +1020,27 @@ void chronicle_debug() { queue_t *current = queue_head; while (current != NULL) { - printf(" directory %s\n", current->dirname); - printf(" handle %p\n", current); - printf(" blocksize %x\n", current->blocksize); - printf(" version %d\n", current->version); - printf(" create %d\n", current->create); - printf(" dirlist_name %s\n", current->dirlist_name); - printf(" dirlist_fd %d\n", current->dirlist_fd); + printf(" directory %s\n", current->dirname); + printf(" handle %p\n", current); + printf(" blocksize %x\n", current->blocksize); + printf(" version %d\n", current->version); + printf(" create %d\n", current->create); + printf(" dirlist_name %s\n", current->dirlist_name); + printf(" dirlist_fd %d\n", current->dirlist_fd); printf(" dirlist_sz %" PRIu64 "\n", (uint64_t)current->dirlist_statbuf.st_size); - printf(" dirlist %p\n", current->dirlist); + printf(" dirlist %p\n", current->dirlist); printf(" cycle-low %" PRIu64 "\n", current->lowest_cycle); printf(" cycle-high %" PRIu64 "\n", current->highest_cycle); printf(" modcount %" PRIu64 "\n", current->modcount); - printf(" queuefile_pattern %s\n", current->queuefile_pattern); - printf(" cycle_shift %d\n", current->cycle_shift); - printf(" roll_epoch %d\n", current->roll_epoch); - printf(" roll_length (ms) %d\n", current->roll_length); - printf(" roll_format %s\n", current->roll_format); - printf(" roll_name %s\n", current->roll_name); - printf(" roll_strftime %s\n", current->roll_strftime); - printf(" index_count %d\n", current->index_count); - printf(" index_spacing %d\n", current->index_spacing); + printf(" queuefile_pattern %s\n", current->queuefile_pattern); + printf(" cycle_shift %d\n", current->cycle_shift); + printf(" roll_epoch %d\n", current->roll_epoch); + printf(" roll_length (ms) %d\n", current->roll_length); + printf(" roll_format %s\n", current->roll_format); + printf(" roll_name %s\n", current->roll_name); + printf(" roll_strftime %s\n", current->roll_strftime); + printf(" index_count %d\n", current->index_count); + printf(" index_spacing %d\n", current->index_spacing); printf(" tailers:\n"); tailer_t *tailer = current->tailers; // shortcut to save both collections @@ -1013,22 +1056,22 @@ void chronicle_debug() { } } -void chronicle_debug_tailer(queue_t* queue, tailer_t* tailer) { - const char* state_text = tailer_state_messages[tailer->state]; - printf(" dispatcher %p\n", tailer->dispatcher); +void chronicle_debug_tailer(queue_t *queue, tailer_t *tailer) { + const char *state_text = tailer_state_messages[tailer->state]; + printf(" dispatcher %p\n", tailer->dispatcher); uint cycle = tailer->dispatch_after >> queue->cycle_shift; uint seqnum = tailer->dispatch_after & queue->seqnum_mask; printf(" dispatch_after %" PRIu64 " (cycle %u, seqnum %u)\n", tailer->dispatch_after, cycle, seqnum); printf(" state %d - %s\n", tailer->state, state_text); - printf(" qf_fn %s\n", tailer->qf_fn); - printf(" qf_fd %d\n", tailer->qf_fd); + printf(" qf_fn %s\n", tailer->qf_fn); + printf(" qf_fd %d\n", tailer->qf_fd); printf(" qf_statbuf_sz %" PRIu64 "\n", (uint64_t)tailer->qf_statbuf.st_size); printf(" qf_tip %" PRIu64 "\n", tailer->qf_tip); cycle = tailer->qf_index >> queue->cycle_shift; seqnum = tailer->qf_index & queue->seqnum_mask; printf(" qf_index %" PRIu64 " (cycle %u, seqnum %u)\n", tailer->qf_index, cycle, seqnum); - printf(" qf_buf %p\n", tailer->qf_buf); - printf(" extent %p\n", tailer->qf_buf+tailer->qf_mmapsz); + printf(" qf_buf %p\n", tailer->qf_buf); + printf(" extent %p\n", tailer->qf_buf + tailer->qf_mmapsz); printf(" qf_mmapsz %" PRIx64 "\n", tailer->qf_mmapsz); printf(" qf_mmapoff %" PRIx64 "\n", tailer->qf_mmapoff); } @@ -1039,7 +1082,8 @@ uint64_t chronicle_append(queue_t *queue, COBJ msg) { } uint64_t chronicle_append_ts(queue_t *queue, COBJ msg, long ms) { - if (queue == NULL) return chronicle_err("queue is NULL"); + if (queue == NULL) + return chronicle_err("queue is NULL"); // Appending logic // 0) catch up to the end of the current file. @@ -1066,8 +1110,10 @@ uint64_t chronicle_append_ts(queue_t *queue, COBJ msg, long ms) { // caution: encodecheck may tweak blocksize, do not redorder below shmipc_peek_tailer size_t write_sz = queue->append_sizeof(msg); - if (write_sz < 0) return 0; - if (write_sz > HD_MASK_META) return chronicle_err("`shm msg sz > 30bit"); + if (write_sz < 0) + return 0; + if (write_sz > HD_MASK_META) + return chronicle_err("`shm msg sz > 30bit"); while (write_sz > queue->blocksize) queue_double_blocksize(queue); @@ -1077,8 +1123,9 @@ uint64_t chronicle_append_ts(queue_t *queue, COBJ msg, long ms) { // build a special tailer with the protection bits and file descriptor set to allow // writing. if (queue->appender == NULL) { - tailer_t* tailer = malloc(sizeof(tailer_t)); - if (tailer == NULL) return chronicle_err("am fail"); + tailer_t *tailer = malloc(sizeof(tailer_t)); + if (tailer == NULL) + return chronicle_err("am fail"); bzero(tailer, sizeof(tailer_t)); // compat: writers do an extended lookback to patch missing EOFs @@ -1095,27 +1142,30 @@ uint64_t chronicle_append_ts(queue_t *queue, COBJ msg, long ms) { printf("shmipc: rw dir listing %d %s\n", x, cerr_msg); return -1; } - if (debug) printf("shmipc: appender created\n"); + if (debug) + printf("shmipc: appender created\n"); } - tailer_t* appender = queue->appender; + tailer_t *appender = queue->appender; // poll the appender while (1) { int r = chronicle_peek_queue_tailer(queue, appender); // TODO: 2nd call defensive to ensure 1 whole blocksize is available to put r = chronicle_peek_queue_tailer(queue, appender); - if (debug) printf("shmipc: writeloop appender in state %d\n", r); + if (debug) + printf("shmipc: writeloop appender in state %d\n", r); if (r == TS_AWAITING_QUEUEFILE) { // our cycle is pointing to a queuefile that does not exist // as we are writer, create it with temporary filename, atomically // move it to the desired name, then bump the global highest_cycle // value if rename succeeded - char* fn_buf; + char *fn_buf; asprintf(&fn_buf, "%s.%d.tmp", appender->qf_fn, pid_header); // if queuefile_init fails, re-throw the error and abort the write - if (queuefile_init(fn_buf, queue) != 0) return -1; + if (queuefile_init(fn_buf, queue) != 0) + return -1; if (rename(fn_buf, appender->qf_fn) != 0) { // rename failed, maybe raced with another writer, delay and try again @@ -1178,19 +1228,20 @@ uint64_t chronicle_append_ts(queue_t *queue, COBJ msg, long ms) { // Note that we do not extended qf_buf or qf_index after the write. Let the // tailer log handle the entry we've just written in the normal way, since that will // adjust the buffer window/mmap for us. - unsigned char* ptr = (appender->qf_tip - appender->qf_mmapoff) + appender->qf_buf; - uint32_t ret = lock_cmpxchgl(ptr, HD_UNALLOCATED, HD_WORKING); + unsigned char *ptr = (appender->qf_tip - appender->qf_mmapoff) + appender->qf_buf; + uint32_t ret = lock_cmpxchgl(ptr, HD_UNALLOCATED, HD_WORKING); // cmpxchg returns the original value in memory, so we can tell if we succeeded // by looking for HD_UNALLOCATED. If we read a working bit or finished size, we lost. if (ret == HD_UNALLOCATED) { - asm volatile ("mfence" ::: "memory"); + asm volatile("mfence" ::: "memory"); // if given a clock, test if we should write EOF and advance cycle if (ms > 0) { uint64_t cyc = chronicle_cycle_from_ms(queue, ms); if (cyc > appender->qf_index >> queue->cycle_shift) { - printf("shmipc: appender setting cycle from timestamp: current %" PRIu64 " proposed %" PRIu64 "\n", appender->qf_index >> queue->cycle_shift, cyc); + printf("shmipc: appender setting cycle from timestamp: current %" PRIu64 " proposed %" PRIu64 "\n", + appender->qf_index >> queue->cycle_shift, cyc); appender->qf_index = cyc << queue->cycle_shift; printf("shmipc: got write lock, writing EOF to start roll\n"); @@ -1212,13 +1263,14 @@ uint64_t chronicle_append_ts(queue_t *queue, COBJ msg, long ms) { continue; // retry write in next queuefile } - queue->append_write(ptr+4, msg, write_sz); + queue->append_write(ptr + 4, msg, write_sz); - asm volatile ("mfence" ::: "memory"); + asm volatile("mfence" ::: "memory"); uint32_t header = write_sz & HD_MASK_LENGTH; memcpy(ptr, &header, sizeof(header)); - if (debug) printf("shmipc: wrote %zu bytes as index %" PRIu64 "\n", write_sz, appender->qf_index); + if (debug) + printf("shmipc: wrote %zu bytes as index %" PRIu64 "\n", write_sz, appender->qf_index); break; } @@ -1230,8 +1282,9 @@ uint64_t chronicle_append_ts(queue_t *queue, COBJ msg, long ms) { return appender->qf_index; } -tailer_t* chronicle_tailer(queue_t *queue, cdispatch_f dispatcher, void* dispatch_ctx, uint64_t index) { - if (queue == NULL) return chronicle_perr("queue is not valid"); +tailer_t *chronicle_tailer(queue_t *queue, cdispatch_f dispatcher, void *dispatch_ctx, uint64_t index) { + if (queue == NULL) + return chronicle_perr("queue is not valid"); // decompose index into cycle (file) and seqnum within file int cycle = index >> queue->cycle_shift; @@ -1246,8 +1299,9 @@ tailer_t* chronicle_tailer(queue_t *queue, cdispatch_f dispatcher, void* dispatc } // allocate struct, we'll link if all checks pass - tailer_t* tailer = malloc(sizeof(tailer_t)); - if (tailer == NULL) return chronicle_perr("tm fail"); + tailer_t *tailer = malloc(sizeof(tailer_t)); + if (tailer == NULL) + return chronicle_perr("tm fail"); bzero(tailer, sizeof(tailer_t)); tailer->dispatch_after = index - 1; @@ -1259,7 +1313,8 @@ tailer_t* chronicle_tailer(queue_t *queue, cdispatch_f dispatcher, void* dispatc tailer->next = queue->tailers; // linked list tailer->prev = NULL; - if (queue->tailers) queue->tailers->prev = tailer; + if (queue->tailers) + queue->tailers->prev = tailer; queue->tailers = tailer; tailer->queue = queue; // parent pointer @@ -1267,14 +1322,17 @@ tailer_t* chronicle_tailer(queue_t *queue, cdispatch_f dispatcher, void* dispatc } COBJ chronicle_collect(tailer_t *tailer, collected_t *collected) { - if (tailer == NULL) return chronicle_perr("null tailer"); - if (collected == NULL) return chronicle_perr("null collected"); + if (tailer == NULL) + return chronicle_perr("null tailer"); + if (collected == NULL) + return chronicle_perr("null collected"); tailer->collect = collected; uint64_t delaycount = 0; while (1) { int r = chronicle_peek_tailer(tailer); - if (debug) printf("collect value returns %d into object %p\n", r, tailer->collect); + if (debug) + printf("collect value returns %d into object %p\n", r, tailer->collect); if (r == TS_COLLECTED) { break; } @@ -1293,15 +1351,11 @@ void chronicle_return(tailer_t *tailer, collected_t *collected) { } } -tailstate_t chronicle_tailer_state(tailer_t* tailer) { - return tailer->state; -} +tailstate_t chronicle_tailer_state(tailer_t *tailer) { return tailer->state; } -uint64_t chronicle_tailer_index(tailer_t* tailer) { - return tailer->qf_index; -} +uint64_t chronicle_tailer_index(tailer_t *tailer) { return tailer->qf_index; } -void chronicle_tailer_close(tailer_t* tailer) { +void chronicle_tailer_close(tailer_t *tailer) { if (tailer->qf_fn) { // if next filename cached... free(tailer->qf_fn); } @@ -1323,12 +1377,13 @@ void chronicle_tailer_close(tailer_t* tailer) { free(tailer); } -int chronicle_cleanup(queue_t* queue_delete) { - if (queue_delete == NULL) return chronicle_err("queue is NULL"); +int chronicle_cleanup(queue_t *queue_delete) { + if (queue_delete == NULL) + return chronicle_err("queue is NULL"); // check if queue already open queue_t **parent = &queue_head; // pointer to a queue_t pointer - queue_t *queue = queue_head; + queue_t *queue = queue_head; while (queue != NULL) { if (queue == queue_delete) { @@ -1337,13 +1392,14 @@ int chronicle_cleanup(queue_t* queue_delete) { // delete tailers tailer_t *tailer = queue->tailers; // shortcut to save both collections while (tailer != NULL) { - tailer_t* next_tmp = tailer->next; + tailer_t *next_tmp = tailer->next; chronicle_tailer_close(tailer); tailer = next_tmp; } queue->tailers = NULL; - if (queue->appender) chronicle_tailer_close(queue->appender); + if (queue->appender) + chronicle_tailer_close(queue->appender); // kill queue munmap(queue->dirlist, queue->dirlist_statbuf.st_size); @@ -1367,7 +1423,7 @@ int chronicle_cleanup(queue_t* queue_delete) { return chronicle_err("chronicle_cleanup: queue not found"); } -int queuefile_init(char* fn, queue_t* queue) { +int queuefile_init(char *fn, queue_t *queue) { int fd; int mode = 0777; @@ -1397,7 +1453,7 @@ int queuefile_init(char* fn, queue_t* queue) { return 0; } -int directory_listing_init(queue_t* queue, uint64_t cycle) { +int directory_listing_init(queue_t *queue, uint64_t cycle) { int fd; int mode = 0777; @@ -1406,35 +1462,35 @@ int directory_listing_init(queue_t* queue, uint64_t cycle) { return chronicle_err("shmipc: directory_listing_init open failed"); } - wirepad_t* pad = wirepad_init(1024); + wirepad_t *pad = wirepad_init(1024); // deliberately not updating queue->roll_epoch here so that we verify // we can read them back from the directory_listing file - int roll_epoch = (queue->roll_epoch == -1) ? 0 : queue->roll_epoch; - int modcount = 1; + int roll_epoch = (queue->roll_epoch == -1) ? 0 : queue->roll_epoch; + int modcount = 1; // single metadata message wirepad_qc_start(pad, 1); wirepad_event_name(pad, "header"); wirepad_type_prefix(pad, "STStore"); - wirepad_nest_enter(pad); //header - wirepad_field_type_enum(pad, "wireType", "WireType", "BINARY_LIGHT"); - // field metadata, type prefix SCQMeta, nesting begin - wirepad_field(pad, "metadata"); - wirepad_type_prefix(pad, "SCQMeta"); - wirepad_nest_enter(pad); - wirepad_field(pad, "roll"); - wirepad_type_prefix(pad, "SCQSRoll"); - wirepad_nest_enter(pad); - wirepad_field_varint(pad, "length", queue->roll_length); - wirepad_field_text(pad, "format", queue->roll_format); - wirepad_field_varint(pad, "epoch", roll_epoch); - wirepad_nest_exit(pad); - wirepad_field_varint(pad, "deltaCheckpointInterval", 64); - wirepad_field_varint(pad, "sourceId", 0); - wirepad_nest_exit(pad); - wirepad_pad_to_x8(pad); // feels wrong - should be automatic? - wirepad_nest_exit(pad); + wirepad_nest_enter(pad); // header + wirepad_field_type_enum(pad, "wireType", "WireType", "BINARY_LIGHT"); + // field metadata, type prefix SCQMeta, nesting begin + wirepad_field(pad, "metadata"); + wirepad_type_prefix(pad, "SCQMeta"); + wirepad_nest_enter(pad); + wirepad_field(pad, "roll"); + wirepad_type_prefix(pad, "SCQSRoll"); + wirepad_nest_enter(pad); + wirepad_field_varint(pad, "length", queue->roll_length); + wirepad_field_text(pad, "format", queue->roll_format); + wirepad_field_varint(pad, "epoch", roll_epoch); + wirepad_nest_exit(pad); + wirepad_field_varint(pad, "deltaCheckpointInterval", 64); + wirepad_field_varint(pad, "sourceId", 0); + wirepad_nest_exit(pad); + wirepad_pad_to_x8(pad); // feels wrong - should be automatic? + wirepad_nest_exit(pad); wirepad_qc_finish(pad); // 6 data messages @@ -1475,7 +1531,7 @@ int directory_listing_init(queue_t* queue, uint64_t cycle) { return 0; } -int directory_listing_reopen(queue_t* queue, int open_flags, int mmap_prot) { +int directory_listing_reopen(queue_t *queue, int open_flags, int mmap_prot) { if ((queue->dirlist_fd = open(queue->dirlist_name, open_flags)) < 0) { return chronicle_err("directory_listing_reopen open failed"); } @@ -1483,10 +1539,12 @@ int directory_listing_reopen(queue_t* queue, int open_flags, int mmap_prot) { // find size of dirlist and mmap if (fstat(queue->dirlist_fd, &queue->dirlist_statbuf) < 0) return chronicle_err("dirlist fstat"); - if ((queue->dirlist = mmap(0, queue->dirlist_statbuf.st_size, mmap_prot, MAP_SHARED, queue->dirlist_fd, 0)) == MAP_FAILED) + if ((queue->dirlist = mmap(0, queue->dirlist_statbuf.st_size, mmap_prot, MAP_SHARED, queue->dirlist_fd, 0)) == + MAP_FAILED) return chronicle_err("dirlist mmap fail"); - if (debug) printf("shmipc: parsing dirlist %s\n", queue->dirlist_name); + if (debug) + printf("shmipc: parsing dirlist %s\n", queue->dirlist_name); parse_dirlist(queue); // check the polled fields in header section were all resolved to pointers within the map @@ -1497,16 +1555,12 @@ int directory_listing_reopen(queue_t* queue, int open_flags, int mmap_prot) { return 0; } -COBJ chronicle_decoder_default_parse(unsigned char* base, int lim) { - char* msg = calloc(1, lim+1); +COBJ chronicle_decoder_default_parse(unsigned char *base, int lim) { + char *msg = calloc(1, lim + 1); memcpy(msg, base, lim); return msg; } -size_t chronicle_encoder_default_sizeof(COBJ msg) { - return strlen(msg); -} +size_t chronicle_encoder_default_sizeof(COBJ msg) { return strlen(msg); } -void chronicle_encoder_default_write(unsigned char* base, COBJ msg, size_t sz) { - memcpy(base, msg, sz); -} +void chronicle_encoder_default_write(unsigned char *base, COBJ msg, size_t sz) { memcpy(base, msg, sz); } diff --git a/native/libchronicle.h b/native/libchronicle.h index 3dc37b9..a09ca92 100644 --- a/native/libchronicle.h +++ b/native/libchronicle.h @@ -20,15 +20,15 @@ #define __STDC_FORMAT_MACROS #include +#include +#include #include #include -#include #include -#include -#include #include #include -#include +#include +#include #define MAXDATASIZE 1000 // max number of bytes we can get at once @@ -48,27 +48,25 @@ #define CHRONICLE_FLAGS_RW #define CHRONICLE_FLAGS_CREATE - - // Public interface // your glue code will need to cast COBJ in callbacks, by implmenting // four callbacks. -typedef void* COBJ; -typedef void* DISPATCH_CTX; +typedef void *COBJ; +typedef void *DISPATCH_CTX; // // cparse_f takes void* and returns custom object. Deserialise, memcpy // or return same ptr to dispatch ref valid for callback. // csizeof_f tells library how many bytes required to serialise user object // cappend_f takes custom object and writes bytes to void* // cdispatch_f takes custom object and index, delivers to application with user data -typedef COBJ (*cparse_f) (unsigned char*, int); -typedef void (*cparsefree_f)(COBJ); -typedef size_t (*csizeof_f) (COBJ); -typedef void (*cappend_f) (unsigned char*,COBJ,size_t); -typedef int (*cdispatch_f) (DISPATCH_CTX,uint64_t,COBJ); +typedef COBJ (*cparse_f)(unsigned char *, int); +typedef void (*cparsefree_f)(COBJ); +typedef size_t (*csizeof_f)(COBJ); +typedef void (*cappend_f)(unsigned char *, COBJ, size_t); +typedef int (*cdispatch_f)(DISPATCH_CTX, uint64_t, COBJ); // forward definition of queue -typedef struct queue queue_t; +typedef struct queue queue_t; typedef struct tailer tailer_t; // return codes exposed via. chronicle_tailer_state @@ -80,48 +78,56 @@ typedef struct tailer tailer_t; // 5 not yet polled // 6 queuefile at fid needs extending on disk // 7 a value was collected -typedef enum {TS_AWAITING_ENTRY, TS_BUSY, TS_AWAITING_QUEUEFILE, TS_E_STAT, TS_E_MMAP, TS_PEEK, TS_EXTEND_FAIL, TS_COLLECTED} tailstate_t; +typedef enum { + TS_AWAITING_ENTRY, + TS_BUSY, + TS_AWAITING_QUEUEFILE, + TS_E_STAT, + TS_E_MMAP, + TS_PEEK, + TS_EXTEND_FAIL, + TS_COLLECTED +} tailstate_t; // collect structure - we complete values for the caller typedef struct { - COBJ msg; - size_t sz; + COBJ msg; + size_t sz; uint64_t index; } collected_t; -queue_t* chronicle_init(char* dir); -void chronicle_set_version(queue_t* queue, int version); -int chronicle_set_roll_scheme(queue_t* queue, char* scheme); -int chronicle_set_roll_dateformat(queue_t* queue, char* scheme); -void chronicle_set_encoder(queue_t* queue, csizeof_f append_sizeof, cappend_f append_write); -void chronicle_set_decoder(queue_t* queue, cparse_f parser, cparsefree_f parsefree); -void chronicle_set_create(queue_t* queue, int create); -int chronicle_open(queue_t* queue); -int chronicle_cleanup(queue_t* queue); - -COBJ chronicle_decoder_default_parse(unsigned char*, int); +queue_t *chronicle_init(char *dir); +void chronicle_set_version(queue_t *queue, int version); +int chronicle_set_roll_scheme(queue_t *queue, char *scheme); +int chronicle_set_roll_dateformat(queue_t *queue, char *scheme); +void chronicle_set_encoder(queue_t *queue, csizeof_f append_sizeof, cappend_f append_write); +void chronicle_set_decoder(queue_t *queue, cparse_f parser, cparsefree_f parsefree); +void chronicle_set_create(queue_t *queue, int create); +int chronicle_open(queue_t *queue); +int chronicle_cleanup(queue_t *queue); + +COBJ chronicle_decoder_default_parse(unsigned char *, int); size_t chronicle_encoder_default_sizeof(COBJ); -void chronicle_encoder_default_write(unsigned char*,COBJ,size_t); - -int chronicle_get_version(queue_t* queue); -char* chronicle_get_roll_scheme(queue_t* queue); -char* chronicle_get_roll_format(queue_t* queue); -char* chronicle_get_cycle_fn(queue_t* queue, int cycle); +void chronicle_encoder_default_write(unsigned char *, COBJ, size_t); +int chronicle_get_version(queue_t *queue); +char *chronicle_get_roll_scheme(queue_t *queue); +char *chronicle_get_roll_format(queue_t *queue); +char *chronicle_get_cycle_fn(queue_t *queue, int cycle); -const char* chronicle_strerror(); +const char *chronicle_strerror(); -tailer_t* chronicle_tailer(queue_t *queue, cdispatch_f dispatcher, DISPATCH_CTX dispatch_ctx, uint64_t index); -void chronicle_tailer_close(tailer_t* tailer); -tailstate_t chronicle_tailer_state(tailer_t* tailer); -uint64_t chronicle_tailer_index(tailer_t* tailer); +tailer_t *chronicle_tailer(queue_t *queue, cdispatch_f dispatcher, DISPATCH_CTX dispatch_ctx, uint64_t index); +void chronicle_tailer_close(tailer_t *tailer); +tailstate_t chronicle_tailer_state(tailer_t *tailer); +uint64_t chronicle_tailer_index(tailer_t *tailer); void chronicle_peek(); void chronicle_peek_queue(queue_t *queue); int chronicle_peek_tailer(tailer_t *tailer); void chronicle_debug(); -void chronicle_debug_tailer(queue_t* queue, tailer_t* tailer); +void chronicle_debug_tailer(queue_t *queue, tailer_t *tailer); uint64_t chronicle_append(queue_t *queue, COBJ msg); uint64_t chronicle_append_ts(queue_t *queue, COBJ msg, long ms); @@ -130,8 +136,8 @@ COBJ chronicle_collect(tailer_t *tailer, collected_t *collect); void chronicle_return(tailer_t *tailer, collected_t *collect); struct ROLL_SCHEME { - char* name; - char* formatstr; + char *name; + char *formatstr; uint32_t roll_length_secs; uint32_t entries; uint32_t index; diff --git a/native/mock_k.h b/native/mock_k.h index 38d3b47..db51c9e 100644 --- a/native/mock_k.h +++ b/native/mock_k.h @@ -23,10 +23,10 @@ #define KERR -128 // globals -int kxx_errno = 0; -char* kxx_msg = NULL; +int kxx_errno = 0; +char *kxx_msg = NULL; -K krr(const S msg) { +K krr(const S msg) { printf("'%s\n", msg); kxx_errno = -1; kxx_msg = msg; @@ -57,25 +57,27 @@ K kj(long long i) { r->j = i; return r; } -K kss(const char* ss) { +K kss(const char *ss) { K r = ktn(-KS, 0); - r->s = (char*)ss; + r->s = (char *)ss; return r; } -K dl(void* fnptr, long long n) { - K r = ktn(100,0); +K dl(void *fnptr, long long n) { + K r = ktn(100, 0); r->s = fnptr; r->a = n; return r; } typedef K (*kfunc_1arg)(K); -typedef K (*kfunc_2arg)(K,K); -typedef K (*kfunc_3arg)(K,K,K); +typedef K (*kfunc_2arg)(K, K); +typedef K (*kfunc_3arg)(K, K, K); K dot(K x, K y) { // call function pointer in x with args in mixed list y - if (x->t != 100) return krr("x must be fptr"); - if (y->t != 0) return krr("y must be list"); + if (x->t != 100) + return krr("x must be fptr"); + if (y->t != 0) + return krr("y must be list"); if (x->a == 2) { kfunc_2arg fptr = (kfunc_2arg)x->s; return fptr(kK(y)[0], kK(y)[1]); @@ -85,53 +87,60 @@ K dot(K x, K y) { // call function pointer in x with args in mixed list y K knk(int n, ...) { // create a mixed list from K's in varg va_list ap; - K r = ktn(0, n); - va_start(ap, n); //Requires the last fixed parameter (to get the address) - for(int j=0; j 19) ? 8 : sizefor[abs(type)]; - K r = malloc(sizeof(struct k0) + n*sz); + K r = malloc(sizeof(struct k0) + n * sz); r->r = 0; r->t = type; - if (n > 0) r->n = n; // keep: trap accessing n for atom in valgrind + if (n > 0) + r->n = n; // keep: trap accessing n for atom in valgrind return r; } // dummy serialiser returns a single byte array [SOH] K b9(I mode, K obj) { - K r = ktn(KB,1); + K r = ktn(KB, 1); r->G0[0] = 1; return r; } -K d9(K obj) { - return ki(1); -} +K d9(K obj) { return ki(1); } int okx(K obj) { return 1; } // repl equivelent wrapper (protected eval, with and without gc) -K pe(K x) { - if (kxx_errno != 0) exit(-1); +K pe(K x) { + if (kxx_errno != 0) + exit(-1); return x; } void per(K x) { pe(x); - if (x != NULL) r0(x); + if (x != NULL) + r0(x); } void r0(K x) { // Decrement the object‘s reference count - if (x == 0) { printf("Bug r0 of null pointer %p\n", x); return; } - if (x->r < 0) printf("Bug double-free of %p\n", x); + if (x == 0) { + printf("Bug r0 of null pointer %p\n", x); + return; + } + if (x->r < 0) + printf("Bug double-free of %p\n", x); if (x->r == 0) { - if (x->t == 0) for (int i = 0; i < x->n; i++) r0(kK(x)[i]); + if (x->t == 0) + for (int i = 0; i < x->n; i++) + r0(kK(x)[i]); // flip? // dict? free(x); @@ -143,4 +152,3 @@ K r1(K x) { // Increment the object‘s reference count x->r++; return x; } - diff --git a/native/serdes_k.h b/native/serdes_k.h index 0697e69..f571b7d 100644 --- a/native/serdes_k.h +++ b/native/serdes_k.h @@ -23,10 +23,10 @@ #include "libchronicle.h" -COBJ parse_kx(unsigned char* base, int lim) { +COBJ parse_kx(unsigned char *base, int lim) { // prep args and fire callback K msg = ktn(KG, lim); - memcpy((char*)msg->G0, base, lim); + memcpy((char *)msg->G0, base, lim); int ok = okx(msg); if (ok) { K out = d9(msg); @@ -41,9 +41,9 @@ COBJ parse_kx(unsigned char* base, int lim) { // the encoding via. b9 happens in shmipc_append, so here we just // write the bytes -void append_kx(unsigned char* base, COBJ msg, size_t lim) { +void append_kx(unsigned char *base, COBJ msg, size_t lim) { K m = (K)msg; - memcpy(base, (char*)m->G0, m->n); + memcpy(base, (char *)m->G0, m->n); } size_t sizeof_kx(COBJ msg) { @@ -56,4 +56,3 @@ void free_kx(COBJ msg) { K m = (K)msg; r0(m); } - diff --git a/native/shm_example_reader.c b/native/shm_example_reader.c index fef174a..8e8e434 100644 --- a/native/shm_example_reader.c +++ b/native/shm_example_reader.c @@ -1,42 +1,38 @@ #include -#include #include +#include static volatile int keepRunning = 1; // This is a stand-alone tool for reading a queue // queue data is null-terminated strings, embedded nulls will truncate printing -void* parse_msg(unsigned char* base, int lim) { - char* msg = calloc(1, lim+1); +void *parse_msg(unsigned char *base, int lim) { + char *msg = calloc(1, lim + 1); memcpy(msg, base, lim); return msg; } -void free_msg(void* msg) { - free(msg); -} +void free_msg(void *msg) { free(msg); } -int print_msg(void* ctx, uint64_t index, void* msg) { - printf("[%" PRIu64 "] %s\n", index, (char*)msg); +int print_msg(void *ctx, uint64_t index, void *msg) { + printf("[%" PRIu64 "] %s\n", index, (char *)msg); return 0; } -void sigint_handler(int dummy) { - keepRunning = 0; -} +void sigint_handler(int dummy) { keepRunning = 0; } -int main(const int argc, char **argv) { +int main(const int argc, char **argv) { signal(SIGINT, sigint_handler); - queue_t* queue = chronicle_init(argv[1]); + queue_t *queue = chronicle_init(argv[1]); chronicle_set_decoder(queue, &parse_msg, &free_msg); - if (chronicle_open(queue) != 0) exit(-1); + if (chronicle_open(queue) != 0) + exit(-1); chronicle_tailer(queue, &print_msg, NULL, 0); while (keepRunning) { - usleep(500*1000); + usleep(500 * 1000); chronicle_peek(); } printf("exiting\n"); chronicle_cleanup(queue); } - diff --git a/native/shm_example_writer.c b/native/shm_example_writer.c index 6f15ac0..bcf731e 100644 --- a/native/shm_example_writer.c +++ b/native/shm_example_writer.c @@ -3,29 +3,27 @@ // This is a stand-alone tool for writing a queue // queue data is null-terminated strings, embedded nulls will truncate printing -void append_msg(unsigned char* base, void* msg, size_t sz) { - memcpy(base, msg, sz); -} +void append_msg(unsigned char *base, void *msg, size_t sz) { memcpy(base, msg, sz); } -size_t sizeof_msg(void* msg) { - return strlen(msg); -} +size_t sizeof_msg(void *msg) { return strlen(msg); } -int main(const int argc, char **argv) { - queue_t* queue = chronicle_init(argv[1]); +int main(const int argc, char **argv) { + queue_t *queue = chronicle_init(argv[1]); chronicle_set_encoder(queue, &sizeof_msg, &append_msg); chronicle_set_version(queue, 5); chronicle_set_roll_scheme(queue, "FAST_HOURLY"); chronicle_set_create(queue, 1); - if (chronicle_open(queue) != 0) exit(-1); + if (chronicle_open(queue) != 0) + exit(-1); char line[1024]; while (1) { - char* g = fgets(line, 1024, stdin); - if (g == NULL) break; + char *g = fgets(line, 1024, stdin); + if (g == NULL) + break; line[strlen(line) - 1] = 0; // remove line break long int index = chronicle_append(queue, line); - printf("[%" PRIu64 "] %s\n", index, (char*)g); + printf("[%" PRIu64 "] %s\n", index, (char *)g); } chronicle_cleanup(queue); } diff --git a/native/shmmain.c b/native/shmmain.c index e42bd70..14d776d 100644 --- a/native/shmmain.c +++ b/native/shmmain.c @@ -12,30 +12,30 @@ // See the License for the specific language governing permissions and // limitations under the License. +#include #include -#include #include -#include +#include // This is a stand-alone tool for replaying a queue, and optionally writing to it. // It is compatible with java InputMain / OutputMain, ie. the data payloads are // wire-format encoded. we use wire.h to encode/decode this. -int print_msg(void* ctx, uint64_t index, COBJ y) { - printf("[%" PRIu64 "] %s\n", index, (char*)y); +int print_msg(void *ctx, uint64_t index, COBJ y) { + printf("[%" PRIu64 "] %s\n", index, (char *)y); return 0; } int main(const int argc, char **argv) { int c; opterr = 0; - int verboseflag = 0; - int followflag = 0; - char* append = NULL; + int verboseflag = 0; + int followflag = 0; + char *append = NULL; uint64_t index = 0; - int can_create = 0; + int can_create = 0; while ((c = getopt(argc, argv, "i:va:cf")) != -1) - switch (c) { + switch (c) { case 'i': index = strtoull(optarg, NULL, 0); break; @@ -57,7 +57,7 @@ int main(const int argc, char **argv) { default: fprintf(stderr, "Unknown option '%c'\n", c); exit(3); - } + } if (optind + 1 > argc) { printf("Missing mandatory argument.\n Expected: %s [-d] [-m] [-i INDEX] [-v] [-a text] [-f] QUEUE\n", argv[0]); @@ -77,20 +77,20 @@ int main(const int argc, char **argv) { exit(1); } - char* dir = argv[optind]; - queue_t* queue = chronicle_init(dir); + char *dir = argv[optind]; + queue_t *queue = chronicle_init(dir); chronicle_set_encoder(queue, &wirepad_sizeof, &wirepad_write); chronicle_set_decoder(queue, &wire_parse_textonly, &free); chronicle_set_create(queue, can_create); chronicle_set_version(queue, 5); chronicle_set_roll_scheme(queue, "FAST_DAILY"); - + if (chronicle_open(queue) != 0) { printf("failed to open %s", chronicle_strerror()); exit(-1); } - wirepad_t* pad = wirepad_init(1024); + wirepad_t *pad = wirepad_init(1024); chronicle_tailer(queue, &print_msg, NULL, index); chronicle_peek(); @@ -105,11 +105,12 @@ int main(const int argc, char **argv) { chronicle_peek(); while (followflag) { - usleep(500*1000); + usleep(500 * 1000); chronicle_peek(); } - if (verboseflag) chronicle_debug(); + if (verboseflag) + chronicle_debug(); chronicle_cleanup(queue); return 0; diff --git a/native/test/test_buffer.c b/native/test/test_buffer.c index c989fe6..c541be4 100644 --- a/native/test/test_buffer.c +++ b/native/test/test_buffer.c @@ -1,9 +1,9 @@ #define _GNU_SOURCE +#include +#include #include #include -#include -#include #include #include #include @@ -12,38 +12,33 @@ static void test_buffer_hello(void **state) { - char* f = formatbuf("hello world", 12); - assert_string_equal(f, - "00000000 68 65 6c 6c 6f 20 77 6f 72 6c 64 00 hello wo rld. \n"); + char *f = formatbuf("hello world", 12); + assert_string_equal(f, "00000000 68 65 6c 6c 6f 20 77 6f 72 6c 64 00 hello wo rld. \n"); free(f); f = formatbuf("hello", 0); - assert_string_equal(f, - "00000000 \n"); + assert_string_equal(f, "00000000 \n"); free(f); - char* p = "The chances of a neutrino actually hitting something... are roughly " - "comparable to that of dropping a ball bearing at random from a cruising " - "747 and hitting, say, an egg sandwich."; + char *p = "The chances of a neutrino actually hitting something... are roughly " + "comparable to that of dropping a ball bearing at random from a cruising " + "747 and hitting, say, an egg sandwich."; f = formatbuf(p, strlen(p)); - assert_string_equal(f, - "00000000 54 68 65 20 63 68 61 6e 63 65 73 20 6f 66 20 61 The chan ces of a\n" - "00000010 20 6e 65 75 74 72 69 6e 6f 20 61 63 74 75 61 6c neutrin o actual\n" - "00000020 6c 79 20 68 69 74 74 69 6e 67 20 73 6f 6d 65 74 ly hitti ng somet\n" - "00000030 68 69 6e 67 2e 2e 2e 20 61 72 65 20 72 6f 75 67 hing... are roug\n" - "00000040 68 6c 79 20 63 6f 6d 70 61 72 61 62 6c 65 20 74 hly comp arable t\n" - "00000050 6f 20 74 68 61 74 20 6f 66 20 64 72 6f 70 70 69 o that o f droppi\n" - "00000060 6e 67 20 61 20 62 61 6c 6c 20 62 65 61 72 69 6e ng a bal l bearin\n" - "00000070 67 20 61 74 20 72 61 6e 64 6f 6d 20 66 72 6f 6d g at ran dom from\n" - "00000080 20 61 20 63 72 75 69 73 69 6e 67 20 37 34 37 20 a cruis ing 747 \n" - "00000090 61 6e 64 20 68 69 74 74 69 6e 67 2c 20 73 61 79 and hitt ing, say\n" - "000000a0 2c 20 61 6e 20 65 67 67 20 73 61 6e 64 77 69 63 , an egg sandwic\n" - "000000b0 68 2e h. \n" - ); + assert_string_equal(f, "00000000 54 68 65 20 63 68 61 6e 63 65 73 20 6f 66 20 61 The chan ces of a\n" + "00000010 20 6e 65 75 74 72 69 6e 6f 20 61 63 74 75 61 6c neutrin o actual\n" + "00000020 6c 79 20 68 69 74 74 69 6e 67 20 73 6f 6d 65 74 ly hitti ng somet\n" + "00000030 68 69 6e 67 2e 2e 2e 20 61 72 65 20 72 6f 75 67 hing... are roug\n" + "00000040 68 6c 79 20 63 6f 6d 70 61 72 61 62 6c 65 20 74 hly comp arable t\n" + "00000050 6f 20 74 68 61 74 20 6f 66 20 64 72 6f 70 70 69 o that o f droppi\n" + "00000060 6e 67 20 61 20 62 61 6c 6c 20 62 65 61 72 69 6e ng a bal l bearin\n" + "00000070 67 20 61 74 20 72 61 6e 64 6f 6d 20 66 72 6f 6d g at ran dom from\n" + "00000080 20 61 20 63 72 75 69 73 69 6e 67 20 37 34 37 20 a cruis ing 747 \n" + "00000090 61 6e 64 20 68 69 74 74 69 6e 67 2c 20 73 61 79 and hitt ing, say\n" + "000000a0 2c 20 61 6e 20 65 67 67 20 73 61 6e 64 77 69 63 , an egg sandwic\n" + "000000b0 68 2e h. \n"); free(f); } - int main(void) { const struct CMUnitTest tests[] = { cmocka_unit_test(test_buffer_hello), diff --git a/native/test/test_queue.c b/native/test/test_queue.c index 3fa5a9f..c945317 100644 --- a/native/test/test_queue.c +++ b/native/test/test_queue.c @@ -1,30 +1,30 @@ #define _GNU_SOURCE +#include +#include #include #include -#include -#include #include +#include "testdata.h" #include #include -#include "testdata.h" -char* argv0; +char *argv0; -int print_msg(void* ctx, uint64_t index, COBJ y) { - printf("[%" PRIu64 "] %s\n", index, (char*)y); +int print_msg(void *ctx, uint64_t index, COBJ y) { + printf("[%" PRIu64 "] %s\n", index, (char *)y); return 0; } static void queue_init_cleanup(void **state) { - queue_t* queue = chronicle_init("q2"); + queue_t *queue = chronicle_init("q2"); assert_non_null(queue); chronicle_cleanup(queue); } static void queue_not_exist(void **state) { - queue_t* queue = chronicle_init("q2"); + queue_t *queue = chronicle_init("q2"); assert_non_null(queue); assert_int_not_equal(chronicle_open(queue), 0); assert_string_equal(chronicle_strerror(), "dir stat fail"); @@ -32,10 +32,10 @@ static void queue_not_exist(void **state) { } static void queue_is_file(void **state) { - char* temp_dir; + char *temp_dir; asprintf(&temp_dir, "%s/chronicle.test.XXXXXX", P_tmpdir); mkstemp(temp_dir); - queue_t* queue = chronicle_init(temp_dir); + queue_t *queue = chronicle_init(temp_dir); assert_non_null(queue); assert_int_not_equal(chronicle_open(queue), 0); assert_string_equal(chronicle_strerror(), "dir is not a directory"); @@ -47,13 +47,14 @@ static void queue_is_file(void **state) { static void queue_empty_dir_no_ver(void **state) { // this will be an empty directory, so we can't tell the version - char* temp_dir; + char *temp_dir; asprintf(&temp_dir, "%s/chronicle.test.XXXXXX", P_tmpdir); temp_dir = mkdtemp(temp_dir); - queue_t* queue = chronicle_init(temp_dir); + queue_t *queue = chronicle_init(temp_dir); assert_non_null(queue); assert_int_not_equal(chronicle_open(queue), 0); - assert_string_equal(chronicle_strerror(), "queue should exist (no permission to create), but version detect failed"); + assert_string_equal(chronicle_strerror(), + "queue should exist (no permission to create), but version detect failed"); assert_int_equal(chronicle_get_version(queue), 0); chronicle_cleanup(queue); @@ -63,15 +64,15 @@ static void queue_empty_dir_no_ver(void **state) { } static void queue_cqv5_sample_input(void **state) { - uint64_t idx = 0; + uint64_t idx = 0; collected_t result; - char* test_queuedir = unpack_test_data("cqv5-sample-input.tar.bz2", argv0); + char *test_queuedir = unpack_test_data("cqv5-sample-input.tar.bz2", argv0); assert_non_null(test_queuedir); - char* queuedir; + char *queuedir; asprintf(&queuedir, "%s/qv5", test_queuedir); - queue_t* queue = chronicle_init(queuedir); + queue_t *queue = chronicle_init(queuedir); assert_non_null(queue); chronicle_set_decoder(queue, &wire_parse_textonly, &free); @@ -80,54 +81,54 @@ static void queue_cqv5_sample_input(void **state) { assert_int_equal(chronicle_get_version(queue), 5); assert_string_equal(chronicle_get_roll_scheme(queue), "FAST_DAILY"); - tailer_t* tailer = chronicle_tailer(queue, NULL, NULL, 0); + tailer_t *tailer = chronicle_tailer(queue, NULL, NULL, 0); assert_non_null(tailer); assert_int_equal(chronicle_tailer_state(tailer), TS_PEEK); - char* p = (char*)chronicle_collect(tailer, &result); + char *p = (char *)chronicle_collect(tailer, &result); assert_string_equal("one", p); assert_int_equal(result.index, 0x4A0500000000); assert_int_equal(chronicle_tailer_state(tailer), TS_COLLECTED); chronicle_return(tailer, &result); - p = (char*)chronicle_collect(tailer, &result); + p = (char *)chronicle_collect(tailer, &result); assert_int_equal(result.index, 0x4A0500000001); assert_string_equal("two", result.msg); assert_int_equal(4, result.sz); // TODO: bug, should parser be able to re-write this? chronicle_return(tailer, &result); - p = (char*)chronicle_collect(tailer, &result); + p = (char *)chronicle_collect(tailer, &result); assert_int_equal(result.index, 0x4A0500000002); assert_string_equal("three", result.msg); chronicle_return(tailer, &result); - p = (char*)chronicle_collect(tailer, &result); + p = (char *)chronicle_collect(tailer, &result); assert_int_equal(result.index, 0x4A0500000003); assert_string_equal("a much longer item that will need encoding as variable length text", p); chronicle_return(tailer, &result); - wirepad_t* pad = wirepad_init(1024); + wirepad_t *pad = wirepad_init(1024); wirepad_text(pad, "four five"); idx = chronicle_append_ts(queue, pad, 1637267400000L); - assert_int_equal(idx, 0x4A0500000004); // easier to see cycle/index split in hex + assert_int_equal(idx, 0x4A0500000004); // easier to see cycle/index split in hex // write with timestamp that will match recording of 20211118F wirepad_clear(pad); wirepad_text(pad, "six"); idx = chronicle_append_ts(queue, pad, 1637267400000L); // 20211118T203000 - assert_int_equal(idx, 0x4A0500000005); // easier to see cycle/seqnum split in hex + assert_int_equal(idx, 0x4A0500000005); // easier to see cycle/seqnum split in hex // write next day's timestamp wirepad_clear(pad); wirepad_text(pad, "seven"); idx = chronicle_append_ts(queue, pad, 1637308800000L); // 20211119T080000 - assert_int_equal(idx, 0x4a0600000000); // cycle rolled, seqnum reset + assert_int_equal(idx, 0x4a0600000000); // cycle rolled, seqnum reset - p = (char*)chronicle_collect(tailer, &result); + p = (char *)chronicle_collect(tailer, &result); assert_string_equal("four five", p); chronicle_return(tailer, &result); - p = (char*)chronicle_collect(tailer, &result); + p = (char *)chronicle_collect(tailer, &result); assert_string_equal("six", p); chronicle_return(tailer, &result); @@ -139,7 +140,7 @@ static void queue_cqv5_sample_input(void **state) { chronicle_tailer_close(tailer); // add a new tailer starting from midway through - tailer_t* tailer2 = chronicle_tailer(queue, NULL, NULL, 0x4A0500000003); + tailer_t *tailer2 = chronicle_tailer(queue, NULL, NULL, 0x4A0500000003); chronicle_collect(tailer2, &result); assert_int_equal(result.index, 0x4A0500000003); assert_string_equal("a much longer item that will need encoding as variable length text", result.msg); @@ -165,18 +166,18 @@ static void queue_cqv5_sample_input(void **state) { wirepad_free(pad); } -void* parse_cqv4_textonly(unsigned char* base, int lim) { - char* text_result = strndup((const char*) base, lim); +void *parse_cqv4_textonly(unsigned char *base, int lim) { + char *text_result = strndup((const char *)base, lim); return text_result; } static void queue_cqv4_sample_input(void **state) { - char* test_queuedir = unpack_test_data("cqv4-sample-input.tar.bz2", argv0); + char *test_queuedir = unpack_test_data("cqv4-sample-input.tar.bz2", argv0); assert_non_null(test_queuedir); - char* queuedir; + char *queuedir; asprintf(&queuedir, "%s/cqv4", test_queuedir); - queue_t* queue = chronicle_init(queuedir); + queue_t *queue = chronicle_init(queuedir); assert_non_null(queue); chronicle_set_decoder(queue, &parse_cqv4_textonly, &free); chronicle_set_encoder(queue, &wirepad_sizeof, &wirepad_write); @@ -186,10 +187,10 @@ static void queue_cqv4_sample_input(void **state) { collected_t result; - tailer_t* tailer = chronicle_tailer(queue, NULL, NULL, 0); + tailer_t *tailer = chronicle_tailer(queue, NULL, NULL, 0); assert_non_null(tailer); - char* p = (char*)chronicle_collect(tailer, &result); + char *p = (char *)chronicle_collect(tailer, &result); assert_non_null(p); assert_string_equal("one", p); chronicle_return(tailer, &result); @@ -198,11 +199,11 @@ static void queue_cqv4_sample_input(void **state) { assert_string_equal("two", result.msg); chronicle_return(tailer, &result); - p = (char*)chronicle_collect(tailer, &result); + p = (char *)chronicle_collect(tailer, &result); assert_string_equal("three", p); chronicle_return(tailer, &result); - p = (char*)chronicle_collect(tailer, &result); + p = (char *)chronicle_collect(tailer, &result); assert_string_equal("a much longer item that will need encoding as variable length text", p); chronicle_return(tailer, &result); @@ -214,7 +215,7 @@ static void queue_cqv4_sample_input(void **state) { } static void queue_init_rollscheme(void **state) { - queue_t* queue = chronicle_init("/tmp"); + queue_t *queue = chronicle_init("/tmp"); assert_non_null(queue); assert_int_equal(chronicle_get_version(queue), 0); @@ -231,7 +232,7 @@ static void queue_init_rollscheme(void **state) { chronicle_set_roll_scheme(queue, "FAST_HOURLY"); assert_string_equal(chronicle_get_roll_scheme(queue), "FAST_HOURLY"); assert_string_equal(chronicle_get_roll_format(queue), "yyyyMMdd-HH'F'"); - char* p; + char *p; p = chronicle_get_cycle_fn(queue, 0); assert_string_equal(p, "/tmp/19700101-00F.cq4"); free(p); @@ -277,10 +278,10 @@ static void queue_init_rollscheme(void **state) { static void queue_cqv5_new_queue(void **state) { // create queue in an empty directory - char* temp_dir; + char *temp_dir; asprintf(&temp_dir, "%s/chronicle.test.XXXXXX", P_tmpdir); temp_dir = mkdtemp(temp_dir); - queue_t* queue = chronicle_init(temp_dir); + queue_t *queue = chronicle_init(temp_dir); assert_non_null(queue); chronicle_set_version(queue, 5); chronicle_set_roll_scheme(queue, "DAILY"); @@ -291,7 +292,7 @@ static void queue_cqv5_new_queue(void **state) { assert_string_equal(chronicle_get_roll_scheme(queue), "DAILY"); // append will use local clock - we cannot predict the returned index - wirepad_t* pad = wirepad_init(1024); + wirepad_t *pad = wirepad_init(1024); wirepad_text(pad, "four five"); uint64_t idx = chronicle_append(queue, pad); wirepad_free(pad); @@ -308,7 +309,7 @@ static void queue_cqv5_new_queue(void **state) { collected_t result; - tailer_t* tailer = chronicle_tailer(queue, NULL, NULL, 0); + tailer_t *tailer = chronicle_tailer(queue, NULL, NULL, 0); chronicle_collect(tailer, &result); assert_string_equal("four five", result.msg); @@ -323,11 +324,11 @@ static void queue_cqv5_new_queue(void **state) { static void queue_cqv5_new_test4_queue_nodata(void **state) { // create queue in an empty directory - char* temp_dir; + char *temp_dir; asprintf(&temp_dir, "%s/chronicle.test.XXXXXX", P_tmpdir); temp_dir = mkdtemp(temp_dir); - queue_t* queue = chronicle_init(temp_dir); + queue_t *queue = chronicle_init(temp_dir); chronicle_set_version(queue, 5); chronicle_set_roll_scheme(queue, "TEST4_SECONDLY"); chronicle_set_encoder(queue, &wirepad_sizeof, &wirepad_write); @@ -348,7 +349,7 @@ static void queue_cqv5_new_test4_queue_nodata(void **state) { free(temp_dir); } -int main(int argc, char* argv[]) { +int main(int argc, char *argv[]) { argv0 = argv[0]; const struct CMUnitTest tests[] = { cmocka_unit_test(queue_init_cleanup), diff --git a/native/test/test_wire.c b/native/test/test_wire.c index 8546d53..dafbb6f 100644 --- a/native/test/test_wire.c +++ b/native/test/test_wire.c @@ -1,25 +1,26 @@ #define _GNU_SOURCE +#include +#include #include #include -#include -#include #include -#include #include +#include #include -void handle_text(char* buf, int sz, char* data, int dsz, wirecallbacks_t* cbs) { - int* res = (int*)cbs->userdata; +void handle_text(char *buf, int sz, char *data, int dsz, wirecallbacks_t *cbs) { + int *res = (int *)cbs->userdata; *res = (strncmp(buf, "hello", sz) == 0) ? 5 : 1; - if (wire_trace) printf(" got text cb %.*s\n", dsz, data); + if (wire_trace) + printf(" got text cb %.*s\n", dsz, data); } static void test_wirepad_text(void **state) { wire_trace = 0; - wirepad_t* pad = wirepad_init(1024); + wirepad_t *pad = wirepad_init(1024); assert_non_null(pad); assert_int_equal(wirepad_sizeof(pad), 0); @@ -27,18 +28,16 @@ static void test_wirepad_text(void **state) { wirepad_text(pad, "hello"); // this might be padding from the QC layer, perhaps shouldn't be in this test wirepad_pad_to_x8_00(pad); - assert_int_equal(wirepad_sizeof(pad), 1+5+2); - - //wirepad_clear(pad); - //assert_int_equal(wirepad_sizeof(pad), 0); - char* dump = wirepad_hexformat(pad); - assert_string_equal(dump, - "00000000 e5 68 65 6c 6c 6f 00 00 .hello.. \n" - ); + assert_int_equal(wirepad_sizeof(pad), 1 + 5 + 2); + + // wirepad_clear(pad); + // assert_int_equal(wirepad_sizeof(pad), 0); + char *dump = wirepad_hexformat(pad); + assert_string_equal(dump, "00000000 e5 68 65 6c 6c 6f 00 00 .hello.. \n"); free(dump); // run the parser over the pad and check we get one callback - int result = 0; + int result = 0; wirecallbacks_t hcbs; bzero(&hcbs, sizeof(hcbs)); @@ -54,24 +53,21 @@ static void test_wirepad_text(void **state) { static void test_wirepad_fields(void **state) { // example from https://github.com/OpenHFT/Chronicle-Wire#simple-use-case - wirepad_t* pad = wirepad_init(1024); - wirepad_field_text(pad, "message", "Hello World"); - wirepad_field_varint(pad, "number", 1234567890L); - wirepad_field_enum(pad, "code", "SECONDS"); - wirepad_field_float64(pad, "price", 10.50); - - char* dump = wirepad_hexformat(pad); - assert_string_equal(dump, - "00000000 c7 6d 65 73 73 61 67 65 eb 48 65 6c 6c 6f 20 57 .message .Hello W\n" - "00000010 6f 72 6c 64 c6 6e 75 6d 62 65 72 a6 d2 02 96 49 orld.num ber....I\n" - "00000020 c4 63 6f 64 65 e7 53 45 43 4f 4e 44 53 c5 70 72 .code.SE CONDS.pr\n" - "00000030 69 63 65 90 00 00 28 41 ice...(A \n" - ); + wirepad_t *pad = wirepad_init(1024); + wirepad_field_text(pad, "message", "Hello World"); + wirepad_field_varint(pad, "number", 1234567890L); + wirepad_field_enum(pad, "code", "SECONDS"); + wirepad_field_float64(pad, "price", 10.50); + + char *dump = wirepad_hexformat(pad); + assert_string_equal(dump, "00000000 c7 6d 65 73 73 61 67 65 eb 48 65 6c 6c 6f 20 57 .message .Hello W\n" + "00000010 6f 72 6c 64 c6 6e 75 6d 62 65 72 a6 d2 02 96 49 orld.num ber....I\n" + "00000020 c4 63 6f 64 65 e7 53 45 43 4f 4e 44 53 c5 70 72 .code.SE CONDS.pr\n" + "00000030 69 63 65 90 00 00 28 41 ice...(A \n"); free(dump); wirepad_free(pad); }; - static void test_wirepad_metadata(void **state) { wire_trace = 0; // extract of the data written to metadata.cq4t (v5) up to the last non-zero byte @@ -109,41 +105,41 @@ static void test_wirepad_metadata(void **state) { // 00000190 6f 77 6c 65 64 67 65 64 49 6e 64 65 78 52 65 70 owledged IndexRep // 000001a0 6c 69 63 61 74 65 64 a7 ff ff ff ff ff ff ff ff licated. ........ - char* buf="\254\000\000@\271\006header\266\007STStore\202\226\000\000\000\310wireType\266\010" - "WireType\354BINARY_LIGHT\310metadata\266\007SCQMeta\202]\000\000\000\304roll\266\010" - "SCQSRoll\202&\000\000\000\306length\246\000\\&\005\306format\353yyyyMMdd'F'\305epoch" - "\000\327deltaCheckpointInterval@\310sourceId\000\217\217$\000\000\000\271\024" - "listing.highestCycle\216\000\000\000\000\247\375I\000\000\000\000\000\000$\000\000\000\271\023" - "listing.lowestCycle\216\001\000\000\000\000\247\375I\000\000\000\000\000\000\034\000\000\000\271\020" - "listing.modCount\217\247\001\000\000\000\000\000\000\000$\000\000\000\271\024" - "chronicle.write.lock\216\000\000\000\000\247\000\000\000\000\000\000\000\200,\000\000\000\271\035" - "chronicle.lastIndexReplicated\217\217\217\217\247\377\377\377\377\377\377\377\3774\000\000\000\271)" - "chronicle.lastAcknowledgedIndexReplicated\247\377\377\377\377\377\377\377\377"; + char *buf = "\254\000\000@\271\006header\266\007STStore\202\226\000\000\000\310wireType\266\010" + "WireType\354BINARY_LIGHT\310metadata\266\007SCQMeta\202]\000\000\000\304roll\266\010" + "SCQSRoll\202&\000\000\000\306length\246\000\\&\005\306format\353yyyyMMdd'F'\305epoch" + "\000\327deltaCheckpointInterval@\310sourceId\000\217\217$\000\000\000\271\024" + "listing.highestCycle\216\000\000\000\000\247\375I\000\000\000\000\000\000$\000\000\000\271\023" + "listing.lowestCycle\216\001\000\000\000\000\247\375I\000\000\000\000\000\000\034\000\000\000\271\020" + "listing.modCount\217\247\001\000\000\000\000\000\000\000$\000\000\000\271\024" + "chronicle.write.lock\216\000\000\000\000\247\000\000\000\000\000\000\000\200,\000\000\000\271\035" + "chronicle.lastIndexReplicated\217\217\217\217\247\377\377\377\377\377\377\377\3774\000\000\000\271)" + "chronicle.lastAcknowledgedIndexReplicated\247\377\377\377\377\377\377\377\377"; - wirepad_t* pad = wirepad_init(1024); + wirepad_t *pad = wirepad_init(1024); // single metadata message wirepad_qc_start(pad, 1); wirepad_event_name(pad, "header"); wirepad_type_prefix(pad, "STStore"); - wirepad_nest_enter(pad); //header - wirepad_field_type_enum(pad, "wireType", "WireType", "BINARY_LIGHT"); - // field metadata, type prefix SCQMeta, nesting begin - wirepad_field(pad, "metadata"); - wirepad_type_prefix(pad, "SCQMeta"); - wirepad_nest_enter(pad); - wirepad_field(pad, "roll"); - wirepad_type_prefix(pad, "SCQSRoll"); - wirepad_nest_enter(pad); - wirepad_field_varint(pad, "length", 86400000); - wirepad_field_text(pad, "format", "yyyyMMdd'F'"); - wirepad_field_varint(pad, "epoch", 0); - wirepad_nest_exit(pad); - wirepad_field_varint(pad, "deltaCheckpointInterval", 64); - wirepad_field_varint(pad, "sourceId", 0); - wirepad_nest_exit(pad); - wirepad_pad_to_x8(pad); // feels wrong - should be automatic? - wirepad_nest_exit(pad); + wirepad_nest_enter(pad); // header + wirepad_field_type_enum(pad, "wireType", "WireType", "BINARY_LIGHT"); + // field metadata, type prefix SCQMeta, nesting begin + wirepad_field(pad, "metadata"); + wirepad_type_prefix(pad, "SCQMeta"); + wirepad_nest_enter(pad); + wirepad_field(pad, "roll"); + wirepad_type_prefix(pad, "SCQSRoll"); + wirepad_nest_enter(pad); + wirepad_field_varint(pad, "length", 86400000); + wirepad_field_text(pad, "format", "yyyyMMdd'F'"); + wirepad_field_varint(pad, "epoch", 0); + wirepad_nest_exit(pad); + wirepad_field_varint(pad, "deltaCheckpointInterval", 64); + wirepad_field_varint(pad, "sourceId", 0); + wirepad_nest_exit(pad); + wirepad_pad_to_x8(pad); // feels wrong - should be automatic? + wirepad_nest_exit(pad); wirepad_qc_finish(pad); // 6 data messages @@ -180,7 +176,6 @@ static void test_wirepad_metadata(void **state) { assert_memory_equal(buf, wirepad_base(pad), wirepad_sizeof(pad)); wirepad_free(pad); - } int main(void) { diff --git a/native/test/testdata.h b/native/test/testdata.h index 6556f3b..8155244 100644 --- a/native/test/testdata.h +++ b/native/test/testdata.h @@ -1,20 +1,20 @@ #define _GNU_SOURCE +#include #include #include #include #include #include -#include #include #include #include static int copy_data(struct archive *ar, struct archive *aw) { - int r; + int r; const void *buff; - size_t size; + size_t size; #if ARCHIVE_VERSION_NUMBER >= 3000000 int64_t offset; #else @@ -34,23 +34,23 @@ static int copy_data(struct archive *ar, struct archive *aw) { } } -char* unpack_test_data(char* file, char* argv0) { +char *unpack_test_data(char *file, char *argv0) { - char* appdir = strdup(argv0); - char* test_archive; + char *appdir = strdup(argv0); + char *test_archive; asprintf(&test_archive, "%s/%s", dirname(appdir), file); - char* temp_dir; + char *temp_dir; asprintf(&temp_dir, "%s/chronicle.test.XXXXXX", P_tmpdir); temp_dir = mkdtemp(temp_dir); printf("Unpacking test data from %s to %s\n", test_archive, temp_dir); // https://github.com/libarchive/libarchive/wiki/Examples - struct archive *a; - struct archive *ext; + struct archive *a; + struct archive *ext; struct archive_entry *entry; - int r; + int r; a = archive_read_new(); archive_read_support_format_tar(a); @@ -64,15 +64,15 @@ char* unpack_test_data(char* file, char* argv0) { fprintf(stderr, "%s\n", archive_error_string(a)); return NULL; } - char* dest_file; + char *dest_file; for (;;) { r = archive_read_next_header(a, &entry); if (r == ARCHIVE_EOF) - break; + break; if (r < ARCHIVE_OK) - fprintf(stderr, "%s\n", archive_error_string(a)); + fprintf(stderr, "%s\n", archive_error_string(a)); if (r < ARCHIVE_WARN) - return NULL; + return NULL; asprintf(&dest_file, "%s/%s", temp_dir, archive_entry_pathname(entry)); archive_entry_set_pathname(entry, dest_file); @@ -80,19 +80,19 @@ char* unpack_test_data(char* file, char* argv0) { r = archive_write_header(ext, entry); if (r < ARCHIVE_OK) - fprintf(stderr, "%s\n", archive_error_string(ext)); - else if (archive_entry_size(entry) > 0) { - r = copy_data(a, ext); - if (r < ARCHIVE_OK) fprintf(stderr, "%s\n", archive_error_string(ext)); - if (r < ARCHIVE_WARN) - exit(1); + else if (archive_entry_size(entry) > 0) { + r = copy_data(a, ext); + if (r < ARCHIVE_OK) + fprintf(stderr, "%s\n", archive_error_string(ext)); + if (r < ARCHIVE_WARN) + exit(1); } r = archive_write_finish_entry(ext); if (r < ARCHIVE_OK) - fprintf(stderr, "%s\n", archive_error_string(ext)); + fprintf(stderr, "%s\n", archive_error_string(ext)); if (r < ARCHIVE_WARN) - return NULL; + return NULL; free(dest_file); } @@ -107,20 +107,19 @@ char* unpack_test_data(char* file, char* argv0) { } static int rmFiles(const char *pathname, const struct stat *sbuf, int type, struct FTW *ftwb) { - if(remove(pathname) < 0) { + if (remove(pathname) < 0) { fprintf(stderr, "ERROR: remove failed at %s", pathname); return FTW_STOP; } return FTW_CONTINUE; } -int delete_test_data(char* queuedir) { - // Delete the directory and its contents by traversing the tree in reverse order, without crossing mount boundaries and symbolic links - if (nftw(queuedir, rmFiles, 10, FTW_DEPTH|FTW_MOUNT|FTW_PHYS|FTW_ACTIONRETVAL) < 0) { +int delete_test_data(char *queuedir) { + // Delete the directory and its contents by traversing the tree in reverse order, without crossing mount boundaries + // and symbolic links + if (nftw(queuedir, rmFiles, 10, FTW_DEPTH | FTW_MOUNT | FTW_PHYS | FTW_ACTIONRETVAL) < 0) { perror("ERROR: ntfw"); return 1; } return 0; } - - diff --git a/native/wire.c b/native/wire.c index fe6567d..7303ea6 100644 --- a/native/wire.c +++ b/native/wire.c @@ -16,198 +16,225 @@ #include #include -#include #include +#include -#include #include +#include // enables full printf tracing during parsing of wire data (slow) int wire_trace = 0; - -int read_stop_uint(unsigned char* p, int *stopsz) { +int read_stop_uint(unsigned char *p, int *stopsz) { *stopsz = 0; int n = 0; do { n++; - *stopsz = (*stopsz << 7) + (p[n-1] & 0x7F); - } while ((p[n-1] & 0x80) != 0x00); + *stopsz = (*stopsz << 7) + (p[n - 1] & 0x7F); + } while ((p[n - 1] & 0x80) != 0x00); // printf("stopsz %d bytes %d\n", *stopsz, n); return n; } // used for reading index and header structures as well as directory-index.cq4t -void wire_parse(unsigned char* base, int lim, wirecallbacks_t* cbs) { +void wire_parse(unsigned char *base, int lim, wirecallbacks_t *cbs) { // constants from // https://github.com/OpenHFT/Chronicle-Wire/blob/master/src/main/java/net/openhft/chronicle/wire/BinaryWireCode.java // decoder details // https://github.com/OpenHFT/Chronicle-Wire/blob/ea/src/main/java/net/openhft/chronicle/wire/BinaryWire.java - unsigned char* p = base; - uint8_t control; - - char* field_name = NULL; - int field_name_sz = 0; - char* ev_name = NULL; - int ev_name_sz = 0; - char* type_name = NULL; - int type_name_sz = 0; - char* field_text = NULL; - int field_text_sz = 0; - - uint16_t padding16 = 0; - uint32_t padding32 = 0; - uint64_t padding64 = 0; - uint64_t jlong2 = 0; - float float32 = 0; + unsigned char *p = base; + uint8_t control; + + char *field_name = NULL; + int field_name_sz = 0; + char *ev_name = NULL; + int ev_name_sz = 0; + char *type_name = NULL; + int type_name_sz = 0; + char *field_text = NULL; + int field_text_sz = 0; + + uint16_t padding16 = 0; + uint32_t padding32 = 0; + uint64_t padding64 = 0; + uint64_t jlong2 = 0; + float float32 = 0; // track nesting stack - int nest = 0; - unsigned char* pop_pos[10]; - pop_pos[nest] = base+lim; - if (cbs->reset_nesting) cbs->reset_nesting(); + int nest = 0; + unsigned char *pop_pos[10]; + pop_pos[nest] = base + lim; + if (cbs->reset_nesting) + cbs->reset_nesting(); while (p < base + lim) { - control = p[0]; p++; - if (wire_trace) printf(" control 0x%02x\n", control); - switch(control) { - case 0x00 ... 0x7F: // NUM - if (wire_trace) printf(" Field %.*s = %d (uint8)\n", (int)field_name_sz, field_name, control); - if (cbs->field_uint64) cbs->field_uint64(field_name, field_name_sz, (uint64_t)control, cbs); - break; - case 0xB9: // EVENT_NAME - p+= read_stop_uint(p, &ev_name_sz); - ev_name = (char*)p; - field_name = (char*)p; - field_name_sz = ev_name_sz; - p += ev_name_sz; - if (cbs->event_name) cbs->event_name(ev_name, ev_name_sz, cbs); - if (wire_trace) printf("Event name '%.*s'\n", ev_name_sz, ev_name); - break; - case 0x8F: // PADDING - break; - case 0x8E: // PADDING_32 - memcpy(&padding32, p, sizeof(padding32)); - p += 4 + padding32; // not a bug, padding 4 bytes is a counter of extra bytes to skip - break; - case 0x82: // BYTES_LENGTH32, introduces nested structure with length - memcpy(&padding32, p, sizeof(padding32)); - p += 4; - if (wire_trace) printf(" Enter nesting '%.*s' for %d bytes\n", field_name_sz, field_name, padding32); - pop_pos[++nest] = p+padding32; - break; - case 0x8D: // I64_ARRAY - // length/used a, then length*64-bit ints. used for index structured, so expose ptr - memcpy(&padding64, p, sizeof(padding64)); - memcpy(&jlong2, p+8, sizeof(jlong2)); - if (wire_trace) printf(" Field %.*s = [...] (I64_ARRAY) used %" PRIu64 "/%" PRIu64 "\n", field_name_sz, field_name, jlong2, padding64); - p += 16; - if (cbs->ptr_uint64arr) cbs->ptr_uint64arr(field_name, field_name_sz, jlong2, padding64, p, cbs); - p += 8*padding64; - break; - case 0x90: // FLOAT32 - memcpy(&float32, p, sizeof(float32)); - if (wire_trace) printf(" Field %.*s = %f (FLOAT32)\n", field_name_sz, field_name, float32); - // if (cbs->field_float) cbs->field_float(field_name, field_name_sz, float32, cbs); - p += 4; - break; - //case 0x91: // FLOAT64 - // break; - case 0xA5: // INT16 - memcpy(&padding16, p, sizeof(padding16)); - if (wire_trace) printf(" Field %.*s = %hu (uint16)\n", (int)field_name_sz, field_name, padding16); - if (cbs->field_uint64) cbs->field_uint64(field_name, field_name_sz, (uint64_t)padding16, cbs); - p += 2; - break; - case 0xA6: // INT32 - memcpy(&padding32, p, sizeof(padding32)); - if (wire_trace) printf(" Field %.*s = %u (uint32)\n", (int)field_name_sz, field_name, padding32); - if (cbs->field_uint64) cbs->field_uint64(field_name, field_name_sz, (uint64_t)padding32, cbs); - p += 4; - break; - case 0xA7: // INT64 - memcpy(&padding64, p, sizeof(padding64)); - if (wire_trace) printf(" Field %.*s = %" PRIu64 " (uint64)\n", field_name_sz, field_name, padding64); - if (cbs->ptr_uint64) cbs->ptr_uint64(ev_name, ev_name_sz, p, cbs); - if (cbs->field_uint64) cbs->field_uint64(field_name, field_name_sz, padding64, cbs); - p += 8; - break; - case 0xB6: // TYPE_PREFIX - p+= read_stop_uint(p, &type_name_sz); - type_name = (char*)p; - p += type_name_sz; - if (wire_trace) printf("Type prefix !%.*s\n", type_name_sz, type_name); - if (cbs->type_prefix) cbs->type_prefix(type_name, type_name_sz, cbs); - break; - case 0xC0 ... 0xDF: // Small Field start, UTF length encoded in control - field_name = (char*)p; - field_name_sz = control - 0xC0; - // TODO: for UTF characters in utflen, we need to skip additional bytes here - // see BytesInternal.parse8bit_SB1 - // printf(" Field %.*s \n", field_name_sz, field_name); - p += field_name_sz; - break; - case 0xE0 ... 0xFF: // Text field value - field_text = (char*)p; - field_text_sz = control - 0xE0; - if (wire_trace) printf(" Field %.*s = %.*s (text)\n", (int)field_name_sz, field_name, field_text_sz, field_text); - if (cbs->field_char) cbs->field_char(field_name, field_name_sz, field_text, field_text_sz, cbs); - p += field_text_sz; - break; - case 0xB8: // Text any length - p+= read_stop_uint(p, &field_text_sz); - field_text = (char*)p; - if (wire_trace) printf(" Field %.*s = %.*s (text)\n", (int)field_name_sz, field_name, field_text_sz, field_text); - if (cbs->field_char) cbs->field_char(field_name, field_name_sz, field_text, field_text_sz, cbs); - p += field_text_sz; - break; - default: - printf("Aborted at %p (+%04lx) unknown control word %d 0x%02x\n", p-1, p-base, control, control); - p = base + lim; - break; + control = p[0]; + p++; + if (wire_trace) + printf(" control 0x%02x\n", control); + switch (control) { + case 0x00 ... 0x7F: // NUM + if (wire_trace) + printf(" Field %.*s = %d (uint8)\n", (int)field_name_sz, field_name, control); + if (cbs->field_uint64) + cbs->field_uint64(field_name, field_name_sz, (uint64_t)control, cbs); + break; + case 0xB9: // EVENT_NAME + p += read_stop_uint(p, &ev_name_sz); + ev_name = (char *)p; + field_name = (char *)p; + field_name_sz = ev_name_sz; + p += ev_name_sz; + if (cbs->event_name) + cbs->event_name(ev_name, ev_name_sz, cbs); + if (wire_trace) + printf("Event name '%.*s'\n", ev_name_sz, ev_name); + break; + case 0x8F: // PADDING + break; + case 0x8E: // PADDING_32 + memcpy(&padding32, p, sizeof(padding32)); + p += 4 + padding32; // not a bug, padding 4 bytes is a counter of extra bytes to skip + break; + case 0x82: // BYTES_LENGTH32, introduces nested structure with length + memcpy(&padding32, p, sizeof(padding32)); + p += 4; + if (wire_trace) + printf(" Enter nesting '%.*s' for %d bytes\n", field_name_sz, field_name, padding32); + pop_pos[++nest] = p + padding32; + break; + case 0x8D: // I64_ARRAY + // length/used a, then length*64-bit ints. used for index structured, so expose ptr + memcpy(&padding64, p, sizeof(padding64)); + memcpy(&jlong2, p + 8, sizeof(jlong2)); + if (wire_trace) + printf(" Field %.*s = [...] (I64_ARRAY) used %" PRIu64 "/%" PRIu64 "\n", field_name_sz, field_name, + jlong2, padding64); + p += 16; + if (cbs->ptr_uint64arr) + cbs->ptr_uint64arr(field_name, field_name_sz, jlong2, padding64, p, cbs); + p += 8 * padding64; + break; + case 0x90: // FLOAT32 + memcpy(&float32, p, sizeof(float32)); + if (wire_trace) + printf(" Field %.*s = %f (FLOAT32)\n", field_name_sz, field_name, float32); + // if (cbs->field_float) cbs->field_float(field_name, field_name_sz, float32, cbs); + p += 4; + break; + // case 0x91: // FLOAT64 + // break; + case 0xA5: // INT16 + memcpy(&padding16, p, sizeof(padding16)); + if (wire_trace) + printf(" Field %.*s = %hu (uint16)\n", (int)field_name_sz, field_name, padding16); + if (cbs->field_uint64) + cbs->field_uint64(field_name, field_name_sz, (uint64_t)padding16, cbs); + p += 2; + break; + case 0xA6: // INT32 + memcpy(&padding32, p, sizeof(padding32)); + if (wire_trace) + printf(" Field %.*s = %u (uint32)\n", (int)field_name_sz, field_name, padding32); + if (cbs->field_uint64) + cbs->field_uint64(field_name, field_name_sz, (uint64_t)padding32, cbs); + p += 4; + break; + case 0xA7: // INT64 + memcpy(&padding64, p, sizeof(padding64)); + if (wire_trace) + printf(" Field %.*s = %" PRIu64 " (uint64)\n", field_name_sz, field_name, padding64); + if (cbs->ptr_uint64) + cbs->ptr_uint64(ev_name, ev_name_sz, p, cbs); + if (cbs->field_uint64) + cbs->field_uint64(field_name, field_name_sz, padding64, cbs); + p += 8; + break; + case 0xB6: // TYPE_PREFIX + p += read_stop_uint(p, &type_name_sz); + type_name = (char *)p; + p += type_name_sz; + if (wire_trace) + printf("Type prefix !%.*s\n", type_name_sz, type_name); + if (cbs->type_prefix) + cbs->type_prefix(type_name, type_name_sz, cbs); + break; + case 0xC0 ... 0xDF: // Small Field start, UTF length encoded in control + field_name = (char *)p; + field_name_sz = control - 0xC0; + // TODO: for UTF characters in utflen, we need to skip additional bytes here + // see BytesInternal.parse8bit_SB1 + // printf(" Field %.*s \n", field_name_sz, field_name); + p += field_name_sz; + break; + case 0xE0 ... 0xFF: // Text field value + field_text = (char *)p; + field_text_sz = control - 0xE0; + if (wire_trace) + printf(" Field %.*s = %.*s (text)\n", (int)field_name_sz, field_name, field_text_sz, field_text); + if (cbs->field_char) + cbs->field_char(field_name, field_name_sz, field_text, field_text_sz, cbs); + p += field_text_sz; + break; + case 0xB8: // Text any length + p += read_stop_uint(p, &field_text_sz); + field_text = (char *)p; + if (wire_trace) + printf(" Field %.*s = %.*s (text)\n", (int)field_name_sz, field_name, field_text_sz, field_text); + if (cbs->field_char) + cbs->field_char(field_name, field_name_sz, field_text, field_text_sz, cbs); + p += field_text_sz; + break; + default: + printf("Aborted at %p (+%04lx) unknown control word %d 0x%02x\n", p - 1, p - base, control, control); + p = base + lim; + break; } while (p >= pop_pos[nest] && nest > 0) { nest--; - if (wire_trace) printf(" nesting pop\n"); + if (wire_trace) + printf(" nesting pop\n"); } } } struct wirepad { - int sz; - unsigned char* pos; - unsigned char* base; + int sz; + unsigned char *pos; + unsigned char *base; - int nest; - unsigned char* nest_enter_pos[10]; + int nest; + unsigned char *nest_enter_pos[10]; }; -wirepad_t* wirepad_init(int initial_sz) { - wirepad_t* pad = malloc(sizeof(wirepad_t)); +wirepad_t *wirepad_init(int initial_sz) { + wirepad_t *pad = malloc(sizeof(wirepad_t)); pad->sz = initial_sz; pad->base = NULL; pad->base = realloc(pad->base, pad->sz); - if (wire_trace) printf("wire pad created at %p sz %d\n", pad->base, pad->sz); + if (wire_trace) + printf("wire pad created at %p sz %d\n", pad->base, pad->sz); pad->pos = pad->base; pad->nest = 0; return pad; } -void wirepad_free(wirepad_t* pad) { +void wirepad_free(wirepad_t *pad) { free(pad->base); free(pad); } -void wirepad_clear(wirepad_t* pad) { +void wirepad_clear(wirepad_t *pad) { pad->pos = pad->base; pad->nest = 0; } -void wirepad_extent(wirepad_t* pad, int sz) { +void wirepad_extent(wirepad_t *pad, int sz) { int used = pad->pos - pad->base; int remain = pad->sz - used - sz; - if (wire_trace) printf(" wirepad_extent base=%p used=%d need=+%d remain=%d\n", pad->base, used, sz, remain); + if (wire_trace) + printf(" wirepad_extent base=%p used=%d need=+%d remain=%d\n", pad->base, used, sz, remain); while (remain < 0) { pad->sz = pad->sz * 2; pad->base = realloc(pad->base, pad->sz); @@ -215,11 +242,12 @@ void wirepad_extent(wirepad_t* pad, int sz) { } } -void wirepad_text(wirepad_t* pad, char* text) { +void wirepad_text(wirepad_t *pad, char *text) { int d = strlen(text); int overhead = d < 0x1F ? 1 : 4; - if (wire_trace) printf("wirepad_text pos=%p writing d=%d overhead=%d\n", pad->pos, d, overhead); - wirepad_extent(pad, d+overhead); + if (wire_trace) + printf("wirepad_text pos=%p writing d=%d overhead=%d\n", pad->pos, d, overhead); + wirepad_extent(pad, d + overhead); if (d < 0x1F) { pad->pos[0] = 0xE0 + d; } else { @@ -228,27 +256,29 @@ void wirepad_text(wirepad_t* pad, char* text) { printf("long text encoding?"); abort(); } - memcpy(pad->pos+overhead, text, d); + memcpy(pad->pos + overhead, text, d); pad->pos = pad->pos + d + overhead; } -void wirepad_field(wirepad_t* pad, char* text) { +void wirepad_field(wirepad_t *pad, char *text) { int d = strlen(text); int overhead = d < 0x1F ? 1 : 4; - if (wire_trace) printf("wirepad_field pos=%p writing text=%s d=%d overhead=%d\n", pad->pos, text, d, overhead); - wirepad_extent(pad, d+overhead); + if (wire_trace) + printf("wirepad_field pos=%p writing text=%s d=%d overhead=%d\n", pad->pos, text, d, overhead); + wirepad_extent(pad, d + overhead); if (d < 0x1F) { pad->pos[0] = 0xC0 + d; } else { printf("long field encoding?"); abort(); } - memcpy(pad->pos+overhead, text, d); + memcpy(pad->pos + overhead, text, d); pad->pos = pad->pos + overhead + d; } -void wirepad_uint64_aligned(wirepad_t* pad, uint64_t v) { - if (wire_trace) printf("wirepad_uint64 pos=%p writing uint64=%" PRIu64 "\n", pad->pos, v); +void wirepad_uint64_aligned(wirepad_t *pad, uint64_t v) { + if (wire_trace) + printf("wirepad_uint64 pos=%p writing uint64=%" PRIu64 "\n", pad->pos, v); wirepad_extent(pad, 16); // align the 8 data bytes to a multiple of 8 bytes, which @@ -269,18 +299,20 @@ void wirepad_uint64_aligned(wirepad_t* pad, uint64_t v) { uint32_t header = padding - 5; memcpy(pad->pos + 1, &header, sizeof(header)); // set possible byte 5, 6, 7 to zero - for(int i = 5; i < padding; i++) pad->pos[i] = 0; + for (int i = 5; i < padding; i++) + pad->pos[i] = 0; pad->pos += padding; } pad->pos[0] = 0xA7; // INT64 - memcpy(pad->pos+1, &v, sizeof(v)); + memcpy(pad->pos + 1, &v, sizeof(v)); pad->pos = pad->pos + 9; } -void wirepad_varint(wirepad_t* pad, uint64_t v) { +void wirepad_varint(wirepad_t *pad, uint64_t v) { // compacted value representation, unaligned - if (wire_trace) printf("wirepad_varint pos=%p writing varint=%" PRIu64 "\n", pad->pos, v); + if (wire_trace) + printf("wirepad_varint pos=%p writing varint=%" PRIu64 "\n", pad->pos, v); wirepad_extent(pad, 9); if (v <= 0x7F) { @@ -288,66 +320,65 @@ void wirepad_varint(wirepad_t* pad, uint64_t v) { pad->pos = pad->pos + 1; } else if (v <= 0xFFFF) { pad->pos[0] = 0xa5; // INT16 - pad->pos[1] = (v >> 0) & 0xff; - pad->pos[2] = (v >> 8) & 0xff; + pad->pos[1] = (v >> 0) & 0xff; + pad->pos[2] = (v >> 8) & 0xff; pad->pos = pad->pos + 3; } else if (v <= 0xFFFFFFFF) { // 1234567890L => 49 96 02 D2 // a6 d2 02 96 49 pad->pos[0] = 0xa6; // INT32 - pad->pos[1] = (v >> 0) & 0xff; - pad->pos[2] = (v >> 8) & 0xff; + pad->pos[1] = (v >> 0) & 0xff; + pad->pos[2] = (v >> 8) & 0xff; pad->pos[3] = (v >> 16) & 0xff; pad->pos[4] = (v >> 24) & 0xff; pad->pos = pad->pos + 5; } else { pad->pos[0] = 0xA7; // INT64 - memcpy(pad->pos+1, &v, sizeof(v)); + memcpy(pad->pos + 1, &v, sizeof(v)); pad->pos = pad->pos + 9; } } -void wirepad_field_text(wirepad_t* pad, char* field, char* text) { +void wirepad_field_text(wirepad_t *pad, char *field, char *text) { wirepad_field(pad, field); wirepad_text(pad, text); } -void wirepad_field_enum(wirepad_t* pad, char* field, char* text) { +void wirepad_field_enum(wirepad_t *pad, char *field, char *text) { wirepad_field(pad, field); wirepad_text(pad, text); } -void wirepad_field_type_enum(wirepad_t* pad, char* field, char* type, char* text) { +void wirepad_field_type_enum(wirepad_t *pad, char *field, char *type, char *text) { wirepad_field(pad, field); wirepad_type_prefix(pad, type); wirepad_text(pad, text); } -void wirepad_field_uint64(wirepad_t* pad, char* field, uint64_t v) { +void wirepad_field_uint64(wirepad_t *pad, char *field, uint64_t v) { wirepad_field(pad, field); wirepad_uint64_aligned(pad, v); } -void wirepad_field_float64(wirepad_t* pad, char* field, double v) { +void wirepad_field_float64(wirepad_t *pad, char *field, double v) { wirepad_field(pad, field); float f = v; if (f == v) { pad->pos[0] = 0x90; // FLOAT32 - memcpy(pad->pos+1, &f, sizeof(f)); + memcpy(pad->pos + 1, &f, sizeof(f)); pad->pos = pad->pos + 5; } else { printf("float64 large encoding?"); abort(); } - } -void wirepad_field_varint(wirepad_t* pad, char* field, int v) { +void wirepad_field_varint(wirepad_t *pad, char *field, int v) { wirepad_field(pad, field); wirepad_varint(pad, v); } -void wirepad_pad_to_x8(wirepad_t* pad) { +void wirepad_pad_to_x8(wirepad_t *pad) { int padding = -(pad->pos - pad->base) & 0x07; for (int i = 0; i < padding; i++) { pad->pos[i] = 0x8F; @@ -355,7 +386,7 @@ void wirepad_pad_to_x8(wirepad_t* pad) { pad->pos = pad->pos + padding; } -void wirepad_pad_to_x8_00(wirepad_t* pad) { +void wirepad_pad_to_x8_00(wirepad_t *pad) { int padding = -(pad->pos - pad->base) & 0x07; for (int i = 0; i < padding; i++) { pad->pos[i] = 0x00; @@ -363,49 +394,53 @@ void wirepad_pad_to_x8_00(wirepad_t* pad) { pad->pos = pad->pos + padding; } -void wirepad_event_name(wirepad_t* pad, char* event_name) { +void wirepad_event_name(wirepad_t *pad, char *event_name) { int d = strlen(event_name); - if (wire_trace) printf("wirepad_event_name pos=%p name=%s\n", pad->pos, event_name); + if (wire_trace) + printf("wirepad_event_name pos=%p name=%s\n", pad->pos, event_name); if (d > 0xFF) { printf("event_name large encoding?"); abort(); } pad->pos[0] = 0xB9; pad->pos[1] = d & 0xFF; - memcpy(pad->pos+2, event_name, d); + memcpy(pad->pos + 2, event_name, d); pad->pos = pad->pos + 2 + d; } -void wirepad_type_prefix(wirepad_t* pad, char* type_prefix) { +void wirepad_type_prefix(wirepad_t *pad, char *type_prefix) { int d = strlen(type_prefix); - if (wire_trace) printf("wirepad_type_prefix pos=%p name=%s\n", pad->pos, type_prefix); + if (wire_trace) + printf("wirepad_type_prefix pos=%p name=%s\n", pad->pos, type_prefix); if (d > 0xFF) { printf("type_prefix large encoding?"); abort(); } pad->pos[0] = 0xB6; pad->pos[1] = d & 0xFF; - memcpy(pad->pos+2, type_prefix, d); + memcpy(pad->pos + 2, type_prefix, d); pad->pos = pad->pos + 2 + d; } -const uint32_t QC_HD_WORKING = 0x80000000; +const uint32_t QC_HD_WORKING = 0x80000000; const uint32_t QC_HD_METADATA = 0x40000000; -void wirepad_qc_start(wirepad_t* pad, int metadata) { - uint32_t header = QC_HD_WORKING | (metadata==0 ? 0 : QC_HD_METADATA); - if (wire_trace) printf("wirepad_qc_start pos=%p meta=%d nest=%d header=0x%x\n", pad->pos, metadata, pad->nest, header); +void wirepad_qc_start(wirepad_t *pad, int metadata) { + uint32_t header = QC_HD_WORKING | (metadata == 0 ? 0 : QC_HD_METADATA); + if (wire_trace) + printf("wirepad_qc_start pos=%p meta=%d nest=%d header=0x%x\n", pad->pos, metadata, pad->nest, header); wirepad_extent(pad, 9); memcpy(pad->pos, &header, sizeof(header)); pad->nest_enter_pos[pad->nest++] = pad->pos; pad->pos = pad->pos + 4; } -void wirepad_qc_finish(wirepad_t* pad) { +void wirepad_qc_finish(wirepad_t *pad) { pad->nest--; - unsigned char* entered = pad->nest_enter_pos[pad->nest]; - int len = pad->pos - entered - 4; - if (wire_trace) printf("wirepad_qc_finish pos=%p entered=%p len=%d (0x%x) nest=%d\n", pad->pos, entered, len, len, pad->nest); + unsigned char *entered = pad->nest_enter_pos[pad->nest]; + int len = pad->pos - entered - 4; + if (wire_trace) + printf("wirepad_qc_finish pos=%p entered=%p len=%d (0x%x) nest=%d\n", pad->pos, entered, len, len, pad->nest); uint32_t header = 0; memcpy(&header, entered, sizeof(header)); @@ -415,47 +450,45 @@ void wirepad_qc_finish(wirepad_t* pad) { // pad->pos hasn't moved! } -void wirepad_nest_enter(wirepad_t* pad) { - if (wire_trace) printf("wirepad_nest_start pos=%p nest=%d\n", pad->pos, pad->nest); +void wirepad_nest_enter(wirepad_t *pad) { + if (wire_trace) + printf("wirepad_nest_start pos=%p nest=%d\n", pad->pos, pad->nest); wirepad_extent(pad, 5); uint32_t header = 0; pad->pos[0] = 0x82; - memcpy(pad->pos+1, &header, sizeof(header)); - pad->nest_enter_pos[pad->nest++] = pad->pos+1; + memcpy(pad->pos + 1, &header, sizeof(header)); + pad->nest_enter_pos[pad->nest++] = pad->pos + 1; pad->pos = pad->pos + 5; } -void wirepad_nest_exit(wirepad_t* pad) { +void wirepad_nest_exit(wirepad_t *pad) { pad->nest--; - unsigned char* entered = pad->nest_enter_pos[pad->nest]; - int len = pad->pos - entered - 4; - if (wire_trace) printf("wirepad_nest_exit pos=%p entered=%p len=%d (0x%x) nest=%d\n", pad->pos, entered, len, len, pad->nest); + unsigned char *entered = pad->nest_enter_pos[pad->nest]; + int len = pad->pos - entered - 4; + if (wire_trace) + printf("wirepad_nest_exit pos=%p entered=%p len=%d (0x%x) nest=%d\n", pad->pos, entered, len, len, pad->nest); uint32_t header = len; memcpy(entered, &header, sizeof(header)); } -void wirepad_parse(wirepad_t* pad, wirecallbacks_t* cbs) { - wire_parse(pad->base, pad->pos-pad->base, cbs); -} +void wirepad_parse(wirepad_t *pad, wirecallbacks_t *cbs) { wire_parse(pad->base, pad->pos - pad->base, cbs); } -void wirepad_dump(wirepad_t* pad) { - int n = pad->pos-pad->base; +void wirepad_dump(wirepad_t *pad) { + int n = pad->pos - pad->base; printf("wirepad_dump base=%p pos=%p (+%d) sz=%d\n", pad->base, pad->pos, n, pad->sz); printbuf((char *)pad->base, n); - char* p = formatbuf((char*)pad->base, n); + char *p = formatbuf((char *)pad->base, n); printf("%s\n", p); free(p); } -char* wirepad_hexformat(wirepad_t* pad) { - int n = pad->pos-pad->base; - return formatbuf((char*)pad->base, n); +char *wirepad_hexformat(wirepad_t *pad) { + int n = pad->pos - pad->base; + return formatbuf((char *)pad->base, n); } -unsigned char* wirepad_base(wirepad_t* pad) { - return pad->base; -} +unsigned char *wirepad_base(wirepad_t *pad) { return pad->base; } // Integration with queues using BinaryWire for their data pages // these are not strictly part of wire.h but ease integration with @@ -463,28 +496,29 @@ unsigned char* wirepad_base(wirepad_t* pad) { // // writer // sizeof and write take a wirepad_t argument, cast down to void* here for ease of use with libchronicle -size_t wirepad_sizeof(void* msg) { - wirepad_t* pad = (wirepad_t*)msg; +size_t wirepad_sizeof(void *msg) { + wirepad_t *pad = (wirepad_t *)msg; return pad->pos - pad->base; } -void wirepad_write(unsigned char* base, void* msg, size_t sz) { - wirepad_t* pad = (wirepad_t*)msg; +void wirepad_write(unsigned char *base, void *msg, size_t sz) { + wirepad_t *pad = (wirepad_t *)msg; memcpy(base, pad->base, sz); } // parser // an implementation of cparse_f where we capture text and return to new object -void wire_parse_textonly_text(char* field, int fsz, char* data, int dsz, wirecallbacks_t* cbs) { - char** res = (char**)cbs->userdata; +void wire_parse_textonly_text(char *field, int fsz, char *data, int dsz, wirecallbacks_t *cbs) { + char **res = (char **)cbs->userdata; // avoid leak if multiple text entries - return last - if (*res != NULL) free(*res); + if (*res != NULL) + free(*res); *res = strndup(data, dsz); } -void* wire_parse_textonly(unsigned char* base, int lim) { +void *wire_parse_textonly(unsigned char *base, int lim) { wirecallbacks_t cbs; - char* text_result = NULL; + char *text_result = NULL; bzero(&cbs, sizeof(cbs)); cbs.field_char = &wire_parse_textonly_text; diff --git a/native/wire.h b/native/wire.h index 555ae7b..9abfde0 100644 --- a/native/wire.h +++ b/native/wire.h @@ -15,34 +15,33 @@ #ifndef FILE_WIRE_SEEN #define FILE_WIRE_SEEN -#include #include +#include // this library implements the "BinaryWire" self-describing wire protocol // used for chronicle-queue metadata messages. - // global options - debug trace extern int wire_trace; // wire reader - set callbacks then invoke parse_wire typedef struct wirecallbacks { - void (*event_name)(char*,int,struct wirecallbacks*); - void (*type_prefix)(char*,int,struct wirecallbacks*); + void (*event_name)(char *, int, struct wirecallbacks *); + void (*type_prefix)(char *, int, struct wirecallbacks *); - void (*field_uint64)(char*,int,uint64_t,struct wirecallbacks*); - void (*field_char)(char*,int,char*,int,struct wirecallbacks*); + void (*field_uint64)(char *, int, uint64_t, struct wirecallbacks *); + void (*field_char)(char *, int, char *, int, struct wirecallbacks *); // direct pointer access for headers, index arrays - void (*ptr_uint64)(char*,int,unsigned char*,struct wirecallbacks*); - void (*ptr_uint64arr)(char*,int,uint64_t,uint64_t,unsigned char*,struct wirecallbacks*); + void (*ptr_uint64)(char *, int, unsigned char *, struct wirecallbacks *); + void (*ptr_uint64arr)(char *, int, uint64_t, uint64_t, unsigned char *, struct wirecallbacks *); void (*reset_nesting)(); - void* userdata; + void *userdata; } wirecallbacks_t; -void wire_parse(unsigned char* base, int lim, wirecallbacks_t* cbs); +void wire_parse(unsigned char *base, int lim, wirecallbacks_t *cbs); // wire writer // create a wirepad using wirepad_init, then write headers and fields @@ -50,47 +49,47 @@ void wire_parse(unsigned char* base, int lim, wirecallbacks_t* cbs); // over content with wirepad_parse typedef struct wirepad wirepad_t; -wirepad_t* wirepad_init(int initial_sz); -void wirepad_clear(wirepad_t* pad); -void wirepad_dump(wirepad_t* pad); -char* wirepad_hexformat(wirepad_t* pad); -void wirepad_free(wirepad_t* pad); +wirepad_t *wirepad_init(int initial_sz); +void wirepad_clear(wirepad_t *pad); +void wirepad_dump(wirepad_t *pad); +char *wirepad_hexformat(wirepad_t *pad); +void wirepad_free(wirepad_t *pad); -void wirepad_pad_to_x8(wirepad_t* pad); -void wirepad_pad_to_x8_00(wirepad_t* pad); -void wirepad_event_name(wirepad_t* pad, char* event_name); -void wirepad_type_prefix(wirepad_t* pad, char* type_prefix); +void wirepad_pad_to_x8(wirepad_t *pad); +void wirepad_pad_to_x8_00(wirepad_t *pad); +void wirepad_event_name(wirepad_t *pad, char *event_name); +void wirepad_type_prefix(wirepad_t *pad, char *type_prefix); -void wirepad_text(wirepad_t* pad, char* text); -void wirepad_uint64_aligned(wirepad_t* pad, uint64_t v); +void wirepad_text(wirepad_t *pad, char *text); +void wirepad_uint64_aligned(wirepad_t *pad, uint64_t v); -void wirepad_qc_start(wirepad_t* pad, int metadata); -void wirepad_qc_finish(wirepad_t* pad); +void wirepad_qc_start(wirepad_t *pad, int metadata); +void wirepad_qc_finish(wirepad_t *pad); -void wirepad_nest_enter(wirepad_t* pad); -void wirepad_nest_exit(wirepad_t* pad); +void wirepad_nest_enter(wirepad_t *pad); +void wirepad_nest_exit(wirepad_t *pad); -void wirepad_field(wirepad_t* pad, char* text); -void wirepad_field_varint(wirepad_t* pad, char* field, int v); -void wirepad_field_text(wirepad_t* pad, char* field, char* text); -void wirepad_field_uint64(wirepad_t* pad, char* field, uint64_t v); -void wirepad_field_float64(wirepad_t* pad, char* field, double v); -void wirepad_field_enum(wirepad_t* pad, char* field, char* v); -void wirepad_field_type_enum(wirepad_t* pad, char* field, char* type, char* v); +void wirepad_field(wirepad_t *pad, char *text); +void wirepad_field_varint(wirepad_t *pad, char *field, int v); +void wirepad_field_text(wirepad_t *pad, char *field, char *text); +void wirepad_field_uint64(wirepad_t *pad, char *field, uint64_t v); +void wirepad_field_float64(wirepad_t *pad, char *field, double v); +void wirepad_field_enum(wirepad_t *pad, char *field, char *v); +void wirepad_field_type_enum(wirepad_t *pad, char *field, char *type, char *v); -unsigned char* wirepad_base(wirepad_t* pad); -void wirepad_parse(wirepad_t* pad, wirecallbacks_t* cbs); +unsigned char *wirepad_base(wirepad_t *pad); +void wirepad_parse(wirepad_t *pad, wirecallbacks_t *cbs); // Integration with queues using BinaryWire for their data pages // // cparse_f for the common case where the wire format is just text, // and we want to capture the text into a new object. Object must // be freed by the callback cdispatch_f -void* wire_parse_textonly(unsigned char* base, int lim); +void *wire_parse_textonly(unsigned char *base, int lim); // sizeof_f and append_f where the object to be written is a wirepad_t // which is cast down to void* here for ease of use with libchronicle -size_t wirepad_sizeof(void* pad); -void wirepad_write(unsigned char* base, void* pad, size_t sz); +size_t wirepad_sizeof(void *pad); +void wirepad_write(unsigned char *base, void *pad, size_t sz); #endif \ No newline at end of file From f2bc2d37fda607ad3228c93839ff9fc0bdb50a24 Mon Sep 17 00:00:00 2001 From: Chris Shucksmith Date: Mon, 18 Nov 2024 14:51:23 +0000 Subject: [PATCH 2/3] restore #include order, set SortIncludes: Never --- .clang-format | 1 + native/test/test_buffer.c | 3 ++- native/test/test_queue.c | 2 +- native/test/test_wire.c | 2 +- 4 files changed, 5 insertions(+), 3 deletions(-) diff --git a/.clang-format b/.clang-format index 8a6642c..5b6f7a0 100644 --- a/.clang-format +++ b/.clang-format @@ -1,4 +1,5 @@ BasedOnStyle: LLVM +SortIncludes: Never IndentWidth: 4 TabWidth: 4 UseTab: Never diff --git a/native/test/test_buffer.c b/native/test/test_buffer.c index c541be4..e8c5a22 100644 --- a/native/test/test_buffer.c +++ b/native/test/test_buffer.c @@ -1,6 +1,5 @@ #define _GNU_SOURCE -#include #include #include #include @@ -8,6 +7,8 @@ #include #include +#include + #include static void test_buffer_hello(void **state) { diff --git a/native/test/test_queue.c b/native/test/test_queue.c index c945317..a2f9b3f 100644 --- a/native/test/test_queue.c +++ b/native/test/test_queue.c @@ -1,11 +1,11 @@ #define _GNU_SOURCE -#include #include #include #include #include +#include #include "testdata.h" #include #include diff --git a/native/test/test_wire.c b/native/test/test_wire.c index dafbb6f..962f47f 100644 --- a/native/test/test_wire.c +++ b/native/test/test_wire.c @@ -1,6 +1,5 @@ #define _GNU_SOURCE -#include #include #include #include @@ -8,6 +7,7 @@ #include #include +#include #include void handle_text(char *buf, int sz, char *data, int dsz, wirecallbacks_t *cbs) { From 9c634aec1a4af6875937339cffcb8bef66be6092 Mon Sep 17 00:00:00 2001 From: Chris Shucksmith Date: Mon, 18 Nov 2024 15:06:50 +0000 Subject: [PATCH 3/3] bump LLVM --- .github/workflows/make.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/make.yml b/.github/workflows/make.yml index 4265274..0a790b9 100644 --- a/.github/workflows/make.yml +++ b/.github/workflows/make.yml @@ -11,6 +11,9 @@ jobs: steps: - uses: actions/checkout@v2 + - name: LLVM upgrade + shell: bash + run: sudo bash -c "$(wget -O - https://apt.llvm.org/llvm.sh)" - name: Prep shell: bash run: sudo apt install libcmocka-dev libarchive-dev