Add dynamic estimated size for the token array, XML attribute fixes

Author: pascal-fb-martin

README.md

@@ -313,13 +313,19 @@ The echttp library provides functions to handle JSON and XML data: a small JSON
 These parsers support UTF-8 only.
 
 ```
+int echttp_json_estimate (const char *json);
 const char *echttp_json_parse (char *json, ParserToken *token, int *count);
 ```
-Parses the provided JSON string and populate the array of token. The content of the string is modified during the parsing. The variable pointed by count must contain the size of the token array before the call, and is set to the actual number of JSON items found by the parser. The parser return a null pointer on success, or an error message on failure. The error message container is a static buffer and it thus overwritten on the next call.
+Parses the provided JSON string and populate the array of token. The content of the string is modified during the parsing. The variable pointed by count must contain the size of the token array before the call, and is set to the actual number of JSON items found by the parser (even on error). The parser return a null pointer on success, or an error message on failure. The error message container is a static buffer and it thus overwritten on the next call.
+
+The token array must be large enough to hold all the tokens found, or else an error is returned. The `echttp_json_estimate()` function calculates an estimated size for the token array that should be sufficient. if the same token array is used multiple times, it is recommended to allocate an initial size large enough for most common cases and rely on the estimate only to protect against larger data sets. This avoids trashing the heap with frequent re-allocations.
 ```
+int echttp_xml_estimate ((const char *xml);
 const char *echttp_xml_parse (char *xml, ParserToken *token, int *count);
 ```
-Parses the provided XML string and populate the array of token. The content of the string is modified during the parsing. The variable pointed by count must contain the size of the token array before the call, and is set to the actual number of JSON items found by the parser. The parser return a null pointer on success, or an error message on failure. The error message container is a static buffer and it thus overwritten on the next call.
+Parses the provided XML string and populate the array of token. The content of the string is modified during the parsing. The variable pointed by count must contain the size of the token array before the call, and is set to the actual number of JSON items found by the parser (even on error). The parser return a null pointer on success, or an error message on failure. The error message container is a static buffer and it thus overwritten on the next call.
+
+The `echttp_xml_estimate()` function is the XML equivalent of the `echttp_json_estimate()` function.
 
 The ParserToken type is defined as follows:
 ```

echttp_json.c

@@ -26,6 +26,8 @@
  * but is a totally independent implementation, using recursive descent
  * instead of a state machine.
  *
+ * int echttp_json_estimate (const char *json);
+ *
  * const char *echttp_json_parse (char *json, ParserToken *token, int *count);
  *
  *    Decode a JSON string and return a list of tokens. The decoding breaks
@@ -423,6 +425,27 @@ static const char *echttp_json_object (ParserContext context) {
     return "object processing error";
 }
 
+int echttp_json_estimate (const char *json) {
+    // This method of counting does not escape the literal strings content
+    // and does not account for a ',' after an object or array: it might
+    // overestimates the number of tokens needed. This is OK because
+    // we are looking for enough space, not for the smallest space.
+    int count = 0;
+    for (;;) {
+        switch (*(json++)) {
+            case ']':
+            case '}':
+                count += 2; // the item before, plus the object/array.
+                break;
+            case ',':
+                count += 1;
+                break;
+            case 0:
+                return count;
+        }
+    }
+}
+
 void echttp_json_enable_debug (void) {
     echttp_json_debug = 1;
 }
@@ -443,6 +466,7 @@ const char *echttp_json_parse (char *json, ParserToken *token, int *count) {
    context.max = *count;
 
    echttp_json_error_text[0] = 0;
+   *count = 0;
 
    token[0].key = 0;
 

echttp_json.h

@@ -10,6 +10,7 @@
 
 void echttp_json_enable_debug (void);
 
+int echttp_json_estimate (const char *json);
 const char *echttp_json_parse (char *json, ParserToken *token, int *count);
 
 int echttp_json_search (const ParserToken *parent, const char *path);

echttp_print.c

@@ -41,8 +41,6 @@ static char *inbuffer = 0;
 static char *outbuffer = 0;
 static int outbuffer_size = 0;
 
-#define PRINT_MAX 20480
-
 
 static void print_tokens (ParserToken *token, int count) {
     int i;
@@ -90,11 +88,14 @@ int main (int argc, const char **argv) {
     int size;
     int count;
     int show_tokens = 0;
-    int xml_input = 0;
-    int consume_xml = 0;
+    int force_xml = 0; // default is JSON.
+    int xml_input;
     int pretty = PRINT_OPTION_PRETTY;
+    int silent = 0;
     const char *error;
-    ParserToken token[PRINT_MAX];
+    ParserToken *token = 0;
+    int max = 0;
+    int estimated;
 
     for (i = 1; i < argc; ++i) {
         if (strcmp (argv[i], "-d") == 0) {
@@ -107,14 +108,18 @@ int main (int argc, const char **argv) {
             continue;
         }
         if (strcmp (argv[i], "-x") == 0) {
-            consume_xml = 1;
+            force_xml = 1; // Force XML no matter what.
             continue;
         }
         if (strcmp (argv[i], "-r") == 0) {
             pretty = 0;
             continue;
         }
-        xml_input = consume_xml;
+        if (strcmp (argv[i], "-s") == 0) {
+            silent = 1;
+            continue;
+        }
+        xml_input = force_xml;
         if (strstr(argv[i], ".xml")) xml_input = 1;
 
         inbuffer = echttp_parser_load (argv[i]);
@@ -128,26 +133,44 @@ int main (int argc, const char **argv) {
             outbuffer = (char *) realloc (outbuffer, outbuffer_size);
         }
 
-        count = PRINT_MAX;
-        if (consume_xml)
+        if (xml_input) {
+            int estimated = echttp_xml_estimate (inbuffer);
+            printf ("// File %s: estimated %d XML tokens\n", argv[i], estimated);
+            if (estimated > max) {
+                token = realloc (token, estimated * sizeof(*token));
+                max = estimated;
+            }
+            count = max;
             error = echttp_xml_parse (inbuffer, token, &count);
-        else
+        } else {
+            int estimated = echttp_json_estimate (inbuffer);
+            printf ("// File %s: estimated %d JSON tokens\n", argv[i], estimated);
+            if (estimated > max) {
+                token = realloc (token, estimated * sizeof(*token));
+                max = estimated;
+            }
+            count = max;
             error = echttp_json_parse (inbuffer, token, &count);
+        }
         if (error) {
-            fprintf (stderr, "Cannot decode %s: %s\n", argv[i], error);
+            fprintf (stderr,
+                     "%s: error after %d tokens, %s\n", argv[i], count, error);
             continue;
         }
         if (show_tokens) print_tokens (token, count);
         printf ("// File %s (%d characters, %d tokens)\n",
                 argv[i], size, count);
 
-        error = echttp_json_format (token, count, outbuffer, outbuffer_size, pretty);
-        if (error) {
-            fprintf (stderr, "Cannot format: %s: %s\n", argv[i], error);
-            continue;
+        if (!silent) {
+            error = echttp_json_format
+                        (token, count, outbuffer, outbuffer_size, pretty);
+            if (error) {
+                fprintf (stderr, "Cannot format: %s: %s\n", argv[i], error);
+                continue;
+            }
+            printf ("%s", outbuffer);
+            echttp_parser_free (inbuffer);
         }
-        printf ("%s", outbuffer);
-        echttp_parser_free (inbuffer);
     }
 }
 

echttp_xml.c

@@ -26,6 +26,8 @@
  * but is a totally independent implementation, using recursive descent
  * instead of a state machine.
  *
+ * int echttp_xml_estimate (const char *xml);
+ *
  * const char *echttp_xml_parse (char *xml, ParserToken *token, int *count);
  *
  *    Decode a XML string and return a list of tokens. The decoding breaks
@@ -209,6 +211,10 @@ static const char *echttp_xml_tagname (ParserContext context, int index) {
     return 0;
 }
 
+static int echttp_xml_valid_attribute (char c) {
+    return isalnum(c) || c == '_' || c == ':' || c == '-';
+}
+
 static const char *echttp_xml_attributes (ParserContext context, int parent) {
     char *xml = context->xml;
     ParserToken *token = context->token;
@@ -234,7 +240,7 @@ static const char *echttp_xml_attributes (ParserContext context, int parent) {
         token[this_attribute].key = xml + context->cursor;
         do {
             context->cursor += 1;
-        } while (isalnum(xml[context->cursor]));
+        } while (echttp_xml_valid_attribute(xml[context->cursor]));
         if (xml[context->cursor] != '=') return "invalid attributes syntax";
         xml[context->cursor] = 0;
         context->cursor += 1;
@@ -311,6 +317,27 @@ static const char *echttp_xml_element (ParserContext context, int parent) {
     return echttp_xml_content (context, index);
 }
 
+int echttp_xml_estimate (const char *xml) {
+    int count = 1; // Implicit outer object.
+    // This method of counting does not escape the literal strings content,
+    // always assumes a token for attributes and accounts for 4 items per
+    // start/end tag pair instead of the max 3.
+    // This is not a problem because the goal is to estimate a sufficient
+    // space, not the smallest possible space.
+    for (;;) {
+        switch (*(xml++)) {
+            case '=': // Tell-all sign of an attribute.
+                count += 1;
+                break;
+            case '>':
+                count += 2; // total of 4 when content is present (start + end)
+                break;
+            case 0:
+                return count;
+        }
+    }
+}
+
 const char *echttp_xml_parse (char *xml, ParserToken *token, int *count) {
 
    const char *error;
@@ -327,6 +354,7 @@ const char *echttp_xml_parse (char *xml, ParserToken *token, int *count) {
    context.max = *count;
 
    echttp_xml_error_text[0] = 0;
+   *count = 0;
 
    token[0].key = 0;
 
@@ -364,6 +392,7 @@ const char *echttp_xml_parse (char *xml, ParserToken *token, int *count) {
           break;
        default:  return "probably not XML data";
    }
+   *count = context.count;
 
    if (! error) {
        if (next_word(&context) != 0) error = "data left at the end of input";
@@ -375,7 +404,6 @@ const char *echttp_xml_parse (char *xml, ParserToken *token, int *count) {
                  error, context.line_count, context.cursor-context.line_start);
        return echttp_xml_error_text;
    }
-   *count = context.count;
    return 0;
 }
 

echttp_xml.h

@@ -10,5 +10,6 @@
 
 void echttp_xml_enable_debug (void);
 
+int echttp_xml_estimate (const char *xml);
 const char *echttp_xml_parse (char *xml, ParserToken *token, int *count);