From f7b71740380dd9f109e675208b1ed70391daaebc Mon Sep 17 00:00:00 2001 From: Phil Shafer Date: Sun, 20 Mar 2016 16:56:57 -0400 Subject: [PATCH 01/59] Add The Argument Modifier ({a:}); other nits --- doc/libxo.txt | 66 +++++++++++++++++++++++++++++++++++---------------- 1 file changed, 45 insertions(+), 21 deletions(-) diff --git a/doc/libxo.txt b/doc/libxo.txt index 7d60285f..d558e51d 100644 --- a/doc/libxo.txt +++ b/doc/libxo.txt @@ -699,25 +699,26 @@ XOF_WARN is set, a warning will be generated. Field modifiers are flags which modify the way content emitted for particular output styles: -|---+---------------+-------------------------------------------------| -| M | Name | Description | -|---+---------------+-------------------------------------------------| -| c | colon | A colon (":") is appended after the label | -| d | display | Only emit field for display styles (text/HTML) | -| e | encoding | Only emit for encoding styles (XML/JSON) | -| g | gettext | Call gettext on field's render content | -| h | humanize (hn) | Format large numbers in human-readable style | -| | hn-space | Humanize: Place space between numeric and unit | -| | hn-decimal | Humanize: Add a decimal digit, if number < 10 | -| | hn-1000 | Humanize: Use 1000 as divisor instead of 1024 | -| k | key | Field is a key, suitable for XPath predicates | -| l | leaf-list | Field is a leaf-list | -| n | no-quotes | Do not quote the field when using JSON style | -| p | plural | Gettext: Use comma-separated plural form | -| q | quotes | Quote the field when using JSON style | -| t | trim | Trim leading and trailing whitespace | -| w | white | A blank (" ") is appended after the label | -|---+---------------+-------------------------------------------------| +|---+---------------+--------------------------------------------------| +| M | Name | Description | +|---+---------------+--------------------------------------------------| +| a | argument | The content appears as a 'const char *' argument | +| c | colon | A colon (":") is appended after the label | +| d | display | Only emit field for display styles (text/HTML) | +| e | encoding | Only emit for encoding styles (XML/JSON) | +| g | gettext | Call gettext on field's render content | +| h | humanize (hn) | Format large numbers in human-readable style | +| | hn-space | Humanize: Place space between numeric and unit | +| | hn-decimal | Humanize: Add a decimal digit, if number < 10 | +| | hn-1000 | Humanize: Use 1000 as divisor instead of 1024 | +| k | key | Field is a key, suitable for XPath predicates | +| l | leaf-list | Field is a leaf-list | +| n | no-quotes | Do not quote the field when using JSON style | +| p | plural | Gettext: Use comma-separated plural form | +| q | quotes | Quote the field when using JSON style | +| t | trim | Trim leading and trailing whitespace | +| w | white | A blank (" ") is appended after the label | +|---+---------------+--------------------------------------------------| Roles and modifiers can also use more verbose names, when preceeded by a comma. For example, the modifier string "Lwc" (or "L,white,colon") @@ -727,6 +728,27 @@ modifier string "Vkq" (or ":key,quote") means the field has a value role (the default role), that it is a key for the current instance, and that the value should be quoted when encoded for JSON. +**** The Argument Modifier ({a:}) + +The argument modifier indicates that the content of the field +descriptor will be placed as a UTF-8 string (const char *) argument +within the xo_emit parameters. + + EXAMPLE: + xo_emit("{La:} {a:}\n", "Label text", "label", "value"); + TEXT: + Label text value + JSON: + "label": "value" + XML: + + +The argument modifier allows field names for value fields to be passed +on the stack, avoiding the need to build a field descriptor using +snprintf. For many field roles, the argument modifier is not needed, +since those roles have specific mechanisms for arguments, such as +"{C:fg-%s}". + **** The Colon Modifier ({c:}) The colon modifier appends a single colon to the data value: @@ -1663,7 +1685,7 @@ string, since an inappropriate cast can ruin your day. The vap argument to xo_emit_hv() points to a variable argument list that can be used to retrieve arguments via va_arg(). -*** Additional Emitting Functions +*** Single Field Emitting Functions (xo_emit_field) @xo_emit_field@ The following functions can also make output, but only make a single field at a time: @@ -1684,11 +1706,13 @@ would otherwise need to compose a format descriptors using snprintf(). The individual parts of the format descriptor are passed in distinctly. + xo_emit("T", "Host name is ", NULL, NULL); + xo_emit("V", "host-name", NULL, NULL, host-name); + *** Attributes (xo_attr) @xo_attr@ The xo_attr() function emits attributes for the XML output style. - int xo_attr (const char *name, const char *fmt, ...); int xo_attr_h (xo_handle_t *xop, const char *name, const char *fmt, ...); From e4391bf6cbc689ee0bc58ccb6032a38d40bd8e66 Mon Sep 17 00:00:00 2001 From: Phil Shafer Date: Sun, 20 Mar 2016 16:57:07 -0400 Subject: [PATCH 02/59] Add The Argument Modifier ({a:}) --- libxo/libxo.c | 76 +++++++++++++++++++++++++++++++-------------------- 1 file changed, 46 insertions(+), 30 deletions(-) diff --git a/libxo/libxo.c b/libxo/libxo.c index 60cc51d1..b6238e91 100644 --- a/libxo/libxo.c +++ b/libxo/libxo.c @@ -340,6 +340,7 @@ typedef unsigned long xo_xff_flags_t; #define XFF_GT_FIELD (1<<19) /* Call gettext() on a field */ #define XFF_GT_PLURAL (1<<20) /* Call dngettext to find plural form */ +#define XFF_ARGUMENT (1<<21) /* Content provided via argument */ /* Flags to turn off when we don't want i18n processing */ #define XFF_GT_FLAGS (XFF_GT_FIELD | XFF_GT_PLURAL) @@ -3661,10 +3662,9 @@ xo_format_text (xo_handle_t *xop, const char *str, int len) } static void -xo_format_title (xo_handle_t *xop, xo_field_info_t *xfip) +xo_format_title (xo_handle_t *xop, xo_field_info_t *xfip, + const char *str, unsigned len) { - const char *str = xfip->xfi_content; - unsigned len = xfip->xfi_clen; const char *fmt = xfip->xfi_format; unsigned flen = xfip->xfi_flen; xo_xff_flags_t flags = xfip->xfi_flags; @@ -4131,10 +4131,9 @@ xo_format_value (xo_handle_t *xop, const char *name, int nlen, } static void -xo_set_gettext_domain (xo_handle_t *xop, xo_field_info_t *xfip) +xo_set_gettext_domain (xo_handle_t *xop, xo_field_info_t *xfip, + const char *str, unsigned len) { - const char *str = xfip->xfi_content; - unsigned len = xfip->xfi_clen; const char *fmt = xfip->xfi_format; unsigned flen = xfip->xfi_flen; @@ -4389,7 +4388,7 @@ xo_colors_handle_text (xo_handle_t *xop UNUSED, xo_colors_t *newp) char *cp = buf, *ep = buf + sizeof(buf); unsigned i, bit; xo_colors_t *oldp = &xop->xo_colors; - const char *code; + const char *code = NULL; /* * Start the buffer with an escape. We don't want to add the '[' @@ -4508,10 +4507,9 @@ xo_colors_handle_html (xo_handle_t *xop, xo_colors_t *newp) } static void -xo_format_colors (xo_handle_t *xop, xo_field_info_t *xfip) +xo_format_colors (xo_handle_t *xop, xo_field_info_t *xfip, + const char *str, unsigned len) { - const char *str = xfip->xfi_content; - unsigned len = xfip->xfi_clen; const char *fmt = xfip->xfi_format; unsigned flen = xfip->xfi_flen; @@ -4582,10 +4580,9 @@ xo_format_colors (xo_handle_t *xop, xo_field_info_t *xfip) } static void -xo_format_units (xo_handle_t *xop, xo_field_info_t *xfip) +xo_format_units (xo_handle_t *xop, xo_field_info_t *xfip, + const char *str, unsigned len) { - const char *str = xfip->xfi_content; - unsigned len = xfip->xfi_clen; const char *fmt = xfip->xfi_format; unsigned flen = xfip->xfi_flen; xo_xff_flags_t flags = xfip->xfi_flags; @@ -4637,10 +4634,9 @@ xo_format_units (xo_handle_t *xop, xo_field_info_t *xfip) } static int -xo_find_width (xo_handle_t *xop, xo_field_info_t *xfip) +xo_find_width (xo_handle_t *xop, xo_field_info_t *xfip, + const char *str, unsigned len) { - const char *str = xfip->xfi_content; - unsigned len = xfip->xfi_clen; const char *fmt = xfip->xfi_format; unsigned flen = xfip->xfi_flen; @@ -4687,7 +4683,8 @@ xo_anchor_clear (xo_handle_t *xop) * format it when the end anchor tag is seen. */ static void -xo_anchor_start (xo_handle_t *xop, xo_field_info_t *xfip) +xo_anchor_start (xo_handle_t *xop, xo_field_info_t *xfip, + const char *str, unsigned len) { if (xo_style(xop) != XO_STYLE_TEXT && xo_style(xop) != XO_STYLE_HTML) return; @@ -4704,11 +4701,12 @@ xo_anchor_start (xo_handle_t *xop, xo_field_info_t *xfip) * Now we find the width, if possible. If it's not there, * we'll get it on the end anchor. */ - xop->xo_anchor_min_width = xo_find_width(xop, xfip); + xop->xo_anchor_min_width = xo_find_width(xop, xfip, str, len); } static void -xo_anchor_stop (xo_handle_t *xop, xo_field_info_t *xfip) +xo_anchor_stop (xo_handle_t *xop, xo_field_info_t *xfip, + const char *str, unsigned len) { if (xo_style(xop) != XO_STYLE_TEXT && xo_style(xop) != XO_STYLE_HTML) return; @@ -4720,7 +4718,7 @@ xo_anchor_stop (xo_handle_t *xop, xo_field_info_t *xfip) XOIF_CLEAR(xop, XOIF_UNITS_PENDING); - int width = xo_find_width(xop, xfip); + int width = xo_find_width(xop, xfip, str, len); if (width == 0) width = xop->xo_anchor_min_width; @@ -4835,6 +4833,7 @@ static xo_mapping_t xo_role_names[] = { #define XO_ROLE_NEWLINE '\n' static xo_mapping_t xo_modifier_names[] = { + { XFF_ARGUMENT, "argument" }, { XFF_COLON, "colon" }, { XFF_COMMA, "comma" }, { XFF_DISPLAY_ONLY, "display" }, @@ -4906,6 +4905,7 @@ xo_count_fields (xo_handle_t *xop UNUSED, const char *fmt) * '[': start a section of anchored text * ']': end a section of anchored text * The following modifiers are also supported: + * 'a': content is provided via argument (const char *), not descriptor * 'c': flag: emit a colon after the label * 'd': field is only emitted for display styles (text and html) * 'e': field is only emitted for encoding styles (xml and json) @@ -5009,6 +5009,10 @@ xo_parse_roles (xo_handle_t *xop, const char *fmt, fnum = (fnum * 10) + (*sp - '0'); break; + case 'a': + flags |= XFF_ARGUMENT; + break; + case 'c': flags |= XFF_COLON; break; @@ -5314,7 +5318,7 @@ xo_parse_fields (xo_handle_t *xop, xo_field_info_t *fields, xfip->xfi_next = ++sp; /* If we have content, then we have a default format */ - if (xfip->xfi_clen || format) { + if (xfip->xfi_clen || format || (xfip->xfi_flags & XFF_ARGUMENT)) { if (format) { xfip->xfi_format = format; xfip->xfi_flen = flen; @@ -5783,6 +5787,18 @@ xo_do_emit_fields (xo_handle_t *xop, xo_field_info_t *fields, min_fstart = field; } + const char *content = xfip->xfi_content; + int clen = xfip->xfi_clen; + + if (flags & XFF_ARGUMENT) { + /* + * Argument flag means the content isn't given in the descriptor, + * but as a UTF-8 string ('const char *') argument in xo_vap. + */ + content = va_arg(xop->xo_vap, char *); + clen = content ? strlen(content) : 0; + } + if (ftype == XO_ROLE_NEWLINE) { xo_line_close(xop); if (flush_line && xo_flush_h(xop) < 0) @@ -5811,15 +5827,15 @@ xo_do_emit_fields (xo_handle_t *xop, xo_field_info_t *fields, } if (ftype == 'V') - xo_format_value(xop, xfip->xfi_content, xfip->xfi_clen, + xo_format_value(xop, content, clen, xfip->xfi_format, xfip->xfi_flen, xfip->xfi_encoding, xfip->xfi_elen, flags); else if (ftype == '[') - xo_anchor_start(xop, xfip); + xo_anchor_start(xop, xfip, content, clen); else if (ftype == ']') - xo_anchor_stop(xop, xfip); + xo_anchor_stop(xop, xfip, content, clen); else if (ftype == 'C') - xo_format_colors(xop, xfip); + xo_format_colors(xop, xfip, content, clen); else if (ftype == 'G') { /* @@ -5830,7 +5846,7 @@ xo_do_emit_fields (xo_handle_t *xop, xo_field_info_t *fields, * Since gettext returns strings in a static buffer, we make * a copy in new_fmt. */ - xo_set_gettext_domain(xop, xfip); + xo_set_gettext_domain(xop, xfip, content, clen); if (!gettext_inuse) { /* Only translate once */ gettext_inuse = 1; @@ -5881,17 +5897,17 @@ xo_do_emit_fields (xo_handle_t *xop, xo_field_info_t *fields, } continue; - } else if (xfip->xfi_clen || xfip->xfi_format) { + } else if (clen || xfip->xfi_format) { const char *class_name = xo_class_name(ftype); if (class_name) xo_format_content(xop, class_name, xo_tag_name(ftype), - xfip->xfi_content, xfip->xfi_clen, + content, clen, xfip->xfi_format, xfip->xfi_flen, flags); else if (ftype == 'T') - xo_format_title(xop, xfip); + xo_format_title(xop, xfip, content, clen); else if (ftype == 'U') - xo_format_units(xop, xfip); + xo_format_units(xop, xfip, content, clen); else xo_failure(xop, "unknown field type: '%c'", ftype); } From 993237ca2dfccb9d6fb6a6add2be4470b644bd64 Mon Sep 17 00:00:00 2001 From: Phil Shafer Date: Sun, 20 Mar 2016 16:57:28 -0400 Subject: [PATCH 03/59] The Argument Modifier ({a:}); go deep with nroff backslashes --- libxo/xo_format.5 | 318 +++++++++++++++++++++++++--------------------- 1 file changed, 171 insertions(+), 147 deletions(-) diff --git a/libxo/xo_format.5 b/libxo/xo_format.5 index 8c3cbea3..d0e6a33a 100644 --- a/libxo/xo_format.5 +++ b/libxo/xo_format.5 @@ -51,14 +51,14 @@ field descriptions within the format string. .Pp The field description is given as follows: .Bd -literal -offset indent - '{' [ role | modifier ]* [',' long-names ]* ':' [ content ] - [ '/' field-format [ '/' encoding-format ]] '}' + \(aq{\(aq [ role | modifier ]* [\(aq,\(aq long\-names ]* \(aq:\(aq [ content ] + [ \(aq/\(aq field\-format [ \(aq/\(aq encoding\-format ]] \(aq}\(aq .Ed .Pp The role describes the function of the field, while the modifiers enable optional behaviors. -The contents, field-format, and -encoding-format are used in varying ways, based on the role. +The contents, field\-format, and +encoding\-format are used in varying ways, based on the role. These are described in the following sections. .Pp Braces can be escaped by using double braces, similar to "%%" in @@ -68,26 +68,26 @@ The format string "{{braces}}" would emit "{braces}". In the following example, three field descriptors appear. The first is a padding field containing three spaces of padding, the second is a -label ("In stock"), and the third is a value field ("in-stock"). -The in-stock field has a "%u" format that will parse the next argument +label ("In stock"), and the third is a value field ("in\-stock"). +The in\-stock field has a "%u" format that will parse the next argument passed to the .Xr xo_emit 3 , function as an unsigned integer. .Bd -literal -offset indent - xo_emit("{P: }{Lwc:In stock}{:in-stock/%u}\\n", 65); + xo_emit("{P: }{Lwc:In stock}{:in\-stock/%u}\\n", 65); .Ed .Pp This single line of code can generate text ("In stock: 65\\n"), XML -("65"), JSON ('"in-stock": 65'), or HTML (too +("65"), JSON (\(aq"in\-stock": 65\(aq), or HTML (too lengthy to be listed here). .Pp While roles and modifiers typically use single character for brevity, there are alternative names for each which allow more verbose formatting strings. These names must be preceded by a comma, and may follow any -single-character values: +single\-character values: .Bd -literal -offset indent - xo_emit("{L,white,colon:In stock}{,key:in-stock/%u}\n", 65); + xo_emit("{L,white,colon:In stock}{,key:in\-stock/%u}\\n", 65); .Ed .Ss "Field Roles" Field roles are optional, and indicate the role and formatting of the @@ -96,7 +96,7 @@ The roles are listed below; only one role is permitted: .Bl -column "M" "Name12341234" .It Sy "M" "Name " "Description" .It C "color " "Field is a color or effect" -.It D "decoration " "Field is non-text (e.g. colon, comma)" +.It D "decoration " "Field is non\-text (e.g. colon, comma)" .It E "error " "Field is an error message" .It L "label " "Field is text that prefixes a value" .It N "note " "Field is text that follows a value" @@ -105,12 +105,12 @@ The roles are listed below; only one role is permitted: .It U "units " "Field is the units for the previous value field" .It V "value " "Field is the name of field (the default)" .It W "warning " "Field is a warning message" -.It \&[ "start-anchor" "Begin a section of anchored variable-width text" -.It \&] "stop-anchor " "End a section of anchored variable-width text" +.It \&[ "start\-anchor" "Begin a section of anchored variable\-width text" +.It \&] "stop\-anchor " "End a section of anchored variable\-width text" .El .Bd -literal -offset indent EXAMPLE: - xo_emit("{L:Free}{D::}{P: }{:free/%u} {U:Blocks}\n", + xo_emit("{L:Free}{D::}{P: }{:free/%u} {U:Blocks}\\n", free_blocks); .Ed .Pp @@ -121,50 +121,50 @@ a comma: .Bd -literal -offset indent EXAMPLE: xo_emit("{,label:Free}{,decoration::}{,padding: }" - "{,value:free/%u} {,units:Blocks}\n", + "{,value:free/%u} {,units:Blocks}\\n", free_blocks); .Ed .Ss "The Color Role ({C:})" Colors and effects control how text values are displayed; they are used for display styles (TEXT and HTML). .Bd -literal -offset indent - xo_emit("{C:bold}{:value}{C:no-bold}\n", value); + xo_emit("{C:bold}{:value}{C:no\-bold}\\n", value); .Ed .Pp -Colors and effects remain in effect until modified by other "C"-role +Colors and effects remain in effect until modified by other "C"\-role fields. .Bd -literal -offset indent - xo_emit("{C:bold}{C:inverse}both{C:no-bold}only inverse\n"); + xo_emit("{C:bold}{C:inverse}both{C:no\-bold}only inverse\\n"); .Ed .Pp If the content is empty, the "reset" action is performed. .Bd -literal -offset indent - xo_emit("{C:both,underline}{:value}{C:}\n", value); + xo_emit("{C:both,underline}{:value}{C:}\\n", value); .Ed .Pp -The content should be a comma-separated list of zero or more colors or +The content should be a comma\-separated list of zero or more colors or display effects. .Bd -literal -offset indent - xo_emit("{C:bold,underline,inverse}All three{C:no-bold,no-inverse}\n"); + xo_emit("{C:bold,underline,inverse}All three{C:no\-bold,no\-inverse}\\n"); .Ed .Pp The color content can be either static, when placed directly within -the field descriptor, or a printf-style format descriptor can be used, +the field descriptor, or a printf\-style format descriptor can be used, if preceded by a slash ("/"): .Bd -literal -offset indent xo_emit("{C:/%s%s}{:value}{C:}", need_bold ? "bold" : "", need_underline ? "underline" : "", value); .Ed .Pp -Color names are prefixed with either "fg-" or "bg-" to change the +Color names are prefixed with either "fg\-" or "bg\-" to change the foreground and background colors, respectively. .Bd -literal -offset indent - xo_emit("{C:/fg-%s,bg-%s}{Lwc:Cost}{:cost/%u}{C:reset}\n", + xo_emit("{C:/fg\-%s,bg\-%s}{Lwc:Cost}{:cost/%u}{C:reset}\\n", fg_color, bg_color, cost); .Ed .Pp The following table lists the supported effects: -.Bl -column "no-underline" +.Bl -column "no\-underline" .It Sy "Name " "Description" .It "bg\-xxxxx " "Change background color" .It "bold " "Start bold text effect" @@ -179,7 +179,7 @@ The following table lists the supported effects: .El .Pp The following color names are supported: -.Bl -column "no-underline" +.Bl -column "no\-underline" .It Sy "Name" .It black .It blue @@ -193,7 +193,7 @@ The following color names are supported: .El .Ss "The Decoration Role ({D:})" Decorations are typically punctuation marks such as colons, -semi-colons, and commas used to decorate the text and make it simpler +semi\-colons, and commas used to decorate the text and make it simpler for human readers. By marking these distinctly, HTML usage scenarios can use CSS to direct their display parameters. @@ -219,22 +219,23 @@ change such as changing "/%06d" to "/%08d" should not force hand inspection of all .po files. .Pp The simplified version can be generated for a single message using the -"xopo -s " command, or an entire .pot can be translated using -the "xopo -f -o " command. +"xopo \-s " command, or an entire .pot can be translated using +the "xopo \-f \-o " command. .Bd -literal -offset indent - xo_emit("{G:}Invalid token\n"); + xo_emit("{G:}Invalid token\\n"); .Ed +.Pp The {G:} role allows a domain name to be set. .Fn gettext calls will continue to use that domain name until the current format string processing is complete, enabling a library function to emit strings -using it's own catalog. +using it\(aqs own catalog. The domain name can be either static as the content of the field, or a format can be used to get the domain name from the arguments. .Bd -literal -offset indent - xo_emit("{G:libc}Service unavailable in restricted mode\n"); + xo_emit("{G:libc}Service unavailable in restricted mode\\n"); .Ed .Ss "The Label Role ({L:})" Labels are text that appears before a value. @@ -249,7 +250,7 @@ Notes are text that appears after a value. .Ss "The Padding Role ({P:})" Padding represents whitespace used before and between fields. The padding content can be either static, when placed directly within -the field descriptor, or a printf-style format descriptor can be used, +the field descriptor, or a printf\-style format descriptor can be used, if preceded by a slash ("/"): .Bd -literal -offset indent xo_emit("{P: }{Lwc:Cost}{:cost/%u}\\n", cost); @@ -259,7 +260,7 @@ if preceded by a slash ("/"): Titles are heading or column headers that are meant to be displayed to the user. The title can be either static, when placed directly within -the field descriptor, or a printf-style format descriptor can be used, +the field descriptor, or a printf\-style format descriptor can be used, if preceded by a slash ("/"): .Bd -literal -offset indent xo_emit("{T:Interface Statistics}\\n"); @@ -274,7 +275,7 @@ for the previous value field. xo_emit("{Lwc:Distance}{:distance/%u}{Uw:miles}\\n", miles); .Ed .Pp -Note that the sense of the 'w' modifier is reversed for units; +Note that the sense of the \(aqw\(aq modifier is reversed for units; a blank is added before the contents, rather than after it. .Pp When the @@ -286,14 +287,14 @@ attribute: 50 .Ed .Pp -Units can also be rendered in HTML as the "data-units" attribute: +Units can also be rendered in HTML as the "data\-units" attribute: .Bd -literal -offset indent -
50
+
50
.Ed .Ss "The Value Role ({V:} and {:})" The value role is used to represent the a data value that is -interesting for the non-display output styles (XML and JSON). +interesting for the non\-display output styles (XML and JSON). Value is the default role; if no other role designation is given, the field is a value. @@ -356,16 +357,17 @@ Field modifiers are flags which modify the way content emitted for particular output styles: .Bl -column M "Name123456789" .It Sy M "Name " "Description" +.It a "argument " "The content appears as a ""const char *"" argument" .It c "colon " "A colon ("":"") is appended after the label" .It d "display " "Only emit field for display styles (text/HTML)" .It e "encoding " "Only emit for encoding styles (XML/JSON)" -.It h "humanize (hn) " "Format large numbers in human-readable style" -.It " " "hn-space " "Humanize: Place space between numeric and unit" -.It " " "hn-decimal " "Humanize: Add a decimal digit, if number < 10" -.It " " "hn-1000 " "Humanize: Use 1000 as divisor instead of 1024" +.It h "humanize (hn) " "Format large numbers in human\-readable style" +.It " " "hn\-space " "Humanize: Place space between numeric and unit" +.It " " "hn\-decimal " "Humanize: Add a decimal digit, if number < 10" +.It " " "hn\-1000 " "Humanize: Use 1000 as divisor instead of 1024" .It k "key " "Field is a key, suitable for XPath predicates" -.It l "leaf-list " "Field is a leaf-list, a list of leaf values" -.It n "no-quotes " "Do not quote the field when using JSON style" +.It l "leaf\-list " "Field is a leaf\-list, a list of leaf values" +.It n "no\-quotes " "Do not quote the field when using JSON style" .It q "quotes " "Quote the field when using JSON style" .It t "trim " "Trim leading and trailing whitespace" .It w "white space " "A blank ("" "") is appended after the label" @@ -373,7 +375,7 @@ particular output styles: .Pp For example, the modifier string "Lwc" means the field has a label role (text that describes the next field) and should be followed by a -colon ('c') and a space ('w'). +colon (\(aqc\(aq) and a space (\(aqw\(aq). The modifier string "Vkq" means the field has a value role, that it is a key for the current instance, and that the value should be quoted when encoded for JSON. @@ -382,10 +384,31 @@ Roles and modifiers can also use more verbose names, when preceeded by a comma. For example, the modifier string "Lwc" (or "L,white,colon") means the field has a label role (text that describes the next field) -and should be followed by a colon ('c') and a space ('w'). +and should be followed by a colon (\(aqc\(aq) and a space (\(aqw\(aq). The modifier string "Vkq" (or ":key,quote") means the field has a value role (the default role), that it is a key for the current instance, and that the value should be quoted when encoded for JSON. +.Ss "The Argument Modifier ({a:})" +The argument modifier indicates that the content of the field +descriptor will be placed as a UTF\-8 string (const char *) argument +within the xo_emit parameters. +.Bd -literal -offset indent + EXAMPLE: + xo_emit("{La:} {a:}\\n", "Label text", "label", "value"); + TEXT: + Label text value + JSON: + "label": "value" + XML: + +.Ed +.Pp +The argument modifier allows field names for value fields to be passed +on the stack, avoiding the need to build a field descriptor using +.Xr snprintf 1 . +For many field roles, the argument modifier is not needed, +since those roles have specific mechanisms for arguments, +such as "{C:fg\-%s}". .Ss "The Colon Modifier ({c:})" The colon modifier appends a single colon to the data value: .Bd -literal -offset indent @@ -397,7 +420,7 @@ The colon modifier appends a single colon to the data value: .Pp The colon modifier is only used for the TEXT and HTML output styles. -It is commonly combined with the space modifier ('{w:}'). +It is commonly combined with the space modifier (\(aq{w:}\(aq). It is purely a convenience feature. .Ss "The Display Modifier ({d:})" The display modifier indicated the field should only be generated for @@ -429,39 +452,39 @@ The encoding modifier is the opposite of the display modifier, and they are often used to give to distinct views of the underlying data. .Ss "The Humanize Modifier ({h:})" The humanize modifier is used to render large numbers as in a -human-readable format. +human\-readable format. While numbers like "44470272" are completely readable to computers and savants, humans will generally find "44M" more meaningful. .Pp "hn" can be used as an alias for "humanize". .Pp The humanize modifier only affects display styles (TEXT and HMTL). -The "no-humanize" option will block the function of the humanize modifier. +The "no\-humanize" option will block the function of the humanize modifier. .Pp There are a number of modifiers that affect details of humanization. These are only available in as full names, not single characters. -The "hn-space" modifier places a space between the number and any +The "hn\-space" modifier places a space between the number and any multiplier symbol, such as "M" or "K" (ex: "44 K"). -The "hn-decimal" modifier will add a decimal point and a single tenths digit +The "hn\-decimal" modifier will add a decimal point and a single tenths digit when the number is less than 10 (ex: "4.4K"). -The "hn-1000" modifier will use 1000 as divisor instead of 1024, following the -JEDEC-standard instead of the more natural binary powers-of-two +The "hn\-1000" modifier will use 1000 as divisor instead of 1024, following the +JEDEC\-standard instead of the more natural binary powers\-of\-two tradition. .Bd -literal -offset indent EXAMPLE: - xo_emit("{h:input/%u}, {h,hn-space:output/%u}, " - "{h,hn-decimal:errors/%u}, {h,hn-1000:capacity/%u}, " - "{h,hn-decimal:remaining/%u}\n", + xo_emit("{h:input/%u}, {h,hn\-space:output/%u}, " + "{h,hn\-decimal:errors/%u}, {h,hn\-1000:capacity/%u}, " + "{h,hn\-decimal:remaining/%u}\\n", input, output, errors, capacity, remaining); TEXT: 21, 57 K, 96M, 44M, 1.2G .Ed .Pp In the HTML style, the original numeric value is rendered in the -"data-number" attribute on the
element: +"data\-number" attribute on the
element: .Bd -literal -offset indent -
96M
+
96M
.Ed .Ss "The Gettext Modifier ({g:})" The gettext modifier is used to translate individual fields using the @@ -476,9 +499,9 @@ translation. In the following example, the strings "State" and "full" are passed to .Fn gettext -to find locale-based translated strings. +to find locale\-based translated strings. .Bd -literal -offset indent - xo_emit("{Lgwc:State}{g:state}\n", "full"); + xo_emit("{Lgwc:State}{g:state}\\n", "full"); .Ed .Ss "The Key Modifier ({k:})" The key modifier is used to indicate that a particular field helps @@ -499,15 +522,15 @@ Currently the key modifier is only used when generating XPath values for the HTML output style when .Dv XOF_XPATH is set, but other uses are likely in the near future. -.Ss "The Leaf-List Modifier ({l:})" -The leaf-list modifier is used to distinguish lists where each +.Ss "The Leaf\-List Modifier ({l:})" +The leaf\-list modifier is used to distinguish lists where each instance consists of only a single value. In XML, these are rendered as single elements, where JSON renders them as arrays. .Bd -literal -offset indent EXAMPLE: xo_open_list("user"); for (i = 0; i < num_users; i++) { - xo_emit("Member {l:name}\n", user[i].u_name); + xo_emit("Member {l:name}\\n", user[i].u_name); } xo_close_list("user"); XML: @@ -516,8 +539,8 @@ rendered as single elements, where JSON renders them as arrays. JSON: "user": [ "phil", "pallavi" ] .Ed -.Ss "The No-Quotes Modifier ({n:})" -The no-quotes modifier (and its twin, the 'quotes' modifier) affect +.Ss "The No\-Quotes Modifier ({n:})" +The no\-quotes modifier (and its twin, the \(aqquotes\(aq modifier) affect the quoting of values in the JSON output style. JSON uses quotes for string values, but no quotes for numeric, boolean, and null data. @@ -538,8 +561,9 @@ language settings. The contents of the field should be the singular and plural English values, separated by a comma: .Bd -literal -offset indent - xo_emit("{:bytes} {Ngp:byte,bytes}\n", bytes); + xo_emit("{:bytes} {Ngp:byte,bytes}\\n", bytes); .Ed +.Pp The plural modifier is meant to work with the gettext modifier ({g:}) but can work independently. .Pp @@ -554,7 +578,7 @@ function is called to handle the heavy lifting, using the message catalog to convert the singular and plural forms into the native language. .Ss "The Quotes Modifier ({q:})" -The quotes modifier (and its twin, the 'no-quotes' modifier) affect +The quotes modifier (and its twin, the \(aqno-quotes\(aq modifier) affect the quoting of values in the JSON output style. JSON uses quotes for string values, but no quotes for numeric, boolean, and null data. @@ -578,23 +602,23 @@ The white space modifier appends a single space to the data value: .Pp The white space modifier is only used for the TEXT and HTML output styles. -It is commonly combined with the colon modifier ('{c:}'). +It is commonly combined with the colon modifier (\(aq{c:}\(aq). It is purely a convenience feature. .Pp -Note that the sense of the 'w' modifier is reversed for the units role +Note that the sense of the \(aqw\(aq modifier is reversed for the units role ({Uw:}); a blank is added before the contents, rather than after it. .Ss "Field Formatting" The field format is similar to the format string for .Xr printf 3 . Its use varies based on the role of the field, but generally is used to -format the field's contents. +format the field\(aqs contents. .Pp If the format string is not provided for a value field, it defaults to "%s". .Pp -Note a field definition can contain zero or more printf-style +Note a field definition can contain zero or more printf\-style .Dq directives , -which are sequences that start with a '%' and end with +which are sequences that start with a \(aq%\(aq and end with one of following characters: "diouxXDOUeEfFgGaAcCsSp". Each directive is matched by one of more arguments to the @@ -603,54 +627,54 @@ function. .Pp The format string has the form: .Bd -literal -offset indent - '%' format-modifier * format-character + \(aq%\(aq format\-modifier * format\-character .Ed .Pp -The format- modifier can be: +The format\- modifier can be: .Bl -bullet .It -a '#' character, indicating the output value should be prefixed with +a \(aq#\(aq character, indicating the output value should be prefixed with "0x", typically to indicate a base 16 (hex) value. .It -a minus sign ('-'), indicating the output value should be padded on +a minus sign (\(aq\-\(aq), indicating the output value should be padded on the right instead of the left. .It -a leading zero ('0') indicating the output value should be padded on the -left with zeroes instead of spaces (' '). +a leading zero (\(aq0\(aq) indicating the output value should be padded on the +left with zeroes instead of spaces (\(aq \(aq). .It -one or more digits ('0' - '9') indicating the minimum width of the +one or more digits (\(aq0\(aq \- \(aq9\(aq) indicating the minimum width of the argument. If the width in columns of the output value is less than the minimum width, the value will be padded to reach the minimum. .It a period followed by one or more digits indicating the maximum number of bytes which will be examined for a string argument, or the maximum -width for a non-string argument. +width for a non\-string argument. When handling ASCII strings this -functions as the field width but for multi-byte characters, a single +functions as the field width but for multi\-byte characters, a single character may be composed of multiple bytes. .Xr xo_emit 3 will never dereference memory beyond the given number of bytes. .It a second period followed by one or more digits indicating the maximum width for a string argument. -This modifier cannot be given for non-string arguments. +This modifier cannot be given for non\-string arguments. .It -one or more 'h' characters, indicating shorter input data. +one or more \(aqh\(aq characters, indicating shorter input data. .It -one or more 'l' characters, indicating longer input data. +one or more \(aql\(aq characters, indicating longer input data. .It -a 'z' character, indicating a 'size_t' argument. +a \(aqz\(aq character, indicating a \(aqsize_t\(aq argument. .It -a 't' character, indicating a 'ptrdiff_t' argument. +a \(aqt\(aq character, indicating a \(aqptrdiff_t\(aq argument. .It -a ' ' character, indicating a space should be emitted before +a \(aq \(aq character, indicating a space should be emitted before positive numbers. .It -a '+' character, indicating sign should emitted before any number. +a \(aq+\(aq character, indicating sign should emitted before any number. .El .Pp -Note that 'q', 'D', 'O', and 'U' are considered deprecated and will be +Note that \(aqq\(aq, \(aqD\(aq, \(aqO\(aq, and \(aqU\(aq are considered deprecated and will be removed eventually. .Pp The format character is described in the following table: @@ -665,22 +689,22 @@ The format character is described in the following table: .It D "long " "base 10 (decimal)" .It O "unsigned long " "base 8 (octal)" .It U "unsigned long " "base 10 (decimal)" -.It e "double " "[-]d.ddde+-dd" -.It E "double " "[-]d.dddE+-dd" -.It f "double " "[-]ddd.ddd" -.It F "double " "[-]ddd.ddd" -.It g "double " "as 'e' or 'f'" -.It G "double " "as 'E' or 'F'" -.It a "double " "[-]0xh.hhhp[+-]d" -.It A "double " "[-]0Xh.hhhp[+-]d" +.It e "double " "[\-]d.ddde+\-dd" +.It E "double " "[\-]d.dddE+\-dd" +.It f "double " "[\-]ddd.ddd" +.It F "double " "[\-]ddd.ddd" +.It g "double " "as \(aqe\(aq or \(aqf\(aq" +.It G "double " "as \(aqE\(aq or \(aqF\(aq" +.It a "double " "[\-]0xh.hhhp[+\-]d" +.It A "double " "[\-]0Xh.hhhp[+\-]d" .It c "unsigned char " "a character" .It C "wint_t " "a character" -.It s "char * " "a UTF-8 string" +.It s "char * " "a UTF\-8 string" .It S "wchar_t * " "a unicode/WCS string" -.It p "void * " "'%#lx'" +.It p "void * " "\(aq%#lx\(aq" .El .Pp -The 'h' and 'l' modifiers affect the size and treatment of the +The \(aqh\(aq and \(aql\(aq modifiers affect the size and treatment of the argument: .Bl -column "Mod" "d, i " "o, u, x, X " .It Sy "Mod" "d, i " "o, u, x, X" @@ -693,27 +717,27 @@ argument: .It "z " "size_t " "size_t" .It "q " "quad_t " "u_quad_t" .El -.Ss "UTF-8 and Locale Strings" +.Ss "UTF\-8 and Locale Strings" All strings for .Nm libxo -must be UTF-8. +must be UTF\-8. .Nm libxo will handle turning them -into locale-based strings for display to the user. +into locale\-based strings for display to the user. .Pp -For strings, the 'h' and 'l' modifiers affect the interpretation of +For strings, the \(aqh\(aq and \(aql\(aq modifiers affect the interpretation of the bytes pointed to argument. -The default '%s' string is a 'char *' -pointer to a string encoded as UTF-8. -Since UTF-8 is compatible with +The default \(aq%s\(aq string is a \(aqchar *\(aq +pointer to a string encoded as UTF\-8. +Since UTF\-8 is compatible with .Em ASCII -data, a normal 7-bit +data, a normal 7\-bit .Em ASCII string can be used. "%ls" expects a -"wchar_t *" pointer to a wide-character string, encoded as 32-bit +"wchar_t *" pointer to a wide\-character string, encoded as 32\-bit Unicode values. -"%hs" expects a "char *" pointer to a multi-byte +"%hs" expects a "char *" pointer to a multi\-byte string encoded with the current locale, as given by the .Ev LC_CTYPE , .Ev LANG , @@ -722,22 +746,22 @@ or environment variables. The first of this list of variables is used and if none of the variables are set, the locale defaults to -.Em UTF-8 . +.Em UTF\-8 . .Pp .Nm libxo will -convert these arguments as needed to either UTF-8 (for XML, JSON, and -HTML styles) or locale-based strings for display in text style. +convert these arguments as needed to either UTF\-8 (for XML, JSON, and +HTML styles) or locale\-based strings for display in text style. .Bd -literal -offset indent - xo_emit("All strings are utf-8 content {:tag/%ls}", + xo_emit("All strings are utf\-8 content {:tag/%ls}", L"except for wide strings"); .Ed .Pp "%S" is equivalent to "%ls". .Pp -For example, a function is passed a locale-base name, a hat size, +For example, a function is passed a locale\-base name, a hat size, and a time value. -The hat size is formatted in a UTF-8 (ASCII) +The hat size is formatted in a UTF\-8 (ASCII) string, and the time value is formatted into a wchar_t string. .Bd -literal -offset indent void print_order (const char *name, int size, @@ -755,7 +779,7 @@ string, and the time value is formatted into a wchar_t string. xo_emit("The hat for {:name/%hs} is {:size/%s}.\\n", name, size_val); - xo_emit("It was ordered on {:order-time/%ls}.\\n", + xo_emit("It was ordered on {:order\-time/%ls}.\\n", when); } .Ed @@ -766,11 +790,11 @@ will perform the conversion required to make appropriate output. Text style output uses the current locale (as described above), while XML, JSON, and HTML use -UTF-8. +UTF\-8. .Pp -UTF-8 and locale-encoded strings can use multiple bytes to encode one +UTF\-8 and locale\-encoded strings can use multiple bytes to encode one column of data. -The traditional "precision'" (aka "max-width") value +The traditional "precision" (aka "max\-width") value for "%s" printf formatting becomes overloaded since it specifies both the number of bytes that can be safely referenced and the maximum number of columns to emit. @@ -800,12 +824,12 @@ For HTML, these characters are placed in a
with class "text". "size": "extra small" HTML:
The hat is
-
extra small
+
extra small
.
.Ed -.Ss "'%n' is Not Supported" +.Ss "\(aq%n\(aq is Not Supported" .Nm libxo -does not support the '%n' directive. +does not support the \(aq%n\(aq directive. It is a bad idea and we just do not do it. .Ss "The Encoding Format (eformat)" @@ -817,7 +841,7 @@ If the primary is not given, both default to "%s". .Sh EXAMPLE In this example, the value for the number of items in stock is emitted: .Bd -literal -offset indent - xo_emit("{P: }{Lwc:In stock}{:in-stock/%u}\\n", + xo_emit("{P: }{Lwc:In stock}{:in\-stock/%u}\\n", instock); .Ed .Pp @@ -826,16 +850,16 @@ This call will generate the following output: TEXT: In stock: 144 XML: - 144 + 144 JSON: - "in-stock": 144, + "in\-stock": 144, HTML:
In stock
:
-
144
+
144
.Ed .Pp @@ -846,10 +870,10 @@ or .Dv XOF_INFO data, which would expand the penultimate line to: .Bd -literal -offset indent -
144
+
144
.Ed .Sh WHAT MAKES A GOOD FIELD NAME? To make useful, consistent field names, follow these guidelines: @@ -867,23 +891,23 @@ But the raw field name should use hyphens. .Ss "Use full words" Do not abbreviate especially when the abbreviation is not obvious or not widely used. -Use "data-size", not "dsz" or "dsize". +Use "data\-size", not "dsz" or "dsize". Use -"interface" instead of "ifname", "if-name", "iface", "if", or "intf". -.Ss "Use -" -Using the form - or -- helps in +"interface" instead of "ifname", "if\-name", "iface", "if", or "intf". +.Ss "Use \-" +Using the form \- or \-\- helps in making consistent, useful names, avoiding the situation where one app -uses "sent-packet" and another "packets-sent" and another -"packets-we-have-sent". +uses "sent\-packet" and another "packets\-sent" and another +"packets\-we\-have\-sent". The can be dropped when it is obvious, as can obvious words in the classification. -Use "receive-after-window-packets" instead of -"received-packets-of-data-after-window". +Use "receive\-after\-window\-packets" instead of +"received\-packets\-of\-data\-after\-window". .Ss "Reuse existing field names" Nothing is worse than writing expressions like: .Bd -literal -offset indent if ($src1/process[pid == $pid]/name == - $src2/proc-table/proc/p[process-id == $pid]/proc-name) { + $src2/proc\-table/proc/p[process\-id == $pid]/proc\-name) { ... } .Ed @@ -903,7 +927,7 @@ calls or "{e:}" fields to make the data useful. .Ss "Do not use an arbitrary number postfix" What does "errors2" mean? No one will know. -"errors-after-restart" would be a better choice. +"errors\-after\-restart" would be a better choice. Think of your users, and think of the future. If you make "errors2", the next guy will happily make "errors3" and before you know it, someone will be asking what is the @@ -913,7 +937,7 @@ Think of your field vocabulary as an API. You want it useful, expressive, meaningful, direct, and obvious. You want the client -application's programmer to move between without the need to +application\(aqs programmer to move between without the need to understand a variety of opinions on how fields are named. They should see the system as a single cohesive whole, not a sack of cats. @@ -925,12 +949,12 @@ By choosing wise names now, you are making their lives better. After using .Xr xolint 1 to find errors in your field descriptors, use -.Dq "xolint -V" +.Dq "xolint \-V" to spell check your field names and to detect different names for the same data. -.Dq dropped-short +.Dq dropped\-short and -.Dq dropped-too-short +.Dq dropped\-too\-short are both reasonable names, but using them both will lead users to ask the difference between the two fields. If there is no difference, From 0fdd479fb9ff10f1fa4014805aae373875bc2ada Mon Sep 17 00:00:00 2001 From: Phil Shafer Date: Sun, 20 Mar 2016 16:57:33 -0400 Subject: [PATCH 04/59] update test cases --- tests/core/saved/test_01.E.out | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/tests/core/saved/test_01.E.out b/tests/core/saved/test_01.E.out index fa0ce656..de23baad 100644 --- a/tests/core/saved/test_01.E.out +++ b/tests/core/saved/test_01.E.out @@ -1,5 +1,10 @@ op create: [] [] op open_container: [top] [] +op string: [host] [my-box] +op string: [domain] [example.com] +op string: [host] [my-box] +op string: [domain] [example.com] +op string: [label] [value] op string: [max-chaos] [very] op content: [min-chaos] [42] op string: [some-chaos] [[42]] From 1b042b88fb3f50d8fd49180a648909a5484de4ab Mon Sep 17 00:00:00 2001 From: Phil Shafer Date: Sun, 20 Mar 2016 16:57:35 -0400 Subject: [PATCH 05/59] update test cases --- tests/core/saved/test_01.H.out | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/core/saved/test_01.H.out b/tests/core/saved/test_01.H.out index d0d9b3af..7c0b3de0 100644 --- a/tests/core/saved/test_01.H.out +++ b/tests/core/saved/test_01.H.out @@ -1,2 +1,2 @@ -
very
42
42 +
testing argument modifier
my-box
.
example.com
...
testing argument modifier with encoding to
.
example.com
...
Label text
value
very
42
42
Connecting to
my-box
.
example.com
...
Item
Total Sold
In Stock
On Order
SKU
gum
1412
54
10
GRO-000-415
rope
85
4
2
HRD-000-212
ladder
0
2
1
HRD-000-517
bolt
4123
144
42
HRD-000-632
water
17
14
2
GRO-000-2331
Item
'
gum
':
Total sold
:
1412.0
In stock
:
54
On order
:
10
SKU
:
GRO-000-415
Item
'
rope
':
Total sold
:
85.0
In stock
:
4
On order
:
2
SKU
:
HRD-000-212
Item
'
ladder
':
Total sold
:
0
In stock
:
2
On order
:
1
SKU
:
HRD-000-517
Item
'
bolt
':
Total sold
:
4123.0
In stock
:
144
On order
:
42
SKU
:
HRD-000-632
Item
'
water
':
Total sold
:
17.0
In stock
:
14
On order
:
2
SKU
:
GRO-000-2331
Item
'
fish
':
Total sold
:
1321.0
In stock
:
45
On order
:
1
SKU
:
GRO-000-533
Item
:
gum
Item
:
rope
Item
:
ladder
Item
:
bolt
Item
:
water
X
X
X
X
X
X
X
X
X
X
Cost
:
425
X
X
Cost
:
455
links
user
group
/some/file
1
user
group
\ No newline at end of file From 07ff4fd39be6a11b07b365ff2e421dc2958fe42f Mon Sep 17 00:00:00 2001 From: Phil Shafer Date: Sun, 20 Mar 2016 16:57:36 -0400 Subject: [PATCH 06/59] update test cases --- tests/core/saved/test_01.HIPx.out | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/tests/core/saved/test_01.HIPx.out b/tests/core/saved/test_01.HIPx.out index d2da572c..f6c72903 100644 --- a/tests/core/saved/test_01.HIPx.out +++ b/tests/core/saved/test_01.HIPx.out @@ -1,3 +1,21 @@ +
+
testing argument modifier
+
my-box
+
.
+
example.com
+
...
+
+
+
testing argument modifier with encoding to
+
.
+
example.com
+
...
+
+
+
Label text
+
+
value
+
very
42
From 21eb4046bf9c44fe36864b7d5a7fdf63b89c28be Mon Sep 17 00:00:00 2001 From: Phil Shafer Date: Sun, 20 Mar 2016 16:57:37 -0400 Subject: [PATCH 07/59] update test cases --- tests/core/saved/test_01.HP.out | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/tests/core/saved/test_01.HP.out b/tests/core/saved/test_01.HP.out index 85a31fa0..0fdcbd18 100644 --- a/tests/core/saved/test_01.HP.out +++ b/tests/core/saved/test_01.HP.out @@ -1,3 +1,21 @@ +
+
testing argument modifier
+
my-box
+
.
+
example.com
+
...
+
+
+
testing argument modifier with encoding to
+
.
+
example.com
+
...
+
+
+
Label text
+
+
value
+
very
42
From 532b7807d1248b529801cc0faadf8418b2350798 Mon Sep 17 00:00:00 2001 From: Phil Shafer Date: Sun, 20 Mar 2016 16:57:38 -0400 Subject: [PATCH 08/59] update test cases --- tests/core/saved/test_01.J.out | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/core/saved/test_01.J.out b/tests/core/saved/test_01.J.out index d799f6ee..3fc12a17 100644 --- a/tests/core/saved/test_01.J.out +++ b/tests/core/saved/test_01.J.out @@ -1,2 +1,2 @@ -{"top": {"max-chaos":"very","min-chaos":42,"some-chaos":"[42]","host":"my-box","domain":"example.com", "data": {"item": [{"sku":"GRO-000-415","name":"gum","sold":1412,"in-stock":54,"on-order":10}, {"sku":"HRD-000-212","name":"rope","sold":85,"in-stock":4,"on-order":2}, {"sku":"HRD-000-517","name":"ladder","sold":0,"in-stock":2,"on-order":1}, {"sku":"HRD-000-632","name":"bolt","sold":4123,"in-stock":144,"on-order":42}, {"sku":"GRO-000-2331","name":"water","sold":17,"in-stock":14,"on-order":2}]}, "data2": {"item": [{"sku":"GRO-000-415","name":"gum","sold":1412.0,"in-stock":54,"on-order":10}, {"sku":"HRD-000-212","name":"rope","sold":85.0,"in-stock":4,"on-order":2}, {"sku":"HRD-000-517","name":"ladder","sold":0,"in-stock":2,"on-order":1}, {"sku":"HRD-000-632","name":"bolt","sold":4123.0,"in-stock":144,"on-order":42}, {"sku":"GRO-000-2331","name":"water","sold":17.0,"in-stock":14,"on-order":2}]}, "data3": {"item": [{"sku":"GRO-000-533","name":"fish","sold":1321.0,"in-stock":45,"on-order":1}]}, "data4": {"item": ["gum","rope","ladder","bolt","water"]},"cost":425,"cost":455,"mode":"mode","mode_octal":"octal","links":"links","user":"user","group":"group","mode":"/some/file","mode_octal":640,"links":1,"user":"user","group":"group"} +{"top": {"host":"my-box","domain":"example.com","host":"my-box","domain":"example.com","label":"value","max-chaos":"very","min-chaos":42,"some-chaos":"[42]","host":"my-box","domain":"example.com", "data": {"item": [{"sku":"GRO-000-415","name":"gum","sold":1412,"in-stock":54,"on-order":10}, {"sku":"HRD-000-212","name":"rope","sold":85,"in-stock":4,"on-order":2}, {"sku":"HRD-000-517","name":"ladder","sold":0,"in-stock":2,"on-order":1}, {"sku":"HRD-000-632","name":"bolt","sold":4123,"in-stock":144,"on-order":42}, {"sku":"GRO-000-2331","name":"water","sold":17,"in-stock":14,"on-order":2}]}, "data2": {"item": [{"sku":"GRO-000-415","name":"gum","sold":1412.0,"in-stock":54,"on-order":10}, {"sku":"HRD-000-212","name":"rope","sold":85.0,"in-stock":4,"on-order":2}, {"sku":"HRD-000-517","name":"ladder","sold":0,"in-stock":2,"on-order":1}, {"sku":"HRD-000-632","name":"bolt","sold":4123.0,"in-stock":144,"on-order":42}, {"sku":"GRO-000-2331","name":"water","sold":17.0,"in-stock":14,"on-order":2}]}, "data3": {"item": [{"sku":"GRO-000-533","name":"fish","sold":1321.0,"in-stock":45,"on-order":1}]}, "data4": {"item": ["gum","rope","ladder","bolt","water"]},"cost":425,"cost":455,"mode":"mode","mode_octal":"octal","links":"links","user":"user","group":"group","mode":"/some/file","mode_octal":640,"links":1,"user":"user","group":"group"} } From 93c47fde55b434bfd5f7c04d381b40a4ea1bc7e0 Mon Sep 17 00:00:00 2001 From: Phil Shafer Date: Sun, 20 Mar 2016 16:57:39 -0400 Subject: [PATCH 09/59] update test cases --- tests/core/saved/test_01.JP.out | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/tests/core/saved/test_01.JP.out b/tests/core/saved/test_01.JP.out index 1d325257..2c7397f5 100644 --- a/tests/core/saved/test_01.JP.out +++ b/tests/core/saved/test_01.JP.out @@ -1,5 +1,10 @@ { "top": { + "host": "my-box", + "domain": "example.com", + "host": "my-box", + "domain": "example.com", + "label": "value", "max-chaos": "very", "min-chaos": 42, "some-chaos": "[42]", From 5525ffd26e3a6943d30225c4f5b1ef993f2544fc Mon Sep 17 00:00:00 2001 From: Phil Shafer Date: Sun, 20 Mar 2016 16:57:41 -0400 Subject: [PATCH 10/59] update test cases --- tests/core/saved/test_01.T.out | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tests/core/saved/test_01.T.out b/tests/core/saved/test_01.T.out index dcd8048b..71cd130a 100644 --- a/tests/core/saved/test_01.T.out +++ b/tests/core/saved/test_01.T.out @@ -1,3 +1,6 @@ +testing argument modifier my-box.example.com... +testing argument modifier with encoding to .example.com... +Label text value very 4242 Connecting to my-box.example.com... Item Total Sold In Stock On Order SKU From 8b0edb8b4376a15de4ae92604cbeda6db79d2ad4 Mon Sep 17 00:00:00 2001 From: Phil Shafer Date: Sun, 20 Mar 2016 16:57:42 -0400 Subject: [PATCH 11/59] update test cases --- tests/core/saved/test_01.X.out | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/core/saved/test_01.X.out b/tests/core/saved/test_01.X.out index a0535af8..da80e681 100644 --- a/tests/core/saved/test_01.X.out +++ b/tests/core/saved/test_01.X.out @@ -1 +1 @@ -very42[42]my-boxexample.comGRO-000-415gum14125410HRD-000-212rope8542HRD-000-517ladder021HRD-000-632bolt412314442GRO-000-2331water17142GRO-000-415gum1412.05410HRD-000-212rope85.042HRD-000-517ladder021HRD-000-632bolt4123.014442GRO-000-2331water17.0142GRO-000-533fish1321.0451gumropeladderboltwater425455modeoctallinksusergroup/some/file6401usergroup \ No newline at end of file +my-boxexample.commy-boxexample.comvery42[42]my-boxexample.comGRO-000-415gum14125410HRD-000-212rope8542HRD-000-517ladder021HRD-000-632bolt412314442GRO-000-2331water17142GRO-000-415gum1412.05410HRD-000-212rope85.042HRD-000-517ladder021HRD-000-632bolt4123.014442GRO-000-2331water17.0142GRO-000-533fish1321.0451gumropeladderboltwater425455modeoctallinksusergroup/some/file6401usergroup \ No newline at end of file From 9551fac0956e30f76df0a9eaca00ace1c510a042 Mon Sep 17 00:00:00 2001 From: Phil Shafer Date: Sun, 20 Mar 2016 16:57:44 -0400 Subject: [PATCH 12/59] update test cases --- tests/core/saved/test_01.XP.out | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/tests/core/saved/test_01.XP.out b/tests/core/saved/test_01.XP.out index a263bf5d..c331dcea 100644 --- a/tests/core/saved/test_01.XP.out +++ b/tests/core/saved/test_01.XP.out @@ -1,4 +1,9 @@ + my-box + example.com + my-box + example.com + very 42 [42] From a3f4ba920f897fe3b98ccac08a0b96a8a00ef137 Mon Sep 17 00:00:00 2001 From: Phil Shafer Date: Sun, 20 Mar 2016 16:57:46 -0400 Subject: [PATCH 13/59] update test cases --- tests/core/test_01.c | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/tests/core/test_01.c b/tests/core/test_01.c index e465ffae..05d778a0 100644 --- a/tests/core/test_01.c +++ b/tests/core/test_01.c @@ -79,6 +79,14 @@ main (int argc, char **argv) xo_open_container_h(NULL, "top"); + xo_emit("testing argument modifier {a:}.{a:}...\n", + "host", "my-box", "domain", "example.com"); + + xo_emit("testing argument modifier with encoding to {ea:}.{a:}...\n", + "host", "my-box", "domain", "example.com"); + + xo_emit("{La:} {a:}\n", "Label text", "label", "value"); + xo_emit_field("Vt", "max-chaos", NULL, NULL, " very "); xo_emit_field("V", "min-chaos", "%d", NULL, 42); xo_emit_field("V", "some-chaos", "%d\n", "[%d]", 42); From 60b716cc2a3dd0fc19c9c39c4e817d4dbdedd8fb Mon Sep 17 00:00:00 2001 From: Phil Shafer Date: Wed, 23 Mar 2016 00:04:09 -0400 Subject: [PATCH 14/59] Use "ULL" for 32 bit check --- encoder/cbor/enc_cbor.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/encoder/cbor/enc_cbor.c b/encoder/cbor/enc_cbor.c index 513063fb..7c0a1d33 100644 --- a/encoder/cbor/enc_cbor.c +++ b/encoder/cbor/enc_cbor.c @@ -135,7 +135,7 @@ cbor_encode_uint (xo_buffer_t *xbp, uint64_t minor, unsigned limit) char *bp = xbp->xb_curp; int i, m; - if (minor > (1UL<<32)) { + if (minor > (1ULL << 32)) { *bp++ |= CBOR_LEN64; m = 64; From 45d0846634fa532321ea42d15ff240705691ea3a Mon Sep 17 00:00:00 2001 From: Phil Shafer Date: Wed, 23 Mar 2016 00:04:45 -0400 Subject: [PATCH 15/59] Add {R:} == retain, which retains a copy of the parse fields --- libxo/libxo.c | 227 ++++++++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 222 insertions(+), 5 deletions(-) diff --git a/libxo/libxo.c b/libxo/libxo.c index b6238e91..7f41a0df 100644 --- a/libxo/libxo.c +++ b/libxo/libxo.c @@ -341,6 +341,7 @@ typedef unsigned long xo_xff_flags_t; #define XFF_GT_PLURAL (1<<20) /* Call dngettext to find plural form */ #define XFF_ARGUMENT (1<<21) /* Content provided via argument */ +#define XFF_RETAIN (1<<22) /* Retain parsed format information */ /* Flags to turn off when we don't want i18n processing */ #define XFF_GT_FLAGS (XFF_GT_FIELD | XFF_GT_PLURAL) @@ -1284,6 +1285,191 @@ xo_data_escape (xo_handle_t *xop, const char *str, int len) xo_buf_escape(xop, &xop->xo_data, str, len, 0); } +#ifdef LIBXO_NO_RETAIN +/* + * Empty implementations of the retain logic + */ + +void +xo_retain_clear_all (void) +{ + return; +} + +void +xo_retain_clear (const char *fmt UNUSED) +{ + return; +} +static void +xo_retain_add (const char *fmt UNUSED, xo_field_info_t *fields UNUSED, + unsigned num_fields UNUSED) +{ + return; +} + +static int +xo_retain_find (const char *fmt UNUSED, xo_field_info_t **valp UNUSED, + unsigned *nump UNUSED) +{ + return -1; +} + +#else /* !LIBXO_NO_RETAIN */ +/* + * Retain: We retain parsed field definitions to enhance performance, + * especially inside loops. We depend on the caller treating the format + * strings as immutable, so that we can retain pointers into them. We + * hold the pointers in a hash table, so allow quick access. Retained + * information is retained until xo_retain_clear is called. + */ + +/* + * xo_retain_entry_t holds information about one retained set of + * parsed fields. + */ +typedef struct xo_retain_entry_s { + struct xo_retain_entry_s *xre_next; /* Pointer to next (older) entry */ + unsigned long xre_hits; /* Number of times we've hit */ + const char *xre_format; /* Pointer to format string */ + unsigned xre_num_fields; /* Number of fields saved */ + xo_field_info_t *xre_fields; /* Pointer to fields */ +} xo_retain_entry_t; + +/* + * xo_retain_t holds a complete set of parsed fields as a hash table. + */ +#define RETAIN_HASH_SIZE 64 +typedef struct xo_retain_s { + xo_retain_entry_t *xr_bucket[RETAIN_HASH_SIZE]; +} xo_retain_t; + +static THREAD_LOCAL(xo_retain_t) xo_retain; +static THREAD_LOCAL(unsigned) xo_retain_count; + +/* + * Simple hash function based on Thomas Wang's paper. The original is + * gone, but an archive is available on the Way Back Machine: + * + * http://web.archive.org/web/20071223173210/\ + * http://www.concentric.net/~Ttwang/tech/inthash.htm + * + * For our purposes, we can assume the low four bits are uninteresting + * since any string less that 16 bytes wouldn't be worthy of + * retaining. We toss the high bits also, since these bits are likely + * to be common among constant format strings. We then run Wang's + * algorithm, and cap the result at RETAIN_HASH_SIZE. + */ +static unsigned +xo_retain_hash (const char *fmt UNUSED) +{ + volatile uintptr_t iptr = (uintptr_t) (const void *) fmt; + + /* Discard low four bits and high bits; they aren't interesting */ + uint32_t val = (uint32_t) ((iptr >> 4) & (((1 << 24) - 1))); + + val = (val ^ 61) ^ (val >> 16); + val = val + (val << 3); + val = val ^ (val >> 4); + val = val * 0x3a8f05c5; /* My large prime number */ + val = val ^ (val >> 15); + val &= RETAIN_HASH_SIZE - 1; + + return val; +} + +/* + * Walk all buckets, clearing all retained entries + */ +void +xo_retain_clear_all (void) +{ + int i; + xo_retain_entry_t *xrep, *next; + + for (i = 0; i < RETAIN_HASH_SIZE; i++) { + for (xrep = xo_retain.xr_bucket[i]; xrep; xrep = next) { + next = xrep->xre_next; + xo_free(xrep); + } + xo_retain.xr_bucket[i] = NULL; + } + xo_retain_count = 0; +} + +/* + * Walk all buckets, clearing all retained entries + */ +void +xo_retain_clear (const char *fmt) +{ + xo_retain_entry_t **xrepp; + unsigned hash = xo_retain_hash(fmt); + + for (xrepp = &xo_retain.xr_bucket[hash]; *xrepp; + xrepp = &(*xrepp)->xre_next) { + if ((*xrepp)->xre_format == fmt) { + *xrepp = (*xrepp)->xre_next; + xo_retain_count -= 1; + return; + } + } +} + +/* + * Search the hash for an entry matching 'fmt'; return it's fields. + */ +static int +xo_retain_find (const char *fmt, xo_field_info_t **valp, unsigned *nump) +{ + if (xo_retain_count == 0) + return -1; + + unsigned hash = xo_retain_hash(fmt); + xo_retain_entry_t *xrep; + + for (xrep = xo_retain.xr_bucket[hash]; xrep != NULL; + xrep = xrep->xre_next) { + if (xrep->xre_format == fmt) { + *valp = xrep->xre_fields; + *nump = xrep->xre_num_fields; + xrep->xre_hits += 1; + return 0; + } + } + + return -1; +} + +static void +xo_retain_add (const char *fmt, xo_field_info_t *fields, unsigned num_fields) +{ + unsigned hash = xo_retain_hash(fmt); + xo_retain_entry_t *xrep; + unsigned sz = sizeof(*xrep) + (num_fields + 1) * sizeof(*fields); + xo_field_info_t *xfip; + + xrep = xo_realloc(NULL, sz); + if (xrep == NULL) + return; + + xfip = (xo_field_info_t *) &xrep[1]; + memcpy(xfip, fields, num_fields * sizeof(*fields)); + + bzero(xrep, sizeof(*xrep)); + + xrep->xre_format = fmt; + xrep->xre_fields = xfip; + xrep->xre_num_fields = num_fields; + + /* Record the field info in the retain bucket */ + xrep->xre_next = xo_retain.xr_bucket[hash]; + xo_retain.xr_bucket[hash] = xrep; + xo_retain_count += 1; +} + +#endif /* !LIBXO_NO_RETAIN */ + /* * Generate a warning. Normally, this is a text message written to * standard error. If the XOF_WARN_XML flag is set, then we generate @@ -4819,6 +5005,7 @@ static xo_mapping_t xo_role_names[] = { { 'L', "label" }, { 'N', "note" }, { 'P', "padding" }, + { 'R', "retain" }, { 'T', "title" }, { 'U', "units" }, { 'V', "value" }, @@ -4898,6 +5085,7 @@ xo_count_fields (xo_handle_t *xop UNUSED, const char *fmt) * 'L': label; text preceding data * 'N': note; text following data * 'P': padding; whitespace + * 'R': retain; record the compiled field info * 'T': Title, where 'content' is a column title * 'U': Units, where 'content' is the unit label * 'V': value, where 'content' is the name of the field (the default) @@ -4982,6 +5170,7 @@ xo_parse_roles (xo_handle_t *xop, const char *fmt, case 'L': case 'N': case 'P': + case 'R': case 'T': case 'U': case 'V': @@ -5340,6 +5529,12 @@ xo_parse_fields (xo_handle_t *xop, xo_field_info_t *fields, if (seen_fnum) rc = xo_parse_field_numbers(xop, fmt, fields, field); + /* + * If the first field is a 'retain' role, then we retain the info + */ + if (fields->xfi_ftype == 'R') + xo_retain_add(fmt, fields, field); + return rc; } @@ -5837,6 +6032,9 @@ xo_do_emit_fields (xo_handle_t *xop, xo_field_info_t *fields, else if (ftype == 'C') xo_format_colors(xop, xfip, content, clen); + else if (ftype == 'R') + /* 'retain'; do nothing */; + else if (ftype == 'G') { /* * A {G:domain} field; disect the domain name and translate @@ -5967,13 +6165,32 @@ xo_do_emit (xo_handle_t *xop, const char *fmt) xop->xo_columns = 0; /* Always reset it */ xop->xo_errno = errno; /* Save for "%m" */ - unsigned max_fields = xo_count_fields(xop, fmt); - xo_field_info_t fields[max_fields]; + if (fmt == NULL) + return 0; - bzero(fields, max_fields * sizeof(fields[0])); + unsigned max_fields; + xo_field_info_t *fields = NULL; - if (xo_parse_fields(xop, fields, max_fields, fmt)) - return -1; /* Warning already displayed */ + /* + * Look for the magic role "{R:}" for retain, telling us to + * retain the field information. If we've already saved it, + * then we can avoid re-parsing the format string. + * + * This check is a bit naive, but will do for now, since only {R:} + * needs to be first and can't be combined with others. + */ + if (strncmp(fmt, "{R:}", 4) != 0 + || xo_retain_find(fmt, &fields, &max_fields) != 0 + || fields == NULL) { + + /* Nothing retained; parse the format string */ + max_fields = xo_count_fields(xop, fmt); + fields = alloca(max_fields * sizeof(fields[0])); + bzero(fields, max_fields * sizeof(fields[0])); + + if (xo_parse_fields(xop, fields, max_fields, fmt)) + return -1; /* Warning already displayed */ + } return xo_do_emit_fields(xop, fields, max_fields, fmt); } From 8db51fe3f9deb8ff8a4153d89a350194ab4c0342 Mon Sep 17 00:00:00 2001 From: Phil Shafer Date: Wed, 23 Mar 2016 00:05:00 -0400 Subject: [PATCH 16/59] add xo_retain_clear and xo_retain_clear_all --- libxo/xo.h | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/libxo/xo.h b/libxo/xo.h index 8f5007aa..8aab5e10 100644 --- a/libxo/xo.h +++ b/libxo/xo.h @@ -612,4 +612,10 @@ int xo_emit_field (const char *rolmod, const char *contents, const char *fmt, const char *efmt, ...); +void +xo_retain_clear_all (void); + +void +xo_retain_clear (const char *fmt); + #endif /* INCLUDE_XO_H */ From ccac1d1ba51a8ed227af233462cfe793826e1225 Mon Sep 17 00:00:00 2001 From: Phil Shafer Date: Wed, 23 Mar 2016 00:05:08 -0400 Subject: [PATCH 17/59] add test_12 --- tests/core/Makefile.am | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tests/core/Makefile.am b/tests/core/Makefile.am index 0131a6f5..7e61f558 100644 --- a/tests/core/Makefile.am +++ b/tests/core/Makefile.am @@ -22,7 +22,8 @@ test_07.c \ test_08.c \ test_09.c \ test_10.c \ -test_11.c +test_11.c \ +test_12.c test_01_test_SOURCES = test_01.c test_02_test_SOURCES = test_02.c @@ -35,6 +36,7 @@ test_08_test_SOURCES = test_08.c test_09_test_SOURCES = test_09.c test_10_test_SOURCES = test_10.c test_11_test_SOURCES = test_11.c +test_12_test_SOURCES = test_12.c # TEST_CASES := $(shell cd ${srcdir} ; echo *.c ) From b7e8e6d32c2f0709422f6dad575830ed32399ab9 Mon Sep 17 00:00:00 2001 From: Phil Shafer Date: Wed, 23 Mar 2016 00:05:17 -0400 Subject: [PATCH 18/59] update test cases --- tests/core/saved/test_12.E.err | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 tests/core/saved/test_12.E.err diff --git a/tests/core/saved/test_12.E.err b/tests/core/saved/test_12.E.err new file mode 100644 index 00000000..e69de29b From 6e703456eba38ce0fa5e0a61a9ef0c30cf7740d1 Mon Sep 17 00:00:00 2001 From: Phil Shafer Date: Wed, 23 Mar 2016 00:05:18 -0400 Subject: [PATCH 19/59] update test cases --- tests/core/saved/test_12.E.out | 89 ++++++++++++++++++++++++++++++++++ 1 file changed, 89 insertions(+) create mode 100644 tests/core/saved/test_12.E.out diff --git a/tests/core/saved/test_12.E.out b/tests/core/saved/test_12.E.out new file mode 100644 index 00000000..1c899a16 --- /dev/null +++ b/tests/core/saved/test_12.E.out @@ -0,0 +1,89 @@ +op create: [] [] +op open_container: [top] [] +op open_container: [data] [] +op open_list: [thing] [] +op open_instance: [thing] [] +op string: [name] [thing] +op string: [color] [green] +op content: [time] [2:15] +op string: [hand] [left] +op string: [color] [blue] +op content: [time] [3:45] +op close_instance: [thing] [] +op open_instance: [thing] [] +op string: [name] [thing] +op string: [color] [green] +op content: [time] [2:15] +op string: [hand] [left] +op string: [color] [blue] +op content: [time] [3:45] +op close_instance: [thing] [] +op open_instance: [thing] [] +op string: [name] [thing] +op string: [color] [green] +op content: [time] [2:15] +op string: [hand] [left] +op string: [color] [blue] +op content: [time] [3:45] +op close_instance: [thing] [] +op open_instance: [thing] [] +op string: [name] [thing] +op string: [color] [green] +op content: [time] [2:15] +op string: [hand] [left] +op string: [color] [blue] +op content: [time] [3:45] +op close_instance: [thing] [] +op open_instance: [thing] [] +op string: [name] [thing] +op string: [color] [green] +op content: [time] [2:15] +op string: [hand] [left] +op string: [color] [blue] +op content: [time] [3:45] +op close_instance: [thing] [] +op open_instance: [thing] [] +op string: [name] [thing] +op string: [color] [green] +op content: [time] [2:15] +op string: [hand] [left] +op string: [color] [blue] +op content: [time] [3:45] +op close_instance: [thing] [] +op open_instance: [thing] [] +op string: [name] [thing] +op string: [color] [green] +op content: [time] [2:15] +op string: [hand] [left] +op string: [color] [blue] +op content: [time] [3:45] +op close_instance: [thing] [] +op open_instance: [thing] [] +op string: [name] [thing] +op string: [color] [green] +op content: [time] [2:15] +op string: [hand] [left] +op string: [color] [blue] +op content: [time] [3:45] +op close_instance: [thing] [] +op open_instance: [thing] [] +op string: [name] [thing] +op string: [color] [green] +op content: [time] [2:15] +op string: [hand] [left] +op string: [color] [blue] +op content: [time] [3:45] +op close_instance: [thing] [] +op open_instance: [thing] [] +op string: [name] [thing] +op string: [color] [green] +op content: [time] [2:15] +op string: [hand] [left] +op string: [color] [blue] +op content: [time] [3:45] +op close_instance: [thing] [] +op close_list: [thing] [] +op close_container: [data] [] +op close_container: [top] [] +op finish: [] [] +op flush: [] [] From e326e8e4695e9337c7467db85eec69466e87743d Mon Sep 17 00:00:00 2001 From: Phil Shafer Date: Wed, 23 Mar 2016 00:05:18 -0400 Subject: [PATCH 20/59] update test cases --- tests/core/saved/test_12.H.err | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 tests/core/saved/test_12.H.err diff --git a/tests/core/saved/test_12.H.err b/tests/core/saved/test_12.H.err new file mode 100644 index 00000000..e69de29b From bdbd2d9ae3227eb6ca5413c30e4d0acecdd94c4f Mon Sep 17 00:00:00 2001 From: Phil Shafer Date: Wed, 23 Mar 2016 00:05:19 -0400 Subject: [PATCH 21/59] update test cases --- tests/core/saved/test_12.H.out | 1 + 1 file changed, 1 insertion(+) create mode 100644 tests/core/saved/test_12.H.out diff --git a/tests/core/saved/test_12.H.out b/tests/core/saved/test_12.H.out new file mode 100644 index 00000000..5cbac179 --- /dev/null +++ b/tests/core/saved/test_12.H.out @@ -0,0 +1 @@ +
The
thing
is
green
til
02:15
My
left
hand is
blue
til
03:45
The
thing
is
green
til
02:15
My
left
hand is
blue
til
03:45
The
thing
is
green
til
02:15
My
left
hand is
blue
til
03:45
The
thing
is
green
til
02:15
My
left
hand is
blue
til
03:45
The
thing
is
green
til
02:15
My
left
hand is
blue
til
03:45
The
thing
is
green
til
02:15
My
left
hand is
blue
til
03:45
The
thing
is
green
til
02:15
My
left
hand is
blue
til
03:45
The
thing
is
green
til
02:15
My
left
hand is
blue
til
03:45
The
thing
is
green
til
02:15
My
left
hand is
blue
til
03:45
The
thing
is
green
til
02:15
My
left
hand is
blue
til
03:45
\ No newline at end of file From 3a6387901f97d657dfa5d452ebe776910812df49 Mon Sep 17 00:00:00 2001 From: Phil Shafer Date: Wed, 23 Mar 2016 00:05:20 -0400 Subject: [PATCH 22/59] update test cases --- tests/core/saved/test_12.HIPx.err | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 tests/core/saved/test_12.HIPx.err diff --git a/tests/core/saved/test_12.HIPx.err b/tests/core/saved/test_12.HIPx.err new file mode 100644 index 00000000..e69de29b From 4a8af52f6ad5f68d1d14a01c5c705b05cb378790 Mon Sep 17 00:00:00 2001 From: Phil Shafer Date: Wed, 23 Mar 2016 00:05:21 -0400 Subject: [PATCH 23/59] update test cases --- tests/core/saved/test_12.HIPx.out | 160 ++++++++++++++++++++++++++++++ 1 file changed, 160 insertions(+) create mode 100644 tests/core/saved/test_12.HIPx.out diff --git a/tests/core/saved/test_12.HIPx.out b/tests/core/saved/test_12.HIPx.out new file mode 100644 index 00000000..9b5fea1c --- /dev/null +++ b/tests/core/saved/test_12.HIPx.out @@ -0,0 +1,160 @@ +
+
The
+
thing
+
is
+
green
+
til
+
02:15
+
+
+
My
+
left
+
hand is
+
blue
+
til
+
03:45
+
+
+
The
+
thing
+
is
+
green
+
til
+
02:15
+
+
+
My
+
left
+
hand is
+
blue
+
til
+
03:45
+
+
+
The
+
thing
+
is
+
green
+
til
+
02:15
+
+
+
My
+
left
+
hand is
+
blue
+
til
+
03:45
+
+
+
The
+
thing
+
is
+
green
+
til
+
02:15
+
+
+
My
+
left
+
hand is
+
blue
+
til
+
03:45
+
+
+
The
+
thing
+
is
+
green
+
til
+
02:15
+
+
+
My
+
left
+
hand is
+
blue
+
til
+
03:45
+
+
+
The
+
thing
+
is
+
green
+
til
+
02:15
+
+
+
My
+
left
+
hand is
+
blue
+
til
+
03:45
+
+
+
The
+
thing
+
is
+
green
+
til
+
02:15
+
+
+
My
+
left
+
hand is
+
blue
+
til
+
03:45
+
+
+
The
+
thing
+
is
+
green
+
til
+
02:15
+
+
+
My
+
left
+
hand is
+
blue
+
til
+
03:45
+
+
+
The
+
thing
+
is
+
green
+
til
+
02:15
+
+
+
My
+
left
+
hand is
+
blue
+
til
+
03:45
+
+
+
The
+
thing
+
is
+
green
+
til
+
02:15
+
+
+
My
+
left
+
hand is
+
blue
+
til
+
03:45
+
From 53d9131db12a8edbfcc9cec09d7894c9d66e72d9 Mon Sep 17 00:00:00 2001 From: Phil Shafer Date: Wed, 23 Mar 2016 00:05:21 -0400 Subject: [PATCH 24/59] update test cases --- tests/core/saved/test_12.HP.err | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 tests/core/saved/test_12.HP.err diff --git a/tests/core/saved/test_12.HP.err b/tests/core/saved/test_12.HP.err new file mode 100644 index 00000000..e69de29b From c22a2643cd2f807fa0304664e3105dd902153e2f Mon Sep 17 00:00:00 2001 From: Phil Shafer Date: Wed, 23 Mar 2016 00:05:22 -0400 Subject: [PATCH 25/59] update test cases --- tests/core/saved/test_12.HP.out | 160 ++++++++++++++++++++++++++++++++ 1 file changed, 160 insertions(+) create mode 100644 tests/core/saved/test_12.HP.out diff --git a/tests/core/saved/test_12.HP.out b/tests/core/saved/test_12.HP.out new file mode 100644 index 00000000..1e0e9233 --- /dev/null +++ b/tests/core/saved/test_12.HP.out @@ -0,0 +1,160 @@ +
+
The
+
thing
+
is
+
green
+
til
+
02:15
+
+
+
My
+
left
+
hand is
+
blue
+
til
+
03:45
+
+
+
The
+
thing
+
is
+
green
+
til
+
02:15
+
+
+
My
+
left
+
hand is
+
blue
+
til
+
03:45
+
+
+
The
+
thing
+
is
+
green
+
til
+
02:15
+
+
+
My
+
left
+
hand is
+
blue
+
til
+
03:45
+
+
+
The
+
thing
+
is
+
green
+
til
+
02:15
+
+
+
My
+
left
+
hand is
+
blue
+
til
+
03:45
+
+
+
The
+
thing
+
is
+
green
+
til
+
02:15
+
+
+
My
+
left
+
hand is
+
blue
+
til
+
03:45
+
+
+
The
+
thing
+
is
+
green
+
til
+
02:15
+
+
+
My
+
left
+
hand is
+
blue
+
til
+
03:45
+
+
+
The
+
thing
+
is
+
green
+
til
+
02:15
+
+
+
My
+
left
+
hand is
+
blue
+
til
+
03:45
+
+
+
The
+
thing
+
is
+
green
+
til
+
02:15
+
+
+
My
+
left
+
hand is
+
blue
+
til
+
03:45
+
+
+
The
+
thing
+
is
+
green
+
til
+
02:15
+
+
+
My
+
left
+
hand is
+
blue
+
til
+
03:45
+
+
+
The
+
thing
+
is
+
green
+
til
+
02:15
+
+
+
My
+
left
+
hand is
+
blue
+
til
+
03:45
+
From 6cbe58505f2d0f4b3c14e3e6fc051cafb2d4a554 Mon Sep 17 00:00:00 2001 From: Phil Shafer Date: Wed, 23 Mar 2016 00:05:23 -0400 Subject: [PATCH 26/59] update test cases --- tests/core/saved/test_12.J.err | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 tests/core/saved/test_12.J.err diff --git a/tests/core/saved/test_12.J.err b/tests/core/saved/test_12.J.err new file mode 100644 index 00000000..e69de29b From fd06f6b229beb233b39f39cf603483fc72877669 Mon Sep 17 00:00:00 2001 From: Phil Shafer Date: Wed, 23 Mar 2016 00:05:24 -0400 Subject: [PATCH 27/59] update test cases --- tests/core/saved/test_12.J.out | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 tests/core/saved/test_12.J.out diff --git a/tests/core/saved/test_12.J.out b/tests/core/saved/test_12.J.out new file mode 100644 index 00000000..118bb760 --- /dev/null +++ b/tests/core/saved/test_12.J.out @@ -0,0 +1,2 @@ +{"top": {"data": {"thing": [{"name":"thing","color":"green","time":2:15,"hand":"left","color":"blue","time":3:45}, {"name":"thing","color":"green","time":2:15,"hand":"left","color":"blue","time":3:45}, {"name":"thing","color":"green","time":2:15,"hand":"left","color":"blue","time":3:45}, {"name":"thing","color":"green","time":2:15,"hand":"left","color":"blue","time":3:45}, {"name":"thing","color":"green","time":2:15,"hand":"left","color":"blue","time":3:45}, {"name":"thing","color":"green","time":2:15,"hand":"left","color":"blue","time":3:45}, {"name":"thing","color":"green","time":2:15,"hand":"left","color":"blue","time":3:45}, {"name":"thing","color":"green","time":2:15,"hand":"left","color":"blue","time":3:45}, {"name":"thing","color":"green","time":2:15,"hand":"left","color":"blue","time":3:45}, {"name":"thing","color":"green","time":2:15,"hand":"left","color":"blue","time":3:45}]}} +} From a9e4803bc05f9bf5553c7747b41c44145fbb55c6 Mon Sep 17 00:00:00 2001 From: Phil Shafer Date: Wed, 23 Mar 2016 00:05:26 -0400 Subject: [PATCH 28/59] update test cases --- tests/core/saved/test_12.JP.err | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 tests/core/saved/test_12.JP.err diff --git a/tests/core/saved/test_12.JP.err b/tests/core/saved/test_12.JP.err new file mode 100644 index 00000000..e69de29b From f3806c5d80e8af6e04b2cabb1a7ae48b8cb54145 Mon Sep 17 00:00:00 2001 From: Phil Shafer Date: Wed, 23 Mar 2016 00:05:41 -0400 Subject: [PATCH 29/59] update test cases --- tests/core/saved/test_12.JP.out | 88 +++++++++++++++++++++++++++++++++ 1 file changed, 88 insertions(+) create mode 100644 tests/core/saved/test_12.JP.out diff --git a/tests/core/saved/test_12.JP.out b/tests/core/saved/test_12.JP.out new file mode 100644 index 00000000..3e15e0d5 --- /dev/null +++ b/tests/core/saved/test_12.JP.out @@ -0,0 +1,88 @@ +{ + "top": { + "data": { + "thing": [ + { + "name": "thing", + "color": "green", + "time": 2:15, + "hand": "left", + "color": "blue", + "time": 3:45 + }, + { + "name": "thing", + "color": "green", + "time": 2:15, + "hand": "left", + "color": "blue", + "time": 3:45 + }, + { + "name": "thing", + "color": "green", + "time": 2:15, + "hand": "left", + "color": "blue", + "time": 3:45 + }, + { + "name": "thing", + "color": "green", + "time": 2:15, + "hand": "left", + "color": "blue", + "time": 3:45 + }, + { + "name": "thing", + "color": "green", + "time": 2:15, + "hand": "left", + "color": "blue", + "time": 3:45 + }, + { + "name": "thing", + "color": "green", + "time": 2:15, + "hand": "left", + "color": "blue", + "time": 3:45 + }, + { + "name": "thing", + "color": "green", + "time": 2:15, + "hand": "left", + "color": "blue", + "time": 3:45 + }, + { + "name": "thing", + "color": "green", + "time": 2:15, + "hand": "left", + "color": "blue", + "time": 3:45 + }, + { + "name": "thing", + "color": "green", + "time": 2:15, + "hand": "left", + "color": "blue", + "time": 3:45 + }, + { + "name": "thing", + "color": "green", + "time": 2:15, + "hand": "left", + "color": "blue", + "time": 3:45 + } + ] + } + } +} From 9ea626d71d9962cb30fc28724f699c694a1f819e Mon Sep 17 00:00:00 2001 From: Phil Shafer Date: Wed, 23 Mar 2016 00:05:43 -0400 Subject: [PATCH 30/59] update test cases --- tests/core/saved/test_12.T.err | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 tests/core/saved/test_12.T.err diff --git a/tests/core/saved/test_12.T.err b/tests/core/saved/test_12.T.err new file mode 100644 index 00000000..e69de29b From 413609a25ff2a697861da1b078613037ac3d6fb7 Mon Sep 17 00:00:00 2001 From: Phil Shafer Date: Wed, 23 Mar 2016 00:05:45 -0400 Subject: [PATCH 31/59] update test cases --- tests/core/saved/test_12.T.out | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) create mode 100644 tests/core/saved/test_12.T.out diff --git a/tests/core/saved/test_12.T.out b/tests/core/saved/test_12.T.out new file mode 100644 index 00000000..6f777c7a --- /dev/null +++ b/tests/core/saved/test_12.T.out @@ -0,0 +1,20 @@ +The thing is green til 02:15 +My left hand is blue til 03:45 +The thing is green til 02:15 +My left hand is blue til 03:45 +The thing is green til 02:15 +My left hand is blue til 03:45 +The thing is green til 02:15 +My left hand is blue til 03:45 +The thing is green til 02:15 +My left hand is blue til 03:45 +The thing is green til 02:15 +My left hand is blue til 03:45 +The thing is green til 02:15 +My left hand is blue til 03:45 +The thing is green til 02:15 +My left hand is blue til 03:45 +The thing is green til 02:15 +My left hand is blue til 03:45 +The thing is green til 02:15 +My left hand is blue til 03:45 From 8a03570a4334de024d9ce949b233a13c482e7b64 Mon Sep 17 00:00:00 2001 From: Phil Shafer Date: Wed, 23 Mar 2016 00:05:46 -0400 Subject: [PATCH 32/59] update test cases --- tests/core/saved/test_12.X.err | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 tests/core/saved/test_12.X.err diff --git a/tests/core/saved/test_12.X.err b/tests/core/saved/test_12.X.err new file mode 100644 index 00000000..e69de29b From 0d8d08edfe50f1ae0293711bf9d1cf8414dd21b4 Mon Sep 17 00:00:00 2001 From: Phil Shafer Date: Wed, 23 Mar 2016 00:05:47 -0400 Subject: [PATCH 33/59] update test cases --- tests/core/saved/test_12.X.out | 1 + 1 file changed, 1 insertion(+) create mode 100644 tests/core/saved/test_12.X.out diff --git a/tests/core/saved/test_12.X.out b/tests/core/saved/test_12.X.out new file mode 100644 index 00000000..ed2d8e44 --- /dev/null +++ b/tests/core/saved/test_12.X.out @@ -0,0 +1 @@ +thinggreenleftbluethinggreenleftbluethinggreenleftbluethinggreenleftbluethinggreenleftbluethinggreenleftbluethinggreenleftbluethinggreenleftbluethinggreenleftbluethinggreenleftblue \ No newline at end of file From a8a5821b2e42727e485831386effa966e663b1a5 Mon Sep 17 00:00:00 2001 From: Phil Shafer Date: Wed, 23 Mar 2016 00:05:47 -0400 Subject: [PATCH 34/59] update test cases --- tests/core/saved/test_12.XP.err | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 tests/core/saved/test_12.XP.err diff --git a/tests/core/saved/test_12.XP.err b/tests/core/saved/test_12.XP.err new file mode 100644 index 00000000..e69de29b From aec4cf888dc46a3bb9d89f1b169c4e8d2126e884 Mon Sep 17 00:00:00 2001 From: Phil Shafer Date: Wed, 23 Mar 2016 00:05:48 -0400 Subject: [PATCH 35/59] update test cases --- tests/core/saved/test_12.XP.out | 84 +++++++++++++++++++++++++++++++++ 1 file changed, 84 insertions(+) create mode 100644 tests/core/saved/test_12.XP.out diff --git a/tests/core/saved/test_12.XP.out b/tests/core/saved/test_12.XP.out new file mode 100644 index 00000000..73eec4c6 --- /dev/null +++ b/tests/core/saved/test_12.XP.out @@ -0,0 +1,84 @@ + + + + thing + green + + left + blue + + + + thing + green + + left + blue + + + + thing + green + + left + blue + + + + thing + green + + left + blue + + + + thing + green + + left + blue + + + + thing + green + + left + blue + + + + thing + green + + left + blue + + + + thing + green + + left + blue + + + + thing + green + + left + blue + + + + thing + green + + left + blue + + + + From 7837a19a4daeb3fd545575a15783b41adf6591b4 Mon Sep 17 00:00:00 2001 From: Phil Shafer Date: Wed, 23 Mar 2016 00:05:52 -0400 Subject: [PATCH 36/59] update test cases --- tests/core/test_12.c | 79 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 79 insertions(+) create mode 100644 tests/core/test_12.c diff --git a/tests/core/test_12.c b/tests/core/test_12.c new file mode 100644 index 00000000..3613633f --- /dev/null +++ b/tests/core/test_12.c @@ -0,0 +1,79 @@ +/* + * Copyright (c) 2014, Juniper Networks, Inc. + * All rights reserved. + * This SOFTWARE is licensed under the LICENSE provided in the + * ../Copyright file. By downloading, installing, copying, or otherwise + * using the SOFTWARE, you agree to be bound by the terms of that + * LICENSE. + * Phil Shafer, July 2014 + */ + +#include +#include +#include +#include + +#include "xo.h" + +int +main (int argc, char **argv) +{ + int i, count = 10; + int retain = 1; + + argc = xo_parse_args(argc, argv); + if (argc < 0) + return 1; + + for (argc = 1; argv[argc]; argc++) { + if (strcmp(argv[argc], "xml") == 0) + xo_set_style(NULL, XO_STYLE_XML); + else if (strcmp(argv[argc], "json") == 0) + xo_set_style(NULL, XO_STYLE_JSON); + else if (strcmp(argv[argc], "text") == 0) + xo_set_style(NULL, XO_STYLE_TEXT); + else if (strcmp(argv[argc], "html") == 0) + xo_set_style(NULL, XO_STYLE_HTML); + else if (strcmp(argv[argc], "pretty") == 0) + xo_set_flags(NULL, XOF_PRETTY); + else if (strcmp(argv[argc], "xpath") == 0) + xo_set_flags(NULL, XOF_XPATH); + else if (strcmp(argv[argc], "info") == 0) + xo_set_flags(NULL, XOF_INFO); + else if (strcmp(argv[argc], "no-retain") == 0) + retain = 0; + else if (strcmp(argv[argc], "big") == 0) { + if (argv[argc + 1]) + count = atoi(argv[++argc]); + } + } + + xo_set_flags(NULL, XOF_UNITS); /* Always test w/ this */ + xo_set_file(stdout); + + xo_open_container("top"); + xo_open_container("data"); + + const char *fmt1 = "{R:}The {C:fg-red}{k:name}{C:reset} is " + "{C:/fg-%s}{:color}{C:reset} til {:time/%02d:%02d}\n"; + const char *fmt2 = "{R:}My {C:fg-red}{:hand}{C:reset} hand is " + "{C:/fg-%s}{:color}{C:reset} til {:time/%02d:%02d}\n"; + + if (!retain) { + fmt1 += 4; + fmt2 += 4; + } + + for (i = 0; i < count; i++) { + xo_open_instance("thing"); + xo_emit(fmt1, "thing", "green", "green", 2, 15); + xo_emit(fmt2, "left", "blue", "blue", 3, 45); + } + + xo_close_container("data"); + xo_close_container_h(NULL, "top"); + + xo_finish(); + + return 0; +} From c21e143cc83cab2e67fbb6bd4caf9909d99c7558 Mon Sep 17 00:00:00 2001 From: Phil Shafer Date: Wed, 23 Mar 2016 14:49:54 -0400 Subject: [PATCH 37/59] Add test for monitor.h --- configure.ac | 1 + 1 file changed, 1 insertion(+) diff --git a/configure.ac b/configure.ac index 2175572c..9cc600bc 100644 --- a/configure.ac +++ b/configure.ac @@ -74,6 +74,7 @@ AC_CHECK_HEADERS([ctype.h errno.h stdio.h stdlib.h]) AC_CHECK_HEADERS([string.h sys/param.h unistd.h ]) AC_CHECK_HEADERS([sys/sysctl.h]) AC_CHECK_HEADERS([threads.h]) +AC_CHECK_HEADERS([monitor.h]) dnl humanize_number(3) is a great function, but it's not standard. dnl Note Macosx has the function in libutil.a but doesn't ship the From 60a64064921e2e9149d9a48c684fdb7ed7bc1919 Mon Sep 17 00:00:00 2001 From: Phil Shafer Date: Wed, 23 Mar 2016 14:50:19 -0400 Subject: [PATCH 38/59] Add call to moncontrol() if "monitor" is an argument --- tests/core/test_12.c | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/tests/core/test_12.c b/tests/core/test_12.c index 3613633f..f72a95b5 100644 --- a/tests/core/test_12.c +++ b/tests/core/test_12.c @@ -13,13 +13,21 @@ #include #include +#include "xo_config.h" #include "xo.h" +#ifdef HAVE_MONITOR_H +#include +#else +#define moncontrol(x) /* Nothing */ +#endif /* HAVE_MONITOR_H */ + int main (int argc, char **argv) { int i, count = 10; int retain = 1; + int mon = 0; argc = xo_parse_args(argc, argv); if (argc < 0) @@ -42,12 +50,17 @@ main (int argc, char **argv) xo_set_flags(NULL, XOF_INFO); else if (strcmp(argv[argc], "no-retain") == 0) retain = 0; + else if (strcmp(argv[argc], "monitor") == 0) + mon = 1; else if (strcmp(argv[argc], "big") == 0) { if (argv[argc + 1]) count = atoi(argv[++argc]); } } + if (mon) + moncontrol(1); + xo_set_flags(NULL, XOF_UNITS); /* Always test w/ this */ xo_set_file(stdout); @@ -75,5 +88,8 @@ main (int argc, char **argv) xo_finish(); + if (mon) + moncontrol(0); + return 0; } From ed7c528364d1668d1fe8b2bffd5270ac01b599ed Mon Sep 17 00:00:00 2001 From: Phil Shafer Date: Wed, 23 Mar 2016 15:02:36 -0400 Subject: [PATCH 39/59] Add {R:} role --- doc/libxo.txt | 31 +++++++++++++++++++++++++++++++ 1 file changed, 31 insertions(+) diff --git a/doc/libxo.txt b/doc/libxo.txt index d558e51d..85815254 100644 --- a/doc/libxo.txt +++ b/doc/libxo.txt @@ -455,6 +455,7 @@ content. The roles are listed below; only one role is permitted: | L | label | Field is text that prefixes a value | | N | note | Field is text that follows a value | | P | padding | Field is spaces needed for vertical alignment | +| R | retain | Retain parsed information on the format string | | T | title | Field is a title value for headings | | U | units | Field is the units for the previous value field | | V | value | Field is the name of field (the default) | @@ -607,6 +608,36 @@ if preceded by a slash ("/"): xo_emit("{P: }{Lwc:Cost}{:cost/%u}\n", cost); xo_emit("{P:/%30s}{Lwc:Cost}{:cost/%u}\n", "", cost); +**** The Retain Role ({R:}) + +The {R:} role requests that xo_emit() retain the parsed internal +information related to the given format string. On subsequent xo_emit +calls, the retained information is used, avoiding repetitive parsing +of the format string. + +If used, the string "{R:}" must be the first four bytes of the format +string. This restriction is required to avoid parsing the string only +to find out that we can avoid parsing it. + +The format string be immutable across multiple calls to xo_emit(), +since the library retains the string. Typically this is done by using +static constant strings, such as string literals. If the string is not +immutable, the {R:} role must not be used. + +The functions xo_retain_clear() and xo_retain_clear_all() release +internal information on either a single format string or all format +strings, respectively. Neither is required, but the library will +retain this information until it is cleared or the process exits. + + const char *fmt = "{R:}{:name} {:count}\n"; + for (i = 0; i < 1000; i++) { + xo_open_instance("item"); + xo_emit(fmt, name[i], count[i]); + } + xo_retain_clear(fmt); + +The retained information is kept as thread-specific data. + **** The Title Role ({T:}) Title are heading or column headers that are meant to be displayed to From cfc660d6b44296b5fd25b2e03a1b6be1faf4085b Mon Sep 17 00:00:00 2001 From: Phil Shafer Date: Thu, 24 Mar 2016 21:55:42 -0400 Subject: [PATCH 40/59] minor optimizations --- libxo/libxo.c | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/libxo/libxo.c b/libxo/libxo.c index 7f41a0df..831e35d6 100644 --- a/libxo/libxo.c +++ b/libxo/libxo.c @@ -1050,7 +1050,7 @@ xo_is_utf8 (char ch) return (ch & 0x80); } -static int +static inline int xo_utf8_to_wc_len (const char *buf) { unsigned b = (unsigned char) *buf; @@ -1109,9 +1109,13 @@ xo_buf_utf8_len (xo_handle_t *xop, const char *buf, int bufsiz) * bits we pull off the first character is dependent on the length, * but we put 6 bits off all other bytes. */ -static wchar_t +static inline wchar_t xo_utf8_char (const char *buf, int len) { + /* Most common case: singleton byte */ + if (len == 1) + return (unsigned char) buf[0]; + int i; wchar_t wc; const unsigned char *cp = (const unsigned char *) buf; From b79a65b123c8af8423b9883127e94e65cccaf61b Mon Sep 17 00:00:00 2001 From: Phil Shafer Date: Thu, 24 Mar 2016 21:55:59 -0400 Subject: [PATCH 41/59] remove moncontrol calls --- tests/core/test_12.c | 14 -------------- 1 file changed, 14 deletions(-) diff --git a/tests/core/test_12.c b/tests/core/test_12.c index f72a95b5..c4164b67 100644 --- a/tests/core/test_12.c +++ b/tests/core/test_12.c @@ -16,12 +16,6 @@ #include "xo_config.h" #include "xo.h" -#ifdef HAVE_MONITOR_H -#include -#else -#define moncontrol(x) /* Nothing */ -#endif /* HAVE_MONITOR_H */ - int main (int argc, char **argv) { @@ -50,17 +44,12 @@ main (int argc, char **argv) xo_set_flags(NULL, XOF_INFO); else if (strcmp(argv[argc], "no-retain") == 0) retain = 0; - else if (strcmp(argv[argc], "monitor") == 0) - mon = 1; else if (strcmp(argv[argc], "big") == 0) { if (argv[argc + 1]) count = atoi(argv[++argc]); } } - if (mon) - moncontrol(1); - xo_set_flags(NULL, XOF_UNITS); /* Always test w/ this */ xo_set_file(stdout); @@ -88,8 +77,5 @@ main (int argc, char **argv) xo_finish(); - if (mon) - moncontrol(0); - return 0; } From 543f3c86a7858c8c03f58a6c2e56bbe853af6525 Mon Sep 17 00:00:00 2001 From: Phil Shafer Date: Fri, 25 Mar 2016 01:17:08 -0400 Subject: [PATCH 42/59] nuke some unused UNUSEDs --- libxo/libxo.c | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/libxo/libxo.c b/libxo/libxo.c index 831e35d6..5aa82dec 100644 --- a/libxo/libxo.c +++ b/libxo/libxo.c @@ -1365,7 +1365,7 @@ static THREAD_LOCAL(unsigned) xo_retain_count; * algorithm, and cap the result at RETAIN_HASH_SIZE. */ static unsigned -xo_retain_hash (const char *fmt UNUSED) +xo_retain_hash (const char *fmt) { volatile uintptr_t iptr = (uintptr_t) (const void *) fmt; @@ -4572,7 +4572,7 @@ xo_colors_enabled (xo_handle_t *xop UNUSED) } static void -xo_colors_handle_text (xo_handle_t *xop UNUSED, xo_colors_t *newp) +xo_colors_handle_text (xo_handle_t *xop, xo_colors_t *newp) { char buf[BUFSIZ]; char *cp = buf, *ep = buf + sizeof(buf); @@ -5818,9 +5818,8 @@ xo_gettext_combine_formats (xo_handle_t *xop, const char *fmt UNUSED, * Summary: i18n aighn't cheap. */ static const char * -xo_gettext_build_format (xo_handle_t *xop UNUSED, - xo_field_info_t *fields UNUSED, - int this_field UNUSED, +xo_gettext_build_format (xo_handle_t *xop, + xo_field_info_t *fields, int this_field, const char *fmt, char **new_fmtp) { if (xo_style_is_encoding(xop)) @@ -6768,7 +6767,7 @@ xo_open_list_hf (xo_handle_t *xop, xo_xsf_flags_t flags, const char *name) } int -xo_open_list_h (xo_handle_t *xop, const char *name UNUSED) +xo_open_list_h (xo_handle_t *xop, const char *name) { return xo_open_list_hf(xop, 0, name); } @@ -6780,7 +6779,7 @@ xo_open_list (const char *name) } int -xo_open_list_hd (xo_handle_t *xop, const char *name UNUSED) +xo_open_list_hd (xo_handle_t *xop, const char *name) { return xo_open_list_hf(xop, XOF_DTRT, name); } @@ -7805,7 +7804,7 @@ xo_set_program (const char *name) } void -xo_set_version_h (xo_handle_t *xop, const char *version UNUSED) +xo_set_version_h (xo_handle_t *xop, const char *version) { xop = xo_default(xop); From ea58472fff95d0cbe949faabd537999c096c6d87 Mon Sep 17 00:00:00 2001 From: Phil Shafer Date: Sat, 26 Mar 2016 01:13:58 -0400 Subject: [PATCH 43/59] nits --- doc/libxo.txt | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/doc/libxo.txt b/doc/libxo.txt index 85815254..0280e23a 100644 --- a/doc/libxo.txt +++ b/doc/libxo.txt @@ -2641,23 +2641,23 @@ In 2001, we added an XML API to the JUNOS operating system, which is built on top of FreeBSD. Eventually this API became standardized as the NETCONF API (RFC 6241). As part of this effort, we modified many FreeBSD utilities to emit XML, typically via a "-X" switch. The -results were mixed. The cost of maintaining this code, updating it +results were mixed. The cost of maintaining this code, updating it, and carrying it were non-trivial, and contributed to our expense (and the associated delay) with upgrading the version of FreeBSD on which each release of JUNOS is based. A recent (2014) effort within JUNOS aims at removing our modifications to the underlying FreeBSD code as a means of reducing the expense and -delay. JUNOS is structured to have system components generate XML -that is rendered by the CLI (think: login shell) into human-readable -text. This allows the API to use the same plumbing as the CLI, and -ensures that all components emit XML, and that it is emitted with -knowledge of the consumer of that XML, yielding an API that have no -incremental cost or feature delay. +delay in tracking HEAD. JUNOS is structured to have system components +generate XML that is rendered by the CLI (think: login shell) into +human-readable text. This allows the API to use the same plumbing as +the CLI, and ensures that all components emit XML, and that it is +emitted with knowledge of the consumer of that XML, yielding an API +that have no incremental cost or feature delay. libxo is an effort to mix the best aspects of the JUNOS strategy into FreeBSD in a seemless way, allowing commands to make printf-like -output calls without needing to care how the output is rendered. +output calls with a single code path. *** Did the complex semantics of format strings evolve over time? From 1abf64f08bb0b33ead2cc43f4e57a8c1fab60b88 Mon Sep 17 00:00:00 2001 From: Phil Shafer Date: Sat, 26 Mar 2016 20:25:28 -0400 Subject: [PATCH 44/59] Add --with-retain-size to set the size (in bits) of the retain hash buckets --- configure.ac | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/configure.ac b/configure.ac index 9cc600bc..6e215107 100644 --- a/configure.ac +++ b/configure.ac @@ -288,6 +288,18 @@ if test "${LIBXO_WCWIDTH}" != "no"; then AC_DEFINE([LIBXO_WCWIDTH], [1], [Enable local wcwidth implementation]) fi +AC_MSG_CHECKING([retain hash bucket size]) +AC_ARG_WITH(retain-size, + [ --with-retain-size=[DIR] Specify retain hash bucket size (in bits)], + [XO_RETAIN_SIZE=$withval], + [XO_RETAIN_SIZE=default] +) + +AC_MSG_RESULT([$XO_RETAIN_SIZE]) +if test "${XO_RETAIN_SIZE}" != "default"; then + AC_DEFINE_UNQUOTED([XO_RETAIN_SIZE], ${XO_RETAIN_SIZE}, [Retain hash bucket size]) +fi + AC_CHECK_LIB([m], [lrint]) AM_CONDITIONAL([HAVE_LIBM], [test "$HAVE_LIBM" != "no"]) @@ -453,4 +465,5 @@ AC_MSG_NOTICE([summary of build options: isthreaded: ${HAVE_ISTHREADED:-no} thread-local: ${THREAD_LOCAL:-no} local wcwidth: ${LIBXO_WCWIDTH:-no} + retain size: ${XO_RETAIN_SIZE:-no} ]) From d6c3fe9d844f00c4bc8feee32295e62f0cdc6c7f Mon Sep 17 00:00:00 2001 From: Phil Shafer Date: Sat, 26 Mar 2016 20:26:26 -0400 Subject: [PATCH 45/59] make 'retain' a flag (XOEF_RETAIN) instead of a role; it's simpler, and doesn't feel as tacky. "{R:}" was painful to document, which means it's painful to use. --- doc/libxo.txt | 111 +++++++++++++++++++++++++++++--------------------- 1 file changed, 65 insertions(+), 46 deletions(-) diff --git a/doc/libxo.txt b/doc/libxo.txt index 0280e23a..9d9d0e1a 100644 --- a/doc/libxo.txt +++ b/doc/libxo.txt @@ -455,7 +455,6 @@ content. The roles are listed below; only one role is permitted: | L | label | Field is text that prefixes a value | | N | note | Field is text that follows a value | | P | padding | Field is spaces needed for vertical alignment | -| R | retain | Retain parsed information on the format string | | T | title | Field is a title value for headings | | U | units | Field is the units for the previous value field | | V | value | Field is the name of field (the default) | @@ -608,36 +607,6 @@ if preceded by a slash ("/"): xo_emit("{P: }{Lwc:Cost}{:cost/%u}\n", cost); xo_emit("{P:/%30s}{Lwc:Cost}{:cost/%u}\n", "", cost); -**** The Retain Role ({R:}) - -The {R:} role requests that xo_emit() retain the parsed internal -information related to the given format string. On subsequent xo_emit -calls, the retained information is used, avoiding repetitive parsing -of the format string. - -If used, the string "{R:}" must be the first four bytes of the format -string. This restriction is required to avoid parsing the string only -to find out that we can avoid parsing it. - -The format string be immutable across multiple calls to xo_emit(), -since the library retains the string. Typically this is done by using -static constant strings, such as string literals. If the string is not -immutable, the {R:} role must not be used. - -The functions xo_retain_clear() and xo_retain_clear_all() release -internal information on either a single format string or all format -strings, respectively. Neither is required, but the library will -retain this information until it is cleared or the process exits. - - const char *fmt = "{R:}{:name} {:count}\n"; - for (i = 0; i < 1000; i++) { - xo_open_instance("item"); - xo_emit(fmt, name[i], count[i]); - } - xo_retain_clear(fmt); - -The retained information is kept as thread-specific data. - **** The Title Role ({T:}) Title are heading or column headers that are meant to be displayed to @@ -1092,6 +1061,24 @@ LANG, or LC_ALL environment varibles. The first of this list of variables is used and if none of the variables are set, the locale defaults to "UTF-8". +libxo will convert these arguments as needed to either UTF-8 (for XML, +JSON, and HTML styles) or locale-based strings for display in text +style. + + xo_emit("Alll strings are utf-8 content {:tag/%ls}", + L"except for wide strings"); + +"%S" is equivalent to "%ls". + +|--------+-----------------+-------------------------------| +| Format | Argument Type | Argument Contents | +|--------+-----------------+-------------------------------| +| %s | const char * | UTF-8 string | +| %S | const char * | UTF-8 string (alias for '%s') | +| %ls | const wchar_t * | Wide character UNICODE string | +| %hs | const char * | locale-based string | +|--------+-----------------+-------------------------------| + For example, a function is passed a locale-base name, a hat size, and a time value. The hat size is formatted in a UTF-8 (ASCII) string, and the time value is formatted into a wchar_t string. @@ -1226,6 +1213,53 @@ variants might be wise. | xo_emit_errc | xo_emit_errc_p | |------------------+------------------------| +*** Retaining Parsed Format Information @retain@ + +libxo can retain the parsed internal information related to the given +format string, allowing subsequent xo_emit calls, the retained +information is used, avoiding repetitive parsing of the format string. + + SYNTAX: + int xo_emit_f(xo_emit_flags_t flags, const char fmt, ...); + EXAMPLE: + xo_emit_f(XOEF_RETAIN, "{:some/%02d}{:thing/%-6s}{:fancy}\n", + some, thing, fancy); + +To retain parsed format information, use the XOEF_RETAIN flag to the +xo_emit_f() function. A complete set of xo_emit_f functions exist to +match all the xo_emit function signatures (with handles, varadic +argument, and printf-like flags): + +|------------------+------------------------| +| Function | printf-like Equivalent | +|------------------+------------------------| +| xo_emit_hv | xo_emit_hvf | +| xo_emit_h | xo_emit_hf | +| xo_emit | xo_emit_f | +| xo_emit_hvp | xo_emit_hvfp | +| xo_emit_hp | xo_emit_hfp | +| xo_emit_p | xo_emit_fp | +|------------------+------------------------| + +The format string be immutable across multiple calls to xo_emit_f(), +since the library retains the string. Typically this is done by using +static constant strings, such as string literals. If the string is not +immutable, the XOEF_RETAIN flag must not be used. + +The functions xo_retain_clear() and xo_retain_clear_all() release +internal information on either a single format string or all format +strings, respectively. Neither is required, but the library will +retain this information until it is cleared or the process exits. + + const char *fmt = "{:name} {:count/%d}\n"; + for (i = 0; i < 1000; i++) { + xo_open_instance("item"); + xo_emit_f(XOEF_RETAIN, fmt, name[i], count[i]); + } + xo_retain_clear(fmt); + +The retained information is kept as thread-specific data. + *** Example In this example, the value for the number of items in stock is emitted: @@ -1467,21 +1501,6 @@ For example, the following are equivalent: Handles are created using xo_create() and destroy using xo_destroy(). -** UTF-8 - -All strings for libxo must be UTF-8. libxo will handle turning them -into locale-based strings for display to the user. - -The only exception is argument formatted using the "%ls" format, which -require a wide character string (wchar_t *) as input. libxo will -convert these arguments as needed to either UTF-8 (for XML, JSON, and -HTML styles) or locale-based strings for display in text style. - - xo_emit("Alll strings are utf-8 content {:tag/%ls}", - L"except for wide strings"); - -"%S" is equivalent to "%ls". - * The libxo API This section gives details about the functions in libxo, how to call From 82b3c51410c1841043249eeb7ddd3a212ccf15b6 Mon Sep 17 00:00:00 2001 From: Phil Shafer Date: Sat, 26 Mar 2016 20:26:28 -0400 Subject: [PATCH 46/59] make 'retain' a flag (XOEF_RETAIN) instead of a role; it's simpler, and doesn't feel as tacky. "{R:}" was painful to document, which means it's painful to use. --- libxo/libxo.c | 77 ++++++++++++++++++++++++++++++++++++++------------- 1 file changed, 58 insertions(+), 19 deletions(-) diff --git a/libxo/libxo.c b/libxo/libxo.c index 5aa82dec..26e31212 100644 --- a/libxo/libxo.c +++ b/libxo/libxo.c @@ -341,7 +341,6 @@ typedef unsigned long xo_xff_flags_t; #define XFF_GT_PLURAL (1<<20) /* Call dngettext to find plural form */ #define XFF_ARGUMENT (1<<21) /* Content provided via argument */ -#define XFF_RETAIN (1<<22) /* Retain parsed format information */ /* Flags to turn off when we don't want i18n processing */ #define XFF_GT_FLAGS (XFF_GT_FIELD | XFF_GT_PLURAL) @@ -1343,7 +1342,11 @@ typedef struct xo_retain_entry_s { /* * xo_retain_t holds a complete set of parsed fields as a hash table. */ -#define RETAIN_HASH_SIZE 64 +#ifndef XO_RETAIN_SIZE +#define XO_RETAIN_SIZE 6 +#endif /* XO_RETAIN_SIZE */ +#define RETAIN_HASH_SIZE (1<xfi_ftype == 'R') - xo_retain_add(fmt, fields, field); - return rc; } @@ -6035,9 +6029,6 @@ xo_do_emit_fields (xo_handle_t *xop, xo_field_info_t *fields, else if (ftype == 'C') xo_format_colors(xop, xfip, content, clen); - else if (ftype == 'R') - /* 'retain'; do nothing */; - else if (ftype == 'G') { /* * A {G:domain} field; disect the domain name and translate @@ -6163,7 +6154,7 @@ xo_do_emit_fields (xo_handle_t *xop, xo_field_info_t *fields, * Parse and emit a set of fields */ static int -xo_do_emit (xo_handle_t *xop, const char *fmt) +xo_do_emit (xo_handle_t *xop, xo_emit_flags_t flags, const char *fmt) { xop->xo_columns = 0; /* Always reset it */ xop->xo_errno = errno; /* Save for "%m" */ @@ -6182,7 +6173,7 @@ xo_do_emit (xo_handle_t *xop, const char *fmt) * This check is a bit naive, but will do for now, since only {R:} * needs to be first and can't be combined with others. */ - if (strncmp(fmt, "{R:}", 4) != 0 + if (!(flags & XOEF_RETAIN) || xo_retain_find(fmt, &fields, &max_fields) != 0 || fields == NULL) { @@ -6193,6 +6184,11 @@ xo_do_emit (xo_handle_t *xop, const char *fmt) if (xo_parse_fields(xop, fields, max_fields, fmt)) return -1; /* Warning already displayed */ + + if (flags & XOEF_RETAIN) { + /* Retain the info */ + xo_retain_add(fmt, fields, max_fields); + } } return xo_do_emit_fields(xop, fields, max_fields, fmt); @@ -6238,7 +6234,7 @@ xo_emit_hv (xo_handle_t *xop, const char *fmt, va_list vap) xop = xo_default(xop); va_copy(xop->xo_vap, vap); - rc = xo_do_emit(xop, fmt); + rc = xo_do_emit(xop, 0, fmt); va_end(xop->xo_vap); bzero(&xop->xo_vap, sizeof(xop->xo_vap)); @@ -6252,7 +6248,7 @@ xo_emit_h (xo_handle_t *xop, const char *fmt, ...) xop = xo_default(xop); va_start(xop->xo_vap, fmt); - rc = xo_do_emit(xop, fmt); + rc = xo_do_emit(xop, 0, fmt); va_end(xop->xo_vap); bzero(&xop->xo_vap, sizeof(xop->xo_vap)); @@ -6266,7 +6262,50 @@ xo_emit (const char *fmt, ...) int rc; va_start(xop->xo_vap, fmt); - rc = xo_do_emit(xop, fmt); + rc = xo_do_emit(xop, 0, fmt); + va_end(xop->xo_vap); + bzero(&xop->xo_vap, sizeof(xop->xo_vap)); + + return rc; +} + +int +xo_emit_hvf (xo_handle_t *xop, xo_emit_flags_t flags, + const char *fmt, va_list vap) +{ + int rc; + + xop = xo_default(xop); + va_copy(xop->xo_vap, vap); + rc = xo_do_emit(xop, flags, fmt); + va_end(xop->xo_vap); + bzero(&xop->xo_vap, sizeof(xop->xo_vap)); + + return rc; +} + +int +xo_emit_hf (xo_handle_t *xop, xo_emit_flags_t flags, const char *fmt, ...) +{ + int rc; + + xop = xo_default(xop); + va_start(xop->xo_vap, fmt); + rc = xo_do_emit(xop, flags, fmt); + va_end(xop->xo_vap); + bzero(&xop->xo_vap, sizeof(xop->xo_vap)); + + return rc; +} + +int +xo_emit_f (xo_emit_flags_t flags, const char *fmt, ...) +{ + xo_handle_t *xop = xo_default(NULL); + int rc; + + va_start(xop->xo_vap, fmt); + rc = xo_do_emit(xop, flags, fmt); va_end(xop->xo_vap); bzero(&xop->xo_vap, sizeof(xop->xo_vap)); From 74f777bc369ed5b209074887e0807674eefe6d81 Mon Sep 17 00:00:00 2001 From: Phil Shafer Date: Sat, 26 Mar 2016 20:26:29 -0400 Subject: [PATCH 47/59] make 'retain' a flag (XOEF_RETAIN) instead of a role; it's simpler, and doesn't feel as tacky. "{R:}" was painful to document, which means it's painful to use. --- libxo/xo.h | 43 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 43 insertions(+) diff --git a/libxo/xo.h b/libxo/xo.h index 8aab5e10..d3f01fad 100644 --- a/libxo/xo.h +++ b/libxo/xo.h @@ -95,6 +95,9 @@ typedef unsigned long long xo_xof_flags_t; #define XOF_LOG_GETTEXT XOF_BIT(28) /** Log (stderr) gettext lookup strings */ #define XOF_UTF8 XOF_BIT(29) /** Force text output to be UTF8 */ +typedef unsigned xo_emit_flags_t; /* Flags to xo_emit() and friends */ +#define XOEF_RETAIN (1<<0) /* Retain parsed formatting information */ + /* * The xo_info_t structure provides a mapping between names and * additional data emitted via HTML. @@ -186,6 +189,16 @@ xo_emit_h (xo_handle_t *xop, const char *fmt, ...); int xo_emit (const char *fmt, ...); +int +xo_emit_hvf (xo_handle_t *xop, xo_emit_flags_t flags, + const char *fmt, va_list vap); + +int +xo_emit_hf (xo_handle_t *xop, xo_emit_flags_t flags, const char *fmt, ...); + +int +xo_emit_f (xo_emit_flags_t flags, const char *fmt, ...); + PRINTFLIKE(2, 0) static inline int xo_emit_hvp (xo_handle_t *xop, const char *fmt, va_list vap) @@ -215,6 +228,36 @@ xo_emit_p (const char *fmt, ...) return rc; } +PRINTFLIKE(3, 0) +static inline int +xo_emit_hvfp (xo_handle_t *xop, xo_emit_flags_t flags, + const char *fmt, va_list vap) +{ + return xo_emit_hvf(xop, flags, fmt, vap); +} + +PRINTFLIKE(3, 4) +static inline int +xo_emit_hfp (xo_handle_t *xop, xo_emit_flags_t flags, const char *fmt, ...) +{ + va_list vap; + va_start(vap, fmt); + int rc = xo_emit_hvf(xop, flags, fmt, vap); + va_end(vap); + return rc; +} + +PRINTFLIKE(2, 3) +static inline int +xo_emit_fp (xo_emit_flags_t flags, const char *fmt, ...) +{ + va_list vap; + va_start(vap, fmt); + int rc = xo_emit_hvf(NULL, flags, fmt, vap); + va_end(vap); + return rc; +} + int xo_open_container_h (xo_handle_t *xop, const char *name); From 0fcaa168389daec8cbfcad00b6205c9fa457f0c0 Mon Sep 17 00:00:00 2001 From: Phil Shafer Date: Sat, 26 Mar 2016 20:26:31 -0400 Subject: [PATCH 48/59] make 'retain' a flag (XOEF_RETAIN) instead of a role; it's simpler, and doesn't feel as tacky. "{R:}" was painful to document, which means it's painful to use. --- tests/core/test_12.c | 17 ++++++----------- 1 file changed, 6 insertions(+), 11 deletions(-) diff --git a/tests/core/test_12.c b/tests/core/test_12.c index c4164b67..17d26a6d 100644 --- a/tests/core/test_12.c +++ b/tests/core/test_12.c @@ -20,8 +20,8 @@ int main (int argc, char **argv) { int i, count = 10; - int retain = 1; int mon = 0; + xo_emit_flags_t flags = XOEF_RETAIN; argc = xo_parse_args(argc, argv); if (argc < 0) @@ -43,7 +43,7 @@ main (int argc, char **argv) else if (strcmp(argv[argc], "info") == 0) xo_set_flags(NULL, XOF_INFO); else if (strcmp(argv[argc], "no-retain") == 0) - retain = 0; + flags &= ~XOEF_RETAIN; else if (strcmp(argv[argc], "big") == 0) { if (argv[argc + 1]) count = atoi(argv[++argc]); @@ -56,20 +56,15 @@ main (int argc, char **argv) xo_open_container("top"); xo_open_container("data"); - const char *fmt1 = "{R:}The {C:fg-red}{k:name}{C:reset} is " + const char *fmt1 = "The {C:fg-red}{k:name}{C:reset} is " "{C:/fg-%s}{:color}{C:reset} til {:time/%02d:%02d}\n"; - const char *fmt2 = "{R:}My {C:fg-red}{:hand}{C:reset} hand is " + const char *fmt2 = "My {C:fg-red}{:hand}{C:reset} hand is " "{C:/fg-%s}{:color}{C:reset} til {:time/%02d:%02d}\n"; - if (!retain) { - fmt1 += 4; - fmt2 += 4; - } - for (i = 0; i < count; i++) { xo_open_instance("thing"); - xo_emit(fmt1, "thing", "green", "green", 2, 15); - xo_emit(fmt2, "left", "blue", "blue", 3, 45); + xo_emit_f(flags, fmt1, "thing", "green", "green", 2, 15); + xo_emit_f(flags, fmt2, "left", "blue", "blue", 3, 45); } xo_close_container("data"); From 934d58e96e233d3b8264d4f9ba9df1da222c6175 Mon Sep 17 00:00:00 2001 From: Phil Shafer Date: Sat, 26 Mar 2016 22:56:20 -0400 Subject: [PATCH 49/59] Fix comment re: retain flag --- libxo/libxo.c | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/libxo/libxo.c b/libxo/libxo.c index 26e31212..9284598b 100644 --- a/libxo/libxo.c +++ b/libxo/libxo.c @@ -6166,12 +6166,9 @@ xo_do_emit (xo_handle_t *xop, xo_emit_flags_t flags, const char *fmt) xo_field_info_t *fields = NULL; /* - * Look for the magic role "{R:}" for retain, telling us to - * retain the field information. If we've already saved it, - * then we can avoid re-parsing the format string. - * - * This check is a bit naive, but will do for now, since only {R:} - * needs to be first and can't be combined with others. + * Check for 'retain' flag, telling us to retain the field + * information. If we've already saved it, then we can avoid + * re-parsing the format string. */ if (!(flags & XOEF_RETAIN) || xo_retain_find(fmt, &fields, &max_fields) != 0 From c2dbf81290037bbe9435151d669aebd09ba270af Mon Sep 17 00:00:00 2001 From: Phil Shafer Date: Sun, 27 Mar 2016 00:03:24 -0400 Subject: [PATCH 50/59] combine two 'handles' section; move command line argument section --- doc/libxo.txt | 120 ++++++++++++++++++++++++-------------------------- 1 file changed, 57 insertions(+), 63 deletions(-) diff --git a/doc/libxo.txt b/doc/libxo.txt index 9d9d0e1a..db95c082 100644 --- a/doc/libxo.txt +++ b/doc/libxo.txt @@ -1293,46 +1293,6 @@ penultimate line to: data-type="number" data-help="Number of items in stock">144
-** Command-line Arguments - -libxo uses command line options to trigger rendering behavior. The -following options are recognised: - -- --libxo -- --libxo= -- --libxo: - -Options is a comma-separated list of tokens that correspond to output -styles, flags, or features: - -|-------------+-------------------------------------------------------| -| Token | Action | -|-------------+-------------------------------------------------------| -| color | Enable colors/effects for display styles (TEXT, HTML) | -| dtrt | Enable "Do The Right Thing" mode | -| html | Emit HTML output | -| indent=xx | Set the indentation level | -| info | Add info attributes (HTML) | -| json | Emit JSON output | -| keys | Emit the key attribute for keys (XML) | -| log-gettext | Log (via stderr) each gettext(3) string lookup | -| log-syslog | Log (via stderr) each syslog message (via xo_syslog) | -| no-humanize | Ignore the {h:} modifier (TEXT, HTML) | -| no-locale | Do not initialize the locale setting | -| no-top | Do not emit a top set of braces (JSON) | -| not-first | Pretend the 1st output item was not 1st (JSON) | -| pretty | Emit pretty-printed output | -| text | Emit TEXT output | -| underscores | Replace XML-friendly "-"s with JSON friendly "_"s e | -| units | Add the 'units' (XML) or 'data-units (HTML) attribute | -| warn | Emit warnings when libxo detects bad calls | -| warn-xml | Emit warnings in XML | -| xml | Emit XML output | -| xpath | Add XPath expressions (HTML) | -|-------------+-------------------------------------------------------| - -The brief options are detailed in ^LIBXO_OPTIONS^. - ** Representing Hierarchy For XML and JSON, individual fields appear inside hierarchies which @@ -1479,53 +1439,87 @@ properly. xo_close_marker("fish-guts"); } -** Handles @handles@ +** Command-line Arguments -libxo uses "handles" to control its rendering functionality. The -handle contains state and buffered data, as well as callback functions -to process data. +libxo uses command line options to trigger rendering behavior. The +following options are recognised: -A default handle is used when a NULL is passed to functions accepting -a handle. This handle is initialized to write its data to stdout -using the default style of text (XO_STYLE_TEXT). +- --libxo +- --libxo= +- --libxo: -For the convenience of callers, the libxo library includes handle-less -functions that implicitly use the default handle. Any function that -takes a handle will use the default handle is a value of NULL is -passed in place of a valid handle. +Programs using libxo are expecting to call the xo_parse_args function +to parse these arguments. See ^xo_parse_args^ for details. -For example, the following are equivalent: +Options is a comma-separated list of tokens that correspond to output +styles, flags, or features: - xo_emit("test"); - xo_emit_h(NULL, "test"); +|-------------+-------------------------------------------------------| +| Token | Action | +|-------------+-------------------------------------------------------| +| color | Enable colors/effects for display styles (TEXT, HTML) | +| dtrt | Enable "Do The Right Thing" mode | +| html | Emit HTML output | +| indent=xx | Set the indentation level | +| info | Add info attributes (HTML) | +| json | Emit JSON output | +| keys | Emit the key attribute for keys (XML) | +| log-gettext | Log (via stderr) each gettext(3) string lookup | +| log-syslog | Log (via stderr) each syslog message (via xo_syslog) | +| no-humanize | Ignore the {h:} modifier (TEXT, HTML) | +| no-locale | Do not initialize the locale setting | +| no-top | Do not emit a top set of braces (JSON) | +| not-first | Pretend the 1st output item was not 1st (JSON) | +| pretty | Emit pretty-printed output | +| text | Emit TEXT output | +| underscores | Replace XML-friendly "-"s with JSON friendly "_"s e | +| units | Add the 'units' (XML) or 'data-units (HTML) attribute | +| warn | Emit warnings when libxo detects bad calls | +| warn-xml | Emit warnings in XML | +| xml | Emit XML output | +| xpath | Add XPath expressions (HTML) | +|-------------+-------------------------------------------------------| -Handles are created using xo_create() and destroy using xo_destroy(). +The brief options are detailed in ^LIBXO_OPTIONS^. * The libxo API This section gives details about the functions in libxo, how to call them, and the actions they perform. -** Handles +** Handles @handles@ + +libxo uses "handles" to control its rendering functionality. The +handle contains state and buffered data, as well as callback functions +to process data. Handles give an abstraction for libxo that encapsulates the state of a stream of output. Handles have the data type "xo_handle_t" and are opaque to the caller. The library has a default handle that is automatically initialized. -By default, this handle will send text style output to standard output. -The xo_set_style and xo_set_flags functions can be used to change this -behavior. - -Many libxo functions take a handle as their first parameter; most that -do not use the default handle. Any function taking a handle can -be passed NULL to access the default handle. +By default, this handle will send text style output (XO_STYLE_TEXT) to +standard output. The xo_set_style and xo_set_flags functions can be +used to change this behavior. For the typical command that is generating output on standard output, there is no need to create an explicit handle, but they are available when needed, e.g., for daemons that generate multiple streams of output. +Many libxo functions take a handle as their first parameter; most that +do not use the default handle. Any function taking a handle can be +passed NULL to access the default handle. For the convenience of +callers, the libxo library includes handle-less functions that +implicitly use the default handle. + +For example, the following are equivalent: + + xo_emit("test"); + xo_emit_h(NULL, "test"); + +Handles are created using xo_create() and destroy using xo_destroy(). + *** xo_create A handle can be allocated using the xo_create() function: From d7e7a9107c359880ce292901bc02e3a6ab550bbf Mon Sep 17 00:00:00 2001 From: Phil Shafer Date: Wed, 30 Mar 2016 01:40:50 -0400 Subject: [PATCH 51/59] Document quote heuristic --- doc/libxo.txt | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/doc/libxo.txt b/doc/libxo.txt index db95c082..9ce18566 100644 --- a/doc/libxo.txt +++ b/doc/libxo.txt @@ -929,6 +929,11 @@ needed, but often this needs to be controlled by the caller. JSON: "year": "2014" +The heuristic is based on the format; if the format uses any of the +following conversion specifiers, then no quotes are used: + + d i o u x X D O U e E f F g G a A c C p + **** The Trim Modifier ({t:}) The trim modifier removes any leading or trailing whitespace from From bd0ca9c6f2f75d174afb576b0fa8d40340679171 Mon Sep 17 00:00:00 2001 From: Phil Shafer Date: Wed, 30 Mar 2016 01:40:57 -0400 Subject: [PATCH 52/59] path must be static --- tests/gettext/gt_01.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/gettext/gt_01.c b/tests/gettext/gt_01.c index a0200c29..d6367474 100644 --- a/tests/gettext/gt_01.c +++ b/tests/gettext/gt_01.c @@ -26,7 +26,7 @@ int main (int argc, char **argv) { static char domainname[] = "gt_01"; - char path[MAXPATHLEN]; + static char path[MAXPATHLEN]; const char *tzone = "EST"; const char *lang = "pig_latin"; From 431eacdb959cc5cc3816db112d1bb81cdb1fb4c0 Mon Sep 17 00:00:00 2001 From: Phil Shafer Date: Mon, 4 Apr 2016 19:05:54 -0400 Subject: [PATCH 53/59] handle GETTEXT when msgfmt isn't where it's supposed to be (FreeBSD) --- configure.ac | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/configure.ac b/configure.ac index 6e215107..cc371100 100644 --- a/configure.ac +++ b/configure.ac @@ -149,10 +149,18 @@ fi AC_SUBST(GETTEXT_CFLAGS) AC_SUBST(GETTEXT_LIBS) -GETTEXT_BINDIR=${GETTEXT_PREFIX}/bin -AC_SUBST(GETTEXT_BINDIR) GETTEXT_LIBDIR=${GETTEXT_PREFIX}/lib AC_SUBST(GETTEXT_LIBDIR) +if test -x ${GETTEXT_PREFIX}/bin/msgfmt ; then + GETTEXT_BINDIR=${GETTEXT_PREFIX}/bin +elif test -x ${GETTEXT_PREFIX}/local/bin/msgfmt ; then + GETTEXT_BINDIR=${GETTEXT_PREFIX}/local/bin +else + AC_MSG_NOTICE("could not find msgfmt tool") + # Use a (bad) fall back value + GETTEXT_BINDIR=${GETTEXT_PREFIX}/bin +fi +AC_SUBST(GETTEXT_BINDIR) AM_CONDITIONAL([HAVE_GETTEXT], [test "$HAVE_GETTEXT" = "yes"]) From b3de21435dd795cb5af4c84959c66d4049d1825a Mon Sep 17 00:00:00 2001 From: Phil Shafer Date: Fri, 15 Apr 2016 12:44:10 -0400 Subject: [PATCH 54/59] Add retain and no-retain; fix table heading --- doc/libxo.txt | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/doc/libxo.txt b/doc/libxo.txt index 9ce18566..ba637029 100644 --- a/doc/libxo.txt +++ b/doc/libxo.txt @@ -1236,7 +1236,7 @@ match all the xo_emit function signatures (with handles, varadic argument, and printf-like flags): |------------------+------------------------| -| Function | printf-like Equivalent | +| Function | Flags Equivalent | |------------------+------------------------| | xo_emit_hv | xo_emit_hvf | | xo_emit_h | xo_emit_hf | @@ -1246,7 +1246,7 @@ argument, and printf-like flags): | xo_emit_p | xo_emit_fp | |------------------+------------------------| -The format string be immutable across multiple calls to xo_emit_f(), +The format string must be immutable across multiple calls to xo_emit_f(), since the library retains the string. Typically this is done by using static constant strings, such as string literals. If the string is not immutable, the XOEF_RETAIN flag must not be used. @@ -1473,9 +1473,11 @@ styles, flags, or features: | log-syslog | Log (via stderr) each syslog message (via xo_syslog) | | no-humanize | Ignore the {h:} modifier (TEXT, HTML) | | no-locale | Do not initialize the locale setting | +| no-retain | Prevent retaining formatting information | | no-top | Do not emit a top set of braces (JSON) | | not-first | Pretend the 1st output item was not 1st (JSON) | | pretty | Emit pretty-printed output | +| retain | Force retaining formatting information | | text | Emit TEXT output | | underscores | Replace XML-friendly "-"s with JSON friendly "_"s e | | units | Add the 'units' (XML) or 'data-units (HTML) attribute | From c4d8e1fd22f5f2762f6c004cfccf1e6a86657e8d Mon Sep 17 00:00:00 2001 From: Phil Shafer Date: Fri, 15 Apr 2016 12:44:26 -0400 Subject: [PATCH 55/59] add retain and no-retain flags --- libxo/libxo.c | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/libxo/libxo.c b/libxo/libxo.c index 9284598b..194a0962 100644 --- a/libxo/libxo.c +++ b/libxo/libxo.c @@ -2066,9 +2066,11 @@ static xo_mapping_t xo_xof_names[] = { { XOF_LOG_SYSLOG, "log-syslog" }, { XOF_NO_HUMANIZE, "no-humanize" }, { XOF_NO_LOCALE, "no-locale" }, + { XOF_RETAIN_NONE, "no-retain" }, { XOF_NO_TOP, "no-top" }, { XOF_NOT_FIRST, "not-first" }, { XOF_PRETTY, "pretty" }, + { XOF_RETAIN_ALL, "retain" }, { XOF_UNDERSCORES, "underscores" }, { XOF_UNITS, "units" }, { XOF_WARN, "warn" }, @@ -6165,6 +6167,12 @@ xo_do_emit (xo_handle_t *xop, xo_emit_flags_t flags, const char *fmt) unsigned max_fields; xo_field_info_t *fields = NULL; + /* Adjust XOEF_RETAIN based on global flags */ + if (XOF_ISSET(xop, XOF_RETAIN_ALL)) + flags |= XOEF_RETAIN; + if (XOF_ISSET(xop, XOF_RETAIN_NONE)) + flags &= ~XOEF_RETAIN; + /* * Check for 'retain' flag, telling us to retain the field * information. If we've already saved it, then we can avoid From 370c2d906b034a19fcfe85f79fcbb8c8588a3582 Mon Sep 17 00:00:00 2001 From: Phil Shafer Date: Fri, 15 Apr 2016 12:44:29 -0400 Subject: [PATCH 56/59] add retain and no-retain flags --- libxo/xo.h | 2 ++ 1 file changed, 2 insertions(+) diff --git a/libxo/xo.h b/libxo/xo.h index d3f01fad..310b21ca 100644 --- a/libxo/xo.h +++ b/libxo/xo.h @@ -94,6 +94,8 @@ typedef unsigned long long xo_xof_flags_t; #define XOF_LOG_GETTEXT XOF_BIT(28) /** Log (stderr) gettext lookup strings */ #define XOF_UTF8 XOF_BIT(29) /** Force text output to be UTF8 */ +#define XOF_RETAIN_ALL XOF_BIT(30) /** Force use of XOEF_RETAIN */ +#define XOF_RETAIN_NONE XOF_BIT(31) /** Prevent use of XOEF_RETAIN */ typedef unsigned xo_emit_flags_t; /* Flags to xo_emit() and friends */ #define XOEF_RETAIN (1<<0) /* Retain parsed formatting information */ From a8f415e29a90001c1f4a7e79f1907366c3bd969b Mon Sep 17 00:00:00 2001 From: Phil Shafer Date: Fri, 15 Apr 2016 12:44:32 -0400 Subject: [PATCH 57/59] add retain and no-retain flags --- libxo/xo_parse_args.3 | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/libxo/xo_parse_args.3 b/libxo/xo_parse_args.3 index 80dceca9..487b50ba 100644 --- a/libxo/xo_parse_args.3 +++ b/libxo/xo_parse_args.3 @@ -83,12 +83,16 @@ Log (via stderr) each syslog message (via Ignore the {h:} modifier (TEXT, HTML) .It Dv no-locale Do not initialize the locale setting +.It Dv no-retain +Prevent retaining formatting information .It Dv no-top Do not emit a top set of braces (JSON) .It Dv not-first Pretend the 1st output item was not 1st (JSON) .It Dv pretty Emit pretty-printed output +.It Dv retain +Force retaining formatting information .It Dv text Emit TEXT output .If Dv underscores From 596dad66bea1bce40f3e51009b1d8319bb5fee68 Mon Sep 17 00:00:00 2001 From: Phil Shafer Date: Fri, 15 Apr 2016 12:44:49 -0400 Subject: [PATCH 58/59] new xo_emit_f functions --- libxo/xo_emit_f.3 | 111 ++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 111 insertions(+) create mode 100644 libxo/xo_emit_f.3 diff --git a/libxo/xo_emit_f.3 b/libxo/xo_emit_f.3 new file mode 100644 index 00000000..01a3f79f --- /dev/null +++ b/libxo/xo_emit_f.3 @@ -0,0 +1,111 @@ +.\" # +.\" # Copyright (c) 2016, Juniper Networks, Inc. +.\" # All rights reserved. +.\" # This SOFTWARE is licensed under the LICENSE provided in the +.\" # ../Copyright file. By downloading, installing, copying, or +.\" # using the SOFTWARE, you agree to be bound by the terms of that +.\" # LICENSE. +.\" # Phil Shafer, April 2016 +.\" +.Dd April 15, 2016 +.Dt LIBXO 3 +.Os +.Sh NAME +.Nm xo_emit_f , xo_emit_hf , xo_emit_hvf +.Nd emit formatted output based on format string and arguments +.Sh LIBRARY +.Lb libxo +.Sh SYNOPSIS +.In libxo/xo.h +.Ft int +.Fn xo_emit_f "xo_emit_flags_t flags" "const char *fmt" "..." +.Ft int +.Fn xo_emit_hf "xo_handle_t *xop" "xo_emit_flags_t flags" "const char *fmt" "..." +.Ft int +.Fn xo_emit_hvf "xo_handle_t *xop" "xo_emit_flags_t flags" "const char *fmt" "va_list vap" +.Ft void +.Fn xo_retain_clear_all "void" +.Ft void +.Fn xo_retain_clear "const char *fmt" +.Sh DESCRIPTION +These functions allow callers to pass a set of flags to +.Nm +emitting functions. These processing of arguments, except for +.Fa flags , +is identical to the base functions. +See +.Xr xo_emit 3 +for additional information. +.Pp +The only currently defined flag is +.Dv XOEF_RETAIN . +.Nm +can retain the parsed internal information related to the given +format string, allowing subsequent +.Xr xo_emit 3 +calls, the retained +information is used, avoiding repetitive parsing of the format string. +To retain parsed format information, use the +.Dv XOEF_RETAIN +flag to the +.Fn xo_emit_f +function. +.Pp +The format string must be immutable across multiple calls to +.Xn xo_emit_f , +since the library retains the string. +Typically this is done by using +static constant strings, such as string literals. If the string is not +immutable, the +.Dv XOEF_RETAIN +flag must not be used. +.Pp +The functions +.Fn xo_retain_clear +and +.Fn xo_retain_clear_all +release internal information on either a single format string or all +format strings, respectively. +Neither is required, but the library will +retain this information until it is cleared or the process exits. +.Pp +The retained information is kept as thread-specific data. +.Pp +Use +.Fn xo_retain_clear +and +.Fn xo_retain_clear_all +to clear the retained information, clearing the retained information +for either a specific format string or all format strings, respectively. +These functions are only needed when the calling application wants to +clear this information; they are not generally needed. +.Sh EXAMPLES +.Bd -literal -offset indent + for (i = 0; i < 1000; i++) { + xo_open_instance("item"); + xo_emit_f(XOEF_RETAIN, "{:name} {:count/%d}\n", + name[i], count[i]); + } +.Ed +In this example, the caller desires to clear the retained information. +.Bd -literal -offset indent + const char *fmt = "{:name} {:count/%d}\n"; + for (i = 0; i < 1000; i++) { + xo_open_instance("item"); + xo_emit_f(XOEF_RETAIN, fmt, name[i], count[i]); + } + xo_retain_clear(fmt); +.Ed +.Sh RETURN CODE +.Nm +returns a negative value on error. If the +.Nm XOF_COLUMNS +flag has been turned on for the specific handle using +.Xr xo_set_flags 3 , +then the number of display columns consumed by the output will be returned. +.Sh SEE ALSO +.Xr xo_emit 3 , +.Xr xo_open_container 3 , +.Xr xo_open_list 3 , +.Xr xo_format 5 , +.Xr libxo 3 From b80549ced677dc3e88e2d0c0058e398b41474971 Mon Sep 17 00:00:00 2001 From: Phil Shafer Date: Fri, 15 Apr 2016 12:58:49 -0400 Subject: [PATCH 59/59] Fix man page --- libxo/xo_emit_f.3 | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/libxo/xo_emit_f.3 b/libxo/xo_emit_f.3 index 01a3f79f..08795463 100644 --- a/libxo/xo_emit_f.3 +++ b/libxo/xo_emit_f.3 @@ -80,6 +80,7 @@ for either a specific format string or all format strings, respectively. These functions are only needed when the calling application wants to clear this information; they are not generally needed. .Sh EXAMPLES +.Pp .Bd -literal -offset indent for (i = 0; i < 1000; i++) { xo_open_instance("item"); @@ -87,6 +88,7 @@ clear this information; they are not generally needed. name[i], count[i]); } .Ed +.Pp In this example, the caller desires to clear the retained information. .Bd -literal -offset indent const char *fmt = "{:name} {:count/%d}\n"; @@ -97,12 +99,10 @@ In this example, the caller desires to clear the retained information. xo_retain_clear(fmt); .Ed .Sh RETURN CODE -.Nm -returns a negative value on error. If the -.Nm XOF_COLUMNS -flag has been turned on for the specific handle using -.Xr xo_set_flags 3 , -then the number of display columns consumed by the output will be returned. +The return values for these functions is identical to those of their +traditional counterparts. See +.Xr xo_emit 3 +for details. .Sh SEE ALSO .Xr xo_emit 3 , .Xr xo_open_container 3 ,