Skip to content

parser: add IANA time_zone support for naive timestamps #11639

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
arnikola wants to merge 1 commit into
base: 4.2
Choose a base branch
from
Open

parser: add IANA time_zone support for naive timestamps #11639

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
19 changes: 5 additions & 14 deletions include/fluent-bit/flb_parser.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -49,6 +49,7 @@ struct flb_parser {
char *time_key; /* field name that contains the time */
int time_offset; /* fixed UTC offset */
int time_system_timezone; /* use the system timezone as a fallback */
char *time_zone; /* IANA timezone for naive timestamps (e.g. America/New_York) */
int time_keep; /* keep time field */
int time_strict; /* parse time field strictly */
int logfmt_no_bare_keys; /* in logfmt parsers, require all keys to have values */
Expand All @@ -75,21 +76,10 @@ enum {
FLB_PARSER_TYPE_HEX,
};

static inline time_t flb_parser_tm2time(const struct flb_tm *src,
int use_system_timezone)
{
struct tm tmp;
time_t res;
time_t flb_parser_tm2time_simple(const struct flb_tm *src,
int use_system_timezone);

tmp = src->tm;
if (use_system_timezone) {
tmp.tm_isdst = -1;
res = mktime(&tmp);
} else {
res = timegm(&tmp) - flb_tm_gmtoff(src);
}
return res;
}
time_t flb_parser_tm2time(const struct flb_tm *src, struct flb_parser *parser);


struct flb_parser *flb_parser_create(const char *name, const char *format,
Expand All @@ -100,6 +90,7 @@ struct flb_parser *flb_parser_create(const char *name, const char *format,
int time_keep,
int time_strict,
int time_system_timezone,
const char *time_zone,
int logfmt_no_bare_keys,
struct flb_parser_types *types,
int types_len,
Expand Down
2 changes: 1 addition & 1 deletion plugins/in_kubernetes_events/kubernetes_events.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -227,7 +227,7 @@ static int record_get_field_time(msgpack_object *obj, const char *fieldname, str
return -2;
}

val->tm.tv_sec = flb_parser_tm2time(&tm, FLB_FALSE);
val->tm.tv_sec = flb_parser_tm2time_simple(&tm, FLB_FALSE);
val->tm.tv_nsec = 0;

return 0;
Expand Down
139 changes: 133 additions & 6 deletions src/flb_parser.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -42,8 +42,71 @@
#include <sys/types.h>
#include <sys/stat.h>
#include <limits.h>
#include <stdlib.h>
#include <string.h>

#include <fluent-bit/flb_pthread.h>

static pthread_mutex_t flb_time_zone_mutex = PTHREAD_MUTEX_INITIALIZER;

time_t flb_parser_tm2time_simple(const struct flb_tm *src, int use_system_timezone)
{
struct tm tmp;
time_t res;

tmp = src->tm;
if (use_system_timezone) {
tmp.tm_isdst = -1;
res = mktime(&tmp);
}
else {
res = timegm(&tmp) - flb_tm_gmtoff(src);
}
return res;
}

time_t flb_parser_tm2time(const struct flb_tm *src, struct flb_parser *parser)
{
#ifndef FLB_SYSTEM_WINDOWS
if (parser->time_zone) {
char *old_tz = NULL;
const char *prev;
struct tm tmp;
Comment on lines +71 to +74
Copy link
Copy Markdown

@chatgpt-codex-connector chatgpt-codex-connector bot Mar 31, 2026

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

P1 Badge Skip time_zone conversion when timestamp already has offset

The new time_zone branch always uses mktime in the configured zone, even when time_format includes %z/%Z and the parsed record already carries an explicit offset. In that case flb_parser_time_lookup() has already parsed the source zone, but this path ignores it and reinterprets the wall clock in parser->time_zone, producing shifted epochs (for example, a +0000 input gets converted as local New York time). Please either reject time_zone with %z/%Z formats in flb_parser_create() or bypass this branch when parser->time_with_tz is true.

Useful? React with 👍 / 👎.

time_t res;

pthread_mutex_lock(&flb_time_zone_mutex);
prev = getenv("TZ");
if (prev) {
old_tz = flb_strdup(prev);
if (!old_tz) {
pthread_mutex_unlock(&flb_time_zone_mutex);
return (time_t) -1;
}
}
if (setenv("TZ", parser->time_zone, 1) != 0) {
flb_free(old_tz);
pthread_mutex_unlock(&flb_time_zone_mutex);
return (time_t) -1;
}
Comment on lines +86 to +90
Copy link
Copy Markdown

@chatgpt-codex-connector chatgpt-codex-connector bot Mar 31, 2026

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

P1 Badge Avoid mutating process TZ during per-record parsing

This code changes TZ and calls tzset() for every parsed record, which is process-global state. The mutex only serializes this function, but other paths still call mktime without this lock (e.g. flb_parser_tm2time_simple for time_system_timezone), so concurrent parsing can observe the temporary timezone and compute wrong epochs. This creates cross-parser timestamp corruption when one thread uses time_zone and another thread parses local/system timestamps at the same time.

Useful? React with 👍 / 👎.

tzset();
tmp = src->tm;
tmp.tm_isdst = -1;
res = mktime(&tmp);
if (old_tz) {
setenv("TZ", old_tz, 1);
flb_free(old_tz);
}
else {
unsetenv("TZ");
}
tzset();
pthread_mutex_unlock(&flb_time_zone_mutex);
return res;
}
#endif
return flb_parser_tm2time_simple(src, parser->time_system_timezone);
}

static inline uint32_t digits10(uint64_t v) {
if (v < 10) return 1;
if (v < 100) return 2;
Expand Down Expand Up @@ -140,6 +203,9 @@ static void flb_interim_parser_destroy(struct flb_parser *parser)
if (parser->time_key) {
flb_free(parser->time_key);
}
if (parser->time_zone) {
flb_free(parser->time_zone);
}

mk_list_del(&parser->_head);
flb_free(parser);
Expand All @@ -153,6 +219,7 @@ struct flb_parser *flb_parser_create(const char *name, const char *format,
int time_keep,
int time_strict,
int time_system_timezone,
const char *time_zone,
int logfmt_no_bare_keys,
struct flb_parser_types *types,
int types_len,
Expand Down Expand Up @@ -231,6 +298,24 @@ struct flb_parser *flb_parser_create(const char *name, const char *format,

p->name = flb_strdup(name);

#if defined(FLB_SYSTEM_WINDOWS)
if (time_zone && time_zone[0]) {
flb_error("[parser:%s] time_zone is not supported on Windows", name);
mk_list_del(&p->_head);
flb_free(p->name);
flb_free(p);
return NULL;
}
#endif

if (time_zone && time_zone[0] && !time_fmt) {
flb_error("[parser:%s] time_zone requires time_format", name);
mk_list_del(&p->_head);
flb_free(p->name);
flb_free(p);
return NULL;
}

if (time_fmt) {
p->time_fmt_full = flb_strdup(time_fmt);
if (!p->time_fmt_full) {
Expand Down Expand Up @@ -319,11 +404,33 @@ struct flb_parser *flb_parser_create(const char *name, const char *format,
*/
p->time_system_timezone = time_system_timezone;

if (time_zone && time_zone[0]) {
if (time_system_timezone) {
flb_error("[parser:%s] time_zone cannot be combined with "
"time_system_timezone",
name);
flb_interim_parser_destroy(p);
return NULL;
}
if (time_offset && time_offset[0]) {
flb_error("[parser:%s] time_zone cannot be combined with "
"time_offset",
name);
flb_interim_parser_destroy(p);
return NULL;
}
p->time_zone = flb_strdup(time_zone);
if (!p->time_zone) {
flb_interim_parser_destroy(p);
return NULL;
}
}

/*
* Optional fixed timezone offset, only applied if
* not falling back to system timezone.
* not falling back to system timezone or IANA time_zone.
*/
if (!p->time_system_timezone && time_offset) {
if (!p->time_system_timezone && !p->time_zone && time_offset) {
diff = 0;
len = strlen(time_offset);
ret = flb_parser_tzone_offset(time_offset, len, &diff);
Expand Down Expand Up @@ -367,6 +474,9 @@ void flb_parser_destroy(struct flb_parser *parser)
if (parser->time_key) {
flb_free(parser->time_key);
}
if (parser->time_zone) {
flb_free(parser->time_zone);
}
if (parser->types_len != 0) {
for (i=0; i<parser->types_len; i++){
flb_free(parser->types[i].key);
Expand Down Expand Up @@ -492,6 +602,7 @@ int flb_parser_load_parser_definitions(const char *cfg, struct flb_cf *cf,
flb_sds_t time_fmt;
flb_sds_t time_key;
flb_sds_t time_offset;
flb_sds_t time_zone;
flb_sds_t types_str;
flb_sds_t tmp_str;
int skip_empty;
Expand All @@ -513,6 +624,7 @@ int flb_parser_load_parser_definitions(const char *cfg, struct flb_cf *cf,
time_fmt = NULL;
time_key = NULL;
time_offset = NULL;
time_zone = NULL;
types_str = NULL;
tmp_str = NULL;

Expand Down Expand Up @@ -582,6 +694,9 @@ int flb_parser_load_parser_definitions(const char *cfg, struct flb_cf *cf,
/* time_offset (UTC offset) */
time_offset = get_parser_key(config, cf, s, "time_offset");

/* time_zone (IANA region, DST-aware; Unix only) */
time_zone = get_parser_key(config, cf, s, "time_zone");

/* logfmt_no_bare_keys */
logfmt_no_bare_keys = FLB_FALSE;
tmp_str = get_parser_key(config, cf, s, "logfmt_no_bare_keys");
Expand All @@ -605,7 +720,8 @@ int flb_parser_load_parser_definitions(const char *cfg, struct flb_cf *cf,
/* Create the parser context */
if (!flb_parser_create(name, format, regex, skip_empty,
time_fmt, time_key, time_offset, time_keep, time_strict,
time_system_timezone, logfmt_no_bare_keys, types, types_len,
time_system_timezone, time_zone, logfmt_no_bare_keys,
types, types_len,
decoders, config)) {
goto fconf_error;
}
Expand All @@ -627,6 +743,9 @@ int flb_parser_load_parser_definitions(const char *cfg, struct flb_cf *cf,
if (time_offset) {
flb_sds_destroy(time_offset);
}
if (time_zone) {
flb_sds_destroy(time_zone);
}
if (types_str) {
flb_sds_destroy(types_str);
}
Expand All @@ -635,8 +754,8 @@ int flb_parser_load_parser_definitions(const char *cfg, struct flb_cf *cf,

return 0;

/* Use early exit before call to flb_parser_create */
fconf_early_error:
/* Use early exit before call to flb_parser_create */
fconf_early_error:
if (name) {
flb_sds_destroy(name);
}
Expand All @@ -663,6 +782,9 @@ int flb_parser_load_parser_definitions(const char *cfg, struct flb_cf *cf,
if (time_offset) {
flb_sds_destroy(time_offset);
}
if (time_zone) {
flb_sds_destroy(time_zone);
}
if (types_str) {
flb_sds_destroy(types_str);
}
Expand Down Expand Up @@ -1271,7 +1393,12 @@ int flb_parser_time_lookup(const char *time_str, size_t tsize,
}

if (parser->time_with_tz == FLB_FALSE) {
flb_tm_gmtoff(tm) = parser->time_offset;
if (parser->time_zone) {
flb_tm_gmtoff(tm) = 0;
}
else {
flb_tm_gmtoff(tm) = parser->time_offset;
}
}

return 0;
Expand Down
2 changes: 1 addition & 1 deletion src/flb_parser_json.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -211,7 +211,7 @@ int flb_parser_json_do(struct flb_parser *parser,
skip = map_size;
}
else {
time_lookup = flb_parser_tm2time(&tm, parser->time_system_timezone);
time_lookup = flb_parser_tm2time(&tm, parser);
}

/* Compose a new map without the time_key field */
Expand Down
2 changes: 1 addition & 1 deletion src/flb_parser_logfmt.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -166,7 +166,7 @@ static int logfmt_parser(struct flb_parser *parser,
parser->name, parser->time_fmt_full);
return -1;
}
*time_lookup = flb_parser_tm2time(&tm, parser->time_system_timezone);
*time_lookup = flb_parser_tm2time(&tm, parser);
}
time_found = FLB_TRUE;
}
Expand Down
2 changes: 1 addition & 1 deletion src/flb_parser_ltsv.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -139,7 +139,7 @@ static int ltsv_parser(struct flb_parser *parser,
parser->name, parser->time_fmt_full);
return -1;
}
*time_lookup = flb_parser_tm2time(&tm, parser->time_system_timezone);
*time_lookup = flb_parser_tm2time(&tm, parser);
}
time_found = FLB_TRUE;
}
Expand Down
2 changes: 1 addition & 1 deletion src/flb_parser_regex.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -87,7 +87,7 @@ static void cb_results(const char *name, const char *value,
}

pcb->time_frac = frac;
pcb->time_lookup = flb_parser_tm2time(&tm, parser->time_system_timezone);
pcb->time_lookup = flb_parser_tm2time(&tm, parser);

if (parser->time_keep == FLB_FALSE) {
pcb->num_skipped++;
Expand Down
1 change: 1 addition & 0 deletions src/multiline/flb_ml_parser_cri.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -41,6 +41,7 @@ static struct flb_parser *cri_parser_create(struct flb_config *config)
FLB_TRUE, /* time keep */
FLB_FALSE, /* time strict */
FLB_FALSE, /* time system timezone */
NULL, /* time zone (IANA) */
FLB_FALSE, /* no bare keys */
NULL, /* parser types */
0, /* types len */
Expand Down
1 change: 1 addition & 0 deletions src/multiline/flb_ml_parser_docker.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -36,6 +36,7 @@ static struct flb_parser *docker_parser_create(struct flb_config *config)
FLB_TRUE, /* time keep */
FLB_FALSE, /* time strict */
FLB_FALSE, /* time system timezone */
NULL, /* time zone (IANA) */
FLB_FALSE, /* no bare keys */
NULL, /* parser types */
0, /* types len */
Expand Down
2 changes: 1 addition & 1 deletion tests/internal/fuzzers/engine_fuzzer.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -143,7 +143,7 @@ int LLVMFuzzerInitialize(int *argc, char ***argv) {

parser = flb_parser_create("timestamp", "regex", "^(?<time>.*)$", FLB_TRUE,
"%s.%L", "time", NULL, MK_FALSE, 0, FLB_FALSE,
FLB_FALSE, NULL, 0, NULL, ctx->config);
NULL, FLB_FALSE, NULL, 0, NULL, ctx->config);
filter_ffd = flb_filter(ctx, (char *) "parser", NULL);
int ret;
ret = flb_filter_set(ctx, filter_ffd, "Match", "test",
Expand Down
2 changes: 1 addition & 1 deletion tests/internal/fuzzers/parse_json_fuzzer.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -51,7 +51,7 @@ int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size){
}

fuzz_parser = flb_parser_create("fuzzer", "json", NULL, FLB_TRUE, NULL,
NULL, NULL, MK_FALSE, MK_TRUE, FLB_FALSE, FLB_FALSE,
NULL, NULL, MK_FALSE, MK_TRUE, FLB_FALSE, NULL, FLB_FALSE,
NULL, 0, NULL, fuzz_config);
if (fuzz_parser) {
flb_parser_do(fuzz_parser, (char*)data, size,
Expand Down
2 changes: 1 addition & 1 deletion tests/internal/fuzzers/parse_logfmt_fuzzer.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -32,7 +32,7 @@ int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size){
}
fuzz_parser = flb_parser_create("fuzzer", "logfmt", NULL, FLB_TRUE,
NULL, NULL, NULL, MK_FALSE,
MK_TRUE, FLB_FALSE, FLB_FALSE, NULL, 0,
MK_TRUE, FLB_FALSE, NULL, FLB_FALSE, NULL, 0,
NULL, fuzz_config);
if (fuzz_parser) {
flb_parser_do(fuzz_parser, (char*)data, size,
Expand Down
4 changes: 2 additions & 2 deletions tests/internal/fuzzers/parse_ltsv_fuzzer.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,8 +18,8 @@ int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size){
/* ltsvc parser */
fuzz_config = flb_config_init();
fuzz_parser = flb_parser_create("fuzzer", "ltsv", NULL, FLB_TRUE,
NULL, NULL, NULL, MK_FALSE,
MK_TRUE, FLB_FALSE, FLB_FALSE, NULL, 0,
NULL, NULL, NULL, MK_FALSE,
MK_TRUE, FLB_FALSE, NULL, FLB_FALSE, NULL, 0,
NULL, fuzz_config);
flb_parser_do(fuzz_parser, (char*)data, size,
&out_buf, &out_size, &out_time);
Expand Down
2 changes: 1 addition & 1 deletion tests/internal/fuzzers/parser_fuzzer.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -154,7 +154,7 @@ int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size)
/* now call into the parser */
fuzz_parser = flb_parser_create("fuzzer", format, pregex, FLB_TRUE,
time_fmt, time_key, time_offset, time_keep, 0, FLB_FALSE,
FLB_FALSE, types, types_len, list, fuzz_config);
NULL, FLB_FALSE, types, types_len, list, fuzz_config);

/* Second step is to use the random parser to parse random input */
if (fuzz_parser != NULL) {
Expand Down
Loading
Loading