-: ---------- > 1: aa553b7700 Revert ECPG's use of pnstrdup() -: ---------- > 2: d6cae9157e Remove fe_memutils from libpgcommon_shlib 1: 9efb0d64e5 ! 3: 5543539169 common/jsonapi: support libpq as a client @@ Commit message us to track allocation failures so that we can return JSON_OUT_OF_MEMORY as needed rather than exit()ing. + Co-authored-by: Michael Paquier + Co-authored-by: Daniel Gustafsson + ## src/bin/pg_combinebackup/Makefile ## @@ src/bin/pg_combinebackup/Makefile: include $(top_builddir)/src/Makefile.global @@ src/common/jsonapi.c #endif +/* -+ * In backend, we will use palloc/pfree along with StringInfo. In frontend, use -+ * malloc and PQExpBuffer, and return JSON_OUT_OF_MEMORY on out-of-memory. ++ * In backend, we will use palloc/pfree along with StringInfo. In frontend, ++ * use malloc and PQExpBuffer, and return JSON_OUT_OF_MEMORY on out-of-memory. + */ +#ifdef FRONTEND + +#define STRDUP(s) strdup(s) +#define ALLOC(size) malloc(size) ++#define ALLOC0(size) calloc(1, size) ++#define REALLOC realloc ++#define FREE(s) free(s) + -+#define appendStrVal appendPQExpBuffer -+#define appendBinaryStrVal appendBinaryPQExpBuffer -+#define appendStrValChar appendPQExpBufferChar -+#define createStrVal createPQExpBuffer -+#define resetStrVal resetPQExpBuffer -+#define destroyStrVal destroyPQExpBuffer ++#define appendStrVal appendPQExpBuffer ++#define appendBinaryStrVal appendBinaryPQExpBuffer ++#define appendStrValChar appendPQExpBufferChar ++/* XXX should we add a macro version to PQExpBuffer? */ ++#define appendStrValCharMacro appendPQExpBufferChar ++#define createStrVal createPQExpBuffer ++#define initStrVal initPQExpBuffer ++#define resetStrVal resetPQExpBuffer ++#define termStrVal termPQExpBuffer ++#define destroyStrVal destroyPQExpBuffer + +#else /* !FRONTEND */ + +#define STRDUP(s) pstrdup(s) +#define ALLOC(size) palloc(size) ++#define ALLOC0(size) palloc0(size) ++#define REALLOC repalloc + -+#define appendStrVal appendStringInfo -+#define appendBinaryStrVal appendBinaryStringInfo -+#define appendStrValChar appendStringInfoChar -+#define createStrVal makeStringInfo -+#define resetStrVal resetStringInfo -+#define destroyStrVal destroyStringInfo ++/* ++ * Backend pfree() doesn't handle NULL pointers like the frontend's does; smooth ++ * that over to reduce mental gymnastics. Avoid multiple evaluation of the macro ++ * argument to avoid future hair-pulling. ++ */ ++#define FREE(s) do { \ ++ void *__v = (s); \ ++ if (__v) \ ++ pfree(__v); \ ++} while (0) ++ ++#define appendStrVal appendStringInfo ++#define appendBinaryStrVal appendBinaryStringInfo ++#define appendStrValChar appendStringInfoChar ++#define appendStrValCharMacro appendStringInfoCharMacro ++#define createStrVal makeStringInfo ++#define initStrVal initStringInfo ++#define resetStrVal resetStringInfo ++#define termStrVal(s) pfree((s)->data) ++#define destroyStrVal destroyStringInfo + +#endif + /* * The context of the parser is maintained by the recursive descent * mechanism, but is passed explicitly to the error reporting routine -@@ src/common/jsonapi.c: makeJsonLexContextCstringLen(JsonLexContext *lex, char *json, +@@ src/common/jsonapi.c: struct JsonIncrementalState + { + bool is_last_chunk; + bool partial_completed; +- StringInfoData partial_token; ++ StrValType partial_token; + }; + + /* +@@ src/common/jsonapi.c: static JsonParseErrorType parse_object(JsonLexContext *lex, JsonSemAction *sem); + static JsonParseErrorType parse_array_element(JsonLexContext *lex, JsonSemAction *sem); + static JsonParseErrorType parse_array(JsonLexContext *lex, JsonSemAction *sem); + static JsonParseErrorType report_parse_error(JsonParseContext ctx, JsonLexContext *lex); ++static bool allocate_incremental_state(JsonLexContext *lex); + + /* the null action object used for pure validation */ + JsonSemAction nullSemAction = +@@ src/common/jsonapi.c: IsValidJsonNumber(const char *str, size_t len) + { + bool numeric_error; + size_t total_len; +- JsonLexContext dummy_lex; ++ JsonLexContext dummy_lex = {0}; + + if (len <= 0) + return false; + +- dummy_lex.incremental = false; +- dummy_lex.inc_state = NULL; +- dummy_lex.pstack = NULL; +- + /* + * json_lex_number expects a leading '-' to have been eaten already. + * +@@ src/common/jsonapi.c: IsValidJsonNumber(const char *str, size_t len) + * responsible for freeing the returned struct, either by calling + * freeJsonLexContext() or (in backend environment) via memory context + * cleanup. ++ * ++ * In frontend code this can return NULL on OOM, so callers must inspect the ++ * returned pointer. + */ + JsonLexContext * + makeJsonLexContextCstringLen(JsonLexContext *lex, const char *json, +@@ src/common/jsonapi.c: makeJsonLexContextCstringLen(JsonLexContext *lex, const char *json, + { + if (lex == NULL) + { +- lex = palloc0(sizeof(JsonLexContext)); ++ lex = ALLOC0(sizeof(JsonLexContext)); ++ if (!lex) ++ return NULL; + lex->flags |= JSONLEX_FREE_STRUCT; + } + else +@@ src/common/jsonapi.c: makeJsonLexContextCstringLen(JsonLexContext *lex, const char *json, lex->input_encoding = encoding; if (need_escapes) { @@ src/common/jsonapi.c: makeJsonLexContextCstringLen(JsonLexContext *lex, char *js lex->flags |= JSONLEX_FREE_STRVAL; + lex->parse_strval = true; } -+ lex->errormsg = NULL; return lex; } -@@ src/common/jsonapi.c: makeJsonLexContextCstringLen(JsonLexContext *lex, char *json, + ++/* ++ * Allocates the internal bookkeeping structures for incremental parsing. This ++ * can only fail in-band with FRONTEND code. ++ */ ++#define JS_STACK_CHUNK_SIZE 64 ++#define JS_MAX_PROD_LEN 10 /* more than we need */ ++#define JSON_TD_MAX_STACK 6400 /* hard coded for now - this is a REALLY high ++ * number */ ++static bool ++allocate_incremental_state(JsonLexContext *lex) ++{ ++ void *pstack, ++ *prediction, ++ *fnames, ++ *fnull; ++ ++ lex->inc_state = ALLOC0(sizeof(JsonIncrementalState)); ++ pstack = ALLOC(sizeof(JsonParserStack)); ++ prediction = ALLOC(JS_STACK_CHUNK_SIZE * JS_MAX_PROD_LEN); ++ fnames = ALLOC(JS_STACK_CHUNK_SIZE * sizeof(char *)); ++ fnull = ALLOC(JS_STACK_CHUNK_SIZE * sizeof(bool)); ++ ++#ifdef FRONTEND ++ if (!lex->inc_state ++ || !pstack ++ || !prediction ++ || !fnames ++ || !fnull) ++ { ++ FREE(lex->inc_state); ++ FREE(pstack); ++ FREE(prediction); ++ FREE(fnames); ++ FREE(fnull); ++ ++ return false; ++ } ++#endif ++ ++ initStrVal(&(lex->inc_state->partial_token)); ++ lex->pstack = pstack; ++ lex->pstack->stack_size = JS_STACK_CHUNK_SIZE; ++ lex->pstack->prediction = prediction; ++ lex->pstack->pred_index = 0; ++ lex->pstack->fnames = fnames; ++ lex->pstack->fnull = fnull; ++ ++ lex->incremental = true; ++ return true; ++} ++ + + /* + * makeJsonLexContextIncremental +@@ src/common/jsonapi.c: makeJsonLexContextCstringLen(JsonLexContext *lex, const char *json, + * we don't need the input, that will be handed in bit by bit to the + * parse routine. We also need an accumulator for partial tokens in case + * the boundary between chunks happens to fall in the middle of a token. ++ * ++ * In frontend code this can return NULL on OOM, so callers must inspect the ++ * returned pointer. + */ +-#define JS_STACK_CHUNK_SIZE 64 +-#define JS_MAX_PROD_LEN 10 /* more than we need */ +-#define JSON_TD_MAX_STACK 6400 /* hard coded for now - this is a REALLY high +- * number */ +- + JsonLexContext * + makeJsonLexContextIncremental(JsonLexContext *lex, int encoding, + bool need_escapes) + { + if (lex == NULL) + { +- lex = palloc0(sizeof(JsonLexContext)); ++ lex = ALLOC0(sizeof(JsonLexContext)); ++ if (!lex) ++ return NULL; ++ + lex->flags |= JSONLEX_FREE_STRUCT; + } + else +@@ src/common/jsonapi.c: makeJsonLexContextIncremental(JsonLexContext *lex, int encoding, + + lex->line_number = 1; + lex->input_encoding = encoding; +- lex->incremental = true; +- lex->inc_state = palloc0(sizeof(JsonIncrementalState)); +- initStringInfo(&(lex->inc_state->partial_token)); +- lex->pstack = palloc(sizeof(JsonParserStack)); +- lex->pstack->stack_size = JS_STACK_CHUNK_SIZE; +- lex->pstack->prediction = palloc(JS_STACK_CHUNK_SIZE * JS_MAX_PROD_LEN); +- lex->pstack->pred_index = 0; +- lex->pstack->fnames = palloc(JS_STACK_CHUNK_SIZE * sizeof(char *)); +- lex->pstack->fnull = palloc(JS_STACK_CHUNK_SIZE * sizeof(bool)); ++ ++ if (!allocate_incremental_state(lex)) ++ { ++ if (lex->flags & JSONLEX_FREE_STRUCT) ++ FREE(lex); ++ return NULL; ++ } ++ + if (need_escapes) + { +- lex->strval = makeStringInfo(); ++ /* ++ * This call can fail in FRONTEND code. We defer error handling to ++ * time of use (json_lex_string()) since we might not need to parse ++ * any strings anyway. ++ */ ++ lex->strval = createStrVal(); + lex->flags |= JSONLEX_FREE_STRVAL; ++ lex->parse_strval = true; + } ++ + return lex; + } + +-static inline void ++static inline bool + inc_lex_level(JsonLexContext *lex) + { +- lex->lex_level += 1; +- +- if (lex->incremental && lex->lex_level >= lex->pstack->stack_size) ++ if (lex->incremental && (lex->lex_level + 1) >= lex->pstack->stack_size) + { +- lex->pstack->stack_size += JS_STACK_CHUNK_SIZE; +- lex->pstack->prediction = +- repalloc(lex->pstack->prediction, +- lex->pstack->stack_size * JS_MAX_PROD_LEN); +- if (lex->pstack->fnames) +- lex->pstack->fnames = +- repalloc(lex->pstack->fnames, +- lex->pstack->stack_size * sizeof(char *)); +- if (lex->pstack->fnull) +- lex->pstack->fnull = +- repalloc(lex->pstack->fnull, lex->pstack->stack_size * sizeof(bool)); ++ size_t new_stack_size; ++ char *new_prediction; ++ char **new_fnames; ++ bool *new_fnull; ++ ++ new_stack_size = lex->pstack->stack_size + JS_STACK_CHUNK_SIZE; ++ ++ new_prediction = REALLOC(lex->pstack->prediction, ++ new_stack_size * JS_MAX_PROD_LEN); ++ new_fnames = REALLOC(lex->pstack->fnames, ++ new_stack_size * sizeof(char *)); ++ new_fnull = REALLOC(lex->pstack->fnull, new_stack_size * sizeof(bool)); ++ ++#ifdef FRONTEND ++ if (!new_prediction || !new_fnames || !new_fnull) ++ return false; ++#endif ++ ++ lex->pstack->stack_size = new_stack_size; ++ lex->pstack->prediction = new_prediction; ++ lex->pstack->fnames = new_fnames; ++ lex->pstack->fnull = new_fnull; + } ++ ++ lex->lex_level += 1; ++ return true; + } + + static inline void +@@ src/common/jsonapi.c: get_fnull(JsonLexContext *lex) void freeJsonLexContext(JsonLexContext *lex) { + static const JsonLexContext empty = {0}; ++ ++ if (!lex) ++ return; + if (lex->flags & JSONLEX_FREE_STRVAL) - destroyStringInfo(lex->strval); @@ src/common/jsonapi.c: makeJsonLexContextCstringLen(JsonLexContext *lex, char *js - destroyStringInfo(lex->errormsg); + destroyStrVal(lex->errormsg); + if (lex->incremental) + { +- pfree(lex->inc_state->partial_token.data); +- pfree(lex->inc_state); +- pfree(lex->pstack->prediction); +- pfree(lex->pstack->fnames); +- pfree(lex->pstack->fnull); +- pfree(lex->pstack); ++ termStrVal(&lex->inc_state->partial_token); ++ FREE(lex->inc_state); ++ FREE(lex->pstack->prediction); ++ FREE(lex->pstack->fnames); ++ FREE(lex->pstack->fnull); ++ FREE(lex->pstack); + } + if (lex->flags & JSONLEX_FREE_STRUCT) - pfree(lex); +- pfree(lex); ++ FREE(lex); + else + *lex = empty; } /* +@@ src/common/jsonapi.c: JsonParseErrorType + pg_parse_json(JsonLexContext *lex, JsonSemAction *sem) + { + #ifdef FORCE_JSON_PSTACK +- +- lex->incremental = true; +- lex->inc_state = palloc0(sizeof(JsonIncrementalState)); +- + /* + * We don't need partial token processing, there is only one chunk. But we + * still need to init the partial token string so that freeJsonLexContext +- * works. ++ * works, so perform the full incremental initialization. + */ +- initStringInfo(&(lex->inc_state->partial_token)); +- lex->pstack = palloc(sizeof(JsonParserStack)); +- lex->pstack->stack_size = JS_STACK_CHUNK_SIZE; +- lex->pstack->prediction = palloc(JS_STACK_CHUNK_SIZE * JS_MAX_PROD_LEN); +- lex->pstack->pred_index = 0; +- lex->pstack->fnames = palloc(JS_STACK_CHUNK_SIZE * sizeof(char *)); +- lex->pstack->fnull = palloc(JS_STACK_CHUNK_SIZE * sizeof(bool)); ++ if (!allocate_incremental_state(lex)) ++ return JSON_OUT_OF_MEMORY; + + return pg_parse_json_incremental(lex, sem, lex->input, lex->input_length, true); + @@ src/common/jsonapi.c: json_count_array_elements(JsonLexContext *lex, int *elements) * etc, so doing this with a copy makes that safe. */ @@ src/common/jsonapi.c: json_count_array_elements(JsonLexContext *lex, int *elemen copylex.lex_level++; count = 0; +@@ src/common/jsonapi.c: pg_parse_json_incremental(JsonLexContext *lex, + if (result != JSON_SUCCESS) + return result; + } +- inc_lex_level(lex); ++ ++ if (!inc_lex_level(lex)) ++ return JSON_OUT_OF_MEMORY; + } + break; + case JSON_SEM_OEND: +@@ src/common/jsonapi.c: pg_parse_json_incremental(JsonLexContext *lex, + if (result != JSON_SUCCESS) + return result; + } +- inc_lex_level(lex); ++ ++ if (!inc_lex_level(lex)) ++ return JSON_OUT_OF_MEMORY; + } + break; + case JSON_SEM_AEND: +@@ src/common/jsonapi.c: pg_parse_json_incremental(JsonLexContext *lex, + json_ofield_action ostart = sem->object_field_start; + json_ofield_action oend = sem->object_field_end; + +- if ((ostart != NULL || oend != NULL) && lex->strval != NULL) ++ if ((ostart != NULL || oend != NULL) && lex->parse_strval) + { +- fname = pstrdup(lex->strval->data); ++ fname = STRDUP(lex->strval->data); ++ if (fname == NULL) ++ return JSON_OUT_OF_MEMORY; + } + set_fname(lex, fname); + } +@@ src/common/jsonapi.c: pg_parse_json_incremental(JsonLexContext *lex, + */ + if (tok == JSON_TOKEN_STRING) + { +- if (lex->strval != NULL) +- pstack->scalar_val = pstrdup(lex->strval->data); ++ if (lex->parse_strval) ++ { ++ pstack->scalar_val = STRDUP(lex->strval->data); ++ if (pstack->scalar_val == NULL) ++ return JSON_OUT_OF_MEMORY; ++ } + } + else + { + ptrdiff_t tlen = (lex->token_terminator - lex->token_start); + +- pstack->scalar_val = palloc(tlen + 1); ++ pstack->scalar_val = ALLOC(tlen + 1); ++ if (pstack->scalar_val == NULL) ++ return JSON_OUT_OF_MEMORY; ++ + memcpy(pstack->scalar_val, lex->token_start, tlen); + pstack->scalar_val[tlen] = '\0'; + } @@ src/common/jsonapi.c: parse_scalar(JsonLexContext *lex, JsonSemAction *sem) /* extract the de-escaped string value, or the raw lexeme */ if (lex_peek(lex) == JSON_TOKEN_STRING) @@ src/common/jsonapi.c: parse_object(JsonLexContext *lex, JsonSemAction *sem) check_stack_depth(); #endif +@@ src/common/jsonapi.c: json_lex(JsonLexContext *lex) + const char *const end = lex->input + lex->input_length; + JsonParseErrorType result; + +- if (lex->incremental && lex->inc_state->partial_completed) ++ if (lex->incremental) + { +- /* +- * We just lexed a completed partial token on the last call, so reset +- * everything +- */ +- resetStringInfo(&(lex->inc_state->partial_token)); +- lex->token_terminator = lex->input; +- lex->inc_state->partial_completed = false; ++ if (lex->inc_state->partial_completed) ++ { ++ /* ++ * We just lexed a completed partial token on the last call, so ++ * reset everything ++ */ ++ resetStrVal(&(lex->inc_state->partial_token)); ++ lex->token_terminator = lex->input; ++ lex->inc_state->partial_completed = false; ++ } ++ ++#ifdef FRONTEND ++ /* Make sure our partial token buffer is valid before using it below. */ ++ if (PQExpBufferDataBroken(lex->inc_state->partial_token)) ++ return JSON_OUT_OF_MEMORY; ++#endif + } + + s = lex->token_terminator; +@@ src/common/jsonapi.c: json_lex(JsonLexContext *lex) + * We have a partial token. Extend it and if completed lex it by a + * recursive call + */ +- StringInfo ptok = &(lex->inc_state->partial_token); ++ StrValType *ptok = &(lex->inc_state->partial_token); + size_t added = 0; + bool tok_done = false; + JsonLexContext dummy_lex; +@@ src/common/jsonapi.c: json_lex(JsonLexContext *lex) + { + char c = lex->input[i]; + +- appendStringInfoCharMacro(ptok, c); ++ appendStrValCharMacro(ptok, c); + added++; + if (c == '"' && escapes % 2 == 0) + { +@@ src/common/jsonapi.c: json_lex(JsonLexContext *lex) + case '8': + case '9': + { +- appendStringInfoCharMacro(ptok, cc); ++ appendStrValCharMacro(ptok, cc); + added++; + } + break; +@@ src/common/jsonapi.c: json_lex(JsonLexContext *lex) + + if (JSON_ALPHANUMERIC_CHAR(cc)) + { +- appendStringInfoCharMacro(ptok, cc); ++ appendStrValCharMacro(ptok, cc); + added++; + } + else +@@ src/common/jsonapi.c: json_lex(JsonLexContext *lex) + dummy_lex.input_length = ptok->len; + dummy_lex.input_encoding = lex->input_encoding; + dummy_lex.incremental = false; ++ dummy_lex.parse_strval = lex->parse_strval; + dummy_lex.strval = lex->strval; + + partial_result = json_lex(&dummy_lex); +@@ src/common/jsonapi.c: json_lex(JsonLexContext *lex) + if (lex->incremental && !lex->inc_state->is_last_chunk && + p == lex->input + lex->input_length) + { +- appendBinaryStringInfo( +- &(lex->inc_state->partial_token), s, end - s); ++ appendBinaryStrVal( ++ &(lex->inc_state->partial_token), s, end - s); + return JSON_INCOMPLETE; + } + +@@ src/common/jsonapi.c: json_lex_string(JsonLexContext *lex) + do { \ + if (lex->incremental && !lex->inc_state->is_last_chunk) \ + { \ +- appendBinaryStringInfo(&lex->inc_state->partial_token, \ +- lex->token_start, end - lex->token_start); \ ++ appendBinaryStrVal(&lex->inc_state->partial_token, \ ++ lex->token_start, end - lex->token_start); \ + return JSON_INCOMPLETE; \ + } \ + lex->token_terminator = s; \ @@ src/common/jsonapi.c: json_lex_string(JsonLexContext *lex) return code; \ } while (0) @@ src/common/jsonapi.c: json_lex_string(JsonLexContext *lex) /* Hooray, we found the end of the string! */ lex->prev_token_terminator = lex->token_terminator; lex->token_terminator = s + 1; +@@ src/common/jsonapi.c: json_lex_number(JsonLexContext *lex, const char *s, + if (lex->incremental && !lex->inc_state->is_last_chunk && + len >= lex->input_length) + { +- appendBinaryStringInfo(&lex->inc_state->partial_token, +- lex->token_start, s - lex->token_start); ++ appendBinaryStrVal(&lex->inc_state->partial_token, ++ lex->token_start, s - lex->token_start); + if (num_err != NULL) + *num_err = error; + @@ src/common/jsonapi.c: report_parse_error(JsonParseContext ctx, JsonLexContext *lex) char * json_errdetail(JsonParseErrorType error, JsonLexContext *lex) @@ src/common/jsonapi.c: report_parse_error(JsonParseContext ctx, JsonLexContext *l * A helper for error messages that should print the current token. The * format must contain exactly one %.*s specifier. */ - #define token_error(lex, format) \ + #define json_token_error(lex, format) \ - appendStringInfo((lex)->errormsg, _(format), \ - (int) ((lex)->token_terminator - (lex)->token_start), \ - (lex)->token_start); @@ src/common/jsonapi.c: report_parse_error(JsonParseContext ctx, JsonLexContext *l switch (error) { @@ src/common/jsonapi.c: json_errdetail(JsonParseErrorType error, JsonLexContext *lex) - token_error(lex, "Escape sequence \"\\%.*s\" is invalid."); + json_token_error(lex, "Escape sequence \"\\%.*s\" is invalid."); break; case JSON_ESCAPING_REQUIRED: - appendStringInfo(lex->errormsg, @@ src/common/jsonapi.c: json_errdetail(JsonParseErrorType error, JsonLexContext *l + (unsigned char) *(lex->token_terminator)); break; case JSON_EXPECTED_END: - token_error(lex, "Expected end of input, but found \"%.*s\"."); + json_token_error(lex, "Expected end of input, but found \"%.*s\"."); @@ src/common/jsonapi.c: json_errdetail(JsonParseErrorType error, JsonLexContext *lex) case JSON_INVALID_TOKEN: - token_error(lex, "Token \"%.*s\" is invalid."); + json_token_error(lex, "Token \"%.*s\" is invalid."); break; + case JSON_OUT_OF_MEMORY: + /* should have been handled above; use the error path */ @@ src/common/jsonapi.c: json_errdetail(JsonParseErrorType error, JsonLexContext *l case JSON_UNICODE_ESCAPE_FORMAT: @@ src/common/jsonapi.c: json_errdetail(JsonParseErrorType error, JsonLexContext *lex) } - #undef token_error + #undef json_token_error - /* - * We don't use a default: case, so that the compiler will warn about @@ src/common/jsonapi.c: json_errdetail(JsonParseErrorType error, JsonLexContext *l - */ - if (lex->errormsg->len == 0) - appendStringInfo(lex->errormsg, -- _("unexpected json parse error type: %d"), +- "unexpected json parse error type: %d", - (int) error); + /* Note that lex->errormsg can be NULL in FRONTEND code. */ + if (lex->errormsg && lex->errormsg->len == 0) @@ src/common/jsonapi.c: json_errdetail(JsonParseErrorType error, JsonLexContext *l + * the possibility of an incorrect input. + */ + appendStrVal(lex->errormsg, -+ _("unexpected json parse error type: %d"), ++ "unexpected json parse error type: %d", + (int) error); + } + @@ src/common/meson.build: foreach name, opts : pgcommon_variants 'dependencies': opts['dependencies'] + [ssl], } + ## src/common/parse_manifest.c ## +@@ src/common/parse_manifest.c: json_parse_manifest_incremental_init(JsonManifestParseContext *context) + parse->state = JM_EXPECT_TOPLEVEL_START; + parse->saw_version_field = false; + +- makeJsonLexContextIncremental(&(incstate->lex), PG_UTF8, true); ++ if (!makeJsonLexContextIncremental(&(incstate->lex), PG_UTF8, true)) ++ context->error_cb(context, "out of memory"); + + incstate->sem.semstate = parse; + incstate->sem.object_start = json_manifest_object_start; +@@ src/common/parse_manifest.c: json_parse_manifest(JsonManifestParseContext *context, const char *buffer, + + /* Create a JSON lexing context. */ + lex = makeJsonLexContextCstringLen(NULL, buffer, size, PG_UTF8, true); ++ if (!lex) ++ json_manifest_parse_failure(context, "out of memory"); + + /* Set up semantic actions. */ + sem.semstate = &parse; + ## src/include/common/jsonapi.h ## @@ #ifndef JSONAPI_H @@ src/include/common/jsonapi.h: typedef enum JsonParseErrorType JSON_UNICODE_ESCAPE_FORMAT, JSON_UNICODE_HIGH_ESCAPE, @@ src/include/common/jsonapi.h: typedef enum JsonParseErrorType - JSON_SEM_ACTION_FAILED, /* error should already be reported */ - } JsonParseErrorType; + typedef struct JsonParserStack JsonParserStack; + typedef struct JsonIncrementalState JsonIncrementalState; +/* + * Don't depend on the internal type header for strval; if callers need access @@ src/include/common/jsonapi.h: typedef enum JsonParseErrorType +#endif + +typedef struct StrValType StrValType; - ++ /* * All the fields in this structure should be treated as read-only. + * @@ src/include/common/jsonapi.h: typedef struct JsonLexContext - bits32 flags; - int line_number; /* line number, starting from 1 */ - char *line_start; /* where that line starts within input */ + const char *line_start; /* where that line starts within input */ + JsonParserStack *pstack; + JsonIncrementalState *inc_state; - StringInfo strval; - StringInfo errormsg; + bool parse_strval; @@ src/include/common/jsonapi.h: typedef struct JsonLexContext } JsonLexContext; typedef JsonParseErrorType (*json_struct_action) (void *state); + + ## src/test/modules/test_json_parser/Makefile ## +@@ src/test/modules/test_json_parser/Makefile: include $(top_builddir)/src/Makefile.global + include $(top_srcdir)/contrib/contrib-global.mk + endif + ++# TODO: fix this properly ++LDFLAGS_INTERNAL += -lpgcommon $(libpq_pgport) ++ + all: test_json_parser_incremental$(X) test_json_parser_perf$(X) + + %.o: $(top_srcdir)/$(subdir)/%.c + + ## src/test/modules/test_json_parser/meson.build ## +@@ src/test/modules/test_json_parser/meson.build: endif + + test_json_parser_incremental = executable('test_json_parser_incremental', + test_json_parser_incremental_sources, +- dependencies: [frontend_code], ++ dependencies: [frontend_code, libpq], + kwargs: default_bin_args + { + 'install': false, + }, +@@ src/test/modules/test_json_parser/meson.build: endif + + test_json_parser_perf = executable('test_json_parser_perf', + test_json_parser_perf_sources, +- dependencies: [frontend_code], ++ dependencies: [frontend_code, libpq], + kwargs: default_bin_args + { + 'install': false, + }, 2: 31f2ffdf0b ! 4: 1639f7eb9a libpq: add OAUTHBEARER SASL mechanism @@ src/interfaces/libpq/Makefile: endif endif ## src/interfaces/libpq/exports.txt ## -@@ src/interfaces/libpq/exports.txt: PQcancelSocket 199 - PQcancelErrorMessage 200 - PQcancelReset 201 - PQcancelFinish 202 -+PQsetAuthDataHook 203 -+PQgetAuthDataHook 204 -+PQdefaultAuthDataHook 205 +@@ src/interfaces/libpq/exports.txt: PQcancelFinish 202 + PQsocketPoll 203 + PQsetChunkedRowsMode 204 + PQgetCurrentTimeUSec 205 ++PQsetAuthDataHook 206 ++PQgetAuthDataHook 207 ++PQdefaultAuthDataHook 208 ## src/interfaces/libpq/fe-auth-oauth-curl.c (new) ## @@ @@ src/interfaces/libpq/fe-connect.c: keep_going: /* We will come back to here + } +#endif + - #ifdef ENABLE_GSS - - /* + CONNECTION_FAILED(); + } + else if (beresp == PqMsg_NegotiateProtocolVersion) @@ src/interfaces/libpq/fe-connect.c: keep_going: /* We will come back to here until there is * Note that conn->pghost must be non-NULL if we are going to * avoid the Kerberos code doing a hostname look-up. @@ src/interfaces/libpq/fe-connect.c: PQsocket(const PGconn *conn) ## src/interfaces/libpq/fe-misc.c ## @@ src/interfaces/libpq/fe-misc.c: static int - pqSocketCheck(PGconn *conn, int forRead, int forWrite, time_t end_time) + pqSocketCheck(PGconn *conn, int forRead, int forWrite, pg_usec_time_t end_time) { int result; + pgsocket sock; @@ src/interfaces/libpq/fe-misc.c: static int { libpq_append_conn_error(conn, "invalid socket"); return -1; -@@ src/interfaces/libpq/fe-misc.c: pqSocketCheck(PGconn *conn, int forRead, int forWrite, time_t end_time) +@@ src/interfaces/libpq/fe-misc.c: pqSocketCheck(PGconn *conn, int forRead, int forWrite, pg_usec_time_t end_time) /* We will retry as long as we get EINTR */ do -- result = pqSocketPoll(conn->sock, forRead, forWrite, end_time); -+ result = pqSocketPoll(sock, forRead, forWrite, end_time); +- result = PQsocketPoll(conn->sock, forRead, forWrite, end_time); ++ result = PQsocketPoll(sock, forRead, forWrite, end_time); while (result < 0 && SOCK_ERRNO == EINTR); if (result < 0) @@ src/interfaces/libpq/libpq-fe.h: extern "C" /* * Option flags for PQcopyResult @@ src/interfaces/libpq/libpq-fe.h: typedef enum - CONNECTION_CHECK_TARGET, /* Internal state: checking target server - * properties. */ CONNECTION_CHECK_STANDBY, /* Checking if server is in standby mode. */ -- CONNECTION_ALLOCATED /* Waiting for connection attempt to be -+ CONNECTION_ALLOCATED, /* Waiting for connection attempt to be + CONNECTION_ALLOCATED, /* Waiting for connection attempt to be * started. */ + CONNECTION_AUTHENTICATING, /* Authentication is in progress with some + * external system. */ 3: 96892c6249 ! 5: 044d8b08e9 backend: add OAUTHBEARER SASL mechanism @@ Commit message Co-authored-by: Daniel Gustafsson + ## .cirrus.tasks.yml ## +@@ .cirrus.tasks.yml: task: + chown root:postgres /tmp/cores + sysctl kern.corefile='/tmp/cores/%N.%P.core' + setup_additional_packages_script: | +- #pkg install -y ... ++ pkg install -y curl + + # NB: Intentionally build without -Dllvm. The freebsd image size is already + # large enough to make VM startup slow, and even without llvm freebsd +@@ .cirrus.tasks.yml: task: + -Dcassert=true -Dinjection_points=true \ + -Duuid=bsd -Dtcl_version=tcl86 -Ddtrace=auto \ + -DPG_TEST_EXTRA="$PG_TEST_EXTRA" \ ++ -Doauth=curl \ + -Dextra_lib_dirs=/usr/local/lib -Dextra_include_dirs=/usr/local/include/ \ + build + EOF +@@ .cirrus.tasks.yml: LINUX_CONFIGURE_FEATURES: &LINUX_CONFIGURE_FEATURES >- + --with-libxslt + --with-llvm + --with-lz4 ++ --with-oauth=curl + --with-pam + --with-perl + --with-python +@@ .cirrus.tasks.yml: LINUX_CONFIGURE_FEATURES: &LINUX_CONFIGURE_FEATURES >- + + LINUX_MESON_FEATURES: &LINUX_MESON_FEATURES >- + -Dllvm=enabled ++ -Doauth=curl + -Duuid=e2fs + + +@@ .cirrus.tasks.yml: task: + EOF + + setup_additional_packages_script: | +- #apt-get update +- #DEBIAN_FRONTEND=noninteractive apt-get -y install ... ++ apt-get update ++ DEBIAN_FRONTEND=noninteractive apt-get -y install \ ++ libcurl4-openssl-dev \ ++ libcurl4-openssl-dev:i386 \ + + matrix: + - name: Linux - Debian Bullseye - Autoconf +@@ .cirrus.tasks.yml: task: + folder: $CCACHE_DIR + + setup_additional_packages_script: | +- #apt-get update +- #DEBIAN_FRONTEND=noninteractive apt-get -y install ... ++ apt-get update ++ DEBIAN_FRONTEND=noninteractive apt-get -y install libcurl4-openssl-dev + + ### + # Test that code can be built with gcc/clang without warnings + ## src/backend/libpq/Makefile ## @@ src/backend/libpq/Makefile: include $(top_builddir)/src/Makefile.global # be-fsstubs is here for historical reasons, probably belongs elsewhere @@ src/backend/utils/misc/guc_tables.c #include "nodes/queryjumble.h" #include "optimizer/cost.h" @@ src/backend/utils/misc/guc_tables.c: struct config_string ConfigureNamesString[] = - check_standby_slot_names, assign_standby_slot_names, NULL + check_synchronized_standby_slots, assign_synchronized_standby_slots, NULL }, + { @@ src/include/libpq/sasl.h: typedef struct pg_be_sasl_mech /* Common implementation for auth.c */ + ## src/test/modules/Makefile ## +@@ src/test/modules/Makefile: SUBDIRS = \ + dummy_index_am \ + dummy_seclabel \ + libpq_pipeline \ ++ oauth_validator \ + plsample \ + spgist_name_ops \ + test_bloomfilter \ + ## src/test/modules/meson.build ## @@ src/test/modules/meson.build: subdir('gin') subdir('injection_points') @@ src/test/modules/oauth_validator/.gitignore (new) ## src/test/modules/oauth_validator/Makefile (new) ## @@ +export PYTHON ++export with_oauth + +MODULES = validator +PGFILEDESC = "validator - test OAuth validator module" @@ src/test/modules/oauth_validator/meson.build (new) + ], + 'env': { + 'PYTHON': python.path(), ++ 'with_oauth': oauth_library, + }, + }, +} @@ src/test/modules/oauth_validator/t/001_server.pl (new) +use PostgreSQL::Test::OAuthServer; +use Test::More; + ++if ($ENV{with_oauth} ne 'curl') ++{ ++ plan skip_all => 'client-side OAuth not supported by this build'; ++} ++ +my $node = PostgreSQL::Test::Cluster->new('primary'); +$node->init; +$node->append_conf('postgresql.conf', "log_connections = on\n"); 4: 5677e59152 ! 6: 84c6893325 Review comments @@ Commit message * Implement a version check for libcurl in autoconf, the equivalent check for Meson is still a TODO. * Address a few TODOs in the code - * libpq JSON support memory management fixups + * libpq JSON support memory management fixups [ed: these have been moved + to an earlier commit] ## config/programs.m4 ## @@ config/programs.m4: if test "$pgac_cv_ldap_safe" != yes; then @@ src/backend/libpq/auth-oauth.c: validate(Port *port, const char *auth) } - ## src/common/jsonapi.c ## -@@ - #endif - - /* -- * In backend, we will use palloc/pfree along with StringInfo. In frontend, use -- * malloc and PQExpBuffer, and return JSON_OUT_OF_MEMORY on out-of-memory. -+ * In backend, we will use palloc/pfree along with StringInfo. In frontend, -+ * use malloc and PQExpBuffer, and return JSON_OUT_OF_MEMORY on out-of-memory. - */ - #ifdef FRONTEND - - #define STRDUP(s) strdup(s) - #define ALLOC(size) malloc(size) -+#define ALLOC0(size) calloc(1, size) -+#define FREE(s) free(s) - - #define appendStrVal appendPQExpBuffer - #define appendBinaryStrVal appendBinaryPQExpBuffer -@@ - - #define STRDUP(s) pstrdup(s) - #define ALLOC(size) palloc(size) -+#define ALLOC0(size) palloc0(size) -+#define FREE(s) pfree(s) - - #define appendStrVal appendStringInfo - #define appendBinaryStrVal appendBinaryStringInfo -@@ src/common/jsonapi.c: IsValidJsonNumber(const char *str, int len) - * responsible for freeing the returned struct, either by calling - * freeJsonLexContext() or (in backend environment) via memory context - * cleanup. -+ * -+ * In frontend code this can return NULL on OOM, so callers must inspect the -+ * returned pointer. - */ - JsonLexContext * - makeJsonLexContextCstringLen(JsonLexContext *lex, char *json, -@@ src/common/jsonapi.c: makeJsonLexContextCstringLen(JsonLexContext *lex, char *json, - { - if (lex == NULL) - { -- lex = palloc0(sizeof(JsonLexContext)); -+ lex = ALLOC0(sizeof(JsonLexContext)); -+ if (!lex) -+ return NULL; - lex->flags |= JSONLEX_FREE_STRUCT; - } - else -@@ src/common/jsonapi.c: freeJsonLexContext(JsonLexContext *lex) - { - static const JsonLexContext empty = {0}; - -+ if (!lex) -+ return; -+ - if (lex->flags & JSONLEX_FREE_STRVAL) - destroyStrVal(lex->strval); - -@@ src/common/jsonapi.c: freeJsonLexContext(JsonLexContext *lex) - destroyStrVal(lex->errormsg); - - if (lex->flags & JSONLEX_FREE_STRUCT) -- pfree(lex); -+ FREE(lex); - else - *lex = empty; - } - - ## src/common/parse_manifest.c ## -@@ src/common/parse_manifest.c: json_parse_manifest(JsonManifestParseContext *context, char *buffer, - - /* Create a JSON lexing context. */ - lex = makeJsonLexContextCstringLen(NULL, buffer, size, PG_UTF8, true); -+ if (!lex) -+ json_manifest_parse_failure(context, "out of memory"); - - /* Set up semantic actions. */ - sem.semstate = &parse; - ## src/include/common/oauth-common.h ## @@ * oauth-common.h