From 16df5bb881b8255e57d408a019a23ba029e9cb9b Mon Sep 17 00:00:00 2001 From: Dridi Boukelmoune Date: Tue, 4 Jan 2022 08:24:47 +0100 Subject: [PATCH 1/8] vcc: Refine tracking of constant expressions This tracking was very approximate, leading to many constant expressions being considered variable. Instead of marking all expressions variable right away, we can keep track of expressions that are edited on top of a constant expression. If a variable expression is simply the edition of a constant expression, for example a constant STRING edited into a STRANDS, the variable one keeps track of its constant origins. --- lib/libvcc/vcc_expr.c | 27 ++++++++++++++++++++++++--- 1 file changed, 24 insertions(+), 3 deletions(-) diff --git a/lib/libvcc/vcc_expr.c b/lib/libvcc/vcc_expr.c index 1b5926499c8..8ccb002a502 100644 --- a/lib/libvcc/vcc_expr.c +++ b/lib/libvcc/vcc_expr.c @@ -46,6 +46,7 @@ struct expr { vcc_type_t fmt; struct vsb *vsb; uint8_t constant; + struct expr *constant_expr; #define EXPR_VAR (1<<0) #define EXPR_CONST (1<<1) #define EXPR_STR_CONST (1<<2) // Last string elem is "..." @@ -95,7 +96,6 @@ vcc_new_expr(vcc_type_t fmt) AN(e); e->vsb = VSB_new_auto(); e->fmt = fmt; - e->constant = EXPR_VAR; return (e); } @@ -106,6 +106,7 @@ vcc_mk_expr(vcc_type_t fmt, const char *str, ...) struct expr *e; e = vcc_new_expr(fmt); + e->constant = EXPR_VAR; va_start(ap, str); VSB_vprintf(e->vsb, str, ap); va_end(ap); @@ -119,6 +120,7 @@ vcc_delete_expr(struct expr *e) if (e == NULL) return; CHECK_OBJ(e, EXPR_MAGIC); + vcc_delete_expr(e->constant_expr); VSB_destroy(&e->vsb); FREE_OBJ(e); } @@ -149,9 +151,10 @@ vcc_delete_expr(struct expr *e) */ static void -vcc_strands_edit(const struct expr *e1, const struct expr *e2) +vcc_strands_edit(struct expr *e1, const struct expr *e2) { + e1->constant = EXPR_VAR; if (e2->nstr == 1) { VSB_printf(e1->vsb, "TOSTRAND(%s)", VSB_data(e2->vsb)); return; @@ -172,6 +175,15 @@ vcc_expr_edit(struct vcc *tl, vcc_type_t fmt, const char *p, struct expr *e1, AN(e1); e = vcc_new_expr(fmt); + + /* NB: An expression with two operands is enough to be considered + * variable, even if both operands are constant. + */ + if ((e1->constant & EXPR_VAR) || e2 != NULL) + e->constant = EXPR_VAR; + else + e->constant = e1->constant; + while (*p != '\0') { if (*p != '\v') { if (*p != '\n' || !nl) @@ -222,7 +234,10 @@ vcc_expr_edit(struct vcc *tl, vcc_type_t fmt, const char *p, struct expr *e1, e->t2 = e1->t2; if (e2 != NULL) e->t2 = e2->t2; - vcc_delete_expr(e1); + if (!(e1->constant & EXPR_VAR) && e2 == NULL) + e->constant_expr = e1; + else + vcc_delete_expr(e1); vcc_delete_expr(e2); return (e); } @@ -271,6 +286,7 @@ vcc_expr_tobool(struct vcc *tl, struct expr **e) if ((*e)->fmt == BOOL) return; + (*e)->constant = EXPR_VAR; if ((*e)->fmt == BACKEND || (*e)->fmt == INT) *e = vcc_expr_edit(tl, BOOL, "(\v1 != 0)", *e, NULL); else if ((*e)->fmt == DURATION) @@ -948,6 +964,11 @@ vcc_expr4(struct vcc *tl, struct expr **e, vcc_type_t fmt) return; } + /* NB: we have no idea what method calls do, at this point all + * bets are off. + */ + (*e)->constant = EXPR_VAR; + AN(sym->eval); sym->eval(tl, e, tl->t, sym, sym->type); ERRCHK(tl); From d1289a71824b84672c2172437931b89c1f1ab991 Mon Sep 17 00:00:00 2001 From: Dridi Boukelmoune Date: Tue, 4 Jan 2022 08:35:35 +0100 Subject: [PATCH 2/8] vcc: Allow vcc_Expr() call sites to edit expressions If the expression was successfully parsed, it is possible to define how to edit it, whether it is variable or constant. This allows vcc_Expr() call sites to pass edit strings instead of making calls to Fb() and intertwine them with INDENT adjustments. --- lib/libvcc/vcc_compile.h | 1 + lib/libvcc/vcc_expr.c | 31 ++++++++++++++++++++++++++++++- 2 files changed, 31 insertions(+), 1 deletion(-) diff --git a/lib/libvcc/vcc_compile.h b/lib/libvcc/vcc_compile.h index 76b2cab734d..904fe3904b6 100644 --- a/lib/libvcc/vcc_compile.h +++ b/lib/libvcc/vcc_compile.h @@ -348,6 +348,7 @@ char *TlDup(struct vcc *tl, const char *s); /* vcc_expr.c */ void vcc_Expr(struct vcc *tl, vcc_type_t typ); +void vcc_ExprEdit(struct vcc *tl, vcc_type_t typ, const char *, const char *); sym_act_f vcc_Act_Call; sym_act_f vcc_Act_Obj; void vcc_Expr_Init(struct vcc *tl); diff --git a/lib/libvcc/vcc_expr.c b/lib/libvcc/vcc_expr.c index 8ccb002a502..c51ee4c242f 100644 --- a/lib/libvcc/vcc_expr.c +++ b/lib/libvcc/vcc_expr.c @@ -1510,13 +1510,42 @@ vcc_expr_typecheck(struct vcc *tl, struct expr **e, vcc_type_t fmt, void vcc_Expr(struct vcc *tl, vcc_type_t fmt) { - struct expr *e = NULL; + + vcc_ExprEdit(tl, fmt, NULL, NULL); +} + +void +vcc_ExprEdit(struct vcc *tl, vcc_type_t fmt, const char *var_edit, + const char *const_edit) +{ + struct expr *e = NULL, *e1 = NULL; + const char *edit = NULL; assert(fmt != VOID); assert(fmt != STRINGS); vcc_expr0(tl, &e, fmt); ERRCHK(tl); assert(e->fmt == fmt); + if (const_edit != NULL) + AN(var_edit); + + if (e->constant & EXPR_VAR) { + if (e->constant_expr != NULL) { + edit = const_edit; + e1 = e->constant_expr; + } + } else { + edit = const_edit; + e1 = e; + } + + if (edit == NULL) { + edit = var_edit; + e1 = e; + } + + if (edit != NULL) + e = vcc_expr_edit(tl, e1->fmt, edit, e1, NULL); vcc_expr_fmt(tl->fb, tl->indent, e); VSB_cat(tl->fb, "\n"); From 9fb94393452c37e52df26d626bcd131d9e6c1f93 Mon Sep 17 00:00:00 2001 From: Dridi Boukelmoune Date: Tue, 4 Jan 2022 08:37:17 +0100 Subject: [PATCH 3/8] vcc: Parse assignment expressions with edit string Go one step further in the reuse of the expression edit syntax and have a proper "\v2" sequence to refer to the symbol on the RHS in the assign table. --- lib/libvcc/vcc_action.c | 72 ++++++++++++++++++++--------------------- 1 file changed, 36 insertions(+), 36 deletions(-) diff --git a/lib/libvcc/vcc_action.c b/lib/libvcc/vcc_action.c index f7c2ce29fbb..25e0891e6d7 100644 --- a/lib/libvcc/vcc_action.c +++ b/lib/libvcc/vcc_action.c @@ -91,35 +91,38 @@ vcc_act_call(struct vcc *tl, struct token *t, struct symbol *sym) Fb(tl, 1, "}\n"); } -/*--------------------------------------------------------------------*/ +/*-------------------------------------------------------------------- + * We borrow the edit string syntax from vcc_expr.c and use the \v2 + * sequence to refer to the symbol as itself on the RHS. + */ static const struct assign { vcc_type_t type; unsigned oper; vcc_type_t want; - const char *expr; + const char *edit; } assign[] = { - { INT, T_INCR, INT, "\v + " }, - { INT, T_DECR, INT, "\v - " }, - { INT, T_MUL, INT, "\v * " }, - { INT, T_DIV, INT, "\v / " }, + { INT, T_INCR, INT, "\v2 + " }, + { INT, T_DECR, INT, "\v2 - " }, + { INT, T_MUL, INT, "\v2 * " }, + { INT, T_DIV, INT, "\v2 / " }, { INT, '=', INT }, { INT, 0, INT }, - { TIME, T_INCR, DURATION, "\v + " }, - { TIME, T_DECR, DURATION, "\v - " }, - { TIME, T_MUL, REAL, "\v * " }, - { TIME, T_DIV, REAL, "\v / " }, + { TIME, T_INCR, DURATION, "\v2 + " }, + { TIME, T_DECR, DURATION, "\v2 - " }, + { TIME, T_MUL, REAL, "\v2 * " }, + { TIME, T_DIV, REAL, "\v2 / " }, { TIME, '=', TIME }, { TIME, 0, TIME }, - { DURATION, T_INCR, DURATION, "\v + " }, - { DURATION, T_DECR, DURATION, "\v - " }, - { DURATION, T_MUL, REAL, "\v * " }, - { DURATION, T_DIV, REAL, "\v / " }, + { DURATION, T_INCR, DURATION, "\v2 + " }, + { DURATION, T_DECR, DURATION, "\v2 - " }, + { DURATION, T_MUL, REAL, "\v2 * " }, + { DURATION, T_DIV, REAL, "\v2 / " }, { DURATION, '=', DURATION }, { DURATION, 0, DURATION }, - { STRING, T_INCR, STRANDS, "\v,\n" }, + { STRING, T_INCR, STRANDS, "\v2,\n" }, { STRING, '=', STRANDS, "0,\n" }, - { HEADER, T_INCR, STRANDS, "VRT_GetHdr(ctx, \v),\n" }, + { HEADER, T_INCR, STRANDS, "VRT_GetHdr(ctx, \v2),\n" }, { HEADER, '=', STRANDS, "0,\n" }, { BODY, '=', BODY, "LBODY_SET_" }, { BODY, T_INCR, BODY, "LBODY_ADD_" }, @@ -127,23 +130,20 @@ static const struct assign { }; static void -vcc_assign_expr(struct vcc *tl, struct symbol *sym, const struct assign *ap) +vcc_assign_edit(struct vsb *vsb, struct symbol *sym, const struct assign *ap) { const char *e; - unsigned indent = 1; - e = ap->expr; - if (e == NULL) - return; - - while (*e != '\0') { - if (*e == '\v') - Fb(tl, indent, "%s", sym->rname); - else - Fb(tl, indent, "%c", *e); - indent = 0; - e++; + VSB_printf(vsb, "%s\n\v+", sym->lname); + for (e = ap->edit; e != NULL && *e != '\0'; e++) { + if (*e == '\v' && e[1] == '2') { + VSB_cat(vsb, sym->rname); + e++; + continue; + } + VSB_putc(vsb, *e); } + VSB_cat(vsb, "\v1\v-);\n"); } /*--------------------------------------------------------------------*/ @@ -152,6 +152,7 @@ static void v_matchproto_(sym_act_f) vcc_act_set(struct vcc *tl, struct token *t, struct symbol *sym) { const struct assign *ap; + struct vsb *vsb; vcc_type_t type; (void)t; @@ -176,13 +177,12 @@ vcc_act_set(struct vcc *tl, struct token *t, struct symbol *sym) if (ap->type == VOID) SkipToken(tl, ap->oper); - Fb(tl, 1, "%s\n", sym->lname); - tl->indent += INDENT; - vcc_assign_expr(tl, sym, ap); - vcc_Expr(tl, type); - ERRCHK(tl); - tl->indent -= INDENT; - Fb(tl, 1, ");\n"); + vsb = VSB_new_auto(); + AN(vsb); + vcc_assign_edit(vsb, sym, ap); + AZ(VSB_finish(vsb)); + vcc_ExprEdit(tl, type, VSB_data(vsb), NULL); + VSB_destroy(&vsb); SkipToken(tl, ';'); } From 5863a2851623f4ee093bcc03384e88a9dea86b9e Mon Sep 17 00:00:00 2001 From: Dridi Boukelmoune Date: Tue, 4 Jan 2022 08:40:18 +0100 Subject: [PATCH 4/8] vcc: Retire the noindent flag for the BODY type Now that assignments parse expressions with their own edit string, there is no longer a problematic boundary in the middle of an LBODY_* enum. --- lib/libvcc/vcc_compile.h | 1 - lib/libvcc/vcc_expr.c | 6 ++---- lib/libvcc/vcc_types.c | 1 - 3 files changed, 2 insertions(+), 6 deletions(-) diff --git a/lib/libvcc/vcc_compile.h b/lib/libvcc/vcc_compile.h index 904fe3904b6..0d1c4047779 100644 --- a/lib/libvcc/vcc_compile.h +++ b/lib/libvcc/vcc_compile.h @@ -120,7 +120,6 @@ struct type { vcc_type_t multype; int stringform; int bodyform; - int noindent; }; #define VCC_TYPE(UC, lc) extern const struct type UC[1]; diff --git a/lib/libvcc/vcc_expr.c b/lib/libvcc/vcc_expr.c index c51ee4c242f..b7b79390c73 100644 --- a/lib/libvcc/vcc_expr.c +++ b/lib/libvcc/vcc_expr.c @@ -252,10 +252,8 @@ vcc_expr_fmt(struct vsb *d, int ind, const struct expr *e1) char *p; int i; - if (!e1->fmt->noindent) { - for (i = 0; i < ind; i++) - VSB_putc(d, ' '); - } + for (i = 0; i < ind; i++) + VSB_putc(d, ' '); p = VSB_data(e1->vsb); while (*p != '\0') { if (*p == '\n') { diff --git a/lib/libvcc/vcc_types.c b/lib/libvcc/vcc_types.c index cfa6714b39f..da7bbc57883 100644 --- a/lib/libvcc/vcc_types.c +++ b/lib/libvcc/vcc_types.c @@ -85,7 +85,6 @@ const struct type BLOB[1] = {{ const struct type BODY[1] = {{ .magic = TYPE_MAGIC, .name = "BODY", - .noindent = 1, }}; const struct type BOOL[1] = {{ From 26a624703e18f09f5a5c5a4c55753db7a258fc2e Mon Sep 17 00:00:00 2001 From: Dridi Boukelmoune Date: Tue, 4 Jan 2022 08:45:21 +0100 Subject: [PATCH 5/8] vcc: Symbols can edit const assignment expressions --- lib/libvcc/vcc_action.c | 7 ++++++- lib/libvcc/vcc_compile.h | 1 + 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/lib/libvcc/vcc_action.c b/lib/libvcc/vcc_action.c index 25e0891e6d7..cdf8e550a03 100644 --- a/lib/libvcc/vcc_action.c +++ b/lib/libvcc/vcc_action.c @@ -151,6 +151,7 @@ vcc_assign_edit(struct vsb *vsb, struct symbol *sym, const struct assign *ap) static void v_matchproto_(sym_act_f) vcc_act_set(struct vcc *tl, struct token *t, struct symbol *sym) { + const char *const_assign = NULL; const struct assign *ap; struct vsb *vsb; vcc_type_t type; @@ -181,7 +182,11 @@ vcc_act_set(struct vcc *tl, struct token *t, struct symbol *sym) AN(vsb); vcc_assign_edit(vsb, sym, ap); AZ(VSB_finish(vsb)); - vcc_ExprEdit(tl, type, VSB_data(vsb), NULL); + + if (ap->oper == '=') + const_assign = sym->const_assign; + + vcc_ExprEdit(tl, type, VSB_data(vsb), const_assign); VSB_destroy(&vsb); SkipToken(tl, ';'); } diff --git a/lib/libvcc/vcc_compile.h b/lib/libvcc/vcc_compile.h index 0d1c4047779..2d143639f64 100644 --- a/lib/libvcc/vcc_compile.h +++ b/lib/libvcc/vcc_compile.h @@ -200,6 +200,7 @@ struct symbol { unsigned w_methods; const char *uname; unsigned u_methods; + const char *const_assign; }; VTAILQ_HEAD(tokenhead, token); From 31ba1a7254aa89f2f73d0e70f74c4f0cd253a26f Mon Sep 17 00:00:00 2001 From: Dridi Boukelmoune Date: Tue, 4 Jan 2022 09:15:13 +0100 Subject: [PATCH 6/8] vrt: VRT_SetHeader() wrapper for http_SetHeader() --- bin/varnishd/cache/cache_vrt.c | 26 ++++++++++++++++++++++++++ include/vrt.h | 2 ++ 2 files changed, 28 insertions(+) diff --git a/bin/varnishd/cache/cache_vrt.c b/bin/varnishd/cache/cache_vrt.c index bd856193c09..a4f81d0a795 100644 --- a/bin/varnishd/cache/cache_vrt.c +++ b/bin/varnishd/cache/cache_vrt.c @@ -634,6 +634,32 @@ VRT_SetHdr(VRT_CTX, VCL_HEADER hs, const char *pfx, VCL_STRANDS s) http_SetHeader(hp, b); } +VCL_VOID +VRT_SetHeader(VRT_CTX, VCL_HEADER hs, const char *hdr) +{ + VCL_HTTP hp; + unsigned wl, hl; + + CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC); + AN(hs); + AN(hs->what); + AN(hdr); + + hl = strlen(hdr); + wl = hs->what[0]; + assert(hl > wl); + AZ(strncasecmp(hs->what + 1, hdr, wl)); + hp = VRT_selecthttp(ctx, hs->where); + CHECK_OBJ_NOTNULL(hp, HTTP_MAGIC); + + if (FEATURE(FEATURE_VALIDATE_HEADERS) && !validhdr(hdr + wl)) { + VRT_fail(ctx, "Bad static header %s", hdr); + return; + } + http_Unset(hp, hs->what); + http_SetHeader(hp, hdr); +} + /*--------------------------------------------------------------------*/ VCL_VOID diff --git a/include/vrt.h b/include/vrt.h index 25fbabfb693..e00fa14bfa9 100644 --- a/include/vrt.h +++ b/include/vrt.h @@ -65,6 +65,7 @@ * BODY can either be a BLOB or a STRANDS, but only a STRANDS * can take a non-NULL const char * prefix. The changes to BODY * assignments doesn't break the ABI or the API. + * VRT_SetHeader() added * * 14.0 (2021-09-15) * VIN_n_Arg() no directly returns the directory name. @@ -646,6 +647,7 @@ VCL_VOID VRT_hit_for_pass(VRT_CTX, VCL_DURATION); VCL_BOOL VRT_ValidHdr(VRT_CTX, VCL_STRANDS); VCL_VOID VRT_UnsetHdr(VRT_CTX, VCL_HEADER); VCL_VOID VRT_SetHdr(VRT_CTX, VCL_HEADER, const char *pfx, VCL_STRANDS); +VCL_VOID VRT_SetHeader(VRT_CTX, VCL_HEADER, const char *); VCL_VOID VRT_handling(VRT_CTX, unsigned hand); unsigned VRT_handled(VRT_CTX); VCL_VOID VRT_fail(VRT_CTX, const char *fmt, ...) v_printflike_(2,3); From 40832a909351b09794860e2bd90ca2bc2ed34311 Mon Sep 17 00:00:00 2001 From: Dridi Boukelmoune Date: Tue, 4 Jan 2022 10:14:38 +0100 Subject: [PATCH 7/8] vcc: Teach HEADER symbols to accept constant strings Static strings are unfortunately copied into workspace simply because they don't belong to it. Since it is safe to reference a static string, or at the very least a string that is guaranteed to outlive a VCL transaction since in this case it has the same scope as the VCL shared object, we can bypass the regular STRANDS assignment. When a transaction runs out of workspace, fails and goes to vcl_synth, we don't want to fail again because of a static VCL assignment of the content-type and retry-after headers from the built-in VCL. Refs #3765 --- bin/varnishtest/tests/b00040.vtc | 2 +- lib/libvcc/vcc_var.c | 5 +++++ 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/bin/varnishtest/tests/b00040.vtc b/bin/varnishtest/tests/b00040.vtc index 7edce650947..f73bc891111 100644 --- a/bin/varnishtest/tests/b00040.vtc +++ b/bin/varnishtest/tests/b00040.vtc @@ -35,7 +35,7 @@ logexpect l1 -v v1 -g raw { expect * 1012 BogoHeader {Header has ctrl char 0x0d} expect * 1014 BogoHeader {Header has ctrl char 0x0d} expect * 1016 BogoHeader {Missing header name:.*} - expect * 1018 VCL_Error {Bad header foo:} + expect * 1018 VCL_Error {Bad static header foo:} } -start client c1 { diff --git a/lib/libvcc/vcc_var.c b/lib/libvcc/vcc_var.c index 286949e49e8..2f39643d265 100644 --- a/lib/libvcc/vcc_var.c +++ b/lib/libvcc/vcc_var.c @@ -96,5 +96,10 @@ vcc_Var_Wildcard(struct vcc *tl, struct symbol *parent, struct symbol *sym) VSB_printf(vsb, "VRT_UnsetHdr(ctx, %s)", sym->rname); AZ(VSB_finish(vsb)); sym->uname = TlDup(tl, VSB_data(vsb)); + VSB_clear(vsb); + VSB_printf(vsb, "VRT_SetHeader(ctx, %s, \"%s: \" \v1);", + sym->rname, sym->name); + AZ(VSB_finish(vsb)); + sym->const_assign = TlDup(tl, VSB_data(vsb)); VSB_destroy(&vsb); } From cef6f007b722a7358d2ba083343cc9b956bf4f1e Mon Sep 17 00:00:00 2001 From: Dridi Boukelmoune Date: Wed, 5 Jan 2022 18:25:09 +0100 Subject: [PATCH 8/8] vcc: Generate ban() actions with an edit string And get rid of a trailing space in the generated C code. --- lib/libvcc/vcc_action.c | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/lib/libvcc/vcc_action.c b/lib/libvcc/vcc_action.c index cdf8e550a03..5072f363a18 100644 --- a/lib/libvcc/vcc_action.c +++ b/lib/libvcc/vcc_action.c @@ -223,14 +223,9 @@ vcc_act_ban(struct vcc *tl, struct token *t, struct symbol *sym) (void)sym; SkipToken(tl, '('); - - Fb(tl, 1, "(void) VRT_ban_string(ctx, \n"); - tl->indent += INDENT; - vcc_Expr(tl, STRING); - tl->indent -= INDENT; + vcc_ExprEdit(tl, STRING, + "(void) VRT_ban_string(ctx,\v+\n\v1\v-);\n", NULL); ERRCHK(tl); - Fb(tl, 1, ");\n"); - SkipToken(tl, ')'); SkipToken(tl, ';'); }