diff --git a/api/next/76821.txt b/api/next/76821.txt new file mode 100644 index 00000000000000..50177b9c0a2d11 --- /dev/null +++ b/api/next/76821.txt @@ -0,0 +1,9 @@ +pkg math/big, method (*Int) Divide(*Int, *Int, *Int, RoundingMode) (*Int, *Int) #76821 +pkg math/big, const Trunc = 2 #76821 +pkg math/big, const Trunc RoundingMode #76821 +pkg math/big, const Floor = 4 #76821 +pkg math/big, const Floor RoundingMode #76821 +pkg math/big, const Round = 0 #76821 +pkg math/big, const Round RoundingMode #76821 +pkg math/big, const Ceil = 5 #76821 +pkg math/big, const Ceil RoundingMode #76821 diff --git a/api/next/79042.txt b/api/next/79042.txt new file mode 100644 index 00000000000000..068b01dfcd4113 --- /dev/null +++ b/api/next/79042.txt @@ -0,0 +1 @@ +pkg go/constant, func StringLen(Value) int64 #79042 diff --git a/doc/next/6-stdlib/99-minor/go/constant/79042.md b/doc/next/6-stdlib/99-minor/go/constant/79042.md new file mode 100644 index 00000000000000..1ff423bf716d8d --- /dev/null +++ b/doc/next/6-stdlib/99-minor/go/constant/79042.md @@ -0,0 +1 @@ +The new [StringLen] function returns the length of a string [Value]. For an [Unknown] value, the length is 0. diff --git a/doc/next/6-stdlib/99-minor/math/big/76821.md b/doc/next/6-stdlib/99-minor/math/big/76821.md new file mode 100644 index 00000000000000..7ae8b932bf90cc --- /dev/null +++ b/doc/next/6-stdlib/99-minor/math/big/76821.md @@ -0,0 +1,3 @@ + +[Int] now has method [Int.Divide] to compute quotient and remainder of two [Int] values. +It supports rounding modes [Trunc], [Floor], [Round] and [Ceil]. diff --git a/src/cmd/compile/internal/importer/genmeth_test.go b/src/cmd/compile/internal/importer/genmeth_test.go index ffcc099cc49f0b..fa827a7adf204c 100644 --- a/src/cmd/compile/internal/importer/genmeth_test.go +++ b/src/cmd/compile/internal/importer/genmeth_test.go @@ -2,6 +2,8 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. +//go:build goexperiment.genericmethods + package importer import ( diff --git a/src/cmd/compile/internal/noder/unified.go b/src/cmd/compile/internal/noder/unified.go index c942819a802878..ca601ef9cc1a55 100644 --- a/src/cmd/compile/internal/noder/unified.go +++ b/src/cmd/compile/internal/noder/unified.go @@ -26,12 +26,12 @@ import ( ) // uirVersion is the unified IR version to use for encoding/decoding. -// Use V4 for generic methods. Revert to V3 if the GOEXPERIMENT is enabled. +// Use V4 for generic methods if the GOEXPERIMENT is enabled. var uirVersion = func() pkgbits.Version { if buildcfg.Experiment.GenericMethods { - return pkgbits.V3 + return pkgbits.V4 } - return pkgbits.V4 + return pkgbits.V3 }() // localPkgReader holds the package reader used for reading the local diff --git a/src/cmd/compile/internal/types2/builtins.go b/src/cmd/compile/internal/types2/builtins.go index 4968c6a6305d54..2f44d3ea3b10b9 100644 --- a/src/cmd/compile/internal/types2/builtins.go +++ b/src/cmd/compile/internal/types2/builtins.go @@ -153,7 +153,7 @@ func (check *Checker) builtin(x *operand, call *syntax.CallExpr, id builtinId) ( if isString(t) && id == _Len { if x.mode() == constant_ { mode = constant_ - val = constant.MakeInt64(int64(len(constant.StringVal(x.val)))) + val = constant.MakeInt64(constant.StringLen(x.val)) } else { mode = value } diff --git a/src/cmd/compile/internal/types2/const.go b/src/cmd/compile/internal/types2/const.go index 374cb89d2ced5f..dc459798c8b0c1 100644 --- a/src/cmd/compile/internal/types2/const.go +++ b/src/cmd/compile/internal/types2/const.go @@ -49,16 +49,15 @@ func (check *Checker) overflow(x *operand, opPos syntax.Pos) { return } - const maxLen = int(2e9) // cmd/internal/obj.MaxSymSize - // Disable the length check for now, as calling constant.StringVal - // eagerly constructs the string and can lead to significant memory - // usage increase. We may want a StringLen function. - // TODO(go.dev/issue/78346): reenable the check. - if false && x.val.Kind() == constant.String && len(constant.StringVal(x.val)) > maxLen { - check.errorf(atPos(opPos), InvalidConstVal, "constant string too long (%d bytes > %d bytes)", - len(constant.StringVal(x.val)), maxLen) - x.val = constant.MakeUnknown() - return + // String values must not become arbitrarily long (go.dev/issue/78346). + const maxLen = int64(2e9) // cmd/internal/obj.MaxSymSize + if x.val.Kind() == constant.String { + len := constant.StringLen(x.val) + if len > maxLen { + check.errorf(atPos(opPos), InvalidConstVal, "constant string too long (%d bytes > %d bytes)", len, maxLen) + x.val = constant.MakeUnknown() + return + } } } diff --git a/src/cmd/compile/internal/types2/index.go b/src/cmd/compile/internal/types2/index.go index fd92bef2cb1206..0ea6372b1df36b 100644 --- a/src/cmd/compile/internal/types2/index.go +++ b/src/cmd/compile/internal/types2/index.go @@ -77,7 +77,7 @@ func (check *Checker) indexExpr(x *operand, e *syntax.IndexExpr) (isFuncInst boo if isString(typ) { valid = true if x.mode() == constant_ { - length = int64(len(constant.StringVal(x.val))) + length = constant.StringLen(x.val) } // an indexed string always yields a byte value // (not a constant) even if the string and the @@ -302,7 +302,7 @@ func (check *Checker) sliceExpr(x *operand, e *syntax.SliceExpr) { } valid = true if x.mode() == constant_ { - length = int64(len(constant.StringVal(x.val))) + length = constant.StringLen(x.val) } // spec: "For untyped string operands the result // is a non-constant value of type string." diff --git a/src/cmd/go.mod b/src/cmd/go.mod index 930247f4c895b7..938fdb2955f766 100644 --- a/src/cmd/go.mod +++ b/src/cmd/go.mod @@ -6,7 +6,7 @@ require ( github.com/google/pprof v0.0.0-20260115054156-294ebfa9ad83 golang.org/x/arch v0.23.1-0.20260109160903-657d90bd6695 golang.org/x/build v0.0.0-20260122183339-3ba88df37c64 - golang.org/x/mod v0.35.0 + golang.org/x/mod v0.36.1-0.20260513122029-343ee60345a1 golang.org/x/sync v0.20.0 golang.org/x/sys v0.44.0 golang.org/x/telemetry v0.0.0-20260409153401-be6f6cb8b1fa diff --git a/src/cmd/go.sum b/src/cmd/go.sum index 89e5f334cfffbb..d78128ce1beace 100644 --- a/src/cmd/go.sum +++ b/src/cmd/go.sum @@ -10,8 +10,8 @@ golang.org/x/arch v0.23.1-0.20260109160903-657d90bd6695 h1:q45HsUyFzBjBk4mHGgUew golang.org/x/arch v0.23.1-0.20260109160903-657d90bd6695/go.mod h1:dNHoOeKiyja7GTvF9NJS1l3Z2yntpQNzgrjh1cU103A= golang.org/x/build v0.0.0-20260122183339-3ba88df37c64 h1:BNhBATNmH/VtzGolB+ksQPPvn6ZyffiR8TmKenqNo+A= golang.org/x/build v0.0.0-20260122183339-3ba88df37c64/go.mod h1:3QmSbNil8ZWqC94m80Glej1v8b92gYzPIQPTtSa0c+4= -golang.org/x/mod v0.35.0 h1:Ww1D637e6Pg+Zb2KrWfHQUnH2dQRLBQyAtpr/haaJeM= -golang.org/x/mod v0.35.0/go.mod h1:+GwiRhIInF8wPm+4AoT6L0FA1QWAad3OMdTRx4tFYlU= +golang.org/x/mod v0.36.1-0.20260513122029-343ee60345a1 h1:C0TwvxhsI0bHc1TbK4QEa5PCMrHiST7y/lpX4MVW3KM= +golang.org/x/mod v0.36.1-0.20260513122029-343ee60345a1/go.mod h1:moc6ELqsWcOw5Ef3xVprK5ul/MvtVvkIXLziUOICjUQ= golang.org/x/sync v0.20.0 h1:e0PTpb7pjO8GAtTs2dQ6jYa5BWYlMuX047Dco/pItO4= golang.org/x/sync v0.20.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0= golang.org/x/sys v0.44.0 h1:ildZl3J4uzeKP07r2F++Op7E9B29JRUy+a27EibtBTQ= diff --git a/src/cmd/go/internal/modload/init.go b/src/cmd/go/internal/modload/init.go index 2fbbdfa6d23613..f1130fca1aa6e6 100644 --- a/src/cmd/go/internal/modload/init.go +++ b/src/cmd/go/internal/modload/init.go @@ -1989,8 +1989,10 @@ func UpdateGoModFromReqs(ld *Loader, ctx context.Context, opts WriteOpts) (befor // Update require blocks. if gover.Compare(goVersion, gover.SeparateIndirectVersion) < 0 { modFile.SetRequire(list) - } else { + } else if gover.Compare(goVersion, gover.SimplifyRequireVersion) < 0 { modFile.SetRequireSeparateIndirect(list) + } else { + modFile.SetRequireAtMostTwo(list) } modFile.Cleanup() after, err = modFile.Format() diff --git a/src/cmd/go/testdata/script/mod_simplify_require.txt b/src/cmd/go/testdata/script/mod_simplify_require.txt new file mode 100644 index 00000000000000..fe1de470cf1597 --- /dev/null +++ b/src/cmd/go/testdata/script/mod_simplify_require.txt @@ -0,0 +1,155 @@ +# If go.mod has go 1.27 or higher, multiple require blocks should be +# consolidated. Even those with comments. + +cp go.mod.127 go.mod +go mod tidy +cmp go.mod go.mod.127tidy + +# If go.mod has go 1.26, blocks with comments should be preserved. +cp go.mod.126 go.mod +go mod tidy +cmp go.mod go.mod.126tidy + +-- go.mod.127 -- +module example.com/m + +go 1.27 + +require example.com/a v1.0.0 + +// Block comment +require ( + example.com/b v1.0.0 +) + +// Another block comment +require ( + example.com/d v1.0.0 // an inline comment + example.com/c v1.0.0 // indirect +) + +// A third block comment +require ( + example.com/e v1.0.0 // indirect +) + +replace ( + example.com/a v1.0.0 => ./a + example.com/b v1.0.0 => ./b + example.com/c v1.0.0 => ./c + example.com/d v1.0.0 => ./d + example.com/e v1.0.0 => ./e +) +-- go.mod.127tidy -- +module example.com/m + +go 1.27 + +// Block comment +// +// Another block comment +require ( + example.com/a v1.0.0 + example.com/b v1.0.0 + example.com/d v1.0.0 // an inline comment +) + +// A third block comment +require ( + example.com/c v1.0.0 // indirect + example.com/e v1.0.0 // indirect +) + +replace ( + example.com/a v1.0.0 => ./a + example.com/b v1.0.0 => ./b + example.com/c v1.0.0 => ./c + example.com/d v1.0.0 => ./d + example.com/e v1.0.0 => ./e +) +-- go.mod.126 -- +module example.com/m + +go 1.26 + +require example.com/a v1.0.0 + +// Block comment +require ( + example.com/b v1.0.0 + example.com/d v1.0.0 +) + +require example.com/c v1.0.0 // indirect + +replace ( + example.com/a v1.0.0 => ./a + example.com/b v1.0.0 => ./b + example.com/c v1.0.0 => ./c + example.com/d v1.0.0 => ./d + example.com/e v1.0.0 => ./e +) +-- go.mod.126tidy -- +module example.com/m + +go 1.26 + +require example.com/a v1.0.0 + +// Block comment +require ( + example.com/b v1.0.0 + example.com/d v1.0.0 +) + +require ( + example.com/c v1.0.0 // indirect + example.com/e v1.0.0 // indirect +) + +replace ( + example.com/a v1.0.0 => ./a + example.com/b v1.0.0 => ./b + example.com/c v1.0.0 => ./c + example.com/d v1.0.0 => ./d + example.com/e v1.0.0 => ./e +) +-- m.go -- +package m +import _ "example.com/a" +import _ "example.com/b" +import _ "example.com/d" + +-- a/go.mod -- +module example.com/a +go 1.26 +require example.com/c v1.0.0 +require example.com/e v1.0.0 +-- a/a.go -- +package a +import _ "example.com/c" +import _ "example.com/e" + +-- b/go.mod -- +module example.com/b +go 1.26 +-- b/b.go -- +package b + +-- c/go.mod -- +module example.com/c +go 1.26 +-- c/c.go -- +package c + +-- d/go.mod -- +module example.com/d +go 1.26 +-- d/d.go -- +package d + +-- e/go.mod -- +module example.com/e +go 1.26 +-- e/e.go -- +package e diff --git a/src/cmd/vendor/golang.org/x/mod/modfile/rule.go b/src/cmd/vendor/golang.org/x/mod/modfile/rule.go index c5b8305de725db..65eb72b48ca697 100644 --- a/src/cmd/vendor/golang.org/x/mod/modfile/rule.go +++ b/src/cmd/vendor/golang.org/x/mod/modfile/rule.go @@ -327,6 +327,7 @@ func parseToFile(file string, data []byte, fix VersionFixer, strict bool) (parse } var GoVersionRE = lazyregexp.New(`^([1-9][0-9]*)\.(0|[1-9][0-9]*)(\.(0|[1-9][0-9]*))?([a-z]+[0-9]+)?$`) + var laxGoVersionRE = lazyregexp.New(`^v?(([1-9][0-9]*)\.(0|[1-9][0-9]*))([^0-9].*)$`) // Toolchains must be named beginning with `go1`, @@ -1272,6 +1273,17 @@ func (f *File) SetRequire(req []*Require) { // SetRequireSeparateIndirect will split it into a direct-only and indirect-only // block. This aids in the transition to separate blocks. func (f *File) SetRequireSeparateIndirect(req []*Require) { + f.setRequireSeparateIndirect(req, false) +} + +// SetRequireAtMostTwo is like SetRequireSeparateIndirect but it aggressively +// consolidates all requirements into at most two blocks (one direct, one indirect). +// It ignores existing blocks and comments when deciding where to place requirements. +func (f *File) SetRequireAtMostTwo(req []*Require) { + f.setRequireSeparateIndirect(req, true) +} + +func (f *File) setRequireSeparateIndirect(req []*Require, simplify bool) { // hasComments returns whether a line or block has comments // other than "indirect". hasComments := func(c Comments) bool { @@ -1304,6 +1316,17 @@ func (f *File) SetRequireSeparateIndirect(req []*Require) { } // Examine existing require lines and blocks. + need := make(map[string]*Require) + for _, r := range req { + need[r.Mod.Path] = r + } + lineIndirect := make(map[*Line]bool) + for _, r := range f.Require { + if n := need[r.Mod.Path]; n != nil { + lineIndirect[r.Syntax] = n.Indirect + } + } + var ( // We may insert new requirements into the last uncommented // direct-only and indirect-only blocks. We may also move requirements @@ -1321,7 +1344,9 @@ func (f *File) SetRequireSeparateIndirect(req []*Require) { // Track the block each requirement belongs to (if any) so we can // move them later. - lineToBlock = make(map[*Line]*LineBlock) + lineToBlock = make(map[*Line]*LineBlock) + directBlockComments []Comment + indirectBlockComments []Comment ) for i, stmt := range f.Syntax.Stmt { switch stmt := stmt.(type) { @@ -1364,6 +1389,24 @@ func (f *File) SetRequireSeparateIndirect(req []*Require) { if allIndirect { lastIndirectIndex = i } + if simplify { + anyDirect := false + for _, line := range stmt.Line { + if ind, ok := lineIndirect[line]; ok && !ind { + anyDirect = true + break + } + } + target := &directBlockComments + if !anyDirect && len(stmt.Line) > 0 { + target = &indirectBlockComments + } + if len(*target) > 0 && len(stmt.Comments.Before) > 0 { + *target = append(*target, Comment{Token: "//"}) + } + *target = append(*target, stmt.Comments.Before...) + stmt.Comments.Before = nil + } } } @@ -1422,6 +1465,15 @@ func (f *File) SetRequireSeparateIndirect(req []*Require) { lastIndirectBlock = ensureBlock(lastIndirectIndex) } + if simplify { + if len(directBlockComments) > 0 { + lastDirectBlock.Comments.Before = append(lastDirectBlock.Comments.Before, directBlockComments...) + } + if len(indirectBlockComments) > 0 { + lastIndirectBlock.Comments.Before = append(lastIndirectBlock.Comments.Before, indirectBlockComments...) + } + } + // Delete requirements we don't want anymore. // Update versions and indirect comments on requirements we want to keep. // If a requirement is in last{Direct,Indirect}Block with the wrong @@ -1430,10 +1482,6 @@ func (f *File) SetRequireSeparateIndirect(req []*Require) { // correct block. // // Some blocks may be empty after this. Cleanup will remove them. - need := make(map[string]*Require) - for _, r := range req { - need[r.Mod.Path] = r - } have := make(map[string]*Require) for _, r := range f.Require { path := r.Mod.Path @@ -1446,10 +1494,10 @@ func (f *File) SetRequireSeparateIndirect(req []*Require) { r.setVersion(need[path].Mod.Version) r.setIndirect(need[path].Indirect) if need[path].Indirect && - (oneFlatUncommentedBlock || lineToBlock[r.Syntax] == lastDirectBlock) { + (simplify || oneFlatUncommentedBlock || lineToBlock[r.Syntax] == lastDirectBlock) { moveReq(r, lastIndirectBlock) } else if !need[path].Indirect && - (oneFlatUncommentedBlock || lineToBlock[r.Syntax] == lastIndirectBlock) { + (simplify || oneFlatUncommentedBlock || lineToBlock[r.Syntax] == lastIndirectBlock) { moveReq(r, lastDirectBlock) } } diff --git a/src/cmd/vendor/modules.txt b/src/cmd/vendor/modules.txt index 6ba72c229f6240..68ae04853cc802 100644 --- a/src/cmd/vendor/modules.txt +++ b/src/cmd/vendor/modules.txt @@ -28,7 +28,7 @@ golang.org/x/arch/x86/x86asm # golang.org/x/build v0.0.0-20260122183339-3ba88df37c64 ## explicit; go 1.24.9 golang.org/x/build/relnote -# golang.org/x/mod v0.35.0 +# golang.org/x/mod v0.36.1-0.20260513122029-343ee60345a1 ## explicit; go 1.25.0 golang.org/x/mod/internal/lazyregexp golang.org/x/mod/modfile diff --git a/src/encoding/json/internal/jsontest/testdata/canada_geometry.json.zst b/src/encoding/json/internal/jsontest/_embed/canada_geometry.json.zst similarity index 100% rename from src/encoding/json/internal/jsontest/testdata/canada_geometry.json.zst rename to src/encoding/json/internal/jsontest/_embed/canada_geometry.json.zst diff --git a/src/encoding/json/internal/jsontest/testdata/citm_catalog.json.zst b/src/encoding/json/internal/jsontest/_embed/citm_catalog.json.zst similarity index 100% rename from src/encoding/json/internal/jsontest/testdata/citm_catalog.json.zst rename to src/encoding/json/internal/jsontest/_embed/citm_catalog.json.zst diff --git a/src/encoding/json/internal/jsontest/testdata/golang_source.json.zst b/src/encoding/json/internal/jsontest/_embed/golang_source.json.zst similarity index 100% rename from src/encoding/json/internal/jsontest/testdata/golang_source.json.zst rename to src/encoding/json/internal/jsontest/_embed/golang_source.json.zst diff --git a/src/encoding/json/internal/jsontest/testdata/string_escaped.json.zst b/src/encoding/json/internal/jsontest/_embed/string_escaped.json.zst similarity index 100% rename from src/encoding/json/internal/jsontest/testdata/string_escaped.json.zst rename to src/encoding/json/internal/jsontest/_embed/string_escaped.json.zst diff --git a/src/encoding/json/internal/jsontest/testdata/string_unicode.json.zst b/src/encoding/json/internal/jsontest/_embed/string_unicode.json.zst similarity index 100% rename from src/encoding/json/internal/jsontest/testdata/string_unicode.json.zst rename to src/encoding/json/internal/jsontest/_embed/string_unicode.json.zst diff --git a/src/encoding/json/internal/jsontest/testdata/synthea_fhir.json.zst b/src/encoding/json/internal/jsontest/_embed/synthea_fhir.json.zst similarity index 100% rename from src/encoding/json/internal/jsontest/testdata/synthea_fhir.json.zst rename to src/encoding/json/internal/jsontest/_embed/synthea_fhir.json.zst diff --git a/src/encoding/json/internal/jsontest/testdata/twitter_status.json.zst b/src/encoding/json/internal/jsontest/_embed/twitter_status.json.zst similarity index 100% rename from src/encoding/json/internal/jsontest/testdata/twitter_status.json.zst rename to src/encoding/json/internal/jsontest/_embed/twitter_status.json.zst diff --git a/src/encoding/json/internal/jsontest/testdata.go b/src/encoding/json/internal/jsontest/testdata.go index 74de366136ce97..39638905f230c3 100644 --- a/src/encoding/json/internal/jsontest/testdata.go +++ b/src/encoding/json/internal/jsontest/testdata.go @@ -25,7 +25,13 @@ import ( // by other packages such that the location of testdata may change relative // to the working directory of the test itself. // -//go:embed testdata/*.json.zst +// Various tools assume that testdata directories are unnecessary if you don't +// need to run tests. This includes cmd/internal/bootstrap_test, which runs go +// install std on a GOROOT excluding testdata directories. Since this is an +// importable package rather than a test, to avoid breaking that case we must +// not actually name the directory testdata. +// +//go:embed _embed/*.json.zst var testdataFS embed.FS type Entry struct { @@ -43,7 +49,7 @@ func mustGet[T any](v T, err error) T { // Data is a list of JSON testdata. var Data = func() (entries []Entry) { - fis := mustGet(fs.ReadDir(testdataFS, "testdata")) + fis := mustGet(fs.ReadDir(testdataFS, "_embed")) slices.SortFunc(fis, func(x, y fs.DirEntry) int { return strings.Compare(x.Name(), y.Name()) }) for _, fi := range fis { var entry Entry @@ -57,7 +63,7 @@ var Data = func() (entries []Entry) { // Lazily read and decompress the test data. entry.Data = sync.OnceValue(func() []byte { - filePath := path.Join("testdata", fi.Name()) + filePath := path.Join("_embed", fi.Name()) b := mustGet(fs.ReadFile(testdataFS, filePath)) zr := zstd.NewReader(bytes.NewReader(b)) return mustGet(io.ReadAll(zr)) diff --git a/src/go/constant/value.go b/src/go/constant/value.go index fb9a0523ced63d..eb6d4cf3dfeaea 100644 --- a/src/go/constant/value.go +++ b/src/go/constant/value.go @@ -137,15 +137,13 @@ func (x *stringVal) String() string { // concatenation. See golang.org/issue/23348. func (x *stringVal) string() string { x.mu.Lock() + defer x.mu.Unlock() if x.l != nil { x.s = strings.Join(reverse(x.appendReverse(nil)), "") x.l = nil x.r = nil } - s := x.s - x.mu.Unlock() - - return s + return x.s } // reverse reverses x in place and returns it. @@ -609,6 +607,30 @@ func Val(x Value) any { } } +// StringLen returns the length of x if x is a [String]. +// If x is [Unknown], the result is 0. +// In all other cases, the function panics. +func StringLen(x Value) int64 { + switch x := x.(type) { + case *stringVal: + return x.len() + case unknownVal: + return 0 + default: + panic(fmt.Sprintf("%v not a String", x)) + } +} + +// len computes and returns the length of x without constructing the entire string. +func (x *stringVal) len() int64 { + x.mu.Lock() + defer x.mu.Unlock() + if x.l != nil { + return x.l.len() + x.r.len() + } + return int64(len(x.s)) +} + // Make returns the [Value] for x. // // type of x result Kind diff --git a/src/go/constant/value_test.go b/src/go/constant/value_test.go index 0f50281ee27988..3a43a1d144103b 100644 --- a/src/go/constant/value_test.go +++ b/src/go/constant/value_test.go @@ -437,6 +437,27 @@ func TestString(t *testing.T) { } } +func TestStringLen(t *testing.T) { + tests := []struct { + x Value + want int64 + }{ + {MakeUnknown(), 0}, + {val(`""`), 0}, + {val(`"foo"`), 3}, + {val(`"世界"`), 6}, + {BinaryOp(val(`"foo"`), token.ADD, val(`"bar"`)), 6}, + {BinaryOp(val(`"世界"`), token.ADD, val(`"!"`)), 7}, + {BinaryOp(val(`"a"`), token.ADD, BinaryOp(val(`"b"`), token.ADD, val(`"c"`))), 3}, + } + + for _, test := range tests { + if got := StringLen(test.x); got != test.want { + t.Errorf("StringLen(%v): got %d; want %d", test.x, got, test.want) + } + } +} + // ---------------------------------------------------------------------------- // Support functions diff --git a/src/go/internal/gcimporter/genmeth_test.go b/src/go/internal/gcimporter/genmeth_test.go index 07b053d5e574dd..63b8858af3c0d2 100644 --- a/src/go/internal/gcimporter/genmeth_test.go +++ b/src/go/internal/gcimporter/genmeth_test.go @@ -2,6 +2,8 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. +//go:build goexperiment.genericmethods + package gcimporter_test import ( diff --git a/src/go/types/builtins.go b/src/go/types/builtins.go index ce70311cc16b22..e751161095dd0c 100644 --- a/src/go/types/builtins.go +++ b/src/go/types/builtins.go @@ -156,7 +156,7 @@ func (check *Checker) builtin(x *operand, call *ast.CallExpr, id builtinId) (_ b if isString(t) && id == _Len { if x.mode() == constant_ { mode = constant_ - val = constant.MakeInt64(int64(len(constant.StringVal(x.val)))) + val = constant.MakeInt64(constant.StringLen(x.val)) } else { mode = value } diff --git a/src/go/types/const.go b/src/go/types/const.go index e4e8e86ecc9930..056fd5bf6f5b1e 100644 --- a/src/go/types/const.go +++ b/src/go/types/const.go @@ -51,16 +51,15 @@ func (check *Checker) overflow(x *operand, opPos token.Pos) { return } - const maxLen = int(2e9) // cmd/internal/obj.MaxSymSize - // Disable the length check for now, as calling constant.StringVal - // eagerly constructs the string and can lead to significant memory - // usage increase. We may want a StringLen function. - // TODO(go.dev/issue/78346): reenable the check. - if false && x.val.Kind() == constant.String && len(constant.StringVal(x.val)) > maxLen { - check.errorf(atPos(opPos), InvalidConstVal, "constant string too long (%d bytes > %d bytes)", - len(constant.StringVal(x.val)), maxLen) - x.val = constant.MakeUnknown() - return + // String values must not become arbitrarily long (go.dev/issue/78346). + const maxLen = int64(2e9) // cmd/internal/obj.MaxSymSize + if x.val.Kind() == constant.String { + len := constant.StringLen(x.val) + if len > maxLen { + check.errorf(atPos(opPos), InvalidConstVal, "constant string too long (%d bytes > %d bytes)", len, maxLen) + x.val = constant.MakeUnknown() + return + } } } diff --git a/src/go/types/index.go b/src/go/types/index.go index 3c8bf4ffe41b46..524e93153a0c2e 100644 --- a/src/go/types/index.go +++ b/src/go/types/index.go @@ -78,7 +78,7 @@ func (check *Checker) indexExpr(x *operand, e *indexedExpr) (isFuncInst bool) { if isString(typ) { valid = true if x.mode() == constant_ { - length = int64(len(constant.StringVal(x.val))) + length = constant.StringLen(x.val) } // an indexed string always yields a byte value // (not a constant) even if the string and the @@ -307,7 +307,7 @@ func (check *Checker) sliceExpr(x *operand, e *ast.SliceExpr) { } valid = true if x.mode() == constant_ { - length = int64(len(constant.StringVal(x.val))) + length = constant.StringLen(x.val) } // spec: "For untyped string operands the result // is a non-constant value of type string." diff --git a/src/internal/buildcfg/exp.go b/src/internal/buildcfg/exp.go index 3c7e2aa7d442d2..f45dc62f2a4882 100644 --- a/src/internal/buildcfg/exp.go +++ b/src/internal/buildcfg/exp.go @@ -85,6 +85,7 @@ func ParseGOEXPERIMENT(goos, goarch, goexp string) (*ExperimentFlags, error) { RandomizedHeapBase64: true, GreenTeaGC: true, JSONv2: true, + GenericMethods: true, } flags := &ExperimentFlags{ Flags: baseline, diff --git a/src/math/big/int.go b/src/math/big/int.go index 3c6f0d40283e54..eb796c6dc3eee1 100644 --- a/src/math/big/int.go +++ b/src/math/big/int.go @@ -1308,3 +1308,85 @@ func (z *Int) Sqrt(x *Int) *Int { z.abs = z.abs.sqrt(nil, x.abs) return z } + +// Rounding modes that determine how the integer quotient is adjusted in an integer division. +// See Daan Leijen, “Division and Modulus for Computer Scientists”, for details. +const ( + Trunc = ToZero // T-division (same as Go division) + Floor = ToNegativeInf // F-division + Round = ToNearestEven // R-division + Ceil = ToPositiveInf // C-division +) + +// Divide computes the integer quotient q and remainder r such that +// +// q = f(x/y) +// r = x - y*q +// +// where f is described by the rounding mode, +// which must be one of [Trunc], [Floor], [Round] or [Ceil]. +// Divide sets z to q if z != nil, updates r if r != nil, +// and returns the pair (z, r) if y != 0. +// If y == 0, a division-by-zero run-time panic occurs. +func (z *Int) Divide(x, y, r *Int, mode RoundingMode) (*Int, *Int) { + var z_abs nat + if z != nil { + z_abs = z.abs + } + var r_neg bool + var r_abs nat + if r != nil { + r_abs = r.abs + } + y_abs := y.abs // save y + if z == y || alias(z_abs, y.abs) { + y_abs = nat(nil).set(y.abs) + } + neg := x.neg != y.neg + z_abs, r_abs = z_abs.div(nil, r_abs, x.abs, y.abs) + if len(r_abs) > 0 { + switch mode { + case Trunc: + r_neg = x.neg + case Floor: + r_neg = y.neg + if neg { + z_abs = z_abs.add(z_abs, natOne) + r_abs = r_abs.sub(y_abs, r_abs) + } + case Ceil: + r_neg = !y.neg + if !neg { + z_abs = z_abs.add(z_abs, natOne) + r_abs = r_abs.sub(y_abs, r_abs) + } + case Round: + switch nat(nil).mul(nil, r_abs, natTwo).cmp(y_abs) { + case -1: + r_neg = x.neg + case 0: + even := len(z_abs) == 0 || z_abs[0]&1 == 0 + if even { + r_neg = x.neg + break + } + fallthrough + case 1: + r_neg = !x.neg + z_abs = z_abs.add(z_abs, natOne) + r_abs = r_abs.sub(y_abs, r_abs) + } + default: + panic("unsupported rounding mode") + } + } + if z != nil { + z.abs = z_abs + z.neg = neg && len(z_abs) > 0 // 0 has no sign + } + if r != nil { + r.abs = r_abs + r.neg = r_neg + } + return z, r +} diff --git a/src/math/big/int_test.go b/src/math/big/int_test.go index f6865d1ed9b980..b593ea625cac89 100644 --- a/src/math/big/int_test.go +++ b/src/math/big/int_test.go @@ -2011,3 +2011,87 @@ func TestFloat64(t *testing.T) { } } } + +func TestIntDivide(t *testing.T) { + x := new(Int) + y := new(Int) + q := new(Int) + r := new(Int) + qExp := new(Int) + rExp := new(Int) + factor, _ := new(Int).SetString("123_456_789_012_345_678_901", 0) + msg := "%v(%v/%v): got q = %v r = %v, want q = %v r = %v" + for i := int64(-10); i <= 10; i++ { + for j := int64(-10); j <= 10; j++ { + if j == 0 { + continue + } + x.SetInt64(i) + y.SetInt64(j) + qExp.SetInt64(i / j) + rExp.SetInt64(i % j) + q, r = q.Divide(x, y, r, Trunc) + if q.Cmp(qExp) != 0 || r.Cmp(rExp) != 0 { + t.Errorf(msg, "trunc", x, y, q, r, qExp, rExp) + } + x.Mul(x, factor) + y.Mul(y, factor) + rExp.Mul(rExp, factor) + q, r = q.Divide(x, y, r, Trunc) + if q.Cmp(qExp) != 0 || r.Cmp(rExp) != 0 { + t.Errorf(msg, "trunc", x, y, q, r, qExp, rExp) + } + + x.SetInt64(i) + y.SetInt64(j) + floor := int64(math.Floor(float64(i) / float64(j))) + qExp.SetInt64(floor) + rExp.SetInt64(i - j*floor) + q, r = q.Divide(x, y, r, Floor) + if q.Cmp(qExp) != 0 || r.Cmp(rExp) != 0 { + t.Errorf(msg, "floor", x, y, q, r, qExp, rExp) + } + x.Mul(x, factor) + y.Mul(y, factor) + rExp.Mul(rExp, factor) + q, r = q.Divide(x, y, r, Floor) + if q.Cmp(qExp) != 0 || r.Cmp(rExp) != 0 { + t.Errorf(msg, "floor", x, y, q, r, qExp, rExp) + } + + x.SetInt64(i) + y.SetInt64(j) + ceil := int64(math.Ceil(float64(i) / float64(j))) + qExp.SetInt64(ceil) + rExp.SetInt64(i - j*ceil) + q, r = q.Divide(x, y, r, Ceil) + if q.Cmp(qExp) != 0 || r.Cmp(rExp) != 0 { + t.Errorf(msg, "ceil", x, y, q, r, qExp, rExp) + } + x.Mul(x, factor) + y.Mul(y, factor) + rExp.Mul(rExp, factor) + q, r = q.Divide(x, y, r, Ceil) + if q.Cmp(qExp) != 0 || r.Cmp(rExp) != 0 { + t.Errorf(msg, "ceil", x, y, q, r, qExp, rExp) + } + + x.SetInt64(i) + y.SetInt64(j) + round := int64(math.RoundToEven(float64(i) / float64(j))) + qExp.SetInt64(round) + rExp.SetInt64(i - j*round) + q, r = q.Divide(x, y, r, Round) + if q.Cmp(qExp) != 0 || r.Cmp(rExp) != 0 { + t.Errorf(msg, "round", x, y, q, r, qExp, rExp) + } + x.Mul(x, factor) + y.Mul(y, factor) + rExp.Mul(rExp, factor) + q, r = q.Divide(x, y, r, Round) + if q.Cmp(qExp) != 0 || r.Cmp(rExp) != 0 { + t.Errorf(msg, "round", x, y, q, r, qExp, rExp) + } + } + } +} diff --git a/src/runtime/_mkmalloc/go.mod b/src/runtime/_mkmalloc/go.mod index 61b1c550521756..f6bc431b1e07ea 100644 --- a/src/runtime/_mkmalloc/go.mod +++ b/src/runtime/_mkmalloc/go.mod @@ -1,5 +1,5 @@ module _mkmalloc -go 1.24 +go 1.26 require golang.org/x/tools v0.33.0 diff --git a/src/runtime/_mkmalloc/mkmalloc.go b/src/runtime/_mkmalloc/mkmalloc.go index aa10ece2340d05..e99383544a6410 100644 --- a/src/runtime/_mkmalloc/mkmalloc.go +++ b/src/runtime/_mkmalloc/mkmalloc.go @@ -166,7 +166,9 @@ func specializedMallocConfig(classes []class, sizeToSizeClass []uint8) generator templateFunc: "mallocStub", name: name, ops: []op{ - {inlineFunc, "inlinedMalloc", "smallScanNoHeaderStub"}, + {inlineFunc, "inlinedMalloc", "smallStub"}, + {inlineFunc, "postMallocgc", "postMallocgc"}, + {foldCondition, "isNoScan_", str(false)}, {inlineFunc, "heapSetTypeNoHeaderStub", "heapSetTypeNoHeaderStub"}, {inlineFunc, "nextFreeFastStub", "nextFreeFastStub"}, {inlineFunc, "writeHeapBitsSmallStub", "writeHeapBitsSmallStub"}, @@ -194,6 +196,8 @@ func specializedMallocConfig(classes []class, sizeToSizeClass []uint8) generator ops: []op{ {inlineFunc, "inlinedMalloc", "tinyStub"}, {inlineFunc, "nextFreeFastTiny", "nextFreeFastTiny"}, + {inlineFunc, "postMallocgc", "postMallocgc"}, + {inlineFunc, "nextFreeFastStub", "nextFreeFastStub"}, {inlineFunc, "deductAssistCredit", "deductAssistCredit"}, {subBasicLit, "elemsize_", str(elemsize)}, {subBasicLit, "sizeclass_", str(tinySizeClass)}, @@ -211,7 +215,9 @@ func specializedMallocConfig(classes []class, sizeToSizeClass []uint8) generator templateFunc: "mallocStub", name: name, ops: []op{ - {inlineFunc, "inlinedMalloc", "smallNoScanStub"}, + {inlineFunc, "inlinedMalloc", "smallStub"}, + {inlineFunc, "postMallocgc", "postMallocgc"}, + {foldCondition, "isNoScan_", str(true)}, {inlineFunc, "nextFreeFastStub", "nextFreeFastStub"}, {subBasicLit, "elemsize_", str(elemsize)}, {subBasicLit, "sizeclass_", str(sc)}, @@ -305,12 +311,15 @@ func substituteWithBasicLit(node ast.Node, from, to string) ast.Node { if err != nil { log.Fatalf("parsing expr %q: %v", to, err) } - if _, ok := toExpr.(*ast.BasicLit); !ok { + toLit, ok := toExpr.(*ast.BasicLit) + if !ok { log.Fatalf("op 'to' expr %q is not a basic literal", to) } return astutil.Apply(node, func(cursor *astutil.Cursor) bool { - if isIdentWithName(cursor.Node(), from) { - cursor.Replace(toExpr) + if ident, ok := cursor.Node().(*ast.Ident); ok && ident.Name == from { + replacement := *toLit + replacement.ValuePos = ident.NamePos + cursor.Replace(new(replacement)) } return true }, nil) @@ -384,6 +393,23 @@ func inlineFunction(node ast.Node, from string, toDecl *ast.FuncDecl) ast.Node { replaceCallExprStmt(cursor, toDecl) } return false + case *ast.ReturnStmt: + if len(node.Results) == 1 && isCallTo(node.Results[0], from) { + args := node.Results[0].(*ast.CallExpr).Args + if !argsMatchParameters(args, toDecl.Type.Params) { + log.Fatalf("applying op: arguments to %v don't match parameter names of %v: %v", from, toDecl.Name, debugPrint(args...)) + } + replaceTailCall(cursor, toDecl) + } + return false + case *ast.CallExpr: + if isCallTo(node, from) { + switch cursor.Parent().(type) { + case *ast.AssignStmt, *ast.ExprStmt: + default: + log.Fatalf("applying op: all calls to function %q being replaced must appear in an assignment or expression statement, appears in %T", from, cursor.Parent()) + } + } } return true }, nil) @@ -438,6 +464,27 @@ func replaceCallExprStmt(cursor *astutil.Cursor, funcdecl *ast.FuncDecl) { cursor.Delete() } +func replaceTailCall(cursor *astutil.Cursor, funcdecl *ast.FuncDecl) { + if !hasTerminatingReturn(funcdecl.Body) { + log.Fatal("function being inlined must have a return at the end") + } + + body := internalastutil.CloneNode(funcdecl.Body) + if len(body.List) < 1 { + log.Fatal("replacing with empty bodied function") + } + + // The op happens in two steps: first we insert the body of the function being inlined (except for + // the final return) before the assignment, and then we change the assignment statement to replace the function call + // with the expressions being returned. + + // Insert the body up to the final return. + for _, stmt := range body.List { + cursor.InsertBefore(stmt) + } + cursor.Delete() +} + // replaceAssignment replaces an assignment statement where the right hand side is a function call // whose arguments have the same names as the parameters to funcdecl with the body of funcdecl. // It sets the left hand side of the assignment to the return values of the function. diff --git a/src/runtime/malloc_bench_generated_test.go b/src/runtime/malloc_bench_generated_test.go index e7e012daf1c6fc..34755f25feef53 100644 --- a/src/runtime/malloc_bench_generated_test.go +++ b/src/runtime/malloc_bench_generated_test.go @@ -11,7 +11,6 @@ import ( func benchmarkMallocgcNoscan8(b *testing.B) { const size = 8 - b.Run("kind=new", func(b *testing.B) { for b.Loop() { @@ -32,7 +31,6 @@ func benchmarkMallocgcNoscan8(b *testing.B) { func benchmarkMallocgcScan8(b *testing.B) { const size = 8 - b.Run("kind=new", func(b *testing.B) { for b.Loop() { @@ -53,7 +51,6 @@ func benchmarkMallocgcScan8(b *testing.B) { func benchmarkMallocgcNoscan16(b *testing.B) { const size = 16 - b.Run("kind=new", func(b *testing.B) { for b.Loop() { @@ -74,7 +71,6 @@ func benchmarkMallocgcNoscan16(b *testing.B) { func benchmarkMallocgcScan16(b *testing.B) { const size = 16 - b.Run("kind=new", func(b *testing.B) { for b.Loop() { @@ -95,7 +91,6 @@ func benchmarkMallocgcScan16(b *testing.B) { func benchmarkMallocgcNoscan24(b *testing.B) { const size = 24 - b.Run("kind=new", func(b *testing.B) { for b.Loop() { @@ -116,7 +111,6 @@ func benchmarkMallocgcNoscan24(b *testing.B) { func benchmarkMallocgcScan24(b *testing.B) { const size = 24 - b.Run("kind=new", func(b *testing.B) { for b.Loop() { @@ -137,7 +131,6 @@ func benchmarkMallocgcScan24(b *testing.B) { func benchmarkMallocgcNoscan32(b *testing.B) { const size = 32 - b.Run("kind=new", func(b *testing.B) { for b.Loop() { @@ -158,7 +151,6 @@ func benchmarkMallocgcNoscan32(b *testing.B) { func benchmarkMallocgcScan32(b *testing.B) { const size = 32 - b.Run("kind=new", func(b *testing.B) { for b.Loop() { @@ -179,7 +171,6 @@ func benchmarkMallocgcScan32(b *testing.B) { func benchmarkMallocgcNoscan48(b *testing.B) { const size = 48 - b.Run("kind=new", func(b *testing.B) { for b.Loop() { @@ -200,7 +191,6 @@ func benchmarkMallocgcNoscan48(b *testing.B) { func benchmarkMallocgcScan48(b *testing.B) { const size = 48 - b.Run("kind=new", func(b *testing.B) { for b.Loop() { @@ -221,7 +211,6 @@ func benchmarkMallocgcScan48(b *testing.B) { func benchmarkMallocgcNoscan64(b *testing.B) { const size = 64 - b.Run("kind=new", func(b *testing.B) { for b.Loop() { @@ -242,7 +231,6 @@ func benchmarkMallocgcNoscan64(b *testing.B) { func benchmarkMallocgcScan64(b *testing.B) { const size = 64 - b.Run("kind=new", func(b *testing.B) { for b.Loop() { @@ -263,7 +251,6 @@ func benchmarkMallocgcScan64(b *testing.B) { func benchmarkMallocgcNoscan80(b *testing.B) { const size = 80 - b.Run("kind=new", func(b *testing.B) { for b.Loop() { @@ -284,7 +271,6 @@ func benchmarkMallocgcNoscan80(b *testing.B) { func benchmarkMallocgcScan80(b *testing.B) { const size = 80 - b.Run("kind=new", func(b *testing.B) { for b.Loop() { @@ -305,7 +291,6 @@ func benchmarkMallocgcScan80(b *testing.B) { func benchmarkMallocgcNoscan96(b *testing.B) { const size = 96 - b.Run("kind=new", func(b *testing.B) { for b.Loop() { @@ -326,7 +311,6 @@ func benchmarkMallocgcNoscan96(b *testing.B) { func benchmarkMallocgcScan96(b *testing.B) { const size = 96 - b.Run("kind=new", func(b *testing.B) { for b.Loop() { @@ -347,7 +331,6 @@ func benchmarkMallocgcScan96(b *testing.B) { func benchmarkMallocgcNoscan112(b *testing.B) { const size = 112 - b.Run("kind=new", func(b *testing.B) { for b.Loop() { @@ -368,7 +351,6 @@ func benchmarkMallocgcNoscan112(b *testing.B) { func benchmarkMallocgcScan112(b *testing.B) { const size = 112 - b.Run("kind=new", func(b *testing.B) { for b.Loop() { @@ -389,7 +371,6 @@ func benchmarkMallocgcScan112(b *testing.B) { func benchmarkMallocgcNoscan128(b *testing.B) { const size = 128 - b.Run("kind=new", func(b *testing.B) { for b.Loop() { @@ -410,7 +391,6 @@ func benchmarkMallocgcNoscan128(b *testing.B) { func benchmarkMallocgcScan128(b *testing.B) { const size = 128 - b.Run("kind=new", func(b *testing.B) { for b.Loop() { @@ -431,7 +411,6 @@ func benchmarkMallocgcScan128(b *testing.B) { func benchmarkMallocgcNoscan144(b *testing.B) { const size = 144 - b.Run("kind=new", func(b *testing.B) { for b.Loop() { @@ -452,7 +431,6 @@ func benchmarkMallocgcNoscan144(b *testing.B) { func benchmarkMallocgcScan144(b *testing.B) { const size = 144 - b.Run("kind=new", func(b *testing.B) { for b.Loop() { @@ -473,7 +451,6 @@ func benchmarkMallocgcScan144(b *testing.B) { func benchmarkMallocgcNoscan160(b *testing.B) { const size = 160 - b.Run("kind=new", func(b *testing.B) { for b.Loop() { @@ -494,7 +471,6 @@ func benchmarkMallocgcNoscan160(b *testing.B) { func benchmarkMallocgcScan160(b *testing.B) { const size = 160 - b.Run("kind=new", func(b *testing.B) { for b.Loop() { @@ -515,7 +491,6 @@ func benchmarkMallocgcScan160(b *testing.B) { func benchmarkMallocgcNoscan176(b *testing.B) { const size = 176 - b.Run("kind=new", func(b *testing.B) { for b.Loop() { @@ -536,7 +511,6 @@ func benchmarkMallocgcNoscan176(b *testing.B) { func benchmarkMallocgcScan176(b *testing.B) { const size = 176 - b.Run("kind=new", func(b *testing.B) { for b.Loop() { @@ -557,7 +531,6 @@ func benchmarkMallocgcScan176(b *testing.B) { func benchmarkMallocgcNoscan192(b *testing.B) { const size = 192 - b.Run("kind=new", func(b *testing.B) { for b.Loop() { @@ -578,7 +551,6 @@ func benchmarkMallocgcNoscan192(b *testing.B) { func benchmarkMallocgcScan192(b *testing.B) { const size = 192 - b.Run("kind=new", func(b *testing.B) { for b.Loop() { @@ -599,7 +571,6 @@ func benchmarkMallocgcScan192(b *testing.B) { func benchmarkMallocgcNoscan208(b *testing.B) { const size = 208 - b.Run("kind=new", func(b *testing.B) { for b.Loop() { @@ -620,7 +591,6 @@ func benchmarkMallocgcNoscan208(b *testing.B) { func benchmarkMallocgcScan208(b *testing.B) { const size = 208 - b.Run("kind=new", func(b *testing.B) { for b.Loop() { @@ -641,7 +611,6 @@ func benchmarkMallocgcScan208(b *testing.B) { func benchmarkMallocgcNoscan224(b *testing.B) { const size = 224 - b.Run("kind=new", func(b *testing.B) { for b.Loop() { @@ -662,7 +631,6 @@ func benchmarkMallocgcNoscan224(b *testing.B) { func benchmarkMallocgcScan224(b *testing.B) { const size = 224 - b.Run("kind=new", func(b *testing.B) { for b.Loop() { @@ -683,7 +651,6 @@ func benchmarkMallocgcScan224(b *testing.B) { func benchmarkMallocgcNoscan240(b *testing.B) { const size = 240 - b.Run("kind=new", func(b *testing.B) { for b.Loop() { @@ -704,7 +671,6 @@ func benchmarkMallocgcNoscan240(b *testing.B) { func benchmarkMallocgcScan240(b *testing.B) { const size = 240 - b.Run("kind=new", func(b *testing.B) { for b.Loop() { @@ -725,7 +691,6 @@ func benchmarkMallocgcScan240(b *testing.B) { func benchmarkMallocgcNoscan256(b *testing.B) { const size = 256 - b.Run("kind=new", func(b *testing.B) { for b.Loop() { @@ -746,7 +711,6 @@ func benchmarkMallocgcNoscan256(b *testing.B) { func benchmarkMallocgcScan256(b *testing.B) { const size = 256 - b.Run("kind=new", func(b *testing.B) { for b.Loop() { @@ -767,7 +731,6 @@ func benchmarkMallocgcScan256(b *testing.B) { func benchmarkMallocgcNoscan288(b *testing.B) { const size = 288 - b.Run("kind=new", func(b *testing.B) { for b.Loop() { @@ -788,7 +751,6 @@ func benchmarkMallocgcNoscan288(b *testing.B) { func benchmarkMallocgcScan288(b *testing.B) { const size = 288 - b.Run("kind=new", func(b *testing.B) { for b.Loop() { @@ -809,7 +771,6 @@ func benchmarkMallocgcScan288(b *testing.B) { func benchmarkMallocgcNoscan320(b *testing.B) { const size = 320 - b.Run("kind=new", func(b *testing.B) { for b.Loop() { @@ -830,7 +791,6 @@ func benchmarkMallocgcNoscan320(b *testing.B) { func benchmarkMallocgcScan320(b *testing.B) { const size = 320 - b.Run("kind=new", func(b *testing.B) { for b.Loop() { @@ -851,7 +811,6 @@ func benchmarkMallocgcScan320(b *testing.B) { func benchmarkMallocgcNoscan352(b *testing.B) { const size = 352 - b.Run("kind=new", func(b *testing.B) { for b.Loop() { @@ -872,7 +831,6 @@ func benchmarkMallocgcNoscan352(b *testing.B) { func benchmarkMallocgcScan352(b *testing.B) { const size = 352 - b.Run("kind=new", func(b *testing.B) { for b.Loop() { @@ -893,7 +851,6 @@ func benchmarkMallocgcScan352(b *testing.B) { func benchmarkMallocgcNoscan384(b *testing.B) { const size = 384 - b.Run("kind=new", func(b *testing.B) { for b.Loop() { @@ -914,7 +871,6 @@ func benchmarkMallocgcNoscan384(b *testing.B) { func benchmarkMallocgcScan384(b *testing.B) { const size = 384 - b.Run("kind=new", func(b *testing.B) { for b.Loop() { @@ -935,7 +891,6 @@ func benchmarkMallocgcScan384(b *testing.B) { func benchmarkMallocgcNoscan416(b *testing.B) { const size = 416 - b.Run("kind=new", func(b *testing.B) { for b.Loop() { @@ -956,7 +911,6 @@ func benchmarkMallocgcNoscan416(b *testing.B) { func benchmarkMallocgcScan416(b *testing.B) { const size = 416 - b.Run("kind=new", func(b *testing.B) { for b.Loop() { @@ -977,7 +931,6 @@ func benchmarkMallocgcScan416(b *testing.B) { func benchmarkMallocgcNoscan448(b *testing.B) { const size = 448 - b.Run("kind=new", func(b *testing.B) { for b.Loop() { @@ -998,7 +951,6 @@ func benchmarkMallocgcNoscan448(b *testing.B) { func benchmarkMallocgcScan448(b *testing.B) { const size = 448 - b.Run("kind=new", func(b *testing.B) { for b.Loop() { @@ -1019,7 +971,6 @@ func benchmarkMallocgcScan448(b *testing.B) { func benchmarkMallocgcNoscan480(b *testing.B) { const size = 480 - b.Run("kind=new", func(b *testing.B) { for b.Loop() { @@ -1040,7 +991,6 @@ func benchmarkMallocgcNoscan480(b *testing.B) { func benchmarkMallocgcScan480(b *testing.B) { const size = 480 - b.Run("kind=new", func(b *testing.B) { for b.Loop() { @@ -1061,7 +1011,6 @@ func benchmarkMallocgcScan480(b *testing.B) { func benchmarkMallocgcNoscan512(b *testing.B) { const size = 512 - b.Run("kind=new", func(b *testing.B) { for b.Loop() { @@ -1082,7 +1031,6 @@ func benchmarkMallocgcNoscan512(b *testing.B) { func benchmarkMallocgcScan512(b *testing.B) { const size = 512 - b.Run("kind=new", func(b *testing.B) { for b.Loop() { @@ -1103,7 +1051,6 @@ func benchmarkMallocgcScan512(b *testing.B) { func benchmarkMallocgcTiny1(b *testing.B) { const size = 1 - type s struct { v [size]byte } @@ -1122,7 +1069,6 @@ func benchmarkMallocgcTiny1(b *testing.B) { func benchmarkMallocgcTiny2(b *testing.B) { const size = 2 - type s struct { v [size]byte } @@ -1141,7 +1087,6 @@ func benchmarkMallocgcTiny2(b *testing.B) { func benchmarkMallocgcTiny3(b *testing.B) { const size = 3 - type s struct { v [size]byte } @@ -1160,7 +1105,6 @@ func benchmarkMallocgcTiny3(b *testing.B) { func benchmarkMallocgcTiny4(b *testing.B) { const size = 4 - type s struct { v [size]byte } @@ -1179,7 +1123,6 @@ func benchmarkMallocgcTiny4(b *testing.B) { func benchmarkMallocgcTiny5(b *testing.B) { const size = 5 - type s struct { v [size]byte } @@ -1198,7 +1141,6 @@ func benchmarkMallocgcTiny5(b *testing.B) { func benchmarkMallocgcTiny6(b *testing.B) { const size = 6 - type s struct { v [size]byte } @@ -1217,7 +1159,6 @@ func benchmarkMallocgcTiny6(b *testing.B) { func benchmarkMallocgcTiny7(b *testing.B) { const size = 7 - type s struct { v [size]byte } @@ -1236,7 +1177,6 @@ func benchmarkMallocgcTiny7(b *testing.B) { func benchmarkMallocgcTiny8(b *testing.B) { const size = 8 - type s struct { v [size]byte } @@ -1255,7 +1195,6 @@ func benchmarkMallocgcTiny8(b *testing.B) { func benchmarkMallocgcTiny9(b *testing.B) { const size = 9 - type s struct { v [size]byte } @@ -1274,7 +1213,6 @@ func benchmarkMallocgcTiny9(b *testing.B) { func benchmarkMallocgcTiny10(b *testing.B) { const size = 10 - type s struct { v [size]byte } @@ -1293,7 +1231,6 @@ func benchmarkMallocgcTiny10(b *testing.B) { func benchmarkMallocgcTiny11(b *testing.B) { const size = 11 - type s struct { v [size]byte } @@ -1312,7 +1249,6 @@ func benchmarkMallocgcTiny11(b *testing.B) { func benchmarkMallocgcTiny12(b *testing.B) { const size = 12 - type s struct { v [size]byte } @@ -1331,7 +1267,6 @@ func benchmarkMallocgcTiny12(b *testing.B) { func benchmarkMallocgcTiny13(b *testing.B) { const size = 13 - type s struct { v [size]byte } @@ -1350,7 +1285,6 @@ func benchmarkMallocgcTiny13(b *testing.B) { func benchmarkMallocgcTiny14(b *testing.B) { const size = 14 - type s struct { v [size]byte } @@ -1369,7 +1303,6 @@ func benchmarkMallocgcTiny14(b *testing.B) { func benchmarkMallocgcTiny15(b *testing.B) { const size = 15 - type s struct { v [size]byte } diff --git a/src/runtime/malloc_generated.go b/src/runtime/malloc_generated.go index 4b7b547b027c58..9642386cb77a65 100644 --- a/src/runtime/malloc_generated.go +++ b/src/runtime/malloc_generated.go @@ -31,12 +31,13 @@ func mallocgcSmallScanNoHeaderSC1(size uintptr, typ *_type, needzero bool) unsaf } const sizeclass = 1 - const elemsize = 8 mp := acquirem() if doubleCheckMalloc { + doubleCheckSmallScanNoHeader(size, typ, mp) + } mp.mallocing = 1 @@ -55,9 +56,7 @@ func mallocgcSmallScanNoHeaderSC1(size uintptr, typ *_type, needzero bool) unsaf span.allocCache >>= uint(theBit + 1) span.freeindex = freeidx span.allocCount++ - nextFreeFastResult = gclinkptr(uintptr(result)* - 8 + - span.base()) + nextFreeFastResult = gclinkptr(uintptr(result)*8 + span.base()) } } } @@ -66,6 +65,7 @@ func mallocgcSmallScanNoHeaderSC1(size uintptr, typ *_type, needzero bool) unsaf v, span, checkGCTrigger = c.nextFree(spc) } x := unsafe.Pointer(v) + if span.needzero != 0 { memclrNoHeapPointers(x, elemsize) } @@ -162,10 +162,6 @@ func mallocgcSmallScanNoHeaderSC1(size uintptr, typ *_type, needzero bool) unsaf addSecret(x, size) } - if valgrindenabled { - valgrindMalloc(x, size) - } - if gcBlackenEnabled != 0 && elemsize != 0 { if assistG := getg().m.curg; assistG != nil { assistG.gcAssistBytes -= int64(elemsize - size) @@ -175,6 +171,7 @@ func mallocgcSmallScanNoHeaderSC1(size uintptr, typ *_type, needzero bool) unsaf if debug.malloc { postMallocgcDebug(x, elemsize, typ) } + return x } @@ -199,12 +196,13 @@ func mallocgcSmallScanNoHeaderSC2(size uintptr, typ *_type, needzero bool) unsaf } const sizeclass = 2 - const elemsize = 16 mp := acquirem() if doubleCheckMalloc { + doubleCheckSmallScanNoHeader(size, typ, mp) + } mp.mallocing = 1 @@ -223,9 +221,7 @@ func mallocgcSmallScanNoHeaderSC2(size uintptr, typ *_type, needzero bool) unsaf span.allocCache >>= uint(theBit + 1) span.freeindex = freeidx span.allocCount++ - nextFreeFastResult = gclinkptr(uintptr(result)* - 16 + - span.base()) + nextFreeFastResult = gclinkptr(uintptr(result)*16 + span.base()) } } } @@ -234,6 +230,7 @@ func mallocgcSmallScanNoHeaderSC2(size uintptr, typ *_type, needzero bool) unsaf v, span, checkGCTrigger = c.nextFree(spc) } x := unsafe.Pointer(v) + if span.needzero != 0 { memclrNoHeapPointers(x, elemsize) } @@ -330,10 +327,6 @@ func mallocgcSmallScanNoHeaderSC2(size uintptr, typ *_type, needzero bool) unsaf addSecret(x, size) } - if valgrindenabled { - valgrindMalloc(x, size) - } - if gcBlackenEnabled != 0 && elemsize != 0 { if assistG := getg().m.curg; assistG != nil { assistG.gcAssistBytes -= int64(elemsize - size) @@ -343,6 +336,7 @@ func mallocgcSmallScanNoHeaderSC2(size uintptr, typ *_type, needzero bool) unsaf if debug.malloc { postMallocgcDebug(x, elemsize, typ) } + return x } @@ -367,12 +361,13 @@ func mallocgcSmallScanNoHeaderSC3(size uintptr, typ *_type, needzero bool) unsaf } const sizeclass = 3 - const elemsize = 24 mp := acquirem() if doubleCheckMalloc { + doubleCheckSmallScanNoHeader(size, typ, mp) + } mp.mallocing = 1 @@ -391,9 +386,7 @@ func mallocgcSmallScanNoHeaderSC3(size uintptr, typ *_type, needzero bool) unsaf span.allocCache >>= uint(theBit + 1) span.freeindex = freeidx span.allocCount++ - nextFreeFastResult = gclinkptr(uintptr(result)* - 24 + - span.base()) + nextFreeFastResult = gclinkptr(uintptr(result)*24 + span.base()) } } } @@ -402,6 +395,7 @@ func mallocgcSmallScanNoHeaderSC3(size uintptr, typ *_type, needzero bool) unsaf v, span, checkGCTrigger = c.nextFree(spc) } x := unsafe.Pointer(v) + if span.needzero != 0 { memclrNoHeapPointers(x, elemsize) } @@ -498,10 +492,6 @@ func mallocgcSmallScanNoHeaderSC3(size uintptr, typ *_type, needzero bool) unsaf addSecret(x, size) } - if valgrindenabled { - valgrindMalloc(x, size) - } - if gcBlackenEnabled != 0 && elemsize != 0 { if assistG := getg().m.curg; assistG != nil { assistG.gcAssistBytes -= int64(elemsize - size) @@ -511,6 +501,7 @@ func mallocgcSmallScanNoHeaderSC3(size uintptr, typ *_type, needzero bool) unsaf if debug.malloc { postMallocgcDebug(x, elemsize, typ) } + return x } @@ -535,12 +526,13 @@ func mallocgcSmallScanNoHeaderSC4(size uintptr, typ *_type, needzero bool) unsaf } const sizeclass = 4 - const elemsize = 32 mp := acquirem() if doubleCheckMalloc { + doubleCheckSmallScanNoHeader(size, typ, mp) + } mp.mallocing = 1 @@ -559,9 +551,7 @@ func mallocgcSmallScanNoHeaderSC4(size uintptr, typ *_type, needzero bool) unsaf span.allocCache >>= uint(theBit + 1) span.freeindex = freeidx span.allocCount++ - nextFreeFastResult = gclinkptr(uintptr(result)* - 32 + - span.base()) + nextFreeFastResult = gclinkptr(uintptr(result)*32 + span.base()) } } } @@ -570,6 +560,7 @@ func mallocgcSmallScanNoHeaderSC4(size uintptr, typ *_type, needzero bool) unsaf v, span, checkGCTrigger = c.nextFree(spc) } x := unsafe.Pointer(v) + if span.needzero != 0 { memclrNoHeapPointers(x, elemsize) } @@ -666,10 +657,6 @@ func mallocgcSmallScanNoHeaderSC4(size uintptr, typ *_type, needzero bool) unsaf addSecret(x, size) } - if valgrindenabled { - valgrindMalloc(x, size) - } - if gcBlackenEnabled != 0 && elemsize != 0 { if assistG := getg().m.curg; assistG != nil { assistG.gcAssistBytes -= int64(elemsize - size) @@ -679,6 +666,7 @@ func mallocgcSmallScanNoHeaderSC4(size uintptr, typ *_type, needzero bool) unsaf if debug.malloc { postMallocgcDebug(x, elemsize, typ) } + return x } @@ -703,12 +691,13 @@ func mallocgcSmallScanNoHeaderSC5(size uintptr, typ *_type, needzero bool) unsaf } const sizeclass = 5 - const elemsize = 48 mp := acquirem() if doubleCheckMalloc { + doubleCheckSmallScanNoHeader(size, typ, mp) + } mp.mallocing = 1 @@ -727,9 +716,7 @@ func mallocgcSmallScanNoHeaderSC5(size uintptr, typ *_type, needzero bool) unsaf span.allocCache >>= uint(theBit + 1) span.freeindex = freeidx span.allocCount++ - nextFreeFastResult = gclinkptr(uintptr(result)* - 48 + - span.base()) + nextFreeFastResult = gclinkptr(uintptr(result)*48 + span.base()) } } } @@ -738,6 +725,7 @@ func mallocgcSmallScanNoHeaderSC5(size uintptr, typ *_type, needzero bool) unsaf v, span, checkGCTrigger = c.nextFree(spc) } x := unsafe.Pointer(v) + if span.needzero != 0 { memclrNoHeapPointers(x, elemsize) } @@ -834,10 +822,6 @@ func mallocgcSmallScanNoHeaderSC5(size uintptr, typ *_type, needzero bool) unsaf addSecret(x, size) } - if valgrindenabled { - valgrindMalloc(x, size) - } - if gcBlackenEnabled != 0 && elemsize != 0 { if assistG := getg().m.curg; assistG != nil { assistG.gcAssistBytes -= int64(elemsize - size) @@ -847,6 +831,7 @@ func mallocgcSmallScanNoHeaderSC5(size uintptr, typ *_type, needzero bool) unsaf if debug.malloc { postMallocgcDebug(x, elemsize, typ) } + return x } @@ -871,12 +856,13 @@ func mallocgcSmallScanNoHeaderSC6(size uintptr, typ *_type, needzero bool) unsaf } const sizeclass = 6 - const elemsize = 64 mp := acquirem() if doubleCheckMalloc { + doubleCheckSmallScanNoHeader(size, typ, mp) + } mp.mallocing = 1 @@ -895,9 +881,7 @@ func mallocgcSmallScanNoHeaderSC6(size uintptr, typ *_type, needzero bool) unsaf span.allocCache >>= uint(theBit + 1) span.freeindex = freeidx span.allocCount++ - nextFreeFastResult = gclinkptr(uintptr(result)* - 64 + - span.base()) + nextFreeFastResult = gclinkptr(uintptr(result)*64 + span.base()) } } } @@ -906,6 +890,7 @@ func mallocgcSmallScanNoHeaderSC6(size uintptr, typ *_type, needzero bool) unsaf v, span, checkGCTrigger = c.nextFree(spc) } x := unsafe.Pointer(v) + if span.needzero != 0 { memclrNoHeapPointers(x, elemsize) } @@ -1002,10 +987,6 @@ func mallocgcSmallScanNoHeaderSC6(size uintptr, typ *_type, needzero bool) unsaf addSecret(x, size) } - if valgrindenabled { - valgrindMalloc(x, size) - } - if gcBlackenEnabled != 0 && elemsize != 0 { if assistG := getg().m.curg; assistG != nil { assistG.gcAssistBytes -= int64(elemsize - size) @@ -1015,6 +996,7 @@ func mallocgcSmallScanNoHeaderSC6(size uintptr, typ *_type, needzero bool) unsaf if debug.malloc { postMallocgcDebug(x, elemsize, typ) } + return x } @@ -1039,12 +1021,13 @@ func mallocgcSmallScanNoHeaderSC7(size uintptr, typ *_type, needzero bool) unsaf } const sizeclass = 7 - const elemsize = 80 mp := acquirem() if doubleCheckMalloc { + doubleCheckSmallScanNoHeader(size, typ, mp) + } mp.mallocing = 1 @@ -1063,9 +1046,7 @@ func mallocgcSmallScanNoHeaderSC7(size uintptr, typ *_type, needzero bool) unsaf span.allocCache >>= uint(theBit + 1) span.freeindex = freeidx span.allocCount++ - nextFreeFastResult = gclinkptr(uintptr(result)* - 80 + - span.base()) + nextFreeFastResult = gclinkptr(uintptr(result)*80 + span.base()) } } } @@ -1074,6 +1055,7 @@ func mallocgcSmallScanNoHeaderSC7(size uintptr, typ *_type, needzero bool) unsaf v, span, checkGCTrigger = c.nextFree(spc) } x := unsafe.Pointer(v) + if span.needzero != 0 { memclrNoHeapPointers(x, elemsize) } @@ -1170,10 +1152,6 @@ func mallocgcSmallScanNoHeaderSC7(size uintptr, typ *_type, needzero bool) unsaf addSecret(x, size) } - if valgrindenabled { - valgrindMalloc(x, size) - } - if gcBlackenEnabled != 0 && elemsize != 0 { if assistG := getg().m.curg; assistG != nil { assistG.gcAssistBytes -= int64(elemsize - size) @@ -1183,6 +1161,7 @@ func mallocgcSmallScanNoHeaderSC7(size uintptr, typ *_type, needzero bool) unsaf if debug.malloc { postMallocgcDebug(x, elemsize, typ) } + return x } @@ -1207,12 +1186,13 @@ func mallocgcSmallScanNoHeaderSC8(size uintptr, typ *_type, needzero bool) unsaf } const sizeclass = 8 - const elemsize = 96 mp := acquirem() if doubleCheckMalloc { + doubleCheckSmallScanNoHeader(size, typ, mp) + } mp.mallocing = 1 @@ -1231,9 +1211,7 @@ func mallocgcSmallScanNoHeaderSC8(size uintptr, typ *_type, needzero bool) unsaf span.allocCache >>= uint(theBit + 1) span.freeindex = freeidx span.allocCount++ - nextFreeFastResult = gclinkptr(uintptr(result)* - 96 + - span.base()) + nextFreeFastResult = gclinkptr(uintptr(result)*96 + span.base()) } } } @@ -1242,6 +1220,7 @@ func mallocgcSmallScanNoHeaderSC8(size uintptr, typ *_type, needzero bool) unsaf v, span, checkGCTrigger = c.nextFree(spc) } x := unsafe.Pointer(v) + if span.needzero != 0 { memclrNoHeapPointers(x, elemsize) } @@ -1338,10 +1317,6 @@ func mallocgcSmallScanNoHeaderSC8(size uintptr, typ *_type, needzero bool) unsaf addSecret(x, size) } - if valgrindenabled { - valgrindMalloc(x, size) - } - if gcBlackenEnabled != 0 && elemsize != 0 { if assistG := getg().m.curg; assistG != nil { assistG.gcAssistBytes -= int64(elemsize - size) @@ -1351,6 +1326,7 @@ func mallocgcSmallScanNoHeaderSC8(size uintptr, typ *_type, needzero bool) unsaf if debug.malloc { postMallocgcDebug(x, elemsize, typ) } + return x } @@ -1375,12 +1351,13 @@ func mallocgcSmallScanNoHeaderSC9(size uintptr, typ *_type, needzero bool) unsaf } const sizeclass = 9 - const elemsize = 112 mp := acquirem() if doubleCheckMalloc { + doubleCheckSmallScanNoHeader(size, typ, mp) + } mp.mallocing = 1 @@ -1399,9 +1376,7 @@ func mallocgcSmallScanNoHeaderSC9(size uintptr, typ *_type, needzero bool) unsaf span.allocCache >>= uint(theBit + 1) span.freeindex = freeidx span.allocCount++ - nextFreeFastResult = gclinkptr(uintptr(result)* - 112 + - span.base()) + nextFreeFastResult = gclinkptr(uintptr(result)*112 + span.base()) } } } @@ -1410,6 +1385,7 @@ func mallocgcSmallScanNoHeaderSC9(size uintptr, typ *_type, needzero bool) unsaf v, span, checkGCTrigger = c.nextFree(spc) } x := unsafe.Pointer(v) + if span.needzero != 0 { memclrNoHeapPointers(x, elemsize) } @@ -1506,10 +1482,6 @@ func mallocgcSmallScanNoHeaderSC9(size uintptr, typ *_type, needzero bool) unsaf addSecret(x, size) } - if valgrindenabled { - valgrindMalloc(x, size) - } - if gcBlackenEnabled != 0 && elemsize != 0 { if assistG := getg().m.curg; assistG != nil { assistG.gcAssistBytes -= int64(elemsize - size) @@ -1519,6 +1491,7 @@ func mallocgcSmallScanNoHeaderSC9(size uintptr, typ *_type, needzero bool) unsaf if debug.malloc { postMallocgcDebug(x, elemsize, typ) } + return x } @@ -1543,12 +1516,13 @@ func mallocgcSmallScanNoHeaderSC10(size uintptr, typ *_type, needzero bool) unsa } const sizeclass = 10 - const elemsize = 128 mp := acquirem() if doubleCheckMalloc { + doubleCheckSmallScanNoHeader(size, typ, mp) + } mp.mallocing = 1 @@ -1567,9 +1541,7 @@ func mallocgcSmallScanNoHeaderSC10(size uintptr, typ *_type, needzero bool) unsa span.allocCache >>= uint(theBit + 1) span.freeindex = freeidx span.allocCount++ - nextFreeFastResult = gclinkptr(uintptr(result)* - 128 + - span.base()) + nextFreeFastResult = gclinkptr(uintptr(result)*128 + span.base()) } } } @@ -1578,6 +1550,7 @@ func mallocgcSmallScanNoHeaderSC10(size uintptr, typ *_type, needzero bool) unsa v, span, checkGCTrigger = c.nextFree(spc) } x := unsafe.Pointer(v) + if span.needzero != 0 { memclrNoHeapPointers(x, elemsize) } @@ -1674,10 +1647,6 @@ func mallocgcSmallScanNoHeaderSC10(size uintptr, typ *_type, needzero bool) unsa addSecret(x, size) } - if valgrindenabled { - valgrindMalloc(x, size) - } - if gcBlackenEnabled != 0 && elemsize != 0 { if assistG := getg().m.curg; assistG != nil { assistG.gcAssistBytes -= int64(elemsize - size) @@ -1687,6 +1656,7 @@ func mallocgcSmallScanNoHeaderSC10(size uintptr, typ *_type, needzero bool) unsa if debug.malloc { postMallocgcDebug(x, elemsize, typ) } + return x } @@ -1752,33 +1722,24 @@ func mallocgcTinySC2(size uintptr, typ *_type, needzero bool) unsafe.Pointer { mp.mallocing = 0 releasem(mp) const elemsize = 0 - { - - if valgrindenabled { - valgrindMalloc(x, size) - } - - if gcBlackenEnabled != 0 && elemsize != 0 { - if assistG := getg().m.curg; assistG != nil { - assistG.gcAssistBytes -= int64(elemsize - size) - } + if gcBlackenEnabled != 0 && elemsize != 0 { + if assistG := getg().m.curg; assistG != nil { + assistG.gcAssistBytes -= int64(elemsize - size) } + } - if debug.malloc { - postMallocgcDebug(x, elemsize, typ) - } - return x + if debug.malloc { + postMallocgcDebug(x, elemsize, typ) } + return x } checkGCTrigger := false span := c.alloc[tinySpanClass] const nbytes = 8192 - const nelems = uint16((nbytes - unsafe.Sizeof(spanInlineMarkBits{})) / - 16, - ) + const nelems = uint16((nbytes - unsafe.Sizeof(spanInlineMarkBits{})) / 16) var nextFreeFastResult gclinkptr if span.allocCache != 0 { theBit := sys.TrailingZeros64(span.allocCache) @@ -1789,9 +1750,7 @@ func mallocgcTinySC2(size uintptr, typ *_type, needzero bool) unsafe.Pointer { span.allocCache >>= uint(theBit + 1) span.freeindex = freeidx span.allocCount++ - nextFreeFastResult = gclinkptr(uintptr(result)* - 16 + - span.base()) + nextFreeFastResult = gclinkptr(uintptr(result)*16 + span.base()) } } } @@ -1831,15 +1790,6 @@ func mallocgcTinySC2(size uintptr, typ *_type, needzero bool) unsafe.Pointer { gcStart(t) } } - - if raceenabled { - - x = add(x, elemsize-size) - } - if valgrindenabled { - valgrindMalloc(x, size) - } - if gcBlackenEnabled != 0 && elemsize != 0 { if assistG := getg().m.curg; assistG != nil { assistG.gcAssistBytes -= int64(elemsize - size) @@ -1849,6 +1799,7 @@ func mallocgcTinySC2(size uintptr, typ *_type, needzero bool) unsafe.Pointer { if debug.malloc { postMallocgcDebug(x, elemsize, typ) } + return x } @@ -1873,12 +1824,13 @@ func mallocgcSmallNoScanSC2(size uintptr, typ *_type, needzero bool) unsafe.Poin } const sizeclass = 2 - const elemsize = 16 mp := acquirem() if doubleCheckMalloc { + doubleCheckSmallNoScan(typ, mp) + } mp.mallocing = 1 @@ -1889,34 +1841,26 @@ func mallocgcSmallNoScanSC2(size uintptr, typ *_type, needzero bool) unsafe.Poin if runtimeFreegcEnabled && c.hasReusableNoscan(spc) { - v := mallocgcSmallNoscanReuse(c, span, spc, elemsize, needzero) + x := mallocgcSmallNoscanReuse(c, span, spc, elemsize, needzero) mp.mallocing = 0 releasem(mp) - x := v - { - - gp := getg() - if goexperiment.RuntimeSecret && gp.secret > 0 { + gp := getg() + if goexperiment.RuntimeSecret && gp.secret > 0 { - addSecret(x, size) - } - - if valgrindenabled { - valgrindMalloc(x, size) - } + addSecret(x, size) + } - if gcBlackenEnabled != 0 && elemsize != 0 { - if assistG := getg().m.curg; assistG != nil { - assistG.gcAssistBytes -= int64(elemsize - size) - } + if gcBlackenEnabled != 0 && elemsize != 0 { + if assistG := getg().m.curg; assistG != nil { + assistG.gcAssistBytes -= int64(elemsize - size) } + } - if debug.malloc { - postMallocgcDebug(x, elemsize, typ) - } - return x + if debug.malloc { + postMallocgcDebug(x, elemsize, typ) } + return x } var nextFreeFastResult gclinkptr @@ -1929,9 +1873,7 @@ func mallocgcSmallNoScanSC2(size uintptr, typ *_type, needzero bool) unsafe.Poin span.allocCache >>= uint(theBit + 1) span.freeindex = freeidx span.allocCount++ - nextFreeFastResult = gclinkptr(uintptr(result)* - 16 + - span.base()) + nextFreeFastResult = gclinkptr(uintptr(result)*16 + span.base()) } } } @@ -1940,6 +1882,7 @@ func mallocgcSmallNoScanSC2(size uintptr, typ *_type, needzero bool) unsafe.Poin v, span, checkGCTrigger = c.nextFree(spc) } x := unsafe.Pointer(v) + if needzero && span.needzero != 0 { memclrNoHeapPointers(x, elemsize) } @@ -1972,10 +1915,6 @@ func mallocgcSmallNoScanSC2(size uintptr, typ *_type, needzero bool) unsafe.Poin addSecret(x, size) } - if valgrindenabled { - valgrindMalloc(x, size) - } - if gcBlackenEnabled != 0 && elemsize != 0 { if assistG := getg().m.curg; assistG != nil { assistG.gcAssistBytes -= int64(elemsize - size) @@ -1985,6 +1924,7 @@ func mallocgcSmallNoScanSC2(size uintptr, typ *_type, needzero bool) unsafe.Poin if debug.malloc { postMallocgcDebug(x, elemsize, typ) } + return x } @@ -2009,12 +1949,13 @@ func mallocgcSmallNoScanSC3(size uintptr, typ *_type, needzero bool) unsafe.Poin } const sizeclass = 3 - const elemsize = 24 mp := acquirem() if doubleCheckMalloc { + doubleCheckSmallNoScan(typ, mp) + } mp.mallocing = 1 @@ -2025,34 +1966,26 @@ func mallocgcSmallNoScanSC3(size uintptr, typ *_type, needzero bool) unsafe.Poin if runtimeFreegcEnabled && c.hasReusableNoscan(spc) { - v := mallocgcSmallNoscanReuse(c, span, spc, elemsize, needzero) + x := mallocgcSmallNoscanReuse(c, span, spc, elemsize, needzero) mp.mallocing = 0 releasem(mp) - x := v - { - - gp := getg() - if goexperiment.RuntimeSecret && gp.secret > 0 { - - addSecret(x, size) - } + gp := getg() + if goexperiment.RuntimeSecret && gp.secret > 0 { - if valgrindenabled { - valgrindMalloc(x, size) - } + addSecret(x, size) + } - if gcBlackenEnabled != 0 && elemsize != 0 { - if assistG := getg().m.curg; assistG != nil { - assistG.gcAssistBytes -= int64(elemsize - size) - } + if gcBlackenEnabled != 0 && elemsize != 0 { + if assistG := getg().m.curg; assistG != nil { + assistG.gcAssistBytes -= int64(elemsize - size) } + } - if debug.malloc { - postMallocgcDebug(x, elemsize, typ) - } - return x + if debug.malloc { + postMallocgcDebug(x, elemsize, typ) } + return x } var nextFreeFastResult gclinkptr @@ -2065,9 +1998,7 @@ func mallocgcSmallNoScanSC3(size uintptr, typ *_type, needzero bool) unsafe.Poin span.allocCache >>= uint(theBit + 1) span.freeindex = freeidx span.allocCount++ - nextFreeFastResult = gclinkptr(uintptr(result)* - 24 + - span.base()) + nextFreeFastResult = gclinkptr(uintptr(result)*24 + span.base()) } } } @@ -2076,6 +2007,7 @@ func mallocgcSmallNoScanSC3(size uintptr, typ *_type, needzero bool) unsafe.Poin v, span, checkGCTrigger = c.nextFree(spc) } x := unsafe.Pointer(v) + if needzero && span.needzero != 0 { memclrNoHeapPointers(x, elemsize) } @@ -2108,10 +2040,6 @@ func mallocgcSmallNoScanSC3(size uintptr, typ *_type, needzero bool) unsafe.Poin addSecret(x, size) } - if valgrindenabled { - valgrindMalloc(x, size) - } - if gcBlackenEnabled != 0 && elemsize != 0 { if assistG := getg().m.curg; assistG != nil { assistG.gcAssistBytes -= int64(elemsize - size) @@ -2121,6 +2049,7 @@ func mallocgcSmallNoScanSC3(size uintptr, typ *_type, needzero bool) unsafe.Poin if debug.malloc { postMallocgcDebug(x, elemsize, typ) } + return x } @@ -2145,12 +2074,13 @@ func mallocgcSmallNoScanSC4(size uintptr, typ *_type, needzero bool) unsafe.Poin } const sizeclass = 4 - const elemsize = 32 mp := acquirem() if doubleCheckMalloc { + doubleCheckSmallNoScan(typ, mp) + } mp.mallocing = 1 @@ -2161,34 +2091,26 @@ func mallocgcSmallNoScanSC4(size uintptr, typ *_type, needzero bool) unsafe.Poin if runtimeFreegcEnabled && c.hasReusableNoscan(spc) { - v := mallocgcSmallNoscanReuse(c, span, spc, elemsize, needzero) + x := mallocgcSmallNoscanReuse(c, span, spc, elemsize, needzero) mp.mallocing = 0 releasem(mp) - x := v - { - - gp := getg() - if goexperiment.RuntimeSecret && gp.secret > 0 { + gp := getg() + if goexperiment.RuntimeSecret && gp.secret > 0 { - addSecret(x, size) - } - - if valgrindenabled { - valgrindMalloc(x, size) - } + addSecret(x, size) + } - if gcBlackenEnabled != 0 && elemsize != 0 { - if assistG := getg().m.curg; assistG != nil { - assistG.gcAssistBytes -= int64(elemsize - size) - } + if gcBlackenEnabled != 0 && elemsize != 0 { + if assistG := getg().m.curg; assistG != nil { + assistG.gcAssistBytes -= int64(elemsize - size) } + } - if debug.malloc { - postMallocgcDebug(x, elemsize, typ) - } - return x + if debug.malloc { + postMallocgcDebug(x, elemsize, typ) } + return x } var nextFreeFastResult gclinkptr @@ -2201,9 +2123,7 @@ func mallocgcSmallNoScanSC4(size uintptr, typ *_type, needzero bool) unsafe.Poin span.allocCache >>= uint(theBit + 1) span.freeindex = freeidx span.allocCount++ - nextFreeFastResult = gclinkptr(uintptr(result)* - 32 + - span.base()) + nextFreeFastResult = gclinkptr(uintptr(result)*32 + span.base()) } } } @@ -2212,6 +2132,7 @@ func mallocgcSmallNoScanSC4(size uintptr, typ *_type, needzero bool) unsafe.Poin v, span, checkGCTrigger = c.nextFree(spc) } x := unsafe.Pointer(v) + if needzero && span.needzero != 0 { memclrNoHeapPointers(x, elemsize) } @@ -2244,10 +2165,6 @@ func mallocgcSmallNoScanSC4(size uintptr, typ *_type, needzero bool) unsafe.Poin addSecret(x, size) } - if valgrindenabled { - valgrindMalloc(x, size) - } - if gcBlackenEnabled != 0 && elemsize != 0 { if assistG := getg().m.curg; assistG != nil { assistG.gcAssistBytes -= int64(elemsize - size) @@ -2257,6 +2174,7 @@ func mallocgcSmallNoScanSC4(size uintptr, typ *_type, needzero bool) unsafe.Poin if debug.malloc { postMallocgcDebug(x, elemsize, typ) } + return x } @@ -2281,12 +2199,13 @@ func mallocgcSmallNoScanSC5(size uintptr, typ *_type, needzero bool) unsafe.Poin } const sizeclass = 5 - const elemsize = 48 mp := acquirem() if doubleCheckMalloc { + doubleCheckSmallNoScan(typ, mp) + } mp.mallocing = 1 @@ -2297,34 +2216,26 @@ func mallocgcSmallNoScanSC5(size uintptr, typ *_type, needzero bool) unsafe.Poin if runtimeFreegcEnabled && c.hasReusableNoscan(spc) { - v := mallocgcSmallNoscanReuse(c, span, spc, elemsize, needzero) + x := mallocgcSmallNoscanReuse(c, span, spc, elemsize, needzero) mp.mallocing = 0 releasem(mp) - x := v - { + gp := getg() + if goexperiment.RuntimeSecret && gp.secret > 0 { - gp := getg() - if goexperiment.RuntimeSecret && gp.secret > 0 { - - addSecret(x, size) - } - - if valgrindenabled { - valgrindMalloc(x, size) - } + addSecret(x, size) + } - if gcBlackenEnabled != 0 && elemsize != 0 { - if assistG := getg().m.curg; assistG != nil { - assistG.gcAssistBytes -= int64(elemsize - size) - } + if gcBlackenEnabled != 0 && elemsize != 0 { + if assistG := getg().m.curg; assistG != nil { + assistG.gcAssistBytes -= int64(elemsize - size) } + } - if debug.malloc { - postMallocgcDebug(x, elemsize, typ) - } - return x + if debug.malloc { + postMallocgcDebug(x, elemsize, typ) } + return x } var nextFreeFastResult gclinkptr @@ -2337,9 +2248,7 @@ func mallocgcSmallNoScanSC5(size uintptr, typ *_type, needzero bool) unsafe.Poin span.allocCache >>= uint(theBit + 1) span.freeindex = freeidx span.allocCount++ - nextFreeFastResult = gclinkptr(uintptr(result)* - 48 + - span.base()) + nextFreeFastResult = gclinkptr(uintptr(result)*48 + span.base()) } } } @@ -2348,6 +2257,7 @@ func mallocgcSmallNoScanSC5(size uintptr, typ *_type, needzero bool) unsafe.Poin v, span, checkGCTrigger = c.nextFree(spc) } x := unsafe.Pointer(v) + if needzero && span.needzero != 0 { memclrNoHeapPointers(x, elemsize) } @@ -2380,10 +2290,6 @@ func mallocgcSmallNoScanSC5(size uintptr, typ *_type, needzero bool) unsafe.Poin addSecret(x, size) } - if valgrindenabled { - valgrindMalloc(x, size) - } - if gcBlackenEnabled != 0 && elemsize != 0 { if assistG := getg().m.curg; assistG != nil { assistG.gcAssistBytes -= int64(elemsize - size) @@ -2393,6 +2299,7 @@ func mallocgcSmallNoScanSC5(size uintptr, typ *_type, needzero bool) unsafe.Poin if debug.malloc { postMallocgcDebug(x, elemsize, typ) } + return x } @@ -2417,12 +2324,13 @@ func mallocgcSmallNoScanSC6(size uintptr, typ *_type, needzero bool) unsafe.Poin } const sizeclass = 6 - const elemsize = 64 mp := acquirem() if doubleCheckMalloc { + doubleCheckSmallNoScan(typ, mp) + } mp.mallocing = 1 @@ -2433,34 +2341,26 @@ func mallocgcSmallNoScanSC6(size uintptr, typ *_type, needzero bool) unsafe.Poin if runtimeFreegcEnabled && c.hasReusableNoscan(spc) { - v := mallocgcSmallNoscanReuse(c, span, spc, elemsize, needzero) + x := mallocgcSmallNoscanReuse(c, span, spc, elemsize, needzero) mp.mallocing = 0 releasem(mp) - x := v - { + gp := getg() + if goexperiment.RuntimeSecret && gp.secret > 0 { - gp := getg() - if goexperiment.RuntimeSecret && gp.secret > 0 { - - addSecret(x, size) - } - - if valgrindenabled { - valgrindMalloc(x, size) - } + addSecret(x, size) + } - if gcBlackenEnabled != 0 && elemsize != 0 { - if assistG := getg().m.curg; assistG != nil { - assistG.gcAssistBytes -= int64(elemsize - size) - } + if gcBlackenEnabled != 0 && elemsize != 0 { + if assistG := getg().m.curg; assistG != nil { + assistG.gcAssistBytes -= int64(elemsize - size) } + } - if debug.malloc { - postMallocgcDebug(x, elemsize, typ) - } - return x + if debug.malloc { + postMallocgcDebug(x, elemsize, typ) } + return x } var nextFreeFastResult gclinkptr @@ -2473,9 +2373,7 @@ func mallocgcSmallNoScanSC6(size uintptr, typ *_type, needzero bool) unsafe.Poin span.allocCache >>= uint(theBit + 1) span.freeindex = freeidx span.allocCount++ - nextFreeFastResult = gclinkptr(uintptr(result)* - 64 + - span.base()) + nextFreeFastResult = gclinkptr(uintptr(result)*64 + span.base()) } } } @@ -2484,6 +2382,7 @@ func mallocgcSmallNoScanSC6(size uintptr, typ *_type, needzero bool) unsafe.Poin v, span, checkGCTrigger = c.nextFree(spc) } x := unsafe.Pointer(v) + if needzero && span.needzero != 0 { memclrNoHeapPointers(x, elemsize) } @@ -2516,10 +2415,6 @@ func mallocgcSmallNoScanSC6(size uintptr, typ *_type, needzero bool) unsafe.Poin addSecret(x, size) } - if valgrindenabled { - valgrindMalloc(x, size) - } - if gcBlackenEnabled != 0 && elemsize != 0 { if assistG := getg().m.curg; assistG != nil { assistG.gcAssistBytes -= int64(elemsize - size) @@ -2529,6 +2424,7 @@ func mallocgcSmallNoScanSC6(size uintptr, typ *_type, needzero bool) unsafe.Poin if debug.malloc { postMallocgcDebug(x, elemsize, typ) } + return x } @@ -2553,12 +2449,13 @@ func mallocgcSmallNoScanSC7(size uintptr, typ *_type, needzero bool) unsafe.Poin } const sizeclass = 7 - const elemsize = 80 mp := acquirem() if doubleCheckMalloc { + doubleCheckSmallNoScan(typ, mp) + } mp.mallocing = 1 @@ -2569,34 +2466,26 @@ func mallocgcSmallNoScanSC7(size uintptr, typ *_type, needzero bool) unsafe.Poin if runtimeFreegcEnabled && c.hasReusableNoscan(spc) { - v := mallocgcSmallNoscanReuse(c, span, spc, elemsize, needzero) + x := mallocgcSmallNoscanReuse(c, span, spc, elemsize, needzero) mp.mallocing = 0 releasem(mp) - x := v - { - - gp := getg() - if goexperiment.RuntimeSecret && gp.secret > 0 { - - addSecret(x, size) - } + gp := getg() + if goexperiment.RuntimeSecret && gp.secret > 0 { - if valgrindenabled { - valgrindMalloc(x, size) - } + addSecret(x, size) + } - if gcBlackenEnabled != 0 && elemsize != 0 { - if assistG := getg().m.curg; assistG != nil { - assistG.gcAssistBytes -= int64(elemsize - size) - } + if gcBlackenEnabled != 0 && elemsize != 0 { + if assistG := getg().m.curg; assistG != nil { + assistG.gcAssistBytes -= int64(elemsize - size) } + } - if debug.malloc { - postMallocgcDebug(x, elemsize, typ) - } - return x + if debug.malloc { + postMallocgcDebug(x, elemsize, typ) } + return x } var nextFreeFastResult gclinkptr @@ -2609,9 +2498,7 @@ func mallocgcSmallNoScanSC7(size uintptr, typ *_type, needzero bool) unsafe.Poin span.allocCache >>= uint(theBit + 1) span.freeindex = freeidx span.allocCount++ - nextFreeFastResult = gclinkptr(uintptr(result)* - 80 + - span.base()) + nextFreeFastResult = gclinkptr(uintptr(result)*80 + span.base()) } } } @@ -2620,6 +2507,7 @@ func mallocgcSmallNoScanSC7(size uintptr, typ *_type, needzero bool) unsafe.Poin v, span, checkGCTrigger = c.nextFree(spc) } x := unsafe.Pointer(v) + if needzero && span.needzero != 0 { memclrNoHeapPointers(x, elemsize) } @@ -2652,10 +2540,6 @@ func mallocgcSmallNoScanSC7(size uintptr, typ *_type, needzero bool) unsafe.Poin addSecret(x, size) } - if valgrindenabled { - valgrindMalloc(x, size) - } - if gcBlackenEnabled != 0 && elemsize != 0 { if assistG := getg().m.curg; assistG != nil { assistG.gcAssistBytes -= int64(elemsize - size) @@ -2665,6 +2549,7 @@ func mallocgcSmallNoScanSC7(size uintptr, typ *_type, needzero bool) unsafe.Poin if debug.malloc { postMallocgcDebug(x, elemsize, typ) } + return x } @@ -2689,12 +2574,13 @@ func mallocgcSmallNoScanSC8(size uintptr, typ *_type, needzero bool) unsafe.Poin } const sizeclass = 8 - const elemsize = 96 mp := acquirem() if doubleCheckMalloc { + doubleCheckSmallNoScan(typ, mp) + } mp.mallocing = 1 @@ -2705,34 +2591,26 @@ func mallocgcSmallNoScanSC8(size uintptr, typ *_type, needzero bool) unsafe.Poin if runtimeFreegcEnabled && c.hasReusableNoscan(spc) { - v := mallocgcSmallNoscanReuse(c, span, spc, elemsize, needzero) + x := mallocgcSmallNoscanReuse(c, span, spc, elemsize, needzero) mp.mallocing = 0 releasem(mp) - x := v - { - - gp := getg() - if goexperiment.RuntimeSecret && gp.secret > 0 { + gp := getg() + if goexperiment.RuntimeSecret && gp.secret > 0 { - addSecret(x, size) - } - - if valgrindenabled { - valgrindMalloc(x, size) - } + addSecret(x, size) + } - if gcBlackenEnabled != 0 && elemsize != 0 { - if assistG := getg().m.curg; assistG != nil { - assistG.gcAssistBytes -= int64(elemsize - size) - } + if gcBlackenEnabled != 0 && elemsize != 0 { + if assistG := getg().m.curg; assistG != nil { + assistG.gcAssistBytes -= int64(elemsize - size) } + } - if debug.malloc { - postMallocgcDebug(x, elemsize, typ) - } - return x + if debug.malloc { + postMallocgcDebug(x, elemsize, typ) } + return x } var nextFreeFastResult gclinkptr @@ -2745,9 +2623,7 @@ func mallocgcSmallNoScanSC8(size uintptr, typ *_type, needzero bool) unsafe.Poin span.allocCache >>= uint(theBit + 1) span.freeindex = freeidx span.allocCount++ - nextFreeFastResult = gclinkptr(uintptr(result)* - 96 + - span.base()) + nextFreeFastResult = gclinkptr(uintptr(result)*96 + span.base()) } } } @@ -2756,6 +2632,7 @@ func mallocgcSmallNoScanSC8(size uintptr, typ *_type, needzero bool) unsafe.Poin v, span, checkGCTrigger = c.nextFree(spc) } x := unsafe.Pointer(v) + if needzero && span.needzero != 0 { memclrNoHeapPointers(x, elemsize) } @@ -2788,10 +2665,6 @@ func mallocgcSmallNoScanSC8(size uintptr, typ *_type, needzero bool) unsafe.Poin addSecret(x, size) } - if valgrindenabled { - valgrindMalloc(x, size) - } - if gcBlackenEnabled != 0 && elemsize != 0 { if assistG := getg().m.curg; assistG != nil { assistG.gcAssistBytes -= int64(elemsize - size) @@ -2801,6 +2674,7 @@ func mallocgcSmallNoScanSC8(size uintptr, typ *_type, needzero bool) unsafe.Poin if debug.malloc { postMallocgcDebug(x, elemsize, typ) } + return x } @@ -2825,12 +2699,13 @@ func mallocgcSmallNoScanSC9(size uintptr, typ *_type, needzero bool) unsafe.Poin } const sizeclass = 9 - const elemsize = 112 mp := acquirem() if doubleCheckMalloc { + doubleCheckSmallNoScan(typ, mp) + } mp.mallocing = 1 @@ -2841,34 +2716,26 @@ func mallocgcSmallNoScanSC9(size uintptr, typ *_type, needzero bool) unsafe.Poin if runtimeFreegcEnabled && c.hasReusableNoscan(spc) { - v := mallocgcSmallNoscanReuse(c, span, spc, elemsize, needzero) + x := mallocgcSmallNoscanReuse(c, span, spc, elemsize, needzero) mp.mallocing = 0 releasem(mp) - x := v - { + gp := getg() + if goexperiment.RuntimeSecret && gp.secret > 0 { - gp := getg() - if goexperiment.RuntimeSecret && gp.secret > 0 { - - addSecret(x, size) - } - - if valgrindenabled { - valgrindMalloc(x, size) - } + addSecret(x, size) + } - if gcBlackenEnabled != 0 && elemsize != 0 { - if assistG := getg().m.curg; assistG != nil { - assistG.gcAssistBytes -= int64(elemsize - size) - } + if gcBlackenEnabled != 0 && elemsize != 0 { + if assistG := getg().m.curg; assistG != nil { + assistG.gcAssistBytes -= int64(elemsize - size) } + } - if debug.malloc { - postMallocgcDebug(x, elemsize, typ) - } - return x + if debug.malloc { + postMallocgcDebug(x, elemsize, typ) } + return x } var nextFreeFastResult gclinkptr @@ -2881,9 +2748,7 @@ func mallocgcSmallNoScanSC9(size uintptr, typ *_type, needzero bool) unsafe.Poin span.allocCache >>= uint(theBit + 1) span.freeindex = freeidx span.allocCount++ - nextFreeFastResult = gclinkptr(uintptr(result)* - 112 + - span.base()) + nextFreeFastResult = gclinkptr(uintptr(result)*112 + span.base()) } } } @@ -2892,6 +2757,7 @@ func mallocgcSmallNoScanSC9(size uintptr, typ *_type, needzero bool) unsafe.Poin v, span, checkGCTrigger = c.nextFree(spc) } x := unsafe.Pointer(v) + if needzero && span.needzero != 0 { memclrNoHeapPointers(x, elemsize) } @@ -2924,10 +2790,6 @@ func mallocgcSmallNoScanSC9(size uintptr, typ *_type, needzero bool) unsafe.Poin addSecret(x, size) } - if valgrindenabled { - valgrindMalloc(x, size) - } - if gcBlackenEnabled != 0 && elemsize != 0 { if assistG := getg().m.curg; assistG != nil { assistG.gcAssistBytes -= int64(elemsize - size) @@ -2937,6 +2799,7 @@ func mallocgcSmallNoScanSC9(size uintptr, typ *_type, needzero bool) unsafe.Poin if debug.malloc { postMallocgcDebug(x, elemsize, typ) } + return x } @@ -2961,12 +2824,13 @@ func mallocgcSmallNoScanSC10(size uintptr, typ *_type, needzero bool) unsafe.Poi } const sizeclass = 10 - const elemsize = 128 mp := acquirem() if doubleCheckMalloc { + doubleCheckSmallNoScan(typ, mp) + } mp.mallocing = 1 @@ -2977,34 +2841,26 @@ func mallocgcSmallNoScanSC10(size uintptr, typ *_type, needzero bool) unsafe.Poi if runtimeFreegcEnabled && c.hasReusableNoscan(spc) { - v := mallocgcSmallNoscanReuse(c, span, spc, elemsize, needzero) + x := mallocgcSmallNoscanReuse(c, span, spc, elemsize, needzero) mp.mallocing = 0 releasem(mp) - x := v - { - - gp := getg() - if goexperiment.RuntimeSecret && gp.secret > 0 { - - addSecret(x, size) - } + gp := getg() + if goexperiment.RuntimeSecret && gp.secret > 0 { - if valgrindenabled { - valgrindMalloc(x, size) - } + addSecret(x, size) + } - if gcBlackenEnabled != 0 && elemsize != 0 { - if assistG := getg().m.curg; assistG != nil { - assistG.gcAssistBytes -= int64(elemsize - size) - } + if gcBlackenEnabled != 0 && elemsize != 0 { + if assistG := getg().m.curg; assistG != nil { + assistG.gcAssistBytes -= int64(elemsize - size) } + } - if debug.malloc { - postMallocgcDebug(x, elemsize, typ) - } - return x + if debug.malloc { + postMallocgcDebug(x, elemsize, typ) } + return x } var nextFreeFastResult gclinkptr @@ -3017,9 +2873,7 @@ func mallocgcSmallNoScanSC10(size uintptr, typ *_type, needzero bool) unsafe.Poi span.allocCache >>= uint(theBit + 1) span.freeindex = freeidx span.allocCount++ - nextFreeFastResult = gclinkptr(uintptr(result)* - 128 + - span.base()) + nextFreeFastResult = gclinkptr(uintptr(result)*128 + span.base()) } } } @@ -3028,6 +2882,7 @@ func mallocgcSmallNoScanSC10(size uintptr, typ *_type, needzero bool) unsafe.Poi v, span, checkGCTrigger = c.nextFree(spc) } x := unsafe.Pointer(v) + if needzero && span.needzero != 0 { memclrNoHeapPointers(x, elemsize) } @@ -3060,10 +2915,6 @@ func mallocgcSmallNoScanSC10(size uintptr, typ *_type, needzero bool) unsafe.Poi addSecret(x, size) } - if valgrindenabled { - valgrindMalloc(x, size) - } - if gcBlackenEnabled != 0 && elemsize != 0 { if assistG := getg().m.curg; assistG != nil { assistG.gcAssistBytes -= int64(elemsize - size) @@ -3073,5 +2924,6 @@ func mallocgcSmallNoScanSC10(size uintptr, typ *_type, needzero bool) unsafe.Poi if debug.malloc { postMallocgcDebug(x, elemsize, typ) } + return x } diff --git a/src/runtime/malloc_stubs.go b/src/runtime/malloc_stubs.go index e77c7bd00a8077..5456bbe9f9a193 100644 --- a/src/runtime/malloc_stubs.go +++ b/src/runtime/malloc_stubs.go @@ -10,11 +10,11 @@ // To generate the specialized mallocgc functions, do 'go run .' inside runtime/_mkmalloc. // // To assemble a mallocgc function, the mallocStub function is cloned, and the call to -// inlinedMalloc is replaced with the inlined body of smallScanNoHeaderStub, -// smallNoScanStub or tinyStub, depending on the parameters being specialized. +// inlinedMalloc is replaced with the inlined body of smallStub or tinyStub, +// depending on the parameters being specialized. // -// The size_ (for the tiny case) and elemsize_, sizeclass_, and noscanint_ (for all three cases) -// identifiers are replaced with the value of the parameter in the specialized case. +// The size_ (for the tiny case) and elemsize_, sizeclass_, noscanint_, and isNoScan_ (for all +// three cases) identifiers are replaced with the value of the parameter in the specialized case. // The nextFreeFastStub, nextFreeFastTiny, heapSetTypeNoHeaderStub, and writeHeapBitsSmallStub // functions are also inlined by _mkmalloc. @@ -36,6 +36,7 @@ import ( const elemsize_ = 8 const sizeclass_ = 0 const noscanint_ = 0 +const isNoScan_ = false const size_ = 0 const isTiny_ = false @@ -94,8 +95,10 @@ func mallocStub(size uintptr, typ *_type, needzero bool) unsafe.Pointer { } // Actually do the allocation. - x, elemsize := inlinedMalloc(size, typ, needzero) + return inlinedMalloc(size, typ, needzero) +} +func postMallocgc(x unsafe.Pointer, typ *_type, size uintptr, elemsize uintptr) { if !isTiny_ { gp := getg() if goexperiment.RuntimeSecret && gp.secret > 0 { @@ -105,13 +108,6 @@ func mallocStub(size uintptr, typ *_type, needzero bool) unsafe.Pointer { } } - // Notify valgrind, if enabled. - // To allow the compiler to not know about valgrind, we do valgrind instrumentation - // unlike the other sanitizers. - if valgrindenabled { - valgrindMalloc(x, size) - } - // Adjust our GC assist debt to account for internal fragmentation. if gcBlackenEnabled != 0 && elemsize != 0 { if assistG := getg().m.curg; assistG != nil { @@ -123,7 +119,6 @@ func mallocStub(size uintptr, typ *_type, needzero bool) unsafe.Pointer { if debug.malloc { postMallocgcDebug(x, elemsize, typ) } - return x } // deductAssistCredit reduces the current G's GC assist credit @@ -146,11 +141,11 @@ func deductAssistCredit(size uintptr) { // inlinedMalloc will never be called. It is defined just so that the compiler can compile // the mallocStub function, which will also never be called, but instead used as a template // to generate a size-specialized malloc function. The call to inlinedMalloc in mallocStub -// will be replaced with the inlined body of smallScanNoHeaderStub, smallNoScanStub, or tinyStub -// when generating the size-specialized malloc function. See the comment at the top of this -// file for more information. -func inlinedMalloc(size uintptr, typ *_type, needzero bool) (unsafe.Pointer, uintptr) { - return unsafe.Pointer(uintptr(0)), 0 +// will be replaced with the inlined body of smallStub or tinyStub when generating the +// size-specialized malloc function. See the comment at the top of this file for more +// information. +func inlinedMalloc(size uintptr, typ *_type, needzero bool) unsafe.Pointer { + return unsafe.Pointer(uintptr(0)) } func doubleCheckSmallScanNoHeader(size uintptr, typ *_type, mp *m) { @@ -168,14 +163,19 @@ func doubleCheckSmallScanNoHeader(size uintptr, typ *_type, mp *m) { } } -func smallScanNoHeaderStub(size uintptr, typ *_type, needzero bool) (unsafe.Pointer, uintptr) { +func smallStub(size uintptr, typ *_type, needzero bool) unsafe.Pointer { const sizeclass = sizeclass_ const elemsize = elemsize_ // Set mp.mallocing to keep from being preempted by GC. mp := acquirem() if doubleCheckMalloc { - doubleCheckSmallScanNoHeader(size, typ, mp) + if isNoScan_ { + doubleCheckSmallNoScan(typ, mp) + } + if !isNoScan_ { + doubleCheckSmallScanNoHeader(size, typ, mp) + } } mp.mallocing = 1 @@ -183,23 +183,51 @@ func smallScanNoHeaderStub(size uintptr, typ *_type, needzero bool) (unsafe.Poin c := getMCache(mp) const spc = spanClass(sizeclass<<1) | spanClass(noscanint_) span := c.alloc[spc] + + if isNoScan_ { + // First, check for a reusable object. + if runtimeFreegcEnabled && c.hasReusableNoscan(spc) { + // We have a reusable object, use it. + x := mallocgcSmallNoscanReuse(c, span, spc, elemsize, needzero) + mp.mallocing = 0 + releasem(mp) + + // TODO(thepudds): note that the generated return path is essentially duplicated + // by the generator. For example, see the two postMallocgcDebug calls and + // related duplicated code on the return path currently in the generated + // mallocgcSmallNoScanSC2 function. One set of those correspond to this + // return here. We might be able to de-duplicate the generated return path + // by updating the generator, perhaps by jumping to a shared return or similar. + postMallocgc(x, typ, size, elemsize) + + return x + } + } + v := nextFreeFastStub(span) if v == 0 { v, span, checkGCTrigger = c.nextFree(spc) } x := unsafe.Pointer(v) - if span.needzero != 0 { - memclrNoHeapPointers(x, elemsize) + if isNoScan_ { + if needzero && span.needzero != 0 { + memclrNoHeapPointers(x, elemsize) + } } - if goarch.PtrSize == 8 && sizeclass == 1 { - // initHeapBits already set the pointer bits for the 8-byte sizeclass - // on 64-bit platforms. - c.scanAlloc += 8 - } else { - dataSize := size // make the inliner happy - x := uintptr(x) - scanSize := heapSetTypeNoHeaderStub(x, dataSize, typ, span) - c.scanAlloc += scanSize + if !isNoScan_ { + if span.needzero != 0 { + memclrNoHeapPointers(x, elemsize) + } + if goarch.PtrSize == 8 && sizeclass == 1 { + // initHeapBits already set the pointer bits for the 8-byte sizeclass + // on 64-bit platforms. + c.scanAlloc += 8 + } else { + dataSize := size // make the inliner happy + x := uintptr(x) + scanSize := heapSetTypeNoHeaderStub(x, dataSize, typ, span) + c.scanAlloc += scanSize + } } // Ensure that the stores above that initialize x to @@ -250,7 +278,9 @@ func smallScanNoHeaderStub(size uintptr, typ *_type, needzero bool) (unsafe.Poin } } - return x, elemsize + postMallocgc(x, typ, size, elemsize) + + return x } func doubleCheckSmallNoScan(typ *_type, mp *m) { @@ -265,102 +295,6 @@ func doubleCheckSmallNoScan(typ *_type, mp *m) { } } -func smallNoScanStub(size uintptr, typ *_type, needzero bool) (unsafe.Pointer, uintptr) { - // TODO(matloob): Add functionality to mkmalloc to allow us to inline a non-constant - // sizeclass_ and elemsize_ value (instead just set to the expressions to look up the size class - // and elemsize. We'd also need to teach mkmalloc that values that are touched by these (specifically - // spc below) should turn into vars. This would allow us to generate mallocgcSmallNoScan itself, - // so that its code could not diverge from the generated functions. - const sizeclass = sizeclass_ - const elemsize = elemsize_ - - // Set mp.mallocing to keep from being preempted by GC. - mp := acquirem() - if doubleCheckMalloc { - doubleCheckSmallNoScan(typ, mp) - } - mp.mallocing = 1 - - checkGCTrigger := false - c := getMCache(mp) - const spc = spanClass(sizeclass<<1) | spanClass(noscanint_) - span := c.alloc[spc] - - // First, check for a reusable object. - if runtimeFreegcEnabled && c.hasReusableNoscan(spc) { - // We have a reusable object, use it. - v := mallocgcSmallNoscanReuse(c, span, spc, elemsize, needzero) - mp.mallocing = 0 - releasem(mp) - - // TODO(thepudds): note that the generated return path is essentially duplicated - // by the generator. For example, see the two postMallocgcDebug calls and - // related duplicated code on the return path currently in the generated - // mallocgcSmallNoScanSC2 function. One set of those correspond to this - // return here. We might be able to de-duplicate the generated return path - // by updating the generator, perhaps by jumping to a shared return or similar. - return v, elemsize - } - - v := nextFreeFastStub(span) - if v == 0 { - v, span, checkGCTrigger = c.nextFree(spc) - } - x := unsafe.Pointer(v) - if needzero && span.needzero != 0 { - memclrNoHeapPointers(x, elemsize) - } - - // Ensure that the stores above that initialize x to - // type-safe memory and set the heap bits occur before - // the caller can make x observable to the garbage - // collector. Otherwise, on weakly ordered machines, - // the garbage collector could follow a pointer to x, - // but see uninitialized memory or stale heap bits. - publicationBarrier() - - if writeBarrier.enabled { - // Allocate black during GC. - // All slots hold nil so no scanning is needed. - // This may be racing with GC so do it atomically if there can be - // a race marking the bit. - gcmarknewobject(span, uintptr(x)) - } else { - // Track the last free index before the mark phase. This field - // is only used by the garbage collector. During the mark phase - // this is used by the conservative scanner to filter out objects - // that are both free and recently-allocated. It's safe to do that - // because we allocate-black if the GC is enabled. The conservative - // scanner produces pointers out of thin air, so without additional - // synchronization it might otherwise observe a partially-initialized - // object, which could crash the program. - span.freeIndexForScan = span.freeindex - } - - // Note cache c only valid while m acquired; see #47302 - // - // N.B. Use the full size because that matches how the GC - // will update the mem profile on the "free" side. - // - // TODO(mknyszek): We should really count the header as part - // of gc_sys or something. The code below just pretends it is - // internal fragmentation and matches the GC's accounting by - // using the whole allocation slot. - c.nextSample -= int64(elemsize) - if c.nextSample < 0 || MemProfileRate != c.memProfRate { - profilealloc(mp, x, elemsize) - } - mp.mallocing = 0 - releasem(mp) - - if checkGCTrigger { - if t := (gcTrigger{kind: gcTriggerHeap}); t.test() { - gcStart(t) - } - } - return x, elemsize -} - func doubleCheckTiny(size uintptr, typ *_type, mp *m) { if mp.mallocing != 0 { throw("malloc deadlock") @@ -373,7 +307,7 @@ func doubleCheckTiny(size uintptr, typ *_type, mp *m) { } } -func tinyStub(size uintptr, typ *_type, needzero bool) (unsafe.Pointer, uintptr) { +func tinyStub(size uintptr, typ *_type, needzero bool) unsafe.Pointer { const elemsize = elemsize_ // Set mp.mallocing to keep from being preempted by GC. @@ -437,7 +371,9 @@ func tinyStub(size uintptr, typ *_type, needzero bool) (unsafe.Pointer, uintptr) c.tinyAllocs++ mp.mallocing = 0 releasem(mp) - return x, 0 + const elemsize = 0 + postMallocgc(x, typ, size, elemsize) + return x } // Allocate a new maxTinySize block. checkGCTrigger := false @@ -505,22 +441,9 @@ func tinyStub(size uintptr, typ *_type, needzero bool) (unsafe.Pointer, uintptr) } } - if raceenabled { - // Pad tinysize allocations so they are aligned with the end - // of the tinyalloc region. This ensures that any arithmetic - // that goes off the top end of the object will be detectable - // by checkptr (issue 38872). - // Note that we disable tinyalloc when raceenabled for this to work. - // TODO: This padding is only performed when the race detector - // is enabled. It would be nice to enable it if any package - // was compiled with checkptr, but there's no easy way to - // detect that (especially at compile time). - // TODO: enable this padding for all allocations, not just - // tinyalloc ones. It's tricky because of pointer maps. - // Maybe just all noscan objects? - x = add(x, elemsize-size) - } - return x, elemsize + postMallocgc(x, typ, size, elemsize) + + return x } // TODO(matloob): Should we let the go compiler inline this instead of using mkmalloc? diff --git a/test/genmeth.go b/test/genmeth.go index c37ea60d430132..85c6402e7c2be2 100644 --- a/test/genmeth.go +++ b/test/genmeth.go @@ -1,4 +1,4 @@ -// run +// run -goexperiment genericmethods // Copyright 2026 The Go Authors. All rights reserved. // Use of this source code is governed by a BSD-style diff --git a/test/genmeth1.go b/test/genmeth1.go index 1963f5b4acdb54..2a2207b8df146b 100644 --- a/test/genmeth1.go +++ b/test/genmeth1.go @@ -1,4 +1,4 @@ -// run +// run -goexperiment genericmethods // Copyright 2026 The Go Authors. All rights reserved. // Use of this source code is governed by a BSD-style diff --git a/test/genmeth2.go b/test/genmeth2.go index a67181b275f836..db3118859c4c27 100644 --- a/test/genmeth2.go +++ b/test/genmeth2.go @@ -1,4 +1,4 @@ -// run +// run -goexperiment genericmethods // Copyright 2026 The Go Authors. All rights reserved. // Use of this source code is governed by a BSD-style