feat: more complete function loader (bytedance#354)

* follow complete implementation of go symtab
* support go1.20

.github/workflows/push-check-go118.yml

@@ -25,8 +25,8 @@ jobs:
 
       - name: Unit Test
         run: |
-          GOMAXPROCS=4 go test -v -gcflags=-d=checkptr=0 -race ./...
-          GOMAXPROCS=4 go test -v -gcflags=-d=checkptr=0 -race ./external_jsonlib_test/...
+          GOMAXPROCS=4 go test -v -gcflags=-d=checkptr=0 ./...
+          GOMAXPROCS=4 go test -v -gcflags=-d=checkptr=0 ./external_jsonlib_test/...
       
       - name: Generic Test
-        run: GOMAXPROCS=4 go test -v -gcflags=-d=checkptr=0 -race ./generic_test
+        run: GOMAXPROCS=4 go test -v -gcflags=-d=checkptr=0 ./generic_test

.github/workflows/push-check-linux-arm64.yml

@@ -6,7 +6,7 @@ jobs:
   build:
     strategy:
       matrix:
-        go-version: [1.15.x, 1.19.x]
+        go-version: [1.15.x, 1.20.x]
         os: [arm]
     runs-on: ${{ matrix.os }}
     steps:

.github/workflows/push-check-linux-x64.yml

@@ -6,7 +6,7 @@ jobs:
   build:
     strategy:
       matrix:
-        go-version: [1.15.x, 1.16.x, 1.17.x, 1.19.x]
+        go-version: [1.15.x, 1.16.x, 1.17.x, 1.19.x, 1.20.x]
     runs-on: [self-hosted, X64]
     steps:
       - name: Clear repository

.github/workflows/push-check-qemu.yml

@@ -6,7 +6,7 @@ jobs:
   build:
     strategy:
       matrix:
-        go-version: [1.15.x, 1.19.x]
+        go-version: [1.15.x, 1.20.x]
         os: [arm]
     runs-on: ${{ matrix.os }}
     steps:

.github/workflows/push-check-windows.yml

@@ -6,7 +6,7 @@ jobs:
   build:
     strategy:
       matrix:
-        go-version: [1.15.x, 1.19.x]
+        go-version: [1.15.x, 1.20.x]
     runs-on: windows-latest
     steps:
       - uses: actions/checkout@v2

ast/api_amd64_test.go

@@ -10,7 +10,8 @@ import (
     `github.com/stretchr/testify/assert`
 )
 
-func TestSortNodeTwitter(t *testing.T) {root, err := NewSearcher(_TwitterJson).GetByPath()
+func TestSortNodeTwitter(t *testing.T) {
+    root, err := NewSearcher(_TwitterJson).GetByPath()
     if err != nil {
         t.Fatal(err)
     }

ast/encode.go

@@ -19,6 +19,8 @@ package ast
 import (
     `sync`
     `unicode/utf8`
+
+    `github.com/bytedance/sonic/internal/rt`
 )
 
 const (
@@ -163,7 +165,7 @@ func (self *Node) encodeFalse(buf *[]byte) error {
 }
 
 func (self *Node) encodeNumber(buf *[]byte) error {
-    str := addr2str(self.p, self.v)
+    str := rt.StrFrom(self.p, self.v)
     *buf = append(*buf, str...)
     return nil
 }
@@ -174,7 +176,7 @@ func (self *Node) encodeString(buf *[]byte) error {
         return nil
     }
 
-    quote(buf, addr2str(self.p, self.v))
+    quote(buf, rt.StrFrom(self.p, self.v))
     return nil
 }
 

ast/node.go

@@ -21,6 +21,7 @@ import (
     `fmt`
     `strconv`
     `unsafe`
+    `reflect`
 
     `github.com/bytedance/sonic/internal/native/types`
     `github.com/bytedance/sonic/internal/rt`
@@ -60,6 +61,10 @@ const (
     V_ANY    = int(_V_ANY)
 )
 
+var (
+    byteType = rt.UnpackType(reflect.TypeOf(byte(0)))
+)
+
 type Node struct {
     v int64
     t types.ValueType
@@ -151,7 +156,7 @@ func (self *Node) Raw() (string, error) {
         buf, err := self.MarshalJSON()
         return rt.Mem2Str(buf), err
     }
-    return addr2str(self.p, self.v), nil
+    return rt.StrFrom(self.p, self.v), nil
 }
 
 func (self *Node) checkRaw() error {
@@ -183,7 +188,7 @@ func (self *Node) Bool() (bool, error) {
             } else {
                 return false, err
             }
-        case types.V_STRING: return strconv.ParseBool(addr2str(self.p, self.v))
+        case types.V_STRING: return strconv.ParseBool(rt.StrFrom(self.p, self.v))
         case _V_ANY        :   
             any := self.packAny()     
             switch v := any.(type) {
@@ -389,7 +394,7 @@ func (self *Node) String() (string, error) {
         case types.V_NULL    : return "" , nil
         case types.V_TRUE    : return "true" , nil
         case types.V_FALSE   : return "false", nil
-        case types.V_STRING, _V_NUMBER  : return addr2str(self.p, self.v), nil
+        case types.V_STRING, _V_NUMBER  : return rt.StrFrom(self.p, self.v), nil
         case _V_ANY          :        
         any := self.packAny()
         switch v := any.(type) {
@@ -421,7 +426,7 @@ func (self *Node) StrictString() (string, error) {
         return "", err
     }
     switch self.t {
-        case types.V_STRING  : return addr2str(self.p, self.v), nil
+        case types.V_STRING  : return rt.StrFrom(self.p, self.v), nil
         case _V_ANY          :        
             if v, ok := self.packAny().(string); ok {
                 return v, nil
@@ -836,7 +841,7 @@ func (self *Node) UnsafeMap() ([]Pair, error) {
     if err := self.skipAllKey(); err != nil {
         return nil, err
     }
-    s := ptr2slice(self.p, int(self.len()), self.cap())
+    s := rt.Ptr2SlicePtr(self.p, int(self.len()), self.cap())
     return *(*[]Pair)(s), nil
 }
 
@@ -935,7 +940,7 @@ func (self *Node) UnsafeArray() ([]Node, error) {
     if err := self.skipAllIndex(); err != nil {
         return nil, err
     }
-    s := ptr2slice(self.p, self.len(), self.cap())
+    s := rt.Ptr2SlicePtr(self.p, self.len(), self.cap())
     return *(*[]Node)(s), nil
 }
 
@@ -953,7 +958,7 @@ func (self *Node) Interface() (interface{}, error) {
         case types.V_FALSE   : return false, nil
         case types.V_ARRAY   : return self.toGenericArray()
         case types.V_OBJECT  : return self.toGenericObject()
-        case types.V_STRING  : return addr2str(self.p, self.v), nil
+        case types.V_STRING  : return rt.StrFrom(self.p, self.v), nil
         case _V_NUMBER       : 
             v, err := numberToFloat64(self)
             if err != nil {
@@ -997,7 +1002,7 @@ func (self *Node) InterfaceUseNumber() (interface{}, error) {
         case types.V_FALSE   : return false, nil
         case types.V_ARRAY   : return self.toGenericArrayUseNumber()
         case types.V_OBJECT  : return self.toGenericObjectUseNumber()
-        case types.V_STRING  : return addr2str(self.p, self.v), nil
+        case types.V_STRING  : return rt.StrFrom(self.p, self.v), nil
         case _V_NUMBER       : return toNumber(self), nil
         case _V_ARRAY_LAZY   :
             if err := self.loadAllIndex(); err != nil {
@@ -1597,13 +1602,13 @@ func NewBool(v bool) Node {
 func NewNumber(v string) Node {
     return Node{
         v: int64(len(v) & _LEN_MASK),
-        p: str2ptr(v),
+        p: rt.StrPtr(v),
         t: _V_NUMBER,
     }
 }
 
 func toNumber(node *Node) json.Number {
-    return json.Number(addr2str(node.p, node.v))
+    return json.Number(rt.StrFrom(node.p, node.v))
 }
 
 func numberToFloat64(node *Node) (float64, error) {
@@ -1637,7 +1642,7 @@ func newBytes(v []byte) Node {
 func NewString(v string) Node {
     return Node{
         t: types.V_STRING,
-        p: str2ptr(v),
+        p: rt.StrPtr(v),
         v: int64(len(v) & _LEN_MASK),
     }
 }
@@ -1727,13 +1732,13 @@ func (self *Node) setLazyObject(p *Parser, v []Pair) {
 func newRawNode(str string, typ types.ValueType) Node {
     return Node{
         t: _V_RAW | typ,
-        p: str2ptr(str),
+        p: rt.StrPtr(str),
         v: int64(len(str) & _LEN_MASK),
     }
 }
 
 func (self *Node) parseRaw(full bool) {
-    raw := addr2str(self.p, self.v)
+    raw := rt.StrFrom(self.p, self.v)
     parser := NewParser(raw)
     if full {
         parser.noLazy = true

ast/stubs.go → ast/stubs_go115.go

@@ -1,3 +1,5 @@
+// +build go1.15,!go1.20
+
 /*
  * Copyright 2021 ByteDance Inc.
  *
@@ -18,16 +20,11 @@ package ast
 
 import (
     `unsafe`
-    `reflect`
     `unicode/utf8`
 
     `github.com/bytedance/sonic/internal/rt`
 )
 
-var (
-    byteType = rt.UnpackType(reflect.TypeOf(byte(0)))
-)
-
 //go:noescape
 //go:linkname memmove runtime.memmove
 //goland:noinspection GoUnusedParameter
@@ -46,29 +43,6 @@ func mem2ptr(s []byte) unsafe.Pointer {
     return (*rt.GoSlice)(unsafe.Pointer(&s)).Ptr
 }
 
-//go:nosplit
-func ptr2slice(s unsafe.Pointer, l int, c int) unsafe.Pointer {
-    slice := &rt.GoSlice{
-        Ptr: s,
-        Len: l,
-        Cap: c,
-    }
-    return unsafe.Pointer(slice)
-}
-
-//go:nosplit
-func str2ptr(s string) unsafe.Pointer {
-    return (*rt.GoString)(unsafe.Pointer(&s)).Ptr
-}
-
-//go:nosplit
-func addr2str(p unsafe.Pointer, n int64) (s string) {
-    (*rt.GoString)(unsafe.Pointer(&s)).Ptr = p
-    (*rt.GoString)(unsafe.Pointer(&s)).Len = int(n)
-    return
-}
-
-
 var (
     //go:linkname safeSet encoding/json.safeSet
     safeSet [utf8.RuneSelf]bool

ast/stubs_go120.go

@@ -0,0 +1,55 @@
+// +build go1.20
+
+/*
+ * Copyright 2021 ByteDance Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package ast
+
+import (
+    `unsafe`
+    `unicode/utf8`
+
+    `github.com/bytedance/sonic/internal/rt`
+)
+
+//go:noescape
+//go:linkname memmove runtime.memmove
+//goland:noinspection GoUnusedParameter
+func memmove(to unsafe.Pointer, from unsafe.Pointer, n uintptr)
+
+//go:linkname unsafe_NewArray reflect.unsafe_NewArray
+//goland:noinspection GoUnusedParameter
+func unsafe_NewArray(typ *rt.GoType, n int) unsafe.Pointer
+
+//go:linkname growslice reflect.growslice
+//goland:noinspection GoUnusedParameter
+func growslice(et *rt.GoType, old rt.GoSlice, cap int) rt.GoSlice
+
+//go:nosplit
+func mem2ptr(s []byte) unsafe.Pointer {
+    return (*rt.GoSlice)(unsafe.Pointer(&s)).Ptr
+}
+
+var (
+    //go:linkname safeSet encoding/json.safeSet
+    safeSet [utf8.RuneSelf]bool
+
+    //go:linkname hex encoding/json.hex
+    hex string
+)
+
+//go:linkname unquoteBytes encoding/json.unquoteBytes
+func unquoteBytes(s []byte) (t []byte, ok bool)

decoder/assembler_amd64_go116.go

@@ -215,6 +215,7 @@ var (
 type _Assembler struct {
     jit.BaseAssembler
     p _Program
+    name string
 }
 
 func newAssembler(p _Program) *_Assembler {
@@ -224,7 +225,7 @@ func newAssembler(p _Program) *_Assembler {
 /** Assembler Interface **/
 
 func (self *_Assembler) Load() _Decoder {
-    return ptodec(self.BaseAssembler.LoadWithFaker("json_decoder", _FP_size, _FP_args, _Decoder_Shadow))
+    return ptodec(self.BaseAssembler.Load("decode_"+self.name, _FP_size, _FP_args, argPtrs, localPtrs))
 }
 
 func (self *_Assembler) Init(p _Program) *_Assembler {