PHI/webrtc
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

first working

Browse Source
This commit is contained in:
kaedwen
2023-07-04 19:53:26 +02:00
parent b7abd95296
commit 957bf18721
2528 changed files with 787399 additions and 155 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.vscode/launch.json
Vendored
+3 -1
View File
@@ -11,7 +11,9 @@
"mode": "auto",
"program": "${workspaceFolder}/main.go",
"env": {
"HTTP_STATIC": "${workspaceFolder}/static/dist"
"HTTP_STATIC": "${workspaceFolder}/static/dist",
"VIDEO_DEVICE": "/dev/video4",
"AUDIO_DEVICE": "plughw:0,1,0"
}
},
{
example/example-webrtc-applications
-1
Submodule example/example-webrtc-applications deleted from 0a280dd1f9
go.mod
+6 -6
View File
@@ -1,22 +1,25 @@
module gitea.heinrich.blue/PHI/webrtc-gst
module github.com/kaedwen/webrtc
go 1.20
require (
github.com/alexflint/go-arg v1.4.3
github.com/gin-gonic/gin v1.9.1
github.com/pion/rtcp v1.2.10
github.com/pion/webrtc/v3 v3.2.10
github.com/tinyzimmer/go-glib v0.0.25
github.com/tinyzimmer/go-gst v0.2.33
go.uber.org/zap v1.24.0
nhooyr.io/websocket v1.8.7
)
require (
github.com/alexflint/go-arg v1.4.3 // indirect
github.com/alexflint/go-scalar v1.1.0 // indirect
github.com/bytedance/sonic v1.9.2 // indirect
github.com/chenzhuoyu/base64x v0.0.0-20221115062448-fe3a3abad311 // indirect
github.com/davecgh/go-spew v1.1.1 // indirect
github.com/gabriel-vasile/mimetype v1.4.2 // indirect
github.com/gin-contrib/sse v0.1.0 // indirect
github.com/gin-gonic/gin v1.9.1 // indirect
github.com/go-playground/locales v0.14.1 // indirect
github.com/go-playground/universal-translator v0.18.1 // indirect
github.com/go-playground/validator/v10 v10.14.1 // indirect
@@ -38,7 +41,6 @@ require (
github.com/pion/logging v0.2.2 // indirect
github.com/pion/mdns v0.0.7 // indirect
github.com/pion/randutil v0.1.0 // indirect
github.com/pion/rtcp v1.2.10 // indirect
github.com/pion/rtp v1.7.13 // indirect
github.com/pion/sctp v1.8.7 // indirect
github.com/pion/sdp/v3 v3.0.6 // indirect
@@ -46,7 +48,6 @@ require (
github.com/pion/stun v0.6.0 // indirect
github.com/pion/transport/v2 v2.2.1 // indirect
github.com/pion/turn/v2 v2.1.0 // indirect
github.com/pion/webrtc/v3 v3.2.10 // indirect
github.com/pmezard/go-difflib v1.0.0 // indirect
github.com/stretchr/testify v1.8.4 // indirect
github.com/twitchyliquid64/golang-asm v0.15.1 // indirect
@@ -60,5 +61,4 @@ require (
golang.org/x/text v0.10.0 // indirect
google.golang.org/protobuf v1.31.0 // indirect
gopkg.in/yaml.v3 v3.0.1 // indirect
nhooyr.io/websocket v1.8.7 // indirect
)
go.sum
+11 -5
View File
@@ -22,6 +22,7 @@ github.com/gin-gonic/gin v1.6.3/go.mod h1:75u5sXoLsGZoRN5Sgbi1eraJ4GU3++wFwWzhwv
github.com/gin-gonic/gin v1.9.1 h1:4idEAncQnU5cB7BeOkPtxjfCSye0AAm1R0RVIqJ+Jmg=
github.com/gin-gonic/gin v1.9.1/go.mod h1:hPrL7YrpYKXt5YId3A/Tnip5kqbEAP+KLuI3SUcPTeU=
github.com/go-playground/assert/v2 v2.0.1/go.mod h1:VDjEfimB/XKnb+ZQfWdccd7VUvScMdVu0Titje2rxJ4=
github.com/go-playground/assert/v2 v2.2.0 h1:JvknZsQTYeFEAhQwI4qEt9cyV5ONwRHC+lYKSsYSR8s=
github.com/go-playground/locales v0.13.0/go.mod h1:taPMhCMXrRLJO55olJkUXHZBHCxTMfnGwq/HNwmWNS8=
github.com/go-playground/locales v0.14.1 h1:EWaQ/wswjilfKLTECiXz7Rh+3BjFhfDFKv/oXslEjJA=
github.com/go-playground/locales v0.14.1/go.mod h1:hxrqLVvrK65+Rwrd5Fc6F2O76J/NuW9t0sjnWqG1slY=
@@ -32,8 +33,11 @@ github.com/go-playground/validator/v10 v10.2.0/go.mod h1:uOYAAleCW8F/7oMFd6aG0GO
github.com/go-playground/validator/v10 v10.14.1 h1:9c50NUPC30zyuKprjL3vNZ0m5oG+jU0zvx4AqHGnv4k=
github.com/go-playground/validator/v10 v10.14.1/go.mod h1:9iXMNT7sEkjXb0I+enO7QXmzG6QCsPWY4zveKFVRSyU=
github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0/go.mod h1:fyg7847qk6SyHyPtNmDHnmrv/HOrqktSC+C9fM+CJOE=
github.com/gobwas/httphead v0.0.0-20180130184737-2c6c146eadee h1:s+21KNqlpePfkah2I+gwHF8xmJWRjooY+5248k6m4A0=
github.com/gobwas/httphead v0.0.0-20180130184737-2c6c146eadee/go.mod h1:L0fX3K22YWvt/FAX9NnzrNzcI4wNYi9Yku4O0LKYflo=
github.com/gobwas/pool v0.2.0 h1:QEmUOlnSjWtnpRGHF3SauEiOsy82Cup83Vf2LcMlnc8=
github.com/gobwas/pool v0.2.0/go.mod h1:q8bcK0KcYlCgd9e7WYLm9LpyS+YeLd8JVDW6WezmKEw=
github.com/gobwas/ws v1.0.2 h1:CoAavW/wd/kulfZmSIBt6p24n4j7tHgNVCjsfHVNUbo=
github.com/gobwas/ws v1.0.2/go.mod h1:szmBTxLgaFppYjEmNtny/v3w89xOydFnnZMcgRRu/EM=
github.com/goccy/go-json v0.10.2 h1:CrxCmQqYDkv1z7lO7Wbh2HN93uovUHgrECaO5ZrCXAU=
github.com/goccy/go-json v0.10.2/go.mod h1:6MelG93GURQebXPDq3khkgXZkazVtN9CRI+MGFi0w8I=
@@ -47,14 +51,17 @@ github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:W
github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0=
github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI=
github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk=
github.com/golang/protobuf v1.5.2 h1:ROPKBNFfQgOUMifHyP+KYbvpjbdoFNs+aK7DXlji0Tw=
github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY=
github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
github.com/google/go-cmp v0.5.5 h1:Khx7svrCpmxxtHBq5j2mp/xVjsi8hQMfNLvJFAlrGgU=
github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I=
github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
github.com/gorilla/websocket v1.4.1 h1:q7AeDBpnBk8AogcD4DSag/Ukw/KV+YhzLj2bP5HvKCM=
github.com/gorilla/websocket v1.4.1/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE=
github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU=
github.com/json-iterator/go v1.1.9/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
@@ -65,8 +72,10 @@ github.com/klauspost/compress v1.10.3/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYs
github.com/klauspost/cpuid/v2 v2.0.9/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg=
github.com/klauspost/cpuid/v2 v2.2.5 h1:0E5MSMDEoAulmXNFquVs//DdoomxaoTY1kUhbc/qbZg=
github.com/klauspost/cpuid/v2 v2.2.5/go.mod h1:Lcz8mBdAVJIBVzewtcLocK12l3Y+JytZYpaMropDUws=
github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI=
github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE=
github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
github.com/leodido/go-urn v1.2.0/go.mod h1:+8+nEpDfqqsY+g338gtMEUOtuK+4dEMhiQEgxpxOKII=
github.com/leodido/go-urn v1.2.4 h1:XlAE/cm/ms7TE/VMVoduSpNBoyc2dOxHs5MZSwAN63Q=
@@ -145,7 +154,6 @@ github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81P
github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA=
github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
github.com/stretchr/testify v1.8.0 h1:pSgiaMZlXftHpm5L7V1+rVB+AZJydKsMxsQBIJw4PKk=
github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=
github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=
github.com/stretchr/testify v1.8.2/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=
@@ -158,7 +166,6 @@ github.com/tinyzimmer/go-gst v0.2.33 h1:wdwUYoN7dkWGUTrZIgB9Mp5LMRr/Sld5PVGRsE7/
github.com/tinyzimmer/go-gst v0.2.33/go.mod h1:0hI+orMYVT61TEh429LvmoV9UmyqjeTqdJ3DW2TX114=
github.com/twitchyliquid64/golang-asm v0.15.1 h1:SU5vSMR7hnwNxj24w34ZyCi/FmDZTkS4MhqMhdFk5YI=
github.com/twitchyliquid64/golang-asm v0.15.1/go.mod h1:a1lVb/DtPvCB8fslRZhAngC2+aY1QWCk3Cedj/Gdt08=
github.com/ugorji/go v1.1.7 h1:/68gy2h+1mWMrwZFeD1kQialdSzAb432dtpeJ42ovdo=
github.com/ugorji/go v1.1.7/go.mod h1:kZn38zHttfInRq0xu/PH0az30d+z6vm202qpg1oXVMw=
github.com/ugorji/go/codec v1.1.7/go.mod h1:Ax+UKWsSmolVDwsd+7N3ZtXu+yMGCf907BLYF3GoBXY=
github.com/ugorji/go/codec v1.2.11 h1:BMaWp1Bb6fHwEtbplGBGJ498wD+LKlNSl25MjdZY4dU=
@@ -180,7 +187,6 @@ golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8U
golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
golang.org/x/crypto v0.8.0/go.mod h1:mRqEX+O9/h5TFCrQhkgjo2yKi0yYA+9ecGkdQoHrywE=
golang.org/x/crypto v0.9.0 h1:LF6fAI+IutBocDJ2OT0Q1g8plpYljMZ4+lty+dsqw3g=
golang.org/x/crypto v0.9.0/go.mod h1:yrmDGqONDYtNj3tH8X9dzUun2m2lzPa9ngI6/RUPGR0=
golang.org/x/crypto v0.10.0 h1:LKqV2xt9+kDzSTfOhx4FrkEBcMrAgHSYgzywV9zcGmM=
golang.org/x/crypto v0.10.0/go.mod h1:o4eNf7Ede1fv+hwOwZsTHl9EsPFO6q6ZvYR8vYfY45I=
@@ -200,7 +206,6 @@ golang.org/x/net v0.5.0/go.mod h1:DivGGAXEgPSlEBzxGzZI+ZLohi+xUj054jfeKui00ws=
golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs=
golang.org/x/net v0.8.0/go.mod h1:QVkue5JL9kW//ek3r6jTKnTFis1tRmNAW2P1shuFdJc=
golang.org/x/net v0.9.0/go.mod h1:d48xBJpPfHeWQsugry2m+kC02ZBRGRgulfHnEXEuWns=
golang.org/x/net v0.10.0 h1:X2//UzNDwYmtCLn7To6G58Wr6f5ahEAQgKNzv9Y951M=
golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg=
golang.org/x/net v0.11.0 h1:Gi2tvZIJyBtO9SDr1q9h5hEQCp/4L2RQ+ar0qjx2oNU=
golang.org/x/net v0.11.0/go.mod h1:2L/ixqYpgIVXmeoSA/4Lu7BzTG4KIyPIryS4IsOd1oQ=
@@ -230,7 +235,6 @@ golang.org/x/sys v0.4.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.7.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.8.0 h1:EBmGv8NaZBZTWvrbjNoL6HVt+IVy3QDQpJs7VRIw3tU=
golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.9.0 h1:KS/R3tvhPqvJvwcKfnBHJwwthS11LRhmM5D59eEXa0s=
golang.org/x/sys v0.9.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
@@ -263,6 +267,7 @@ golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU=
golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 h1:go1bK/D/BFZV2I8cIQd1NKEZ+0owSTG1fDTci4IqFcE=
golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8=
google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0=
@@ -275,6 +280,7 @@ google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQ
google.golang.org/protobuf v1.31.0 h1:g0LDEJHgrBl9N9r17Ru3sqWhkIx2NB67okBHPwC7hs8=
google.golang.org/protobuf v1.31.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo=
gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys=
gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw=
main.go
+4 -4
View File
@@ -5,9 +5,9 @@ import (
"os/signal"
"syscall"
"gitea.heinrich.blue/PHI/webrtc-gst/pkg/common"
"gitea.heinrich.blue/PHI/webrtc-gst/pkg/server"
"gitea.heinrich.blue/PHI/webrtc-gst/pkg/webrtc"
"github.com/kaedwen/webrtc/pkg/common"
"github.com/kaedwen/webrtc/pkg/server"
"github.com/kaedwen/webrtc/pkg/webrtc"
"github.com/tinyzimmer/go-glib/glib"
"go.uber.org/zap"
)
@@ -28,7 +28,7 @@ func main() {
http := server.NewHttpServer(lg.With(zap.String("context", "server")), &cfg)
webrtc.NewWebrtcHandler(ctx, lg.With(zap.String("context", "webrtc")), http.Hndl)
webrtc.NewWebrtcHandler(ctx, lg.With(zap.String("context", "webrtc")), &cfg.Stream, http.Hndl)
http.ListenAndServe(ctx, cfg.Http.Address())
pkg/common/config.go
+34
View File
@@ -9,6 +9,7 @@ import (
type Config struct {
Logging ConfigLogging `arg:"-"`
Stream ConfigStream `arg:"-"`
Http ConfigHTTP `arg:"-"`
}
@@ -24,12 +25,45 @@ type ConfigHTTP struct {
StaticPath *string `arg:"--http-static,env:HTTP_STATIC"`
}
type ConfigStream struct {
VideoOut ConfigVideoOutputStream `arg:"-"`
AudioOut ConfigAudioOutputStream `arg:"-"`
AudioIn ConfigAudioInputStream `arg:"-"`
}
type ConfigVideoOutputStream struct {
Source string `arg:"--video-src,env:VIDEO_SRC" default:"v4l2src"`
Device string `arg:"--video-device,env:VIDEO_DEVICE" default:"/dev/video0"`
Codec string `arg:"--video-codec,env:VIDEO_CODEC" default:"vp8"`
Height uint `arg:"--video-height,env:VIDEO_HEIGHT" default:"480"`
Width uint `arg:"--video-width,env:VIDEO_WIDTH" default:"640"`
}
type ConfigAudioOutputStream struct {
Source string `arg:"--audio-src,env:AUDIO_SRC" default:"alsasrc"`
DeviceName string `arg:"--audio-device-name,env:AUDIO_DEVICE" default:"default"`
Device *string `arg:"--audio-device,env:AUDIO_DEVICE"`
Codec string `arg:"--audio-codec,env:AUDIO_CODEC" default:"opus"`
Channels uint `arg:"--audio-channels,env:AUDIO_CHANNELS" default:"1"`
}
type ConfigAudioInputStream struct {
Sink string `arg:"--audio-src,env:AUDIO_SINK" default:"alsasink"`
DeviceName string `arg:"--audio-device-name,env:AUDIO_DEVICE" default:"default"`
Device *string `arg:"--audio-device,env:AUDIO_DEVICE"`
Codec string `arg:"--audio-codec,env:AUDIO_CODEC" default:"opus"`
Channels uint `arg:"--audio-channels,env:AUDIO_CHANNELS" default:"1"`
}
func (c *ConfigHTTP) Address() string {
return net.JoinHostPort(c.Host, fmt.Sprint(c.Port))
}
func (c *Config) MustParse() {
arg.MustParse(&c.Logging)
arg.MustParse(&c.Stream.VideoOut)
arg.MustParse(&c.Stream.AudioOut)
arg.MustParse(&c.Stream.AudioIn)
arg.MustParse(&c.Http)
arg.MustParse(c)
}
pkg/server/http.go
+1 -1
View File
@@ -11,8 +11,8 @@ import (
"strings"
"time"
"gitea.heinrich.blue/PHI/webrtc-gst/pkg/common"
"github.com/gin-gonic/gin"
"github.com/kaedwen/webrtc/pkg/common"
"github.com/pion/webrtc/v3"
"go.uber.org/zap"
"nhooyr.io/websocket"
pkg/webrtc/gst.go → pkg/streamer/gst-sink.go
+67 -19
View File
@@ -1,4 +1,4 @@
package webrtc
package streamer
import (
"github.com/pion/webrtc/v3/pkg/media"
@@ -6,10 +6,6 @@ import (
"github.com/tinyzimmer/go-gst/gst/app"
)
func init() {
gst.Init(nil)
}
func setCallback(sink *app.Sink, ch chan media.Sample) {
sink.SetCallbacks(&app.SinkCallbacks{
NewSampleFunc: func(sink *app.Sink) gst.FlowReturn {
@@ -38,24 +34,49 @@ func setCallback(sink *app.Sink, ch chan media.Sample) {
})
}
func CreateVideoPipeline(s string) (*gst.Pipeline, <-chan media.Sample, error) {
func CreateVideoPipelineSink(s StreamElement) (*gst.Pipeline, <-chan media.Sample, error) {
// Create a pipeline
pipeline, err := gst.NewPipeline("")
pipeline, err := gst.NewPipeline("pion-video-pipeline")
if err != nil {
return nil, nil, err
}
elems := make([]*gst.Element, 0)
// Create the src
src, err := gst.NewElement(s)
src, err := gst.NewElement(s.Kind)
if err != nil {
return nil, nil, err
}
elems = append(elems, src)
for name, value := range s.Properties {
src.Set(name, value)
}
if s.Caps != nil {
filter, err := gst.NewElement("capsfilter")
if err != nil {
return nil, nil, err
}
filter.Set("caps", s.ToGstCaps())
elems = append(elems, filter)
}
// just to be on the save side
conv, err := gst.NewElement("videoconvert")
if err != nil {
return nil, nil, err
}
elems = append(elems, conv)
// Create the enc
enc, err := gst.NewElement("vp8enc")
if err != nil {
return nil, nil, err
}
elems = append(elems, enc)
enc.SetProperty("error-resilient", "partitions")
enc.SetProperty("keyframe-max-dist", 10)
@@ -64,56 +85,83 @@ func CreateVideoPipeline(s string) (*gst.Pipeline, <-chan media.Sample, error) {
enc.SetProperty("deadline", 1)
// Create the sink
sink, err := app.NewAppSink()
appsink, err := app.NewAppSink()
if err != nil {
return nil, nil, err
}
elems = append(elems, appsink.Element)
ch := make(chan media.Sample, 100)
setCallback(sink, ch)
setCallback(appsink, ch)
// Add the elements to the pipeline
pipeline.AddMany(src, enc, sink.Element)
pipeline.AddMany(elems...)
// link the elements
gst.ElementLinkMany(src, enc, sink.Element)
gst.ElementLinkMany(elems...)
return pipeline, ch, nil
}
func CreateAudioPipeline(s string) (*gst.Pipeline, <-chan media.Sample, error) {
func CreateAudioPipelineSink(s StreamElement) (*gst.Pipeline, <-chan media.Sample, error) {
// Create a pipeline
pipeline, err := gst.NewPipeline("")
pipeline, err := gst.NewPipeline("pion-audio-pipeline")
if err != nil {
return nil, nil, err
}
elems := make([]*gst.Element, 0)
// Create the src
src, err := gst.NewElement(s)
src, err := gst.NewElement(s.Kind)
if err != nil {
return nil, nil, err
}
elems = append(elems, src)
for name, value := range s.Properties {
src.Set(name, value)
}
if s.Caps != nil {
filter, err := gst.NewElement("capsfilter")
if err != nil {
return nil, nil, err
}
filter.Set("caps", s.ToGstCaps())
elems = append(elems, filter)
}
// just to be on the save side
conv, err := gst.NewElement("audioconvert")
if err != nil {
return nil, nil, err
}
elems = append(elems, conv)
// Create the enc
enc, err := gst.NewElement("opusenc")
if err != nil {
return nil, nil, err
}
elems = append(elems, enc)
// Create the sink
sink, err := app.NewAppSink()
appsink, err := app.NewAppSink()
if err != nil {
return nil, nil, err
}
elems = append(elems, appsink.Element)
ch := make(chan media.Sample, 100)
setCallback(sink, ch)
setCallback(appsink, ch)
// Add the elements to the pipeline
pipeline.AddMany(src, enc, sink.Element)
pipeline.AddMany(elems...)
// link the elements
gst.ElementLinkMany(src, enc, sink.Element)
gst.ElementLinkMany(elems...)
return pipeline, ch, nil
}
pkg/streamer/gst-src.go
+79
View File
@@ -0,0 +1,79 @@
package streamer
import (
"fmt"
"github.com/tinyzimmer/go-gst/gst"
"github.com/tinyzimmer/go-gst/gst/app"
)
type SrcPipeline struct {
*gst.Pipeline
src *app.Source
}
func CreateAudioPipelineSrc(dst StreamElement) (*SrcPipeline, error) {
// Create a pipeline
pipeline, err := gst.NewPipeline("")
if err != nil {
return nil, err
}
elems := make([]*gst.Element, 0)
// Create the src
appsrc, err := app.NewAppSrc()
if err != nil {
return nil, err
}
elems = append(elems, appsrc.Element)
appsrc.SetFormat(gst.FormatTime)
appsrc.SetDoTimestamp(true)
appsrc.SetLive(true)
// Create the opus decoder
decCodec, err := gst.NewElement("rtpopusdepay")
if err != nil {
return nil, err
}
elems = append(elems, decCodec)
// Create the bin decoder
decBin, err := gst.NewElement("decodebin")
if err != nil {
return nil, err
}
elems = append(elems, decBin)
// Create the sink
sink, err := gst.NewElement(dst.Kind)
if err != nil {
return nil, err
}
elems = append(elems, sink)
for name, value := range dst.Properties {
sink.SetProperty(name, value)
}
// Add the elements to the pipeline
pipeline.AddMany(elems...)
// link the elements
gst.ElementLinkMany(elems...)
return &SrcPipeline{
Pipeline: pipeline,
src: appsrc,
}, nil
}
func (p *SrcPipeline) Push(data []byte) error {
err := p.src.PushBuffer(gst.NewBufferFromBytes(data))
if err != gst.FlowOK {
return fmt.Errorf("failed to send bytes to gst pipeline - %s", err.String())
}
return nil
}
pkg/streamer/gst.go
+88
View File
@@ -0,0 +1,88 @@
package streamer
import (
"fmt"
"reflect"
"strings"
"time"
"github.com/tinyzimmer/go-gst/gst"
"github.com/tinyzimmer/go-gst/gst/app"
"go.uber.org/zap"
)
const capsTagName = "caps"
func init() {
gst.Init(nil)
}
type StreamElementCaps struct {
Mime string `caps:"mime"`
Height uint `caps:"height"`
Width uint `caps:"width"`
Format string `caps:"format"`
Channels uint `caps:"channels"`
Rate uint `caps:"rate"`
}
type StreamElement struct {
Kind string
Properties map[string]interface{}
Caps *StreamElementCaps
}
func (se StreamElement) ToGstCaps() *gst.Caps {
if se.Caps == nil {
return nil
}
t := reflect.ValueOf(*se.Caps)
var mime string
var caps []string
for i := 0; i < t.NumField(); i++ {
valueField := t.Field(i)
tagField := t.Type().Field(i).Tag.Get(capsTagName)
if tagField == "mime" {
mime = valueField.String()
} else {
if !valueField.IsZero() {
caps = append(caps, fmt.Sprintf("%s=%v", tagField, valueField.Interface()))
}
}
}
return gst.NewCapsFromString(strings.Join(append([]string{mime}, caps...), ","))
}
func handleMessage(msg *gst.Message) error {
switch msg.Type() {
case gst.MessageEOS:
return app.ErrEOS
case gst.MessageError:
return msg.ParseError()
}
return nil
}
func LoopBus(lg *zap.Logger, pipeline *gst.Pipeline) {
// Retrieve the bus from the pipeline
bus := pipeline.GetPipelineBus()
// Loop over messsages from the pipeline
go func() {
for {
msg := bus.TimedPop(time.Duration(-1))
if msg == nil {
return
}
if err := handleMessage(msg); err != nil {
lg.Error("failed to handle message", zap.Error(err))
}
}
}()
}
pkg/webrtc/webrtc.go
+97 -8
View File
@@ -3,8 +3,12 @@ package webrtc
import (
"context"
"sync"
"time"
"gitea.heinrich.blue/PHI/webrtc-gst/pkg/server"
"github.com/kaedwen/webrtc/pkg/common"
"github.com/kaedwen/webrtc/pkg/server"
"github.com/kaedwen/webrtc/pkg/streamer"
"github.com/pion/rtcp"
"github.com/pion/webrtc/v3"
"github.com/pion/webrtc/v3/pkg/media"
"github.com/tinyzimmer/go-gst/gst"
@@ -26,19 +30,19 @@ type PeerHandle struct {
videoTrack *webrtc.TrackLocalStaticSample
}
func NewWebrtcHandler(ctx context.Context, lg *zap.Logger, ch <-chan *server.SignalingHandle) error {
func NewWebrtcHandler(ctx context.Context, lg *zap.Logger, cfg *common.ConfigStream, ch <-chan *server.SignalingHandle) error {
wh := WebrtcHandler{
lg: lg,
mu: &sync.Mutex{},
peerHandles: make(map[string]*PeerHandle, 0),
}
err := wh.handleAudioSamples(ctx, "audiotestsrc")
err := wh.handleAudioSamples(ctx, &cfg.AudioOut)
if err != nil {
return err
}
err = wh.handleVideoSamples(ctx, "videotestsrc")
err = wh.handleVideoSamples(ctx, &cfg.VideoOut)
if err != nil {
return err
}
@@ -99,14 +103,35 @@ func (wh *WebrtcHandler) stopPipelines() {
}
func (wh *WebrtcHandler) handleAudioSamples(ctx context.Context, src string) error {
func (wh *WebrtcHandler) handleAudioSamples(ctx context.Context, cfg *common.ConfigAudioOutputStream) error {
properties := map[string]interface{}{}
if cfg.Source == "alsasrc" {
if cfg.Device != nil {
properties["device"] = *cfg.Device
} else {
properties["device-name"] = cfg.DeviceName
}
}
src := streamer.StreamElement{
Kind: cfg.Source,
Properties: properties,
Caps: &streamer.StreamElementCaps{
Mime: "audio/x-raw",
Channels: cfg.Channels,
Rate: 48000,
},
}
var err error
var audioCh <-chan media.Sample
wh.audioPipeline, audioCh, err = CreateAudioPipeline(src)
wh.audioPipeline, audioCh, err = streamer.CreateAudioPipelineSink(src)
if err != nil {
return err
}
streamer.LoopBus(wh.lg.With(zap.String("sub-context", "audio")), wh.audioPipeline)
go func() {
wh.lg.Info("wait for audio sample")
for {
@@ -127,14 +152,29 @@ func (wh *WebrtcHandler) handleAudioSamples(ctx context.Context, src string) err
return nil
}
func (wh *WebrtcHandler) handleVideoSamples(ctx context.Context, src string) error {
func (wh *WebrtcHandler) handleVideoSamples(ctx context.Context, cfg *common.ConfigVideoOutputStream) error {
src := streamer.StreamElement{
Kind: cfg.Source,
Properties: map[string]interface{}{
"device": cfg.Device,
},
Caps: &streamer.StreamElementCaps{
Mime: "video/x-raw",
Format: "YUY2",
Width: cfg.Width,
Height: cfg.Height,
},
}
var err error
var videoCh <-chan media.Sample
wh.videoPipeline, videoCh, err = CreateVideoPipeline(src)
wh.videoPipeline, videoCh, err = streamer.CreateVideoPipelineSink(src)
if err != nil {
return err
}
streamer.LoopBus(wh.lg.With(zap.String("sub-context", "video")), wh.videoPipeline)
go func() {
wh.lg.Info("wait for video sample")
for {
@@ -170,6 +210,55 @@ func (wh *WebrtcHandler) createPeerHandle(ctx context.Context, sh *server.Signal
return err
}
// Set a handler for when a new remote track starts, this handler creates a gstreamer pipeline
// for the given codec
peerConnection.OnTrack(func(track *webrtc.TrackRemote, receiver *webrtc.RTPReceiver) {
wh.lg.Info("received track", zap.String("kind", track.Kind().String()), zap.String("codec", track.Codec().MimeType))
if track.Codec().MimeType != "audio/opus" {
wh.lg.Error("mimetype not supported", zap.String("mime", track.Codec().MimeType))
return
}
// Send a PLI on an interval so that the publisher is pushing a keyframe every rtcpPLIInterval
go func() {
ticker := time.NewTicker(time.Second * 3)
for {
select {
case <-ticker.C:
if err := peerConnection.WriteRTCP([]rtcp.Packet{&rtcp.PictureLossIndication{MediaSSRC: uint32(track.SSRC())}}); err != nil {
wh.lg.Error("failed to send PLI", zap.Error(err))
}
case <-ctx.Done():
return
}
}
}()
pipeline, err := streamer.CreateAudioPipelineSrc(streamer.StreamElement{
Kind: "autoaudiosink",
})
if err != nil {
wh.lg.Error("failed to create src pipeline", zap.Error(err))
}
pipeline.Start()
buf := make([]byte, 1400)
for {
i, _, readErr := track.Read(buf)
if readErr != nil {
wh.lg.Error("read failed", zap.Error(err))
}
if i > 0 {
pipeline.Push(buf[:i])
if err != nil {
wh.lg.Error("push failed", zap.Error(err))
}
}
}
})
// Set the handler for ICE connection state
// This will notify you when the peer has connected/disconnected
peerConnection.OnICEConnectionStateChange(func(connectionState webrtc.ICEConnectionState) {
static/dist/main.js
Vendored
+197 -44
View File
@@ -12,30 +12,60 @@ __webpack_require__.r(__webpack_exports__);
/* harmony export */ "AppComponent": () => (/* binding */ AppComponent)
/* harmony export */ });
/* harmony import */ var _projects_PRIVATE_webrtc_gst_static_node_modules_babel_runtime_helpers_esm_asyncToGenerator_js__WEBPACK_IMPORTED_MODULE_0__ = __webpack_require__(/*! ./node_modules/@babel/runtime/helpers/esm/asyncToGenerator.js */ 1670);
/* harmony import */ var _angular_core__WEBPACK_IMPORTED_MODULE_2__ = __webpack_require__(/*! @angular/core */ 2560);
/* harmony import */ var _services_signaling_service__WEBPACK_IMPORTED_MODULE_1__ = __webpack_require__(/*! ./services/signaling.service */ 1759);
/* harmony import */ var _angular_core__WEBPACK_IMPORTED_MODULE_4__ = __webpack_require__(/*! @angular/core */ 2560);
/* harmony import */ var _components_video_video_component__WEBPACK_IMPORTED_MODULE_1__ = __webpack_require__(/*! ./components/video/video.component */ 5465);
/* harmony import */ var _components_audio_audio_component__WEBPACK_IMPORTED_MODULE_2__ = __webpack_require__(/*! ./components/audio/audio.component */ 210);
/* harmony import */ var _services_signaling_service__WEBPACK_IMPORTED_MODULE_3__ = __webpack_require__(/*! ./services/signaling.service */ 1759);
const _c0 = ["video"];
const _c1 = ["audio"];
const _c2 = ["start"];
const _c0 = ["vc"];
class AppComponent {
constructor(signaling) {
onClick(_) {
var _this = this;
return (0,_projects_PRIVATE_webrtc_gst_static_node_modules_babel_runtime_helpers_esm_asyncToGenerator_js__WEBPACK_IMPORTED_MODULE_0__["default"])(function* () {
if (!_this.selfAudioRunning) {
yield _this.startAudio();
}
for (const el of _this.audioList) {
el.instance.toggleMute();
}
})();
}
startAudio() {
var _this2 = this;
return (0,_projects_PRIVATE_webrtc_gst_static_node_modules_babel_runtime_helpers_esm_asyncToGenerator_js__WEBPACK_IMPORTED_MODULE_0__["default"])(function* () {
const stream = yield navigator.mediaDevices.getUserMedia({
audio: true,
video: false
});
_this2.pc.addTrack(stream.getAudioTracks()[0]);
_this2.selfAudioRunning = true;
})();
}
constructor(signaling) {
var _this3 = this;
this.signaling = signaling;
this.audioList = [];
this.selfAudioRunning = false;
//this.pc = new RTCPeerConnection({iceServers: [{urls: 'stun:stun.l.google.com:19302'}]});
this.pc = new RTCPeerConnection();
// Once remote track media arrives, show it in remote element.
this.pc.ontrack = event => {
const [remoteStream] = event.streams;
if (event.track.kind === 'video') {
this.videoElem.nativeElement.srcObject = remoteStream;
this.videoElem.nativeElement.muted = true;
this.videoElem.nativeElement.play();
const video = this.vcr.createComponent(_components_video_video_component__WEBPACK_IMPORTED_MODULE_1__.VideoComponent, {});
video.instance.setStream(remoteStream);
video.instance.play();
console.log(video);
} else if (event.track.kind === 'audio') {
this.audioElem.nativeElement.srcObject = remoteStream;
const audio = this.vcr.createComponent(_components_audio_audio_component__WEBPACK_IMPORTED_MODULE_2__.AudioComponent, {});
audio.instance.setStream(remoteStream);
console.log(audio);
this.audioList.push(audio);
}
};
this.pc.oniceconnectionstatechange = e => {
@@ -52,7 +82,7 @@ class AppComponent {
};
// Offer to receive 1 audio, and 1 video track
this.pc.addTransceiver('audio', {
direction: 'recvonly'
direction: 'sendrecv'
});
this.pc.addTransceiver('video', {
direction: 'recvonly'
@@ -63,14 +93,12 @@ class AppComponent {
// Let the "negotiationneeded" event trigger offer generation.
this.pc.onnegotiationneeded = /*#__PURE__*/function () {
var _ref = (0,_projects_PRIVATE_webrtc_gst_static_node_modules_babel_runtime_helpers_esm_asyncToGenerator_js__WEBPACK_IMPORTED_MODULE_0__["default"])(function* (e) {
console.log(e);
// try {
// await this.pc.setLocalDescription();
// // Send the offer to the other peer.
// this.api.send({desc: this.pc.localDescription});
// } catch (err) {
// console.error(err);
// }
try {
yield _this3.pc.setLocalDescription();
signaling.next(_this3.pc.localDescription);
} catch (err) {
console.error(err);
}
});
return function (_x) {
return _ref.apply(this, arguments);
@@ -83,7 +111,7 @@ class AppComponent {
console.log('Received answer');
try {
// the answer
yield _this.pc.setRemoteDescription(x);
yield _this3.pc.setRemoteDescription(x);
} catch (e) {
console.error(e);
}
@@ -97,38 +125,38 @@ class AppComponent {
ngOnInit() {
return (0,_projects_PRIVATE_webrtc_gst_static_node_modules_babel_runtime_helpers_esm_asyncToGenerator_js__WEBPACK_IMPORTED_MODULE_0__["default"])(function* () {})();
}
run(e) {
this.videoElem.nativeElement.play();
}
}
AppComponent.ɵfac = function AppComponent_Factory(t) {
return new (t || AppComponent)(_angular_core__WEBPACK_IMPORTED_MODULE_2__["ɵɵdirectiveInject"](_services_signaling_service__WEBPACK_IMPORTED_MODULE_1__.SignalingService));
return new (t || AppComponent)(_angular_core__WEBPACK_IMPORTED_MODULE_4__["ɵɵdirectiveInject"](_services_signaling_service__WEBPACK_IMPORTED_MODULE_3__.SignalingService));
};
AppComponent.ɵcmp = /*@__PURE__*/_angular_core__WEBPACK_IMPORTED_MODULE_2__["ɵɵdefineComponent"]({
AppComponent.ɵcmp = /*@__PURE__*/_angular_core__WEBPACK_IMPORTED_MODULE_4__["ɵɵdefineComponent"]({
type: AppComponent,
selectors: [["app-root"]],
viewQuery: function AppComponent_Query(rf, ctx) {
if (rf & 1) {
_angular_core__WEBPACK_IMPORTED_MODULE_2__["ɵɵviewQuery"](_c0, 7, _angular_core__WEBPACK_IMPORTED_MODULE_2__.ElementRef);
_angular_core__WEBPACK_IMPORTED_MODULE_2__["ɵɵviewQuery"](_c1, 7, _angular_core__WEBPACK_IMPORTED_MODULE_2__.ElementRef);
_angular_core__WEBPACK_IMPORTED_MODULE_2__["ɵɵviewQuery"](_c2, 7, _angular_core__WEBPACK_IMPORTED_MODULE_2__.ElementRef);
_angular_core__WEBPACK_IMPORTED_MODULE_4__["ɵɵviewQuery"](_c0, 7, _angular_core__WEBPACK_IMPORTED_MODULE_4__.ViewContainerRef);
}
if (rf & 2) {
let _t;
_angular_core__WEBPACK_IMPORTED_MODULE_2__["ɵɵqueryRefresh"](_t = _angular_core__WEBPACK_IMPORTED_MODULE_2__["ɵɵloadQuery"]()) && (ctx.videoElem = _t.first);
_angular_core__WEBPACK_IMPORTED_MODULE_2__["ɵɵqueryRefresh"](_t = _angular_core__WEBPACK_IMPORTED_MODULE_2__["ɵɵloadQuery"]()) && (ctx.audioElem = _t.first);
_angular_core__WEBPACK_IMPORTED_MODULE_2__["ɵɵqueryRefresh"](_t = _angular_core__WEBPACK_IMPORTED_MODULE_2__["ɵɵloadQuery"]()) && (ctx.startElem = _t.first);
_angular_core__WEBPACK_IMPORTED_MODULE_4__["ɵɵqueryRefresh"](_t = _angular_core__WEBPACK_IMPORTED_MODULE_4__["ɵɵloadQuery"]()) && (ctx.vcr = _t.first);
}
},
decls: 4,
hostBindings: function AppComponent_HostBindings(rf, ctx) {
if (rf & 1) {
_angular_core__WEBPACK_IMPORTED_MODULE_4__["ɵɵlistener"]("click", function AppComponent_click_HostBindingHandler($event) {
return ctx.onClick($event.target);
}, false, _angular_core__WEBPACK_IMPORTED_MODULE_4__["ɵɵresolveDocument"]);
}
},
decls: 2,
vars: 0,
consts: [["autoplay", "", "muted", "", "controls", ""], ["video", ""], ["autoplay", "", "controls", ""], ["audio", ""]],
consts: [["vc", ""]],
template: function AppComponent_Template(rf, ctx) {
if (rf & 1) {
_angular_core__WEBPACK_IMPORTED_MODULE_2__["ɵɵelement"](0, "video", 0, 1)(2, "audio", 2, 3);
_angular_core__WEBPACK_IMPORTED_MODULE_4__["ɵɵelementContainer"](0, null, 0);
}
},
styles: ["[_nghost-%COMP%] {\n display: flex;\n align-items: center;\n justify-content: center;\n flex-direction: column;\n height: 100%;\n}\n\nvideo[_ngcontent-%COMP%] {\n max-height: 90%;\n max-width: 80%;\n height: 90%;\n width: auto;\n}\n/*# sourceMappingURL=data:application/json;charset=utf-8;base64,eyJ2ZXJzaW9uIjozLCJzb3VyY2VzIjpbIndlYnBhY2s6Ly8uL3NyYy9hcHAvYXBwLmNvbXBvbmVudC5zY3NzIl0sIm5hbWVzIjpbXSwibWFwcGluZ3MiOiJBQUNBO0VBQ0UsYUFBQTtFQUNBLG1CQUFBO0VBQ0EsdUJBQUE7RUFDQSxzQkFBQTtFQUNBLFlBQUE7QUFBRjs7QUFHQTtFQUNFLGVBQUE7RUFDQSxjQUFBO0VBQ0EsV0FBQTtFQUNBLFdBQUE7QUFBRiIsInNvdXJjZXNDb250ZW50IjpbIlxuOmhvc3Qge1xuICBkaXNwbGF5OiBmbGV4O1xuICBhbGlnbi1pdGVtczogY2VudGVyO1xuICBqdXN0aWZ5LWNvbnRlbnQ6IGNlbnRlcjtcbiAgZmxleC1kaXJlY3Rpb246IGNvbHVtbjtcbiAgaGVpZ2h0OiAxMDAlO1xufVxuXG52aWRlbyB7XG4gIG1heC1oZWlnaHQ6IDkwJTtcbiAgbWF4LXdpZHRoOiA4MCU7XG4gIGhlaWdodDogOTAlO1xuICB3aWR0aDogYXV0bztcbn1cbiJdLCJzb3VyY2VSb290IjoiIn0= */"]
styles: ["[_nghost-%COMP%] {\n display: flex;\n align-items: center;\n justify-content: center;\n flex-direction: column;\n height: 100%;\n width: 100%;\n}\n/*# sourceMappingURL=data:application/json;charset=utf-8;base64,eyJ2ZXJzaW9uIjozLCJzb3VyY2VzIjpbIndlYnBhY2s6Ly8uL3NyYy9hcHAvYXBwLmNvbXBvbmVudC5zY3NzIl0sIm5hbWVzIjpbXSwibWFwcGluZ3MiOiJBQUNBO0VBQ0UsYUFBQTtFQUNBLG1CQUFBO0VBQ0EsdUJBQUE7RUFDQSxzQkFBQTtFQUNBLFlBQUE7RUFDQSxXQUFBO0FBQUYiLCJzb3VyY2VzQ29udGVudCI6WyJcbjpob3N0IHtcbiAgZGlzcGxheTogZmxleDtcbiAgYWxpZ24taXRlbXM6IGNlbnRlcjtcbiAganVzdGlmeS1jb250ZW50OiBjZW50ZXI7XG4gIGZsZXgtZGlyZWN0aW9uOiBjb2x1bW47XG4gIGhlaWdodDogMTAwJTtcbiAgd2lkdGg6IDEwMCU7XG59XG5cbiJdLCJzb3VyY2VSb290IjoiIn0= */"]
});
/***/ }),
@@ -143,11 +171,15 @@ __webpack_require__.r(__webpack_exports__);
/* harmony export */ __webpack_require__.d(__webpack_exports__, {
/* harmony export */ "AppModule": () => (/* binding */ AppModule)
/* harmony export */ });
/* harmony import */ var _angular_platform_browser__WEBPACK_IMPORTED_MODULE_3__ = __webpack_require__(/*! @angular/platform-browser */ 4497);
/* harmony import */ var _angular_common_http__WEBPACK_IMPORTED_MODULE_4__ = __webpack_require__(/*! @angular/common/http */ 8987);
/* harmony import */ var _angular_platform_browser__WEBPACK_IMPORTED_MODULE_5__ = __webpack_require__(/*! @angular/platform-browser */ 4497);
/* harmony import */ var _angular_common_http__WEBPACK_IMPORTED_MODULE_6__ = __webpack_require__(/*! @angular/common/http */ 8987);
/* harmony import */ var _app_component__WEBPACK_IMPORTED_MODULE_0__ = __webpack_require__(/*! ./app.component */ 5041);
/* harmony import */ var _services_signaling_service__WEBPACK_IMPORTED_MODULE_1__ = __webpack_require__(/*! ./services/signaling.service */ 1759);
/* harmony import */ var _angular_core__WEBPACK_IMPORTED_MODULE_2__ = __webpack_require__(/*! @angular/core */ 2560);
/* harmony import */ var _components_video_video_component__WEBPACK_IMPORTED_MODULE_2__ = __webpack_require__(/*! ./components/video/video.component */ 5465);
/* harmony import */ var _components_audio_audio_component__WEBPACK_IMPORTED_MODULE_3__ = __webpack_require__(/*! ./components/audio/audio.component */ 210);
/* harmony import */ var _angular_core__WEBPACK_IMPORTED_MODULE_4__ = __webpack_require__(/*! @angular/core */ 2560);
@@ -157,23 +189,144 @@ class AppModule {}
AppModule.ɵfac = function AppModule_Factory(t) {
return new (t || AppModule)();
};
AppModule.ɵmod = /*@__PURE__*/_angular_core__WEBPACK_IMPORTED_MODULE_2__["ɵɵdefineNgModule"]({
AppModule.ɵmod = /*@__PURE__*/_angular_core__WEBPACK_IMPORTED_MODULE_4__["ɵɵdefineNgModule"]({
type: AppModule,
bootstrap: [_app_component__WEBPACK_IMPORTED_MODULE_0__.AppComponent]
});
AppModule.ɵinj = /*@__PURE__*/_angular_core__WEBPACK_IMPORTED_MODULE_2__["ɵɵdefineInjector"]({
AppModule.ɵinj = /*@__PURE__*/_angular_core__WEBPACK_IMPORTED_MODULE_4__["ɵɵdefineInjector"]({
providers: [_services_signaling_service__WEBPACK_IMPORTED_MODULE_1__.SignalingService],
imports: [_angular_platform_browser__WEBPACK_IMPORTED_MODULE_3__.BrowserModule, _angular_common_http__WEBPACK_IMPORTED_MODULE_4__.HttpClientModule]
imports: [_angular_platform_browser__WEBPACK_IMPORTED_MODULE_5__.BrowserModule, _angular_common_http__WEBPACK_IMPORTED_MODULE_6__.HttpClientModule]
});
(function () {
(typeof ngJitMode === "undefined" || ngJitMode) && _angular_core__WEBPACK_IMPORTED_MODULE_2__["ɵɵsetNgModuleScope"](AppModule, {
declarations: [_app_component__WEBPACK_IMPORTED_MODULE_0__.AppComponent],
imports: [_angular_platform_browser__WEBPACK_IMPORTED_MODULE_3__.BrowserModule, _angular_common_http__WEBPACK_IMPORTED_MODULE_4__.HttpClientModule]
(typeof ngJitMode === "undefined" || ngJitMode) && _angular_core__WEBPACK_IMPORTED_MODULE_4__["ɵɵsetNgModuleScope"](AppModule, {
declarations: [_app_component__WEBPACK_IMPORTED_MODULE_0__.AppComponent, _components_video_video_component__WEBPACK_IMPORTED_MODULE_2__.VideoComponent, _components_audio_audio_component__WEBPACK_IMPORTED_MODULE_3__.AudioComponent],
imports: [_angular_platform_browser__WEBPACK_IMPORTED_MODULE_5__.BrowserModule, _angular_common_http__WEBPACK_IMPORTED_MODULE_6__.HttpClientModule]
});
})();
/***/ }),
/***/ 210:
/*!*****************************************************!*\
!*** ./src/app/components/audio/audio.component.ts ***!
\*****************************************************/
/***/ ((__unused_webpack_module, __webpack_exports__, __webpack_require__) => {
__webpack_require__.r(__webpack_exports__);
/* harmony export */ __webpack_require__.d(__webpack_exports__, {
/* harmony export */ "AudioComponent": () => (/* binding */ AudioComponent)
/* harmony export */ });
/* harmony import */ var _angular_core__WEBPACK_IMPORTED_MODULE_0__ = __webpack_require__(/*! @angular/core */ 2560);
const _c0 = ["audio"];
class AudioComponent {
ngOnInit() {
this.elem.nativeElement.muted = true;
}
setStream(stream) {
this.elem.nativeElement.srcObject = stream;
}
play() {
this.elem.nativeElement.play();
}
pause() {
this.elem.nativeElement.pause();
}
toggleMute() {
this.elem.nativeElement.muted = !this.elem.nativeElement.muted;
console.log('toggle mute to', this.elem.nativeElement.muted);
if (!this.elem.nativeElement.muted) {
this.play();
}
}
}
AudioComponent.ɵfac = function AudioComponent_Factory(t) {
return new (t || AudioComponent)();
};
AudioComponent.ɵcmp = /*@__PURE__*/_angular_core__WEBPACK_IMPORTED_MODULE_0__["ɵɵdefineComponent"]({
type: AudioComponent,
selectors: [["app-audio"]],
viewQuery: function AudioComponent_Query(rf, ctx) {
if (rf & 1) {
_angular_core__WEBPACK_IMPORTED_MODULE_0__["ɵɵviewQuery"](_c0, 7, _angular_core__WEBPACK_IMPORTED_MODULE_0__.ElementRef);
}
if (rf & 2) {
let _t;
_angular_core__WEBPACK_IMPORTED_MODULE_0__["ɵɵqueryRefresh"](_t = _angular_core__WEBPACK_IMPORTED_MODULE_0__["ɵɵloadQuery"]()) && (ctx.elem = _t.first);
}
},
decls: 2,
vars: 0,
consts: [["audio", ""]],
template: function AudioComponent_Template(rf, ctx) {
if (rf & 1) {
_angular_core__WEBPACK_IMPORTED_MODULE_0__["ɵɵelement"](0, "audio", null, 0);
}
},
styles: ["\n/*# sourceMappingURL=data:application/json;charset=utf-8;base64,eyJ2ZXJzaW9uIjozLCJzb3VyY2VzIjpbXSwibmFtZXMiOltdLCJtYXBwaW5ncyI6IiIsInNvdXJjZVJvb3QiOiIifQ== */"]
});
/***/ }),
/***/ 5465:
/*!*****************************************************!*\
!*** ./src/app/components/video/video.component.ts ***!
\*****************************************************/
/***/ ((__unused_webpack_module, __webpack_exports__, __webpack_require__) => {
__webpack_require__.r(__webpack_exports__);
/* harmony export */ __webpack_require__.d(__webpack_exports__, {
/* harmony export */ "VideoComponent": () => (/* binding */ VideoComponent)
/* harmony export */ });
/* harmony import */ var _angular_core__WEBPACK_IMPORTED_MODULE_0__ = __webpack_require__(/*! @angular/core */ 2560);
const _c0 = ["video"];
class VideoComponent {
setStream(stream) {
this.elem.nativeElement.srcObject = stream;
this.elem.nativeElement.muted = true;
}
play() {
this.elem.nativeElement.play();
}
pause() {
this.elem.nativeElement.pause();
}
toggleMute() {
this.elem.nativeElement.muted = !this.elem.nativeElement.muted;
console.log('toggle mute to', this.elem.nativeElement.muted);
}
}
VideoComponent.ɵfac = function VideoComponent_Factory(t) {
return new (t || VideoComponent)();
};
VideoComponent.ɵcmp = /*@__PURE__*/_angular_core__WEBPACK_IMPORTED_MODULE_0__["ɵɵdefineComponent"]({
type: VideoComponent,
selectors: [["app-video"]],
viewQuery: function VideoComponent_Query(rf, ctx) {
if (rf & 1) {
_angular_core__WEBPACK_IMPORTED_MODULE_0__["ɵɵviewQuery"](_c0, 7, _angular_core__WEBPACK_IMPORTED_MODULE_0__.ElementRef);
}
if (rf & 2) {
let _t;
_angular_core__WEBPACK_IMPORTED_MODULE_0__["ɵɵqueryRefresh"](_t = _angular_core__WEBPACK_IMPORTED_MODULE_0__["ɵɵloadQuery"]()) && (ctx.elem = _t.first);
}
},
decls: 2,
vars: 0,
consts: [["video", ""]],
template: function VideoComponent_Template(rf, ctx) {
if (rf & 1) {
_angular_core__WEBPACK_IMPORTED_MODULE_0__["ɵɵelement"](0, "video", null, 0);
}
},
styles: ["[_nghost-%COMP%] {\n max-height: 90%;\n max-width: 80%;\n height: 90%;\n width: auto;\n}\n\nvideo[_ngcontent-%COMP%] {\n height: 100%;\n width: 100%;\n}\n/*# sourceMappingURL=data:application/json;charset=utf-8;base64,eyJ2ZXJzaW9uIjozLCJzb3VyY2VzIjpbIndlYnBhY2s6Ly8uL3NyYy9hcHAvY29tcG9uZW50cy92aWRlby92aWRlby5jb21wb25lbnQuc2NzcyJdLCJuYW1lcyI6W10sIm1hcHBpbmdzIjoiQUFDQTtFQUNFLGVBQUE7RUFDQSxjQUFBO0VBQ0EsV0FBQTtFQUNBLFdBQUE7QUFBRjs7QUFHQTtFQUNFLFlBQUE7RUFDQSxXQUFBO0FBQUYiLCJzb3VyY2VzQ29udGVudCI6WyJcbjpob3N0IHtcbiAgbWF4LWhlaWdodDogOTAlO1xuICBtYXgtd2lkdGg6IDgwJTtcbiAgaGVpZ2h0OiA5MCU7XG4gIHdpZHRoOiBhdXRvO1xufVxuXG52aWRlbyB7XG4gIGhlaWdodDogMTAwJTtcbiAgd2lkdGg6IDEwMCU7XG59Il0sInNvdXJjZVJvb3QiOiIifQ== */"]
});
/***/ }),
/***/ 1759:
/*!***********************************************!*\
!*** ./src/app/services/signaling.service.ts ***!
static/dist/main.js.map
Vendored
+1 -1
View File
File diff suppressed because one or more lines are too long
static/src/app/app.component.html
+1 -2
View File
@@ -1,2 +1 @@
<video #video autoplay muted controls></video>
<audio #audio autoplay controls></audio>
<ng-container #vc></ng-container>
static/src/app/app.component.scss
+1 -6
View File
@@ -5,11 +5,6 @@
justify-content: center;
flex-direction: column;
height: 100%;
width: 100%;
}
video {
max-height: 90%;
max-width: 80%;
height: 90%;
width: auto;
}
static/src/app/app.component.ts
+45 -26
View File
@@ -1,4 +1,6 @@
import { Component, ElementRef, OnInit, ViewChild } from '@angular/core';
import { Component, ComponentRef, HostListener, OnInit, ViewChild, ViewContainerRef } from '@angular/core';
import { VideoComponent } from './components/video/video.component';
import { AudioComponent } from './components/audio/audio.component';
import { SignalingService } from './services/signaling.service';
@Component({
@@ -7,18 +9,36 @@ import { SignalingService } from './services/signaling.service';
styleUrls: ['./app.component.scss']
})
export class AppComponent implements OnInit {
@ViewChild("vc", {static: true, read: ViewContainerRef })
public vcr!: ViewContainerRef;
@ViewChild('video', { static: true, read: ElementRef })
public videoElem!: ElementRef<HTMLVideoElement>;
@ViewChild('audio', { static: true, read: ElementRef })
public audioElem!: ElementRef<HTMLAudioElement>;
@ViewChild('start', { static: true, read: ElementRef })
public startElem!: ElementRef<HTMLButtonElement>;
private audioList: ComponentRef<AudioComponent>[] = [];
private selfAudioRunning = false;
private readonly pc: RTCPeerConnection;
@HostListener('document:click', ['$event.target'])
async onClick(_: KeyboardEvent) {
if (!this.selfAudioRunning) {
await this.startAudio()
}
for (const el of this.audioList) {
el.instance.toggleMute();
}
}
private async startAudio() {
const stream = await navigator.mediaDevices.getUserMedia({
audio: true,
video: false
});
this.pc.addTrack(stream.getAudioTracks()[0])
this.selfAudioRunning = true;
}
constructor(private signaling: SignalingService) {
//this.pc = new RTCPeerConnection({iceServers: [{urls: 'stun:stun.l.google.com:19302'}]});
this.pc = new RTCPeerConnection();
@@ -27,11 +47,16 @@ export class AppComponent implements OnInit {
this.pc.ontrack = (event) => {
const [remoteStream] = event.streams;
if (event.track.kind === 'video') {
this.videoElem.nativeElement.srcObject = remoteStream;
this.videoElem.nativeElement.muted = true;
this.videoElem.nativeElement.play();
const video = this.vcr.createComponent(VideoComponent, {});
video.instance.setStream(remoteStream);
video.instance.play();
console.log(video);
} else if (event.track.kind === 'audio') {
this.audioElem.nativeElement.srcObject = remoteStream;
const audio = this.vcr.createComponent(AudioComponent, {});
audio.instance.setStream(remoteStream);
console.log(audio);
this.audioList.push(audio)
}
}
@@ -48,7 +73,7 @@ export class AppComponent implements OnInit {
};
// Offer to receive 1 audio, and 1 video track
this.pc.addTransceiver('audio', {direction: 'recvonly'})
this.pc.addTransceiver('audio', {direction: 'sendrecv'})
this.pc.addTransceiver('video', {direction: 'recvonly'})
this.pc.createOffer().then(d => this.pc.setLocalDescription(d)).catch((e) => {
console.error(e);
@@ -56,14 +81,12 @@ export class AppComponent implements OnInit {
// Let the "negotiationneeded" event trigger offer generation.
this.pc.onnegotiationneeded = async (e) => {
console.log(e);
// try {
// await this.pc.setLocalDescription();
// // Send the offer to the other peer.
// this.api.send({desc: this.pc.localDescription});
// } catch (err) {
// console.error(err);
// }
try {
await this.pc.setLocalDescription();
signaling.next(this.pc.localDescription);
} catch (err) {
console.error(err);
}
};
this.signaling.subscribe(async (x) => {
@@ -86,8 +109,4 @@ export class AppComponent implements OnInit {
}
public run(e: MouseEvent): void {
this.videoElem.nativeElement.play();
}
}
static/src/app/app.module.ts
+5 -1
View File
@@ -3,10 +3,14 @@ import { BrowserModule } from '@angular/platform-browser';
import { HttpClientModule } from '@angular/common/http';
import { AppComponent } from './app.component';
import { SignalingService } from './services/signaling.service';
import { VideoComponent } from './components/video/video.component';
import { AudioComponent } from './components/audio/audio.component';
@NgModule({
declarations: [
AppComponent
AppComponent,
VideoComponent,
AudioComponent
],
imports: [
BrowserModule,
static/src/app/components/audio/audio.component.scss
View File
static/src/app/components/audio/audio.component.ts
+36
View File
@@ -0,0 +1,36 @@
import { Component, ElementRef, OnInit, ViewChild } from '@angular/core';
@Component({
selector: 'app-audio',
template: '<audio #audio></audio>',
styleUrls: ['./audio.component.scss']
})
export class AudioComponent implements OnInit {
@ViewChild('audio', { static: true, read: ElementRef })
public elem!: ElementRef<HTMLAudioElement>;
ngOnInit(): void {
this.elem.nativeElement.muted = true;
}
public setStream(stream: MediaStream): void {
this.elem.nativeElement.srcObject = stream;
}
public play(): void {
this.elem.nativeElement.play()
}
public pause(): void {
this.elem.nativeElement.pause()
}
public toggleMute(): void {
this.elem.nativeElement.muted = !this.elem.nativeElement.muted;
console.log('toggle mute to', this.elem.nativeElement.muted);
if (!this.elem.nativeElement.muted) {
this.play();
}
}
}
static/src/app/components/video/video.component.scss
+12
View File
@@ -0,0 +1,12 @@
:host {
max-height: 90%;
max-width: 80%;
height: 90%;
width: auto;
}
video {
height: 100%;
width: 100%;
}
static/src/app/components/video/video.component.ts
+29
View File
@@ -0,0 +1,29 @@
import { Component, ElementRef, ViewChild } from '@angular/core';
@Component({
selector: 'app-video',
template: '<video #video></video>',
styleUrls: ['./video.component.scss']
})
export class VideoComponent {
@ViewChild('video', { static: true, read: ElementRef })
public elem!: ElementRef<HTMLVideoElement>;
public setStream(stream: MediaStream): void {
this.elem.nativeElement.srcObject = stream;
this.elem.nativeElement.muted = true;
}
public play(): void {
this.elem.nativeElement.play()
}
public pause(): void {
this.elem.nativeElement.pause()
}
public toggleMute(): void {
this.elem.nativeElement.muted = !this.elem.nativeElement.muted;
console.log('toggle mute to', this.elem.nativeElement.muted);
}
}
test/main.go
-30
View File
@@ -1,30 +0,0 @@
package main
import (
"log"
"gitea.heinrich.blue/PHI/webrtc-gst/pkg/webrtc"
"github.com/tinyzimmer/go-glib/glib"
)
func main() {
loop := glib.NewMainLoop(glib.MainContextDefault(), false)
audioPipeline, audioCh, err := webrtc.CreateVideoPipeline("videotestsrc")
if err != nil {
panic(err)
}
go func() {
for s := range audioCh {
log.Println(s.Timestamp)
}
}()
err = audioPipeline.Start()
if err != nil {
panic(err)
}
loop.Run()
}
vendor/github.com/alexflint/go-arg/.gitignore
Generated
+24
View File
@@ -0,0 +1,24 @@
# Compiled Object files, Static and Dynamic libs (Shared Objects)
*.o
*.a
*.so
# Folders
_obj
_test
# Architecture specific extensions/prefixes
*.[568vq]
[568vq].out
*.cgo1.go
*.cgo2.c
_cgo_defun.c
_cgo_gotypes.go
_cgo_export.*
_testmain.go
*.exe
*.test
*.prof
vendor/github.com/alexflint/go-arg/LICENSE
Generated Vendored
+24
View File
@@ -0,0 +1,24 @@
Copyright (c) 2015, Alex Flint
All rights reserved.
Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions are met:
* Redistributions of source code must retain the above copyright notice, this
list of conditions and the following disclaimer.
* Redistributions in binary form must reproduce the above copyright notice,
this list of conditions and the following disclaimer in the documentation
and/or other materials provided with the distribution.
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
vendor/github.com/alexflint/go-arg/README.md
Generated Vendored
+552
View File
@@ -0,0 +1,552 @@
<h1 align="center">
<img src="./.github/banner.jpg" alt="go-arg" height="250px">
<br>
go-arg
</br>
</h1>
<h4 align="center">Struct-based argument parsing for Go</h4>
<p align="center">
<a href="https://sourcegraph.com/github.com/alexflint/go-arg?badge"><img src="https://sourcegraph.com/github.com/alexflint/go-arg/-/badge.svg" alt="Sourcegraph"></a>
<a href="https://pkg.go.dev/github.com/alexflint/go-arg"><img src="https://img.shields.io/badge/go.dev-reference-007d9c?logo=go&logoColor=white&style=flat-square" alt="Documentation"></a>
<a href="https://github.com/alexflint/go-arg/actions"><img src="https://github.com/alexflint/go-arg/workflows/Go/badge.svg" alt="Build Status"></a>
<a href="https://codecov.io/gh/alexflint/go-arg"><img src="https://codecov.io/gh/alexflint/go-arg/branch/master/graph/badge.svg" alt="Coverage Status"></a>
<a href="https://goreportcard.com/report/github.com/alexflint/go-arg"><img src="https://goreportcard.com/badge/github.com/alexflint/go-arg" alt="Go Report Card"></a>
</p>
<br>
Declare command line arguments for your program by defining a struct.
```go
var args struct {
Foo string
Bar bool
}
arg.MustParse(&args)
fmt.Println(args.Foo, args.Bar)
```
```shell
$ ./example --foo=hello --bar
hello true
```
### Installation
```shell
go get github.com/alexflint/go-arg
```
### Required arguments
```go
var args struct {
ID int `arg:"required"`
Timeout time.Duration
}
arg.MustParse(&args)
```
```shell
$ ./example
Usage: example --id ID [--timeout TIMEOUT]
error: --id is required
```
### Positional arguments
```go
var args struct {
Input string `arg:"positional"`
Output []string `arg:"positional"`
}
arg.MustParse(&args)
fmt.Println("Input:", args.Input)
fmt.Println("Output:", args.Output)
```
```
$ ./example src.txt x.out y.out z.out
Input: src.txt
Output: [x.out y.out z.out]
```
### Environment variables
```go
var args struct {
Workers int `arg:"env"`
}
arg.MustParse(&args)
fmt.Println("Workers:", args.Workers)
```
```
$ WORKERS=4 ./example
Workers: 4
```
```
$ WORKERS=4 ./example --workers=6
Workers: 6
```
You can also override the name of the environment variable:
```go
var args struct {
Workers int `arg:"env:NUM_WORKERS"`
}
arg.MustParse(&args)
fmt.Println("Workers:", args.Workers)
```
```
$ NUM_WORKERS=4 ./example
Workers: 4
```
You can provide multiple values using the CSV (RFC 4180) format:
```go
var args struct {
Workers []int `arg:"env"`
}
arg.MustParse(&args)
fmt.Println("Workers:", args.Workers)
```
```
$ WORKERS='1,99' ./example
Workers: [1 99]
```
### Usage strings
```go
var args struct {
Input string `arg:"positional"`
Output []string `arg:"positional"`
Verbose bool `arg:"-v,--verbose" help:"verbosity level"`
Dataset string `help:"dataset to use"`
Optimize int `arg:"-O" help:"optimization level"`
}
arg.MustParse(&args)
```
```shell
$ ./example -h
Usage: [--verbose] [--dataset DATASET] [--optimize OPTIMIZE] [--help] INPUT [OUTPUT [OUTPUT ...]]
Positional arguments:
INPUT
OUTPUT
Options:
--verbose, -v verbosity level
--dataset DATASET dataset to use
--optimize OPTIMIZE, -O OPTIMIZE
optimization level
--help, -h print this help message
```
### Default values
```go
var args struct {
Foo string `default:"abc"`
Bar bool
}
arg.MustParse(&args)
```
### Default values (before v1.2)
```go
var args struct {
Foo string
Bar bool
}
arg.Foo = "abc"
arg.MustParse(&args)
```
### Combining command line options, environment variables, and default values
You can combine command line arguments, environment variables, and default values. Command line arguments take precedence over environment variables, which take precedence over default values. This means that we check whether a certain option was provided on the command line, then if not, we check for an environment variable (only if an `env` tag was provided), then if none is found, we check for a `default` tag containing a default value.
```go
var args struct {
Test string `arg:"-t,env:TEST" default:"something"`
}
arg.MustParse(&args)
```
### Arguments with multiple values
```go
var args struct {
Database string
IDs []int64
}
arg.MustParse(&args)
fmt.Printf("Fetching the following IDs from %s: %q", args.Database, args.IDs)
```
```shell
./example -database foo -ids 1 2 3
Fetching the following IDs from foo: [1 2 3]
```
### Arguments that can be specified multiple times, mixed with positionals
```go
var args struct {
Commands []string `arg:"-c,separate"`
Files []string `arg:"-f,separate"`
Databases []string `arg:"positional"`
}
arg.MustParse(&args)
```
```shell
./example -c cmd1 db1 -f file1 db2 -c cmd2 -f file2 -f file3 db3 -c cmd3
Commands: [cmd1 cmd2 cmd3]
Files [file1 file2 file3]
Databases [db1 db2 db3]
```
### Arguments with keys and values
```go
var args struct {
UserIDs map[string]int
}
arg.MustParse(&args)
fmt.Println(args.UserIDs)
```
```shell
./example --userids john=123 mary=456
map[john:123 mary:456]
```
### Custom validation
```go
var args struct {
Foo string
Bar string
}
p := arg.MustParse(&args)
if args.Foo == "" && args.Bar == "" {
p.Fail("you must provide either --foo or --bar")
}
```
```shell
./example
Usage: samples [--foo FOO] [--bar BAR]
error: you must provide either --foo or --bar
```
### Version strings
```go
type args struct {
...
}
func (args) Version() string {
return "someprogram 4.3.0"
}
func main() {
var args args
arg.MustParse(&args)
}
```
```shell
$ ./example --version
someprogram 4.3.0
```
### Overriding option names
```go
var args struct {
Short string `arg:"-s"`
Long string `arg:"--custom-long-option"`
ShortAndLong string `arg:"-x,--my-option"`
OnlyShort string `arg:"-o,--"`
}
arg.MustParse(&args)
```
```shell
$ ./example --help
Usage: example [-o ONLYSHORT] [--short SHORT] [--custom-long-option CUSTOM-LONG-OPTION] [--my-option MY-OPTION]
Options:
--short SHORT, -s SHORT
--custom-long-option CUSTOM-LONG-OPTION
--my-option MY-OPTION, -x MY-OPTION
-o ONLYSHORT
--help, -h display this help and exit
```
### Embedded structs
The fields of embedded structs are treated just like regular fields:
```go
type DatabaseOptions struct {
Host string
Username string
Password string
}
type LogOptions struct {
LogFile string
Verbose bool
}
func main() {
var args struct {
DatabaseOptions
LogOptions
}
arg.MustParse(&args)
}
```
As usual, any field tagged with `arg:"-"` is ignored.
### Supported types
The following types may be used as arguments:
- built-in integer types: `int, int8, int16, int32, int64, byte, rune`
- built-in floating point types: `float32, float64`
- strings
- booleans
- URLs represented as `url.URL`
- time durations represented as `time.Duration`
- email addresses represented as `mail.Address`
- MAC addresses represented as `net.HardwareAddr`
- pointers to any of the above
- slices of any of the above
- maps using any of the above as keys and values
- any type that implements `encoding.TextUnmarshaler`
### Custom parsing
Implement `encoding.TextUnmarshaler` to define your own parsing logic.
```go
// Accepts command line arguments of the form "head.tail"
type NameDotName struct {
Head, Tail string
}
func (n *NameDotName) UnmarshalText(b []byte) error {
s := string(b)
pos := strings.Index(s, ".")
if pos == -1 {
return fmt.Errorf("missing period in %s", s)
}
n.Head = s[:pos]
n.Tail = s[pos+1:]
return nil
}
func main() {
var args struct {
Name NameDotName
}
arg.MustParse(&args)
fmt.Printf("%#v\n", args.Name)
}
```
```shell
$ ./example --name=foo.bar
main.NameDotName{Head:"foo", Tail:"bar"}
$ ./example --name=oops
Usage: example [--name NAME]
error: error processing --name: missing period in "oops"
```
### Custom parsing with default values
Implement `encoding.TextMarshaler` to define your own default value strings:
```go
// Accepts command line arguments of the form "head.tail"
type NameDotName struct {
Head, Tail string
}
func (n *NameDotName) UnmarshalText(b []byte) error {
// same as previous example
}
// this is only needed if you want to display a default value in the usage string
func (n *NameDotName) MarshalText() ([]byte, error) {
return []byte(fmt.Sprintf("%s.%s", n.Head, n.Tail)), nil
}
func main() {
var args struct {
Name NameDotName `default:"file.txt"`
}
arg.MustParse(&args)
fmt.Printf("%#v\n", args.Name)
}
```
```shell
$ ./example --help
Usage: test [--name NAME]
Options:
--name NAME [default: file.txt]
--help, -h display this help and exit
$ ./example
main.NameDotName{Head:"file", Tail:"txt"}
```
### Custom placeholders
*Introduced in version 1.3.0*
Use the `placeholder` tag to control which placeholder text is used in the usage text.
```go
var args struct {
Input string `arg:"positional" placeholder:"SRC"`
Output []string `arg:"positional" placeholder:"DST"`
Optimize int `arg:"-O" help:"optimization level" placeholder:"LEVEL"`
MaxJobs int `arg:"-j" help:"maximum number of simultaneous jobs" placeholder:"N"`
}
arg.MustParse(&args)
```
```shell
$ ./example -h
Usage: example [--optimize LEVEL] [--maxjobs N] SRC [DST [DST ...]]
Positional arguments:
SRC
DST
Options:
--optimize LEVEL, -O LEVEL
optimization level
--maxjobs N, -j N maximum number of simultaneous jobs
--help, -h display this help and exit
```
### Description strings
```go
type args struct {
Foo string
}
func (args) Description() string {
return "this program does this and that"
}
func main() {
var args args
arg.MustParse(&args)
}
```
```shell
$ ./example -h
this program does this and that
Usage: example [--foo FOO]
Options:
--foo FOO
--help, -h display this help and exit
```
### Subcommands
*Introduced in version 1.1.0*
Subcommands are commonly used in tools that wish to group multiple functions into a single program. An example is the `git` tool:
```shell
$ git checkout [arguments specific to checking out code]
$ git commit [arguments specific to committing]
$ git push [arguments specific to pushing]
```
The strings "checkout", "commit", and "push" are different from simple positional arguments because the options available to the user change depending on which subcommand they choose.
This can be implemented with `go-arg` as follows:
```go
type CheckoutCmd struct {
Branch string `arg:"positional"`
Track bool `arg:"-t"`
}
type CommitCmd struct {
All bool `arg:"-a"`
Message string `arg:"-m"`
}
type PushCmd struct {
Remote string `arg:"positional"`
Branch string `arg:"positional"`
SetUpstream bool `arg:"-u"`
}
var args struct {
Checkout *CheckoutCmd `arg:"subcommand:checkout"`
Commit *CommitCmd `arg:"subcommand:commit"`
Push *PushCmd `arg:"subcommand:push"`
Quiet bool `arg:"-q"` // this flag is global to all subcommands
}
arg.MustParse(&args)
switch {
case args.Checkout != nil:
fmt.Printf("checkout requested for branch %s\n", args.Checkout.Branch)
case args.Commit != nil:
fmt.Printf("commit requested with message \"%s\"\n", args.Commit.Message)
case args.Push != nil:
fmt.Printf("push requested from %s to %s\n", args.Push.Branch, args.Push.Remote)
}
```
Some additional rules apply when working with subcommands:
* The `subcommand` tag can only be used with fields that are pointers to structs
* Any struct that contains a subcommand must not contain any positionals
This package allows to have a program that accepts subcommands, but also does something else
when no subcommands are specified.
If on the other hand you want the program to terminate when no subcommands are specified,
the recommended way is:
```go
p := arg.MustParse(&args)
if p.Subcommand() == nil {
p.Fail("missing subcommand")
}
```
### API Documentation
https://godoc.org/github.com/alexflint/go-arg
### Rationale
There are many command line argument parsing libraries for Go, including one in the standard library, so why build another?
The `flag` library that ships in the standard library seems awkward to me. Positional arguments must preceed options, so `./prog x --foo=1` does what you expect but `./prog --foo=1 x` does not. It also does not allow arguments to have both long (`--foo`) and short (`-f`) forms.
Many third-party argument parsing libraries are great for writing sophisticated command line interfaces, but feel to me like overkill for a simple script with a few flags.
The idea behind `go-arg` is that Go already has an excellent way to describe data structures using structs, so there is no need to develop additional levels of abstraction. Instead of one API to specify which arguments your program accepts, and then another API to get the values of those arguments, `go-arg` replaces both with a single struct.
### Backward compatibility notes
Earlier versions of this library required the help text to be part of the `arg` tag. This is still supported but is now deprecated. Instead, you should use a separate `help` tag, described above, which removes most of the limits on the text you can write. In particular, you will need to use the new `help` tag if your help text includes any commas.
vendor/github.com/alexflint/go-arg/doc.go
Generated Vendored
+39
View File
@@ -0,0 +1,39 @@
// Package arg parses command line arguments using the fields from a struct.
//
// For example,
//
// var args struct {
// Iter int
// Debug bool
// }
// arg.MustParse(&args)
//
// defines two command line arguments, which can be set using any of
//
// ./example --iter=1 --debug // debug is a boolean flag so its value is set to true
// ./example -iter 1 // debug defaults to its zero value (false)
// ./example --debug=true // iter defaults to its zero value (zero)
//
// The fastest way to see how to use go-arg is to read the examples below.
//
// Fields can be bool, string, any float type, or any signed or unsigned integer type.
// They can also be slices of any of the above, or slices of pointers to any of the above.
//
// Tags can be specified using the `arg` and `help` tag names:
//
// var args struct {
// Input string `arg:"positional"`
// Log string `arg:"positional,required"`
// Debug bool `arg:"-d" help:"turn on debug mode"`
// RealMode bool `arg:"--real"
// Wr io.Writer `arg:"-"`
// }
//
// Any tag string that starts with a single hyphen is the short form for an argument
// (e.g. `./example -d`), and any tag string that starts with two hyphens is the long
// form for the argument (instead of the field name).
//
// Other valid tag strings are `positional` and `required`.
//
// Fields can be excluded from processing with `arg:"-"`.
package arg
vendor/github.com/alexflint/go-arg/parse.go
Generated Vendored
+729
View File
@@ -0,0 +1,729 @@
package arg
import (
"encoding"
"encoding/csv"
"errors"
"fmt"
"os"
"path/filepath"
"reflect"
"strings"
scalar "github.com/alexflint/go-scalar"
)
// path represents a sequence of steps to find the output location for an
// argument or subcommand in the final destination struct
type path struct {
root int // index of the destination struct
fields []reflect.StructField // sequence of struct fields to traverse
}
// String gets a string representation of the given path
func (p path) String() string {
s := "args"
for _, f := range p.fields {
s += "." + f.Name
}
return s
}
// Child gets a new path representing a child of this path.
func (p path) Child(f reflect.StructField) path {
// copy the entire slice of fields to avoid possible slice overwrite
subfields := make([]reflect.StructField, len(p.fields)+1)
copy(subfields, p.fields)
subfields[len(subfields)-1] = f
return path{
root: p.root,
fields: subfields,
}
}
// spec represents a command line option
type spec struct {
dest path
field reflect.StructField // the struct field from which this option was created
long string // the --long form for this option, or empty if none
short string // the -s short form for this option, or empty if none
cardinality cardinality // determines how many tokens will be present (possible values: zero, one, multiple)
required bool // if true, this option must be present on the command line
positional bool // if true, this option will be looked for in the positional flags
separate bool // if true, each slice and map entry will have its own --flag
help string // the help text for this option
env string // the name of the environment variable for this option, or empty for none
defaultVal string // default value for this option
placeholder string // name of the data in help
}
// command represents a named subcommand, or the top-level command
type command struct {
name string
help string
dest path
specs []*spec
subcommands []*command
parent *command
}
// ErrHelp indicates that -h or --help were provided
var ErrHelp = errors.New("help requested by user")
// ErrVersion indicates that --version was provided
var ErrVersion = errors.New("version requested by user")
// MustParse processes command line arguments and exits upon failure
func MustParse(dest ...interface{}) *Parser {
p, err := NewParser(Config{}, dest...)
if err != nil {
fmt.Fprintln(stdout, err)
osExit(-1)
return nil // just in case osExit was monkey-patched
}
err = p.Parse(flags())
switch {
case err == ErrHelp:
p.writeHelpForSubcommand(stdout, p.lastCmd)
osExit(0)
case err == ErrVersion:
fmt.Fprintln(stdout, p.version)
osExit(0)
case err != nil:
p.failWithSubcommand(err.Error(), p.lastCmd)
}
return p
}
// Parse processes command line arguments and stores them in dest
func Parse(dest ...interface{}) error {
p, err := NewParser(Config{}, dest...)
if err != nil {
return err
}
return p.Parse(flags())
}
// flags gets all command line arguments other than the first (program name)
func flags() []string {
if len(os.Args) == 0 { // os.Args could be empty
return nil
}
return os.Args[1:]
}
// Config represents configuration options for an argument parser
type Config struct {
// Program is the name of the program used in the help text
Program string
// IgnoreEnv instructs the library not to read environment variables
IgnoreEnv bool
}
// Parser represents a set of command line options with destination values
type Parser struct {
cmd *command
roots []reflect.Value
config Config
version string
description string
// the following field changes during processing of command line arguments
lastCmd *command
}
// Versioned is the interface that the destination struct should implement to
// make a version string appear at the top of the help message.
type Versioned interface {
// Version returns the version string that will be printed on a line by itself
// at the top of the help message.
Version() string
}
// Described is the interface that the destination struct should implement to
// make a description string appear at the top of the help message.
type Described interface {
// Description returns the string that will be printed on a line by itself
// at the top of the help message.
Description() string
}
// walkFields calls a function for each field of a struct, recursively expanding struct fields.
func walkFields(t reflect.Type, visit func(field reflect.StructField, owner reflect.Type) bool) {
walkFieldsImpl(t, visit, nil)
}
func walkFieldsImpl(t reflect.Type, visit func(field reflect.StructField, owner reflect.Type) bool, path []int) {
for i := 0; i < t.NumField(); i++ {
field := t.Field(i)
field.Index = make([]int, len(path)+1)
copy(field.Index, append(path, i))
expand := visit(field, t)
if expand && field.Type.Kind() == reflect.Struct {
var subpath []int
if field.Anonymous {
subpath = append(path, i)
}
walkFieldsImpl(field.Type, visit, subpath)
}
}
}
// NewParser constructs a parser from a list of destination structs
func NewParser(config Config, dests ...interface{}) (*Parser, error) {
// first pick a name for the command for use in the usage text
var name string
switch {
case config.Program != "":
name = config.Program
case len(os.Args) > 0:
name = filepath.Base(os.Args[0])
default:
name = "program"
}
// construct a parser
p := Parser{
cmd: &command{name: name},
config: config,
}
// make a list of roots
for _, dest := range dests {
p.roots = append(p.roots, reflect.ValueOf(dest))
}
// process each of the destination values
for i, dest := range dests {
t := reflect.TypeOf(dest)
if t.Kind() != reflect.Ptr {
panic(fmt.Sprintf("%s is not a pointer (did you forget an ampersand?)", t))
}
cmd, err := cmdFromStruct(name, path{root: i}, t)
if err != nil {
return nil, err
}
// add nonzero field values as defaults
for _, spec := range cmd.specs {
if v := p.val(spec.dest); v.IsValid() && !isZero(v) {
if defaultVal, ok := v.Interface().(encoding.TextMarshaler); ok {
str, err := defaultVal.MarshalText()
if err != nil {
return nil, fmt.Errorf("%v: error marshaling default value to string: %v", spec.dest, err)
}
spec.defaultVal = string(str)
} else {
spec.defaultVal = fmt.Sprintf("%v", v)
}
}
}
p.cmd.specs = append(p.cmd.specs, cmd.specs...)
p.cmd.subcommands = append(p.cmd.subcommands, cmd.subcommands...)
if dest, ok := dest.(Versioned); ok {
p.version = dest.Version()
}
if dest, ok := dest.(Described); ok {
p.description = dest.Description()
}
}
return &p, nil
}
func cmdFromStruct(name string, dest path, t reflect.Type) (*command, error) {
// commands can only be created from pointers to structs
if t.Kind() != reflect.Ptr {
return nil, fmt.Errorf("subcommands must be pointers to structs but %s is a %s",
dest, t.Kind())
}
t = t.Elem()
if t.Kind() != reflect.Struct {
return nil, fmt.Errorf("subcommands must be pointers to structs but %s is a pointer to %s",
dest, t.Kind())
}
cmd := command{
name: name,
dest: dest,
}
var errs []string
walkFields(t, func(field reflect.StructField, t reflect.Type) bool {
// check for the ignore switch in the tag
tag := field.Tag.Get("arg")
if tag == "-" {
return false
}
// if this is an embedded struct then recurse into its fields, even if
// it is unexported, because exported fields on unexported embedded
// structs are still writable
if field.Anonymous && field.Type.Kind() == reflect.Struct {
return true
}
// ignore any other unexported field
if !isExported(field.Name) {
return false
}
// duplicate the entire path to avoid slice overwrites
subdest := dest.Child(field)
spec := spec{
dest: subdest,
field: field,
long: strings.ToLower(field.Name),
}
help, exists := field.Tag.Lookup("help")
if exists {
spec.help = help
}
defaultVal, hasDefault := field.Tag.Lookup("default")
if hasDefault {
spec.defaultVal = defaultVal
}
// Look at the tag
var isSubcommand bool // tracks whether this field is a subcommand
for _, key := range strings.Split(tag, ",") {
if key == "" {
continue
}
key = strings.TrimLeft(key, " ")
var value string
if pos := strings.Index(key, ":"); pos != -1 {
value = key[pos+1:]
key = key[:pos]
}
switch {
case strings.HasPrefix(key, "---"):
errs = append(errs, fmt.Sprintf("%s.%s: too many hyphens", t.Name(), field.Name))
case strings.HasPrefix(key, "--"):
spec.long = key[2:]
case strings.HasPrefix(key, "-"):
if len(key) != 2 {
errs = append(errs, fmt.Sprintf("%s.%s: short arguments must be one character only",
t.Name(), field.Name))
return false
}
spec.short = key[1:]
case key == "required":
if hasDefault {
errs = append(errs, fmt.Sprintf("%s.%s: 'required' cannot be used when a default value is specified",
t.Name(), field.Name))
return false
}
spec.required = true
case key == "positional":
spec.positional = true
case key == "separate":
spec.separate = true
case key == "help": // deprecated
spec.help = value
case key == "env":
// Use override name if provided
if value != "" {
spec.env = value
} else {
spec.env = strings.ToUpper(field.Name)
}
case key == "subcommand":
// decide on a name for the subcommand
cmdname := value
if cmdname == "" {
cmdname = strings.ToLower(field.Name)
}
// parse the subcommand recursively
subcmd, err := cmdFromStruct(cmdname, subdest, field.Type)
if err != nil {
errs = append(errs, err.Error())
return false
}
subcmd.parent = &cmd
subcmd.help = field.Tag.Get("help")
cmd.subcommands = append(cmd.subcommands, subcmd)
isSubcommand = true
default:
errs = append(errs, fmt.Sprintf("unrecognized tag '%s' on field %s", key, tag))
return false
}
}
placeholder, hasPlaceholder := field.Tag.Lookup("placeholder")
if hasPlaceholder {
spec.placeholder = placeholder
} else if spec.long != "" {
spec.placeholder = strings.ToUpper(spec.long)
} else {
spec.placeholder = strings.ToUpper(spec.field.Name)
}
// Check whether this field is supported. It's good to do this here rather than
// wait until ParseValue because it means that a program with invalid argument
// fields will always fail regardless of whether the arguments it received
// exercised those fields.
if !isSubcommand {
cmd.specs = append(cmd.specs, &spec)
var err error
spec.cardinality, err = cardinalityOf(field.Type)
if err != nil {
errs = append(errs, fmt.Sprintf("%s.%s: %s fields are not supported",
t.Name(), field.Name, field.Type.String()))
return false
}
if spec.cardinality == multiple && hasDefault {
errs = append(errs, fmt.Sprintf("%s.%s: default values are not supported for slice or map fields",
t.Name(), field.Name))
return false
}
}
// if this was an embedded field then we already returned true up above
return false
})
if len(errs) > 0 {
return nil, errors.New(strings.Join(errs, "\n"))
}
// check that we don't have both positionals and subcommands
var hasPositional bool
for _, spec := range cmd.specs {
if spec.positional {
hasPositional = true
}
}
if hasPositional && len(cmd.subcommands) > 0 {
return nil, fmt.Errorf("%s cannot have both subcommands and positional arguments", dest)
}
return &cmd, nil
}
// Parse processes the given command line option, storing the results in the field
// of the structs from which NewParser was constructed
func (p *Parser) Parse(args []string) error {
err := p.process(args)
if err != nil {
// If -h or --help were specified then make sure help text supercedes other errors
for _, arg := range args {
if arg == "-h" || arg == "--help" {
return ErrHelp
}
if arg == "--" {
break
}
}
}
return err
}
// process environment vars for the given arguments
func (p *Parser) captureEnvVars(specs []*spec, wasPresent map[*spec]bool) error {
for _, spec := range specs {
if spec.env == "" {
continue
}
value, found := os.LookupEnv(spec.env)
if !found {
continue
}
if spec.cardinality == multiple {
// expect a CSV string in an environment
// variable in the case of multiple values
var values []string
var err error
if len(strings.TrimSpace(value)) > 0 {
values, err = csv.NewReader(strings.NewReader(value)).Read()
if err != nil {
return fmt.Errorf(
"error reading a CSV string from environment variable %s with multiple values: %v",
spec.env,
err,
)
}
}
if err = setSliceOrMap(p.val(spec.dest), values, !spec.separate); err != nil {
return fmt.Errorf(
"error processing environment variable %s with multiple values: %v",
spec.env,
err,
)
}
} else {
if err := scalar.ParseValue(p.val(spec.dest), value); err != nil {
return fmt.Errorf("error processing environment variable %s: %v", spec.env, err)
}
}
wasPresent[spec] = true
}
return nil
}
// process goes through arguments one-by-one, parses them, and assigns the result to
// the underlying struct field
func (p *Parser) process(args []string) error {
// track the options we have seen
wasPresent := make(map[*spec]bool)
// union of specs for the chain of subcommands encountered so far
curCmd := p.cmd
p.lastCmd = curCmd
// make a copy of the specs because we will add to this list each time we expand a subcommand
specs := make([]*spec, len(curCmd.specs))
copy(specs, curCmd.specs)
// deal with environment vars
if !p.config.IgnoreEnv {
err := p.captureEnvVars(specs, wasPresent)
if err != nil {
return err
}
}
// process each string from the command line
var allpositional bool
var positionals []string
// must use explicit for loop, not range, because we manipulate i inside the loop
for i := 0; i < len(args); i++ {
arg := args[i]
if arg == "--" {
allpositional = true
continue
}
if !isFlag(arg) || allpositional {
// each subcommand can have either subcommands or positionals, but not both
if len(curCmd.subcommands) == 0 {
positionals = append(positionals, arg)
continue
}
// if we have a subcommand then make sure it is valid for the current context
subcmd := findSubcommand(curCmd.subcommands, arg)
if subcmd == nil {
return fmt.Errorf("invalid subcommand: %s", arg)
}
// instantiate the field to point to a new struct
v := p.val(subcmd.dest)
v.Set(reflect.New(v.Type().Elem())) // we already checked that all subcommands are struct pointers
// add the new options to the set of allowed options
specs = append(specs, subcmd.specs...)
// capture environment vars for these new options
if !p.config.IgnoreEnv {
err := p.captureEnvVars(subcmd.specs, wasPresent)
if err != nil {
return err
}
}
curCmd = subcmd
p.lastCmd = curCmd
continue
}
// check for special --help and --version flags
switch arg {
case "-h", "--help":
return ErrHelp
case "--version":
return ErrVersion
}
// check for an equals sign, as in "--foo=bar"
var value string
opt := strings.TrimLeft(arg, "-")
if pos := strings.Index(opt, "="); pos != -1 {
value = opt[pos+1:]
opt = opt[:pos]
}
// lookup the spec for this option (note that the "specs" slice changes as
// we expand subcommands so it is better not to use a map)
spec := findOption(specs, opt)
if spec == nil {
return fmt.Errorf("unknown argument %s", arg)
}
wasPresent[spec] = true
// deal with the case of multiple values
if spec.cardinality == multiple {
var values []string
if value == "" {
for i+1 < len(args) && !isFlag(args[i+1]) && args[i+1] != "--" {
values = append(values, args[i+1])
i++
if spec.separate {
break
}
}
} else {
values = append(values, value)
}
err := setSliceOrMap(p.val(spec.dest), values, !spec.separate)
if err != nil {
return fmt.Errorf("error processing %s: %v", arg, err)
}
continue
}
// if it's a flag and it has no value then set the value to true
// use boolean because this takes account of TextUnmarshaler
if spec.cardinality == zero && value == "" {
value = "true"
}
// if we have something like "--foo" then the value is the next argument
if value == "" {
if i+1 == len(args) {
return fmt.Errorf("missing value for %s", arg)
}
if !nextIsNumeric(spec.field.Type, args[i+1]) && isFlag(args[i+1]) {
return fmt.Errorf("missing value for %s", arg)
}
value = args[i+1]
i++
}
err := scalar.ParseValue(p.val(spec.dest), value)
if err != nil {
return fmt.Errorf("error processing %s: %v", arg, err)
}
}
// process positionals
for _, spec := range specs {
if !spec.positional {
continue
}
if len(positionals) == 0 {
break
}
wasPresent[spec] = true
if spec.cardinality == multiple {
err := setSliceOrMap(p.val(spec.dest), positionals, true)
if err != nil {
return fmt.Errorf("error processing %s: %v", spec.field.Name, err)
}
positionals = nil
} else {
err := scalar.ParseValue(p.val(spec.dest), positionals[0])
if err != nil {
return fmt.Errorf("error processing %s: %v", spec.field.Name, err)
}
positionals = positionals[1:]
}
}
if len(positionals) > 0 {
return fmt.Errorf("too many positional arguments at '%s'", positionals[0])
}
// fill in defaults and check that all the required args were provided
for _, spec := range specs {
if wasPresent[spec] {
continue
}
name := strings.ToLower(spec.field.Name)
if spec.long != "" && !spec.positional {
name = "--" + spec.long
}
if spec.required {
msg := fmt.Sprintf("%s is required", name)
if spec.env != "" {
msg += " (or environment variable " + spec.env + ")"
}
return errors.New(msg)
}
if spec.defaultVal != "" {
err := scalar.ParseValue(p.val(spec.dest), spec.defaultVal)
if err != nil {
return fmt.Errorf("error processing default value for %s: %v", name, err)
}
}
}
return nil
}
func nextIsNumeric(t reflect.Type, s string) bool {
switch t.Kind() {
case reflect.Ptr:
return nextIsNumeric(t.Elem(), s)
case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64, reflect.Float32, reflect.Float64, reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64, reflect.Uintptr:
v := reflect.New(t)
err := scalar.ParseValue(v, s)
return err == nil
default:
return false
}
}
// isFlag returns true if a token is a flag such as "-v" or "--user" but not "-" or "--"
func isFlag(s string) bool {
return strings.HasPrefix(s, "-") && strings.TrimLeft(s, "-") != ""
}
// val returns a reflect.Value corresponding to the current value for the
// given path
func (p *Parser) val(dest path) reflect.Value {
v := p.roots[dest.root]
for _, field := range dest.fields {
if v.Kind() == reflect.Ptr {
if v.IsNil() {
return reflect.Value{}
}
v = v.Elem()
}
v = v.FieldByIndex(field.Index)
}
return v
}
// findOption finds an option from its name, or returns null if no spec is found
func findOption(specs []*spec, name string) *spec {
for _, spec := range specs {
if spec.positional {
continue
}
if spec.long == name || spec.short == name {
return spec
}
}
return nil
}
// findSubcommand finds a subcommand using its name, or returns null if no subcommand is found
func findSubcommand(cmds []*command, name string) *command {
for _, cmd := range cmds {
if cmd.name == name {
return cmd
}
}
return nil
}
vendor/github.com/alexflint/go-arg/reflect.go
Generated Vendored
+107
View File
@@ -0,0 +1,107 @@
package arg
import (
"encoding"
"fmt"
"reflect"
"unicode"
"unicode/utf8"
scalar "github.com/alexflint/go-scalar"
)
var textUnmarshalerType = reflect.TypeOf([]encoding.TextUnmarshaler{}).Elem()
// cardinality tracks how many tokens are expected for a given spec
// - zero is a boolean, which does to expect any value
// - one is an ordinary option that will be parsed from a single token
// - multiple is a slice or map that can accept zero or more tokens
type cardinality int
const (
zero cardinality = iota
one
multiple
unsupported
)
func (k cardinality) String() string {
switch k {
case zero:
return "zero"
case one:
return "one"
case multiple:
return "multiple"
case unsupported:
return "unsupported"
default:
return fmt.Sprintf("unknown(%d)", int(k))
}
}
// cardinalityOf returns true if the type can be parsed from a string
func cardinalityOf(t reflect.Type) (cardinality, error) {
if scalar.CanParse(t) {
if isBoolean(t) {
return zero, nil
}
return one, nil
}
// look inside pointer types
if t.Kind() == reflect.Ptr {
t = t.Elem()
}
// look inside slice and map types
switch t.Kind() {
case reflect.Slice:
if !scalar.CanParse(t.Elem()) {
return unsupported, fmt.Errorf("cannot parse into %v because %v not supported", t, t.Elem())
}
return multiple, nil
case reflect.Map:
if !scalar.CanParse(t.Key()) {
return unsupported, fmt.Errorf("cannot parse into %v because key type %v not supported", t, t.Elem())
}
if !scalar.CanParse(t.Elem()) {
return unsupported, fmt.Errorf("cannot parse into %v because value type %v not supported", t, t.Elem())
}
return multiple, nil
default:
return unsupported, fmt.Errorf("cannot parse into %v", t)
}
}
// isBoolean returns true if the type can be parsed from a single string
func isBoolean(t reflect.Type) bool {
switch {
case t.Implements(textUnmarshalerType):
return false
case t.Kind() == reflect.Bool:
return true
case t.Kind() == reflect.Ptr && t.Elem().Kind() == reflect.Bool:
return true
default:
return false
}
}
// isExported returns true if the struct field name is exported
func isExported(field string) bool {
r, _ := utf8.DecodeRuneInString(field) // returns RuneError for empty string or invalid UTF8
return unicode.IsLetter(r) && unicode.IsUpper(r)
}
// isZero returns true if v contains the zero value for its type
func isZero(v reflect.Value) bool {
t := v.Type()
if t.Kind() == reflect.Slice || t.Kind() == reflect.Map {
return v.IsNil()
}
if !t.Comparable() {
return false
}
return v.Interface() == reflect.Zero(t).Interface()
}
vendor/github.com/alexflint/go-arg/sequence.go
Generated Vendored
+123
View File
@@ -0,0 +1,123 @@
package arg
import (
"fmt"
"reflect"
"strings"
scalar "github.com/alexflint/go-scalar"
)
// setSliceOrMap parses a sequence of strings into a slice or map. If clear is
// true then any values already in the slice or map are first removed.
func setSliceOrMap(dest reflect.Value, values []string, clear bool) error {
if !dest.CanSet() {
return fmt.Errorf("field is not writable")
}
t := dest.Type()
if t.Kind() == reflect.Ptr {
dest = dest.Elem()
t = t.Elem()
}
switch t.Kind() {
case reflect.Slice:
return setSlice(dest, values, clear)
case reflect.Map:
return setMap(dest, values, clear)
default:
return fmt.Errorf("setSliceOrMap cannot insert values into a %v", t)
}
}
// setSlice parses a sequence of strings and inserts them into a slice. If clear
// is true then any values already in the slice are removed.
func setSlice(dest reflect.Value, values []string, clear bool) error {
var ptr bool
elem := dest.Type().Elem()
if elem.Kind() == reflect.Ptr && !elem.Implements(textUnmarshalerType) {
ptr = true
elem = elem.Elem()
}
// clear the slice in case default values exist
if clear && !dest.IsNil() {
dest.SetLen(0)
}
// parse the values one-by-one
for _, s := range values {
v := reflect.New(elem)
if err := scalar.ParseValue(v.Elem(), s); err != nil {
return err
}
if !ptr {
v = v.Elem()
}
dest.Set(reflect.Append(dest, v))
}
return nil
}
// setMap parses a sequence of name=value strings and inserts them into a map.
// If clear is true then any values already in the map are removed.
func setMap(dest reflect.Value, values []string, clear bool) error {
// determine the key and value type
var keyIsPtr bool
keyType := dest.Type().Key()
if keyType.Kind() == reflect.Ptr && !keyType.Implements(textUnmarshalerType) {
keyIsPtr = true
keyType = keyType.Elem()
}
var valIsPtr bool
valType := dest.Type().Elem()
if valType.Kind() == reflect.Ptr && !valType.Implements(textUnmarshalerType) {
valIsPtr = true
valType = valType.Elem()
}
// clear the slice in case default values exist
if clear && !dest.IsNil() {
for _, k := range dest.MapKeys() {
dest.SetMapIndex(k, reflect.Value{})
}
}
// allocate the map if it is not allocated
if dest.IsNil() {
dest.Set(reflect.MakeMap(dest.Type()))
}
// parse the values one-by-one
for _, s := range values {
// split at the first equals sign
pos := strings.Index(s, "=")
if pos == -1 {
return fmt.Errorf("cannot parse %q into a map, expected format key=value", s)
}
// parse the key
k := reflect.New(keyType)
if err := scalar.ParseValue(k.Elem(), s[:pos]); err != nil {
return err
}
if !keyIsPtr {
k = k.Elem()
}
// parse the value
v := reflect.New(valType)
if err := scalar.ParseValue(v.Elem(), s[pos+1:]); err != nil {
return err
}
if !valIsPtr {
v = v.Elem()
}
// add it to the map
dest.SetMapIndex(k, v)
}
return nil
}
vendor/github.com/alexflint/go-arg/subcommand.go
Generated Vendored
+37
View File
@@ -0,0 +1,37 @@
package arg
// Subcommand returns the user struct for the subcommand selected by
// the command line arguments most recently processed by the parser.
// The return value is always a pointer to a struct. If no subcommand
// was specified then it returns the top-level arguments struct. If
// no command line arguments have been processed by this parser then it
// returns nil.
func (p *Parser) Subcommand() interface{} {
if p.lastCmd == nil || p.lastCmd.parent == nil {
return nil
}
return p.val(p.lastCmd.dest).Interface()
}
// SubcommandNames returns the sequence of subcommands specified by the
// user. If no subcommands were given then it returns an empty slice.
func (p *Parser) SubcommandNames() []string {
if p.lastCmd == nil {
return nil
}
// make a list of ancestor commands
var ancestors []string
cur := p.lastCmd
for cur.parent != nil { // we want to exclude the root
ancestors = append(ancestors, cur.name)
cur = cur.parent
}
// reverse the list
out := make([]string, len(ancestors))
for i := 0; i < len(ancestors); i++ {
out[i] = ancestors[len(ancestors)-i-1]
}
return out
}
vendor/github.com/alexflint/go-arg/usage.go
Generated Vendored
+335
View File
@@ -0,0 +1,335 @@
package arg
import (
"fmt"
"io"
"os"
"strings"
)
// the width of the left column
const colWidth = 25
// to allow monkey patching in tests
var (
stdout io.Writer = os.Stdout
stderr io.Writer = os.Stderr
osExit = os.Exit
)
// Fail prints usage information to stderr and exits with non-zero status
func (p *Parser) Fail(msg string) {
p.failWithSubcommand(msg, p.cmd)
}
// FailSubcommand prints usage information for a specified subcommand to stderr,
// then exits with non-zero status. To write usage information for a top-level
// subcommand, provide just the name of that subcommand. To write usage
// information for a subcommand that is nested under another subcommand, provide
// a sequence of subcommand names starting with the top-level subcommand and so
// on down the tree.
func (p *Parser) FailSubcommand(msg string, subcommand ...string) error {
cmd, err := p.lookupCommand(subcommand...)
if err != nil {
return err
}
p.failWithSubcommand(msg, cmd)
return nil
}
// failWithSubcommand prints usage information for the given subcommand to stderr and exits with non-zero status
func (p *Parser) failWithSubcommand(msg string, cmd *command) {
p.writeUsageForSubcommand(stderr, cmd)
fmt.Fprintln(stderr, "error:", msg)
osExit(-1)
}
// WriteUsage writes usage information to the given writer
func (p *Parser) WriteUsage(w io.Writer) {
cmd := p.cmd
if p.lastCmd != nil {
cmd = p.lastCmd
}
p.writeUsageForSubcommand(w, cmd)
}
// WriteUsageForSubcommand writes the usage information for a specified
// subcommand. To write usage information for a top-level subcommand, provide
// just the name of that subcommand. To write usage information for a subcommand
// that is nested under another subcommand, provide a sequence of subcommand
// names starting with the top-level subcommand and so on down the tree.
func (p *Parser) WriteUsageForSubcommand(w io.Writer, subcommand ...string) error {
cmd, err := p.lookupCommand(subcommand...)
if err != nil {
return err
}
p.writeUsageForSubcommand(w, cmd)
return nil
}
// writeUsageForSubcommand writes usage information for the given subcommand
func (p *Parser) writeUsageForSubcommand(w io.Writer, cmd *command) {
var positionals, longOptions, shortOptions []*spec
for _, spec := range cmd.specs {
switch {
case spec.positional:
positionals = append(positionals, spec)
case spec.long != "":
longOptions = append(longOptions, spec)
case spec.short != "":
shortOptions = append(shortOptions, spec)
}
}
if p.version != "" {
fmt.Fprintln(w, p.version)
}
// make a list of ancestor commands so that we print with full context
var ancestors []string
ancestor := cmd
for ancestor != nil {
ancestors = append(ancestors, ancestor.name)
ancestor = ancestor.parent
}
// print the beginning of the usage string
fmt.Fprint(w, "Usage:")
for i := len(ancestors) - 1; i >= 0; i-- {
fmt.Fprint(w, " "+ancestors[i])
}
// write the option component of the usage message
for _, spec := range shortOptions {
// prefix with a space
fmt.Fprint(w, " ")
if !spec.required {
fmt.Fprint(w, "[")
}
fmt.Fprint(w, synopsis(spec, "-"+spec.short))
if !spec.required {
fmt.Fprint(w, "]")
}
}
for _, spec := range longOptions {
// prefix with a space
fmt.Fprint(w, " ")
if !spec.required {
fmt.Fprint(w, "[")
}
fmt.Fprint(w, synopsis(spec, "--"+spec.long))
if !spec.required {
fmt.Fprint(w, "]")
}
}
// When we parse positionals, we check that:
// 1. required positionals come before non-required positionals
// 2. there is at most one multiple-value positional
// 3. if there is a multiple-value positional then it comes after all other positionals
// Here we merely print the usage string, so we do not explicitly re-enforce those rules
// write the positionals in following form:
// REQUIRED1 REQUIRED2
// REQUIRED1 REQUIRED2 [OPTIONAL1 [OPTIONAL2]]
// REQUIRED1 REQUIRED2 REPEATED [REPEATED ...]
// REQUIRED1 REQUIRED2 [REPEATEDOPTIONAL [REPEATEDOPTIONAL ...]]
// REQUIRED1 REQUIRED2 [OPTIONAL1 [REPEATEDOPTIONAL [REPEATEDOPTIONAL ...]]]
var closeBrackets int
for _, spec := range positionals {
fmt.Fprint(w, " ")
if !spec.required {
fmt.Fprint(w, "[")
closeBrackets += 1
}
if spec.cardinality == multiple {
fmt.Fprintf(w, "%s [%s ...]", spec.placeholder, spec.placeholder)
} else {
fmt.Fprint(w, spec.placeholder)
}
}
fmt.Fprint(w, strings.Repeat("]", closeBrackets))
// if the program supports subcommands, give a hint to the user about their existence
if len(cmd.subcommands) > 0 {
fmt.Fprint(w, " <command> [<args>]")
}
fmt.Fprint(w, "\n")
}
func printTwoCols(w io.Writer, left, help string, defaultVal string, envVal string) {
lhs := " " + left
fmt.Fprint(w, lhs)
if help != "" {
if len(lhs)+2 < colWidth {
fmt.Fprint(w, strings.Repeat(" ", colWidth-len(lhs)))
} else {
fmt.Fprint(w, "\n"+strings.Repeat(" ", colWidth))
}
fmt.Fprint(w, help)
}
bracketsContent := []string{}
if defaultVal != "" {
bracketsContent = append(bracketsContent,
fmt.Sprintf("default: %s", defaultVal),
)
}
if envVal != "" {
bracketsContent = append(bracketsContent,
fmt.Sprintf("env: %s", envVal),
)
}
if len(bracketsContent) > 0 {
fmt.Fprintf(w, " [%s]", strings.Join(bracketsContent, ", "))
}
fmt.Fprint(w, "\n")
}
// WriteHelp writes the usage string followed by the full help string for each option
func (p *Parser) WriteHelp(w io.Writer) {
cmd := p.cmd
if p.lastCmd != nil {
cmd = p.lastCmd
}
p.writeHelpForSubcommand(w, cmd)
}
// WriteHelpForSubcommand writes the usage string followed by the full help
// string for a specified subcommand. To write help for a top-level subcommand,
// provide just the name of that subcommand. To write help for a subcommand that
// is nested under another subcommand, provide a sequence of subcommand names
// starting with the top-level subcommand and so on down the tree.
func (p *Parser) WriteHelpForSubcommand(w io.Writer, subcommand ...string) error {
cmd, err := p.lookupCommand(subcommand...)
if err != nil {
return err
}
p.writeHelpForSubcommand(w, cmd)
return nil
}
// writeHelp writes the usage string for the given subcommand
func (p *Parser) writeHelpForSubcommand(w io.Writer, cmd *command) {
var positionals, longOptions, shortOptions []*spec
for _, spec := range cmd.specs {
switch {
case spec.positional:
positionals = append(positionals, spec)
case spec.long != "":
longOptions = append(longOptions, spec)
case spec.short != "":
shortOptions = append(shortOptions, spec)
}
}
if p.description != "" {
fmt.Fprintln(w, p.description)
}
p.writeUsageForSubcommand(w, cmd)
// write the list of positionals
if len(positionals) > 0 {
fmt.Fprint(w, "\nPositional arguments:\n")
for _, spec := range positionals {
printTwoCols(w, spec.placeholder, spec.help, "", "")
}
}
// write the list of options with the short-only ones first to match the usage string
if len(shortOptions)+len(longOptions) > 0 || cmd.parent == nil {
fmt.Fprint(w, "\nOptions:\n")
for _, spec := range shortOptions {
p.printOption(w, spec)
}
for _, spec := range longOptions {
p.printOption(w, spec)
}
}
// obtain a flattened list of options from all ancestors
var globals []*spec
ancestor := cmd.parent
for ancestor != nil {
globals = append(globals, ancestor.specs...)
ancestor = ancestor.parent
}
// write the list of global options
if len(globals) > 0 {
fmt.Fprint(w, "\nGlobal options:\n")
for _, spec := range globals {
p.printOption(w, spec)
}
}
// write the list of built in options
p.printOption(w, &spec{
cardinality: zero,
long: "help",
short: "h",
help: "display this help and exit",
})
if p.version != "" {
p.printOption(w, &spec{
cardinality: zero,
long: "version",
help: "display version and exit",
})
}
// write the list of subcommands
if len(cmd.subcommands) > 0 {
fmt.Fprint(w, "\nCommands:\n")
for _, subcmd := range cmd.subcommands {
printTwoCols(w, subcmd.name, subcmd.help, "", "")
}
}
}
func (p *Parser) printOption(w io.Writer, spec *spec) {
ways := make([]string, 0, 2)
if spec.long != "" {
ways = append(ways, synopsis(spec, "--"+spec.long))
}
if spec.short != "" {
ways = append(ways, synopsis(spec, "-"+spec.short))
}
if len(ways) > 0 {
printTwoCols(w, strings.Join(ways, ", "), spec.help, spec.defaultVal, spec.env)
}
}
// lookupCommand finds a subcommand based on a sequence of subcommand names. The
// first string should be a top-level subcommand, the next should be a child
// subcommand of that subcommand, and so on. If no strings are given then the
// root command is returned. If no such subcommand exists then an error is
// returned.
func (p *Parser) lookupCommand(path ...string) (*command, error) {
cmd := p.cmd
for _, name := range path {
var found *command
for _, child := range cmd.subcommands {
if child.name == name {
found = child
}
}
if found == nil {
return nil, fmt.Errorf("%q is not a subcommand of %s", name, cmd.name)
}
cmd = found
}
return cmd, nil
}
func synopsis(spec *spec, form string) string {
if spec.cardinality == zero {
return form
}
return form + " " + spec.placeholder
}
vendor/github.com/alexflint/go-scalar/.gitignore
Generated
+24
View File
@@ -0,0 +1,24 @@
# Compiled Object files, Static and Dynamic libs (Shared Objects)
*.o
*.a
*.so
# Folders
_obj
_test
# Architecture specific extensions/prefixes
*.[568vq]
[568vq].out
*.cgo1.go
*.cgo2.c
_cgo_defun.c
_cgo_gotypes.go
_cgo_export.*
_testmain.go
*.exe
*.test
*.prof
vendor/github.com/alexflint/go-scalar/LICENSE
Generated Vendored
+24
View File
@@ -0,0 +1,24 @@
Copyright (c) 2015, Alex Flint
All rights reserved.
Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions are met:
* Redistributions of source code must retain the above copyright notice, this
list of conditions and the following disclaimer.
* Redistributions in binary form must reproduce the above copyright notice,
this list of conditions and the following disclaimer in the documentation
and/or other materials provided with the distribution.
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
vendor/github.com/alexflint/go-scalar/README.md
Generated Vendored
+28
View File
@@ -0,0 +1,28 @@
[![GoDoc](https://godoc.org/github.com/alexflint/go-scalar?status.svg)](https://godoc.org/github.com/alexflint/go-scalar)
[![Build Status](https://travis-ci.org/alexflint/go-scalar.svg?branch=master)](https://travis-ci.org/alexflint/go-scalar)
[![Coverage Status](https://coveralls.io/repos/alexflint/go-scalar/badge.svg?branch=master&service=github)](https://coveralls.io/github/alexflint/go-scalar?branch=master)
[![Report Card](https://goreportcard.com/badge/github.com/alexflint/go-scalar)](https://goreportcard.com/badge/github.com/alexflint/go-scalar)
## Scalar parsing library
Scalar is a library for parsing strings into arbitrary scalars (integers,
floats, strings, booleans, etc). It is helpful for tasks such as parsing
strings passed as environment variables or command line arguments.
```shell
go get github.com/alexflint/go-scalar
```
The main API works as follows:
```go
var value int
err := scalar.Parse(&value, "123")
```
There is also a variant that takes a `reflect.Value`:
```go
var value int
err := scalar.ParseValue(reflect.ValueOf(&value), "123")
```
vendor/github.com/alexflint/go-scalar/scalar.go
Generated Vendored
+162
View File
@@ -0,0 +1,162 @@
// Package scalar parses strings into values of scalar type.
package scalar
import (
"encoding"
"errors"
"fmt"
"net"
"net/mail"
"net/url"
"reflect"
"strconv"
"time"
)
// The reflected form of some special types
var (
textUnmarshalerType = reflect.TypeOf([]encoding.TextUnmarshaler{}).Elem()
durationType = reflect.TypeOf(time.Duration(0))
mailAddressType = reflect.TypeOf(mail.Address{})
macType = reflect.TypeOf(net.HardwareAddr{})
urlType = reflect.TypeOf(url.URL{})
)
var (
errNotSettable = errors.New("value is not settable")
errPtrNotSettable = errors.New("value is a nil pointer and is not settable")
)
// Parse assigns a value to v by parsing s.
func Parse(dest interface{}, s string) error {
return ParseValue(reflect.ValueOf(dest), s)
}
// ParseValue assigns a value to v by parsing s.
func ParseValue(v reflect.Value, s string) error {
// If we have a nil pointer then allocate a new object
if v.Kind() == reflect.Ptr && v.IsNil() {
if !v.CanSet() {
return errPtrNotSettable
}
v.Set(reflect.New(v.Type().Elem()))
}
// If it implements encoding.TextUnmarshaler then use that
if scalar, ok := v.Interface().(encoding.TextUnmarshaler); ok {
return scalar.UnmarshalText([]byte(s))
}
// If it's a value instead of a pointer, check that we can unmarshal it
// via TextUnmarshaler as well
if v.CanAddr() {
if scalar, ok := v.Addr().Interface().(encoding.TextUnmarshaler); ok {
return scalar.UnmarshalText([]byte(s))
}
}
// If we have a pointer then dereference it
if v.Kind() == reflect.Ptr {
v = v.Elem()
}
if !v.CanSet() {
return errNotSettable
}
// Switch on concrete type
switch scalar := v.Interface(); scalar.(type) {
case time.Duration:
duration, err := time.ParseDuration(s)
if err != nil {
return err
}
v.Set(reflect.ValueOf(duration))
return nil
case mail.Address:
addr, err := mail.ParseAddress(s)
if err != nil {
return err
}
v.Set(reflect.ValueOf(*addr))
return nil
case net.HardwareAddr:
ip, err := net.ParseMAC(s)
if err != nil {
return err
}
v.Set(reflect.ValueOf(ip))
return nil
case url.URL:
url, err := url.Parse(s)
if err != nil {
return err
}
v.Set(reflect.ValueOf(*url))
return nil
}
// Switch on kind so that we can handle derived types
switch v.Kind() {
case reflect.String:
v.SetString(s)
case reflect.Bool:
x, err := strconv.ParseBool(s)
if err != nil {
return err
}
v.SetBool(x)
case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64:
x, err := strconv.ParseInt(s, 10, v.Type().Bits())
if err != nil {
return err
}
v.SetInt(x)
case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64, reflect.Uintptr:
x, err := strconv.ParseUint(s, 10, v.Type().Bits())
if err != nil {
return err
}
v.SetUint(x)
case reflect.Float32, reflect.Float64:
x, err := strconv.ParseFloat(s, v.Type().Bits())
if err != nil {
return err
}
v.SetFloat(x)
default:
return fmt.Errorf("cannot parse into %v", v.Type())
}
return nil
}
// CanParse returns true if the type can be parsed from a string.
func CanParse(t reflect.Type) bool {
// If it implements encoding.TextUnmarshaler then use that
if t.Implements(textUnmarshalerType) || reflect.PtrTo(t).Implements(textUnmarshalerType) {
return true
}
// If we have a pointer then dereference it
if t.Kind() == reflect.Ptr {
t = t.Elem()
}
// Check for other special types
switch t {
case durationType, mailAddressType, macType, urlType:
return true
}
// Fall back to checking the kind
switch t.Kind() {
case reflect.Bool:
return true
case reflect.String, reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64,
reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64, reflect.Uintptr,
reflect.Float32, reflect.Float64:
return true
}
return false
}
vendor/github.com/bytedance/sonic/.gitignore
Generated
+52
View File
@@ -0,0 +1,52 @@
*.o
*.swp
*.swm
*.swn
*.a
*.so
_obj
_test
*.[568vq]
[568vq].out
*.cgo1.go
*.cgo2.c
_cgo_defun.c
_cgo_gotypes.go
_cgo_export.*
_testmain.go
*.exe
*.exe~
*.test
*.prof
*.rar
*.zip
*.gz
*.psd
*.bmd
*.cfg
*.pptx
*.log
*nohup.out
*settings.pyc
*.sublime-project
*.sublime-workspace
.DS_Store
/.idea/
/.vscode/
/output/
/vendor/
/Gopkg.lock
/Gopkg.toml
coverage.html
coverage.out
coverage.xml
junit.xml
*.profile
*.svg
*.out
ast/test.out
ast/bench.sh
!testdata/*.json.gz
fuzz/testdata
*__debug_bin
vendor/github.com/bytedance/sonic/.gitmodules
Generated
+3
View File
@@ -0,0 +1,3 @@
[submodule "tools/asm2asm"]
path = tools/asm2asm
url = https://github.com/chenzhuoyu/asm2asm
vendor/github.com/bytedance/sonic/.licenserc.yaml
Generated Vendored
+24
View File
@@ -0,0 +1,24 @@
header:
license:
spdx-id: Apache-2.0
copyright-owner: ByteDance Inc.
paths:
- '**/*.go'
- '**/*.s'
paths-ignore:
- 'ast/asm.s' # empty file
- 'decoder/asm.s' # empty file
- 'encoder/asm.s' # empty file
- 'internal/caching/asm.s' # empty file
- 'internal/jit/asm.s' # empty file
- 'internal/native/avx/native_amd64.s' # auto-generated by asm2asm
- 'internal/native/avx/native_subr_amd64.go' # auto-generated by asm2asm
- 'internal/native/avx2/native_amd64.s' # auto-generated by asm2asm
- 'internal/native/avx2/native_subr_amd64.go' # auto-generated by asm2asm
- 'internal/resolver/asm.s' # empty file
- 'internal/rt/asm.s' # empty file
- 'internal/loader/asm.s' # empty file
comment: on-failure
vendor/github.com/bytedance/sonic/CODE_OF_CONDUCT.md
Generated Vendored
+128
View File
@@ -0,0 +1,128 @@
# Contributor Covenant Code of Conduct
## Our Pledge
We as members, contributors, and leaders pledge to make participation in our
community a harassment-free experience for everyone, regardless of age, body
size, visible or invisible disability, ethnicity, sex characteristics, gender
identity and expression, level of experience, education, socio-economic status,
nationality, personal appearance, race, religion, or sexual identity
and orientation.
We pledge to act and interact in ways that contribute to an open, welcoming,
diverse, inclusive, and healthy community.
## Our Standards
Examples of behavior that contributes to a positive environment for our
community include:
* Demonstrating empathy and kindness toward other people
* Being respectful of differing opinions, viewpoints, and experiences
* Giving and gracefully accepting constructive feedback
* Accepting responsibility and apologizing to those affected by our mistakes,
and learning from the experience
* Focusing on what is best not just for us as individuals, but for the
overall community
Examples of unacceptable behavior include:
* The use of sexualized language or imagery, and sexual attention or
advances of any kind
* Trolling, insulting or derogatory comments, and personal or political attacks
* Public or private harassment
* Publishing others' private information, such as a physical or email
address, without their explicit permission
* Other conduct which could reasonably be considered inappropriate in a
professional setting
## Enforcement Responsibilities
Community leaders are responsible for clarifying and enforcing our standards of
acceptable behavior and will take appropriate and fair corrective action in
response to any behavior that they deem inappropriate, threatening, offensive,
or harmful.
Community leaders have the right and responsibility to remove, edit, or reject
comments, commits, code, wiki edits, issues, and other contributions that are
not aligned to this Code of Conduct, and will communicate reasons for moderation
decisions when appropriate.
## Scope
This Code of Conduct applies within all community spaces, and also applies when
an individual is officially representing the community in public spaces.
Examples of representing our community include using an official e-mail address,
posting via an official social media account, or acting as an appointed
representative at an online or offline event.
## Enforcement
Instances of abusive, harassing, or otherwise unacceptable behavior may be
reported to the community leaders responsible for enforcement at
wudi.daniel@bytedance.com.
All complaints will be reviewed and investigated promptly and fairly.
All community leaders are obligated to respect the privacy and security of the
reporter of any incident.
## Enforcement Guidelines
Community leaders will follow these Community Impact Guidelines in determining
the consequences for any action they deem in violation of this Code of Conduct:
### 1. Correction
**Community Impact**: Use of inappropriate language or other behavior deemed
unprofessional or unwelcome in the community.
**Consequence**: A private, written warning from community leaders, providing
clarity around the nature of the violation and an explanation of why the
behavior was inappropriate. A public apology may be requested.
### 2. Warning
**Community Impact**: A violation through a single incident or series
of actions.
**Consequence**: A warning with consequences for continued behavior. No
interaction with the people involved, including unsolicited interaction with
those enforcing the Code of Conduct, for a specified period of time. This
includes avoiding interactions in community spaces as well as external channels
like social media. Violating these terms may lead to a temporary or
permanent ban.
### 3. Temporary Ban
**Community Impact**: A serious violation of community standards, including
sustained inappropriate behavior.
**Consequence**: A temporary ban from any sort of interaction or public
communication with the community for a specified period of time. No public or
private interaction with the people involved, including unsolicited interaction
with those enforcing the Code of Conduct, is allowed during this period.
Violating these terms may lead to a permanent ban.
### 4. Permanent Ban
**Community Impact**: Demonstrating a pattern of violation of community
standards, including sustained inappropriate behavior, harassment of an
individual, or aggression toward or disparagement of classes of individuals.
**Consequence**: A permanent ban from any sort of public interaction within
the community.
## Attribution
This Code of Conduct is adapted from the [Contributor Covenant][homepage],
version 2.0, available at
https://www.contributor-covenant.org/version/2/0/code_of_conduct.html.
Community Impact Guidelines were inspired by [Mozilla's code of conduct
enforcement ladder](https://github.com/mozilla/diversity).
[homepage]: https://www.contributor-covenant.org
For answers to common questions about this code of conduct, see the FAQ at
https://www.contributor-covenant.org/faq. Translations are available at
https://www.contributor-covenant.org/translations.
vendor/github.com/bytedance/sonic/CONTRIBUTING.md
Generated Vendored
+63
View File
@@ -0,0 +1,63 @@
# How to Contribute
## Your First Pull Request
We use GitHub for our codebase. You can start by reading [How To Pull Request](https://docs.github.com/en/github/collaborating-with-issues-and-pull-requests/about-pull-requests).
## Without Semantic Versioning
We keep the stable code in branch `main` like `golang.org/x`. Development base on branch `develop`. We promise the **Forward Compatibility** by adding new package directory with suffix `v2/v3` when code has break changes.
## Branch Organization
We use [git-flow](https://nvie.com/posts/a-successful-git-branching-model/) as our branch organization, as known as [FDD](https://en.wikipedia.org/wiki/Feature-driven_development)
## Bugs
### 1. How to Find Known Issues
We are using [Github Issues](https://github.com/bytedance/sonic/issues) for our public bugs. We keep a close eye on this and try to make it clear when we have an internal fix in progress. Before filing a new task, try to make sure your problem doesn’t already exist.
### 2. Reporting New Issues
Providing a reduced test code is a recommended way for reporting issues. Then can be placed in:
- Just in issues
- [Golang Playground](https://play.golang.org/)
### 3. Security Bugs
Please do not report the safe disclosure of bugs to public issues. Contact us by [Support Email](mailto:sonic@bytedance.com)
## How to Get in Touch
- [Email](mailto:wudi.daniel@bytedance.com)
## Submit a Pull Request
Before you submit your Pull Request (PR) consider the following guidelines:
1. Search [GitHub](https://github.com/bytedance/sonic/pulls) for an open or closed PR that relates to your submission. You don't want to duplicate existing efforts.
2. Be sure that an issue describes the problem you're fixing, or documents the design for the feature you'd like to add. Discussing the design upfront helps to ensure that we're ready to accept your work.
3. [Fork](https://docs.github.com/en/github/getting-started-with-github/fork-a-repo) the bytedance/sonic repo.
4. In your forked repository, make your changes in a new git branch:
```
git checkout -b bugfix/security_bug develop
```
5. Create your patch, including appropriate test cases.
6. Follow our [Style Guides](#code-style-guides).
7. Commit your changes using a descriptive commit message that follows [AngularJS Git Commit Message Conventions](https://docs.google.com/document/d/1QrDFcIiPjSLDn3EL15IJygNPiHORgU1_OOAqWjiDU5Y/edit).
Adherence to these conventions is necessary because release notes will be automatically generated from these messages.
8. Push your branch to GitHub:
```
git push origin bugfix/security_bug
```
9. In GitHub, send a pull request to `sonic:main`
Note: you must use one of `optimize/feature/bugfix/doc/ci/test/refactor` following a slash(`/`) as the branch prefix.
Your pr title and commit message should follow https://www.conventionalcommits.org/.
## Contribution Prerequisites
- Our development environment keeps up with [Go Official](https://golang.org/project/).
- You need fully checking with lint tools before submit your pull request. [gofmt](https://golang.org/pkg/cmd/gofmt/) & [golangci-lint](https://github.com/golangci/golangci-lint)
- You are familiar with [Github](https://github.com)
- Maybe you need familiar with [Actions](https://github.com/features/actions)(our default workflow tool).
## Code Style Guides
See [Go Code Review Comments](https://github.com/golang/go/wiki/CodeReviewComments).
Good resources:
- [Effective Go](https://golang.org/doc/effective_go)
- [Pingcap General advice](https://pingcap.github.io/style-guide/general.html)
- [Uber Go Style Guide](https://github.com/uber-go/guide/blob/master/style.md)
vendor/github.com/bytedance/sonic/CREDITS
Generated Vendored
View File
vendor/github.com/bytedance/sonic/LICENSE
Generated Vendored
+201
View File
@@ -0,0 +1,201 @@
Apache License
Version 2.0, January 2004
http://www.apache.org/licenses/
TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
1. Definitions.
"License" shall mean the terms and conditions for use, reproduction,
and distribution as defined by Sections 1 through 9 of this document.
"Licensor" shall mean the copyright owner or entity authorized by
the copyright owner that is granting the License.
"Legal Entity" shall mean the union of the acting entity and all
other entities that control, are controlled by, or are under common
control with that entity. For the purposes of this definition,
"control" means (i) the power, direct or indirect, to cause the
direction or management of such entity, whether by contract or
otherwise, or (ii) ownership of fifty percent (50%) or more of the
outstanding shares, or (iii) beneficial ownership of such entity.
"You" (or "Your") shall mean an individual or Legal Entity
exercising permissions granted by this License.
"Source" form shall mean the preferred form for making modifications,
including but not limited to software source code, documentation
source, and configuration files.
"Object" form shall mean any form resulting from mechanical
transformation or translation of a Source form, including but
not limited to compiled object code, generated documentation,
and conversions to other media types.
"Work" shall mean the work of authorship, whether in Source or
Object form, made available under the License, as indicated by a
copyright notice that is included in or attached to the work
(an example is provided in the Appendix below).
"Derivative Works" shall mean any work, whether in Source or Object
form, that is based on (or derived from) the Work and for which the
editorial revisions, annotations, elaborations, or other modifications
represent, as a whole, an original work of authorship. For the purposes
of this License, Derivative Works shall not include works that remain
separable from, or merely link (or bind by name) to the interfaces of,
the Work and Derivative Works thereof.
"Contribution" shall mean any work of authorship, including
the original version of the Work and any modifications or additions
to that Work or Derivative Works thereof, that is intentionally
submitted to Licensor for inclusion in the Work by the copyright owner
or by an individual or Legal Entity authorized to submit on behalf of
the copyright owner. For the purposes of this definition, "submitted"
means any form of electronic, verbal, or written communication sent
to the Licensor or its representatives, including but not limited to
communication on electronic mailing lists, source code control systems,
and issue tracking systems that are managed by, or on behalf of, the
Licensor for the purpose of discussing and improving the Work, but
excluding communication that is conspicuously marked or otherwise
designated in writing by the copyright owner as "Not a Contribution."
"Contributor" shall mean Licensor and any individual or Legal Entity
on behalf of whom a Contribution has been received by Licensor and
subsequently incorporated within the Work.
2. Grant of Copyright License. Subject to the terms and conditions of
this License, each Contributor hereby grants to You a perpetual,
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
copyright license to reproduce, prepare Derivative Works of,
publicly display, publicly perform, sublicense, and distribute the
Work and such Derivative Works in Source or Object form.
3. Grant of Patent License. Subject to the terms and conditions of
this License, each Contributor hereby grants to You a perpetual,
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
(except as stated in this section) patent license to make, have made,
use, offer to sell, sell, import, and otherwise transfer the Work,
where such license applies only to those patent claims licensable
by such Contributor that are necessarily infringed by their
Contribution(s) alone or by combination of their Contribution(s)
with the Work to which such Contribution(s) was submitted. If You
institute patent litigation against any entity (including a
cross-claim or counterclaim in a lawsuit) alleging that the Work
or a Contribution incorporated within the Work constitutes direct
or contributory patent infringement, then any patent licenses
granted to You under this License for that Work shall terminate
as of the date such litigation is filed.
4. Redistribution. You may reproduce and distribute copies of the
Work or Derivative Works thereof in any medium, with or without
modifications, and in Source or Object form, provided that You
meet the following conditions:
(a) You must give any other recipients of the Work or
Derivative Works a copy of this License; and
(b) You must cause any modified files to carry prominent notices
stating that You changed the files; and
(c) You must retain, in the Source form of any Derivative Works
that You distribute, all copyright, patent, trademark, and
attribution notices from the Source form of the Work,
excluding those notices that do not pertain to any part of
the Derivative Works; and
(d) If the Work includes a "NOTICE" text file as part of its
distribution, then any Derivative Works that You distribute must
include a readable copy of the attribution notices contained
within such NOTICE file, excluding those notices that do not
pertain to any part of the Derivative Works, in at least one
of the following places: within a NOTICE text file distributed
as part of the Derivative Works; within the Source form or
documentation, if provided along with the Derivative Works; or,
within a display generated by the Derivative Works, if and
wherever such third-party notices normally appear. The contents
of the NOTICE file are for informational purposes only and
do not modify the License. You may add Your own attribution
notices within Derivative Works that You distribute, alongside
or as an addendum to the NOTICE text from the Work, provided
that such additional attribution notices cannot be construed
as modifying the License.
You may add Your own copyright statement to Your modifications and
may provide additional or different license terms and conditions
for use, reproduction, or distribution of Your modifications, or
for any such Derivative Works as a whole, provided Your use,
reproduction, and distribution of the Work otherwise complies with
the conditions stated in this License.
5. Submission of Contributions. Unless You explicitly state otherwise,
any Contribution intentionally submitted for inclusion in the Work
by You to the Licensor shall be under the terms and conditions of
this License, without any additional terms or conditions.
Notwithstanding the above, nothing herein shall supersede or modify
the terms of any separate license agreement you may have executed
with Licensor regarding such Contributions.
6. Trademarks. This License does not grant permission to use the trade
names, trademarks, service marks, or product names of the Licensor,
except as required for reasonable and customary use in describing the
origin of the Work and reproducing the content of the NOTICE file.
7. Disclaimer of Warranty. Unless required by applicable law or
agreed to in writing, Licensor provides the Work (and each
Contributor provides its Contributions) on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
implied, including, without limitation, any warranties or conditions
of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
PARTICULAR PURPOSE. You are solely responsible for determining the
appropriateness of using or redistributing the Work and assume any
risks associated with Your exercise of permissions under this License.
8. Limitation of Liability. In no event and under no legal theory,
whether in tort (including negligence), contract, or otherwise,
unless required by applicable law (such as deliberate and grossly
negligent acts) or agreed to in writing, shall any Contributor be
liable to You for damages, including any direct, indirect, special,
incidental, or consequential damages of any character arising as a
result of this License or out of the use or inability to use the
Work (including but not limited to damages for loss of goodwill,
work stoppage, computer failure or malfunction, or any and all
other commercial damages or losses), even if such Contributor
has been advised of the possibility of such damages.
9. Accepting Warranty or Additional Liability. While redistributing
the Work or Derivative Works thereof, You may choose to offer,
and charge a fee for, acceptance of support, warranty, indemnity,
or other liability obligations and/or rights consistent with this
License. However, in accepting such obligations, You may act only
on Your own behalf and on Your sole responsibility, not on behalf
of any other Contributor, and only if You agree to indemnify,
defend, and hold each Contributor harmless for any liability
incurred by, or claims asserted against, such Contributor by reason
of your accepting any such warranty or additional liability.
END OF TERMS AND CONDITIONS
APPENDIX: How to apply the Apache License to your work.
To apply the Apache License to your work, attach the following
boilerplate notice, with the fields enclosed by brackets "[]"
replaced with your own identifying information. (Don't include
the brackets!) The text should be enclosed in the appropriate
comment syntax for the file format. We also recommend that a
file or class name and description of purpose be included on the
same "printed page" as the copyright notice for easier
identification within third-party archives.
Copyright [yyyy] [name of copyright owner]
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
vendor/github.com/bytedance/sonic/Makefile
Generated Vendored
+116
View File
@@ -0,0 +1,116 @@
#
# Copyright 2021 ByteDance Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
ARCH := avx avx2 sse
TMP_DIR := output
OUT_DIR := internal/native
SRC_FILE := native/native.c
CPU_avx := amd64
CPU_avx2 := amd64
CPU_sse := amd64
TMPL_avx := fastint_amd64_test fastfloat_amd64_test native_amd64_test native_export_amd64
TMPL_avx2 := fastint_amd64_test fastfloat_amd64_test native_amd64_test native_export_amd64
TMPL_sse := fastint_amd64_test fastfloat_amd64_test native_amd64_test native_export_amd64
CFLAGS_avx := -msse -mssse3 -mno-sse4 -mavx -mpclmul -mno-avx2 -DUSE_AVX=1 -DUSE_AVX2=0
CFLAGS_avx2 := -msse -mssse3 -mno-sse4 -mavx -mpclmul -mavx2 -DUSE_AVX=1 -DUSE_AVX2=1
CFLAGS_sse := -msse -mssse3 -mno-sse4 -mno-avx -mno-avx2 -mpclmul
TARGETFLAGS := -target x86_64-apple-macos11 -nostdlib -fno-builtin -fno-asynchronous-unwind-tables
CC_amd64 := clang
ASM2ASM_amd64 := tools/asm2asm/asm2asm.py
CFLAGS := -mno-red-zone
CFLAGS += -fno-exceptions
CFLAGS += -fno-rtti
CFLAGS += -fno-stack-protector
CFLAGS += -O3
CFLAGS += -Wall -Werror
NATIVE_SRC := $(wildcard native/*.h)
NATIVE_SRC += $(wildcard native/*.c)
.PHONY: all clean ${ARCH}
define build_tmpl
$(eval @arch := $(1))
$(eval @tmpl := $(2))
$(eval @dest := $(3))
${@dest}: ${@tmpl}
mkdir -p $(dir ${@dest})
echo '// Code generated by Makefile, DO NOT EDIT.' > ${@dest}
echo >> ${@dest}
sed -e 's/{{PACKAGE}}/${@arch}/g' ${@tmpl} >> ${@dest}
endef
define build_arch
$(eval @cpu := $(value CPU_$(1)))
$(eval @deps := $(foreach tmpl,$(value TMPL_$(1)),${OUT_DIR}/$(1)/${tmpl}.go))
$(eval @asmin := ${TMP_DIR}/$(1)/native.s)
$(eval @asmout := ${OUT_DIR}/$(1)/native_${@cpu}.s)
$(eval @stubin := ${OUT_DIR}/native_${@cpu}.tmpl)
$(eval @stubout := ${OUT_DIR}/$(1)/native_${@cpu}.go)
$(1): ${@asmout} ${@deps}
${@asmout}: ${@stubout} ${NATIVE_SRC}
mkdir -p ${TMP_DIR}/$(1)
$${CC_${@cpu}} $${CFLAGS} $${CFLAGS_$(1)} ${TARGETFLAGS} -S -o ${TMP_DIR}/$(1)/native.s ${SRC_FILE}
$(foreach file,
$(wildcard native/unittest/*),
$${CC_${@cpu}} $${CFLAGS} $${CFLAGS_$(1)} -I./native -o ${TMP_DIR}/$(1)/test $(file)
./${TMP_DIR}/$(1)/test
)
python3 $${ASM2ASM_${@cpu}} ${@asmout} ${TMP_DIR}/$(1)/native.s
asmfmt -w ${@asmout}
$(eval $(call \
build_tmpl, \
$(1), \
${@stubin}, \
${@stubout} \
))
$(foreach \
tmpl, \
$(value TMPL_$(1)), \
$(eval $(call \
build_tmpl, \
$(1), \
${OUT_DIR}/${tmpl}.tmpl, \
${OUT_DIR}/$(1)/${tmpl}.go \
)) \
)
endef
all: ${ARCH}
clean:
for arch in ${ARCH}; do \
rm -vfr ${TMP_DIR}/$${arch}; \
rm -vfr ${OUT_DIR}/$${arch}; \
done
$(foreach \
arch, \
${ARCH}, \
$(eval $(call build_arch,${arch})) \
)
vendor/github.com/bytedance/sonic/README.md
Generated Vendored
+362
View File
@@ -0,0 +1,362 @@
# Sonic
English | [中文](README_ZH_CN.md)
A blazingly fast JSON serializing &amp; deserializing library, accelerated by JIT (just-in-time compiling) and SIMD (single-instruction-multiple-data).
## Requirement
- Go 1.15~1.20
- Linux/MacOS/Windows
- Amd64 ARCH
## Features
- Runtime object binding without code generation
- Complete APIs for JSON value manipulation
- Fast, fast, fast!
## Benchmarks
For **all sizes** of json and **all scenarios** of usage, **Sonic performs best**.
- [Medium](https://github.com/bytedance/sonic/blob/main/decoder/testdata_test.go#L19) (13KB, 300+ key, 6 layers)
```powershell
goversion: 1.17.1
goos: darwin
goarch: amd64
cpu: Intel(R) Core(TM) i9-9880H CPU @ 2.30GHz
BenchmarkEncoder_Generic_Sonic-16 32393 ns/op 402.40 MB/s 11965 B/op 4 allocs/op
BenchmarkEncoder_Generic_Sonic_Fast-16 21668 ns/op 601.57 MB/s 10940 B/op 4 allocs/op
BenchmarkEncoder_Generic_JsonIter-16 42168 ns/op 309.12 MB/s 14345 B/op 115 allocs/op
BenchmarkEncoder_Generic_GoJson-16 65189 ns/op 199.96 MB/s 23261 B/op 16 allocs/op
BenchmarkEncoder_Generic_StdLib-16 106322 ns/op 122.60 MB/s 49136 B/op 789 allocs/op
BenchmarkEncoder_Binding_Sonic-16 6269 ns/op 2079.26 MB/s 14173 B/op 4 allocs/op
BenchmarkEncoder_Binding_Sonic_Fast-16 5281 ns/op 2468.16 MB/s 12322 B/op 4 allocs/op
BenchmarkEncoder_Binding_JsonIter-16 20056 ns/op 649.93 MB/s 9488 B/op 2 allocs/op
BenchmarkEncoder_Binding_GoJson-16 8311 ns/op 1568.32 MB/s 9481 B/op 1 allocs/op
BenchmarkEncoder_Binding_StdLib-16 16448 ns/op 792.52 MB/s 9479 B/op 1 allocs/op
BenchmarkEncoder_Parallel_Generic_Sonic-16 6681 ns/op 1950.93 MB/s 12738 B/op 4 allocs/op
BenchmarkEncoder_Parallel_Generic_Sonic_Fast-16 4179 ns/op 3118.99 MB/s 10757 B/op 4 allocs/op
BenchmarkEncoder_Parallel_Generic_JsonIter-16 9861 ns/op 1321.84 MB/s 14362 B/op 115 allocs/op
BenchmarkEncoder_Parallel_Generic_GoJson-16 18850 ns/op 691.52 MB/s 23278 B/op 16 allocs/op
BenchmarkEncoder_Parallel_Generic_StdLib-16 45902 ns/op 283.97 MB/s 49174 B/op 789 allocs/op
BenchmarkEncoder_Parallel_Binding_Sonic-16 1480 ns/op 8810.09 MB/s 13049 B/op 4 allocs/op
BenchmarkEncoder_Parallel_Binding_Sonic_Fast-16 1209 ns/op 10785.23 MB/s 11546 B/op 4 allocs/op
BenchmarkEncoder_Parallel_Binding_JsonIter-16 6170 ns/op 2112.58 MB/s 9504 B/op 2 allocs/op
BenchmarkEncoder_Parallel_Binding_GoJson-16 3321 ns/op 3925.52 MB/s 9496 B/op 1 allocs/op
BenchmarkEncoder_Parallel_Binding_StdLib-16 3739 ns/op 3486.49 MB/s 9480 B/op 1 allocs/op
BenchmarkDecoder_Generic_Sonic-16 66812 ns/op 195.10 MB/s 57602 B/op 723 allocs/op
BenchmarkDecoder_Generic_Sonic_Fast-16 54523 ns/op 239.07 MB/s 49786 B/op 313 allocs/op
BenchmarkDecoder_Generic_StdLib-16 124260 ns/op 104.90 MB/s 50869 B/op 772 allocs/op
BenchmarkDecoder_Generic_JsonIter-16 91274 ns/op 142.81 MB/s 55782 B/op 1068 allocs/op
BenchmarkDecoder_Generic_GoJson-16 88569 ns/op 147.17 MB/s 66367 B/op 973 allocs/op
BenchmarkDecoder_Binding_Sonic-16 32557 ns/op 400.38 MB/s 28302 B/op 137 allocs/op
BenchmarkDecoder_Binding_Sonic_Fast-16 28649 ns/op 455.00 MB/s 24999 B/op 34 allocs/op
BenchmarkDecoder_Binding_StdLib-16 111437 ns/op 116.97 MB/s 10576 B/op 208 allocs/op
BenchmarkDecoder_Binding_JsonIter-16 35090 ns/op 371.48 MB/s 14673 B/op 385 allocs/op
BenchmarkDecoder_Binding_GoJson-16 28738 ns/op 453.59 MB/s 22039 B/op 49 allocs/op
BenchmarkDecoder_Parallel_Generic_Sonic-16 12321 ns/op 1057.91 MB/s 57233 B/op 723 allocs/op
BenchmarkDecoder_Parallel_Generic_Sonic_Fast-16 10644 ns/op 1224.64 MB/s 49362 B/op 313 allocs/op
BenchmarkDecoder_Parallel_Generic_StdLib-16 57587 ns/op 226.35 MB/s 50874 B/op 772 allocs/op
BenchmarkDecoder_Parallel_Generic_JsonIter-16 38666 ns/op 337.12 MB/s 55789 B/op 1068 allocs/op
BenchmarkDecoder_Parallel_Generic_GoJson-16 30259 ns/op 430.79 MB/s 66370 B/op 974 allocs/op
BenchmarkDecoder_Parallel_Binding_Sonic-16 5965 ns/op 2185.28 MB/s 27747 B/op 137 allocs/op
BenchmarkDecoder_Parallel_Binding_Sonic_Fast-16 5170 ns/op 2521.31 MB/s 24715 B/op 34 allocs/op
BenchmarkDecoder_Parallel_Binding_StdLib-16 27582 ns/op 472.58 MB/s 10576 B/op 208 allocs/op
BenchmarkDecoder_Parallel_Binding_JsonIter-16 13571 ns/op 960.51 MB/s 14685 B/op 385 allocs/op
BenchmarkDecoder_Parallel_Binding_GoJson-16 10031 ns/op 1299.51 MB/s 22111 B/op 49 allocs/op
BenchmarkGetOne_Sonic-16 3276 ns/op 3975.78 MB/s 24 B/op 1 allocs/op
BenchmarkGetOne_Gjson-16 9431 ns/op 1380.81 MB/s 0 B/op 0 allocs/op
BenchmarkGetOne_Jsoniter-16 51178 ns/op 254.46 MB/s 27936 B/op 647 allocs/op
BenchmarkGetOne_Parallel_Sonic-16 216.7 ns/op 60098.95 MB/s 24 B/op 1 allocs/op
BenchmarkGetOne_Parallel_Gjson-16 1076 ns/op 12098.62 MB/s 0 B/op 0 allocs/op
BenchmarkGetOne_Parallel_Jsoniter-16 17741 ns/op 734.06 MB/s 27945 B/op 647 allocs/op
BenchmarkSetOne_Sonic-16 9571 ns/op 1360.61 MB/s 1584 B/op 17 allocs/op
BenchmarkSetOne_Sjson-16 36456 ns/op 357.22 MB/s 52180 B/op 9 allocs/op
BenchmarkSetOne_Jsoniter-16 79475 ns/op 163.86 MB/s 45862 B/op 964 allocs/op
BenchmarkSetOne_Parallel_Sonic-16 850.9 ns/op 15305.31 MB/s 1584 B/op 17 allocs/op
BenchmarkSetOne_Parallel_Sjson-16 18194 ns/op 715.77 MB/s 52247 B/op 9 allocs/op
BenchmarkSetOne_Parallel_Jsoniter-16 33560 ns/op 388.05 MB/s 45892 B/op 964 allocs/op
```
- [Small](https://github.com/bytedance/sonic/blob/main/testdata/small.go) (400B, 11 keys, 3 layers)
![small benchmarks](./docs/imgs/bench-small.png)
- [Large](https://github.com/bytedance/sonic/blob/main/testdata/twitter.json) (635KB, 10000+ key, 6 layers)
![large benchmarks](./docs/imgs/bench-large.png)
See [bench.sh](https://github.com/bytedance/sonic/blob/main/bench.sh) for benchmark codes.
## How it works
See [INTRODUCTION.md](./docs/INTRODUCTION.md).
## Usage
### Marshal/Unmarshal
Default behaviors are mostly consistent with `encoding/json`, except HTML escaping form (see [Escape HTML](https://github.com/bytedance/sonic/blob/main/README.md#escape-html)) and `SortKeys` feature (optional support see [Sort Keys](https://github.com/bytedance/sonic/blob/main/README.md#sort-keys)) that is **NOT** in conformity to [RFC8259](https://datatracker.ietf.org/doc/html/rfc8259).
```go
import "github.com/bytedance/sonic"
var data YourSchema
// Marshal
output, err := sonic.Marshal(&data)
// Unmarshal
err := sonic.Unmarshal(output, &data)
```
### Streaming IO
Sonic supports decoding json from `io.Reader` or encoding objects into `io.`Writer`, aims at handling multiple values as well as reducing memory consumption.
- encoder
```go
var o1 = map[string]interface{}{
"a": "b",
}
var o2 = 1
var w = bytes.NewBuffer(nil)
var enc = sonic.ConfigDefault.NewEncoder(w)
enc.Encode(o1)
enc.Encode(o2)
fmt.Println(w.String())
// Output:
// {"a":"b"}
// 1
```
- decoder
```go
var o = map[string]interface{}{}
var r = strings.NewReader(`{"a":"b"}{"1":"2"}`)
var dec = sonic.ConfigDefault.NewDecoder(r)
dec.Decode(&o)
dec.Decode(&o)
fmt.Printf("%+v", o)
// Output:
// map[1:2 a:b]
```
### Use Number/Use Int64
```go
import "github.com/bytedance/sonic/decoder"
var input = `1`
var data interface{}
// default float64
dc := decoder.NewDecoder(input)
dc.Decode(&data) // data == float64(1)
// use json.Number
dc = decoder.NewDecoder(input)
dc.UseNumber()
dc.Decode(&data) // data == json.Number("1")
// use int64
dc = decoder.NewDecoder(input)
dc.UseInt64()
dc.Decode(&data) // data == int64(1)
root, err := sonic.GetFromString(input)
// Get json.Number
jn := root.Number()
jm := root.InterfaceUseNumber().(json.Number) // jn == jm
// Get float64
fn := root.Float64()
fm := root.Interface().(float64) // jn == jm
```
### Sort Keys
On account of the performance loss from sorting (roughly 10%), sonic doesn't enable this feature by default. If your component depends on it to work (like [zstd](https://github.com/facebook/zstd)), Use it like this:
```go
import "github.com/bytedance/sonic"
import "github.com/bytedance/sonic/encoder"
// Binding map only
m := map[string]interface{}{}
v, err := encoder.Encode(m, encoder.SortMapKeys)
// Or ast.Node.SortKeys() before marshal
var root := sonic.Get(JSON)
err := root.SortKeys()
```
### Escape HTML
On account of the performance loss (roughly 15%), sonic doesn't enable this feature by default. You can use `encoder.EscapeHTML` option to open this feature (align with `encoding/json.HTMLEscape`).
```go
import "github.com/bytedance/sonic"
v := map[string]string{"&&":"<>"}
ret, err := Encode(v, EscapeHTML) // ret == `{"\u0026\u0026":{"X":"\u003c\u003e"}}`
```
### Compact Format
Sonic encodes primitive objects (struct/map...) as compact-format JSON by default, except marshaling `json.RawMessage` or `json.Marshaler`: sonic ensures validating their output JSON but **DONOT** compacting them for performance concerns. We provide the option `encoder.CompactMarshaler` to add compacting process.
### Print Error
If there invalid syntax in input JSON, sonic will return `decoder.SyntaxError`, which supports pretty-printing of error position
```go
import "github.com/bytedance/sonic"
import "github.com/bytedance/sonic/decoder"
var data interface{}
err := sonic.UnmarshalString("[[[}]]", &data)
if err != nil {
/* One line by default */
println(e.Error()) // "Syntax error at index 3: invalid char\n\n\t[[[}]]\n\t...^..\n"
/* Pretty print */
if e, ok := err.(decoder.SyntaxError); ok {
/*Syntax error at index 3: invalid char
[[[}]]
...^..
*/
print(e.Description())
} else if me, ok := err.(*decoder.MismatchTypeError); ok {
// decoder.MismatchTypeError is new to Sonic v1.6.0
print(me.Description())
}
}
```
#### Mismatched Types [Sonic v1.6.0]
If there a **mismatch-typed** value for a given key, sonic will report `decoder.MismatchTypeError` (if there are many, report the last one), but still skip wrong the value and keep decoding next JSON.
```go
import "github.com/bytedance/sonic"
import "github.com/bytedance/sonic/decoder"
var data = struct{
A int
B int
}{}
err := UnmarshalString(`{"A":"1","B":1}`, &data)
println(err.Error()) // Mismatch type int with value string "at index 5: mismatched type with value\n\n\t{\"A\":\"1\",\"B\":1}\n\t.....^.........\n"
fmt.Printf("%+v", data) // {A:0 B:1}
```
### Ast.Node
Sonic/ast.Node is a completely self-contained AST for JSON. It implements serialization and deserialization both and provides robust APIs for obtaining and modification of generic data.
#### Get/Index
Search partial JSON by given paths, which must be non-negative integer or string, or nil
```go
import "github.com/bytedance/sonic"
input := []byte(`{"key1":[{},{"key2":{"key3":[1,2,3]}}]}`)
// no path, returns entire json
root, err := sonic.Get(input)
raw := root.Raw() // == string(input)
// multiple paths
root, err := sonic.Get(input, "key1", 1, "key2")
sub := root.Get("key3").Index(2).Int64() // == 3
```
**Tip**: since `Index()` uses offset to locate data, which is much faster than scanning like `Get()`, we suggest you use it as much as possible. And sonic also provides another API `IndexOrGet()` to underlying use offset as well as ensure the key is matched.
#### Set/Unset
Modify the json content by Set()/Unset()
```go
import "github.com/bytedance/sonic"
// Set
exist, err := root.Set("key4", NewBool(true)) // exist == false
alias1 := root.Get("key4")
println(alias1.Valid()) // true
alias2 := root.Index(1)
println(alias1 == alias2) // true
// Unset
exist, err := root.UnsetByIndex(1) // exist == true
println(root.Get("key4").Check()) // "value not exist"
```
#### Serialize
To encode `ast.Node` as json, use `MarshalJson()` or `json.Marshal()` (MUST pass the node's pointer)
```go
import (
"encoding/json"
"github.com/bytedance/sonic"
)
buf, err := root.MarshalJson()
println(string(buf)) // {"key1":[{},{"key2":{"key3":[1,2,3]}}]}
exp, err := json.Marshal(&root) // WARN: use pointer
println(string(buf) == string(exp)) // true
```
#### APIs
- validation: `Check()`, `Error()`, `Valid()`, `Exist()`
- searching: `Index()`, `Get()`, `IndexPair()`, `IndexOrGet()`, `GetByPath()`
- go-type casting: `Int64()`, `Float64()`, `String()`, `Number()`, `Bool()`, `Map[UseNumber|UseNode]()`, `Array[UseNumber|UseNode]()`, `Interface[UseNumber|UseNode]()`
- go-type packing: `NewRaw()`, `NewNumber()`, `NewNull()`, `NewBool()`, `NewString()`, `NewObject()`, `NewArray()`
- iteration: `Values()`, `Properties()`, `ForEach()`, `SortKeys()`
- modification: `Set()`, `SetByIndex()`, `Add()`
## Compatibility
Sonic **DOES NOT** ensure to support all environments, due to the difficulty of developing high-performance codes. For developers who use sonic to build their applications in different environments, we have the following suggestions:
- Developing on **Mac M1**: Make sure you have Rosetta 2 installed on your machine, and set `GOARCH=amd64` when building your application. Rosetta 2 can automatically translate x86 binaries to arm64 binaries and run x86 applications on Mac M1.
- Developing on **Linux arm64**: You can install qemu and use the `qemu-x86_64 -cpu max` command to convert x86 binaries to amr64 binaries for applications built with sonic. The qemu can achieve a similar transfer effect to Rosetta 2 on Mac M1.
For developers who want to use sonic on Linux arm64 without qemu, or those who want to handle JSON strictly consistent with `encoding/json`, we provide some compatible APIs as `sonic.API`
- `ConfigDefault`: the sonic's default config (`EscapeHTML=false`,`SortKeys=false`...) to run on sonic-supporting environment. It will fall back to `encoding/json` with the corresponding config, and some options like `SortKeys=false` will be invalid.
- `ConfigStd`: the std-compatible config (`EscapeHTML=true`,`SortKeys=true`...) to run on sonic-supporting environment. It will fall back to `encoding/json`.
- `ConfigFastest`: the fastest config (`NoQuoteTextMarshaler=true`) to run on sonic-supporting environment. It will fall back to `encoding/json` with the corresponding config, and some options will be invalid.
## Tips
### Pretouch
Since Sonic uses [golang-asm](https://github.com/twitchyliquid64/golang-asm) as a JIT assembler, which is NOT very suitable for runtime compiling, first-hit running of a huge schema may cause request-timeout or even process-OOM. For better stability, we advise **using `Pretouch()` for huge-schema or compact-memory applications** before `Marshal()/Unmarshal()`.
```go
import (
"reflect"
"github.com/bytedance/sonic"
"github.com/bytedance/sonic/option"
)
func init() {
var v HugeStruct
// For most large types (nesting depth <= option.DefaultMaxInlineDepth)
err := sonic.Pretouch(reflect.TypeOf(v))
// with more CompileOption...
err := sonic.Pretouch(reflect.TypeOf(v),
// If the type is too deep nesting (nesting depth > option.DefaultMaxInlineDepth),
// you can set compile recursive loops in Pretouch for better stability in JIT.
option.WithCompileRecursiveDepth(loop),
// For a large nested struct, try to set a smaller depth to reduce compiling time.
option.WithCompileMaxInlineDepth(depth),
)
}
```
### Copy string
When decoding **string values without any escaped characters**, sonic references them from the origin JSON buffer instead of mallocing a new buffer to copy. This helps a lot for CPU performance but may leave the whole JSON buffer in memory as long as the decoded objects are being used. In practice, we found the extra memory introduced by referring JSON buffer is usually 20% ~ 80% of decoded objects. Once an application holds these objects for a long time (for example, cache the decoded objects for reusing), its in-use memory on the server may go up. We provide the option `decoder.CopyString()` for users to choose not to reference the JSON buffer, which may cause a decline in CPU performance to some degree.
### Pass string or []byte?
For alignment to `encoding/json`, we provide API to pass `[]byte` as an argument, but the string-to-bytes copy is conducted at the same time considering safety, which may lose performance when the origin JSON is huge. Therefore, you can use `UnmarshalString()` and `GetFromString()` to pass a string, as long as your origin data is a string or **nocopy-cast** is safe for your []byte. We also provide API `MarshalString()` for convenient **nocopy-cast** of encoded JSON []byte, which is safe since sonic's output bytes is always duplicated and unique.
### Accelerate `encoding.TextMarshaler`
To ensure data security, sonic.Encoder quotes and escapes string values from `encoding.TextMarshaler` interfaces by default, which may degrade performance much if most of your data is in form of them. We provide `encoder.NoQuoteTextMarshaler` to skip these operations, which means you **MUST** ensure their output string escaped and quoted following [RFC8259](https://datatracker.ietf.org/doc/html/rfc8259).
### Better performance for generic data
In **fully-parsed** scenario, `Unmarshal()` performs better than `Get()`+`Node.Interface()`. But if you only have a part of the schema for specific json, you can combine `Get()` and `Unmarshal()` together:
```go
import "github.com/bytedance/sonic"
node, err := sonic.GetFromString(_TwitterJson, "statuses", 3, "user")
var user User // your partial schema...
err = sonic.UnmarshalString(node.Raw(), &user)
```
Even if you don't have any schema, use `ast.Node` as the container of generic values instead of `map` or `interface`:
```go
import "github.com/bytedance/sonic"
root, err := sonic.GetFromString(_TwitterJson)
user := root.GetByPath("statuses", 3, "user") // === root.Get("status").Index(3).Get("user")
err = user.Check()
// err = user.LoadAll() // only call this when you want to use 'user' concurrently...
go someFunc(user)
```
Why? Because `ast.Node` stores its children using `array`:
- `Array`'s performance is **much better** than `Map` when Inserting (Deserialize) and Scanning (Serialize) data;
- **Hashing** (`map[x]`) is not as efficient as **Indexing** (`array[x]`), which `ast.Node` can conduct on **both array and object**;
- Using `Interface()`/`Map()` means Sonic must parse all the underlying values, while `ast.Node` can parse them **on demand**.
**CAUTION:** `ast.Node` **DOESN'T** ensure concurrent security directly, due to its **lazy-load** design. However, you can call `Node.Load()`/`Node.LoadAll()` to achieve that, which may bring performance reduction while it still works faster than converting to `map` or `interface{}`
## Community
Sonic is a subproject of [CloudWeGo](https://www.cloudwego.io/). We are committed to building a cloud native ecosystem.
vendor/github.com/bytedance/sonic/README_ZH_CN.md
Generated Vendored
+382
View File
@@ -0,0 +1,382 @@
# Sonic
[English](README.md) | 中文
一个速度奇快的 JSON 序列化/反序列化库，由 JIT （即时编译）和 SIMD （单指令流多数据流）加速。
## 依赖
- Go 1.15~1.20
- Linux/MacOS/Windows
- Amd64 架构
## 特色
- 运行时对象绑定，无需代码生成
- 完备的 JSON 操作 API
- 快，更快，还要更快！
## 基准测试
对于**所有大小**的 json 和**所有使用场景**， **Sonic 表现均为最佳**。
- [中型](https://github.com/bytedance/sonic/blob/main/decoder/testdata_test.go#L19) (13kB, 300+ 键, 6 层)
```powershell
goversion: 1.17.1
goos: darwin
goarch: amd64
cpu: Intel(R) Core(TM) i9-9880H CPU @ 2.30GHz
BenchmarkEncoder_Generic_Sonic-16 32393 ns/op 402.40 MB/s 11965 B/op 4 allocs/op
BenchmarkEncoder_Generic_Sonic_Fast-16 21668 ns/op 601.57 MB/s 10940 B/op 4 allocs/op
BenchmarkEncoder_Generic_JsonIter-16 42168 ns/op 309.12 MB/s 14345 B/op 115 allocs/op
BenchmarkEncoder_Generic_GoJson-16 65189 ns/op 199.96 MB/s 23261 B/op 16 allocs/op
BenchmarkEncoder_Generic_StdLib-16 106322 ns/op 122.60 MB/s 49136 B/op 789 allocs/op
BenchmarkEncoder_Binding_Sonic-16 6269 ns/op 2079.26 MB/s 14173 B/op 4 allocs/op
BenchmarkEncoder_Binding_Sonic_Fast-16 5281 ns/op 2468.16 MB/s 12322 B/op 4 allocs/op
BenchmarkEncoder_Binding_JsonIter-16 20056 ns/op 649.93 MB/s 9488 B/op 2 allocs/op
BenchmarkEncoder_Binding_GoJson-16 8311 ns/op 1568.32 MB/s 9481 B/op 1 allocs/op
BenchmarkEncoder_Binding_StdLib-16 16448 ns/op 792.52 MB/s 9479 B/op 1 allocs/op
BenchmarkEncoder_Parallel_Generic_Sonic-16 6681 ns/op 1950.93 MB/s 12738 B/op 4 allocs/op
BenchmarkEncoder_Parallel_Generic_Sonic_Fast-16 4179 ns/op 3118.99 MB/s 10757 B/op 4 allocs/op
BenchmarkEncoder_Parallel_Generic_JsonIter-16 9861 ns/op 1321.84 MB/s 14362 B/op 115 allocs/op
BenchmarkEncoder_Parallel_Generic_GoJson-16 18850 ns/op 691.52 MB/s 23278 B/op 16 allocs/op
BenchmarkEncoder_Parallel_Generic_StdLib-16 45902 ns/op 283.97 MB/s 49174 B/op 789 allocs/op
BenchmarkEncoder_Parallel_Binding_Sonic-16 1480 ns/op 8810.09 MB/s 13049 B/op 4 allocs/op
BenchmarkEncoder_Parallel_Binding_Sonic_Fast-16 1209 ns/op 10785.23 MB/s 11546 B/op 4 allocs/op
BenchmarkEncoder_Parallel_Binding_JsonIter-16 6170 ns/op 2112.58 MB/s 9504 B/op 2 allocs/op
BenchmarkEncoder_Parallel_Binding_GoJson-16 3321 ns/op 3925.52 MB/s 9496 B/op 1 allocs/op
BenchmarkEncoder_Parallel_Binding_StdLib-16 3739 ns/op 3486.49 MB/s 9480 B/op 1 allocs/op
BenchmarkDecoder_Generic_Sonic-16 66812 ns/op 195.10 MB/s 57602 B/op 723 allocs/op
BenchmarkDecoder_Generic_Sonic_Fast-16 54523 ns/op 239.07 MB/s 49786 B/op 313 allocs/op
BenchmarkDecoder_Generic_StdLib-16 124260 ns/op 104.90 MB/s 50869 B/op 772 allocs/op
BenchmarkDecoder_Generic_JsonIter-16 91274 ns/op 142.81 MB/s 55782 B/op 1068 allocs/op
BenchmarkDecoder_Generic_GoJson-16 88569 ns/op 147.17 MB/s 66367 B/op 973 allocs/op
BenchmarkDecoder_Binding_Sonic-16 32557 ns/op 400.38 MB/s 28302 B/op 137 allocs/op
BenchmarkDecoder_Binding_Sonic_Fast-16 28649 ns/op 455.00 MB/s 24999 B/op 34 allocs/op
BenchmarkDecoder_Binding_StdLib-16 111437 ns/op 116.97 MB/s 10576 B/op 208 allocs/op
BenchmarkDecoder_Binding_JsonIter-16 35090 ns/op 371.48 MB/s 14673 B/op 385 allocs/op
BenchmarkDecoder_Binding_GoJson-16 28738 ns/op 453.59 MB/s 22039 B/op 49 allocs/op
BenchmarkDecoder_Parallel_Generic_Sonic-16 12321 ns/op 1057.91 MB/s 57233 B/op 723 allocs/op
BenchmarkDecoder_Parallel_Generic_Sonic_Fast-16 10644 ns/op 1224.64 MB/s 49362 B/op 313 allocs/op
BenchmarkDecoder_Parallel_Generic_StdLib-16 57587 ns/op 226.35 MB/s 50874 B/op 772 allocs/op
BenchmarkDecoder_Parallel_Generic_JsonIter-16 38666 ns/op 337.12 MB/s 55789 B/op 1068 allocs/op
BenchmarkDecoder_Parallel_Generic_GoJson-16 30259 ns/op 430.79 MB/s 66370 B/op 974 allocs/op
BenchmarkDecoder_Parallel_Binding_Sonic-16 5965 ns/op 2185.28 MB/s 27747 B/op 137 allocs/op
BenchmarkDecoder_Parallel_Binding_Sonic_Fast-16 5170 ns/op 2521.31 MB/s 24715 B/op 34 allocs/op
BenchmarkDecoder_Parallel_Binding_StdLib-16 27582 ns/op 472.58 MB/s 10576 B/op 208 allocs/op
BenchmarkDecoder_Parallel_Binding_JsonIter-16 13571 ns/op 960.51 MB/s 14685 B/op 385 allocs/op
BenchmarkDecoder_Parallel_Binding_GoJson-16 10031 ns/op 1299.51 MB/s 22111 B/op 49 allocs/op
BenchmarkGetOne_Sonic-16 3276 ns/op 3975.78 MB/s 24 B/op 1 allocs/op
BenchmarkGetOne_Gjson-16 9431 ns/op 1380.81 MB/s 0 B/op 0 allocs/op
BenchmarkGetOne_Jsoniter-16 51178 ns/op 254.46 MB/s 27936 B/op 647 allocs/op
BenchmarkGetOne_Parallel_Sonic-16 216.7 ns/op 60098.95 MB/s 24 B/op 1 allocs/op
BenchmarkGetOne_Parallel_Gjson-16 1076 ns/op 12098.62 MB/s 0 B/op 0 allocs/op
BenchmarkGetOne_Parallel_Jsoniter-16 17741 ns/op 734.06 MB/s 27945 B/op 647 allocs/op
BenchmarkSetOne_Sonic-16 9571 ns/op 1360.61 MB/s 1584 B/op 17 allocs/op
BenchmarkSetOne_Sjson-16 36456 ns/op 357.22 MB/s 52180 B/op 9 allocs/op
BenchmarkSetOne_Jsoniter-16 79475 ns/op 163.86 MB/s 45862 B/op 964 allocs/op
BenchmarkSetOne_Parallel_Sonic-16 850.9 ns/op 15305.31 MB/s 1584 B/op 17 allocs/op
BenchmarkSetOne_Parallel_Sjson-16 18194 ns/op 715.77 MB/s 52247 B/op 9 allocs/op
BenchmarkSetOne_Parallel_Jsoniter-16 33560 ns/op 388.05 MB/s 45892 B/op 964 allocs/op
```
- [小型](https://github.com/bytedance/sonic/blob/main/testdata/small.go) (400B, 11 个键, 3 层)
![small benchmarks](./docs/imgs/bench-small.png)
- [大型](https://github.com/bytedance/sonic/blob/main/testdata/twitter.json) (635kB, 10000+ 个键, 6 层)
![large benchmarks](./docs/imgs/bench-large.png)
要查看基准测试代码，请参阅 [bench.sh](https://github.com/bytedance/sonic/blob/main/bench.sh) 。
## 工作原理
请参阅 [INTRODUCTION_ZH_CN.md](./docs/INTRODUCTION_ZH_CN.md).
## 使用方式
### 序列化/反序列化
默认的行为基本上与 `encoding/json` 相一致，除了 HTML 转义形式（参见 [Escape HTML](https://github.com/bytedance/sonic/blob/main/README.md#escape-html)) 和 `SortKeys` 功能（参见 [Sort Keys](https://github.com/bytedance/sonic/blob/main/README.md#sort-keys)）**没有**遵循 [RFC8259](https://datatracker.ietf.org/doc/html/rfc8259) 。
```go
import "github.com/bytedance/sonic"
var data YourSchema
// Marshal
output, err := sonic.Marshal(&data)
// Unmarshal
err := sonic.Unmarshal(output, &data)
```
### 流式输入输出
Sonic 支持解码 `io.Reader` 中输入的 json，或将对象编码为 json 后输出至 `io.Writer`，以处理多个值并减少内存消耗。
- 编码器
```go
var o1 = map[string]interface{}{
"a": "b",
}
var o2 = 1
var w = bytes.NewBuffer(nil)
var enc = sonic.ConfigDefault.NewEncoder(w)
enc.Encode(o1)
enc.Encode(o2)
fmt.Println(w.String())
// Output:
// {"a":"b"}
// 1
```
- 解码器
```go
var o = map[string]interface{}{}
var r = strings.NewReader(`{"a":"b"}{"1":"2"}`)
var dec = sonic.ConfigDefault.NewDecoder(r)
dec.Decode(&o)
dec.Decode(&o)
fmt.Printf("%+v", o)
// Output:
// map[1:2 a:b]
```
### 使用 `Number` / `int64`
```go
import "github.com/bytedance/sonic/decoder"
var input = `1`
var data interface{}
// default float64
dc := decoder.NewDecoder(input)
dc.Decode(&data) // data == float64(1)
// use json.Number
dc = decoder.NewDecoder(input)
dc.UseNumber()
dc.Decode(&data) // data == json.Number("1")
// use int64
dc = decoder.NewDecoder(input)
dc.UseInt64()
dc.Decode(&data) // data == int64(1)
root, err := sonic.GetFromString(input)
// Get json.Number
jn := root.Number()
jm := root.InterfaceUseNumber().(json.Number) // jn == jm
// Get float64
fn := root.Float64()
fm := root.Interface().(float64) // jn == jm
```
### 对键排序
考虑到排序带来的性能损失（约 10% ）， sonic 默认不会启用这个功能。如果你的组件依赖这个行为（如 [zstd](https://github.com/facebook/zstd)) ，可以仿照下面的例子：
```go
import "github.com/bytedance/sonic"
import "github.com/bytedance/sonic/encoder"
// Binding map only
m := map[string]interface{}{}
v, err := encoder.Encode(m, encoder.SortMapKeys)
// Or ast.Node.SortKeys() before marshal
var root := sonic.Get(JSON)
err := root.SortKeys()
```
### HTML 转义
考虑到性能损失（约15%）， sonic 默认不会启用这个功能。你可以使用 `encoder.EscapeHTML` 选项来开启（与 `encoding/json.HTMLEscape` 行为一致）。
```go
import "github.com/bytedance/sonic"
v := map[string]string{"&&":"<>"}
ret, err := Encode(v, EscapeHTML) // ret == `{"\u0026\u0026":{"X":"\u003c\u003e"}}`
```
### 紧凑格式
Sonic 默认将基本类型（ `struct` ， `map` 等）编码为紧凑格式的 JSON ，除非使用 `json.RawMessage` or `json.Marshaler` 进行编码： sonic 确保输出的 JSON 合法，但出于性能考虑，**不会**加工成紧凑格式。我们提供选项 `encoder.CompactMarshaler` 来添加此过程，
### 打印错误
如果输入的 JSON 存在无效的语法，sonic 将返回 `decoder.SyntaxError`，该错误支持错误位置的美化输出。
```go
import "github.com/bytedance/sonic"
import "github.com/bytedance/sonic/decoder"
var data interface{}
err := sonic.UnmarshalString("[[[}]]", &data)
if err != nil {
/* One line by default */
println(e.Error()) // "Syntax error at index 3: invalid char\n\n\t[[[}]]\n\t...^..\n"
/* Pretty print */
if e, ok := err.(decoder.SyntaxError); ok {
/*Syntax error at index 3: invalid char
[[[}]]
...^..
*/
print(e.Description())
} else if me, ok := err.(*decoder.MismatchTypeError); ok {
// decoder.MismatchTypeError is new to Sonic v1.6.0
print(me.Description())
}
}
```
#### 类型不匹配 [Sonic v1.6.0]
如果给定键中存在**类型不匹配**的值， sonic 会抛出 `decoder.MismatchTypeError` （如果有多个，只会报告最后一个），但仍会跳过错误的值并解码下一个 JSON 。
```go
import "github.com/bytedance/sonic"
import "github.com/bytedance/sonic/decoder"
var data = struct{
A int
B int
}{}
err := UnmarshalString(`{"A":"1","B":1}`, &data)
println(err.Error()) // Mismatch type int with value string "at index 5: mismatched type with value\n\n\t{\"A\":\"1\",\"B\":1}\n\t.....^.........\n"
fmt.Printf("%+v", data) // {A:0 B:1}
```
### `Ast.Node`
Sonic/ast.Node 是完全独立的 JSON 抽象语法树库。它实现了序列化和反序列化，并提供了获取和修改通用数据的鲁棒的 API。
#### 查找/索引
通过给定的路径搜索 JSON 片段，路径必须为非负整数，字符串或 `nil` 。
```go
import "github.com/bytedance/sonic"
input := []byte(`{"key1":[{},{"key2":{"key3":[1,2,3]}}]}`)
// no path, returns entire json
root, err := sonic.Get(input)
raw := root.Raw() // == string(input)
// multiple paths
root, err := sonic.Get(input, "key1", 1, "key2")
sub := root.Get("key3").Index(2).Int64() // == 3
```
**注意**：由于 `Index()` 使用偏移量来定位数据，比使用扫描的 `Get()` 要快的多，建议尽可能的使用 `Index` 。 Sonic 也提供了另一个 API， `IndexOrGet()` ，以偏移量为基础并且也确保键的匹配。
#### 修改
使用 ` Set()` / `Unset()` 修改 json 的内容
```go
import "github.com/bytedance/sonic"
// Set
exist, err := root.Set("key4", NewBool(true)) // exist == false
alias1 := root.Get("key4")
println(alias1.Valid()) // true
alias2 := root.Index(1)
println(alias1 == alias2) // true
// Unset
exist, err := root.UnsetByIndex(1) // exist == true
println(root.Get("key4").Check()) // "value not exist"
```
#### 序列化
要将 `ast.Node` 编码为 json ，使用 `MarshalJson()` 或者 `json.Marshal()` （必须传递指向节点的指针）
```go
import (
"encoding/json"
"github.com/bytedance/sonic"
)
buf, err := root.MarshalJson()
println(string(buf)) // {"key1":[{},{"key2":{"key3":[1,2,3]}}]}
exp, err := json.Marshal(&root) // WARN: use pointer
println(string(buf) == string(exp)) // true
```
#### APIs
- 合法性检查： `Check()`, `Error()`, `Valid()`, `Exist()`
- 索引： `Index()`, `Get()`, `IndexPair()`, `IndexOrGet()`, `GetByPath()`
- 转换至 go 内置类型： `Int64()`, `Float64()`, `String()`, `Number()`, `Bool()`, `Map[UseNumber|UseNode]()`, `Array[UseNumber|UseNode]()`, `Interface[UseNumber|UseNode]()`
- go 类型打包： `NewRaw()`, `NewNumber()`, `NewNull()`, `NewBool()`, `NewString()`, `NewObject()`, `NewArray()`
- 迭代： `Values()`, `Properties()`, `ForEach()`, `SortKeys()`
- 修改： `Set()`, `SetByIndex()`, `Add()`
## 兼容性
由于开发高性能代码的困难性， Sonic **不**保证对所有环境的支持。对于在不同环境中使用 Sonic 构建应用程序的开发者，我们有以下建议：
- 在 **Mac M1** 上开发：确保在您的计算机上安装了 Rosetta 2，并在构建时设置 `GOARCH=amd64` 。 Rosetta 2 可以自动将 x86 二进制文件转换为 arm64 二进制文件，并在 Mac M1 上运行 x86 应用程序。
- 在 **Linux arm64** 上开发：您可以安装 qemu 并使用 `qemu-x86_64 -cpu max` 命令来将 x86 二进制文件转换为 arm64 二进制文件。qemu可以实现与Mac M1上的Rosetta 2类似的转换效果。
对于希望在不使用 qemu 下使用 sonic 的开发者，或者希望处理 JSON 时与 `encoding/JSON` 严格保持一致的开发者，我们在 `sonic.API` 中提供了一些兼容性 API
- `ConfigDefault`: 在支持 sonic 的环境下 sonic 的默认配置（`EscapeHTML=false`，`SortKeys=false`等）。行为与具有相应配置的 `encoding/json` 一致，一些选项，如 `SortKeys=false` 将无效。
- `ConfigStd`: 在支持 sonic 的环境下与标准库兼容的配置（`EscapeHTML=true`，`SortKeys=true`等）。行为与 `encoding/json` 一致。
- `ConfigFastest`: 在支持 sonic 的环境下运行最快的配置（`NoQuoteTextMarshaler=true`）。行为与具有相应配置的 `encoding/json` 一致，某些选项将无效。
## 注意事项
### 预热
由于 Sonic 使用 [golang-asm](https://github.com/twitchyliquid64/golang-asm) 作为 JIT 汇编器，这个库并不适用于运行时编译，第一次运行一个大型模式可能会导致请求超时甚至进程内存溢出。为了更好地稳定性，我们建议在运行大型模式或在内存有限的应用中，在使用 `Marshal()/Unmarshal()` 前运行 `Pretouch()`。
```go
import (
"reflect"
"github.com/bytedance/sonic"
"github.com/bytedance/sonic/option"
)
func init() {
var v HugeStruct
// For most large types (nesting depth <= option.DefaultMaxInlineDepth)
err := sonic.Pretouch(reflect.TypeOf(v))
// with more CompileOption...
err := sonic.Pretouch(reflect.TypeOf(v),
// If the type is too deep nesting (nesting depth > option.DefaultMaxInlineDepth),
// you can set compile recursive loops in Pretouch for better stability in JIT.
option.WithCompileRecursiveDepth(loop),
// For a large nested struct, try to set a smaller depth to reduce compiling time.
option.WithCompileMaxInlineDepth(depth),
)
}
```
### 拷贝字符串
当解码 **没有转义字符的字符串**时， sonic 会从原始的 JSON 缓冲区内引用而不是复制到新的一个缓冲区中。这对 CPU 的性能方面很有帮助，但是可能因此在解码后对象仍在使用的时候将整个 JSON 缓冲区保留在内存中。实践中我们发现，通过引用 JSON 缓冲区引入的额外内存通常是解码后对象的 20% 至 80% ，一旦应用长期保留这些对象（如缓存以备重用），服务器所使用的内存可能会增加。我们提供了选项 `decoder.CopyString()` 供用户选择，不引用 JSON 缓冲区。这可能在一定程度上降低 CPU 性能。
### 传递字符串还是字节数组？
为了和 `encoding/json` 保持一致，我们提供了传递 `[]byte` 作为参数的 API ，但考虑到安全性，字符串到字节的复制是同时进行的，这在原始 JSON 非常大时可能会导致性能损失。因此，你可以使用 `UnmarshalString()` 和 `GetFromString()` 来传递字符串，只要你的原始数据是字符串，或**零拷贝类型转换**对于你的字节数组是安全的。我们也提供了 `MarshalString()` 的 API ，以便对编码的 JSON 字节数组进行**零拷贝类型转换**，因为 sonic 输出的字节始终是重复并且唯一的，所以这样是安全的。
### 加速 `encoding.TextMarshaler`
为了保证数据安全性， `sonic.Encoder` 默认会对来自 `encoding.TextMarshaler` 接口的字符串进行引用和转义，如果大部分数据都是这种形式那可能会导致很大的性能损失。我们提供了 `encoder.NoQuoteTextMarshaler` 选项来跳过这些操作，但你**必须**保证他们的输出字符串依照 [RFC8259](https://datatracker.ietf.org/doc/html/rfc8259) 进行了转义和引用。
### 泛型的性能优化
在 **完全解析**的场景下， `Unmarshal()` 表现得比 `Get()`+`Node.Interface()` 更好。但是如果你只有特定 JSON 的部分模式，你可以将 `Get()` 和 `Unmarshal()` 结合使用：
```go
import "github.com/bytedance/sonic"
node, err := sonic.GetFromString(_TwitterJson, "statuses", 3, "user")
var user User // your partial schema...
err = sonic.UnmarshalString(node.Raw(), &user)
```
甚至如果你没有任何模式，可以用 `ast.Node` 代替 `map` 或 `interface` 作为泛型的容器：
```go
import "github.com/bytedance/sonic"
root, err := sonic.GetFromString(_TwitterJson)
user := root.GetByPath("statuses", 3, "user") // === root.Get("status").Index(3).Get("user")
err = user.Check()
// err = user.LoadAll() // only call this when you want to use 'user' concurrently...
go someFunc(user)
```
为什么？因为 `ast.Node` 使用 `array` 来存储其子节点：
- 在插入（反序列化）和扫描（序列化）数据时，`Array` 的性能比 `Map` **好得多**；
- **哈希**（`map[x]`）的效率不如**索引**（`array[x]`）高效，而 `ast.Node` 可以在数组和对象上使用索引；
- 使用 `Interface()` / `Map()` 意味着 sonic 必须解析所有的底层值，而 `ast.Node` 可以**按需解析**它们。
**注意**：由于 `ast.Node` 的惰性加载设计，其**不能**直接保证并发安全性，但你可以调用 `Node.Load()` / `Node.LoadAll()` 来实现并发安全。尽管可能会带来性能损失，但仍比转换成 `map` 或 `interface{}` 更为高效。
## 社区
Sonic 是 [CloudWeGo](https://www.cloudwego.io/) 下的一个子项目。我们致力于构建云原生生态系统。
vendor/github.com/bytedance/sonic/api.go
Generated Vendored
+186
View File
@@ -0,0 +1,186 @@
/*
* Copyright 2021 ByteDance Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package sonic
import (
`io`
`github.com/bytedance/sonic/ast`
)
// Config is a combination of sonic/encoder.Options and sonic/decoder.Options
type Config struct {
// EscapeHTML indicates encoder to escape all HTML characters
// after serializing into JSON (see https://pkg.go.dev/encoding/json#HTMLEscape).
// WARNING: This hurts performance A LOT, USE WITH CARE.
EscapeHTML bool
// SortMapKeys indicates encoder that the keys of a map needs to be sorted
// before serializing into JSON.
// WARNING: This hurts performance A LOT, USE WITH CARE.
SortMapKeys bool
// CompactMarshaler indicates encoder that the output JSON from json.Marshaler
// is always compact and needs no validation
CompactMarshaler bool
// NoQuoteTextMarshaler indicates encoder that the output text from encoding.TextMarshaler
// is always escaped string and needs no quoting
NoQuoteTextMarshaler bool
// NoNullSliceOrMap indicates encoder that all empty Array or Object are encoded as '[]' or '{}',
// instead of 'null'
NoNullSliceOrMap bool
// UseInt64 indicates decoder to unmarshal an integer into an interface{} as an
// int64 instead of as a float64.
UseInt64 bool
// UseNumber indicates decoder to unmarshal a number into an interface{} as a
// json.Number instead of as a float64.
UseNumber bool
// UseUnicodeErrors indicates decoder to return an error when encounter invalid
// UTF-8 escape sequences.
UseUnicodeErrors bool
// DisallowUnknownFields indicates decoder to return an error when the destination
// is a struct and the input contains object keys which do not match any
// non-ignored, exported fields in the destination.
DisallowUnknownFields bool
// CopyString indicates decoder to decode string values by copying instead of referring.
CopyString bool
// ValidateString indicates decoder and encoder to valid string values: decoder will return errors
// when unescaped control chars(\u0000-\u001f) in the string value of JSON.
ValidateString bool
}
var (
// ConfigDefault is the default config of APIs, aiming at efficiency and safty.
ConfigDefault = Config{}.Froze()
// ConfigStd is the standard config of APIs, aiming at being compatible with encoding/json.
ConfigStd = Config{
EscapeHTML : true,
SortMapKeys: true,
CompactMarshaler: true,
CopyString : true,
ValidateString : true,
}.Froze()
// ConfigFastest is the fastest config of APIs, aiming at speed.
ConfigFastest = Config{
NoQuoteTextMarshaler: true,
}.Froze()
)
// API is a binding of specific config.
// This interface is inspired by github.com/json-iterator/go,
// and has same behaviors under equavilent config.
type API interface {
// MarshalToString returns the JSON encoding string of v
MarshalToString(v interface{}) (string, error)
// Marshal returns the JSON encoding bytes of v.
Marshal(v interface{}) ([]byte, error)
// MarshalIndent returns the JSON encoding bytes with indent and prefix.
MarshalIndent(v interface{}, prefix, indent string) ([]byte, error)
// UnmarshalFromString parses the JSON-encoded bytes and stores the result in the value pointed to by v.
UnmarshalFromString(str string, v interface{}) error
// Unmarshal parses the JSON-encoded string and stores the result in the value pointed to by v.
Unmarshal(data []byte, v interface{}) error
// NewEncoder create a Encoder holding writer
NewEncoder(writer io.Writer) Encoder
// NewDecoder create a Decoder holding reader
NewDecoder(reader io.Reader) Decoder
// Valid validates the JSON-encoded bytes and reportes if it is valid
Valid(data []byte) bool
}
// Encoder encodes JSON into io.Writer
type Encoder interface {
// Encode writes the JSON encoding of v to the stream, followed by a newline character.
Encode(val interface{}) error
// SetEscapeHTML specifies whether problematic HTML characters
// should be escaped inside JSON quoted strings.
// The default behavior NOT ESCAPE
SetEscapeHTML(on bool)
// SetIndent instructs the encoder to format each subsequent encoded value
// as if indented by the package-level function Indent(dst, src, prefix, indent).
// Calling SetIndent("", "") disables indentation
SetIndent(prefix, indent string)
}
// Decoder decodes JSON from io.Read
type Decoder interface {
// Decode reads the next JSON-encoded value from its input and stores it in the value pointed to by v.
Decode(val interface{}) error
// Buffered returns a reader of the data remaining in the Decoder's buffer.
// The reader is valid until the next call to Decode.
Buffered() io.Reader
// DisallowUnknownFields causes the Decoder to return an error when the destination is a struct
// and the input contains object keys which do not match any non-ignored, exported fields in the destination.
DisallowUnknownFields()
// More reports whether there is another element in the current array or object being parsed.
More() bool
// UseNumber causes the Decoder to unmarshal a number into an interface{} as a Number instead of as a float64.
UseNumber()
}
// Marshal returns the JSON encoding bytes of v.
func Marshal(val interface{}) ([]byte, error) {
return ConfigDefault.Marshal(val)
}
// MarshalString returns the JSON encoding string of v.
func MarshalString(val interface{}) (string, error) {
return ConfigDefault.MarshalToString(val)
}
// Unmarshal parses the JSON-encoded data and stores the result in the value pointed to by v.
// NOTICE: This API copies given buffer by default,
// if you want to pass JSON more efficiently, use UnmarshalString instead.
func Unmarshal(buf []byte, val interface{}) error {
return ConfigDefault.Unmarshal(buf, val)
}
// UnmarshalString is like Unmarshal, except buf is a string.
func UnmarshalString(buf string, val interface{}) error {
return ConfigDefault.UnmarshalFromString(buf, val)
}
// Get searches the given path from json,
// and returns its representing ast.Node.
//
// Each path arg must be integer or string:
// - Integer is target index(>=0), means searching current node as array.
// - String is target key, means searching current node as object.
//
//
// Note, the api expects the json is well-formed at least,
// otherwise it may return unexpected result.
func Get(src []byte, path ...interface{}) (ast.Node, error) {
return GetFromString(string(src), path...)
}
// GetFromString is same with Get except src is string,
// which can reduce unnecessary memory copy.
func GetFromString(src string, path ...interface{}) (ast.Node, error) {
return ast.NewSearcher(src).GetByPath(path...)
}
vendor/github.com/bytedance/sonic/ast/api_amd64.go
Generated Vendored
+151
View File
@@ -0,0 +1,151 @@
// +build amd64,go1.15,!go1.21
/*
* Copyright 2022 ByteDance Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package ast
import (
`runtime`
`unsafe`
`github.com/bytedance/sonic/encoder`
`github.com/bytedance/sonic/internal/native`
`github.com/bytedance/sonic/internal/native/types`
`github.com/bytedance/sonic/internal/rt`
uq `github.com/bytedance/sonic/unquote`
`github.com/chenzhuoyu/base64x`
)
var typeByte = rt.UnpackEface(byte(0)).Type
//go:nocheckptr
func quote(buf *[]byte, val string) {
*buf = append(*buf, '"')
if len(val) == 0 {
*buf = append(*buf, '"')
return
}
sp := rt.IndexChar(val, 0)
nb := len(val)
b := (*rt.GoSlice)(unsafe.Pointer(buf))
// input buffer
for nb > 0 {
// output buffer
dp := unsafe.Pointer(uintptr(b.Ptr) + uintptr(b.Len))
dn := b.Cap - b.Len
// call native.Quote, dn is byte count it outputs
ret := native.Quote(sp, nb, dp, &dn, 0)
// update *buf length
b.Len += dn
// no need more output
if ret >= 0 {
break
}
// double buf size
*b = growslice(typeByte, *b, b.Cap*2)
// ret is the complement of consumed input
ret = ^ret
// update input buffer
nb -= ret
sp = unsafe.Pointer(uintptr(sp) + uintptr(ret))
}
runtime.KeepAlive(buf)
runtime.KeepAlive(sp)
*buf = append(*buf, '"')
}
func unquote(src string) (string, types.ParsingError) {
return uq.String(src)
}
func decodeBase64(src string) ([]byte, error) {
return base64x.StdEncoding.DecodeString(src)
}
func encodeBase64(src []byte) string {
return base64x.StdEncoding.EncodeToString(src)
}
func (self *Parser) decodeValue() (val types.JsonState) {
sv := (*rt.GoString)(unsafe.Pointer(&self.s))
self.p = native.Value(sv.Ptr, sv.Len, self.p, &val, 0)
return
}
func (self *Parser) skip() (int, types.ParsingError) {
fsm := types.NewStateMachine()
start := native.SkipOne(&self.s, &self.p, fsm, 0)
types.FreeStateMachine(fsm)
if start < 0 {
return self.p, types.ParsingError(-start)
}
return start, 0
}
func (self *Node) encodeInterface(buf *[]byte) error {
//WARN: NOT compatible with json.Encoder
return encoder.EncodeInto(buf, self.packAny(), 0)
}
func (self *Parser) skipFast() (int, types.ParsingError) {
start := native.SkipOneFast(&self.s, &self.p)
if start < 0 {
return self.p, types.ParsingError(-start)
}
return start, 0
}
func (self *Parser) getByPath(path ...interface{}) (int, types.ParsingError) {
fsm := types.NewStateMachine()
start := native.GetByPath(&self.s, &self.p, &path, fsm)
types.FreeStateMachine(fsm)
runtime.KeepAlive(path)
if start < 0 {
return self.p, types.ParsingError(-start)
}
return start, 0
}
func (self *Searcher) GetByPath(path ...interface{}) (Node, error) {
var err types.ParsingError
var start int
self.parser.p = 0
start, err = self.parser.getByPath(path...)
if err != 0 {
// for compatibility with old version
if err == types.ERR_NOT_FOUND {
return Node{}, ErrNotExist
}
if err == types.ERR_UNSUPPORT_TYPE {
panic("path must be either int(>=0) or string")
}
return Node{}, self.parser.syntaxError(err)
}
t := switchRawType(self.parser.s[start])
if t == _V_NONE {
return Node{}, self.parser.ExportError(err)
}
return newRawNode(self.parser.s[start:self.parser.p], t), nil
}
vendor/github.com/bytedance/sonic/ast/api_compat.go
Generated Vendored
+120
View File
@@ -0,0 +1,120 @@
// +build !amd64 go1.21
/*
* Copyright 2022 ByteDance Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package ast
import (
`encoding/base64`
`encoding/json`
`fmt`
`github.com/bytedance/sonic/internal/native/types`
`github.com/bytedance/sonic/internal/rt`
)
func quote(buf *[]byte, val string) {
quoteString(buf, val)
}
func unquote(src string) (string, types.ParsingError) {
sp := rt.IndexChar(src, -1)
out, ok := unquoteBytes(rt.BytesFrom(sp, len(src)+2, len(src)+2))
if !ok {
return "", types.ERR_INVALID_ESCAPE
}
return rt.Mem2Str(out), 0
}
func decodeBase64(src string) ([]byte, error) {
return base64.StdEncoding.DecodeString(src)
}
func encodeBase64(src []byte) string {
return base64.StdEncoding.EncodeToString(src)
}
func (self *Parser) decodeValue() (val types.JsonState) {
e, v := decodeValue(self.s, self.p)
if e < 0 {
return v
}
self.p = e
return v
}
func (self *Parser) skip() (int, types.ParsingError) {
e, s := skipValue(self.s, self.p)
if e < 0 {
return self.p, types.ParsingError(-e)
}
self.p = e
return s, 0
}
func (self *Parser) skipFast() (int, types.ParsingError) {
e, s := skipValueFast(self.s, self.p)
if e < 0 {
return self.p, types.ParsingError(-e)
}
self.p = e
return s, 0
}
func (self *Node) encodeInterface(buf *[]byte) error {
out, err := json.Marshal(self.packAny())
if err != nil {
return err
}
*buf = append(*buf, out...)
return nil
}
func (self *Searcher) GetByPath(path ...interface{}) (Node, error) {
self.parser.p = 0
var err types.ParsingError
for _, p := range path {
if idx, ok := p.(int); ok && idx >= 0 {
if err = self.parser.searchIndex(idx); err != 0 {
return Node{}, self.parser.ExportError(err)
}
} else if key, ok := p.(string); ok {
if err = self.parser.searchKey(key); err != 0 {
return Node{}, self.parser.ExportError(err)
}
} else {
panic("path must be either int(>=0) or string")
}
}
var start = self.parser.p
if start, err = self.parser.skip(); err != 0 {
return Node{}, self.parser.ExportError(err)
}
ns := len(self.parser.s)
if self.parser.p > ns || start >= ns || start>=self.parser.p {
return Node{}, fmt.Errorf("skip %d char out of json boundary", start)
}
t := switchRawType(self.parser.s[start])
if t == _V_NONE {
return Node{}, self.parser.ExportError(err)
}
return newRawNode(self.parser.s[start:self.parser.p], t), nil
}
vendor/github.com/bytedance/sonic/ast/asm.s
Generated Vendored
View File
vendor/github.com/bytedance/sonic/ast/decode.go
Generated Vendored
+575
View File
@@ -0,0 +1,575 @@
/*
* Copyright 2022 ByteDance Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package ast
import (
`encoding/base64`
`runtime`
`strconv`
`unsafe`
`github.com/bytedance/sonic/internal/native/types`
`github.com/bytedance/sonic/internal/rt`
)
const _blankCharsMask = (1 << ' ') | (1 << '\t') | (1 << '\r') | (1 << '\n')
const (
bytesNull = "null"
bytesTrue = "true"
bytesFalse = "false"
bytesObject = "{}"
bytesArray = "[]"
)
func isSpace(c byte) bool {
return (int(1<<c) & _blankCharsMask) != 0
}
//go:nocheckptr
func skipBlank(src string, pos int) int {
se := uintptr(rt.IndexChar(src, len(src)))
sp := uintptr(rt.IndexChar(src, pos))
for sp < se {
if !isSpace(*(*byte)(unsafe.Pointer(sp))) {
break
}
sp += 1
}
if sp >= se {
return -int(types.ERR_EOF)
}
runtime.KeepAlive(src)
return int(sp - uintptr(rt.IndexChar(src, 0)))
}
func decodeNull(src string, pos int) (ret int) {
ret = pos + 4
if ret > len(src) {
return -int(types.ERR_EOF)
}
if src[pos:ret] == bytesNull {
return ret
} else {
return -int(types.ERR_INVALID_CHAR)
}
}
func decodeTrue(src string, pos int) (ret int) {
ret = pos + 4
if ret > len(src) {
return -int(types.ERR_EOF)
}
if src[pos:ret] == bytesTrue {
return ret
} else {
return -int(types.ERR_INVALID_CHAR)
}
}
func decodeFalse(src string, pos int) (ret int) {
ret = pos + 5
if ret > len(src) {
return -int(types.ERR_EOF)
}
if src[pos:ret] == bytesFalse {
return ret
}
return -int(types.ERR_INVALID_CHAR)
}
//go:nocheckptr
func decodeString(src string, pos int) (ret int, v string) {
ret, ep := skipString(src, pos)
if ep == -1 {
(*rt.GoString)(unsafe.Pointer(&v)).Ptr = rt.IndexChar(src, pos+1)
(*rt.GoString)(unsafe.Pointer(&v)).Len = ret - pos - 2
return ret, v
}
vv, ok := unquoteBytes(rt.Str2Mem(src[pos:ret]))
if !ok {
return -int(types.ERR_INVALID_CHAR), ""
}
runtime.KeepAlive(src)
return ret, rt.Mem2Str(vv)
}
func decodeBinary(src string, pos int) (ret int, v []byte) {
var vv string
ret, vv = decodeString(src, pos)
if ret < 0 {
return ret, nil
}
var err error
v, err = base64.StdEncoding.DecodeString(vv)
if err != nil {
return -int(types.ERR_INVALID_CHAR), nil
}
return ret, v
}
func isDigit(c byte) bool {
return c >= '0' && c <= '9'
}
//go:nocheckptr
func decodeInt64(src string, pos int) (ret int, v int64, err error) {
sp := uintptr(rt.IndexChar(src, pos))
ss := uintptr(sp)
se := uintptr(rt.IndexChar(src, len(src)))
if uintptr(sp) >= se {
return -int(types.ERR_EOF), 0, nil
}
if c := *(*byte)(unsafe.Pointer(sp)); c == '-' {
sp += 1
}
if sp == se {
return -int(types.ERR_EOF), 0, nil
}
for ; sp < se; sp += uintptr(1) {
if !isDigit(*(*byte)(unsafe.Pointer(sp))) {
break
}
}
if sp < se {
if c := *(*byte)(unsafe.Pointer(sp)); c == '.' || c == 'e' || c == 'E' {
return -int(types.ERR_INVALID_NUMBER_FMT), 0, nil
}
}
var vv string
ret = int(uintptr(sp) - uintptr((*rt.GoString)(unsafe.Pointer(&src)).Ptr))
(*rt.GoString)(unsafe.Pointer(&vv)).Ptr = unsafe.Pointer(ss)
(*rt.GoString)(unsafe.Pointer(&vv)).Len = ret - pos
v, err = strconv.ParseInt(vv, 10, 64)
if err != nil {
//NOTICE: allow overflow here
if err.(*strconv.NumError).Err == strconv.ErrRange {
return ret, 0, err
}
return -int(types.ERR_INVALID_CHAR), 0, err
}
runtime.KeepAlive(src)
return ret, v, nil
}
func isNumberChars(c byte) bool {
return (c >= '0' && c <= '9') || c == '+' || c == '-' || c == 'e' || c == 'E' || c == '.'
}
//go:nocheckptr
func decodeFloat64(src string, pos int) (ret int, v float64, err error) {
sp := uintptr(rt.IndexChar(src, pos))
ss := uintptr(sp)
se := uintptr(rt.IndexChar(src, len(src)))
if uintptr(sp) >= se {
return -int(types.ERR_EOF), 0, nil
}
if c := *(*byte)(unsafe.Pointer(sp)); c == '-' {
sp += 1
}
if sp == se {
return -int(types.ERR_EOF), 0, nil
}
for ; sp < se; sp += uintptr(1) {
if !isNumberChars(*(*byte)(unsafe.Pointer(sp))) {
break
}
}
var vv string
ret = int(uintptr(sp) - uintptr((*rt.GoString)(unsafe.Pointer(&src)).Ptr))
(*rt.GoString)(unsafe.Pointer(&vv)).Ptr = unsafe.Pointer(ss)
(*rt.GoString)(unsafe.Pointer(&vv)).Len = ret - pos
v, err = strconv.ParseFloat(vv, 64)
if err != nil {
//NOTICE: allow overflow here
if err.(*strconv.NumError).Err == strconv.ErrRange {
return ret, 0, err
}
return -int(types.ERR_INVALID_CHAR), 0, err
}
runtime.KeepAlive(src)
return ret, v, nil
}
func decodeValue(src string, pos int) (ret int, v types.JsonState) {
pos = skipBlank(src, pos)
if pos < 0 {
return pos, types.JsonState{Vt: types.ValueType(pos)}
}
switch c := src[pos]; c {
case 'n':
ret = decodeNull(src, pos)
if ret < 0 {
return ret, types.JsonState{Vt: types.ValueType(ret)}
}
return ret, types.JsonState{Vt: types.V_NULL}
case '"':
var ep int
ret, ep = skipString(src, pos)
if ret < 0 {
return ret, types.JsonState{Vt: types.ValueType(ret)}
}
return ret, types.JsonState{Vt: types.V_STRING, Iv: int64(pos + 1), Ep: ep}
case '{':
return pos + 1, types.JsonState{Vt: types.V_OBJECT}
case '[':
return pos + 1, types.JsonState{Vt: types.V_ARRAY}
case 't':
ret = decodeTrue(src, pos)
if ret < 0 {
return ret, types.JsonState{Vt: types.ValueType(ret)}
}
return ret, types.JsonState{Vt: types.V_TRUE}
case 'f':
ret = decodeFalse(src, pos)
if ret < 0 {
return ret, types.JsonState{Vt: types.ValueType(ret)}
}
return ret, types.JsonState{Vt: types.V_FALSE}
case '-', '+', '0', '1', '2', '3', '4', '5', '6', '7', '8', '9':
var iv int64
ret, iv, _ = decodeInt64(src, pos)
if ret >= 0 {
return ret, types.JsonState{Vt: types.V_INTEGER, Iv: iv, Ep: pos}
} else if ret != -int(types.ERR_INVALID_NUMBER_FMT) {
return ret, types.JsonState{Vt: types.ValueType(ret)}
}
var fv float64
ret, fv, _ = decodeFloat64(src, pos)
if ret >= 0 {
return ret, types.JsonState{Vt: types.V_DOUBLE, Dv: fv, Ep: pos}
} else {
return ret, types.JsonState{Vt: types.ValueType(ret)}
}
default:
return -int(types.ERR_INVALID_CHAR), types.JsonState{Vt:-types.ValueType(types.ERR_INVALID_CHAR)}
}
}
//go:nocheckptr
func skipNumber(src string, pos int) (ret int) {
sp := uintptr(rt.IndexChar(src, pos))
se := uintptr(rt.IndexChar(src, len(src)))
if uintptr(sp) >= se {
return -int(types.ERR_EOF)
}
if c := *(*byte)(unsafe.Pointer(sp)); c == '-' {
sp += 1
}
ss := sp
var pointer bool
var exponent bool
var lastIsDigit bool
var nextNeedDigit = true
for ; sp < se; sp += uintptr(1) {
c := *(*byte)(unsafe.Pointer(sp))
if isDigit(c) {
lastIsDigit = true
nextNeedDigit = false
continue
} else if nextNeedDigit {
return -int(types.ERR_INVALID_CHAR)
} else if c == '.' {
if !lastIsDigit || pointer || exponent || sp == ss {
return -int(types.ERR_INVALID_CHAR)
}
pointer = true
lastIsDigit = false
nextNeedDigit = true
continue
} else if c == 'e' || c == 'E' {
if !lastIsDigit || exponent {
return -int(types.ERR_INVALID_CHAR)
}
if sp == se-1 {
return -int(types.ERR_EOF)
}
exponent = true
lastIsDigit = false
nextNeedDigit = false
continue
} else if c == '-' || c == '+' {
if prev := *(*byte)(unsafe.Pointer(sp - 1)); prev != 'e' && prev != 'E' {
return -int(types.ERR_INVALID_CHAR)
}
lastIsDigit = false
nextNeedDigit = true
continue
} else {
break
}
}
if nextNeedDigit {
return -int(types.ERR_EOF)
}
runtime.KeepAlive(src)
return int(uintptr(sp) - uintptr((*rt.GoString)(unsafe.Pointer(&src)).Ptr))
}
//go:nocheckptr
func skipString(src string, pos int) (ret int, ep int) {
if pos+1 >= len(src) {
return -int(types.ERR_EOF), -1
}
sp := uintptr(rt.IndexChar(src, pos))
se := uintptr(rt.IndexChar(src, len(src)))
// not start with quote
if *(*byte)(unsafe.Pointer(sp)) != '"' {
return -int(types.ERR_INVALID_CHAR), -1
}
sp += 1
ep = -1
for sp < se {
c := *(*byte)(unsafe.Pointer(sp))
if c == '\\' {
if ep == -1 {
ep = int(uintptr(sp) - uintptr((*rt.GoString)(unsafe.Pointer(&src)).Ptr))
}
sp += 2
continue
}
sp += 1
if c == '"' {
return int(uintptr(sp) - uintptr((*rt.GoString)(unsafe.Pointer(&src)).Ptr)), ep
}
}
runtime.KeepAlive(src)
// not found the closed quote until EOF
return -int(types.ERR_EOF), -1
}
//go:nocheckptr
func skipPair(src string, pos int, lchar byte, rchar byte) (ret int) {
if pos+1 >= len(src) {
return -int(types.ERR_EOF)
}
sp := uintptr(rt.IndexChar(src, pos))
se := uintptr(rt.IndexChar(src, len(src)))
if *(*byte)(unsafe.Pointer(sp)) != lchar {
return -int(types.ERR_INVALID_CHAR)
}
sp += 1
nbrace := 1
inquote := false
for sp < se {
c := *(*byte)(unsafe.Pointer(sp))
if c == '\\' {
sp += 2
continue
} else if c == '"' {
inquote = !inquote
} else if c == lchar {
if !inquote {
nbrace += 1
}
} else if c == rchar {
if !inquote {
nbrace -= 1
if nbrace == 0 {
sp += 1
break
}
}
}
sp += 1
}
if nbrace != 0 {
return -int(types.ERR_INVALID_CHAR)
}
runtime.KeepAlive(src)
return int(uintptr(sp) - uintptr((*rt.GoString)(unsafe.Pointer(&src)).Ptr))
}
func skipValueFast(src string, pos int) (ret int, start int) {
pos = skipBlank(src, pos)
if pos < 0 {
return pos, -1
}
switch c := src[pos]; c {
case 'n':
ret = decodeNull(src, pos)
case '"':
ret, _ = skipString(src, pos)
case '{':
ret = skipPair(src, pos, '{', '}')
case '[':
ret = skipPair(src, pos, '[', ']')
case 't':
ret = decodeTrue(src, pos)
case 'f':
ret = decodeFalse(src, pos)
case '-', '+', '0', '1', '2', '3', '4', '5', '6', '7', '8', '9':
ret = skipNumber(src, pos)
default:
ret = -int(types.ERR_INVALID_CHAR)
}
return ret, pos
}
func skipValue(src string, pos int) (ret int, start int) {
pos = skipBlank(src, pos)
if pos < 0 {
return pos, -1
}
switch c := src[pos]; c {
case 'n':
ret = decodeNull(src, pos)
case '"':
ret, _ = skipString(src, pos)
case '{':
ret, _ = skipObject(src, pos)
case '[':
ret, _ = skipArray(src, pos)
case 't':
ret = decodeTrue(src, pos)
case 'f':
ret = decodeFalse(src, pos)
case '-', '+', '0', '1', '2', '3', '4', '5', '6', '7', '8', '9':
ret = skipNumber(src, pos)
default:
ret = -int(types.ERR_INVALID_CHAR)
}
return ret, pos
}
func skipObject(src string, pos int) (ret int, start int) {
start = skipBlank(src, pos)
if start < 0 {
return start, -1
}
if src[start] != '{' {
return -int(types.ERR_INVALID_CHAR), -1
}
pos = start + 1
pos = skipBlank(src, pos)
if pos < 0 {
return pos, -1
}
if src[pos] == '}' {
return pos + 1, start
}
for {
pos, _ = skipString(src, pos)
if pos < 0 {
return pos, -1
}
pos = skipBlank(src, pos)
if pos < 0 {
return pos, -1
}
if src[pos] != ':' {
return -int(types.ERR_INVALID_CHAR), -1
}
pos++
pos, _ = skipValue(src, pos)
if pos < 0 {
return pos, -1
}
pos = skipBlank(src, pos)
if pos < 0 {
return pos, -1
}
if src[pos] == '}' {
return pos + 1, start
}
if src[pos] != ',' {
return -int(types.ERR_INVALID_CHAR), -1
}
pos++
pos = skipBlank(src, pos)
if pos < 0 {
return pos, -1
}
}
}
func skipArray(src string, pos int) (ret int, start int) {
start = skipBlank(src, pos)
if start < 0 {
return start, -1
}
if src[start] != '[' {
return -int(types.ERR_INVALID_CHAR), -1
}
pos = start + 1
pos = skipBlank(src, pos)
if pos < 0 {
return pos, -1
}
if src[pos] == ']' {
return pos + 1, start
}
for {
pos, _ = skipValue(src, pos)
if pos < 0 {
return pos, -1
}
pos = skipBlank(src, pos)
if pos < 0 {
return pos, -1
}
if src[pos] == ']' {
return pos + 1, start
}
if src[pos] != ',' {
return -int(types.ERR_INVALID_CHAR), -1
}
pos++
}
}
vendor/github.com/bytedance/sonic/ast/encode.go
Generated Vendored
+259
View File
@@ -0,0 +1,259 @@
/*
* Copyright 2021 ByteDance Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package ast
import (
`sync`
`unicode/utf8`
`github.com/bytedance/sonic/internal/rt`
)
const (
_MaxBuffer = 1024 // 1KB buffer size
)
func quoteString(e *[]byte, s string) {
*e = append(*e, '"')
start := 0
for i := 0; i < len(s); {
if b := s[i]; b < utf8.RuneSelf {
if safeSet[b] {
i++
continue
}
if start < i {
*e = append(*e, s[start:i]...)
}
*e = append(*e, '\\')
switch b {
case '\\', '"':
*e = append(*e, b)
case '\n':
*e = append(*e, 'n')
case '\r':
*e = append(*e, 'r')
case '\t':
*e = append(*e, 't')
default:
// This encodes bytes < 0x20 except for \t, \n and \r.
// If escapeHTML is set, it also escapes <, >, and &
// because they can lead to security holes when
// user-controlled strings are rendered into JSON
// and served to some browsers.
*e = append(*e, `u00`...)
*e = append(*e, hex[b>>4])
*e = append(*e, hex[b&0xF])
}
i++
start = i
continue
}
c, size := utf8.DecodeRuneInString(s[i:])
// if c == utf8.RuneError && size == 1 {
// if start < i {
// e.Write(s[start:i])
// }
// e.WriteString(`\ufffd`)
// i += size
// start = i
// continue
// }
if c == '\u2028' || c == '\u2029' {
if start < i {
*e = append(*e, s[start:i]...)
}
*e = append(*e, `\u202`...)
*e = append(*e, hex[c&0xF])
i += size
start = i
continue
}
i += size
}
if start < len(s) {
*e = append(*e, s[start:]...)
}
*e = append(*e, '"')
}
var bytesPool = sync.Pool{}
func (self *Node) MarshalJSON() ([]byte, error) {
buf := newBuffer()
err := self.encode(buf)
if err != nil {
freeBuffer(buf)
return nil, err
}
ret := make([]byte, len(*buf))
copy(ret, *buf)
freeBuffer(buf)
return ret, err
}
func newBuffer() *[]byte {
if ret := bytesPool.Get(); ret != nil {
return ret.(*[]byte)
} else {
buf := make([]byte, 0, _MaxBuffer)
return &buf
}
}
func freeBuffer(buf *[]byte) {
*buf = (*buf)[:0]
bytesPool.Put(buf)
}
func (self *Node) encode(buf *[]byte) error {
if self.IsRaw() {
return self.encodeRaw(buf)
}
switch self.Type() {
case V_NONE : return ErrNotExist
case V_ERROR : return self.Check()
case V_NULL : return self.encodeNull(buf)
case V_TRUE : return self.encodeTrue(buf)
case V_FALSE : return self.encodeFalse(buf)
case V_ARRAY : return self.encodeArray(buf)
case V_OBJECT: return self.encodeObject(buf)
case V_STRING: return self.encodeString(buf)
case V_NUMBER: return self.encodeNumber(buf)
case V_ANY : return self.encodeInterface(buf)
default : return ErrUnsupportType
}
}
func (self *Node) encodeRaw(buf *[]byte) error {
raw, err := self.Raw()
if err != nil {
return err
}
*buf = append(*buf, raw...)
return nil
}
func (self *Node) encodeNull(buf *[]byte) error {
*buf = append(*buf, bytesNull...)
return nil
}
func (self *Node) encodeTrue(buf *[]byte) error {
*buf = append(*buf, bytesTrue...)
return nil
}
func (self *Node) encodeFalse(buf *[]byte) error {
*buf = append(*buf, bytesFalse...)
return nil
}
func (self *Node) encodeNumber(buf *[]byte) error {
str := rt.StrFrom(self.p, self.v)
*buf = append(*buf, str...)
return nil
}
func (self *Node) encodeString(buf *[]byte) error {
if self.v == 0 {
*buf = append(*buf, '"', '"')
return nil
}
quote(buf, rt.StrFrom(self.p, self.v))
return nil
}
func (self *Node) encodeArray(buf *[]byte) error {
if self.isLazy() {
if err := self.skipAllIndex(); err != nil {
return err
}
}
nb := self.len()
if nb == 0 {
*buf = append(*buf, bytesArray...)
return nil
}
*buf = append(*buf, '[')
var p = (*Node)(self.p)
err := p.encode(buf)
if err != nil {
return err
}
for i := 1; i < nb; i++ {
*buf = append(*buf, ',')
p = p.unsafe_next()
err := p.encode(buf)
if err != nil {
return err
}
}
*buf = append(*buf, ']')
return nil
}
func (self *Pair) encode(buf *[]byte) error {
if len(*buf) == 0 {
*buf = append(*buf, '"', '"', ':')
return self.Value.encode(buf)
}
quote(buf, self.Key)
*buf = append(*buf, ':')
return self.Value.encode(buf)
}
func (self *Node) encodeObject(buf *[]byte) error {
if self.isLazy() {
if err := self.skipAllKey(); err != nil {
return err
}
}
nb := self.len()
if nb == 0 {
*buf = append(*buf, bytesObject...)
return nil
}
*buf = append(*buf, '{')
var p = (*Pair)(self.p)
err := p.encode(buf)
if err != nil {
return err
}
for i := 1; i < nb; i++ {
*buf = append(*buf, ',')
p = p.unsafe_next()
err := p.encode(buf)
if err != nil {
return err
}
}
*buf = append(*buf, '}')
return nil
}
vendor/github.com/bytedance/sonic/ast/error.go
Generated Vendored
+98
View File
@@ -0,0 +1,98 @@
package ast
import (
`fmt`
`strings`
`unsafe`
`github.com/bytedance/sonic/internal/native/types`
)
func (self *Parser) syntaxError(err types.ParsingError) SyntaxError {
return SyntaxError{
Pos : self.p,
Src : self.s,
Code: err,
}
}
func newSyntaxError(err SyntaxError) *Node {
msg := err.Description()
return &Node{
t: V_ERROR,
v: int64(err.Code),
p: unsafe.Pointer(&msg),
}
}
type SyntaxError struct {
Pos int
Src string
Code types.ParsingError
Msg string
}
func (self SyntaxError) Error() string {
return fmt.Sprintf("%q", self.Description())
}
func (self SyntaxError) Description() string {
return "Syntax error " + self.description()
}
func (self SyntaxError) description() string {
i := 16
p := self.Pos - i
q := self.Pos + i
/* check for empty source */
if self.Src == "" {
return fmt.Sprintf("no sources available: %#v", self)
}
/* prevent slicing before the beginning */
if p < 0 {
p, q, i = 0, q - p, i + p
}
/* prevent slicing beyond the end */
if n := len(self.Src); q > n {
n = q - n
q = len(self.Src)
/* move the left bound if possible */
if p > n {
i += n
p -= n
}
}
/* left and right length */
x := clamp_zero(i)
y := clamp_zero(q - p - i - 1)
/* compose the error description */
return fmt.Sprintf(
"at index %d: %s\n\n\t%s\n\t%s^%s\n",
self.Pos,
self.Message(),
self.Src[p:q],
strings.Repeat(".", x),
strings.Repeat(".", y),
)
}
func (self SyntaxError) Message() string {
if self.Msg == "" {
return self.Code.Message()
}
return self.Msg
}
func clamp_zero(v int) int {
if v < 0 {
return 0
} else {
return v
}
}
vendor/github.com/bytedance/sonic/ast/iterator.go
Generated Vendored
+164
View File
@@ -0,0 +1,164 @@
/*
* Copyright 2021 ByteDance Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package ast
import (
`fmt`
`github.com/bytedance/sonic/internal/native/types`
)
type Pair struct {
Key string
Value Node
}
// Values returns iterator for array's children traversal
func (self *Node) Values() (ListIterator, error) {
if err := self.should(types.V_ARRAY, "an array"); err != nil {
return ListIterator{}, err
}
return ListIterator{Iterator{p: self}}, nil
}
// Properties returns iterator for object's children traversal
func (self *Node) Properties() (ObjectIterator, error) {
if err := self.should(types.V_OBJECT, "an object"); err != nil {
return ObjectIterator{}, err
}
return ObjectIterator{Iterator{p: self}}, nil
}
type Iterator struct {
i int
p *Node
}
func (self *Iterator) Pos() int {
return self.i
}
func (self *Iterator) Len() int {
return self.p.len()
}
// HasNext reports if it is the end of iteration or has error.
func (self *Iterator) HasNext() bool {
if !self.p.isLazy() {
return self.p.Valid() && self.i < self.p.len()
} else if self.p.t == _V_ARRAY_LAZY {
return self.p.skipNextNode().Valid()
} else if self.p.t == _V_OBJECT_LAZY {
pair := self.p.skipNextPair()
if pair == nil {
return false
}
return pair.Value.Valid()
}
return false
}
// ListIterator is specialized iterator for V_ARRAY
type ListIterator struct {
Iterator
}
// ObjectIterator is specialized iterator for V_ARRAY
type ObjectIterator struct {
Iterator
}
// Next scans through children of underlying V_ARRAY,
// copies each child to v, and returns .HasNext().
func (self *ListIterator) Next(v *Node) bool {
if !self.HasNext() {
return false
} else {
*v, self.i = *self.p.nodeAt(self.i), self.i + 1
return true
}
}
// Next scans through children of underlying V_OBJECT,
// copies each child to v, and returns .HasNext().
func (self *ObjectIterator) Next(p *Pair) bool {
if !self.HasNext() {
return false
} else {
*p, self.i = *self.p.pairAt(self.i), self.i + 1
return true
}
}
// Sequence represents scanning path of single-layer nodes.
// Index indicates the value's order in both V_ARRAY and V_OBJECT json.
// Key is the value's key (for V_OBJECT json only, otherwise it will be nil).
type Sequence struct {
Index int
Key *string
// Level int
}
// String is string representation of one Sequence
func (s Sequence) String() string {
k := ""
if s.Key != nil {
k = *s.Key
}
return fmt.Sprintf("Sequence(%d, %q)", s.Index, k)
}
type Scanner func(path Sequence, node *Node) bool
// ForEach scans one V_OBJECT node's children from JSON head to tail,
// and pass the Sequence and Node of corresponding JSON value.
//
// Especailly, if the node is not V_ARRAY or V_OBJECT,
// the node itself will be returned and Sequence.Index == -1.
func (self *Node) ForEach(sc Scanner) error {
switch self.itype() {
case types.V_ARRAY:
ns, err := self.UnsafeArray()
if err != nil {
return err
}
for i := range ns {
if !sc(Sequence{i, nil}, &ns[i]) {
return err
}
}
case types.V_OBJECT:
ns, err := self.UnsafeMap()
if err != nil {
return err
}
for i := range ns {
if !sc(Sequence{i, &ns[i].Key}, &ns[i].Value) {
return err
}
}
default:
sc(Sequence{-1, nil}, self)
}
return self.Check()
}
type PairSlice []Pair
func (self PairSlice) Sort() {
radixQsort(self, 0, maxDepth(len(self)))
}
vendor/github.com/bytedance/sonic/ast/node.go
Generated Vendored
+1808
View File
File diff suppressed because it is too large Load Diff
vendor/github.com/bytedance/sonic/ast/parser.go
Generated Vendored
+618
View File
@@ -0,0 +1,618 @@
/*
* Copyright 2021 ByteDance Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package ast
import (
`fmt`
`github.com/bytedance/sonic/internal/native/types`
`github.com/bytedance/sonic/internal/rt`
)
const _DEFAULT_NODE_CAP int = 16
const (
_ERR_NOT_FOUND types.ParsingError = 33
_ERR_UNSUPPORT_TYPE types.ParsingError = 34
)
var (
ErrNotExist error = newError(_ERR_NOT_FOUND, "value not exists")
ErrUnsupportType error = newError(_ERR_UNSUPPORT_TYPE, "unsupported type")
)
type Parser struct {
p int
s string
noLazy bool
skipValue bool
}
/** Parser Private Methods **/
func (self *Parser) delim() types.ParsingError {
n := len(self.s)
p := self.lspace(self.p)
/* check for EOF */
if p >= n {
return types.ERR_EOF
}
/* check for the delimtier */
if self.s[p] != ':' {
return types.ERR_INVALID_CHAR
}
/* update the read pointer */
self.p = p + 1
return 0
}
func (self *Parser) object() types.ParsingError {
n := len(self.s)
p := self.lspace(self.p)
/* check for EOF */
if p >= n {
return types.ERR_EOF
}
/* check for the delimtier */
if self.s[p] != '{' {
return types.ERR_INVALID_CHAR
}
/* update the read pointer */
self.p = p + 1
return 0
}
func (self *Parser) array() types.ParsingError {
n := len(self.s)
p := self.lspace(self.p)
/* check for EOF */
if p >= n {
return types.ERR_EOF
}
/* check for the delimtier */
if self.s[p] != '[' {
return types.ERR_INVALID_CHAR
}
/* update the read pointer */
self.p = p + 1
return 0
}
func (self *Parser) lspace(sp int) int {
ns := len(self.s)
for ; sp<ns && isSpace(self.s[sp]); sp+=1 {}
return sp
}
func (self *Parser) decodeArray(ret []Node) (Node, types.ParsingError) {
sp := self.p
ns := len(self.s)
/* check for EOF */
if self.p = self.lspace(sp); self.p >= ns {
return Node{}, types.ERR_EOF
}
/* check for empty array */
if self.s[self.p] == ']' {
self.p++
return emptyArrayNode, 0
}
/* allocate array space and parse every element */
for {
var val Node
var err types.ParsingError
if self.skipValue {
/* skip the value */
var start int
if start, err = self.skipFast(); err != 0 {
return Node{}, err
}
if self.p > ns {
return Node{}, types.ERR_EOF
}
t := switchRawType(self.s[start])
if t == _V_NONE {
return Node{}, types.ERR_INVALID_CHAR
}
val = newRawNode(self.s[start:self.p], t)
}else{
/* decode the value */
if val, err = self.Parse(); err != 0 {
return Node{}, err
}
}
/* add the value to result */
ret = append(ret, val)
self.p = self.lspace(self.p)
/* check for EOF */
if self.p >= ns {
return Node{}, types.ERR_EOF
}
/* check for the next character */
switch self.s[self.p] {
case ',' : self.p++
case ']' : self.p++; return NewArray(ret), 0
default:
if val.isLazy() {
return newLazyArray(self, ret), 0
}
return Node{}, types.ERR_INVALID_CHAR
}
}
}
func (self *Parser) decodeObject(ret []Pair) (Node, types.ParsingError) {
sp := self.p
ns := len(self.s)
/* check for EOF */
if self.p = self.lspace(sp); self.p >= ns {
return Node{}, types.ERR_EOF
}
/* check for empty object */
if self.s[self.p] == '}' {
self.p++
return emptyObjectNode, 0
}
/* decode each pair */
for {
var val Node
var njs types.JsonState
var err types.ParsingError
/* decode the key */
if njs = self.decodeValue(); njs.Vt != types.V_STRING {
return Node{}, types.ERR_INVALID_CHAR
}
/* extract the key */
idx := self.p - 1
key := self.s[njs.Iv:idx]
/* check for escape sequence */
if njs.Ep != -1 {
if key, err = unquote(key); err != 0 {
return Node{}, err
}
}
/* expect a ':' delimiter */
if err = self.delim(); err != 0 {
return Node{}, err
}
if self.skipValue {
/* skip the value */
var start int
if start, err = self.skipFast(); err != 0 {
return Node{}, err
}
if self.p > ns {
return Node{}, types.ERR_EOF
}
t := switchRawType(self.s[start])
if t == _V_NONE {
return Node{}, types.ERR_INVALID_CHAR
}
val = newRawNode(self.s[start:self.p], t)
} else {
/* decode the value */
if val, err = self.Parse(); err != 0 {
return Node{}, err
}
}
/* add the value to result */
ret = append(ret, Pair{Key: key, Value: val})
self.p = self.lspace(self.p)
/* check for EOF */
if self.p >= ns {
return Node{}, types.ERR_EOF
}
/* check for the next character */
switch self.s[self.p] {
case ',' : self.p++
case '}' : self.p++; return NewObject(ret), 0
default:
if val.isLazy() {
return newLazyObject(self, ret), 0
}
return Node{}, types.ERR_INVALID_CHAR
}
}
}
func (self *Parser) decodeString(iv int64, ep int) (Node, types.ParsingError) {
p := self.p - 1
s := self.s[iv:p]
/* fast path: no escape sequence */
if ep == -1 {
return NewString(s), 0
}
/* unquote the string */
out, err := unquote(s)
/* check for errors */
if err != 0 {
return Node{}, err
} else {
return newBytes(rt.Str2Mem(out)), 0
}
}
/** Parser Interface **/
func (self *Parser) Pos() int {
return self.p
}
func (self *Parser) Parse() (Node, types.ParsingError) {
switch val := self.decodeValue(); val.Vt {
case types.V_EOF : return Node{}, types.ERR_EOF
case types.V_NULL : return nullNode, 0
case types.V_TRUE : return trueNode, 0
case types.V_FALSE : return falseNode, 0
case types.V_STRING : return self.decodeString(val.Iv, val.Ep)
case types.V_ARRAY:
if self.noLazy {
return self.decodeArray(make([]Node, 0, _DEFAULT_NODE_CAP))
}
return newLazyArray(self, make([]Node, 0, _DEFAULT_NODE_CAP)), 0
case types.V_OBJECT:
if self.noLazy {
return self.decodeObject(make([]Pair, 0, _DEFAULT_NODE_CAP))
}
return newLazyObject(self, make([]Pair, 0, _DEFAULT_NODE_CAP)), 0
case types.V_DOUBLE : return NewNumber(self.s[val.Ep:self.p]), 0
case types.V_INTEGER : return NewNumber(self.s[val.Ep:self.p]), 0
default : return Node{}, types.ParsingError(-val.Vt)
}
}
func (self *Parser) searchKey(match string) types.ParsingError {
ns := len(self.s)
if err := self.object(); err != 0 {
return err
}
/* check for EOF */
if self.p = self.lspace(self.p); self.p >= ns {
return types.ERR_EOF
}
/* check for empty object */
if self.s[self.p] == '}' {
self.p++
return _ERR_NOT_FOUND
}
var njs types.JsonState
var err types.ParsingError
/* decode each pair */
for {
/* decode the key */
if njs = self.decodeValue(); njs.Vt != types.V_STRING {
return types.ERR_INVALID_CHAR
}
/* extract the key */
idx := self.p - 1
key := self.s[njs.Iv:idx]
/* check for escape sequence */
if njs.Ep != -1 {
if key, err = unquote(key); err != 0 {
return err
}
}
/* expect a ':' delimiter */
if err = self.delim(); err != 0 {
return err
}
/* skip value */
if key != match {
if _, err = self.skipFast(); err != 0 {
return err
}
} else {
return 0
}
/* check for EOF */
self.p = self.lspace(self.p)
if self.p >= ns {
return types.ERR_EOF
}
/* check for the next character */
switch self.s[self.p] {
case ',':
self.p++
case '}':
self.p++
return _ERR_NOT_FOUND
default:
return types.ERR_INVALID_CHAR
}
}
}
func (self *Parser) searchIndex(idx int) types.ParsingError {
ns := len(self.s)
if err := self.array(); err != 0 {
return err
}
/* check for EOF */
if self.p = self.lspace(self.p); self.p >= ns {
return types.ERR_EOF
}
/* check for empty array */
if self.s[self.p] == ']' {
self.p++
return _ERR_NOT_FOUND
}
var err types.ParsingError
/* allocate array space and parse every element */
for i := 0; i < idx; i++ {
/* decode the value */
if _, err = self.skipFast(); err != 0 {
return err
}
/* check for EOF */
self.p = self.lspace(self.p)
if self.p >= ns {
return types.ERR_EOF
}
/* check for the next character */
switch self.s[self.p] {
case ',':
self.p++
case ']':
self.p++
return _ERR_NOT_FOUND
default:
return types.ERR_INVALID_CHAR
}
}
return 0
}
func (self *Node) skipNextNode() *Node {
if !self.isLazy() {
return nil
}
parser, stack := self.getParserAndArrayStack()
ret := stack.v
sp := parser.p
ns := len(parser.s)
/* check for EOF */
if parser.p = parser.lspace(sp); parser.p >= ns {
return newSyntaxError(parser.syntaxError(types.ERR_EOF))
}
/* check for empty array */
if parser.s[parser.p] == ']' {
parser.p++
self.setArray(ret)
return nil
}
var val Node
/* skip the value */
if start, err := parser.skipFast(); err != 0 {
return newSyntaxError(parser.syntaxError(err))
} else {
t := switchRawType(parser.s[start])
if t == _V_NONE {
return newSyntaxError(parser.syntaxError(types.ERR_INVALID_CHAR))
}
val = newRawNode(parser.s[start:parser.p], t)
}
/* add the value to result */
ret = append(ret, val)
parser.p = parser.lspace(parser.p)
/* check for EOF */
if parser.p >= ns {
return newSyntaxError(parser.syntaxError(types.ERR_EOF))
}
/* check for the next character */
switch parser.s[parser.p] {
case ',':
parser.p++
self.setLazyArray(parser, ret)
return &ret[len(ret)-1]
case ']':
parser.p++
self.setArray(ret)
return &ret[len(ret)-1]
default:
return newSyntaxError(parser.syntaxError(types.ERR_INVALID_CHAR))
}
}
func (self *Node) skipNextPair() (*Pair) {
if !self.isLazy() {
return nil
}
parser, stack := self.getParserAndObjectStack()
ret := stack.v
sp := parser.p
ns := len(parser.s)
/* check for EOF */
if parser.p = parser.lspace(sp); parser.p >= ns {
return &Pair{"", *newSyntaxError(parser.syntaxError(types.ERR_EOF))}
}
/* check for empty object */
if parser.s[parser.p] == '}' {
parser.p++
self.setObject(ret)
return nil
}
/* decode one pair */
var val Node
var njs types.JsonState
var err types.ParsingError
/* decode the key */
if njs = parser.decodeValue(); njs.Vt != types.V_STRING {
return &Pair{"", *newSyntaxError(parser.syntaxError(types.ERR_INVALID_CHAR))}
}
/* extract the key */
idx := parser.p - 1
key := parser.s[njs.Iv:idx]
/* check for escape sequence */
if njs.Ep != -1 {
if key, err = unquote(key); err != 0 {
return &Pair{key, *newSyntaxError(parser.syntaxError(err))}
}
}
/* expect a ':' delimiter */
if err = parser.delim(); err != 0 {
return &Pair{key, *newSyntaxError(parser.syntaxError(err))}
}
/* skip the value */
if start, err := parser.skipFast(); err != 0 {
return &Pair{key, *newSyntaxError(parser.syntaxError(err))}
} else {
t := switchRawType(parser.s[start])
if t == _V_NONE {
return &Pair{key, *newSyntaxError(parser.syntaxError(types.ERR_INVALID_CHAR))}
}
val = newRawNode(parser.s[start:parser.p], t)
}
/* add the value to result */
ret = append(ret, Pair{Key: key, Value: val})
parser.p = parser.lspace(parser.p)
/* check for EOF */
if parser.p >= ns {
return &Pair{key, *newSyntaxError(parser.syntaxError(types.ERR_EOF))}
}
/* check for the next character */
switch parser.s[parser.p] {
case ',':
parser.p++
self.setLazyObject(parser, ret)
return &ret[len(ret)-1]
case '}':
parser.p++
self.setObject(ret)
return &ret[len(ret)-1]
default:
return &Pair{key, *newSyntaxError(parser.syntaxError(types.ERR_INVALID_CHAR))}
}
}
/** Parser Factory **/
// Loads parse all json into interface{}
func Loads(src string) (int, interface{}, error) {
ps := &Parser{s: src}
np, err := ps.Parse()
/* check for errors */
if err != 0 {
return 0, nil, ps.ExportError(err)
} else {
x, err := np.Interface()
if err != nil {
return 0, nil, err
}
return ps.Pos(), x, nil
}
}
// LoadsUseNumber parse all json into interface{}, with numeric nodes casted to json.Number
func LoadsUseNumber(src string) (int, interface{}, error) {
ps := &Parser{s: src}
np, err := ps.Parse()
/* check for errors */
if err != 0 {
return 0, nil, err
} else {
x, err := np.InterfaceUseNumber()
if err != nil {
return 0, nil, err
}
return ps.Pos(), x, nil
}
}
func NewParser(src string) *Parser {
return &Parser{s: src}
}
// ExportError converts types.ParsingError to std Error
func (self *Parser) ExportError(err types.ParsingError) error {
if err == _ERR_NOT_FOUND {
return ErrNotExist
}
return fmt.Errorf("%q", SyntaxError{
Pos : self.p,
Src : self.s,
Code: err,
}.Description())
}
vendor/github.com/bytedance/sonic/ast/search.go
Generated Vendored
+30
View File
@@ -0,0 +1,30 @@
/*
* Copyright 2021 ByteDance Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package ast
type Searcher struct {
parser Parser
}
func NewSearcher(str string) *Searcher {
return &Searcher{
parser: Parser{
s: str,
noLazy: false,
},
}
}
vendor/github.com/bytedance/sonic/ast/sort.go
Generated Vendored
+206
View File
@@ -0,0 +1,206 @@
/*
* Copyright 2021 ByteDance Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package ast
// Algorithm 3-way Radix Quicksort, d means the radix.
// Reference: https://algs4.cs.princeton.edu/51radix/Quick3string.java.html
func radixQsort(kvs PairSlice, d, maxDepth int) {
for len(kvs) > 11 {
// To avoid the worst case of quickSort (time: O(n^2)), use introsort here.
// Reference: https://en.wikipedia.org/wiki/Introsort and
// https://github.com/golang/go/issues/467
if maxDepth == 0 {
heapSort(kvs, 0, len(kvs))
return
}
maxDepth--
p := pivot(kvs, d)
lt, i, gt := 0, 0, len(kvs)
for i < gt {
c := byteAt(kvs[i].Key, d)
if c < p {
swap(kvs, lt, i)
i++
lt++
} else if c > p {
gt--
swap(kvs, i, gt)
} else {
i++
}
}
// kvs[0:lt] < v = kvs[lt:gt] < kvs[gt:len(kvs)]
// Native implemention:
// radixQsort(kvs[:lt], d, maxDepth)
// if p > -1 {
// radixQsort(kvs[lt:gt], d+1, maxDepth)
// }
// radixQsort(kvs[gt:], d, maxDepth)
// Optimize as follows: make recursive calls only for the smaller parts.
// Reference: https://www.geeksforgeeks.org/quicksort-tail-call-optimization-reducing-worst-case-space-log-n/
if p == -1 {
if lt > len(kvs) - gt {
radixQsort(kvs[gt:], d, maxDepth)
kvs = kvs[:lt]
} else {
radixQsort(kvs[:lt], d, maxDepth)
kvs = kvs[gt:]
}
} else {
ml := maxThree(lt, gt-lt, len(kvs)-gt)
if ml == lt {
radixQsort(kvs[lt:gt], d+1, maxDepth)
radixQsort(kvs[gt:], d, maxDepth)
kvs = kvs[:lt]
} else if ml == gt-lt {
radixQsort(kvs[:lt], d, maxDepth)
radixQsort(kvs[gt:], d, maxDepth)
kvs = kvs[lt:gt]
d += 1
} else {
radixQsort(kvs[:lt], d, maxDepth)
radixQsort(kvs[lt:gt], d+1, maxDepth)
kvs = kvs[gt:]
}
}
}
insertRadixSort(kvs, d)
}
func insertRadixSort(kvs PairSlice, d int) {
for i := 1; i < len(kvs); i++ {
for j := i; j > 0 && lessFrom(kvs[j].Key, kvs[j-1].Key, d); j-- {
swap(kvs, j, j-1)
}
}
}
func pivot(kvs PairSlice, d int) int {
m := len(kvs) >> 1
if len(kvs) > 40 {
// Tukey's ``Ninther,'' median of three mediankvs of three.
t := len(kvs) / 8
return medianThree(
medianThree(byteAt(kvs[0].Key, d), byteAt(kvs[t].Key, d), byteAt(kvs[2*t].Key, d)),
medianThree(byteAt(kvs[m].Key, d), byteAt(kvs[m-t].Key, d), byteAt(kvs[m+t].Key, d)),
medianThree(byteAt(kvs[len(kvs)-1].Key, d),
byteAt(kvs[len(kvs)-1-t].Key, d),
byteAt(kvs[len(kvs)-1-2*t].Key, d)))
}
return medianThree(byteAt(kvs[0].Key, d), byteAt(kvs[m].Key, d), byteAt(kvs[len(kvs)-1].Key, d))
}
func medianThree(i, j, k int) int {
if i > j {
i, j = j, i
} // i < j
if k < i {
return i
}
if k > j {
return j
}
return k
}
func maxThree(i, j, k int) int {
max := i
if max < j {
max = j
}
if max < k {
max = k
}
return max
}
// maxDepth returns a threshold at which quicksort should switch
// to heapsort. It returnkvs 2*ceil(lg(n+1)).
func maxDepth(n int) int {
var depth int
for i := n; i > 0; i >>= 1 {
depth++
}
return depth * 2
}
// siftDown implements the heap property on kvs[lo:hi].
// first is an offset into the array where the root of the heap lies.
func siftDown(kvs PairSlice, lo, hi, first int) {
root := lo
for {
child := 2*root + 1
if child >= hi {
break
}
if child+1 < hi && kvs[first+child].Key < kvs[first+child+1].Key {
child++
}
if kvs[first+root].Key >= kvs[first+child].Key {
return
}
swap(kvs, first+root, first+child)
root = child
}
}
func heapSort(kvs PairSlice, a, b int) {
first := a
lo := 0
hi := b - a
// Build heap with the greatest element at top.
for i := (hi - 1) / 2; i >= 0; i-- {
siftDown(kvs, i, hi, first)
}
// Pop elements, the largest first, into end of kvs.
for i := hi - 1; i >= 0; i-- {
swap(kvs, first, first+i)
siftDown(kvs, lo, i, first)
}
}
// Note that Pair.Key is NOT pointed to Pair.m when map key is integer after swap
func swap(kvs PairSlice, a, b int) {
kvs[a].Key, kvs[b].Key = kvs[b].Key, kvs[a].Key
kvs[a].Value, kvs[b].Value = kvs[b].Value, kvs[a].Value
}
// Compare two strings from the pos d.
func lessFrom(a, b string, d int) bool {
l := len(a)
if l > len(b) {
l = len(b)
}
for i := d; i < l; i++ {
if a[i] == b[i] {
continue
}
return a[i] < b[i]
}
return len(a) < len(b)
}
func byteAt(b string, p int) int {
if p < len(b) {
return int(b[p])
}
return -1
}
vendor/github.com/bytedance/sonic/ast/stubs_go115.go
Generated Vendored
+55
View File
@@ -0,0 +1,55 @@
// +build !go1.20
/*
* Copyright 2021 ByteDance Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package ast
import (
`unsafe`
`unicode/utf8`
`github.com/bytedance/sonic/internal/rt`
)
//go:noescape
//go:linkname memmove runtime.memmove
//goland:noinspection GoUnusedParameter
func memmove(to unsafe.Pointer, from unsafe.Pointer, n uintptr)
//go:linkname unsafe_NewArray reflect.unsafe_NewArray
//goland:noinspection GoUnusedParameter
func unsafe_NewArray(typ *rt.GoType, n int) unsafe.Pointer
//go:linkname growslice runtime.growslice
//goland:noinspection GoUnusedParameter
func growslice(et *rt.GoType, old rt.GoSlice, cap int) rt.GoSlice
//go:nosplit
func mem2ptr(s []byte) unsafe.Pointer {
return (*rt.GoSlice)(unsafe.Pointer(&s)).Ptr
}
var (
//go:linkname safeSet encoding/json.safeSet
safeSet [utf8.RuneSelf]bool
//go:linkname hex encoding/json.hex
hex string
)
//go:linkname unquoteBytes encoding/json.unquoteBytes
func unquoteBytes(s []byte) (t []byte, ok bool)
vendor/github.com/bytedance/sonic/ast/stubs_go120.go
Generated Vendored
+55
View File
@@ -0,0 +1,55 @@
// +build go1.20
/*
* Copyright 2021 ByteDance Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package ast
import (
`unsafe`
`unicode/utf8`
`github.com/bytedance/sonic/internal/rt`
)
//go:noescape
//go:linkname memmove runtime.memmove
//goland:noinspection GoUnusedParameter
func memmove(to unsafe.Pointer, from unsafe.Pointer, n uintptr)
//go:linkname unsafe_NewArray reflect.unsafe_NewArray
//goland:noinspection GoUnusedParameter
func unsafe_NewArray(typ *rt.GoType, n int) unsafe.Pointer
//go:linkname growslice reflect.growslice
//goland:noinspection GoUnusedParameter
func growslice(et *rt.GoType, old rt.GoSlice, cap int) rt.GoSlice
//go:nosplit
func mem2ptr(s []byte) unsafe.Pointer {
return (*rt.GoSlice)(unsafe.Pointer(&s)).Ptr
}
var (
//go:linkname safeSet encoding/json.safeSet
safeSet [utf8.RuneSelf]bool
//go:linkname hex encoding/json.hex
hex string
)
//go:linkname unquoteBytes encoding/json.unquoteBytes
func unquoteBytes(s []byte) (t []byte, ok bool)
vendor/github.com/bytedance/sonic/bench-arm.sh
Generated Vendored
+14
View File
@@ -0,0 +1,14 @@
#!/usr/bin/env bash
pwd=$(pwd)
export SONIC_NO_ASYNC_GC=1
cd $pwd/ast
go test -benchmem -run=^$ -benchtime=1000000x -bench "^(BenchmarkGet.*|BenchmarkSet.*)$"
go test -benchmem -run=^$ -benchtime=10000x -bench "^(BenchmarkParser_.*|BenchmarkEncode.*)$"
go test -benchmem -run=^$ -benchtime=10000000x -bench "^(BenchmarkNodeGetByPath|BenchmarkStructGetByPath|BenchmarkNodeIndex|BenchmarkStructIndex|BenchmarkSliceIndex|BenchmarkMapIndex|BenchmarkNodeGet|BenchmarkSliceGet|BenchmarkMapGet|BenchmarkNodeSet|BenchmarkMapSet|BenchmarkNodeSetByIndex|BenchmarkSliceSetByIndex|BenchmarkStructSetByIndex|BenchmarkNodeUnset|BenchmarkMapUnset|BenchmarkNodUnsetByIndex|BenchmarkSliceUnsetByIndex|BenchmarkNodeAdd|BenchmarkSliceAdd|BenchmarkMapAdd)$"
unset SONIC_NO_ASYNC_GC
cd $pwd
vendor/github.com/bytedance/sonic/bench.py
Generated Vendored
+134
View File
@@ -0,0 +1,134 @@
#!/usr/bin/env python3
# Copyright 2022 ByteDance Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import tempfile
import os
import subprocess
import argparse
gbench_prefix = "SONIC_NO_ASYNC_GC=1 go test -benchmem -run=none "
def run(cmd):
print(cmd)
if os.system(cmd):
print ("Failed to run cmd: %s"%(cmd))
exit(1)
def run_s(cmd):
print (cmd)
try:
res = os.popen(cmd)
except subprocess.CalledProcessError as e:
if e.returncode:
print (e.output)
exit(1)
return res.read()
def run_r(cmd):
print (cmd)
try:
cmds = cmd.split(' ')
data = subprocess.check_output(cmds, stderr=subprocess.STDOUT)
except subprocess.CalledProcessError as e:
if e.returncode:
print (e.output)
exit(1)
return data.decode("utf-8")
def compare(args):
# detech current branch.
# result = run_r("git branch")
current_branch = run_s("git status | head -n1 | sed 's/On branch //'")
# for br in result.split('\n'):
# if br.startswith("* "):
# current_branch = br.lstrip('* ')
# break
if not current_branch:
print ("Failed to detech current branch")
return None
# get the current diff
(fd, diff) = tempfile.mkstemp()
run("git diff > %s"%diff)
# early return if currrent is main branch.
print ("Current branch: %s"%(current_branch))
if current_branch == "main":
print ("Cannot compare at the main branch.Please build a new branch")
return None
# benchmark current branch
(fd, target) = tempfile.mkstemp(".target.txt")
run("%s %s ./... 2>&1 | tee %s" %(gbench_prefix, args, target))
# trying to switch to the latest main branch
run("git checkout -- .")
if current_branch != "main":
run("git checkout main")
run("git pull --allow-unrelated-histories origin main")
# benchmark main branch
(fd, main) = tempfile.mkstemp(".main.txt")
run("%s %s ./... 2>&1 | tee %s" %(gbench_prefix, args, main))
# diff the result
# benchstat = "go get golang.org/x/perf/cmd/benchstat && go install golang.org/x/perf/cmd/benchstat"
run( "benchstat -sort=delta %s %s"%(main, target))
run("git checkout -- .")
# restore branch
if current_branch != "main":
run("git checkout %s"%(current_branch))
run("patch -p1 < %s" % (diff))
return target
def main():
argparser = argparse.ArgumentParser(description='Tools to test the performance. Example: ./bench.py -b Decoder_Generic_Sonic -c')
argparser.add_argument('-b', '--bench', dest='filter', required=False,
help='Specify the filter for golang benchmark')
argparser.add_argument('-c', '--compare', dest='compare', action='store_true', required=False,
help='Compare with the main benchmarking')
argparser.add_argument('-t', '--times', dest='times', required=False,
help='benchmark the times')
argparser.add_argument('-r', '--repeat_times', dest='count', required=False,
help='benchmark the count')
args = argparser.parse_args()
if args.filter:
gbench_args = "-bench=%s"%(args.filter)
else:
gbench_args = "-bench=."
if args.times:
gbench_args += " -benchtime=%s"%(args.times)
if args.count:
gbench_args += " -count=%s"%(args.count)
else:
gbench_args += " -count=10"
if args.compare:
target = compare(gbench_args)
else:
target = None
if not target:
(fd, target) = tempfile.mkstemp(".target.txt")
run("%s %s ./... 2>&1 | tee %s" %(gbench_prefix, gbench_args, target))
if __name__ == "__main__":
main()
vendor/github.com/bytedance/sonic/bench.sh
Generated Vendored
+27
View File
@@ -0,0 +1,27 @@
#!/usr/bin/env bash
pwd=$(pwd)
export SONIC_NO_ASYNC_GC=1
cd $pwd/encoder
go test -benchmem -run=^$ -benchtime=100000x -bench "^(BenchmarkEncoder_.*)$"
cd $pwd/decoder
go test -benchmem -run=^$ -benchtime=100000x -bench "^(BenchmarkDecoder_.*)$"
cd $pwd/ast
go test -benchmem -run=^$ -benchtime=1000000x -bench "^(BenchmarkGet.*|BenchmarkSet.*)$"
go test -benchmem -run=^$ -benchtime=10000x -bench "^(BenchmarkParser_.*|BenchmarkEncode.*)$"
go test -benchmem -run=^$ -benchtime=10000000x -bench "^(BenchmarkNodeGetByPath|BenchmarkStructGetByPath|BenchmarkNodeIndex|BenchmarkStructIndex|BenchmarkSliceIndex|BenchmarkMapIndex|BenchmarkNodeGet|BenchmarkSliceGet|BenchmarkMapGet|BenchmarkNodeSet|BenchmarkMapSet|BenchmarkNodeSetByIndex|BenchmarkSliceSetByIndex|BenchmarkStructSetByIndex|BenchmarkNodeUnset|BenchmarkMapUnset|BenchmarkNodUnsetByIndex|BenchmarkSliceUnsetByIndex|BenchmarkNodeAdd|BenchmarkSliceAdd|BenchmarkMapAdd)$"
cd $pwd/external_jsonlib_test/benchmark_test
go test -benchmem -run=^$ -benchtime=100000x -bench "^(BenchmarkEncoder_.*|BenchmarkDecoder_.*)$"
go test -benchmem -run=^$ -benchtime=1000000x -bench "^(BenchmarkGet.*|BenchmarkSet.*)$"
go test -benchmem -run=^$ -benchtime=10000x -bench "^(BenchmarkParser_.*)$"
unset SONIC_NO_ASYNC_GC
cd $pwd
vendor/github.com/bytedance/sonic/check_branch_name.sh
Generated Vendored
+10
View File
@@ -0,0 +1,10 @@
#!/usr/bin/env bash
current=$(git status | head -n1 | sed 's/On branch //')
name=${1:-$current}
if [[ ! $name =~ ^(((opt(imize)?|feat(ure)?|doc|(bug|hot)?fix|test|refact(or)?|ci)/.+)|(main|develop)|(release/.+)|(release-v[0-9]+\.[0-9]+)|(release/v[0-9]+\.[0-9]+\.[0-9]+(-[a-z0-9.]+(\+[a-z0-9.]+)?)?)|revert-[a-z0-9]+)$ ]]; then
echo "branch name '$name' is invalid"
exit 1
else
echo "branch name '$name' is valid"
fi
vendor/github.com/bytedance/sonic/compat.go
Generated Vendored
+131
View File
@@ -0,0 +1,131 @@
// +build !amd64 go1.21
/*
* Copyright 2021 ByteDance Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package sonic
import (
`bytes`
`encoding/json`
`io`
`reflect`
`github.com/bytedance/sonic/option`
)
type frozenConfig struct {
Config
}
// Froze convert the Config to API
func (cfg Config) Froze() API {
api := &frozenConfig{Config: cfg}
return api
}
func (cfg frozenConfig) marshalOptions(val interface{}, prefix, indent string) ([]byte, error) {
w := bytes.NewBuffer([]byte{})
enc := json.NewEncoder(w)
enc.SetEscapeHTML(cfg.EscapeHTML)
enc.SetIndent(prefix, indent)
err := enc.Encode(val)
out := w.Bytes()
// json.Encoder always appends '\n' after encoding,
// which is not same with json.Marshal()
if len(out) > 0 && out[len(out)-1] == '\n' {
out = out[:len(out)-1]
}
return out, err
}
// Marshal is implemented by sonic
func (cfg frozenConfig) Marshal(val interface{}) ([]byte, error) {
if !cfg.EscapeHTML {
return cfg.marshalOptions(val, "", "")
}
return json.Marshal(val)
}
// MarshalToString is implemented by sonic
func (cfg frozenConfig) MarshalToString(val interface{}) (string, error) {
out, err := cfg.Marshal(val)
return string(out), err
}
// MarshalIndent is implemented by sonic
func (cfg frozenConfig) MarshalIndent(val interface{}, prefix, indent string) ([]byte, error) {
if !cfg.EscapeHTML {
return cfg.marshalOptions(val, prefix, indent)
}
return json.MarshalIndent(val, prefix, indent)
}
// UnmarshalFromString is implemented by sonic
func (cfg frozenConfig) UnmarshalFromString(buf string, val interface{}) error {
r := bytes.NewBufferString(buf)
dec := json.NewDecoder(r)
if cfg.UseNumber {
dec.UseNumber()
}
if cfg.DisallowUnknownFields {
dec.DisallowUnknownFields()
}
return dec.Decode(val)
}
// Unmarshal is implemented by sonic
func (cfg frozenConfig) Unmarshal(buf []byte, val interface{}) error {
return cfg.UnmarshalFromString(string(buf), val)
}
// NewEncoder is implemented by sonic
func (cfg frozenConfig) NewEncoder(writer io.Writer) Encoder {
enc := json.NewEncoder(writer)
if !cfg.EscapeHTML {
enc.SetEscapeHTML(cfg.EscapeHTML)
}
return enc
}
// NewDecoder is implemented by sonic
func (cfg frozenConfig) NewDecoder(reader io.Reader) Decoder {
dec := json.NewDecoder(reader)
if cfg.UseNumber {
dec.UseNumber()
}
if cfg.DisallowUnknownFields {
dec.DisallowUnknownFields()
}
return dec
}
// Valid is implemented by sonic
func (cfg frozenConfig) Valid(data []byte) bool {
return json.Valid(data)
}
// Pretouch compiles vt ahead-of-time to avoid JIT compilation on-the-fly, in
// order to reduce the first-hit latency at **amd64** Arch.
// Opts are the compile options, for example, "option.WithCompileRecursiveDepth" is
// a compile option to set the depth of recursive compile for the nested struct type.
// * This is the none implement for !amd64.
// It will be useful for someone who develop with !amd64 arch,like Mac M1.
func Pretouch(vt reflect.Type, opts ...option.CompileOption) error {
return nil
}
vendor/github.com/bytedance/sonic/decoder/decoder_amd64.go
Generated Vendored
+66
View File
@@ -0,0 +1,66 @@
// +build amd64,go1.15,!go1.21
/*
* Copyright 2023 ByteDance Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package decoder
import (
`github.com/bytedance/sonic/internal/decoder`
)
// Decoder is the decoder context object
type Decoder = decoder.Decoder
type MismatchTypeError = decoder.MismatchTypeError
// Options for decode.
type Options = decoder.Options
const (
OptionUseInt64 Options = decoder.OptionUseInt64
OptionUseNumber Options = decoder.OptionUseNumber
OptionUseUnicodeErrors Options = decoder.OptionUseUnicodeErrors
OptionDisableUnknown Options = decoder.OptionDisableUnknown
OptionCopyString Options = decoder.OptionCopyString
OptionValidateString Options = decoder.OptionValidateString
)
// StreamDecoder is the decoder context object for streaming input.
type StreamDecoder = decoder.StreamDecoder
type SyntaxError = decoder.SyntaxError
var (
// NewDecoder creates a new decoder instance.
NewDecoder = decoder.NewDecoder
// NewStreamDecoder adapts to encoding/json.NewDecoder API.
//
// NewStreamDecoder returns a new decoder that reads from r.
NewStreamDecoder = decoder.NewStreamDecoder
// Pretouch compiles vt ahead-of-time to avoid JIT compilation on-the-fly, in
// order to reduce the first-hit latency.
//
// Opts are the compile options, for example, "option.WithCompileRecursiveDepth" is
// a compile option to set the depth of recursive compile for the nested struct type.
Pretouch = decoder.Pretouch
// Skip skips only one json value, and returns first non-blank character position and its ending position if it is valid.
// Otherwise, returns negative error code using start and invalid character position using end
Skip = decoder.Skip
)
vendor/github.com/bytedance/sonic/decoder/decoder_compat.go
Generated Vendored
+196
View File
@@ -0,0 +1,196 @@
// +build !amd64 go1.21
/*
* Copyright 2023 ByteDance Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package decoder
import (
`encoding/json`
`bytes`
`reflect`
`github.com/bytedance/sonic/internal/native/types`
`github.com/bytedance/sonic/option`
`io`
)
const (
_F_use_int64 = iota
_F_use_number
_F_disable_urc
_F_disable_unknown
_F_copy_string
_F_validate_string
_F_allow_control = 31
)
type Options uint64
const (
OptionUseInt64 Options = 1 << _F_use_int64
OptionUseNumber Options = 1 << _F_use_number
OptionUseUnicodeErrors Options = 1 << _F_disable_urc
OptionDisableUnknown Options = 1 << _F_disable_unknown
OptionCopyString Options = 1 << _F_copy_string
OptionValidateString Options = 1 << _F_validate_string
)
func (self *Decoder) SetOptions(opts Options) {
if (opts & OptionUseNumber != 0) && (opts & OptionUseInt64 != 0) {
panic("can't set OptionUseInt64 and OptionUseNumber both!")
}
self.f = uint64(opts)
}
// Decoder is the decoder context object
type Decoder struct {
i int
f uint64
s string
}
// NewDecoder creates a new decoder instance.
func NewDecoder(s string) *Decoder {
return &Decoder{s: s}
}
// Pos returns the current decoding position.
func (self *Decoder) Pos() int {
return self.i
}
func (self *Decoder) Reset(s string) {
self.s = s
self.i = 0
// self.f = 0
}
// NOTE: api fallback do nothing
func (self *Decoder) CheckTrailings() error {
pos := self.i
buf := self.s
/* skip all the trailing spaces */
if pos != len(buf) {
for pos < len(buf) && (types.SPACE_MASK & (1 << buf[pos])) != 0 {
pos++
}
}
/* then it must be at EOF */
if pos == len(buf) {
return nil
}
/* junk after JSON value */
return nil
}
// Decode parses the JSON-encoded data from current position and stores the result
// in the value pointed to by val.
func (self *Decoder) Decode(val interface{}) error {
r := bytes.NewBufferString(self.s)
dec := json.NewDecoder(r)
if (self.f | uint64(OptionUseNumber)) != 0 {
dec.UseNumber()
}
if (self.f | uint64(OptionDisableUnknown)) != 0 {
dec.DisallowUnknownFields()
}
return dec.Decode(val)
}
// UseInt64 indicates the Decoder to unmarshal an integer into an interface{} as an
// int64 instead of as a float64.
func (self *Decoder) UseInt64() {
self.f |= 1 << _F_use_int64
self.f &^= 1 << _F_use_number
}
// UseNumber indicates the Decoder to unmarshal a number into an interface{} as a
// json.Number instead of as a float64.
func (self *Decoder) UseNumber() {
self.f &^= 1 << _F_use_int64
self.f |= 1 << _F_use_number
}
// UseUnicodeErrors indicates the Decoder to return an error when encounter invalid
// UTF-8 escape sequences.
func (self *Decoder) UseUnicodeErrors() {
self.f |= 1 << _F_disable_urc
}
// DisallowUnknownFields indicates the Decoder to return an error when the destination
// is a struct and the input contains object keys which do not match any
// non-ignored, exported fields in the destination.
func (self *Decoder) DisallowUnknownFields() {
self.f |= 1 << _F_disable_unknown
}
// CopyString indicates the Decoder to decode string values by copying instead of referring.
func (self *Decoder) CopyString() {
self.f |= 1 << _F_copy_string
}
// ValidateString causes the Decoder to validate string values when decoding string value
// in JSON. Validation is that, returning error when unescaped control chars(0x00-0x1f) or
// invalid UTF-8 chars in the string value of JSON.
func (self *Decoder) ValidateString() {
self.f |= 1 << _F_validate_string
}
// Pretouch compiles vt ahead-of-time to avoid JIT compilation on-the-fly, in
// order to reduce the first-hit latency.
//
// Opts are the compile options, for example, "option.WithCompileRecursiveDepth" is
// a compile option to set the depth of recursive compile for the nested struct type.
func Pretouch(vt reflect.Type, opts ...option.CompileOption) error {
return nil
}
type StreamDecoder struct {
r io.Reader
buf []byte
scanp int
scanned int64
err error
Decoder
}
// NewStreamDecoder adapts to encoding/json.NewDecoder API.
//
// NewStreamDecoder returns a new decoder that reads from r.
func NewStreamDecoder(r io.Reader) *StreamDecoder {
return &StreamDecoder{r : r}
}
// Decode decodes input stream into val with corresponding data.
// Redundantly bytes may be read and left in its buffer, and can be used at next call.
// Either io error from underlying io.Reader (except io.EOF)
// or syntax error from data will be recorded and stop subsequently decoding.
func (self *StreamDecoder) Decode(val interface{}) (err error) {
dec := json.NewDecoder(self.r)
if (self.f | uint64(OptionUseNumber)) != 0 {
dec.UseNumber()
}
if (self.f | uint64(OptionDisableUnknown)) != 0 {
dec.DisallowUnknownFields()
}
return dec.Decode(val)
}
vendor/github.com/bytedance/sonic/encoder/encoder_amd64.go
Generated Vendored
+108
View File
@@ -0,0 +1,108 @@
// +build amd64,go1.15,!go1.21
/*
* Copyright 2023 ByteDance Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package encoder
import (
`github.com/bytedance/sonic/internal/encoder`
)
// Encoder represents a specific set of encoder configurations.
type Encoder = encoder.Encoder
// StreamEncoder uses io.Writer as input.
type StreamEncoder = encoder.StreamEncoder
// Options is a set of encoding options.
type Options = encoder.Options
const (
// SortMapKeys indicates that the keys of a map needs to be sorted
// before serializing into JSON.
// WARNING: This hurts performance A LOT, USE WITH CARE.
SortMapKeys Options = encoder.SortMapKeys
// EscapeHTML indicates encoder to escape all HTML characters
// after serializing into JSON (see https://pkg.go.dev/encoding/json#HTMLEscape).
// WARNING: This hurts performance A LOT, USE WITH CARE.
EscapeHTML Options = encoder.EscapeHTML
// CompactMarshaler indicates that the output JSON from json.Marshaler
// is always compact and needs no validation
CompactMarshaler Options = encoder.CompactMarshaler
// NoQuoteTextMarshaler indicates that the output text from encoding.TextMarshaler
// is always escaped string and needs no quoting
NoQuoteTextMarshaler Options = encoder.NoQuoteTextMarshaler
// NoNullSliceOrMap indicates all empty Array or Object are encoded as '[]' or '{}',
// instead of 'null'
NoNullSliceOrMap Options = encoder.NoNullSliceOrMap
// ValidateString indicates that encoder should validate the input string
// before encoding it into JSON.
ValidateString Options = encoder.ValidateString
// CompatibleWithStd is used to be compatible with std encoder.
CompatibleWithStd Options = encoder.CompatibleWithStd
)
var (
// Encode returns the JSON encoding of val, encoded with opts.
Encode = encoder.Encode
// EncodeInto is like Encode but uses a user-supplied buffer instead of allocating a new one.
EncodeIndented = encoder.EncodeIndented
// EncodeIndented is like Encode but applies Indent to format the output.
// Each JSON element in the output will begin on a new line beginning with prefix
// followed by one or more copies of indent according to the indentation nesting.
EncodeInto = encoder.EncodeInto
// HTMLEscape appends to dst the JSON-encoded src with <, >, &, U+2028 and U+2029
// characters inside string literals changed to \u003c, \u003e, \u0026, \u2028, \u2029
// so that the JSON will be safe to embed inside HTML <script> tags.
// For historical reasons, web browsers don't honor standard HTML
// escaping within <script> tags, so an alternative JSON encoding must
// be used.
HTMLEscape = encoder.HTMLEscape
// Pretouch compiles vt ahead-of-time to avoid JIT compilation on-the-fly, in
// order to reduce the first-hit latency.
//
// Opts are the compile options, for example, "option.WithCompileRecursiveDepth" is
// a compile option to set the depth of recursive compile for the nested struct type.
Pretouch = encoder.Pretouch
// Quote returns the JSON-quoted version of s.
Quote = encoder.Quote
// Valid validates json and returns first non-blank character position,
// if it is only one valid json value.
// Otherwise returns invalid character position using start.
//
// Note: it does not check for the invalid UTF-8 characters.
Valid = encoder.Valid
// NewStreamEncoder adapts to encoding/json.NewDecoder API.
//
// NewStreamEncoder returns a new encoder that write to w.
NewStreamEncoder = encoder.NewStreamEncoder
)
vendor/github.com/bytedance/sonic/encoder/encoder_compat.go
Generated Vendored
+234
View File
@@ -0,0 +1,234 @@
// +build !amd64 go1.21
/*
* Copyright 2023 ByteDance Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package encoder
import (
`io`
`bytes`
`encoding/json`
`reflect`
`github.com/bytedance/sonic/option`
)
// Options is a set of encoding options.
type Options uint64
const (
bitSortMapKeys = iota
bitEscapeHTML
bitCompactMarshaler
bitNoQuoteTextMarshaler
bitNoNullSliceOrMap
bitValidateString
// used for recursive compile
bitPointerValue = 63
)
const (
// SortMapKeys indicates that the keys of a map needs to be sorted
// before serializing into JSON.
// WARNING: This hurts performance A LOT, USE WITH CARE.
SortMapKeys Options = 1 << bitSortMapKeys
// EscapeHTML indicates encoder to escape all HTML characters
// after serializing into JSON (see https://pkg.go.dev/encoding/json#HTMLEscape).
// WARNING: This hurts performance A LOT, USE WITH CARE.
EscapeHTML Options = 1 << bitEscapeHTML
// CompactMarshaler indicates that the output JSON from json.Marshaler
// is always compact and needs no validation
CompactMarshaler Options = 1 << bitCompactMarshaler
// NoQuoteTextMarshaler indicates that the output text from encoding.TextMarshaler
// is always escaped string and needs no quoting
NoQuoteTextMarshaler Options = 1 << bitNoQuoteTextMarshaler
// NoNullSliceOrMap indicates all empty Array or Object are encoded as '[]' or '{}',
// instead of 'null'
NoNullSliceOrMap Options = 1 << bitNoNullSliceOrMap
// ValidateString indicates that encoder should validate the input string
// before encoding it into JSON.
ValidateString Options = 1 << bitValidateString
// CompatibleWithStd is used to be compatible with std encoder.
CompatibleWithStd Options = SortMapKeys | EscapeHTML | CompactMarshaler
)
// Encoder represents a specific set of encoder configurations.
type Encoder struct {
Opts Options
prefix string
indent string
}
// Encode returns the JSON encoding of v.
func (self *Encoder) Encode(v interface{}) ([]byte, error) {
if self.indent != "" || self.prefix != "" {
return EncodeIndented(v, self.prefix, self.indent, self.Opts)
}
return Encode(v, self.Opts)
}
// SortKeys enables the SortMapKeys option.
func (self *Encoder) SortKeys() *Encoder {
self.Opts |= SortMapKeys
return self
}
// SetEscapeHTML specifies if option EscapeHTML opens
func (self *Encoder) SetEscapeHTML(f bool) {
if f {
self.Opts |= EscapeHTML
} else {
self.Opts &= ^EscapeHTML
}
}
// SetValidateString specifies if option ValidateString opens
func (self *Encoder) SetValidateString(f bool) {
if f {
self.Opts |= ValidateString
} else {
self.Opts &= ^ValidateString
}
}
// SetCompactMarshaler specifies if option CompactMarshaler opens
func (self *Encoder) SetCompactMarshaler(f bool) {
if f {
self.Opts |= CompactMarshaler
} else {
self.Opts &= ^CompactMarshaler
}
}
// SetNoQuoteTextMarshaler specifies if option NoQuoteTextMarshaler opens
func (self *Encoder) SetNoQuoteTextMarshaler(f bool) {
if f {
self.Opts |= NoQuoteTextMarshaler
} else {
self.Opts &= ^NoQuoteTextMarshaler
}
}
// SetIndent instructs the encoder to format each subsequent encoded
// value as if indented by the package-level function EncodeIndent().
// Calling SetIndent("", "") disables indentation.
func (enc *Encoder) SetIndent(prefix, indent string) {
enc.prefix = prefix
enc.indent = indent
}
// Quote returns the JSON-quoted version of s.
func Quote(s string) string {
/* check for empty string */
if s == "" {
return `""`
}
out, _ := json.Marshal(s)
return string(out)
}
// Encode returns the JSON encoding of val, encoded with opts.
func Encode(val interface{}, opts Options) ([]byte, error) {
return json.Marshal(val)
}
// EncodeInto is like Encode but uses a user-supplied buffer instead of allocating
// a new one.
func EncodeInto(buf *[]byte, val interface{}, opts Options) error {
if buf == nil {
panic("user-supplied buffer buf is nil")
}
w := bytes.NewBuffer(*buf)
enc := json.NewEncoder(w)
enc.SetEscapeHTML((opts & EscapeHTML) != 0)
err := enc.Encode(val)
*buf = w.Bytes()
return err
}
// HTMLEscape appends to dst the JSON-encoded src with <, >, &, U+2028 and U+2029
// characters inside string literals changed to \u003c, \u003e, \u0026, \u2028, \u2029
// so that the JSON will be safe to embed inside HTML <script> tags.
// For historical reasons, web browsers don't honor standard HTML
// escaping within <script> tags, so an alternative JSON encoding must
// be used.
func HTMLEscape(dst []byte, src []byte) []byte {
d := bytes.NewBuffer(dst)
json.HTMLEscape(d, src)
return d.Bytes()
}
// EncodeIndented is like Encode but applies Indent to format the output.
// Each JSON element in the output will begin on a new line beginning with prefix
// followed by one or more copies of indent according to the indentation nesting.
func EncodeIndented(val interface{}, prefix string, indent string, opts Options) ([]byte, error) {
w := bytes.NewBuffer([]byte{})
enc := json.NewEncoder(w)
enc.SetEscapeHTML((opts & EscapeHTML) != 0)
enc.SetIndent(prefix, indent)
err := enc.Encode(val)
out := w.Bytes()
return out, err
}
// Pretouch compiles vt ahead-of-time to avoid JIT compilation on-the-fly, in
// order to reduce the first-hit latency.
//
// Opts are the compile options, for example, "option.WithCompileRecursiveDepth" is
// a compile option to set the depth of recursive compile for the nested struct type.
func Pretouch(vt reflect.Type, opts ...option.CompileOption) error {
return nil
}
// Valid validates json and returns first non-blank character position,
// if it is only one valid json value.
// Otherwise returns invalid character position using start.
//
// Note: it does not check for the invalid UTF-8 characters.
func Valid(data []byte) (ok bool, start int) {
return json.Valid(data), 0
}
// StreamEncoder uses io.Writer as
type StreamEncoder struct {
w io.Writer
Encoder
}
// NewStreamEncoder adapts to encoding/json.NewDecoder API.
//
// NewStreamEncoder returns a new encoder that write to w.
func NewStreamEncoder(w io.Writer) *StreamEncoder {
return &StreamEncoder{w: w}
}
// Encode encodes interface{} as JSON to io.Writer
func (enc *StreamEncoder) Encode(val interface{}) (err error) {
jenc := json.NewEncoder(enc.w)
jenc.SetEscapeHTML((enc.Opts & EscapeHTML) != 0)
jenc.SetIndent(enc.prefix, enc.indent)
err = jenc.Encode(val)
return err
}
vendor/github.com/bytedance/sonic/go.work
Generated Vendored
+8
View File
@@ -0,0 +1,8 @@
go 1.18
use (
.
./generic_test
./fuzz
./external_jsonlib_test
)
vendor/github.com/bytedance/sonic/internal/caching/asm.s
Generated Vendored
View File
vendor/github.com/bytedance/sonic/internal/caching/fcache.go
Generated Vendored
+115
View File
@@ -0,0 +1,115 @@
/*
* Copyright 2021 ByteDance Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package caching
import (
`strings`
`unsafe`
`github.com/bytedance/sonic/internal/rt`
)
type FieldMap struct {
N uint64
b unsafe.Pointer
m map[string]int
}
type FieldEntry struct {
ID int
Name string
Hash uint64
}
const (
FieldMap_N = int64(unsafe.Offsetof(FieldMap{}.N))
FieldMap_b = int64(unsafe.Offsetof(FieldMap{}.b))
FieldEntrySize = int64(unsafe.Sizeof(FieldEntry{}))
)
func newBucket(n int) unsafe.Pointer {
v := make([]FieldEntry, n)
return (*rt.GoSlice)(unsafe.Pointer(&v)).Ptr
}
func CreateFieldMap(n int) *FieldMap {
return &FieldMap {
N: uint64(n * 2),
b: newBucket(n * 2), // LoadFactor = 0.5
m: make(map[string]int, n * 2),
}
}
func (self *FieldMap) At(p uint64) *FieldEntry {
off := uintptr(p) * uintptr(FieldEntrySize)
return (*FieldEntry)(unsafe.Pointer(uintptr(self.b) + off))
}
// Get searches FieldMap by name. JIT generated assembly does NOT call this
// function, rather it implements its own version directly in assembly. So
// we must ensure this function stays in sync with the JIT generated one.
func (self *FieldMap) Get(name string) int {
h := StrHash(name)
p := h % self.N
s := self.At(p)
/* find the element;
* the hash map is never full, so the loop will always terminate */
for s.Hash != 0 {
if s.Hash == h && s.Name == name {
return s.ID
} else {
p = (p + 1) % self.N
s = self.At(p)
}
}
/* not found */
return -1
}
func (self *FieldMap) Set(name string, i int) {
h := StrHash(name)
p := h % self.N
s := self.At(p)
/* searching for an empty slot;
* the hash map is never full, so the loop will always terminate */
for s.Hash != 0 {
p = (p + 1) % self.N
s = self.At(p)
}
/* set the value */
s.ID = i
s.Hash = h
s.Name = name
/* add the case-insensitive version, prefer the one with smaller field ID */
key := strings.ToLower(name)
if v, ok := self.m[key]; !ok || i < v {
self.m[key] = i
}
}
func (self *FieldMap) GetCaseInsensitive(name string) int {
if i, ok := self.m[strings.ToLower(name)]; ok {
return i
} else {
return -1
}
}
vendor/github.com/bytedance/sonic/internal/caching/hashing.go
Generated Vendored
+40
View File
@@ -0,0 +1,40 @@
/*
* Copyright 2021 ByteDance Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package caching
import (
`unsafe`
`github.com/bytedance/sonic/internal/rt`
)
var (
V_strhash = rt.UnpackEface(strhash)
S_strhash = *(*uintptr)(V_strhash.Value)
)
//go:noescape
//go:linkname strhash runtime.strhash
func strhash(_ unsafe.Pointer, _ uintptr) uintptr
func StrHash(s string) uint64 {
if v := strhash(unsafe.Pointer(&s), 0); v == 0 {
return 1
} else {
return uint64(v)
}
}
vendor/github.com/bytedance/sonic/internal/caching/pcache.go
Generated Vendored
+173
View File
@@ -0,0 +1,173 @@
/*
* Copyright 2021 ByteDance Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package caching
import (
`sync`
`sync/atomic`
`unsafe`
`github.com/bytedance/sonic/internal/rt`
)
/** Program Map **/
const (
_LoadFactor = 0.5
_InitCapacity = 4096 // must be a power of 2
)
type _ProgramMap struct {
n uint64
m uint32
b []_ProgramEntry
}
type _ProgramEntry struct {
vt *rt.GoType
fn interface{}
}
func newProgramMap() *_ProgramMap {
return &_ProgramMap {
n: 0,
m: _InitCapacity - 1,
b: make([]_ProgramEntry, _InitCapacity),
}
}
func (self *_ProgramMap) copy() *_ProgramMap {
fork := &_ProgramMap{
n: self.n,
m: self.m,
b: make([]_ProgramEntry, len(self.b)),
}
for i, f := range self.b {
fork.b[i] = f
}
return fork
}
func (self *_ProgramMap) get(vt *rt.GoType) interface{} {
i := self.m + 1
p := vt.Hash & self.m
/* linear probing */
for ; i > 0; i-- {
if b := self.b[p]; b.vt == vt {
return b.fn
} else if b.vt == nil {
break
} else {
p = (p + 1) & self.m
}
}
/* not found */
return nil
}
func (self *_ProgramMap) add(vt *rt.GoType, fn interface{}) *_ProgramMap {
p := self.copy()
f := float64(atomic.LoadUint64(&p.n) + 1) / float64(p.m + 1)
/* check for load factor */
if f > _LoadFactor {
p = p.rehash()
}
/* insert the value */
p.insert(vt, fn)
return p
}
func (self *_ProgramMap) rehash() *_ProgramMap {
c := (self.m + 1) << 1
r := &_ProgramMap{m: c - 1, b: make([]_ProgramEntry, int(c))}
/* rehash every entry */
for i := uint32(0); i <= self.m; i++ {
if b := self.b[i]; b.vt != nil {
r.insert(b.vt, b.fn)
}
}
/* rebuild successful */
return r
}
func (self *_ProgramMap) insert(vt *rt.GoType, fn interface{}) {
h := vt.Hash
p := h & self.m
/* linear probing */
for i := uint32(0); i <= self.m; i++ {
if b := &self.b[p]; b.vt != nil {
p += 1
p &= self.m
} else {
b.vt = vt
b.fn = fn
atomic.AddUint64(&self.n, 1)
return
}
}
/* should never happens */
panic("no available slots")
}
/** RCU Program Cache **/
type ProgramCache struct {
m sync.Mutex
p unsafe.Pointer
}
func CreateProgramCache() *ProgramCache {
return &ProgramCache {
m: sync.Mutex{},
p: unsafe.Pointer(newProgramMap()),
}
}
func (self *ProgramCache) Get(vt *rt.GoType) interface{} {
return (*_ProgramMap)(atomic.LoadPointer(&self.p)).get(vt)
}
func (self *ProgramCache) Compute(vt *rt.GoType, compute func(*rt.GoType, ... interface{}) (interface{}, error), ex ...interface{}) (interface{}, error) {
var err error
var val interface{}
/* use defer to prevent inlining of this function */
self.m.Lock()
defer self.m.Unlock()
/* double check with write lock held */
if val = self.Get(vt); val != nil {
return val, nil
}
/* compute the value */
if val, err = compute(vt, ex...); err != nil {
return nil, err
}
/* update the RCU cache */
atomic.StorePointer(&self.p, unsafe.Pointer((*_ProgramMap)(atomic.LoadPointer(&self.p)).add(vt, val)))
return val, nil
}
vendor/github.com/bytedance/sonic/internal/cpu/features.go
Generated Vendored
+40
View File
@@ -0,0 +1,40 @@
/*
* Copyright 2021 ByteDance Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package cpu
import (
`fmt`
`os`
`github.com/klauspost/cpuid/v2`
)
var (
HasAVX = cpuid.CPU.Has(cpuid.AVX)
HasAVX2 = cpuid.CPU.Has(cpuid.AVX2)
HasSSE = cpuid.CPU.Has(cpuid.SSE)
)
func init() {
switch v := os.Getenv("SONIC_MODE"); v {
case "" : break
case "auto" : break
case "noavx" : HasAVX = false; fallthrough
case "noavx2" : HasAVX2 = false
default : panic(fmt.Sprintf("invalid mode: '%s', should be one of 'auto', 'noavx', 'noavx2'", v))
}
}
vendor/github.com/bytedance/sonic/internal/decoder/asm.s
Generated Vendored
View File
vendor/github.com/bytedance/sonic/internal/decoder/assembler_amd64_go116.go
Generated Vendored
+2011
View File
File diff suppressed because it is too large Load Diff
vendor/github.com/bytedance/sonic/internal/decoder/assembler_amd64_go117.go
Generated Vendored
+1990
View File
File diff suppressed because it is too large Load Diff
vendor/github.com/bytedance/sonic/internal/decoder/compiler.go
Generated Vendored
+1155
View File
File diff suppressed because it is too large Load Diff
vendor/github.com/bytedance/sonic/internal/decoder/debug.go
Generated Vendored
+70
View File
@@ -0,0 +1,70 @@
/*
* Copyright 2021 ByteDance Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package decoder
import (
`os`
`runtime`
`runtime/debug`
`strings`
`github.com/bytedance/sonic/internal/jit`
)
var (
debugSyncGC = os.Getenv("SONIC_SYNC_GC") != ""
debugAsyncGC = os.Getenv("SONIC_NO_ASYNC_GC") == ""
)
var (
_Instr_End _Instr = newInsOp(_OP_nil_1)
_F_gc = jit.Func(runtime.GC)
_F_force_gc = jit.Func(debug.FreeOSMemory)
_F_println = jit.Func(println_wrapper)
_F_print = jit.Func(print)
)
func println_wrapper(i int, op1 int, op2 int){
println(i, " Intrs ", op1, _OpNames[op1], "next: ", op2, _OpNames[op2])
}
func print(i int){
println(i)
}
func (self *_Assembler) force_gc() {
self.call_go(_F_gc)
self.call_go(_F_force_gc)
}
func (self *_Assembler) debug_instr(i int, v *_Instr) {
if debugSyncGC {
if (i+1 == len(self.p)) {
self.print_gc(i, v, &_Instr_End)
} else {
next := &(self.p[i+1])
self.print_gc(i, v, next)
name := _OpNames[next.op()]
if strings.Contains(name, "save") {
return
}
}
self.force_gc()
}
}
vendor/github.com/bytedance/sonic/internal/decoder/decoder.go
Generated Vendored
+255
View File
@@ -0,0 +1,255 @@
/*
* Copyright 2021 ByteDance Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package decoder
import (
`unsafe`
`encoding/json`
`reflect`
`runtime`
`github.com/bytedance/sonic/internal/native`
`github.com/bytedance/sonic/internal/native/types`
`github.com/bytedance/sonic/internal/rt`
`github.com/bytedance/sonic/option`
`github.com/bytedance/sonic/utf8`
)
const (
_F_use_int64 = iota
_F_use_number
_F_disable_urc
_F_disable_unknown
_F_copy_string
_F_validate_string
_F_allow_control = 31
)
type Options uint64
const (
OptionUseInt64 Options = 1 << _F_use_int64
OptionUseNumber Options = 1 << _F_use_number
OptionUseUnicodeErrors Options = 1 << _F_disable_urc
OptionDisableUnknown Options = 1 << _F_disable_unknown
OptionCopyString Options = 1 << _F_copy_string
OptionValidateString Options = 1 << _F_validate_string
)
func (self *Decoder) SetOptions(opts Options) {
if (opts & OptionUseNumber != 0) && (opts & OptionUseInt64 != 0) {
panic("can't set OptionUseInt64 and OptionUseNumber both!")
}
self.f = uint64(opts)
}
// Decoder is the decoder context object
type Decoder struct {
i int
f uint64
s string
}
// NewDecoder creates a new decoder instance.
func NewDecoder(s string) *Decoder {
return &Decoder{s: s}
}
// Pos returns the current decoding position.
func (self *Decoder) Pos() int {
return self.i
}
func (self *Decoder) Reset(s string) {
self.s = s
self.i = 0
// self.f = 0
}
func (self *Decoder) CheckTrailings() error {
pos := self.i
buf := self.s
/* skip all the trailing spaces */
if pos != len(buf) {
for pos < len(buf) && (types.SPACE_MASK & (1 << buf[pos])) != 0 {
pos++
}
}
/* then it must be at EOF */
if pos == len(buf) {
return nil
}
/* junk after JSON value */
return SyntaxError {
Src : buf,
Pos : pos,
Code : types.ERR_INVALID_CHAR,
}
}
// Decode parses the JSON-encoded data from current position and stores the result
// in the value pointed to by val.
func (self *Decoder) Decode(val interface{}) error {
/* validate json if needed */
if (self.f & (1 << _F_validate_string)) != 0 && !utf8.ValidateString(self.s){
dbuf := utf8.CorrectWith(nil, rt.Str2Mem(self.s), "\ufffd")
self.s = rt.Mem2Str(dbuf)
}
vv := rt.UnpackEface(val)
vp := vv.Value
/* check for nil type */
if vv.Type == nil {
return &json.InvalidUnmarshalError{}
}
/* must be a non-nil pointer */
if vp == nil || vv.Type.Kind() != reflect.Ptr {
return &json.InvalidUnmarshalError{Type: vv.Type.Pack()}
}
etp := rt.PtrElem(vv.Type)
/* check the defined pointer type for issue 379 */
if vv.Type.IsNamed() {
newp := vp
etp = vv.Type
vp = unsafe.Pointer(&newp)
}
/* create a new stack, and call the decoder */
sb := newStack()
nb, err := decodeTypedPointer(self.s, self.i, etp, vp, sb, self.f)
/* return the stack back */
self.i = nb
freeStack(sb)
/* avoid GC ahead */
runtime.KeepAlive(vv)
return err
}
// UseInt64 indicates the Decoder to unmarshal an integer into an interface{} as an
// int64 instead of as a float64.
func (self *Decoder) UseInt64() {
self.f |= 1 << _F_use_int64
self.f &^= 1 << _F_use_number
}
// UseNumber indicates the Decoder to unmarshal a number into an interface{} as a
// json.Number instead of as a float64.
func (self *Decoder) UseNumber() {
self.f &^= 1 << _F_use_int64
self.f |= 1 << _F_use_number
}
// UseUnicodeErrors indicates the Decoder to return an error when encounter invalid
// UTF-8 escape sequences.
func (self *Decoder) UseUnicodeErrors() {
self.f |= 1 << _F_disable_urc
}
// DisallowUnknownFields indicates the Decoder to return an error when the destination
// is a struct and the input contains object keys which do not match any
// non-ignored, exported fields in the destination.
func (self *Decoder) DisallowUnknownFields() {
self.f |= 1 << _F_disable_unknown
}
// CopyString indicates the Decoder to decode string values by copying instead of referring.
func (self *Decoder) CopyString() {
self.f |= 1 << _F_copy_string
}
// ValidateString causes the Decoder to validate string values when decoding string value
// in JSON. Validation is that, returning error when unescaped control chars(0x00-0x1f) or
// invalid UTF-8 chars in the string value of JSON.
func (self *Decoder) ValidateString() {
self.f |= 1 << _F_validate_string
}
// Pretouch compiles vt ahead-of-time to avoid JIT compilation on-the-fly, in
// order to reduce the first-hit latency.
//
// Opts are the compile options, for example, "option.WithCompileRecursiveDepth" is
// a compile option to set the depth of recursive compile for the nested struct type.
func Pretouch(vt reflect.Type, opts ...option.CompileOption) error {
cfg := option.DefaultCompileOptions()
for _, opt := range opts {
opt(&cfg)
}
return pretouchRec(map[reflect.Type]bool{vt:true}, cfg)
}
func pretouchType(_vt reflect.Type, opts option.CompileOptions) (map[reflect.Type]bool, error) {
/* compile function */
compiler := newCompiler().apply(opts)
decoder := func(vt *rt.GoType, _ ...interface{}) (interface{}, error) {
if pp, err := compiler.compile(_vt); err != nil {
return nil, err
} else {
as := newAssembler(pp)
as.name = _vt.String()
return as.Load(), nil
}
}
/* find or compile */
vt := rt.UnpackType(_vt)
if val := programCache.Get(vt); val != nil {
return nil, nil
} else if _, err := programCache.Compute(vt, decoder); err == nil {
return compiler.rec, nil
} else {
return nil, err
}
}
func pretouchRec(vtm map[reflect.Type]bool, opts option.CompileOptions) error {
if opts.RecursiveDepth < 0 || len(vtm) == 0 {
return nil
}
next := make(map[reflect.Type]bool)
for vt := range(vtm) {
sub, err := pretouchType(vt, opts)
if err != nil {
return err
}
for svt := range(sub) {
next[svt] = true
}
}
opts.RecursiveDepth -= 1
return pretouchRec(next, opts)
}
// Skip skips only one json value, and returns first non-blank character position and its ending position if it is valid.
// Otherwise, returns negative error code using start and invalid character position using end
func Skip(data []byte) (start int, end int) {
s := rt.Mem2Str(data)
p := 0
m := types.NewStateMachine()
ret := native.SkipOne(&s, &p, m, uint64(0))
types.FreeStateMachine(m)
return ret, p
}
vendor/github.com/bytedance/sonic/internal/decoder/errors.go
Generated Vendored
+181
View File
@@ -0,0 +1,181 @@
/*
* Copyright 2021 ByteDance Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package decoder
import (
`encoding/json`
`errors`
`fmt`
`reflect`
`strconv`
`strings`
`github.com/bytedance/sonic/internal/native/types`
`github.com/bytedance/sonic/internal/rt`
)
type SyntaxError struct {
Pos int
Src string
Code types.ParsingError
Msg string
}
func (self SyntaxError) Error() string {
return fmt.Sprintf("%q", self.Description())
}
func (self SyntaxError) Description() string {
return "Syntax error " + self.description()
}
func (self SyntaxError) description() string {
i := 16
p := self.Pos - i
q := self.Pos + i
/* check for empty source */
if self.Src == "" {
return fmt.Sprintf("no sources available: %#v", self)
}
/* prevent slicing before the beginning */
if p < 0 {
p, q, i = 0, q - p, i + p
}
/* prevent slicing beyond the end */
if n := len(self.Src); q > n {
n = q - n
q = len(self.Src)
/* move the left bound if possible */
if p > n {
i += n
p -= n
}
}
/* left and right length */
x := clamp_zero(i)
y := clamp_zero(q - p - i - 1)
/* compose the error description */
return fmt.Sprintf(
"at index %d: %s\n\n\t%s\n\t%s^%s\n",
self.Pos,
self.Message(),
self.Src[p:q],
strings.Repeat(".", x),
strings.Repeat(".", y),
)
}
func (self SyntaxError) Message() string {
if self.Msg == "" {
return self.Code.Message()
}
return self.Msg
}
func clamp_zero(v int) int {
if v < 0 {
return 0
} else {
return v
}
}
/** JIT Error Helpers **/
var stackOverflow = &json.UnsupportedValueError {
Str : "Value nesting too deep",
Value : reflect.ValueOf("..."),
}
//go:nosplit
func error_wrap(src string, pos int, code types.ParsingError) error {
return SyntaxError {
Pos : pos,
Src : src,
Code : code,
}
}
//go:nosplit
func error_type(vt *rt.GoType) error {
return &json.UnmarshalTypeError{Type: vt.Pack()}
}
type MismatchTypeError struct {
Pos int
Src string
Type reflect.Type
}
func swithchJSONType (src string, pos int) string {
var val string
switch src[pos] {
case 'f': fallthrough
case 't': val = "bool"
case '"': val = "string"
case '{': val = "object"
case '[': val = "array"
case '-', '0', '1', '2', '3', '4', '5', '6', '7', '8', '9': val = "number"
}
return val
}
func (self MismatchTypeError) Error() string {
se := SyntaxError {
Pos : self.Pos,
Src : self.Src,
Code : types.ERR_MISMATCH,
}
return fmt.Sprintf("Mismatch type %s with value %s %q", self.Type.String(), swithchJSONType(self.Src, self.Pos), se.description())
}
func (self MismatchTypeError) Description() string {
se := SyntaxError {
Pos : self.Pos,
Src : self.Src,
Code : types.ERR_MISMATCH,
}
return fmt.Sprintf("Mismatch type %s with value %s %s", self.Type.String(), swithchJSONType(self.Src, self.Pos), se.description())
}
//go:nosplit
func error_mismatch(src string, pos int, vt *rt.GoType) error {
return &MismatchTypeError {
Pos : pos,
Src : src,
Type : vt.Pack(),
}
}
//go:nosplit
func error_field(name string) error {
return errors.New("json: unknown field " + strconv.Quote(name))
}
//go:nosplit
func error_value(value string, vtype reflect.Type) error {
return &json.UnmarshalTypeError {
Type : vtype,
Value : value,
}
}
vendor/github.com/bytedance/sonic/internal/decoder/generic_amd64_go116.go
Generated Vendored
+776
View File
@@ -0,0 +1,776 @@
// +build go1.15,!go1.17
/*
* Copyright 2021 ByteDance Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package decoder
import (
`encoding/json`
`fmt`
`reflect`
`strconv`
`github.com/bytedance/sonic/internal/jit`
`github.com/bytedance/sonic/internal/native`
`github.com/bytedance/sonic/internal/native/types`
`github.com/twitchyliquid64/golang-asm/obj`
`github.com/twitchyliquid64/golang-asm/obj/x86`
)
/** Crucial Registers:
*
* ST(BX) : ro, decoder stack
* DF(R10) : ro, decoder flags
* EP(R11) : wo, error pointer
* IP(R12) : ro, input pointer
* IL(R13) : ro, input length
* IC(R14) : rw, input cursor
* VP(R15) : ro, value pointer (to an interface{})
*/
const (
_VD_args = 8 // 8 bytes for passing arguments to this functions
_VD_fargs = 64 // 64 bytes for passing arguments to other Go functions
_VD_saves = 40 // 40 bytes for saving the registers before CALL instructions
_VD_locals = 88 // 88 bytes for local variables
)
const (
_VD_offs = _VD_fargs + _VD_saves + _VD_locals
_VD_size = _VD_offs + 8 // 8 bytes for the parent frame pointer
)
var (
_VAR_ss = _VAR_ss_Vt
_VAR_df = jit.Ptr(_SP, _VD_fargs + _VD_saves)
)
var (
_VAR_ss_Vt = jit.Ptr(_SP, _VD_fargs + _VD_saves + 8)
_VAR_ss_Dv = jit.Ptr(_SP, _VD_fargs + _VD_saves + 16)
_VAR_ss_Iv = jit.Ptr(_SP, _VD_fargs + _VD_saves + 24)
_VAR_ss_Ep = jit.Ptr(_SP, _VD_fargs + _VD_saves + 32)
_VAR_ss_Db = jit.Ptr(_SP, _VD_fargs + _VD_saves + 40)
_VAR_ss_Dc = jit.Ptr(_SP, _VD_fargs + _VD_saves + 48)
)
var (
_VAR_cs_LR = jit.Ptr(_SP, _VD_fargs + _VD_saves + 56)
_VAR_cs_p = jit.Ptr(_SP, _VD_fargs + _VD_saves + 64)
_VAR_cs_n = jit.Ptr(_SP, _VD_fargs + _VD_saves + 72)
_VAR_cs_d = jit.Ptr(_SP, _VD_fargs + _VD_saves + 80)
)
type _ValueDecoder struct {
jit.BaseAssembler
}
func (self *_ValueDecoder) build() uintptr {
self.Init(self.compile)
return *(*uintptr)(self.Load("decode_value", _VD_size, _VD_args, argPtrs_generic, localPtrs_generic))
}
/** Function Calling Helpers **/
func (self *_ValueDecoder) save(r ...obj.Addr) {
for i, v := range r {
if i > _VD_saves / 8 - 1 {
panic("too many registers to save")
} else {
self.Emit("MOVQ", v, jit.Ptr(_SP, _VD_fargs + int64(i) * 8))
}
}
}
func (self *_ValueDecoder) load(r ...obj.Addr) {
for i, v := range r {
if i > _VD_saves / 8 - 1 {
panic("too many registers to load")
} else {
self.Emit("MOVQ", jit.Ptr(_SP, _VD_fargs + int64(i) * 8), v)
}
}
}
func (self *_ValueDecoder) call(fn obj.Addr) {
self.Emit("MOVQ", fn, _AX) // MOVQ ${fn}, AX
self.Rjmp("CALL", _AX) // CALL AX
}
func (self *_ValueDecoder) call_go(fn obj.Addr) {
self.save(_REG_go...) // SAVE $REG_go
self.call(fn) // CALL ${fn}
self.load(_REG_go...) // LOAD $REG_go
}
/** Decoder Assembler **/
const (
_S_val = iota + 1
_S_arr
_S_arr_0
_S_obj
_S_obj_0
_S_obj_delim
_S_obj_sep
)
const (
_S_omask_key = (1 << _S_obj_0) | (1 << _S_obj_sep)
_S_omask_end = (1 << _S_obj_0) | (1 << _S_obj)
_S_vmask = (1 << _S_val) | (1 << _S_arr_0)
)
const (
_A_init_len = 1
_A_init_cap = 16
)
const (
_ST_Sp = 0
_ST_Vt = _PtrBytes
_ST_Vp = _PtrBytes * (types.MAX_RECURSE + 1)
)
var (
_V_true = jit.Imm(int64(pbool(true)))
_V_false = jit.Imm(int64(pbool(false)))
_F_value = jit.Imm(int64(native.S_value))
)
var (
_V_max = jit.Imm(int64(types.V_MAX))
_E_eof = jit.Imm(int64(types.ERR_EOF))
_E_invalid = jit.Imm(int64(types.ERR_INVALID_CHAR))
_E_recurse = jit.Imm(int64(types.ERR_RECURSE_EXCEED_MAX))
)
var (
_F_convTslice = jit.Func(convTslice)
_F_convTstring = jit.Func(convTstring)
_F_invalid_vtype = jit.Func(invalid_vtype)
)
var (
_T_map = jit.Type(reflect.TypeOf((map[string]interface{})(nil)))
_T_bool = jit.Type(reflect.TypeOf(false))
_T_int64 = jit.Type(reflect.TypeOf(int64(0)))
_T_eface = jit.Type(reflect.TypeOf((*interface{})(nil)).Elem())
_T_slice = jit.Type(reflect.TypeOf(([]interface{})(nil)))
_T_string = jit.Type(reflect.TypeOf(""))
_T_number = jit.Type(reflect.TypeOf(json.Number("")))
_T_float64 = jit.Type(reflect.TypeOf(float64(0)))
)
var _R_tab = map[int]string {
'[': "_decode_V_ARRAY",
'{': "_decode_V_OBJECT",
':': "_decode_V_KEY_SEP",
',': "_decode_V_ELEM_SEP",
']': "_decode_V_ARRAY_END",
'}': "_decode_V_OBJECT_END",
}
func (self *_ValueDecoder) compile() {
self.Emit("SUBQ", jit.Imm(_VD_size), _SP) // SUBQ $_VD_size, SP
self.Emit("MOVQ", _BP, jit.Ptr(_SP, _VD_offs)) // MOVQ BP, _VD_offs(SP)
self.Emit("LEAQ", jit.Ptr(_SP, _VD_offs), _BP) // LEAQ _VD_offs(SP), BP
/* initialize the state machine */
self.Emit("XORL", _CX, _CX) // XORL CX, CX
self.Emit("MOVQ", _DF, _VAR_df) // MOVQ DF, df
/* initialize digital buffer first */
self.Emit("MOVQ", jit.Imm(_MaxDigitNums), _VAR_ss_Dc) // MOVQ $_MaxDigitNums, ss.Dcap
self.Emit("LEAQ", jit.Ptr(_ST, _DbufOffset), _AX) // LEAQ _DbufOffset(ST), AX
self.Emit("MOVQ", _AX, _VAR_ss_Db) // MOVQ AX, ss.Dbuf
/* add ST offset */
self.Emit("ADDQ", jit.Imm(_FsmOffset), _ST) // ADDQ _FsmOffset, _ST
self.Emit("MOVQ", _CX, jit.Ptr(_ST, _ST_Sp)) // MOVQ CX, ST.Sp
self.WriteRecNotAX(0, _VP, jit.Ptr(_ST, _ST_Vp), false) // MOVQ VP, ST.Vp[0]
self.Emit("MOVQ", jit.Imm(_S_val), jit.Ptr(_ST, _ST_Vt)) // MOVQ _S_val, ST.Vt[0]
self.Sjmp("JMP" , "_next") // JMP _next
/* set the value from previous round */
self.Link("_set_value") // _set_value:
self.Emit("MOVL" , jit.Imm(_S_vmask), _DX) // MOVL _S_vmask, DX
self.Emit("MOVQ" , jit.Ptr(_ST, _ST_Sp), _CX) // MOVQ ST.Sp, CX
self.Emit("MOVQ" , jit.Sib(_ST, _CX, 8, _ST_Vt), _AX) // MOVQ ST.Vt[CX], AX
self.Emit("BTQ" , _AX, _DX) // BTQ AX, DX
self.Sjmp("JNC" , "_vtype_error") // JNC _vtype_error
self.Emit("XORL" , _SI, _SI) // XORL SI, SI
self.Emit("SUBQ" , jit.Imm(1), jit.Ptr(_ST, _ST_Sp)) // SUBQ $1, ST.Sp
self.Emit("XCHGQ", jit.Sib(_ST, _CX, 8, _ST_Vp), _SI) // XCHGQ ST.Vp[CX], SI
self.Emit("MOVQ" , _R8, jit.Ptr(_SI, 0)) // MOVQ R8, (SI)
self.WriteRecNotAX(1, _R9, jit.Ptr(_SI, 8), false) // MOVQ R9, 8(SI)
/* check for value stack */
self.Link("_next") // _next:
self.Emit("MOVQ" , jit.Ptr(_ST, _ST_Sp), _AX) // MOVQ ST.Sp, AX
self.Emit("TESTQ", _AX, _AX) // TESTQ AX, AX
self.Sjmp("JS" , "_return") // JS _return
/* fast path: test up to 4 characters manually */
self.Emit("CMPQ" , _IC, _IL) // CMPQ IC, IL
self.Sjmp("JAE" , "_decode_V_EOF") // JAE _decode_V_EOF
self.Emit("MOVBQZX", jit.Sib(_IP, _IC, 1, 0), _AX) // MOVBQZX (IP)(IC), AX
self.Emit("MOVQ" , jit.Imm(_BM_space), _DX) // MOVQ _BM_space, DX
self.Emit("CMPQ" , _AX, jit.Imm(' ')) // CMPQ AX, $' '
self.Sjmp("JA" , "_decode_fast") // JA _decode_fast
self.Emit("BTQ" , _AX, _DX) // BTQ _AX, _DX
self.Sjmp("JNC" , "_decode_fast") // JNC _decode_fast
self.Emit("ADDQ" , jit.Imm(1), _IC) // ADDQ $1, IC
/* at least 1 to 3 spaces */
for i := 0; i < 3; i++ {
self.Emit("CMPQ" , _IC, _IL) // CMPQ IC, IL
self.Sjmp("JAE" , "_decode_V_EOF") // JAE _decode_V_EOF
self.Emit("MOVBQZX", jit.Sib(_IP, _IC, 1, 0), _AX) // MOVBQZX (IP)(IC), AX
self.Emit("CMPQ" , _AX, jit.Imm(' ')) // CMPQ AX, $' '
self.Sjmp("JA" , "_decode_fast") // JA _decode_fast
self.Emit("BTQ" , _AX, _DX) // BTQ _AX, _DX
self.Sjmp("JNC" , "_decode_fast") // JNC _decode_fast
self.Emit("ADDQ" , jit.Imm(1), _IC) // ADDQ $1, IC
}
/* at least 4 spaces */
self.Emit("CMPQ" , _IC, _IL) // CMPQ IC, IL
self.Sjmp("JAE" , "_decode_V_EOF") // JAE _decode_V_EOF
self.Emit("MOVBQZX", jit.Sib(_IP, _IC, 1, 0), _AX) // MOVBQZX (IP)(IC), AX
/* fast path: use lookup table to select decoder */
self.Link("_decode_fast") // _decode_fast:
self.Byte(0x48, 0x8d, 0x3d) // LEAQ ?(PC), DI
self.Sref("_decode_tab", 4) // .... &_decode_tab
self.Emit("MOVLQSX", jit.Sib(_DI, _AX, 4, 0), _AX) // MOVLQSX (DI)(AX*4), AX
self.Emit("TESTQ" , _AX, _AX) // TESTQ AX, AX
self.Sjmp("JZ" , "_decode_native") // JZ _decode_native
self.Emit("ADDQ" , jit.Imm(1), _IC) // ADDQ $1, IC
self.Emit("ADDQ" , _DI, _AX) // ADDQ DI, AX
self.Rjmp("JMP" , _AX) // JMP AX
/* decode with native decoder */
self.Link("_decode_native") // _decode_native:
self.Emit("MOVQ", _IP, _DI) // MOVQ IP, DI
self.Emit("MOVQ", _IL, _SI) // MOVQ IL, SI
self.Emit("MOVQ", _IC, _DX) // MOVQ IC, DX
self.Emit("LEAQ", _VAR_ss, _CX) // LEAQ ss, CX
self.Emit("MOVQ", _VAR_df, _R8) // MOVQ $df, R8
self.Emit("BTSQ", jit.Imm(_F_allow_control), _R8) // ANDQ $1<<_F_allow_control, R8
self.call(_F_value) // CALL value
self.Emit("MOVQ", _AX, _IC) // MOVQ AX, IC
/* check for errors */
self.Emit("MOVQ" , _VAR_ss_Vt, _AX) // MOVQ ss.Vt, AX
self.Emit("TESTQ", _AX, _AX) // TESTQ AX, AX
self.Sjmp("JS" , "_parsing_error")
self.Sjmp("JZ" , "_invalid_vtype") // JZ _invalid_vtype
self.Emit("CMPQ" , _AX, _V_max) // CMPQ AX, _V_max
self.Sjmp("JA" , "_invalid_vtype") // JA _invalid_vtype
/* jump table selector */
self.Byte(0x48, 0x8d, 0x3d) // LEAQ ?(PC), DI
self.Sref("_switch_table", 4) // .... &_switch_table
self.Emit("MOVLQSX", jit.Sib(_DI, _AX, 4, -4), _AX) // MOVLQSX -4(DI)(AX*4), AX
self.Emit("ADDQ" , _DI, _AX) // ADDQ DI, AX
self.Rjmp("JMP" , _AX) // JMP AX
/** V_EOF **/
self.Link("_decode_V_EOF") // _decode_V_EOF:
self.Emit("MOVL", _E_eof, _EP) // MOVL _E_eof, EP
self.Sjmp("JMP" , "_error") // JMP _error
/** V_NULL **/
self.Link("_decode_V_NULL") // _decode_V_NULL:
self.Emit("XORL", _R8, _R8) // XORL R8, R8
self.Emit("XORL", _R9, _R9) // XORL R9, R9
self.Emit("LEAQ", jit.Ptr(_IC, -4), _DI) // LEAQ -4(IC), DI
self.Sjmp("JMP" , "_set_value") // JMP _set_value
/** V_TRUE **/
self.Link("_decode_V_TRUE") // _decode_V_TRUE:
self.Emit("MOVQ", _T_bool, _R8) // MOVQ _T_bool, R8
// TODO: maybe modified by users?
self.Emit("MOVQ", _V_true, _R9) // MOVQ _V_true, R9
self.Emit("LEAQ", jit.Ptr(_IC, -4), _DI) // LEAQ -4(IC), DI
self.Sjmp("JMP" , "_set_value") // JMP _set_value
/** V_FALSE **/
self.Link("_decode_V_FALSE") // _decode_V_FALSE:
self.Emit("MOVQ", _T_bool, _R8) // MOVQ _T_bool, R8
self.Emit("MOVQ", _V_false, _R9) // MOVQ _V_false, R9
self.Emit("LEAQ", jit.Ptr(_IC, -5), _DI) // LEAQ -5(IC), DI
self.Sjmp("JMP" , "_set_value") // JMP _set_value
/** V_ARRAY **/
self.Link("_decode_V_ARRAY") // _decode_V_ARRAY
self.Emit("MOVL", jit.Imm(_S_vmask), _DX) // MOVL _S_vmask, DX
self.Emit("MOVQ", jit.Ptr(_ST, _ST_Sp), _CX) // MOVQ ST.Sp, CX
self.Emit("MOVQ", jit.Sib(_ST, _CX, 8, _ST_Vt), _AX) // MOVQ ST.Vt[CX], AX
self.Emit("BTQ" , _AX, _DX) // BTQ AX, DX
self.Sjmp("JNC" , "_invalid_char") // JNC _invalid_char
/* create a new array */
self.Emit("MOVQ", _T_eface, _AX) // MOVQ _T_eface, AX
self.Emit("MOVQ", _AX, jit.Ptr(_SP, 0)) // MOVQ AX, (SP)
self.Emit("MOVQ", jit.Imm(_A_init_len), jit.Ptr(_SP, 8)) // MOVQ _A_init_len, 8(SP)
self.Emit("MOVQ", jit.Imm(_A_init_cap), jit.Ptr(_SP, 16)) // MOVQ _A_init_cap, 16(SP)
self.call_go(_F_makeslice) // CALL_GO runtime.makeslice
self.Emit("MOVQ", jit.Ptr(_SP, 24), _DX) // MOVQ 24(SP), DX
/* pack into an interface */
self.Emit("MOVQ", _DX, jit.Ptr(_SP, 0)) // MOVQ DX, (SP)
self.Emit("MOVQ", jit.Imm(_A_init_len), jit.Ptr(_SP, 8)) // MOVQ _A_init_len, 8(SP)
self.Emit("MOVQ", jit.Imm(_A_init_cap), jit.Ptr(_SP, 16)) // MOVQ _A_init_cap, 16(SP)
self.call_go(_F_convTslice) // CALL_GO runtime.convTslice
self.Emit("MOVQ", jit.Ptr(_SP, 24), _R8) // MOVQ 24(SP), R8
/* replace current state with an array */
self.Emit("MOVQ", jit.Ptr(_ST, _ST_Sp), _CX) // MOVQ ST.Sp, CX
self.Emit("MOVQ", jit.Sib(_ST, _CX, 8, _ST_Vp), _SI) // MOVQ ST.Vp[CX], SI
self.Emit("MOVQ", jit.Imm(_S_arr), jit.Sib(_ST, _CX, 8, _ST_Vt)) // MOVQ _S_arr, ST.Vt[CX]
self.Emit("MOVQ", _T_slice, _AX) // MOVQ _T_slice, AX
self.Emit("MOVQ", _AX, jit.Ptr(_SI, 0)) // MOVQ AX, (SI)
self.WriteRecNotAX(2, _R8, jit.Ptr(_SI, 8), false) // MOVQ R8, 8(SI)
/* add a new slot for the first element */
self.Emit("ADDQ", jit.Imm(1), _CX) // ADDQ $1, CX
self.Emit("CMPQ", _CX, jit.Imm(types.MAX_RECURSE)) // CMPQ CX, ${types.MAX_RECURSE}
self.Sjmp("JAE" , "_stack_overflow") // JA _stack_overflow
self.Emit("MOVQ", jit.Ptr(_R8, 0), _AX) // MOVQ (R8), AX
self.Emit("MOVQ", _CX, jit.Ptr(_ST, _ST_Sp)) // MOVQ CX, ST.Sp
self.WritePtrAX(3, jit.Sib(_ST, _CX, 8, _ST_Vp), false) // MOVQ AX, ST.Vp[CX]
self.Emit("MOVQ", jit.Imm(_S_arr_0), jit.Sib(_ST, _CX, 8, _ST_Vt)) // MOVQ _S_arr_0, ST.Vt[CX]
self.Sjmp("JMP" , "_next") // JMP _next
/** V_OBJECT **/
self.Link("_decode_V_OBJECT") // _decode_V_OBJECT:
self.Emit("MOVL", jit.Imm(_S_vmask), _DX) // MOVL _S_vmask, DX
self.Emit("MOVQ", jit.Ptr(_ST, _ST_Sp), _CX) // MOVQ ST.Sp, CX
self.Emit("MOVQ", jit.Sib(_ST, _CX, 8, _ST_Vt), _AX) // MOVQ ST.Vt[CX], AX
self.Emit("BTQ" , _AX, _DX) // BTQ AX, DX
self.Sjmp("JNC" , "_invalid_char") // JNC _invalid_char
self.call_go(_F_makemap_small) // CALL_GO runtime.makemap_small
self.Emit("MOVQ", jit.Ptr(_SP, 0), _AX) // MOVQ (SP), AX
self.Emit("MOVQ", jit.Ptr(_ST, _ST_Sp), _CX) // MOVQ ST.Sp, CX
self.Emit("MOVQ", jit.Imm(_S_obj_0), jit.Sib(_ST, _CX, 8, _ST_Vt)) // MOVQ _S_obj, ST.Vt[CX]
self.Emit("MOVQ", jit.Sib(_ST, _CX, 8, _ST_Vp), _SI) // MOVQ ST.Vp[CX], SI
self.Emit("MOVQ", _T_map, _DX) // MOVQ _T_map, DX
self.Emit("MOVQ", _DX, jit.Ptr(_SI, 0)) // MOVQ DX, (SI)
self.WritePtrAX(4, jit.Ptr(_SI, 8), false) // MOVQ AX, 8(SI)
self.Sjmp("JMP" , "_next") // JMP _next
/** V_STRING **/
self.Link("_decode_V_STRING") // _decode_V_STRING:
self.Emit("MOVQ", _VAR_ss_Iv, _CX) // MOVQ ss.Iv, CX
self.Emit("MOVQ", _IC, _AX) // MOVQ IC, AX
self.Emit("SUBQ", _CX, _AX) // SUBQ CX, AX
/* check for escapes */
self.Emit("CMPQ", _VAR_ss_Ep, jit.Imm(-1)) // CMPQ ss.Ep, $-1
self.Sjmp("JNE" , "_unquote") // JNE _unquote
self.Emit("SUBQ", jit.Imm(1), _AX) // SUBQ $1, AX
self.Emit("LEAQ", jit.Sib(_IP, _CX, 1, 0), _R8) // LEAQ (IP)(CX), R8
self.Byte(0x48, 0x8d, 0x3d) // LEAQ (PC), DI
self.Sref("_copy_string_end", 4)
self.Emit("BTQ", jit.Imm(_F_copy_string), _VAR_df)
self.Sjmp("JC", "copy_string")
self.Link("_copy_string_end")
self.Emit("XORL", _DX, _DX) // XORL DX, DX
/* strings with no escape sequences */
self.Link("_noescape") // _noescape:
self.Emit("MOVL", jit.Imm(_S_omask_key), _DI) // MOVL _S_omask, DI
self.Emit("MOVQ", jit.Ptr(_ST, _ST_Sp), _CX) // MOVQ ST.Sp, CX
self.Emit("MOVQ", jit.Sib(_ST, _CX, 8, _ST_Vt), _SI) // MOVQ ST.Vt[CX], SI
self.Emit("BTQ" , _SI, _DI) // BTQ SI, DI
self.Sjmp("JC" , "_object_key") // JC _object_key
/* check for pre-packed strings, avoid 1 allocation */
self.Emit("TESTQ", _DX, _DX) // TESTQ DX, DX
self.Sjmp("JNZ" , "_packed_str") // JNZ _packed_str
self.Emit("MOVQ" , _R8, jit.Ptr(_SP, 0)) // MOVQ R8, (SP)
self.Emit("MOVQ" , _AX, jit.Ptr(_SP, 8)) // MOVQ AX, 8(SP)
self.call_go(_F_convTstring) // CALL_GO runtime.convTstring
self.Emit("MOVQ" , jit.Ptr(_SP, 16), _R9) // MOVQ 16(SP), R9
/* packed string already in R9 */
self.Link("_packed_str") // _packed_str:
self.Emit("MOVQ", _T_string, _R8) // MOVQ _T_string, R8
self.Emit("MOVQ", _VAR_ss_Iv, _DI) // MOVQ ss.Iv, DI
self.Emit("SUBQ", jit.Imm(1), _DI) // SUBQ $1, DI
self.Sjmp("JMP" , "_set_value") // JMP _set_value
/* the string is an object key, get the map */
self.Link("_object_key")
self.Emit("MOVQ", jit.Ptr(_ST, _ST_Sp), _CX) // MOVQ ST.Sp, CX
self.Emit("MOVQ", jit.Sib(_ST, _CX, 8, _ST_Vp), _SI) // MOVQ ST.Vp[CX], SI
self.Emit("MOVQ", jit.Ptr(_SI, 8), _SI) // MOVQ 8(SI), SI
/* add a new delimiter */
self.Emit("ADDQ", jit.Imm(1), _CX) // ADDQ $1, CX
self.Emit("CMPQ", _CX, jit.Imm(types.MAX_RECURSE)) // CMPQ CX, ${types.MAX_RECURSE}
self.Sjmp("JAE" , "_stack_overflow") // JA _stack_overflow
self.Emit("MOVQ", _CX, jit.Ptr(_ST, _ST_Sp)) // MOVQ CX, ST.Sp
self.Emit("MOVQ", jit.Imm(_S_obj_delim), jit.Sib(_ST, _CX, 8, _ST_Vt)) // MOVQ _S_obj_delim, ST.Vt[CX]
/* add a new slot int the map */
self.Emit("MOVQ", _T_map, _DX) // MOVQ _T_map, DX
self.Emit("MOVQ", _DX, jit.Ptr(_SP, 0)) // MOVQ DX, (SP)
self.Emit("MOVQ", _SI, jit.Ptr(_SP, 8)) // MOVQ SI, 8(SP)
self.Emit("MOVQ", _R8, jit.Ptr(_SP, 16)) // MOVQ R9, 16(SP)
self.Emit("MOVQ", _AX, jit.Ptr(_SP, 24)) // MOVQ AX, 24(SP)
self.call_go(_F_mapassign_faststr) // CALL_GO runtime.mapassign_faststr
self.Emit("MOVQ", jit.Ptr(_SP, 32), _AX) // MOVQ 32(SP), AX
/* add to the pointer stack */
self.Emit("MOVQ", jit.Ptr(_ST, _ST_Sp), _CX) // MOVQ ST.Sp, CX
self.WritePtrAX(6, jit.Sib(_ST, _CX, 8, _ST_Vp), false) // MOVQ AX, ST.Vp[CX]
self.Sjmp("JMP" , "_next") // JMP _next
/* allocate memory to store the string header and unquoted result */
self.Link("_unquote") // _unquote:
self.Emit("ADDQ", jit.Imm(15), _AX) // ADDQ $15, AX
self.Emit("MOVQ", _T_byte, _CX) // MOVQ _T_byte, CX
self.Emit("MOVQ", _AX, jit.Ptr(_SP, 0)) // MOVQ AX, (SP)
self.Emit("MOVQ", _CX, jit.Ptr(_SP, 8)) // MOVQ CX, 8(SP)
self.Emit("MOVB", jit.Imm(0), jit.Ptr(_SP, 16)) // MOVB $0, 16(SP)
self.call_go(_F_mallocgc) // CALL_GO runtime.mallocgc
self.Emit("MOVQ", jit.Ptr(_SP, 24), _R9) // MOVQ 24(SP), R9
/* prepare the unquoting parameters */
self.Emit("MOVQ" , _VAR_ss_Iv, _CX) // MOVQ ss.Iv, CX
self.Emit("LEAQ" , jit.Sib(_IP, _CX, 1, 0), _DI) // LEAQ (IP)(CX), DI
self.Emit("NEGQ" , _CX) // NEGQ CX
self.Emit("LEAQ" , jit.Sib(_IC, _CX, 1, -1), _SI) // LEAQ -1(IC)(CX), SI
self.Emit("LEAQ" , jit.Ptr(_R9, 16), _DX) // LEAQ 16(R8), DX
self.Emit("LEAQ" , _VAR_ss_Ep, _CX) // LEAQ ss.Ep, CX
self.Emit("XORL" , _R8, _R8) // XORL R8, R8
self.Emit("BTQ" , jit.Imm(_F_disable_urc), _VAR_df) // BTQ ${_F_disable_urc}, fv
self.Emit("SETCC", _R8) // SETCC R8
self.Emit("SHLQ" , jit.Imm(types.B_UNICODE_REPLACE), _R8) // SHLQ ${types.B_UNICODE_REPLACE}, R8
/* unquote the string, with R9 been preserved */
self.save(_R9) // SAVE R9
self.call(_F_unquote) // CALL unquote
self.load(_R9) // LOAD R9
/* check for errors */
self.Emit("TESTQ", _AX, _AX) // TESTQ AX, AX
self.Sjmp("JS" , "_unquote_error") // JS _unquote_error
self.Emit("MOVL" , jit.Imm(1), _DX) // MOVL $1, DX
self.Emit("LEAQ" , jit.Ptr(_R9, 16), _R8) // ADDQ $16, R8
self.Emit("MOVQ" , _R8, jit.Ptr(_R9, 0)) // MOVQ R8, (R9)
self.Emit("MOVQ" , _AX, jit.Ptr(_R9, 8)) // MOVQ AX, 8(R9)
self.Sjmp("JMP" , "_noescape") // JMP _noescape
/** V_DOUBLE **/
self.Link("_decode_V_DOUBLE") // _decode_V_DOUBLE:
self.Emit("BTQ" , jit.Imm(_F_use_number), _VAR_df) // BTQ _F_use_number, df
self.Sjmp("JC" , "_use_number") // JC _use_number
self.Emit("MOVSD", _VAR_ss_Dv, _X0) // MOVSD ss.Dv, X0
self.Sjmp("JMP" , "_use_float64") // JMP _use_float64
/** V_INTEGER **/
self.Link("_decode_V_INTEGER") // _decode_V_INTEGER:
self.Emit("BTQ" , jit.Imm(_F_use_number), _VAR_df) // BTQ _F_use_number, df
self.Sjmp("JC" , "_use_number") // JC _use_number
self.Emit("BTQ" , jit.Imm(_F_use_int64), _VAR_df) // BTQ _F_use_int64, df
self.Sjmp("JC" , "_use_int64") // JC _use_int64
self.Emit("MOVQ" , _VAR_ss_Iv, _AX) // MOVQ ss.Iv, AX
self.Emit("CVTSQ2SD", _AX, _X0) // CVTSQ2SD AX, X0
/* represent numbers as `float64` */
self.Link("_use_float64") // _use_float64:
self.Emit("MOVSD", _X0, jit.Ptr(_SP, 0)) // MOVSD X0, (SP)
self.call_go(_F_convT64) // CALL_GO runtime.convT64
self.Emit("MOVQ" , _T_float64, _R8) // MOVQ _T_float64, R8
self.Emit("MOVQ" , jit.Ptr(_SP, 8), _R9) // MOVQ 8(SP), R9
self.Emit("MOVQ" , _VAR_ss_Ep, _DI) // MOVQ ss.Ep, DI
self.Sjmp("JMP" , "_set_value") // JMP _set_value
/* represent numbers as `json.Number` */
self.Link("_use_number") // _use_number
self.Emit("MOVQ", _VAR_ss_Ep, _AX) // MOVQ ss.Ep, AX
self.Emit("LEAQ", jit.Sib(_IP, _AX, 1, 0), _SI) // LEAQ (IP)(AX), SI
self.Emit("MOVQ", _IC, _CX) // MOVQ IC, CX
self.Emit("SUBQ", _AX, _CX) // SUBQ AX, CX
self.Emit("MOVQ", _SI, jit.Ptr(_SP, 0)) // MOVQ SI, (SP)
self.Emit("MOVQ", _CX, jit.Ptr(_SP, 8)) // MOVQ CX, 8(SP)
self.call_go(_F_convTstring) // CALL_GO runtime.convTstring
self.Emit("MOVQ", _T_number, _R8) // MOVQ _T_number, R8
self.Emit("MOVQ", jit.Ptr(_SP, 16), _R9) // MOVQ 16(SP), R9
self.Emit("MOVQ", _VAR_ss_Ep, _DI) // MOVQ ss.Ep, DI
self.Sjmp("JMP" , "_set_value") // JMP _set_value
/* represent numbers as `int64` */
self.Link("_use_int64") // _use_int64:
self.Emit("MOVQ", _VAR_ss_Iv, _AX) // MOVQ ss.Iv, AX
self.Emit("MOVQ", _AX, jit.Ptr(_SP, 0)) // MOVQ AX, (SP)
self.call_go(_F_convT64) // CALL_GO runtime.convT64
self.Emit("MOVQ", _T_int64, _R8) // MOVQ _T_int64, R8
self.Emit("MOVQ", jit.Ptr(_SP, 8), _R9) // MOVQ 8(SP), R9
self.Emit("MOVQ", _VAR_ss_Ep, _DI) // MOVQ ss.Ep, DI
self.Sjmp("JMP" , "_set_value") // JMP _set_value
/** V_KEY_SEP **/
self.Link("_decode_V_KEY_SEP") // _decode_V_KEY_SEP:
// self.Byte(0xcc)
self.Emit("MOVQ", jit.Ptr(_ST, _ST_Sp), _CX) // MOVQ ST.Sp, CX
self.Emit("MOVQ", jit.Sib(_ST, _CX, 8, _ST_Vt), _AX) // MOVQ ST.Vt[CX], AX
self.Emit("CMPQ", _AX, jit.Imm(_S_obj_delim)) // CMPQ AX, _S_obj_delim
self.Sjmp("JNE" , "_invalid_char") // JNE _invalid_char
self.Emit("MOVQ", jit.Imm(_S_val), jit.Sib(_ST, _CX, 8, _ST_Vt)) // MOVQ _S_val, ST.Vt[CX]
self.Emit("MOVQ", jit.Imm(_S_obj), jit.Sib(_ST, _CX, 8, _ST_Vt - 8)) // MOVQ _S_obj, ST.Vt[CX - 1]
self.Sjmp("JMP" , "_next") // JMP _next
/** V_ELEM_SEP **/
self.Link("_decode_V_ELEM_SEP") // _decode_V_ELEM_SEP:
self.Emit("MOVQ" , jit.Ptr(_ST, _ST_Sp), _CX) // MOVQ ST.Sp, CX
self.Emit("MOVQ" , jit.Sib(_ST, _CX, 8, _ST_Vt), _AX) // MOVQ ST.Vt[CX], AX
self.Emit("CMPQ" , _AX, jit.Imm(_S_arr)) // CMPQ _AX, _S_arr
self.Sjmp("JE" , "_array_sep") // JZ _next
self.Emit("CMPQ" , _AX, jit.Imm(_S_obj)) // CMPQ _AX, _S_arr
self.Sjmp("JNE" , "_invalid_char") // JNE _invalid_char
self.Emit("MOVQ" , jit.Imm(_S_obj_sep), jit.Sib(_ST, _CX, 8, _ST_Vt))
self.Sjmp("JMP" , "_next") // JMP _next
/* arrays */
self.Link("_array_sep")
self.Emit("MOVQ", jit.Sib(_ST, _CX, 8, _ST_Vp), _SI) // MOVQ ST.Vp[CX], SI
self.Emit("MOVQ", jit.Ptr(_SI, 8), _SI) // MOVQ 8(SI), SI
self.Emit("MOVQ", jit.Ptr(_SI, 8), _DX) // MOVQ 8(SI), DX
self.Emit("CMPQ", _DX, jit.Ptr(_SI, 16)) // CMPQ DX, 16(SI)
self.Sjmp("JAE" , "_array_more") // JAE _array_more
/* add a slot for the new element */
self.Link("_array_append") // _array_append:
self.Emit("ADDQ", jit.Imm(1), jit.Ptr(_SI, 8)) // ADDQ $1, 8(SI)
self.Emit("MOVQ", jit.Ptr(_SI, 0), _SI) // MOVQ (SI), SI
self.Emit("ADDQ", jit.Imm(1), _CX) // ADDQ $1, CX
self.Emit("CMPQ", _CX, jit.Imm(types.MAX_RECURSE)) // CMPQ CX, ${types.MAX_RECURSE}
self.Sjmp("JAE" , "_stack_overflow")
self.Emit("SHLQ", jit.Imm(1), _DX) // SHLQ $1, DX
self.Emit("LEAQ", jit.Sib(_SI, _DX, 8, 0), _SI) // LEAQ (SI)(DX*8), SI
self.Emit("MOVQ", _CX, jit.Ptr(_ST, _ST_Sp)) // MOVQ CX, ST.Sp
self.WriteRecNotAX(7 , _SI, jit.Sib(_ST, _CX, 8, _ST_Vp), false) // MOVQ SI, ST.Vp[CX]
self.Emit("MOVQ", jit.Imm(_S_val), jit.Sib(_ST, _CX, 8, _ST_Vt)) // MOVQ _S_val, ST.Vt[CX}
self.Sjmp("JMP" , "_next") // JMP _next
/** V_ARRAY_END **/
self.Link("_decode_V_ARRAY_END") // _decode_V_ARRAY_END:
self.Emit("XORL", _DX, _DX) // XORL DX, DX
self.Emit("MOVQ", jit.Ptr(_ST, _ST_Sp), _CX) // MOVQ ST.Sp, CX
self.Emit("MOVQ", jit.Sib(_ST, _CX, 8, _ST_Vt), _AX) // MOVQ ST.Vt[CX], AX
self.Emit("CMPQ", _AX, jit.Imm(_S_arr_0)) // CMPQ AX, _S_arr_0
self.Sjmp("JE" , "_first_item") // JE _first_item
self.Emit("CMPQ", _AX, jit.Imm(_S_arr)) // CMPQ AX, _S_arr
self.Sjmp("JNE" , "_invalid_char") // JNE _invalid_char
self.Emit("SUBQ", jit.Imm(1), jit.Ptr(_ST, _ST_Sp)) // SUBQ $1, ST.Sp
self.Emit("MOVQ", _DX, jit.Sib(_ST, _CX, 8, _ST_Vp)) // MOVQ DX, ST.Vp[CX]
self.Sjmp("JMP" , "_next") // JMP _next
/* first element of an array */
self.Link("_first_item") // _first_item:
self.Emit("MOVQ", jit.Ptr(_ST, _ST_Sp), _CX) // MOVQ ST.Sp, CX
self.Emit("SUBQ", jit.Imm(2), jit.Ptr(_ST, _ST_Sp)) // SUBQ $2, ST.Sp
self.Emit("MOVQ", jit.Sib(_ST, _CX, 8, _ST_Vp - 8), _SI) // MOVQ ST.Vp[CX - 1], SI
self.Emit("MOVQ", jit.Ptr(_SI, 8), _SI) // MOVQ 8(SI), SI
self.Emit("MOVQ", _DX, jit.Sib(_ST, _CX, 8, _ST_Vp - 8)) // MOVQ DX, ST.Vp[CX - 1]
self.Emit("MOVQ", _DX, jit.Sib(_ST, _CX, 8, _ST_Vp)) // MOVQ DX, ST.Vp[CX]
self.Emit("MOVQ", _DX, jit.Ptr(_SI, 8)) // MOVQ DX, 8(SI)
self.Sjmp("JMP" , "_next") // JMP _next
/** V_OBJECT_END **/
self.Link("_decode_V_OBJECT_END") // _decode_V_OBJECT_END:
self.Emit("MOVL", jit.Imm(_S_omask_end), _DX) // MOVL _S_omask, DI
self.Emit("MOVQ", jit.Ptr(_ST, _ST_Sp), _CX) // MOVQ ST.Sp, CX
self.Emit("MOVQ", jit.Sib(_ST, _CX, 8, _ST_Vt), _AX) // MOVQ ST.Vt[CX], AX
self.Emit("BTQ" , _AX, _DX)
self.Sjmp("JNC" , "_invalid_char") // JNE _invalid_char
self.Emit("XORL", _AX, _AX) // XORL AX, AX
self.Emit("SUBQ", jit.Imm(1), jit.Ptr(_ST, _ST_Sp)) // SUBQ $1, ST.Sp
self.Emit("MOVQ", _AX, jit.Sib(_ST, _CX, 8, _ST_Vp)) // MOVQ AX, ST.Vp[CX]
self.Sjmp("JMP" , "_next") // JMP _next
/* return from decoder */
self.Link("_return") // _return:
self.Emit("XORL", _EP, _EP) // XORL EP, EP
self.Emit("MOVQ", _EP, jit.Ptr(_ST, _ST_Vp)) // MOVQ EP, ST.Vp[0]
self.Link("_epilogue") // _epilogue:
self.Emit("SUBQ", jit.Imm(_FsmOffset), _ST) // SUBQ _FsmOffset, _ST
self.Emit("MOVQ", jit.Ptr(_SP, _VD_offs), _BP) // MOVQ _VD_offs(SP), BP
self.Emit("ADDQ", jit.Imm(_VD_size), _SP) // ADDQ $_VD_size, SP
self.Emit("RET") // RET
/* array expand */
self.Link("_array_more") // _array_more:
self.Emit("MOVQ" , _T_eface, _AX) // MOVQ _T_eface, AX
self.Emit("MOVOU", jit.Ptr(_SI, 0), _X0) // MOVOU (SI), X0
self.Emit("MOVQ" , jit.Ptr(_SI, 16), _DX) // MOVQ 16(SI), DX
self.Emit("MOVQ" , _AX, jit.Ptr(_SP, 0)) // MOVQ AX, (SP)
self.Emit("MOVOU", _X0, jit.Ptr(_SP, 8)) // MOVOU X0, 8(SP)
self.Emit("MOVQ" , _DX, jit.Ptr(_SP, 24)) // MOVQ DX, 24(SP)
self.Emit("SHLQ" , jit.Imm(1), _DX) // SHLQ $1, DX
self.Emit("MOVQ" , _DX, jit.Ptr(_SP, 32)) // MOVQ DX, 32(SP)
self.call_go(_F_growslice) // CALL_GO runtime.growslice
self.Emit("MOVQ" , jit.Ptr(_SP, 40), _DI) // MOVOU 40(SP), DI
self.Emit("MOVQ" , jit.Ptr(_SP, 48), _DX) // MOVOU 48(SP), DX
self.Emit("MOVQ" , jit.Ptr(_SP, 56), _AX) // MOVQ 56(SP), AX
/* update the slice */
self.Emit("MOVQ", jit.Ptr(_ST, _ST_Sp), _CX) // MOVQ ST.Sp, CX
self.Emit("MOVQ", jit.Sib(_ST, _CX, 8, _ST_Vp), _SI) // MOVQ ST.Vp[CX], SI
self.Emit("MOVQ", jit.Ptr(_SI, 8), _SI) // MOVQ 8(SI), SI
self.Emit("MOVQ", _DX, jit.Ptr(_SI, 8)) // MOVQ DX, 8(SI)
self.Emit("MOVQ", _AX, jit.Ptr(_SI, 16)) // MOVQ AX, 16(AX)
self.WriteRecNotAX(8 , _DI, jit.Ptr(_SI, 0), false) // MOVQ R10, (SI)
self.Sjmp("JMP" , "_array_append") // JMP _array_append
/* copy string */
self.Link("copy_string") // pointer: R8, length: AX, return addr: DI
// self.Byte(0xcc)
self.Emit("MOVQ", _R8, _VAR_cs_p)
self.Emit("MOVQ", _AX, _VAR_cs_n)
self.Emit("MOVQ", _DI, _VAR_cs_LR)
self.Emit("MOVQ", _T_byte, jit.Ptr(_SP, 0))
self.Emit("MOVQ", _AX, jit.Ptr(_SP, 8))
self.Emit("MOVQ", _AX, jit.Ptr(_SP, 16))
self.call_go(_F_makeslice)
self.Emit("MOVQ", jit.Ptr(_SP, 24), _R8)
self.Emit("MOVQ", _R8, _VAR_cs_d)
self.Emit("MOVQ", _R8, jit.Ptr(_SP, 0))
self.Emit("MOVQ", _VAR_cs_p, _R8)
self.Emit("MOVQ", _R8, jit.Ptr(_SP, 8))
self.Emit("MOVQ", _VAR_cs_n, _AX)
self.Emit("MOVQ", _AX, jit.Ptr(_SP, 16))
self.call_go(_F_memmove)
self.Emit("MOVQ", _VAR_cs_d, _R8)
self.Emit("MOVQ", _VAR_cs_n, _AX)
self.Emit("MOVQ", _VAR_cs_LR, _DI)
// self.Byte(0xcc)
self.Rjmp("JMP", _DI)
/* error handlers */
self.Link("_stack_overflow")
self.Emit("MOVL" , _E_recurse, _EP) // MOVQ _E_recurse, EP
self.Sjmp("JMP" , "_error") // JMP _error
self.Link("_vtype_error") // _vtype_error:
self.Emit("MOVQ" , _DI, _IC) // MOVQ DI, IC
self.Emit("MOVL" , _E_invalid, _EP) // MOVL _E_invalid, EP
self.Sjmp("JMP" , "_error") // JMP _error
self.Link("_invalid_char") // _invalid_char:
self.Emit("SUBQ" , jit.Imm(1), _IC) // SUBQ $1, IC
self.Emit("MOVL" , _E_invalid, _EP) // MOVL _E_invalid, EP
self.Sjmp("JMP" , "_error") // JMP _error
self.Link("_unquote_error") // _unquote_error:
self.Emit("MOVQ" , _VAR_ss_Iv, _IC) // MOVQ ss.Iv, IC
self.Emit("SUBQ" , jit.Imm(1), _IC) // SUBQ $1, IC
self.Link("_parsing_error") // _parsing_error:
self.Emit("NEGQ" , _AX) // NEGQ AX
self.Emit("MOVQ" , _AX, _EP) // MOVQ AX, EP
self.Link("_error") // _error:
self.Emit("PXOR" , _X0, _X0) // PXOR X0, X0
self.Emit("MOVOU", _X0, jit.Ptr(_VP, 0)) // MOVOU X0, (VP)
self.Sjmp("JMP" , "_epilogue") // JMP _epilogue
/* invalid value type, never returns */
self.Link("_invalid_vtype")
self.Emit("MOVQ", _AX, jit.Ptr(_SP, 0)) // MOVQ AX, (SP)
self.call(_F_invalid_vtype) // CALL invalid_type
self.Emit("UD2") // UD2
/* switch jump table */
self.Link("_switch_table") // _switch_table:
self.Sref("_decode_V_EOF", 0) // SREF &_decode_V_EOF, $0
self.Sref("_decode_V_NULL", -4) // SREF &_decode_V_NULL, $-4
self.Sref("_decode_V_TRUE", -8) // SREF &_decode_V_TRUE, $-8
self.Sref("_decode_V_FALSE", -12) // SREF &_decode_V_FALSE, $-12
self.Sref("_decode_V_ARRAY", -16) // SREF &_decode_V_ARRAY, $-16
self.Sref("_decode_V_OBJECT", -20) // SREF &_decode_V_OBJECT, $-20
self.Sref("_decode_V_STRING", -24) // SREF &_decode_V_STRING, $-24
self.Sref("_decode_V_DOUBLE", -28) // SREF &_decode_V_DOUBLE, $-28
self.Sref("_decode_V_INTEGER", -32) // SREF &_decode_V_INTEGER, $-32
self.Sref("_decode_V_KEY_SEP", -36) // SREF &_decode_V_KEY_SEP, $-36
self.Sref("_decode_V_ELEM_SEP", -40) // SREF &_decode_V_ELEM_SEP, $-40
self.Sref("_decode_V_ARRAY_END", -44) // SREF &_decode_V_ARRAY_END, $-44
self.Sref("_decode_V_OBJECT_END", -48) // SREF &_decode_V_OBJECT_END, $-48
/* fast character lookup table */
self.Link("_decode_tab") // _decode_tab:
self.Sref("_decode_V_EOF", 0) // SREF &_decode_V_EOF, $0
/* generate rest of the tabs */
for i := 1; i < 256; i++ {
if to, ok := _R_tab[i]; ok {
self.Sref(to, -int64(i) * 4)
} else {
self.Byte(0x00, 0x00, 0x00, 0x00)
}
}
}
func (self *_ValueDecoder) WritePtrAX(i int, rec obj.Addr, saveDI bool) {
self.Emit("MOVQ", _V_writeBarrier, _R10)
self.Emit("CMPL", jit.Ptr(_R10, 0), jit.Imm(0))
self.Sjmp("JE", "_no_writeBarrier" + strconv.Itoa(i) + "_{n}")
if saveDI {
self.save(_DI)
}
self.Emit("LEAQ", rec, _DI)
self.Emit("MOVQ", _F_gcWriteBarrierAX, _R10) // MOVQ ${fn}, AX
self.Rjmp("CALL", _R10)
if saveDI {
self.load(_DI)
}
self.Sjmp("JMP", "_end_writeBarrier" + strconv.Itoa(i) + "_{n}")
self.Link("_no_writeBarrier" + strconv.Itoa(i) + "_{n}")
self.Emit("MOVQ", _AX, rec)
self.Link("_end_writeBarrier" + strconv.Itoa(i) + "_{n}")
}
func (self *_ValueDecoder) WriteRecNotAX(i int, ptr obj.Addr, rec obj.Addr, saveDI bool) {
if rec.Reg == x86.REG_AX || rec.Index == x86.REG_AX {
panic("rec contains AX!")
}
self.Emit("MOVQ", _V_writeBarrier, _R10)
self.Emit("CMPL", jit.Ptr(_R10, 0), jit.Imm(0))
self.Sjmp("JE", "_no_writeBarrier" + strconv.Itoa(i) + "_{n}")
self.Emit("MOVQ", ptr, _AX)
if saveDI {
self.save(_DI)
}
self.Emit("LEAQ", rec, _DI)
self.Emit("MOVQ", _F_gcWriteBarrierAX, _R10) // MOVQ ${fn}, AX
self.Rjmp("CALL", _R10)
if saveDI {
self.load(_DI)
}
self.Sjmp("JMP", "_end_writeBarrier" + strconv.Itoa(i) + "_{n}")
self.Link("_no_writeBarrier" + strconv.Itoa(i) + "_{n}")
self.Emit("MOVQ", ptr, rec)
self.Link("_end_writeBarrier" + strconv.Itoa(i) + "_{n}")
}
/** Generic Decoder **/
var (
_subr_decode_value = new(_ValueDecoder).build()
)
//go:nosplit
func invalid_vtype(vt types.ValueType) {
throw(fmt.Sprintf("invalid value type: %d", vt))
}
vendor/github.com/bytedance/sonic/internal/decoder/generic_amd64_go117.go
Generated Vendored
+772
View File
@@ -0,0 +1,772 @@
//go:build go1.17 && !go1.21
// +build go1.17,!go1.21
/*
* Copyright 2021 ByteDance Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package decoder
import (
`encoding/json`
`fmt`
`reflect`
`strconv`
`github.com/bytedance/sonic/internal/jit`
`github.com/bytedance/sonic/internal/native`
`github.com/bytedance/sonic/internal/native/types`
`github.com/twitchyliquid64/golang-asm/obj`
`github.com/twitchyliquid64/golang-asm/obj/x86`
)
/** Crucial Registers:
*
* ST(R13) && 0(SP) : ro, decoder stack
* DF(AX) : ro, decoder flags
* EP(BX) : wo, error pointer
* IP(R10) : ro, input pointer
* IL(R12) : ro, input length
* IC(R11) : rw, input cursor
* VP(R15) : ro, value pointer (to an interface{})
*/
const (
_VD_args = 8 // 8 bytes for passing arguments to this functions
_VD_fargs = 64 // 64 bytes for passing arguments to other Go functions
_VD_saves = 48 // 48 bytes for saving the registers before CALL instructions
_VD_locals = 96 // 96 bytes for local variables
)
const (
_VD_offs = _VD_fargs + _VD_saves + _VD_locals
_VD_size = _VD_offs + 8 // 8 bytes for the parent frame pointer
)
var (
_VAR_ss = _VAR_ss_Vt
_VAR_df = jit.Ptr(_SP, _VD_fargs + _VD_saves)
)
var (
_VAR_ss_Vt = jit.Ptr(_SP, _VD_fargs + _VD_saves + 8)
_VAR_ss_Dv = jit.Ptr(_SP, _VD_fargs + _VD_saves + 16)
_VAR_ss_Iv = jit.Ptr(_SP, _VD_fargs + _VD_saves + 24)
_VAR_ss_Ep = jit.Ptr(_SP, _VD_fargs + _VD_saves + 32)
_VAR_ss_Db = jit.Ptr(_SP, _VD_fargs + _VD_saves + 40)
_VAR_ss_Dc = jit.Ptr(_SP, _VD_fargs + _VD_saves + 48)
)
var (
_VAR_R9 = jit.Ptr(_SP, _VD_fargs + _VD_saves + 56)
)
type _ValueDecoder struct {
jit.BaseAssembler
}
var (
_VAR_cs_LR = jit.Ptr(_SP, _VD_fargs + _VD_saves + 64)
_VAR_cs_p = jit.Ptr(_SP, _VD_fargs + _VD_saves + 72)
_VAR_cs_n = jit.Ptr(_SP, _VD_fargs + _VD_saves + 80)
_VAR_cs_d = jit.Ptr(_SP, _VD_fargs + _VD_saves + 88)
)
func (self *_ValueDecoder) build() uintptr {
self.Init(self.compile)
return *(*uintptr)(self.Load("decode_value", _VD_size, _VD_args, argPtrs_generic, localPtrs_generic))
}
/** Function Calling Helpers **/
func (self *_ValueDecoder) save(r ...obj.Addr) {
for i, v := range r {
if i > _VD_saves / 8 - 1 {
panic("too many registers to save")
} else {
self.Emit("MOVQ", v, jit.Ptr(_SP, _VD_fargs + int64(i) * 8))
}
}
}
func (self *_ValueDecoder) load(r ...obj.Addr) {
for i, v := range r {
if i > _VD_saves / 8 - 1 {
panic("too many registers to load")
} else {
self.Emit("MOVQ", jit.Ptr(_SP, _VD_fargs + int64(i) * 8), v)
}
}
}
func (self *_ValueDecoder) call(fn obj.Addr) {
self.Emit("MOVQ", fn, _R9) // MOVQ ${fn}, AX
self.Rjmp("CALL", _R9) // CALL AX
}
func (self *_ValueDecoder) call_go(fn obj.Addr) {
self.save(_REG_go...) // SAVE $REG_go
self.call(fn) // CALL ${fn}
self.load(_REG_go...) // LOAD $REG_go
}
func (self *_ValueDecoder) callc(fn obj.Addr) {
self.Emit("XCHGQ", _IP, _BP)
self.call(fn)
self.Emit("XCHGQ", _IP, _BP)
}
func (self *_ValueDecoder) call_c(fn obj.Addr) {
self.Emit("XCHGQ", _IC, _BX)
self.callc(fn)
self.Emit("XCHGQ", _IC, _BX)
}
/** Decoder Assembler **/
const (
_S_val = iota + 1
_S_arr
_S_arr_0
_S_obj
_S_obj_0
_S_obj_delim
_S_obj_sep
)
const (
_S_omask_key = (1 << _S_obj_0) | (1 << _S_obj_sep)
_S_omask_end = (1 << _S_obj_0) | (1 << _S_obj)
_S_vmask = (1 << _S_val) | (1 << _S_arr_0)
)
const (
_A_init_len = 1
_A_init_cap = 16
)
const (
_ST_Sp = 0
_ST_Vt = _PtrBytes
_ST_Vp = _PtrBytes * (types.MAX_RECURSE + 1)
)
var (
_V_true = jit.Imm(int64(pbool(true)))
_V_false = jit.Imm(int64(pbool(false)))
_F_value = jit.Imm(int64(native.S_value))
)
var (
_V_max = jit.Imm(int64(types.V_MAX))
_E_eof = jit.Imm(int64(types.ERR_EOF))
_E_invalid = jit.Imm(int64(types.ERR_INVALID_CHAR))
_E_recurse = jit.Imm(int64(types.ERR_RECURSE_EXCEED_MAX))
)
var (
_F_convTslice = jit.Func(convTslice)
_F_convTstring = jit.Func(convTstring)
_F_invalid_vtype = jit.Func(invalid_vtype)
)
var (
_T_map = jit.Type(reflect.TypeOf((map[string]interface{})(nil)))
_T_bool = jit.Type(reflect.TypeOf(false))
_T_int64 = jit.Type(reflect.TypeOf(int64(0)))
_T_eface = jit.Type(reflect.TypeOf((*interface{})(nil)).Elem())
_T_slice = jit.Type(reflect.TypeOf(([]interface{})(nil)))
_T_string = jit.Type(reflect.TypeOf(""))
_T_number = jit.Type(reflect.TypeOf(json.Number("")))
_T_float64 = jit.Type(reflect.TypeOf(float64(0)))
)
var _R_tab = map[int]string {
'[': "_decode_V_ARRAY",
'{': "_decode_V_OBJECT",
':': "_decode_V_KEY_SEP",
',': "_decode_V_ELEM_SEP",
']': "_decode_V_ARRAY_END",
'}': "_decode_V_OBJECT_END",
}
func (self *_ValueDecoder) compile() {
self.Emit("SUBQ", jit.Imm(_VD_size), _SP) // SUBQ $_VD_size, SP
self.Emit("MOVQ", _BP, jit.Ptr(_SP, _VD_offs)) // MOVQ BP, _VD_offs(SP)
self.Emit("LEAQ", jit.Ptr(_SP, _VD_offs), _BP) // LEAQ _VD_offs(SP), BP
/* initialize the state machine */
self.Emit("XORL", _CX, _CX) // XORL CX, CX
self.Emit("MOVQ", _DF, _VAR_df) // MOVQ DF, df
/* initialize digital buffer first */
self.Emit("MOVQ", jit.Imm(_MaxDigitNums), _VAR_ss_Dc) // MOVQ $_MaxDigitNums, ss.Dcap
self.Emit("LEAQ", jit.Ptr(_ST, _DbufOffset), _AX) // LEAQ _DbufOffset(ST), AX
self.Emit("MOVQ", _AX, _VAR_ss_Db) // MOVQ AX, ss.Dbuf
/* add ST offset */
self.Emit("ADDQ", jit.Imm(_FsmOffset), _ST) // ADDQ _FsmOffset, _ST
self.Emit("MOVQ", _CX, jit.Ptr(_ST, _ST_Sp)) // MOVQ CX, ST.Sp
self.WriteRecNotAX(0, _VP, jit.Ptr(_ST, _ST_Vp), false) // MOVQ VP, ST.Vp[0]
self.Emit("MOVQ", jit.Imm(_S_val), jit.Ptr(_ST, _ST_Vt)) // MOVQ _S_val, ST.Vt[0]
self.Sjmp("JMP" , "_next") // JMP _next
/* set the value from previous round */
self.Link("_set_value") // _set_value:
self.Emit("MOVL" , jit.Imm(_S_vmask), _DX) // MOVL _S_vmask, DX
self.Emit("MOVQ" , jit.Ptr(_ST, _ST_Sp), _CX) // MOVQ ST.Sp, CX
self.Emit("MOVQ" , jit.Sib(_ST, _CX, 8, _ST_Vt), _AX) // MOVQ ST.Vt[CX], AX
self.Emit("BTQ" , _AX, _DX) // BTQ AX, DX
self.Sjmp("JNC" , "_vtype_error") // JNC _vtype_error
self.Emit("XORL" , _SI, _SI) // XORL SI, SI
self.Emit("SUBQ" , jit.Imm(1), jit.Ptr(_ST, _ST_Sp)) // SUBQ $1, ST.Sp
self.Emit("XCHGQ", jit.Sib(_ST, _CX, 8, _ST_Vp), _SI) // XCHGQ ST.Vp[CX], SI
self.Emit("MOVQ" , _R8, jit.Ptr(_SI, 0)) // MOVQ R8, (SI)
self.WriteRecNotAX(1, _R9, jit.Ptr(_SI, 8), false) // MOVQ R9, 8(SI)
/* check for value stack */
self.Link("_next") // _next:
self.Emit("MOVQ" , jit.Ptr(_ST, _ST_Sp), _AX) // MOVQ ST.Sp, AX
self.Emit("TESTQ", _AX, _AX) // TESTQ AX, AX
self.Sjmp("JS" , "_return") // JS _return
/* fast path: test up to 4 characters manually */
self.Emit("CMPQ" , _IC, _IL) // CMPQ IC, IL
self.Sjmp("JAE" , "_decode_V_EOF") // JAE _decode_V_EOF
self.Emit("MOVBQZX", jit.Sib(_IP, _IC, 1, 0), _AX) // MOVBQZX (IP)(IC), AX
self.Emit("MOVQ" , jit.Imm(_BM_space), _DX) // MOVQ _BM_space, DX
self.Emit("CMPQ" , _AX, jit.Imm(' ')) // CMPQ AX, $' '
self.Sjmp("JA" , "_decode_fast") // JA _decode_fast
self.Emit("BTQ" , _AX, _DX) // BTQ _AX, _DX
self.Sjmp("JNC" , "_decode_fast") // JNC _decode_fast
self.Emit("ADDQ" , jit.Imm(1), _IC) // ADDQ $1, IC
/* at least 1 to 3 spaces */
for i := 0; i < 3; i++ {
self.Emit("CMPQ" , _IC, _IL) // CMPQ IC, IL
self.Sjmp("JAE" , "_decode_V_EOF") // JAE _decode_V_EOF
self.Emit("MOVBQZX", jit.Sib(_IP, _IC, 1, 0), _AX) // MOVBQZX (IP)(IC), AX
self.Emit("CMPQ" , _AX, jit.Imm(' ')) // CMPQ AX, $' '
self.Sjmp("JA" , "_decode_fast") // JA _decode_fast
self.Emit("BTQ" , _AX, _DX) // BTQ _AX, _DX
self.Sjmp("JNC" , "_decode_fast") // JNC _decode_fast
self.Emit("ADDQ" , jit.Imm(1), _IC) // ADDQ $1, IC
}
/* at least 4 spaces */
self.Emit("CMPQ" , _IC, _IL) // CMPQ IC, IL
self.Sjmp("JAE" , "_decode_V_EOF") // JAE _decode_V_EOF
self.Emit("MOVBQZX", jit.Sib(_IP, _IC, 1, 0), _AX) // MOVBQZX (IP)(IC), AX
/* fast path: use lookup table to select decoder */
self.Link("_decode_fast") // _decode_fast:
self.Byte(0x48, 0x8d, 0x3d) // LEAQ ?(PC), DI
self.Sref("_decode_tab", 4) // .... &_decode_tab
self.Emit("MOVLQSX", jit.Sib(_DI, _AX, 4, 0), _AX) // MOVLQSX (DI)(AX*4), AX
self.Emit("TESTQ" , _AX, _AX) // TESTQ AX, AX
self.Sjmp("JZ" , "_decode_native") // JZ _decode_native
self.Emit("ADDQ" , jit.Imm(1), _IC) // ADDQ $1, IC
self.Emit("ADDQ" , _DI, _AX) // ADDQ DI, AX
self.Rjmp("JMP" , _AX) // JMP AX
/* decode with native decoder */
self.Link("_decode_native") // _decode_native:
self.Emit("MOVQ", _IP, _DI) // MOVQ IP, DI
self.Emit("MOVQ", _IL, _SI) // MOVQ IL, SI
self.Emit("MOVQ", _IC, _DX) // MOVQ IC, DX
self.Emit("LEAQ", _VAR_ss, _CX) // LEAQ ss, CX
self.Emit("MOVQ", _VAR_df, _R8) // MOVQ $df, R8
self.Emit("BTSQ", jit.Imm(_F_allow_control), _R8) // ANDQ $1<<_F_allow_control, R8
self.callc(_F_value) // CALL value
self.Emit("MOVQ", _AX, _IC) // MOVQ AX, IC
/* check for errors */
self.Emit("MOVQ" , _VAR_ss_Vt, _AX) // MOVQ ss.Vt, AX
self.Emit("TESTQ", _AX, _AX) // TESTQ AX, AX
self.Sjmp("JS" , "_parsing_error")
self.Sjmp("JZ" , "_invalid_vtype") // JZ _invalid_vtype
self.Emit("CMPQ" , _AX, _V_max) // CMPQ AX, _V_max
self.Sjmp("JA" , "_invalid_vtype") // JA _invalid_vtype
/* jump table selector */
self.Byte(0x48, 0x8d, 0x3d) // LEAQ ?(PC), DI
self.Sref("_switch_table", 4) // .... &_switch_table
self.Emit("MOVLQSX", jit.Sib(_DI, _AX, 4, -4), _AX) // MOVLQSX -4(DI)(AX*4), AX
self.Emit("ADDQ" , _DI, _AX) // ADDQ DI, AX
self.Rjmp("JMP" , _AX) // JMP AX
/** V_EOF **/
self.Link("_decode_V_EOF") // _decode_V_EOF:
self.Emit("MOVL", _E_eof, _EP) // MOVL _E_eof, EP
self.Sjmp("JMP" , "_error") // JMP _error
/** V_NULL **/
self.Link("_decode_V_NULL") // _decode_V_NULL:
self.Emit("XORL", _R8, _R8) // XORL R8, R8
self.Emit("XORL", _R9, _R9) // XORL R9, R9
self.Emit("LEAQ", jit.Ptr(_IC, -4), _DI) // LEAQ -4(IC), DI
self.Sjmp("JMP" , "_set_value") // JMP _set_value
/** V_TRUE **/
self.Link("_decode_V_TRUE") // _decode_V_TRUE:
self.Emit("MOVQ", _T_bool, _R8) // MOVQ _T_bool, R8
// TODO: maybe modified by users?
self.Emit("MOVQ", _V_true, _R9) // MOVQ _V_true, R9
self.Emit("LEAQ", jit.Ptr(_IC, -4), _DI) // LEAQ -4(IC), DI
self.Sjmp("JMP" , "_set_value") // JMP _set_value
/** V_FALSE **/
self.Link("_decode_V_FALSE") // _decode_V_FALSE:
self.Emit("MOVQ", _T_bool, _R8) // MOVQ _T_bool, R8
self.Emit("MOVQ", _V_false, _R9) // MOVQ _V_false, R9
self.Emit("LEAQ", jit.Ptr(_IC, -5), _DI) // LEAQ -5(IC), DI
self.Sjmp("JMP" , "_set_value") // JMP _set_value
/** V_ARRAY **/
self.Link("_decode_V_ARRAY") // _decode_V_ARRAY
self.Emit("MOVL", jit.Imm(_S_vmask), _DX) // MOVL _S_vmask, DX
self.Emit("MOVQ", jit.Ptr(_ST, _ST_Sp), _CX) // MOVQ ST.Sp, CX
self.Emit("MOVQ", jit.Sib(_ST, _CX, 8, _ST_Vt), _AX) // MOVQ ST.Vt[CX], AX
self.Emit("BTQ" , _AX, _DX) // BTQ AX, DX
self.Sjmp("JNC" , "_invalid_char") // JNC _invalid_char
/* create a new array */
self.Emit("MOVQ", _T_eface, _AX) // MOVQ _T_eface, AX
self.Emit("MOVQ", jit.Imm(_A_init_len), _BX) // MOVQ _A_init_len, BX
self.Emit("MOVQ", jit.Imm(_A_init_cap), _CX) // MOVQ _A_init_cap, CX
self.call_go(_F_makeslice) // CALL_GO runtime.makeslice
/* pack into an interface */
self.Emit("MOVQ", jit.Imm(_A_init_len), _BX) // MOVQ _A_init_len, BX
self.Emit("MOVQ", jit.Imm(_A_init_cap), _CX) // MOVQ _A_init_cap, CX
self.call_go(_F_convTslice) // CALL_GO runtime.convTslice
self.Emit("MOVQ", _AX, _R8) // MOVQ AX, R8
/* replace current state with an array */
self.Emit("MOVQ", jit.Ptr(_ST, _ST_Sp), _CX) // MOVQ ST.Sp, CX
self.Emit("MOVQ", jit.Sib(_ST, _CX, 8, _ST_Vp), _SI) // MOVQ ST.Vp[CX], SI
self.Emit("MOVQ", jit.Imm(_S_arr), jit.Sib(_ST, _CX, 8, _ST_Vt)) // MOVQ _S_arr, ST.Vt[CX]
self.Emit("MOVQ", _T_slice, _AX) // MOVQ _T_slice, AX
self.Emit("MOVQ", _AX, jit.Ptr(_SI, 0)) // MOVQ AX, (SI)
self.WriteRecNotAX(2, _R8, jit.Ptr(_SI, 8), false) // MOVQ R8, 8(SI)
/* add a new slot for the first element */
self.Emit("ADDQ", jit.Imm(1), _CX) // ADDQ $1, CX
self.Emit("CMPQ", _CX, jit.Imm(types.MAX_RECURSE)) // CMPQ CX, ${types.MAX_RECURSE}
self.Sjmp("JAE" , "_stack_overflow") // JA _stack_overflow
self.Emit("MOVQ", jit.Ptr(_R8, 0), _AX) // MOVQ (R8), AX
self.Emit("MOVQ", _CX, jit.Ptr(_ST, _ST_Sp)) // MOVQ CX, ST.Sp
self.WritePtrAX(3, jit.Sib(_ST, _CX, 8, _ST_Vp), false) // MOVQ AX, ST.Vp[CX]
self.Emit("MOVQ", jit.Imm(_S_arr_0), jit.Sib(_ST, _CX, 8, _ST_Vt)) // MOVQ _S_arr_0, ST.Vt[CX]
self.Sjmp("JMP" , "_next") // JMP _next
/** V_OBJECT **/
self.Link("_decode_V_OBJECT") // _decode_V_OBJECT:
self.Emit("MOVL", jit.Imm(_S_vmask), _DX) // MOVL _S_vmask, DX
self.Emit("MOVQ", jit.Ptr(_ST, _ST_Sp), _CX) // MOVQ ST.Sp, CX
self.Emit("MOVQ", jit.Sib(_ST, _CX, 8, _ST_Vt), _AX) // MOVQ ST.Vt[CX], AX
self.Emit("BTQ" , _AX, _DX) // BTQ AX, DX
self.Sjmp("JNC" , "_invalid_char") // JNC _invalid_char
self.call_go(_F_makemap_small) // CALL_GO runtime.makemap_small
self.Emit("MOVQ", jit.Ptr(_ST, _ST_Sp), _CX) // MOVQ ST.Sp, CX
self.Emit("MOVQ", jit.Imm(_S_obj_0), jit.Sib(_ST, _CX, 8, _ST_Vt)) // MOVQ _S_obj_0, ST.Vt[CX]
self.Emit("MOVQ", jit.Sib(_ST, _CX, 8, _ST_Vp), _SI) // MOVQ ST.Vp[CX], SI
self.Emit("MOVQ", _T_map, _DX) // MOVQ _T_map, DX
self.Emit("MOVQ", _DX, jit.Ptr(_SI, 0)) // MOVQ DX, (SI)
self.WritePtrAX(4, jit.Ptr(_SI, 8), false) // MOVQ AX, 8(SI)
self.Sjmp("JMP" , "_next") // JMP _next
/** V_STRING **/
self.Link("_decode_V_STRING") // _decode_V_STRING:
self.Emit("MOVQ", _VAR_ss_Iv, _CX) // MOVQ ss.Iv, CX
self.Emit("MOVQ", _IC, _AX) // MOVQ IC, AX
self.Emit("SUBQ", _CX, _AX) // SUBQ CX, AX
/* check for escapes */
self.Emit("CMPQ", _VAR_ss_Ep, jit.Imm(-1)) // CMPQ ss.Ep, $-1
self.Sjmp("JNE" , "_unquote") // JNE _unquote
self.Emit("SUBQ", jit.Imm(1), _AX) // SUBQ $1, AX
self.Emit("LEAQ", jit.Sib(_IP, _CX, 1, 0), _R8) // LEAQ (IP)(CX), R8
self.Byte(0x48, 0x8d, 0x3d) // LEAQ (PC), DI
self.Sref("_copy_string_end", 4)
self.Emit("BTQ", jit.Imm(_F_copy_string), _VAR_df)
self.Sjmp("JC", "copy_string")
self.Link("_copy_string_end")
self.Emit("XORL", _DX, _DX)
/* strings with no escape sequences */
self.Link("_noescape") // _noescape:
self.Emit("MOVL", jit.Imm(_S_omask_key), _DI) // MOVL _S_omask, DI
self.Emit("MOVQ", jit.Ptr(_ST, _ST_Sp), _CX) // MOVQ ST.Sp, CX
self.Emit("MOVQ", jit.Sib(_ST, _CX, 8, _ST_Vt), _SI) // MOVQ ST.Vt[CX], SI
self.Emit("BTQ" , _SI, _DI) // BTQ SI, DI
self.Sjmp("JC" , "_object_key") // JC _object_key
/* check for pre-packed strings, avoid 1 allocation */
self.Emit("TESTQ", _DX, _DX) // TESTQ DX, DX
self.Sjmp("JNZ" , "_packed_str") // JNZ _packed_str
self.Emit("MOVQ" , _AX, _BX) // MOVQ AX, BX
self.Emit("MOVQ" , _R8, _AX) // MOVQ R8, AX
self.call_go(_F_convTstring) // CALL_GO runtime.convTstring
self.Emit("MOVQ" , _AX, _R9) // MOVQ AX, R9
/* packed string already in R9 */
self.Link("_packed_str") // _packed_str:
self.Emit("MOVQ", _T_string, _R8) // MOVQ _T_string, R8
self.Emit("MOVQ", _VAR_ss_Iv, _DI) // MOVQ ss.Iv, DI
self.Emit("SUBQ", jit.Imm(1), _DI) // SUBQ $1, DI
self.Sjmp("JMP" , "_set_value") // JMP _set_value
/* the string is an object key, get the map */
self.Link("_object_key")
self.Emit("MOVQ", jit.Ptr(_ST, _ST_Sp), _CX) // MOVQ ST.Sp, CX
self.Emit("MOVQ", jit.Sib(_ST, _CX, 8, _ST_Vp), _SI) // MOVQ ST.Vp[CX], SI
self.Emit("MOVQ", jit.Ptr(_SI, 8), _SI) // MOVQ 8(SI), SI
/* add a new delimiter */
self.Emit("ADDQ", jit.Imm(1), _CX) // ADDQ $1, CX
self.Emit("CMPQ", _CX, jit.Imm(types.MAX_RECURSE)) // CMPQ CX, ${types.MAX_RECURSE}
self.Sjmp("JAE" , "_stack_overflow") // JA _stack_overflow
self.Emit("MOVQ", _CX, jit.Ptr(_ST, _ST_Sp)) // MOVQ CX, ST.Sp
self.Emit("MOVQ", jit.Imm(_S_obj_delim), jit.Sib(_ST, _CX, 8, _ST_Vt)) // MOVQ _S_obj_delim, ST.Vt[CX]
/* add a new slot int the map */
self.Emit("MOVQ", _AX, _DI) // MOVQ AX, DI
self.Emit("MOVQ", _T_map, _AX) // MOVQ _T_map, AX
self.Emit("MOVQ", _SI, _BX) // MOVQ SI, BX
self.Emit("MOVQ", _R8, _CX) // MOVQ R9, CX
self.call_go(_F_mapassign_faststr) // CALL_GO runtime.mapassign_faststr
/* add to the pointer stack */
self.Emit("MOVQ", jit.Ptr(_ST, _ST_Sp), _CX) // MOVQ ST.Sp, CX
self.WritePtrAX(6, jit.Sib(_ST, _CX, 8, _ST_Vp), false) // MOVQ AX, ST.Vp[CX]
self.Sjmp("JMP" , "_next") // JMP _next
/* allocate memory to store the string header and unquoted result */
self.Link("_unquote") // _unquote:
self.Emit("ADDQ", jit.Imm(15), _AX) // ADDQ $15, AX
self.Emit("MOVQ", _T_byte, _BX) // MOVQ _T_byte, BX
self.Emit("MOVB", jit.Imm(0), _CX) // MOVB $0, CX
self.call_go(_F_mallocgc) // CALL_GO runtime.mallocgc
self.Emit("MOVQ", _AX, _R9) // MOVQ AX, R9
/* prepare the unquoting parameters */
self.Emit("MOVQ" , _VAR_ss_Iv, _CX) // MOVQ ss.Iv, CX
self.Emit("LEAQ" , jit.Sib(_IP, _CX, 1, 0), _DI) // LEAQ (IP)(CX), DI
self.Emit("NEGQ" , _CX) // NEGQ CX
self.Emit("LEAQ" , jit.Sib(_IC, _CX, 1, -1), _SI) // LEAQ -1(IC)(CX), SI
self.Emit("LEAQ" , jit.Ptr(_R9, 16), _DX) // LEAQ 16(R8), DX
self.Emit("LEAQ" , _VAR_ss_Ep, _CX) // LEAQ ss.Ep, CX
self.Emit("XORL" , _R8, _R8) // XORL R8, R8
self.Emit("BTQ" , jit.Imm(_F_disable_urc), _VAR_df) // BTQ ${_F_disable_urc}, fv
self.Emit("SETCC", _R8) // SETCC R8
self.Emit("SHLQ" , jit.Imm(types.B_UNICODE_REPLACE), _R8) // SHLQ ${types.B_UNICODE_REPLACE}, R8
/* unquote the string, with R9 been preserved */
self.Emit("MOVQ", _R9, _VAR_R9) // SAVE R9
self.call_c(_F_unquote) // CALL unquote
self.Emit("MOVQ", _VAR_R9, _R9) // LOAD R9
/* check for errors */
self.Emit("TESTQ", _AX, _AX) // TESTQ AX, AX
self.Sjmp("JS" , "_unquote_error") // JS _unquote_error
self.Emit("MOVL" , jit.Imm(1), _DX) // MOVL $1, DX
self.Emit("LEAQ" , jit.Ptr(_R9, 16), _R8) // ADDQ $16, R8
self.Emit("MOVQ" , _R8, jit.Ptr(_R9, 0)) // MOVQ R8, (R9)
self.Emit("MOVQ" , _AX, jit.Ptr(_R9, 8)) // MOVQ AX, 8(R9)
self.Sjmp("JMP" , "_noescape") // JMP _noescape
/** V_DOUBLE **/
self.Link("_decode_V_DOUBLE") // _decode_V_DOUBLE:
self.Emit("BTQ" , jit.Imm(_F_use_number), _VAR_df) // BTQ _F_use_number, df
self.Sjmp("JC" , "_use_number") // JC _use_number
self.Emit("MOVSD", _VAR_ss_Dv, _X0) // MOVSD ss.Dv, X0
self.Sjmp("JMP" , "_use_float64") // JMP _use_float64
/** V_INTEGER **/
self.Link("_decode_V_INTEGER") // _decode_V_INTEGER:
self.Emit("BTQ" , jit.Imm(_F_use_number), _VAR_df) // BTQ _F_use_number, df
self.Sjmp("JC" , "_use_number") // JC _use_number
self.Emit("BTQ" , jit.Imm(_F_use_int64), _VAR_df) // BTQ _F_use_int64, df
self.Sjmp("JC" , "_use_int64") // JC _use_int64
//TODO: use ss.Dv directly
self.Emit("MOVSD", _VAR_ss_Dv, _X0) // MOVSD ss.Dv, X0
/* represent numbers as `float64` */
self.Link("_use_float64") // _use_float64:
self.Emit("MOVQ" , _X0, _AX) // MOVQ X0, AX
self.call_go(_F_convT64) // CALL_GO runtime.convT64
self.Emit("MOVQ" , _T_float64, _R8) // MOVQ _T_float64, R8
self.Emit("MOVQ" , _AX, _R9) // MOVQ AX, R9
self.Emit("MOVQ" , _VAR_ss_Ep, _DI) // MOVQ ss.Ep, DI
self.Sjmp("JMP" , "_set_value") // JMP _set_value
/* represent numbers as `json.Number` */
self.Link("_use_number") // _use_number
self.Emit("MOVQ", _VAR_ss_Ep, _AX) // MOVQ ss.Ep, AX
self.Emit("LEAQ", jit.Sib(_IP, _AX, 1, 0), _SI) // LEAQ (IP)(AX), SI
self.Emit("MOVQ", _IC, _CX) // MOVQ IC, CX
self.Emit("SUBQ", _AX, _CX) // SUBQ AX, CX
self.Emit("MOVQ", _SI, _AX) // MOVQ SI, AX
self.Emit("MOVQ", _CX, _BX) // MOVQ CX, BX
self.call_go(_F_convTstring) // CALL_GO runtime.convTstring
self.Emit("MOVQ", _T_number, _R8) // MOVQ _T_number, R8
self.Emit("MOVQ", _AX, _R9) // MOVQ AX, R9
self.Emit("MOVQ", _VAR_ss_Ep, _DI) // MOVQ ss.Ep, DI
self.Sjmp("JMP" , "_set_value") // JMP _set_value
/* represent numbers as `int64` */
self.Link("_use_int64") // _use_int64:
self.Emit("MOVQ", _VAR_ss_Iv, _AX) // MOVQ ss.Iv, AX
self.call_go(_F_convT64) // CALL_GO runtime.convT64
self.Emit("MOVQ", _T_int64, _R8) // MOVQ _T_int64, R8
self.Emit("MOVQ", _AX, _R9) // MOVQ AX, R9
self.Emit("MOVQ", _VAR_ss_Ep, _DI) // MOVQ ss.Ep, DI
self.Sjmp("JMP" , "_set_value") // JMP _set_value
/** V_KEY_SEP **/
self.Link("_decode_V_KEY_SEP") // _decode_V_KEY_SEP:
self.Emit("MOVQ", jit.Ptr(_ST, _ST_Sp), _CX) // MOVQ ST.Sp, CX
self.Emit("MOVQ", jit.Sib(_ST, _CX, 8, _ST_Vt), _AX) // MOVQ ST.Vt[CX], AX
self.Emit("CMPQ", _AX, jit.Imm(_S_obj_delim)) // CMPQ AX, _S_obj_delim
self.Sjmp("JNE" , "_invalid_char") // JNE _invalid_char
self.Emit("MOVQ", jit.Imm(_S_val), jit.Sib(_ST, _CX, 8, _ST_Vt)) // MOVQ _S_val, ST.Vt[CX]
self.Emit("MOVQ", jit.Imm(_S_obj), jit.Sib(_ST, _CX, 8, _ST_Vt - 8)) // MOVQ _S_obj, ST.Vt[CX - 1]
self.Sjmp("JMP" , "_next") // JMP _next
/** V_ELEM_SEP **/
self.Link("_decode_V_ELEM_SEP") // _decode_V_ELEM_SEP:
self.Emit("MOVQ" , jit.Ptr(_ST, _ST_Sp), _CX) // MOVQ ST.Sp, CX
self.Emit("MOVQ" , jit.Sib(_ST, _CX, 8, _ST_Vt), _AX) // MOVQ ST.Vt[CX], AX
self.Emit("CMPQ" , _AX, jit.Imm(_S_arr))
self.Sjmp("JE" , "_array_sep") // JZ _next
self.Emit("CMPQ" , _AX, jit.Imm(_S_obj)) // CMPQ _AX, _S_arr
self.Sjmp("JNE" , "_invalid_char") // JNE _invalid_char
self.Emit("MOVQ" , jit.Imm(_S_obj_sep), jit.Sib(_ST, _CX, 8, _ST_Vt))
self.Sjmp("JMP" , "_next") // JMP _next
/* arrays */
self.Link("_array_sep")
self.Emit("MOVQ", jit.Sib(_ST, _CX, 8, _ST_Vp), _SI) // MOVQ ST.Vp[CX], SI
self.Emit("MOVQ", jit.Ptr(_SI, 8), _SI) // MOVQ 8(SI), SI
self.Emit("MOVQ", jit.Ptr(_SI, 8), _DX) // MOVQ 8(SI), DX
self.Emit("CMPQ", _DX, jit.Ptr(_SI, 16)) // CMPQ DX, 16(SI)
self.Sjmp("JAE" , "_array_more") // JAE _array_more
/* add a slot for the new element */
self.Link("_array_append") // _array_append:
self.Emit("ADDQ", jit.Imm(1), jit.Ptr(_SI, 8)) // ADDQ $1, 8(SI)
self.Emit("MOVQ", jit.Ptr(_SI, 0), _SI) // MOVQ (SI), SI
self.Emit("ADDQ", jit.Imm(1), _CX) // ADDQ $1, CX
self.Emit("CMPQ", _CX, jit.Imm(types.MAX_RECURSE)) // CMPQ CX, ${types.MAX_RECURSE}
self.Sjmp("JAE" , "_stack_overflow") // JA _stack_overflow
self.Emit("SHLQ", jit.Imm(1), _DX) // SHLQ $1, DX
self.Emit("LEAQ", jit.Sib(_SI, _DX, 8, 0), _SI) // LEAQ (SI)(DX*8), SI
self.Emit("MOVQ", _CX, jit.Ptr(_ST, _ST_Sp)) // MOVQ CX, ST.Sp
self.WriteRecNotAX(7 , _SI, jit.Sib(_ST, _CX, 8, _ST_Vp), false) // MOVQ SI, ST.Vp[CX]
self.Emit("MOVQ", jit.Imm(_S_val), jit.Sib(_ST, _CX, 8, _ST_Vt)) // MOVQ _S_val, ST.Vt[CX}
self.Sjmp("JMP" , "_next") // JMP _next
/** V_ARRAY_END **/
self.Link("_decode_V_ARRAY_END") // _decode_V_ARRAY_END:
self.Emit("XORL", _DX, _DX) // XORL DX, DX
self.Emit("MOVQ", jit.Ptr(_ST, _ST_Sp), _CX) // MOVQ ST.Sp, CX
self.Emit("MOVQ", jit.Sib(_ST, _CX, 8, _ST_Vt), _AX) // MOVQ ST.Vt[CX], AX
self.Emit("CMPQ", _AX, jit.Imm(_S_arr_0)) // CMPQ AX, _S_arr_0
self.Sjmp("JE" , "_first_item") // JE _first_item
self.Emit("CMPQ", _AX, jit.Imm(_S_arr)) // CMPQ AX, _S_arr
self.Sjmp("JNE" , "_invalid_char") // JNE _invalid_char
self.Emit("SUBQ", jit.Imm(1), jit.Ptr(_ST, _ST_Sp)) // SUBQ $1, ST.Sp
self.Emit("MOVQ", _DX, jit.Sib(_ST, _CX, 8, _ST_Vp)) // MOVQ DX, ST.Vp[CX]
self.Sjmp("JMP" , "_next") // JMP _next
/* first element of an array */
self.Link("_first_item") // _first_item:
self.Emit("MOVQ", jit.Ptr(_ST, _ST_Sp), _CX) // MOVQ ST.Sp, CX
self.Emit("SUBQ", jit.Imm(2), jit.Ptr(_ST, _ST_Sp)) // SUBQ $2, ST.Sp
self.Emit("MOVQ", jit.Sib(_ST, _CX, 8, _ST_Vp - 8), _SI) // MOVQ ST.Vp[CX - 1], SI
self.Emit("MOVQ", jit.Ptr(_SI, 8), _SI) // MOVQ 8(SI), SI
self.Emit("MOVQ", _DX, jit.Sib(_ST, _CX, 8, _ST_Vp - 8)) // MOVQ DX, ST.Vp[CX - 1]
self.Emit("MOVQ", _DX, jit.Sib(_ST, _CX, 8, _ST_Vp)) // MOVQ DX, ST.Vp[CX]
self.Emit("MOVQ", _DX, jit.Ptr(_SI, 8)) // MOVQ DX, 8(SI)
self.Sjmp("JMP" , "_next") // JMP _next
/** V_OBJECT_END **/
self.Link("_decode_V_OBJECT_END") // _decode_V_OBJECT_END:
self.Emit("MOVL", jit.Imm(_S_omask_end), _DI) // MOVL _S_omask, DI
self.Emit("MOVQ", jit.Ptr(_ST, _ST_Sp), _CX) // MOVQ ST.Sp, CX
self.Emit("MOVQ", jit.Sib(_ST, _CX, 8, _ST_Vt), _AX) // MOVQ ST.Vt[CX], AX
self.Emit("BTQ" , _AX, _DI)
self.Sjmp("JNC" , "_invalid_char") // JNE _invalid_char
self.Emit("XORL", _AX, _AX) // XORL AX, AX
self.Emit("SUBQ", jit.Imm(1), jit.Ptr(_ST, _ST_Sp)) // SUBQ $1, ST.Sp
self.Emit("MOVQ", _AX, jit.Sib(_ST, _CX, 8, _ST_Vp)) // MOVQ AX, ST.Vp[CX]
self.Sjmp("JMP" , "_next") // JMP _next
/* return from decoder */
self.Link("_return") // _return:
self.Emit("XORL", _EP, _EP) // XORL EP, EP
self.Emit("MOVQ", _EP, jit.Ptr(_ST, _ST_Vp)) // MOVQ EP, ST.Vp[0]
self.Link("_epilogue") // _epilogue:
self.Emit("SUBQ", jit.Imm(_FsmOffset), _ST) // SUBQ _FsmOffset, _ST
self.Emit("MOVQ", jit.Ptr(_SP, _VD_offs), _BP) // MOVQ _VD_offs(SP), BP
self.Emit("ADDQ", jit.Imm(_VD_size), _SP) // ADDQ $_VD_size, SP
self.Emit("RET") // RET
/* array expand */
self.Link("_array_more") // _array_more:
self.Emit("MOVQ" , _T_eface, _AX) // MOVQ _T_eface, AX
self.Emit("MOVQ" , jit.Ptr(_SI, 0), _BX) // MOVQ (SI), BX
self.Emit("MOVQ" , jit.Ptr(_SI, 8), _CX) // MOVQ 8(SI), CX
self.Emit("MOVQ" , jit.Ptr(_SI, 16), _DI) // MOVQ 16(SI), DI
self.Emit("MOVQ" , _DI, _SI) // MOVQ DI, 24(SP)
self.Emit("SHLQ" , jit.Imm(1), _SI) // SHLQ $1, SI
self.call_go(_F_growslice) // CALL_GO runtime.growslice
self.Emit("MOVQ" , _AX, _DI) // MOVQ AX, DI
self.Emit("MOVQ" , _BX, _DX) // MOVQ BX, DX
self.Emit("MOVQ" , _CX, _AX) // MOVQ CX, AX
/* update the slice */
self.Emit("MOVQ", jit.Ptr(_ST, _ST_Sp), _CX) // MOVQ ST.Sp, CX
self.Emit("MOVQ", jit.Sib(_ST, _CX, 8, _ST_Vp), _SI) // MOVQ ST.Vp[CX], SI
self.Emit("MOVQ", jit.Ptr(_SI, 8), _SI) // MOVQ 8(SI), SI
self.Emit("MOVQ", _DX, jit.Ptr(_SI, 8)) // MOVQ DX, 8(SI)
self.Emit("MOVQ", _AX, jit.Ptr(_SI, 16)) // MOVQ AX, 16(AX)
self.WriteRecNotAX(8 , _DI, jit.Ptr(_SI, 0), false) // MOVQ R10, (SI)
self.Sjmp("JMP" , "_array_append") // JMP _array_append
/* copy string */
self.Link("copy_string") // pointer: R8, length: AX, return addr: DI
self.Emit("MOVQ", _R8, _VAR_cs_p)
self.Emit("MOVQ", _AX, _VAR_cs_n)
self.Emit("MOVQ", _DI, _VAR_cs_LR)
self.Emit("MOVQ", _AX, _BX)
self.Emit("MOVQ", _AX, _CX)
self.Emit("MOVQ", _T_byte, _AX)
self.call_go(_F_makeslice)
self.Emit("MOVQ", _AX, _VAR_cs_d)
self.Emit("MOVQ", _VAR_cs_p, _BX)
self.Emit("MOVQ", _VAR_cs_n, _CX)
self.call_go(_F_memmove)
self.Emit("MOVQ", _VAR_cs_d, _R8)
self.Emit("MOVQ", _VAR_cs_n, _AX)
self.Emit("MOVQ", _VAR_cs_LR, _DI)
self.Rjmp("JMP", _DI)
/* error handlers */
self.Link("_stack_overflow")
self.Emit("MOVL" , _E_recurse, _EP) // MOVQ _E_recurse, EP
self.Sjmp("JMP" , "_error") // JMP _error
self.Link("_vtype_error") // _vtype_error:
self.Emit("MOVQ" , _DI, _IC) // MOVQ DI, IC
self.Emit("MOVL" , _E_invalid, _EP) // MOVL _E_invalid, EP
self.Sjmp("JMP" , "_error") // JMP _error
self.Link("_invalid_char") // _invalid_char:
self.Emit("SUBQ" , jit.Imm(1), _IC) // SUBQ $1, IC
self.Emit("MOVL" , _E_invalid, _EP) // MOVL _E_invalid, EP
self.Sjmp("JMP" , "_error") // JMP _error
self.Link("_unquote_error") // _unquote_error:
self.Emit("MOVQ" , _VAR_ss_Iv, _IC) // MOVQ ss.Iv, IC
self.Emit("SUBQ" , jit.Imm(1), _IC) // SUBQ $1, IC
self.Link("_parsing_error") // _parsing_error:
self.Emit("NEGQ" , _AX) // NEGQ AX
self.Emit("MOVQ" , _AX, _EP) // MOVQ AX, EP
self.Link("_error") // _error:
self.Emit("PXOR" , _X0, _X0) // PXOR X0, X0
self.Emit("MOVOU", _X0, jit.Ptr(_VP, 0)) // MOVOU X0, (VP)
self.Sjmp("JMP" , "_epilogue") // JMP _epilogue
/* invalid value type, never returns */
self.Link("_invalid_vtype")
self.call_go(_F_invalid_vtype) // CALL invalid_type
self.Emit("UD2") // UD2
/* switch jump table */
self.Link("_switch_table") // _switch_table:
self.Sref("_decode_V_EOF", 0) // SREF &_decode_V_EOF, $0
self.Sref("_decode_V_NULL", -4) // SREF &_decode_V_NULL, $-4
self.Sref("_decode_V_TRUE", -8) // SREF &_decode_V_TRUE, $-8
self.Sref("_decode_V_FALSE", -12) // SREF &_decode_V_FALSE, $-12
self.Sref("_decode_V_ARRAY", -16) // SREF &_decode_V_ARRAY, $-16
self.Sref("_decode_V_OBJECT", -20) // SREF &_decode_V_OBJECT, $-20
self.Sref("_decode_V_STRING", -24) // SREF &_decode_V_STRING, $-24
self.Sref("_decode_V_DOUBLE", -28) // SREF &_decode_V_DOUBLE, $-28
self.Sref("_decode_V_INTEGER", -32) // SREF &_decode_V_INTEGER, $-32
self.Sref("_decode_V_KEY_SEP", -36) // SREF &_decode_V_KEY_SEP, $-36
self.Sref("_decode_V_ELEM_SEP", -40) // SREF &_decode_V_ELEM_SEP, $-40
self.Sref("_decode_V_ARRAY_END", -44) // SREF &_decode_V_ARRAY_END, $-44
self.Sref("_decode_V_OBJECT_END", -48) // SREF &_decode_V_OBJECT_END, $-48
/* fast character lookup table */
self.Link("_decode_tab") // _decode_tab:
self.Sref("_decode_V_EOF", 0) // SREF &_decode_V_EOF, $0
/* generate rest of the tabs */
for i := 1; i < 256; i++ {
if to, ok := _R_tab[i]; ok {
self.Sref(to, -int64(i) * 4)
} else {
self.Byte(0x00, 0x00, 0x00, 0x00)
}
}
}
func (self *_ValueDecoder) WritePtrAX(i int, rec obj.Addr, saveDI bool) {
self.Emit("MOVQ", _V_writeBarrier, _R9)
self.Emit("CMPL", jit.Ptr(_R9, 0), jit.Imm(0))
self.Sjmp("JE", "_no_writeBarrier" + strconv.Itoa(i) + "_{n}")
if saveDI {
self.save(_DI)
}
self.Emit("LEAQ", rec, _DI)
self.call(_F_gcWriteBarrierAX)
if saveDI {
self.load(_DI)
}
self.Sjmp("JMP", "_end_writeBarrier" + strconv.Itoa(i) + "_{n}")
self.Link("_no_writeBarrier" + strconv.Itoa(i) + "_{n}")
self.Emit("MOVQ", _AX, rec)
self.Link("_end_writeBarrier" + strconv.Itoa(i) + "_{n}")
}
func (self *_ValueDecoder) WriteRecNotAX(i int, ptr obj.Addr, rec obj.Addr, saveDI bool) {
if rec.Reg == x86.REG_AX || rec.Index == x86.REG_AX {
panic("rec contains AX!")
}
self.Emit("MOVQ", _V_writeBarrier, _AX)
self.Emit("CMPL", jit.Ptr(_AX, 0), jit.Imm(0))
self.Sjmp("JE", "_no_writeBarrier" + strconv.Itoa(i) + "_{n}")
self.Emit("MOVQ", ptr, _AX)
if saveDI {
self.save(_DI)
}
self.Emit("LEAQ", rec, _DI)
self.call(_F_gcWriteBarrierAX)
if saveDI {
self.load(_DI)
}
self.Sjmp("JMP", "_end_writeBarrier" + strconv.Itoa(i) + "_{n}")
self.Link("_no_writeBarrier" + strconv.Itoa(i) + "_{n}")
self.Emit("MOVQ", ptr, rec)
self.Link("_end_writeBarrier" + strconv.Itoa(i) + "_{n}")
}
/** Generic Decoder **/
var (
_subr_decode_value = new(_ValueDecoder).build()
)
//go:nosplit
func invalid_vtype(vt types.ValueType) {
throw(fmt.Sprintf("invalid value type: %d", vt))
}
vendor/github.com/bytedance/sonic/internal/decoder/generic_amd64_go117_test.s
Generated Vendored
+37
View File
@@ -0,0 +1,37 @@
// +build go1.17,!go1.21
//
// Copyright 2021 ByteDance Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
#include "go_asm.h"
#include "funcdata.h"
#include "textflag.h"
TEXT ·decodeValueStub(SB), NOSPLIT, $0 - 72
NO_LOCAL_POINTERS
PXOR X0, X0
MOVOU X0, rv+48(FP)
MOVQ st+0(FP) , R13
MOVQ sp+8(FP) , R10
MOVQ sn+16(FP), R12
MOVQ ic+24(FP), R11
MOVQ vp+32(FP), R15
MOVQ df+40(FP), AX
MOVQ ·_subr_decode_value(SB), BX
CALL BX
MOVQ R11, rp+48(FP)
MOVQ BX, ex+56(FP)
RET
vendor/github.com/bytedance/sonic/internal/decoder/generic_amd64_test.s
Generated Vendored
+37
View File
@@ -0,0 +1,37 @@
// +build go1.15,!go1.17
//
// Copyright 2021 ByteDance Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
#include "go_asm.h"
#include "funcdata.h"
#include "textflag.h"
TEXT ·decodeValueStub(SB), NOSPLIT, $0 - 72
NO_LOCAL_POINTERS
PXOR X0, X0
MOVOU X0, rv+48(FP)
MOVQ st+0(FP), BX
MOVQ sp+8(FP), R12
MOVQ sn+16(FP), R13
MOVQ ic+24(FP), R14
MOVQ vp+32(FP), R15
MOVQ df+40(FP), R10
MOVQ ·_subr_decode_value(SB), AX
CALL AX
MOVQ R14, rp+48(FP)
MOVQ R11, ex+56(FP)
RET
vendor/github.com/bytedance/sonic/internal/decoder/pools.go
Generated Vendored
+143
View File
@@ -0,0 +1,143 @@
/*
* Copyright 2021 ByteDance Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package decoder
import (
`sync`
`unsafe`
`github.com/bytedance/sonic/internal/caching`
`github.com/bytedance/sonic/internal/native/types`
`github.com/bytedance/sonic/internal/rt`
)
const (
_MinSlice = 2
_MaxStack = 4096 // 4k slots
_MaxStackBytes = _MaxStack * _PtrBytes
_MaxDigitNums = 800 // used in atof fallback algorithm
)
const (
_PtrBytes = _PTR_SIZE / 8
_FsmOffset = (_MaxStack + 1) * _PtrBytes
_DbufOffset = _FsmOffset + int64(unsafe.Sizeof(types.StateMachine{})) + types.MAX_RECURSE * _PtrBytes
_StackSize = unsafe.Sizeof(_Stack{})
)
var (
stackPool = sync.Pool{}
valueCache = []unsafe.Pointer(nil)
fieldCache = []*caching.FieldMap(nil)
fieldCacheMux = sync.Mutex{}
programCache = caching.CreateProgramCache()
)
type _Stack struct {
sp uintptr
sb [_MaxStack]unsafe.Pointer
mm types.StateMachine
vp [types.MAX_RECURSE]unsafe.Pointer
dp [_MaxDigitNums]byte
}
type _Decoder func(
s string,
i int,
vp unsafe.Pointer,
sb *_Stack,
fv uint64,
sv string, // DO NOT pass value to this arguement, since it is only used for local _VAR_sv
vk unsafe.Pointer, // DO NOT pass value to this arguement, since it is only used for local _VAR_vk
) (int, error)
var _KeepAlive struct {
s string
i int
vp unsafe.Pointer
sb *_Stack
fv uint64
sv string
vk unsafe.Pointer
ret int
err error
frame_decoder [_FP_offs]byte
frame_generic [_VD_offs]byte
}
var (
argPtrs = []bool{true, false, false, true, true, false, true, false, true}
localPtrs = []bool{}
)
var (
argPtrs_generic = []bool{true}
localPtrs_generic = []bool{}
)
func newStack() *_Stack {
if ret := stackPool.Get(); ret == nil {
return new(_Stack)
} else {
return ret.(*_Stack)
}
}
func resetStack(p *_Stack) {
memclrNoHeapPointers(unsafe.Pointer(p), _StackSize)
}
func freeStack(p *_Stack) {
p.sp = 0
stackPool.Put(p)
}
func freezeValue(v unsafe.Pointer) uintptr {
valueCache = append(valueCache, v)
return uintptr(v)
}
func freezeFields(v *caching.FieldMap) int64 {
fieldCacheMux.Lock()
fieldCache = append(fieldCache, v)
fieldCacheMux.Unlock()
return referenceFields(v)
}
func referenceFields(v *caching.FieldMap) int64 {
return int64(uintptr(unsafe.Pointer(v)))
}
func makeDecoder(vt *rt.GoType, _ ...interface{}) (interface{}, error) {
if pp, err := newCompiler().compile(vt.Pack()); err != nil {
return nil, err
} else {
return newAssembler(pp).Load(), nil
}
}
func findOrCompile(vt *rt.GoType) (_Decoder, error) {
if val := programCache.Get(vt); val != nil {
return val.(_Decoder), nil
} else if ret, err := programCache.Compute(vt, makeDecoder); err == nil {
return ret.(_Decoder), nil
} else {
return nil, err
}
}
vendor/github.com/bytedance/sonic/internal/decoder/primitives.go
Generated Vendored
+46
View File
@@ -0,0 +1,46 @@
/*
* Copyright 2021 ByteDance Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package decoder
import (
`encoding`
`encoding/json`
`unsafe`
`github.com/bytedance/sonic/internal/native`
`github.com/bytedance/sonic/internal/rt`
)
func decodeTypedPointer(s string, i int, vt *rt.GoType, vp unsafe.Pointer, sb *_Stack, fv uint64) (int, error) {
if fn, err := findOrCompile(vt); err != nil {
return 0, err
} else {
rt.MoreStack(_FP_size + _VD_size + native.MaxFrameSize)
rt.StopProf()
ret, err := fn(s, i, vp, sb, fv, "", nil)
rt.StartProf()
return ret, err
}
}
func decodeJsonUnmarshaler(vv interface{}, s string) error {
return vv.(json.Unmarshaler).UnmarshalJSON(rt.Str2Mem(s))
}
func decodeTextUnmarshaler(vv interface{}, s string) error {
return vv.(encoding.TextUnmarshaler).UnmarshalText(rt.Str2Mem(s))
}
vendor/github.com/bytedance/sonic/internal/decoder/stream.go
Generated Vendored
+228
View File
@@ -0,0 +1,228 @@
/*
* Copyright 2021 ByteDance Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package decoder
import (
`bytes`
`io`
`sync`
`github.com/bytedance/sonic/internal/native`
`github.com/bytedance/sonic/internal/native/types`
`github.com/bytedance/sonic/option`
)
var (
minLeftBufferShift uint = 1
)
// StreamDecoder is the decoder context object for streaming input.
type StreamDecoder struct {
r io.Reader
buf []byte
scanp int
scanned int64
err error
Decoder
}
var bufPool = sync.Pool{
New: func () interface{} {
return make([]byte, 0, option.DefaultDecoderBufferSize)
},
}
// NewStreamDecoder adapts to encoding/json.NewDecoder API.
//
// NewStreamDecoder returns a new decoder that reads from r.
func NewStreamDecoder(r io.Reader) *StreamDecoder {
return &StreamDecoder{r : r}
}
// Decode decodes input stream into val with corresponding data.
// Redundantly bytes may be read and left in its buffer, and can be used at next call.
// Either io error from underlying io.Reader (except io.EOF)
// or syntax error from data will be recorded and stop subsequently decoding.
func (self *StreamDecoder) Decode(val interface{}) (err error) {
if self.err != nil {
return self.err
}
var buf = self.buf[self.scanp:]
var p = 0
var recycle bool
if cap(buf) == 0 {
buf = bufPool.Get().([]byte)
recycle = true
}
var first = true
var repeat = true
read_more:
for {
l := len(buf)
realloc(&buf)
n, err := self.r.Read(buf[l:cap(buf)])
buf = buf[:l+n]
if err != nil {
repeat = false
if err == io.EOF {
if len(buf) == 0 {
return err
}
break
}
self.err = err
return err
}
if n > 0 || first {
break
}
}
first = false
l := len(buf)
if l > 0 {
self.Decoder.Reset(string(buf))
var x int
if ret := native.SkipOneFast(&self.s, &x); ret < 0 {
if repeat {
goto read_more
} else {
err = SyntaxError{x, self.s, types.ParsingError(-ret), ""}
self.err = err
return
}
}
err = self.Decoder.Decode(val)
if err != nil {
self.err = err
}
p = self.Decoder.Pos()
self.scanned += int64(p)
self.scanp = 0
}
if l > p {
// remain undecoded bytes, so copy them into self.buf
self.buf = append(self.buf[:0], buf[p:]...)
} else {
self.buf = nil
recycle = true
}
if recycle {
buf = buf[:0]
bufPool.Put(buf)
}
return err
}
func (self StreamDecoder) repeatable(err error) bool {
if ee, ok := err.(SyntaxError); ok &&
(ee.Code == types.ERR_EOF || (ee.Code == types.ERR_INVALID_CHAR && self.i >= len(self.s)-1)) {
return true
}
return false
}
// InputOffset returns the input stream byte offset of the current decoder position.
// The offset gives the location of the end of the most recently returned token and the beginning of the next token.
func (self *StreamDecoder) InputOffset() int64 {
return self.scanned + int64(self.scanp)
}
// Buffered returns a reader of the data remaining in the Decoder's buffer.
// The reader is valid until the next call to Decode.
func (self *StreamDecoder) Buffered() io.Reader {
return bytes.NewReader(self.buf[self.scanp:])
}
// More reports whether there is another element in the
// current array or object being parsed.
func (self *StreamDecoder) More() bool {
if self.err != nil {
return false
}
c, err := self.peek()
return err == nil && c != ']' && c != '}'
}
func (self *StreamDecoder) peek() (byte, error) {
var err error
for {
for i := self.scanp; i < len(self.buf); i++ {
c := self.buf[i]
if isSpace(c) {
continue
}
self.scanp = i
return c, nil
}
// buffer has been scanned, now report any error
if err != nil {
if err != io.EOF {
self.err = err
}
return 0, err
}
err = self.refill()
}
}
func isSpace(c byte) bool {
return types.SPACE_MASK & (1 << c) != 0
}
func (self *StreamDecoder) refill() error {
// Make room to read more into the buffer.
// First slide down data already consumed.
if self.scanp > 0 {
self.scanned += int64(self.scanp)
n := copy(self.buf, self.buf[self.scanp:])
self.buf = self.buf[:n]
self.scanp = 0
}
// Grow buffer if not large enough.
realloc(&self.buf)
// Read. Delay error for next iteration (after scan).
n, err := self.r.Read(self.buf[len(self.buf):cap(self.buf)])
self.buf = self.buf[0 : len(self.buf)+n]
return err
}
func realloc(buf *[]byte) {
l := uint(len(*buf))
c := uint(cap(*buf))
if c - l <= c >> minLeftBufferShift {
e := l+(l>>minLeftBufferShift)
if e < option.DefaultDecoderBufferSize {
e = option.DefaultDecoderBufferSize
}
tmp := make([]byte, l, e)
copy(tmp, *buf)
*buf = tmp
}
}
vendor/github.com/bytedance/sonic/internal/decoder/stubs_go115.go
Generated Vendored
+111
View File
@@ -0,0 +1,111 @@
// +build go1.15,!go1.20
/*
* Copyright 2021 ByteDance Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package decoder
import (
`unsafe`
`reflect`
_ `github.com/chenzhuoyu/base64x`
`github.com/bytedance/sonic/internal/rt`
)
//go:linkname _subr__b64decode github.com/chenzhuoyu/base64x._subr__b64decode
var _subr__b64decode uintptr
// runtime.maxElementSize
const _max_map_element_size uintptr = 128
func mapfast(vt reflect.Type) bool {
return vt.Elem().Size() <= _max_map_element_size
}
//go:nosplit
//go:linkname throw runtime.throw
//goland:noinspection GoUnusedParameter
func throw(s string)
//go:linkname convT64 runtime.convT64
//goland:noinspection GoUnusedParameter
func convT64(v uint64) unsafe.Pointer
//go:linkname convTslice runtime.convTslice
//goland:noinspection GoUnusedParameter
func convTslice(v []byte) unsafe.Pointer
//go:linkname convTstring runtime.convTstring
//goland:noinspection GoUnusedParameter
func convTstring(v string) unsafe.Pointer
//go:noescape
//go:linkname memequal runtime.memequal
//goland:noinspection GoUnusedParameter
func memequal(a unsafe.Pointer, b unsafe.Pointer, size uintptr) bool
//go:noescape
//go:linkname memmove runtime.memmove
//goland:noinspection GoUnusedParameter
func memmove(to unsafe.Pointer, from unsafe.Pointer, n uintptr)
//go:linkname mallocgc runtime.mallocgc
//goland:noinspection GoUnusedParameter
func mallocgc(size uintptr, typ *rt.GoType, needzero bool) unsafe.Pointer
//go:linkname makeslice runtime.makeslice
//goland:noinspection GoUnusedParameter
func makeslice(et *rt.GoType, len int, cap int) unsafe.Pointer
//go:noescape
//go:linkname growslice runtime.growslice
//goland:noinspection GoUnusedParameter
func growslice(et *rt.GoType, old rt.GoSlice, cap int) rt.GoSlice
//go:linkname makemap_small runtime.makemap_small
func makemap_small() unsafe.Pointer
//go:linkname mapassign runtime.mapassign
//goland:noinspection GoUnusedParameter
func mapassign(t *rt.GoType, h unsafe.Pointer, k unsafe.Pointer) unsafe.Pointer
//go:linkname mapassign_fast32 runtime.mapassign_fast32
//goland:noinspection GoUnusedParameter
func mapassign_fast32(t *rt.GoType, h unsafe.Pointer, k uint32) unsafe.Pointer
//go:linkname mapassign_fast64 runtime.mapassign_fast64
//goland:noinspection GoUnusedParameter
func mapassign_fast64(t *rt.GoType, h unsafe.Pointer, k uint64) unsafe.Pointer
//go:linkname mapassign_fast64ptr runtime.mapassign_fast64ptr
//goland:noinspection GoUnusedParameter
func mapassign_fast64ptr(t *rt.GoType, h unsafe.Pointer, k unsafe.Pointer) unsafe.Pointer
//go:linkname mapassign_faststr runtime.mapassign_faststr
//goland:noinspection GoUnusedParameter
func mapassign_faststr(t *rt.GoType, h unsafe.Pointer, s string) unsafe.Pointer
//go:nosplit
//go:linkname memclrHasPointers runtime.memclrHasPointers
//goland:noinspection GoUnusedParameter
func memclrHasPointers(ptr unsafe.Pointer, n uintptr)
//go:noescape
//go:linkname memclrNoHeapPointers runtime.memclrNoHeapPointers
//goland:noinspection GoUnusedParameter
func memclrNoHeapPointers(ptr unsafe.Pointer, n uintptr)
vendor/github.com/bytedance/sonic/internal/decoder/stubs_go120.go
Generated Vendored
+111
View File
@@ -0,0 +1,111 @@
// +build go1.20
/*
* Copyright 2021 ByteDance Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package decoder
import (
`unsafe`
`reflect`
_ `github.com/chenzhuoyu/base64x`
`github.com/bytedance/sonic/internal/rt`
)
//go:linkname _subr__b64decode github.com/chenzhuoyu/base64x._subr__b64decode
var _subr__b64decode uintptr
// runtime.maxElementSize
const _max_map_element_size uintptr = 128
func mapfast(vt reflect.Type) bool {
return vt.Elem().Size() <= _max_map_element_size
}
//go:nosplit
//go:linkname throw runtime.throw
//goland:noinspection GoUnusedParameter
func throw(s string)
//go:linkname convT64 runtime.convT64
//goland:noinspection GoUnusedParameter
func convT64(v uint64) unsafe.Pointer
//go:linkname convTslice runtime.convTslice
//goland:noinspection GoUnusedParameter
func convTslice(v []byte) unsafe.Pointer
//go:linkname convTstring runtime.convTstring
//goland:noinspection GoUnusedParameter
func convTstring(v string) unsafe.Pointer
//go:noescape
//go:linkname memequal runtime.memequal
//goland:noinspection GoUnusedParameter
func memequal(a unsafe.Pointer, b unsafe.Pointer, size uintptr) bool
//go:noescape
//go:linkname memmove runtime.memmove
//goland:noinspection GoUnusedParameter
func memmove(to unsafe.Pointer, from unsafe.Pointer, n uintptr)
//go:linkname mallocgc runtime.mallocgc
//goland:noinspection GoUnusedParameter
func mallocgc(size uintptr, typ *rt.GoType, needzero bool) unsafe.Pointer
//go:linkname makeslice runtime.makeslice
//goland:noinspection GoUnusedParameter
func makeslice(et *rt.GoType, len int, cap int) unsafe.Pointer
//go:noescape
//go:linkname growslice reflect.growslice
//goland:noinspection GoUnusedParameter
func growslice(et *rt.GoType, old rt.GoSlice, cap int) rt.GoSlice
//go:linkname makemap_small runtime.makemap_small
func makemap_small() unsafe.Pointer
//go:linkname mapassign runtime.mapassign
//goland:noinspection GoUnusedParameter
func mapassign(t *rt.GoType, h unsafe.Pointer, k unsafe.Pointer) unsafe.Pointer
//go:linkname mapassign_fast32 runtime.mapassign_fast32
//goland:noinspection GoUnusedParameter
func mapassign_fast32(t *rt.GoType, h unsafe.Pointer, k uint32) unsafe.Pointer
//go:linkname mapassign_fast64 runtime.mapassign_fast64
//goland:noinspection GoUnusedParameter
func mapassign_fast64(t *rt.GoType, h unsafe.Pointer, k uint64) unsafe.Pointer
//go:linkname mapassign_fast64ptr runtime.mapassign_fast64ptr
//goland:noinspection GoUnusedParameter
func mapassign_fast64ptr(t *rt.GoType, h unsafe.Pointer, k unsafe.Pointer) unsafe.Pointer
//go:linkname mapassign_faststr runtime.mapassign_faststr
//goland:noinspection GoUnusedParameter
func mapassign_faststr(t *rt.GoType, h unsafe.Pointer, s string) unsafe.Pointer
//go:nosplit
//go:linkname memclrHasPointers runtime.memclrHasPointers
//goland:noinspection GoUnusedParameter
func memclrHasPointers(ptr unsafe.Pointer, n uintptr)
//go:noescape
//go:linkname memclrNoHeapPointers runtime.memclrNoHeapPointers
//goland:noinspection GoUnusedParameter
func memclrNoHeapPointers(ptr unsafe.Pointer, n uintptr)
vendor/github.com/bytedance/sonic/internal/decoder/types.go
Generated Vendored
+58
View File
@@ -0,0 +1,58 @@
/*
* Copyright 2021 ByteDance Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package decoder
import (
`encoding`
`encoding/base64`
`encoding/json`
`reflect`
`unsafe`
`github.com/bytedance/sonic/internal/rt`
)
var (
byteType = reflect.TypeOf(byte(0))
intType = reflect.TypeOf(int(0))
int8Type = reflect.TypeOf(int8(0))
int16Type = reflect.TypeOf(int16(0))
int32Type = reflect.TypeOf(int32(0))
int64Type = reflect.TypeOf(int64(0))
uintType = reflect.TypeOf(uint(0))
uint8Type = reflect.TypeOf(uint8(0))
uint16Type = reflect.TypeOf(uint16(0))
uint32Type = reflect.TypeOf(uint32(0))
uint64Type = reflect.TypeOf(uint64(0))
float32Type = reflect.TypeOf(float32(0))
float64Type = reflect.TypeOf(float64(0))
stringType = reflect.TypeOf("")
bytesType = reflect.TypeOf([]byte(nil))
jsonNumberType = reflect.TypeOf(json.Number(""))
base64CorruptInputError = reflect.TypeOf(base64.CorruptInputError(0))
)
var (
errorType = reflect.TypeOf((*error)(nil)).Elem()
jsonUnmarshalerType = reflect.TypeOf((*json.Unmarshaler)(nil)).Elem()
encodingTextUnmarshalerType = reflect.TypeOf((*encoding.TextUnmarshaler)(nil)).Elem()
)
func rtype(t reflect.Type) (*rt.GoItab, *rt.GoType) {
p := (*rt.GoIface)(unsafe.Pointer(&t))
return p.Itab, (*rt.GoType)(p.Value)
}
vendor/github.com/bytedance/sonic/internal/decoder/utils.go
Generated Vendored
+39
View File
@@ -0,0 +1,39 @@
/*
* Copyright 2021 ByteDance Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package decoder
import (
`unsafe`
`github.com/bytedance/sonic/loader`
)
//go:nosplit
func pbool(v bool) uintptr {
return freezeValue(unsafe.Pointer(&v))
}
//go:nosplit
func ptodec(p loader.Function) _Decoder {
return *(*_Decoder)(unsafe.Pointer(&p))
}
func assert_eq(v int64, exp int64, msg string) {
if v != exp {
panic(msg)
}
}
vendor/github.com/bytedance/sonic/internal/encoder/asm.s
Generated Vendored
View File
vendor/github.com/bytedance/sonic/internal/encoder/assembler_amd64_go116.go
Generated Vendored
+1199
View File
File diff suppressed because it is too large Load Diff
vendor/github.com/bytedance/sonic/internal/encoder/assembler_amd64_go117.go
Generated Vendored
+1202
View File
File diff suppressed because it is too large Load Diff
vendor/github.com/bytedance/sonic/internal/encoder/compiler.go
Generated Vendored
+885
View File
@@ -0,0 +1,885 @@
/*
* Copyright 2021 ByteDance Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package encoder
import (
`fmt`
`reflect`
`strconv`
`strings`
`unsafe`
`github.com/bytedance/sonic/internal/resolver`
`github.com/bytedance/sonic/internal/rt`
`github.com/bytedance/sonic/option`
)
type _Op uint8
const (
_OP_null _Op = iota + 1
_OP_empty_arr
_OP_empty_obj
_OP_bool
_OP_i8
_OP_i16
_OP_i32
_OP_i64
_OP_u8
_OP_u16
_OP_u32
_OP_u64
_OP_f32
_OP_f64
_OP_str
_OP_bin
_OP_quote
_OP_number
_OP_eface
_OP_iface
_OP_byte
_OP_text
_OP_deref
_OP_index
_OP_load
_OP_save
_OP_drop
_OP_drop_2
_OP_recurse
_OP_is_nil
_OP_is_nil_p1
_OP_is_zero_1
_OP_is_zero_2
_OP_is_zero_4
_OP_is_zero_8
_OP_is_zero_map
_OP_goto
_OP_map_iter
_OP_map_stop
_OP_map_check_key
_OP_map_write_key
_OP_map_value_next
_OP_slice_len
_OP_slice_next
_OP_marshal
_OP_marshal_p
_OP_marshal_text
_OP_marshal_text_p
_OP_cond_set
_OP_cond_testc
)
const (
_INT_SIZE = 32 << (^uint(0) >> 63)
_PTR_SIZE = 32 << (^uintptr(0) >> 63)
_PTR_BYTE = unsafe.Sizeof(uintptr(0))
)
const (
_MAX_ILBUF = 100000 // cutoff at 100k of IL instructions
_MAX_FIELDS = 50 // cutoff at 50 fields struct
)
var _OpNames = [256]string {
_OP_null : "null",
_OP_empty_arr : "empty_arr",
_OP_empty_obj : "empty_obj",
_OP_bool : "bool",
_OP_i8 : "i8",
_OP_i16 : "i16",
_OP_i32 : "i32",
_OP_i64 : "i64",
_OP_u8 : "u8",
_OP_u16 : "u16",
_OP_u32 : "u32",
_OP_u64 : "u64",
_OP_f32 : "f32",
_OP_f64 : "f64",
_OP_str : "str",
_OP_bin : "bin",
_OP_quote : "quote",
_OP_number : "number",
_OP_eface : "eface",
_OP_iface : "iface",
_OP_byte : "byte",
_OP_text : "text",
_OP_deref : "deref",
_OP_index : "index",
_OP_load : "load",
_OP_save : "save",
_OP_drop : "drop",
_OP_drop_2 : "drop_2",
_OP_recurse : "recurse",
_OP_is_nil : "is_nil",
_OP_is_nil_p1 : "is_nil_p1",
_OP_is_zero_1 : "is_zero_1",
_OP_is_zero_2 : "is_zero_2",
_OP_is_zero_4 : "is_zero_4",
_OP_is_zero_8 : "is_zero_8",
_OP_is_zero_map : "is_zero_map",
_OP_goto : "goto",
_OP_map_iter : "map_iter",
_OP_map_stop : "map_stop",
_OP_map_check_key : "map_check_key",
_OP_map_write_key : "map_write_key",
_OP_map_value_next : "map_value_next",
_OP_slice_len : "slice_len",
_OP_slice_next : "slice_next",
_OP_marshal : "marshal",
_OP_marshal_p : "marshal_p",
_OP_marshal_text : "marshal_text",
_OP_marshal_text_p : "marshal_text_p",
_OP_cond_set : "cond_set",
_OP_cond_testc : "cond_testc",
}
func (self _Op) String() string {
if ret := _OpNames[self]; ret != "" {
return ret
} else {
return "<invalid>"
}
}
func _OP_int() _Op {
switch _INT_SIZE {
case 32: return _OP_i32
case 64: return _OP_i64
default: panic("unsupported int size")
}
}
func _OP_uint() _Op {
switch _INT_SIZE {
case 32: return _OP_u32
case 64: return _OP_u64
default: panic("unsupported uint size")
}
}
func _OP_uintptr() _Op {
switch _PTR_SIZE {
case 32: return _OP_u32
case 64: return _OP_u64
default: panic("unsupported pointer size")
}
}
func _OP_is_zero_ints() _Op {
switch _INT_SIZE {
case 32: return _OP_is_zero_4
case 64: return _OP_is_zero_8
default: panic("unsupported integer size")
}
}
type _Instr struct {
u uint64 // union {op: 8, _: 8, vi: 48}, vi maybe int or len(str)
p unsafe.Pointer // maybe GoString.Ptr, or *GoType
}
func packOp(op _Op) uint64 {
return uint64(op) << 56
}
func newInsOp(op _Op) _Instr {
return _Instr{u: packOp(op)}
}
func newInsVi(op _Op, vi int) _Instr {
return _Instr{u: packOp(op) | rt.PackInt(vi)}
}
func newInsVs(op _Op, vs string) _Instr {
return _Instr {
u: packOp(op) | rt.PackInt(len(vs)),
p: (*rt.GoString)(unsafe.Pointer(&vs)).Ptr,
}
}
func newInsVt(op _Op, vt reflect.Type) _Instr {
return _Instr {
u: packOp(op),
p: unsafe.Pointer(rt.UnpackType(vt)),
}
}
func newInsVp(op _Op, vt reflect.Type, pv bool) _Instr {
i := 0
if pv {
i = 1
}
return _Instr {
u: packOp(op) | rt.PackInt(i),
p: unsafe.Pointer(rt.UnpackType(vt)),
}
}
func (self _Instr) op() _Op {
return _Op(self.u >> 56)
}
func (self _Instr) vi() int {
return rt.UnpackInt(self.u)
}
func (self _Instr) vf() uint8 {
return (*rt.GoType)(self.p).KindFlags
}
func (self _Instr) vs() (v string) {
(*rt.GoString)(unsafe.Pointer(&v)).Ptr = self.p
(*rt.GoString)(unsafe.Pointer(&v)).Len = self.vi()
return
}
func (self _Instr) vk() reflect.Kind {
return (*rt.GoType)(self.p).Kind()
}
func (self _Instr) vt() reflect.Type {
return (*rt.GoType)(self.p).Pack()
}
func (self _Instr) vp() (vt reflect.Type, pv bool) {
return (*rt.GoType)(self.p).Pack(), rt.UnpackInt(self.u) == 1
}
func (self _Instr) i64() int64 {
return int64(self.vi())
}
func (self _Instr) vlen() int {
return int((*rt.GoType)(self.p).Size)
}
func (self _Instr) isBranch() bool {
switch self.op() {
case _OP_goto : fallthrough
case _OP_is_nil : fallthrough
case _OP_is_nil_p1 : fallthrough
case _OP_is_zero_1 : fallthrough
case _OP_is_zero_2 : fallthrough
case _OP_is_zero_4 : fallthrough
case _OP_is_zero_8 : fallthrough
case _OP_map_check_key : fallthrough
case _OP_map_write_key : fallthrough
case _OP_slice_next : fallthrough
case _OP_cond_testc : return true
default : return false
}
}
func (self _Instr) disassemble() string {
switch self.op() {
case _OP_byte : return fmt.Sprintf("%-18s%s", self.op().String(), strconv.QuoteRune(rune(self.vi())))
case _OP_text : return fmt.Sprintf("%-18s%s", self.op().String(), strconv.Quote(self.vs()))
case _OP_index : return fmt.Sprintf("%-18s%d", self.op().String(), self.vi())
case _OP_recurse : fallthrough
case _OP_map_iter : fallthrough
case _OP_marshal : fallthrough
case _OP_marshal_p : fallthrough
case _OP_marshal_text : fallthrough
case _OP_marshal_text_p : return fmt.Sprintf("%-18s%s", self.op().String(), self.vt())
case _OP_goto : fallthrough
case _OP_is_nil : fallthrough
case _OP_is_nil_p1 : fallthrough
case _OP_is_zero_1 : fallthrough
case _OP_is_zero_2 : fallthrough
case _OP_is_zero_4 : fallthrough
case _OP_is_zero_8 : fallthrough
case _OP_is_zero_map : fallthrough
case _OP_cond_testc : fallthrough
case _OP_map_check_key : fallthrough
case _OP_map_write_key : return fmt.Sprintf("%-18sL_%d", self.op().String(), self.vi())
case _OP_slice_next : return fmt.Sprintf("%-18sL_%d, %s", self.op().String(), self.vi(), self.vt())
default : return self.op().String()
}
}
type (
_Program []_Instr
)
func (self _Program) pc() int {
return len(self)
}
func (self _Program) tag(n int) {
if n >= _MaxStack {
panic("type nesting too deep")
}
}
func (self _Program) pin(i int) {
v := &self[i]
v.u &= 0xffff000000000000
v.u |= rt.PackInt(self.pc())
}
func (self _Program) rel(v []int) {
for _, i := range v {
self.pin(i)
}
}
func (self *_Program) add(op _Op) {
*self = append(*self, newInsOp(op))
}
func (self *_Program) key(op _Op) {
*self = append(*self,
newInsVi(_OP_byte, '"'),
newInsOp(op),
newInsVi(_OP_byte, '"'),
)
}
func (self *_Program) int(op _Op, vi int) {
*self = append(*self, newInsVi(op, vi))
}
func (self *_Program) str(op _Op, vs string) {
*self = append(*self, newInsVs(op, vs))
}
func (self *_Program) rtt(op _Op, vt reflect.Type) {
*self = append(*self, newInsVt(op, vt))
}
func (self *_Program) vp(op _Op, vt reflect.Type, pv bool) {
*self = append(*self, newInsVp(op, vt, pv))
}
func (self _Program) disassemble() string {
nb := len(self)
tab := make([]bool, nb + 1)
ret := make([]string, 0, nb + 1)
/* prescan to get all the labels */
for _, ins := range self {
if ins.isBranch() {
tab[ins.vi()] = true
}
}
/* disassemble each instruction */
for i, ins := range self {
if !tab[i] {
ret = append(ret, "\t" + ins.disassemble())
} else {
ret = append(ret, fmt.Sprintf("L_%d:\n\t%s", i, ins.disassemble()))
}
}
/* add the last label, if needed */
if tab[nb] {
ret = append(ret, fmt.Sprintf("L_%d:", nb))
}
/* add an "end" indicator, and join all the strings */
return strings.Join(append(ret, "\tend"), "\n")
}
type _Compiler struct {
opts option.CompileOptions
pv bool
tab map[reflect.Type]bool
rec map[reflect.Type]uint8
}
func newCompiler() *_Compiler {
return &_Compiler {
opts: option.DefaultCompileOptions(),
tab: map[reflect.Type]bool{},
rec: map[reflect.Type]uint8{},
}
}
func (self *_Compiler) apply(opts option.CompileOptions) *_Compiler {
self.opts = opts
if self.opts.RecursiveDepth > 0 {
self.rec = map[reflect.Type]uint8{}
}
return self
}
func (self *_Compiler) rescue(ep *error) {
if val := recover(); val != nil {
if err, ok := val.(error); ok {
*ep = err
} else {
panic(val)
}
}
}
func (self *_Compiler) compile(vt reflect.Type, pv bool) (ret _Program, err error) {
defer self.rescue(&err)
self.compileOne(&ret, 0, vt, pv)
return
}
func (self *_Compiler) compileOne(p *_Program, sp int, vt reflect.Type, pv bool) {
if self.tab[vt] {
p.vp(_OP_recurse, vt, pv)
} else {
self.compileRec(p, sp, vt, pv)
}
}
func (self *_Compiler) compileRec(p *_Program, sp int, vt reflect.Type, pv bool) {
pr := self.pv
pt := reflect.PtrTo(vt)
/* check for addressable `json.Marshaler` with pointer receiver */
if pv && pt.Implements(jsonMarshalerType) {
p.rtt(_OP_marshal_p, pt)
return
}
/* check for `json.Marshaler` */
if vt.Implements(jsonMarshalerType) {
self.compileMarshaler(p, _OP_marshal, vt, jsonMarshalerType)
return
}
/* check for addressable `encoding.TextMarshaler` with pointer receiver */
if pv && pt.Implements(encodingTextMarshalerType) {
p.rtt(_OP_marshal_text_p, pt)
return
}
/* check for `encoding.TextMarshaler` */
if vt.Implements(encodingTextMarshalerType) {
self.compileMarshaler(p, _OP_marshal_text, vt, encodingTextMarshalerType)
return
}
/* enter the recursion, and compile the type */
self.pv = pv
self.tab[vt] = true
self.compileOps(p, sp, vt)
/* exit the recursion */
self.pv = pr
delete(self.tab, vt)
}
func (self *_Compiler) compileOps(p *_Program, sp int, vt reflect.Type) {
switch vt.Kind() {
case reflect.Bool : p.add(_OP_bool)
case reflect.Int : p.add(_OP_int())
case reflect.Int8 : p.add(_OP_i8)
case reflect.Int16 : p.add(_OP_i16)
case reflect.Int32 : p.add(_OP_i32)
case reflect.Int64 : p.add(_OP_i64)
case reflect.Uint : p.add(_OP_uint())
case reflect.Uint8 : p.add(_OP_u8)
case reflect.Uint16 : p.add(_OP_u16)
case reflect.Uint32 : p.add(_OP_u32)
case reflect.Uint64 : p.add(_OP_u64)
case reflect.Uintptr : p.add(_OP_uintptr())
case reflect.Float32 : p.add(_OP_f32)
case reflect.Float64 : p.add(_OP_f64)
case reflect.String : self.compileString (p, vt)
case reflect.Array : self.compileArray (p, sp, vt.Elem(), vt.Len())
case reflect.Interface : self.compileInterface (p, vt)
case reflect.Map : self.compileMap (p, sp, vt)
case reflect.Ptr : self.compilePtr (p, sp, vt.Elem())
case reflect.Slice : self.compileSlice (p, sp, vt.Elem())
case reflect.Struct : self.compileStruct (p, sp, vt)
default : panic (error_type(vt))
}
}
func (self *_Compiler) compileNil(p *_Program, sp int, vt reflect.Type, nil_op _Op, fn func(*_Program, int, reflect.Type)) {
x := p.pc()
p.add(_OP_is_nil)
fn(p, sp, vt)
e := p.pc()
p.add(_OP_goto)
p.pin(x)
p.add(nil_op)
p.pin(e)
}
func (self *_Compiler) compilePtr(p *_Program, sp int, vt reflect.Type) {
self.compileNil(p, sp, vt, _OP_null, self.compilePtrBody)
}
func (self *_Compiler) compilePtrBody(p *_Program, sp int, vt reflect.Type) {
p.tag(sp)
p.add(_OP_save)
p.add(_OP_deref)
self.compileOne(p, sp + 1, vt, true)
p.add(_OP_drop)
}
func (self *_Compiler) compileMap(p *_Program, sp int, vt reflect.Type) {
self.compileNil(p, sp, vt, _OP_empty_obj, self.compileMapBody)
}
func (self *_Compiler) compileMapBody(p *_Program, sp int, vt reflect.Type) {
p.tag(sp + 1)
p.int(_OP_byte, '{')
p.add(_OP_save)
p.rtt(_OP_map_iter, vt)
p.add(_OP_save)
i := p.pc()
p.add(_OP_map_check_key)
u := p.pc()
p.add(_OP_map_write_key)
self.compileMapBodyKey(p, vt.Key())
p.pin(u)
p.int(_OP_byte, ':')
p.add(_OP_map_value_next)
self.compileOne(p, sp + 2, vt.Elem(), false)
j := p.pc()
p.add(_OP_map_check_key)
p.int(_OP_byte, ',')
v := p.pc()
p.add(_OP_map_write_key)
self.compileMapBodyKey(p, vt.Key())
p.pin(v)
p.int(_OP_byte, ':')
p.add(_OP_map_value_next)
self.compileOne(p, sp + 2, vt.Elem(), false)
p.int(_OP_goto, j)
p.pin(i)
p.pin(j)
p.add(_OP_map_stop)
p.add(_OP_drop_2)
p.int(_OP_byte, '}')
}
func (self *_Compiler) compileMapBodyKey(p *_Program, vk reflect.Type) {
if !vk.Implements(encodingTextMarshalerType) {
self.compileMapBodyTextKey(p, vk)
} else {
self.compileMapBodyUtextKey(p, vk)
}
}
func (self *_Compiler) compileMapBodyTextKey(p *_Program, vk reflect.Type) {
switch vk.Kind() {
case reflect.Invalid : panic("map key is nil")
case reflect.Bool : p.key(_OP_bool)
case reflect.Int : p.key(_OP_int())
case reflect.Int8 : p.key(_OP_i8)
case reflect.Int16 : p.key(_OP_i16)
case reflect.Int32 : p.key(_OP_i32)
case reflect.Int64 : p.key(_OP_i64)
case reflect.Uint : p.key(_OP_uint())
case reflect.Uint8 : p.key(_OP_u8)
case reflect.Uint16 : p.key(_OP_u16)
case reflect.Uint32 : p.key(_OP_u32)
case reflect.Uint64 : p.key(_OP_u64)
case reflect.Uintptr : p.key(_OP_uintptr())
case reflect.Float32 : p.key(_OP_f32)
case reflect.Float64 : p.key(_OP_f64)
case reflect.String : self.compileString(p, vk)
default : panic(error_type(vk))
}
}
func (self *_Compiler) compileMapBodyUtextKey(p *_Program, vk reflect.Type) {
if vk.Kind() != reflect.Ptr {
p.rtt(_OP_marshal_text, vk)
} else {
self.compileMapBodyUtextPtr(p, vk)
}
}
func (self *_Compiler) compileMapBodyUtextPtr(p *_Program, vk reflect.Type) {
i := p.pc()
p.add(_OP_is_nil)
p.rtt(_OP_marshal_text, vk)
j := p.pc()
p.add(_OP_goto)
p.pin(i)
p.str(_OP_text, "\"\"")
p.pin(j)
}
func (self *_Compiler) compileSlice(p *_Program, sp int, vt reflect.Type) {
self.compileNil(p, sp, vt, _OP_empty_arr, self.compileSliceBody)
}
func (self *_Compiler) compileSliceBody(p *_Program, sp int, vt reflect.Type) {
if isSimpleByte(vt) {
p.add(_OP_bin)
} else {
self.compileSliceArray(p, sp, vt)
}
}
func (self *_Compiler) compileSliceArray(p *_Program, sp int, vt reflect.Type) {
p.tag(sp)
p.int(_OP_byte, '[')
p.add(_OP_save)
p.add(_OP_slice_len)
i := p.pc()
p.rtt(_OP_slice_next, vt)
self.compileOne(p, sp + 1, vt, true)
j := p.pc()
p.rtt(_OP_slice_next, vt)
p.int(_OP_byte, ',')
self.compileOne(p, sp + 1, vt, true)
p.int(_OP_goto, j)
p.pin(i)
p.pin(j)
p.add(_OP_drop)
p.int(_OP_byte, ']')
}
func (self *_Compiler) compileArray(p *_Program, sp int, vt reflect.Type, nb int) {
p.tag(sp)
p.int(_OP_byte, '[')
p.add(_OP_save)
/* first item */
if nb != 0 {
self.compileOne(p, sp + 1, vt, self.pv)
p.add(_OP_load)
}
/* remaining items */
for i := 1; i < nb; i++ {
p.int(_OP_byte, ',')
p.int(_OP_index, i * int(vt.Size()))
self.compileOne(p, sp + 1, vt, self.pv)
p.add(_OP_load)
}
/* end of array */
p.add(_OP_drop)
p.int(_OP_byte, ']')
}
func (self *_Compiler) compileString(p *_Program, vt reflect.Type) {
if vt != jsonNumberType {
p.add(_OP_str)
} else {
p.add(_OP_number)
}
}
func (self *_Compiler) compileStruct(p *_Program, sp int, vt reflect.Type) {
if sp >= self.opts.MaxInlineDepth || p.pc() >= _MAX_ILBUF || (sp > 0 && vt.NumField() >= _MAX_FIELDS) {
p.vp(_OP_recurse, vt, self.pv)
if self.opts.RecursiveDepth > 0 {
if self.pv {
self.rec[vt] = 1
} else {
self.rec[vt] = 0
}
}
} else {
self.compileStructBody(p, sp, vt)
}
}
func (self *_Compiler) compileStructBody(p *_Program, sp int, vt reflect.Type) {
p.tag(sp)
p.int(_OP_byte, '{')
p.add(_OP_save)
p.add(_OP_cond_set)
/* compile each field */
for _, fv := range resolver.ResolveStruct(vt) {
var s []int
var o resolver.Offset
/* "omitempty" for arrays */
if fv.Type.Kind() == reflect.Array {
if fv.Type.Len() == 0 && (fv.Opts & resolver.F_omitempty) != 0 {
continue
}
}
/* index to the field */
for _, o = range fv.Path {
if p.int(_OP_index, int(o.Size)); o.Kind == resolver.F_deref {
s = append(s, p.pc())
p.add(_OP_is_nil)
p.add(_OP_deref)
}
}
/* check for "omitempty" option */
if fv.Type.Kind() != reflect.Struct && fv.Type.Kind() != reflect.Array && (fv.Opts & resolver.F_omitempty) != 0 {
s = append(s, p.pc())
self.compileStructFieldZero(p, fv.Type)
}
/* add the comma if not the first element */
i := p.pc()
p.add(_OP_cond_testc)
p.int(_OP_byte, ',')
p.pin(i)
/* compile the key and value */
ft := fv.Type
p.str(_OP_text, Quote(fv.Name) + ":")
/* check for "stringnize" option */
if (fv.Opts & resolver.F_stringize) == 0 {
self.compileOne(p, sp + 1, ft, self.pv)
} else {
self.compileStructFieldStr(p, sp + 1, ft)
}
/* patch the skipping jumps and reload the struct pointer */
p.rel(s)
p.add(_OP_load)
}
/* end of object */
p.add(_OP_drop)
p.int(_OP_byte, '}')
}
func (self *_Compiler) compileStructFieldStr(p *_Program, sp int, vt reflect.Type) {
pc := -1
ft := vt
sv := false
/* dereference the pointer if needed */
if ft.Kind() == reflect.Ptr {
ft = ft.Elem()
}
/* check if it can be stringized */
switch ft.Kind() {
case reflect.Bool : sv = true
case reflect.Int : sv = true
case reflect.Int8 : sv = true
case reflect.Int16 : sv = true
case reflect.Int32 : sv = true
case reflect.Int64 : sv = true
case reflect.Uint : sv = true
case reflect.Uint8 : sv = true
case reflect.Uint16 : sv = true
case reflect.Uint32 : sv = true
case reflect.Uint64 : sv = true
case reflect.Uintptr : sv = true
case reflect.Float32 : sv = true
case reflect.Float64 : sv = true
case reflect.String : sv = true
}
/* if it's not, ignore the "string" and follow the regular path */
if !sv {
self.compileOne(p, sp, vt, self.pv)
return
}
/* dereference the pointer */
if vt.Kind() == reflect.Ptr {
pc = p.pc()
vt = vt.Elem()
p.add(_OP_is_nil)
p.add(_OP_deref)
}
/* special case of a double-quoted string */
if ft != jsonNumberType && ft.Kind() == reflect.String {
p.add(_OP_quote)
} else {
self.compileStructFieldQuoted(p, sp, vt)
}
/* the "null" case of the pointer */
if pc != -1 {
e := p.pc()
p.add(_OP_goto)
p.pin(pc)
p.add(_OP_null)
p.pin(e)
}
}
func (self *_Compiler) compileStructFieldZero(p *_Program, vt reflect.Type) {
switch vt.Kind() {
case reflect.Bool : p.add(_OP_is_zero_1)
case reflect.Int : p.add(_OP_is_zero_ints())
case reflect.Int8 : p.add(_OP_is_zero_1)
case reflect.Int16 : p.add(_OP_is_zero_2)
case reflect.Int32 : p.add(_OP_is_zero_4)
case reflect.Int64 : p.add(_OP_is_zero_8)
case reflect.Uint : p.add(_OP_is_zero_ints())
case reflect.Uint8 : p.add(_OP_is_zero_1)
case reflect.Uint16 : p.add(_OP_is_zero_2)
case reflect.Uint32 : p.add(_OP_is_zero_4)
case reflect.Uint64 : p.add(_OP_is_zero_8)
case reflect.Uintptr : p.add(_OP_is_nil)
case reflect.Float32 : p.add(_OP_is_zero_4)
case reflect.Float64 : p.add(_OP_is_zero_8)
case reflect.String : p.add(_OP_is_nil_p1)
case reflect.Interface : p.add(_OP_is_nil_p1)
case reflect.Map : p.add(_OP_is_zero_map)
case reflect.Ptr : p.add(_OP_is_nil)
case reflect.Slice : p.add(_OP_is_nil_p1)
default : panic(error_type(vt))
}
}
func (self *_Compiler) compileStructFieldQuoted(p *_Program, sp int, vt reflect.Type) {
p.int(_OP_byte, '"')
self.compileOne(p, sp, vt, self.pv)
p.int(_OP_byte, '"')
}
func (self *_Compiler) compileInterface(p *_Program, vt reflect.Type) {
x := p.pc()
p.add(_OP_is_nil_p1)
/* iface and efaces are different */
if vt.NumMethod() == 0 {
p.add(_OP_eface)
} else {
p.add(_OP_iface)
}
/* the "null" value */
e := p.pc()
p.add(_OP_goto)
p.pin(x)
p.add(_OP_null)
p.pin(e)
}
func (self *_Compiler) compileMarshaler(p *_Program, op _Op, vt reflect.Type, mt reflect.Type) {
pc := p.pc()
vk := vt.Kind()
/* direct receiver */
if vk != reflect.Ptr {
p.rtt(op, vt)
return
}
/* value receiver with a pointer type, check for nil before calling the marshaler */
p.add(_OP_is_nil)
p.rtt(op, vt)
i := p.pc()
p.add(_OP_goto)
p.pin(pc)
p.add(_OP_null)
p.pin(i)
}
vendor/github.com/bytedance/sonic/internal/encoder/debug_go116.go
Generated Vendored
+66
View File
@@ -0,0 +1,66 @@
// +build go1.15,!go1.17
/*
* Copyright 2021 ByteDance Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package encoder
import (
`os`
`strings`
`runtime`
`runtime/debug`
`github.com/bytedance/sonic/internal/jit`
)
var (
debugSyncGC = os.Getenv("SONIC_SYNC_GC") != ""
debugAsyncGC = os.Getenv("SONIC_NO_ASYNC_GC") == ""
)
var (
_Instr_End _Instr = newInsOp(_OP_null)
_F_gc = jit.Func(runtime.GC)
_F_force_gc = jit.Func(debug.FreeOSMemory)
_F_println = jit.Func(println_wrapper)
)
func println_wrapper(i int, op1 int, op2 int){
println(i, " Intrs ", op1, _OpNames[op1], "next: ", op2, _OpNames[op2])
}
func (self *_Assembler) force_gc() {
self.call_go(_F_gc)
self.call_go(_F_force_gc)
}
func (self *_Assembler) debug_instr(i int, v *_Instr) {
if debugSyncGC {
if (i+1 == len(self.p)) {
self.print_gc(i, v, &_Instr_End)
} else {
next := &(self.p[i+1])
self.print_gc(i, v, next)
name := _OpNames[next.op()]
if strings.Contains(name, "save") {
return
}
}
self.force_gc()
}
}
vendor/github.com/bytedance/sonic/internal/encoder/debug_go117.go
Generated Vendored
+205
View File
@@ -0,0 +1,205 @@
// +build go1.17,!go1.21
/*
* Copyright 2021 ByteDance Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package encoder
import (
`fmt`
`os`
`runtime`
`strings`
`unsafe`
`github.com/bytedance/sonic/internal/jit`
`github.com/twitchyliquid64/golang-asm/obj`
)
const _FP_debug = 128
var (
debugSyncGC = os.Getenv("SONIC_SYNC_GC") != ""
debugAsyncGC = os.Getenv("SONIC_NO_ASYNC_GC") == ""
debugCheckPtr = os.Getenv("SONIC_CHECK_POINTER") != ""
)
var (
_Instr_End = newInsOp(_OP_is_nil)
_F_gc = jit.Func(gc)
_F_println = jit.Func(println_wrapper)
_F_print = jit.Func(print)
)
func (self *_Assembler) dsave(r ...obj.Addr) {
for i, v := range r {
if i > _FP_debug / 8 - 1 {
panic("too many registers to save")
} else {
self.Emit("MOVQ", v, jit.Ptr(_SP, _FP_fargs + _FP_saves + _FP_locals + int64(i) * 8))
}
}
}
func (self *_Assembler) dload(r ...obj.Addr) {
for i, v := range r {
if i > _FP_debug / 8 - 1 {
panic("too many registers to load")
} else {
self.Emit("MOVQ", jit.Ptr(_SP, _FP_fargs + _FP_saves + _FP_locals + int64(i) * 8), v)
}
}
}
func println_wrapper(i int, op1 int, op2 int){
println(i, " Intrs ", op1, _OpNames[op1], "next: ", op2, _OpNames[op2])
}
func print(i int){
println(i)
}
func gc() {
if !debugSyncGC {
return
}
runtime.GC()
// debug.FreeOSMemory()
}
func (self *_Assembler) dcall(fn obj.Addr) {
self.Emit("MOVQ", fn, _R10) // MOVQ ${fn}, R10
self.Rjmp("CALL", _R10) // CALL R10
}
func (self *_Assembler) debug_gc() {
if !debugSyncGC {
return
}
self.dsave(_REG_debug...)
self.dcall(_F_gc)
self.dload(_REG_debug...)
}
func (self *_Assembler) debug_instr(i int, v *_Instr) {
if debugSyncGC {
if i+1 == len(self.p) {
self.print_gc(i, v, &_Instr_End)
} else {
next := &(self.p[i+1])
self.print_gc(i, v, next)
name := _OpNames[next.op()]
if strings.Contains(name, "save") {
return
}
}
// self.debug_gc()
}
}
//go:noescape
//go:linkname checkptrBase runtime.checkptrBase
func checkptrBase(p unsafe.Pointer) uintptr
//go:noescape
//go:linkname findObject runtime.findObject
func findObject(p, refBase, refOff uintptr) (base uintptr, s unsafe.Pointer, objIndex uintptr)
var (
_F_checkptr = jit.Func(checkptr)
_F_printptr = jit.Func(printptr)
)
var (
_R10 = jit.Reg("R10")
)
var _REG_debug = []obj.Addr {
jit.Reg("AX"),
jit.Reg("BX"),
jit.Reg("CX"),
jit.Reg("DX"),
jit.Reg("DI"),
jit.Reg("SI"),
jit.Reg("BP"),
jit.Reg("SP"),
jit.Reg("R8"),
jit.Reg("R9"),
jit.Reg("R10"),
jit.Reg("R11"),
jit.Reg("R12"),
jit.Reg("R13"),
jit.Reg("R14"),
jit.Reg("R15"),
}
func checkptr(ptr uintptr) {
if ptr == 0 {
return
}
fmt.Printf("pointer: %x\n", ptr)
f := checkptrBase(unsafe.Pointer(uintptr(ptr)))
if f == 0 {
fmt.Printf("! unknown-based pointer: %x\n", ptr)
} else if f == 1 {
fmt.Printf("! stack pointer: %x\n", ptr)
} else {
fmt.Printf("base: %x\n", f)
}
findobj(ptr)
}
func findobj(ptr uintptr) {
base, s, objIndex := findObject(ptr, 0, 0)
if s != nil && base == 0 {
fmt.Printf("! invalid pointer: %x\n", ptr)
}
fmt.Printf("objIndex: %d\n", objIndex)
}
func (self *_Assembler) check_ptr(ptr obj.Addr, lea bool) {
if !debugCheckPtr {
return
}
self.dsave(_REG_debug...)
if lea {
self.Emit("LEAQ", ptr, _R10)
} else {
self.Emit("MOVQ", ptr, _R10)
}
self.Emit("MOVQ", _R10, jit.Ptr(_SP, 0))
self.dcall(_F_checkptr)
self.dload(_REG_debug...)
}
func printptr(i int, ptr uintptr) {
fmt.Printf("[%d] ptr: %x\n", i, ptr)
}
func (self *_Assembler) print_ptr(i int, ptr obj.Addr, lea bool) {
self.dsave(_REG_debug...)
if lea {
self.Emit("LEAQ", ptr, _R10)
} else {
self.Emit("MOVQ", ptr, _R10)
}
self.Emit("MOVQ", jit.Imm(int64(i)), _AX)
self.Emit("MOVQ", _R10, _BX)
self.dcall(_F_printptr)
self.dload(_REG_debug...)
}
vendor/github.com/bytedance/sonic/internal/encoder/encoder.go
Generated Vendored
+327
View File
@@ -0,0 +1,327 @@
/*
* Copyright 2021 ByteDance Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package encoder
import (
`bytes`
`encoding/json`
`reflect`
`runtime`
`unsafe`
`github.com/bytedance/sonic/internal/native`
`github.com/bytedance/sonic/internal/native/types`
`github.com/bytedance/sonic/internal/rt`
`github.com/bytedance/sonic/utf8`
`github.com/bytedance/sonic/option`
)
// Options is a set of encoding options.
type Options uint64
const (
bitSortMapKeys = iota
bitEscapeHTML
bitCompactMarshaler
bitNoQuoteTextMarshaler
bitNoNullSliceOrMap
bitValidateString
// used for recursive compile
bitPointerValue = 63
)
const (
// SortMapKeys indicates that the keys of a map needs to be sorted
// before serializing into JSON.
// WARNING: This hurts performance A LOT, USE WITH CARE.
SortMapKeys Options = 1 << bitSortMapKeys
// EscapeHTML indicates encoder to escape all HTML characters
// after serializing into JSON (see https://pkg.go.dev/encoding/json#HTMLEscape).
// WARNING: This hurts performance A LOT, USE WITH CARE.
EscapeHTML Options = 1 << bitEscapeHTML
// CompactMarshaler indicates that the output JSON from json.Marshaler
// is always compact and needs no validation
CompactMarshaler Options = 1 << bitCompactMarshaler
// NoQuoteTextMarshaler indicates that the output text from encoding.TextMarshaler
// is always escaped string and needs no quoting
NoQuoteTextMarshaler Options = 1 << bitNoQuoteTextMarshaler
// NoNullSliceOrMap indicates all empty Array or Object are encoded as '[]' or '{}',
// instead of 'null'
NoNullSliceOrMap Options = 1 << bitNoNullSliceOrMap
// ValidateString indicates that encoder should validate the input string
// before encoding it into JSON.
ValidateString Options = 1 << bitValidateString
// CompatibleWithStd is used to be compatible with std encoder.
CompatibleWithStd Options = SortMapKeys | EscapeHTML | CompactMarshaler
)
// Encoder represents a specific set of encoder configurations.
type Encoder struct {
Opts Options
prefix string
indent string
}
// Encode returns the JSON encoding of v.
func (self *Encoder) Encode(v interface{}) ([]byte, error) {
if self.indent != "" || self.prefix != "" {
return EncodeIndented(v, self.prefix, self.indent, self.Opts)
}
return Encode(v, self.Opts)
}
// SortKeys enables the SortMapKeys option.
func (self *Encoder) SortKeys() *Encoder {
self.Opts |= SortMapKeys
return self
}
// SetEscapeHTML specifies if option EscapeHTML opens
func (self *Encoder) SetEscapeHTML(f bool) {
if f {
self.Opts |= EscapeHTML
} else {
self.Opts &= ^EscapeHTML
}
}
// SetValidateString specifies if option ValidateString opens
func (self *Encoder) SetValidateString(f bool) {
if f {
self.Opts |= ValidateString
} else {
self.Opts &= ^ValidateString
}
}
// SetCompactMarshaler specifies if option CompactMarshaler opens
func (self *Encoder) SetCompactMarshaler(f bool) {
if f {
self.Opts |= CompactMarshaler
} else {
self.Opts &= ^CompactMarshaler
}
}
// SetNoQuoteTextMarshaler specifies if option NoQuoteTextMarshaler opens
func (self *Encoder) SetNoQuoteTextMarshaler(f bool) {
if f {
self.Opts |= NoQuoteTextMarshaler
} else {
self.Opts &= ^NoQuoteTextMarshaler
}
}
// SetIndent instructs the encoder to format each subsequent encoded
// value as if indented by the package-level function EncodeIndent().
// Calling SetIndent("", "") disables indentation.
func (enc *Encoder) SetIndent(prefix, indent string) {
enc.prefix = prefix
enc.indent = indent
}
// Quote returns the JSON-quoted version of s.
func Quote(s string) string {
var n int
var p []byte
/* check for empty string */
if s == "" {
return `""`
}
/* allocate space for result */
n = len(s) + 2
p = make([]byte, 0, n)
/* call the encoder */
_ = encodeString(&p, s)
return rt.Mem2Str(p)
}
// Encode returns the JSON encoding of val, encoded with opts.
func Encode(val interface{}, opts Options) ([]byte, error) {
var ret []byte
buf := newBytes()
err := encodeInto(&buf, val, opts)
/* check for errors */
if err != nil {
freeBytes(buf)
return nil, err
}
/* htmlescape or correct UTF-8 if opts enable */
old := buf
buf = encodeFinish(old, opts)
pbuf := ((*rt.GoSlice)(unsafe.Pointer(&buf))).Ptr
pold := ((*rt.GoSlice)(unsafe.Pointer(&old))).Ptr
/* return when allocated a new buffer */
if pbuf != pold {
freeBytes(old)
return buf, nil
}
/* make a copy of the result */
ret = make([]byte, len(buf))
copy(ret, buf)
freeBytes(buf)
/* return the buffer into pool */
return ret, nil
}
// EncodeInto is like Encode but uses a user-supplied buffer instead of allocating
// a new one.
func EncodeInto(buf *[]byte, val interface{}, opts Options) error {
err := encodeInto(buf, val, opts)
if err != nil {
return err
}
*buf = encodeFinish(*buf, opts)
return err
}
func encodeInto(buf *[]byte, val interface{}, opts Options) error {
stk := newStack()
efv := rt.UnpackEface(val)
err := encodeTypedPointer(buf, efv.Type, &efv.Value, stk, uint64(opts))
/* return the stack into pool */
if err != nil {
resetStack(stk)
}
freeStack(stk)
/* avoid GC ahead */
runtime.KeepAlive(buf)
runtime.KeepAlive(efv)
return err
}
func encodeFinish(buf []byte, opts Options) []byte {
if opts & EscapeHTML != 0 {
buf = HTMLEscape(nil, buf)
}
if opts & ValidateString != 0 && !utf8.Validate(buf) {
buf = utf8.CorrectWith(nil, buf, `\ufffd`)
}
return buf
}
var typeByte = rt.UnpackType(reflect.TypeOf(byte(0)))
// HTMLEscape appends to dst the JSON-encoded src with <, >, &, U+2028 and U+2029
// characters inside string literals changed to \u003c, \u003e, \u0026, \u2028, \u2029
// so that the JSON will be safe to embed inside HTML <script> tags.
// For historical reasons, web browsers don't honor standard HTML
// escaping within <script> tags, so an alternative JSON encoding must
// be used.
func HTMLEscape(dst []byte, src []byte) []byte {
return htmlEscape(dst, src)
}
// EncodeIndented is like Encode but applies Indent to format the output.
// Each JSON element in the output will begin on a new line beginning with prefix
// followed by one or more copies of indent according to the indentation nesting.
func EncodeIndented(val interface{}, prefix string, indent string, opts Options) ([]byte, error) {
var err error
var out []byte
var buf *bytes.Buffer
/* encode into the buffer */
out = newBytes()
err = EncodeInto(&out, val, opts)
/* check for errors */
if err != nil {
freeBytes(out)
return nil, err
}
/* indent the JSON */
buf = newBuffer()
err = json.Indent(buf, out, prefix, indent)
/* check for errors */
if err != nil {
freeBytes(out)
freeBuffer(buf)
return nil, err
}
/* copy to the result buffer */
ret := make([]byte, buf.Len())
copy(ret, buf.Bytes())
/* return the buffers into pool */
freeBytes(out)
freeBuffer(buf)
return ret, nil
}
// Pretouch compiles vt ahead-of-time to avoid JIT compilation on-the-fly, in
// order to reduce the first-hit latency.
//
// Opts are the compile options, for example, "option.WithCompileRecursiveDepth" is
// a compile option to set the depth of recursive compile for the nested struct type.
func Pretouch(vt reflect.Type, opts ...option.CompileOption) error {
cfg := option.DefaultCompileOptions()
for _, opt := range opts {
opt(&cfg)
}
return pretouchRec(map[reflect.Type]uint8{vt: 0}, cfg)
}
// Valid validates json and returns first non-blank character position,
// if it is only one valid json value.
// Otherwise returns invalid character position using start.
//
// Note: it does not check for the invalid UTF-8 characters.
func Valid(data []byte) (ok bool, start int) {
n := len(data)
if n == 0 {
return false, -1
}
s := rt.Mem2Str(data)
p := 0
m := types.NewStateMachine()
ret := native.ValidateOne(&s, &p, m)
types.FreeStateMachine(m)
if ret < 0 {
return false, p-1
}
/* check for trailing spaces */
for ;p < n; p++ {
if (types.SPACE_MASK & (1 << data[p])) == 0 {
return false, p
}
}
return true, ret
}
vendor/github.com/bytedance/sonic/internal/encoder/errors.go
Generated Vendored
+65
View File
@@ -0,0 +1,65 @@
/*
* Copyright 2021 ByteDance Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package encoder
import (
`encoding/json`
`fmt`
`reflect`
`strconv`
`unsafe`
`github.com/bytedance/sonic/internal/rt`
)
var _ERR_too_deep = &json.UnsupportedValueError {
Str : "Value nesting too deep",
Value : reflect.ValueOf("..."),
}
var _ERR_nan_or_infinite = &json.UnsupportedValueError {
Str : "NaN or ±Infinite",
Value : reflect.ValueOf("NaN or ±Infinite"),
}
func error_type(vtype reflect.Type) error {
return &json.UnsupportedTypeError{Type: vtype}
}
func error_number(number json.Number) error {
return &json.UnsupportedValueError {
Str : "invalid number literal: " + strconv.Quote(string(number)),
Value : reflect.ValueOf(number),
}
}
func error_marshaler(ret []byte, pos int) error {
return fmt.Errorf("invalid Marshaler output json syntax at %d: %q", pos, ret)
}
const (
panicNilPointerOfNonEmptyString int = 1 + iota
)
func goPanic(code int, val unsafe.Pointer) {
switch(code){
case panicNilPointerOfNonEmptyString:
panic(fmt.Sprintf("val: %#v has nil pointer while its length is not zero!", (*rt.GoString)(val)))
default:
panic("encoder error!")
}
}

Some files were not shown because too many files have changed in this diff Show More