PHI/webrtc
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

with launch

Browse Source
This commit is contained in:
kaedwen
2024-03-29 16:00:35 +01:00
parent 555fce9f10
commit 424be67dda
113 changed files with 18959 additions and 109 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.gitignore
+3 -1
View File
@@ -1,2 +1,4 @@
service*
renew.local.env
renew.local.env
test
.vscode
.vscode/launch.json
Vendored
-33
View File
@@ -1,33 +0,0 @@
{
// Use IntelliSense to learn about possible attributes.
// Hover to view descriptions of existing attributes.
// For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387
"version": "0.2.0",
"configurations": [
{
"name": "Launch Package",
"type": "go",
"request": "launch",
"mode": "auto",
"program": "${workspaceFolder}/main.go",
"env": {
"HTTP_STATIC": "${workspaceFolder}/static/dist",
"VIDEO_SRC_DEVICE": "/dev/video4",
//"AUDIO_SRC_DEVICE": "plughw:4,0,0",
"AUDIO_SRC": "pulsesrc",
"AUDIO_SINK": "pulsesink",
"INPUT_DEVICE": "/dev/input/event5",
"JINGLE_PATH": "${workspaceFolder}/audio",
"SONOS_TARGET": "Living Room",
"VIDEO_SRC_CODEC": "h264"
}
},
{
"name": "Launch Package Test",
"type": "go",
"request": "launch",
"mode": "auto",
"program": "${workspaceFolder}/test/main.go"
}
]
}
go.mod
+4
View File
@@ -12,6 +12,7 @@ require (
github.com/google/uuid v1.6.0
github.com/hashicorp/mdns v1.0.5
github.com/holoplot/go-evdev v0.0.0-20240213072314-e40ebd251314
github.com/pion/mediadevices v0.6.1
github.com/pion/rtcp v1.2.14
github.com/pion/webrtc/v3 v3.2.28
go.uber.org/zap v1.27.0
@@ -20,6 +21,7 @@ require (
require (
github.com/alexflint/go-scalar v1.2.0 // indirect
github.com/blackjack/webcam v0.5.0 // indirect
github.com/bytedance/sonic v1.11.2 // indirect
github.com/chenzhuoyu/base64x v0.0.0-20230717121745-296ad89f973d // indirect
github.com/chenzhuoyu/iasm v0.9.1 // indirect
@@ -32,6 +34,7 @@ require (
github.com/goccy/go-json v0.10.2 // indirect
github.com/json-iterator/go v1.1.12 // indirect
github.com/klauspost/cpuid/v2 v2.2.7 // indirect
github.com/kr/pretty v0.2.1 // indirect
github.com/leodido/go-urn v1.4.0 // indirect
github.com/mattn/go-isatty v0.0.20 // indirect
github.com/mattn/go-pointer v0.0.1 // indirect
@@ -60,6 +63,7 @@ require (
go.uber.org/multierr v1.11.0 // indirect
golang.org/x/arch v0.7.0 // indirect
golang.org/x/crypto v0.20.0 // indirect
golang.org/x/image v0.15.0 // indirect
golang.org/x/mod v0.15.0 // indirect
golang.org/x/net v0.21.0 // indirect
golang.org/x/sys v0.17.0 // indirect
go.sum
+10 -2
View File
@@ -3,6 +3,8 @@ github.com/alexflint/go-arg v1.4.3/go.mod h1:3PZ/wp/8HuqRZMUUgu7I+e1qcpUbvmS258m
github.com/alexflint/go-scalar v1.1.0/go.mod h1:LoFvNMqS1CPrMVltza4LvnGKhaSpc3oyLEBUZVhhS2o=
github.com/alexflint/go-scalar v1.2.0 h1:WR7JPKkeNpnYIOfHRa7ivM21aWAdHD0gEWHCx+WQBRw=
github.com/alexflint/go-scalar v1.2.0/go.mod h1:LoFvNMqS1CPrMVltza4LvnGKhaSpc3oyLEBUZVhhS2o=
github.com/blackjack/webcam v0.5.0 h1:NImYpsAbWxglejopcQGQ3FClxhJZaBBgr5fV7nsGdvk=
github.com/blackjack/webcam v0.5.0/go.mod h1:zs+RkUZzqpFPHPiwBZ6U5B34ZXXe9i+SiHLKnnukJuI=
github.com/bytedance/sonic v1.5.0/go.mod h1:ED5hyg4y6t3/9Ku1R6dU/4KyJ48DZ4jPhfY1O2AihPM=
github.com/bytedance/sonic v1.10.0-rc/go.mod h1:ElCzW+ufi8qKqNW0FY314xriJhyJhuoJ3gFZdAHF7NM=
github.com/bytedance/sonic v1.11.2 h1:ywfwo0a/3j9HR8wsYGWsIWl2mvRsI950HyoxiBERw5A=
@@ -69,8 +71,9 @@ github.com/klauspost/cpuid/v2 v2.0.9/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa02
github.com/klauspost/cpuid/v2 v2.2.7 h1:ZWSB3igEs+d0qvnxR/ZBzXVmxkgt8DdzP6m9pfuVLDM=
github.com/klauspost/cpuid/v2 v2.2.7/go.mod h1:Lcz8mBdAVJIBVzewtcLocK12l3Y+JytZYpaMropDUws=
github.com/knz/go-libedit v1.10.1/go.mod h1:MZTVkCWyz0oBc7JOWP3wNAzd002ZbM/5hgShxwh4x8M=
github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI=
github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
github.com/kr/pretty v0.2.1 h1:Fmg33tUaq4/8ym9TJN1x7sLJnHVwhP33CNkpYV/7rwI=
github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI=
github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
@@ -114,6 +117,8 @@ github.com/pion/logging v0.2.2 h1:M9+AIj/+pxNsDfAT64+MAVgJO0rsyLnoJKCqf//DoeY=
github.com/pion/logging v0.2.2/go.mod h1:k0/tDVsRCX2Mb2ZEmTqNa7CWsQPc+YYCB7Q+5pahoms=
github.com/pion/mdns v0.0.12 h1:CiMYlY+O0azojWDmxdNr7ADGrnZ+V6Ilfner+6mSVK8=
github.com/pion/mdns v0.0.12/go.mod h1:VExJjv8to/6Wqm1FXK+Ii/Z9tsVk/F5sD/N70cnYFbk=
github.com/pion/mediadevices v0.6.1 h1:mN6Bd5CZF+0uv34NJNCdVRPGJRcE0y98hjcB5TQsOLk=
github.com/pion/mediadevices v0.6.1/go.mod h1:T54oJnAOYl3uSzE4ZOwgSsmmq1R32ho/1P0UQa6u19Y=
github.com/pion/randutil v0.1.0 h1:CFG1UdESneORglEsnimhUjf33Rwjubwj6xfiOXBa3mA=
github.com/pion/randutil v0.1.0/go.mod h1:XcJrSMMbbMRhASFVOlj/5hQial/Y8oH/HVo7TBZq+j8=
github.com/pion/rtcp v1.2.10/go.mod h1:ztfEwXZNLGyF1oQDttz/ZKIBaeeg/oWbRYqzBM9TL1I=
@@ -188,6 +193,8 @@ golang.org/x/crypto v0.12.0/go.mod h1:NF0Gs7EO5K4qLn+Ylc+fih8BSTeIjAP05siRnAh98y
golang.org/x/crypto v0.18.0/go.mod h1:R0j02AL6hcrfOiy9T4ZYp/rcWeMxM3L6QYxlOuEG1mg=
golang.org/x/crypto v0.20.0 h1:jmAMJJZXr5KiCw05dfYK9QnqaqKLYXijU23lsEdcQqg=
golang.org/x/crypto v0.20.0/go.mod h1:Xwo95rrVNIoSMx9wa1JroENMToLWn3RNVrTBpLHgZPQ=
golang.org/x/image v0.15.0 h1:kOELfmgrmJlw4Cdb7g/QGuB3CvDrXbqEIww/pNtNBm8=
golang.org/x/image v0.15.0/go.mod h1:HUYqC05R2ZcZ3ejNQsIHQDQiwWM4JBqmm6MKANTp4LE=
golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4=
golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
@@ -290,8 +297,9 @@ google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQ
google.golang.org/protobuf v1.32.0 h1:pPC6BG5ex8PDFnkbrGU3EixyhKcQ2aDuBS36lqK/C7I=
google.golang.org/protobuf v1.32.0/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo=
gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk=
gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q=
gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys=
gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw=
gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
pkg/streamer/gst-sink.go
+65 -8
View File
@@ -7,6 +7,7 @@ import (
"github.com/go-gst/go-gst/gst/app"
"github.com/kaedwen/webrtc/pkg/common"
"github.com/pion/webrtc/v3/pkg/media"
"go.uber.org/zap"
)
func setCallback(sink *app.Sink, ch chan media.Sample) {
@@ -37,7 +38,25 @@ func setCallback(sink *app.Sink, ch chan media.Sample) {
})
}
func CreateVideoPipelineSink(s StreamElement) (*gst.Pipeline, <-chan media.Sample, error) {
func CreateVideoPipelineSinkWithLaunch(s StreamElement, _ *zap.Logger) (*gst.Pipeline, <-chan media.Sample, error) {
pipeline, err := gst.NewPipelineFromString(`v4l2src device=/dev/video4 ! capsfilter caps="video/x-raw,width=(int)320,height=(int)240" ! videoconvert ! queue ! capsfilter caps="video/x-raw,format=(string)I420" ! x264enc speed-preset=ultrafast tune=zerolatency key-int-max=20 ! capsfilter caps="video/x-h264,stream-format=(string)byte-stream" ! appsink name=appsink`)
if err != nil {
return nil, nil, err
}
elem, err := pipeline.GetElementByName("appsink")
if err != nil {
return nil, nil, err
}
appsink := app.SinkFromElement(elem)
ch := make(chan media.Sample, 100)
setCallback(appsink, ch)
return pipeline, ch, nil
}
func CreateVideoPipelineSink(s StreamElement, lg *zap.Logger) (*gst.Pipeline, <-chan media.Sample, error) {
// Create a pipeline
pipeline, err := gst.NewPipeline("pion-video-pipeline")
if err != nil {
@@ -62,9 +81,14 @@ func CreateVideoPipelineSink(s StreamElement) (*gst.Pipeline, <-chan media.Sampl
if err != nil {
return nil, nil, err
}
filter.Set("caps", s.ToGstCaps())
elems = append(elems, filter)
c := s.Caps.Build()
lg.Info("capsfilter", zap.String("caps", c.String()))
err = filter.SetProperty("caps", c)
if err != nil {
return nil, nil, err
}
}
// just to be on the save side
@@ -98,6 +122,20 @@ func CreateVideoPipelineSink(s StreamElement) (*gst.Pipeline, <-chan media.Sampl
enc.SetProperty("cpu-used", 5)
enc.SetProperty("deadline", 1)
case common.H264:
filterIn, err := gst.NewElement("capsfilter")
if err != nil {
return nil, nil, err
}
elems = append(elems, filterIn)
cIn := gst.NewEmptySimpleCaps("video/x-raw")
cIn.SetValue("format", "I420")
lg.Info("capsfilter", zap.String("caps", cIn.String()))
err = filterIn.SetProperty("caps", cIn)
if err != nil {
return nil, nil, err
}
// Create the enc
enc, err := gst.NewElement("x264enc")
if err != nil {
@@ -107,8 +145,22 @@ func CreateVideoPipelineSink(s StreamElement) (*gst.Pipeline, <-chan media.Sampl
enc.SetProperty("speed-preset", "ultrafast")
enc.SetProperty("tune", "zerolatency")
enc.SetProperty("key-int-max", 2)
enc.SetProperty("bitrate", 300)
enc.SetProperty("key-int-max", 20)
//enc.SetProperty("bitrate", 300)
filterOut, err := gst.NewElement("capsfilter")
if err != nil {
return nil, nil, err
}
elems = append(elems, filterOut)
cOut := gst.NewEmptySimpleCaps("video/x-h264")
cOut.SetValue("stream-format", "byte-stream")
lg.Info("capsfilter", zap.String("caps", cOut.String()))
err = filterOut.SetProperty("caps", cOut)
if err != nil {
return nil, nil, err
}
default:
return nil, nil, fmt.Errorf("unsupported video codec given - %s", s.Codec)
}
@@ -132,7 +184,7 @@ func CreateVideoPipelineSink(s StreamElement) (*gst.Pipeline, <-chan media.Sampl
return pipeline, ch, nil
}
func CreateAudioPipelineSink(s StreamElement) (*gst.Pipeline, <-chan media.Sample, error) {
func CreateAudioPipelineSink(s StreamElement, lg *zap.Logger) (*gst.Pipeline, <-chan media.Sample, error) {
// Create a pipeline
pipeline, err := gst.NewPipeline("pion-audio-pipeline")
if err != nil {
@@ -157,9 +209,14 @@ func CreateAudioPipelineSink(s StreamElement) (*gst.Pipeline, <-chan media.Sampl
if err != nil {
return nil, nil, err
}
filter.Set("caps", s.ToGstCaps())
elems = append(elems, filter)
c := s.Caps.Build()
lg.Info("capsfilter", zap.String("caps", c.String()))
err = filter.SetProperty("caps", c)
if err != nil {
return nil, nil, err
}
}
// just to be on the save side
pkg/streamer/gst.go
-50
View File
@@ -1,66 +1,16 @@
package streamer
import (
"fmt"
"reflect"
"strings"
"github.com/go-gst/go-gst/gst"
"github.com/go-gst/go-gst/gst/app"
"github.com/kaedwen/webrtc/pkg/common"
"go.uber.org/zap"
)
const capsTagName = "caps"
func init() {
gst.Init(nil)
}
type StreamElementCaps struct {
Mime string `caps:"mime"`
Height uint `caps:"height"`
Width uint `caps:"width"`
Format string `caps:"format"`
Channels uint `caps:"channels"`
Rate uint `caps:"rate"`
}
type StreamElement struct {
Kind string
Codec common.StreamCodec
Properties map[string]interface{}
Caps *StreamElementCaps
Queue bool
}
func (se StreamElement) ToGstCaps() *gst.Caps {
if se.Caps == nil {
return nil
}
t := reflect.ValueOf(*se.Caps)
var mime string
var caps []string
for i := 0; i < t.NumField(); i++ {
valueField := t.Field(i)
tagField := t.Type().Field(i).Tag.Get(capsTagName)
if tagField == "mime" {
mime = valueField.String()
} else {
if !valueField.IsZero() {
caps = append(caps, fmt.Sprintf("%s=%v", tagField, valueField.Interface()))
}
}
}
return gst.NewCapsFromString(strings.Join(append([]string{mime}, caps...), ","))
}
func handleMessage(msg *gst.Message) error {
switch msg.Type() {
case gst.MessageEOS:
return app.ErrEOS
pkg/streamer/model.go
+83
View File
@@ -0,0 +1,83 @@
package streamer
import (
"github.com/go-gst/go-gst/gst"
"github.com/kaedwen/webrtc/pkg/common"
)
type StreamElement struct {
Kind string
Codec common.StreamCodec
Properties map[string]interface{}
Caps *CapsBuilder
Queue bool
}
type CapsBuilder struct {
caps Caps
}
type Caps struct {
mime string
values []CapsValue
}
type CapsValue struct {
name string
value any
}
func NewCapsBuilder(mime string) *CapsBuilder {
return &CapsBuilder{
caps: Caps{mime, nil},
}
}
func (b *CapsBuilder) Height(v int) *CapsBuilder {
b.caps.values = append(b.caps.values, CapsValue{
name: "height",
value: v,
})
return b
}
func (b *CapsBuilder) Width(v int) *CapsBuilder {
b.caps.values = append(b.caps.values, CapsValue{
name: "width",
value: v,
})
return b
}
func (b *CapsBuilder) Channels(v int) *CapsBuilder {
b.caps.values = append(b.caps.values, CapsValue{
name: "channels",
value: v,
})
return b
}
func (b *CapsBuilder) Format(v string) *CapsBuilder {
b.caps.values = append(b.caps.values, CapsValue{
name: "format",
value: v,
})
return b
}
func (b *CapsBuilder) Rate(v int) *CapsBuilder {
b.caps.values = append(b.caps.values, CapsValue{
name: "rate",
value: v,
})
return b
}
func (b *CapsBuilder) Build() *gst.Caps {
caps := gst.NewEmptySimpleCaps(b.caps.mime)
for _, v := range b.caps.values {
caps.SetValue(v.name, v.value)
}
return caps
}
pkg/webrtc/webrtc.go
+6 -15
View File
@@ -120,18 +120,14 @@ func (wh *WebrtcHandler) handleAudioSamples(ctx context.Context, cfg *common.Con
src := streamer.StreamElement{
Kind: cfg.Source,
Properties: properties,
Caps: &streamer.StreamElementCaps{
Mime: "audio/x-raw",
Channels: cfg.Channels,
Rate: 48000,
},
Queue: cfg.Queue,
Codec: cfg.Codec,
Caps: streamer.NewCapsBuilder("audio/x-raw").Channels(int(cfg.Channels)).Rate(48000),
Queue: cfg.Queue,
Codec: cfg.Codec,
}
var err error
var audioCh <-chan media.Sample
wh.audioPipeline, audioCh, err = streamer.CreateAudioPipelineSink(src)
wh.audioPipeline, audioCh, err = streamer.CreateAudioPipelineSink(src, wh.lg)
if err != nil {
return err
}
@@ -164,19 +160,14 @@ func (wh *WebrtcHandler) handleVideoSamples(ctx context.Context, cfg *common.Con
Properties: map[string]interface{}{
"device": cfg.Device,
},
Caps: &streamer.StreamElementCaps{
Mime: "video/x-raw",
Format: "YUY2",
Width: cfg.Width,
Height: cfg.Height,
},
Caps: streamer.NewCapsBuilder("video/x-raw").Format("YUY2").Width(int(cfg.Width)).Height(int(cfg.Height)),
Queue: cfg.Queue,
Codec: cfg.Codec,
}
var err error
var videoCh <-chan media.Sample
wh.videoPipeline, videoCh, err = streamer.CreateVideoPipelineSink(src)
wh.videoPipeline, videoCh, err = streamer.CreateVideoPipelineSinkWithLaunch(src, wh.lg)
if err != nil {
return err
}
vendor/github.com/blackjack/webcam/.gitignore
Generated
+1
View File
@@ -0,0 +1 @@
.DS_Store
vendor/github.com/blackjack/webcam/.travis.yml
Generated Vendored
+8
View File
@@ -0,0 +1,8 @@
language: go
before_install:
- sudo apt-get install libv4l-dev
go:
- tip
- 1.9
vendor/github.com/blackjack/webcam/LICENSE
Generated Vendored
+22
View File
@@ -0,0 +1,22 @@
The MIT License (MIT)
Copyright (c) <2015> <Oleksandr Senkovych>
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in
all copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
THE SOFTWARE.
vendor/github.com/blackjack/webcam/README.md
Generated Vendored
+69
View File
@@ -0,0 +1,69 @@
# go-webcam
[![Build Status](https://travis-ci.org/blackjack/webcam.png?branch=master)](https://travis-ci.org/blackjack/webcam) [![GoDoc](https://godoc.org/github.com/google/go-github/github?status.svg)](https://godoc.org/github.com/blackjack/webcam)
This is a **go** library for working with webcams and other video capturing devices.
It depends entirely on [V4L2](http://linuxtv.org/downloads/v4l-dvb-apis/) framework, thus will compile and work only on **Linux** machine.
## Installation
```console
$ go get github.com/blackjack/webcam
```
## Usage
```go
import "github.com/blackjack/webcam"
// ...
cam, err := webcam.Open("/dev/video0") // Open webcam
if err != nil { panic(err.Error()) }
defer cam.Close()
// ...
// Setup webcam image format and frame size here (see examples or documentation)
// ...
err = cam.StartStreaming()
if err != nil { panic(err.Error()) }
for {
err = cam.WaitForFrame(timeout)
switch err.(type) {
case nil:
case *webcam.Timeout:
fmt.Fprint(os.Stderr, err.Error())
continue
default:
panic(err.Error())
}
frame, err := cam.ReadFrame()
if len(frame) != 0 {
// Process frame
} else if err != nil {
panic(err.Error())
}
}
```
For more detailed example see [examples folder](https://github.com/blackjack/webcam/tree/master/examples)
The number of frame buffers used may be set as:
```go
// If already streaming, stop streaming.
if streaming_on {
cam.StopStreaming()
}
err = cam.SetBufferCount(64)
```
## Roadmap
The library is still under development so API changes can happen. Currently library supports streaming
using only MMAP method, which should be sufficient for most of devices available on the market.
Other streaming methods can be added in future (please create issue if you need this).
Also currently image format is defined by 4-byte code received from V4L2, which is good in terms of
compatibility with different versions of Linux kernel, but not very handy if you want to do some image manipulations.
Plans are to aligh V4L2 image format codes with [Image](https://golang.org/pkg/image/) package from Go library.
## License
See LICENSE file
vendor/github.com/blackjack/webcam/errors.go
Generated Vendored
+8
View File
@@ -0,0 +1,8 @@
package webcam
// Timeout error
type Timeout struct{}
func (e *Timeout) Error() string {
return "Timeout occured"
}
vendor/github.com/blackjack/webcam/formats.go
Generated Vendored
+58
View File
@@ -0,0 +1,58 @@
package webcam
import "fmt"
// Represents image format code used by V4L2 subsystem.
// Number of formats can be different in various
// Linux kernel versions
// See /usr/include/linux/videodev2.h for full list
// of supported image formats
type PixelFormat uint32
// Struct that describes frame size supported by a webcam
// For fixed sizes min and max values will be the same and
// step value will be equal to '0'
type FrameSize struct {
MinWidth uint32
MaxWidth uint32
StepWidth uint32
MinHeight uint32
MaxHeight uint32
StepHeight uint32
}
// FrameRate represents all possible framerates supported by a webcam
// for a given pixel format and frame size. For discrete returns min
// and max values will be the same and step value will be equal to '0'
// Stepwise returns are represented as a range of values with a step.
type FrameRate struct {
MinNumerator uint32
MaxNumerator uint32
StepNumerator uint32
MinDenominator uint32
MaxDenominator uint32
StepDenominator uint32
}
// Return string representation of FrameRate struct. e.g.1/30 for
// discrete framerates and [1-2;1]/[10-60;1] for stepwise framerates.
func (f FrameRate) String() string {
if f.StepNumerator == 0 && f.StepDenominator == 0 {
return fmt.Sprintf("%d/%d", f.MinNumerator, f.MinDenominator)
} else {
return fmt.Sprintf("[%d-%d;%d]/[%d-%d;%d]", f.MinNumerator, f.MaxNumerator, f.StepNumerator, f.MinDenominator, f.MaxDenominator, f.StepDenominator)
}
}
// Returns string representation of frame size, e.g.
// 1280x720 for fixed-size frames and
// [320-640;160]x[240-480;160] for stepwise-sized frames
func (s FrameSize) GetString() string {
if s.StepWidth == 0 && s.StepHeight == 0 {
return fmt.Sprintf("%dx%d", s.MaxWidth, s.MaxHeight)
} else {
return fmt.Sprintf("[%d-%d;%d]x[%d-%d;%d]", s.MinWidth, s.MaxWidth, s.StepWidth, s.MinHeight, s.MaxHeight, s.StepHeight)
}
}
vendor/github.com/blackjack/webcam/ioctl/LICENSE
Generated Vendored
+14
View File
@@ -0,0 +1,14 @@
Copyright (c) 2012 Mark Wolfe
Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation
files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy,
modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software
is furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR
IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
vendor/github.com/blackjack/webcam/ioctl/ioctl.go
Generated Vendored
+57
View File
@@ -0,0 +1,57 @@
package ioctl
import "golang.org/x/sys/unix"
const (
typeBits = 8
numberBits = 8
sizeBits = 14
directionBits = 2
typeMask = (1 << typeBits) - 1
numberMask = (1 << numberBits) - 1
sizeMask = (1 << sizeBits) - 1
directionMask = (1 << directionBits) - 1
directionNone = 0
directionWrite = 1
directionRead = 2
numberShift = 0
typeShift = numberShift + numberBits
sizeShift = typeShift + typeBits
directionShift = sizeShift + sizeBits
)
func ioc(dir, t, nr, size uintptr) uintptr {
return (dir << directionShift) | (t << typeShift) | (nr << numberShift) | (size << sizeShift)
}
// Io used for a simple ioctl that sends nothing but the type and number, and receives back nothing but an (integer) retval.
func Io(t, nr uintptr) uintptr {
return ioc(directionNone, t, nr, 0)
}
// IoR used for an ioctl that reads data from the device driver. The driver will be allowed to return sizeof(data_type) bytes to the user.
func IoR(t, nr, size uintptr) uintptr {
return ioc(directionRead, t, nr, size)
}
// IoW used for an ioctl that writes data to the device driver.
func IoW(t, nr, size uintptr) uintptr {
return ioc(directionWrite, t, nr, size)
}
// IoRW a combination of IoR and IoW. That is, data is both written to the driver and then read back from the driver by the client.
func IoRW(t, nr, size uintptr) uintptr {
return ioc(directionRead|directionWrite, t, nr, size)
}
// Ioctl simplified ioct call
func Ioctl(fd, op, arg uintptr) error {
_, _, ep := unix.Syscall(unix.SYS_IOCTL, fd, op, arg)
if ep != 0 {
return ep
}
return nil
}
vendor/github.com/blackjack/webcam/v4l2.go
Generated Vendored
+676
View File
@@ -0,0 +1,676 @@
package webcam
import (
"bytes"
"encoding/binary"
"fmt"
"unsafe"
"github.com/blackjack/webcam/ioctl"
"golang.org/x/sys/unix"
)
/*
V4L2 constants, enums, and structs are derived from the header file
'videodev2.h'. See the following reference for more information:
https://www.kernel.org/doc/html/latest/userspace-api/media/v4l/videodev.html#videodev2-h
*/
type controlType int
const (
c_int controlType = iota
c_bool
c_menu
)
type control struct {
id uint32
name string
c_type controlType
step int32
min int32
max int32
}
const (
V4L2_CAP_VIDEO_CAPTURE uint32 = 0x00000001
V4L2_CAP_STREAMING uint32 = 0x04000000
V4L2_BUF_TYPE_VIDEO_CAPTURE uint32 = 1
V4L2_MEMORY_MMAP uint32 = 1
V4L2_FIELD_ANY uint32 = 0
)
const (
V4L2_FRMSIZE_TYPE_DISCRETE uint32 = 1
V4L2_FRMSIZE_TYPE_CONTINUOUS uint32 = 2
V4L2_FRMSIZE_TYPE_STEPWISE uint32 = 3
)
const (
V4L2_FRMIVAL_TYPE_DISCRETE uint32 = 1
V4L2_FRMIVAL_TYPE_CONTINUOUS uint32 = 2
V4L2_FRMIVAL_TYPE_STEPWISE uint32 = 3
)
const (
V4L2_CID_BASE uint32 = 0x00980900
V4L2_CID_AUTO_WHITE_BALANCE uint32 = V4L2_CID_BASE + 12
V4L2_CID_PRIVATE_BASE uint32 = 0x08000000
)
const (
V4L2_CTRL_TYPE_INTEGER uint32 = 1
V4L2_CTRL_TYPE_BOOLEAN uint32 = 2
V4L2_CTRL_TYPE_MENU uint32 = 3
V4L2_CTRL_TYPE_BUTTON uint32 = 4
V4L2_CTRL_TYPE_INTEGER64 uint32 = 5
V4L2_CTRL_TYPE_CTRL_CLASS uint32 = 6
V4L2_CTRL_TYPE_STRING uint32 = 7
V4L2_CTRL_TYPE_BITMASK uint32 = 8
V4L2_CTRL_TYPE_INTEGER_MENU uint32 = 9
V4L2_CTRL_COMPOUND_TYPES uint32 = 0x0100
V4L2_CTRL_TYPE_U8 uint32 = 0x0100
V4L2_CTRL_TYPE_U16 uint32 = 0x0101
V4L2_CTRL_TYPE_U32 uint32 = 0x0102
)
const (
V4L2_CTRL_FLAG_DISABLED uint32 = 0x00000001
V4L2_CTRL_FLAG_NEXT_CTRL uint32 = 0x80000000
)
var (
VIDIOC_QUERYCAP = ioctl.IoR(uintptr('V'), 0, unsafe.Sizeof(v4l2_capability{}))
VIDIOC_ENUM_FMT = ioctl.IoRW(uintptr('V'), 2, unsafe.Sizeof(v4l2_fmtdesc{}))
VIDIOC_S_FMT = ioctl.IoRW(uintptr('V'), 5, unsafe.Sizeof(v4l2_format{}))
VIDIOC_REQBUFS = ioctl.IoRW(uintptr('V'), 8, unsafe.Sizeof(v4l2_requestbuffers{}))
VIDIOC_QUERYBUF = ioctl.IoRW(uintptr('V'), 9, unsafe.Sizeof(v4l2_buffer{}))
VIDIOC_QBUF = ioctl.IoRW(uintptr('V'), 15, unsafe.Sizeof(v4l2_buffer{}))
VIDIOC_DQBUF = ioctl.IoRW(uintptr('V'), 17, unsafe.Sizeof(v4l2_buffer{}))
VIDIOC_G_PARM = ioctl.IoRW(uintptr('V'), 21, unsafe.Sizeof(v4l2_streamparm{}))
VIDIOC_S_PARM = ioctl.IoRW(uintptr('V'), 22, unsafe.Sizeof(v4l2_streamparm{}))
VIDIOC_G_CTRL = ioctl.IoRW(uintptr('V'), 27, unsafe.Sizeof(v4l2_control{}))
VIDIOC_S_CTRL = ioctl.IoRW(uintptr('V'), 28, unsafe.Sizeof(v4l2_control{}))
VIDIOC_QUERYCTRL = ioctl.IoRW(uintptr('V'), 36, unsafe.Sizeof(v4l2_queryctrl{}))
//sizeof int32
VIDIOC_STREAMON = ioctl.IoW(uintptr('V'), 18, 4)
VIDIOC_STREAMOFF = ioctl.IoW(uintptr('V'), 19, 4)
VIDIOC_G_INPUT = ioctl.IoR(uintptr('V'), 38, 4)
VIDIOC_S_INPUT = ioctl.IoRW(uintptr('V'), 39, 4)
VIDIOC_ENUM_FRAMESIZES = ioctl.IoRW(uintptr('V'), 74, unsafe.Sizeof(v4l2_frmsizeenum{}))
VIDIOC_ENUM_FRAMEINTERVALS = ioctl.IoRW(uintptr('V'), 75, unsafe.Sizeof(v4l2_frmivalenum{}))
__p = unsafe.Pointer(uintptr(0))
NativeByteOrder = getNativeByteOrder()
)
type v4l2_capability struct {
driver [16]uint8
card [32]uint8
bus_info [32]uint8
version uint32
capabilities uint32
device_caps uint32
reserved [3]uint32
}
type v4l2_fmtdesc struct {
index uint32
_type uint32
flags uint32
description [32]uint8
pixelformat uint32
reserved [4]uint32
}
type v4l2_frmsizeenum struct {
index uint32
pixel_format uint32
_type uint32
union [24]uint8
reserved [2]uint32
}
type v4l2_frmsize_discrete struct {
Width uint32
Height uint32
}
type v4l2_frmsize_stepwise struct {
Min_width uint32
Max_width uint32
Step_width uint32
Min_height uint32
Max_height uint32
Step_height uint32
}
// v4l2_frmivalenum that contains a pixel format and size and receives a
// frame interval (time duration between frames of a video stream).
type v4l2_frmivalenum struct {
index uint32
pixel_format uint32
width uint32
height uint32
_type uint32
union [24]uint8
reserved [2]uint32
}
// v4l2_frmival_stepwise represents the frame interval range
// as minimum, maximum, and step size intervals.
type v4l2_frmival_stepwise struct {
min v4l2_fract // minimum frame interval [s]
max v4l2_fract // maximum frame interval [s]
step v4l2_fract // frame interval step size [s]
}
// Hack to make go compiler properly align union
type v4l2_format_aligned_union struct {
data [200 - unsafe.Sizeof(__p)]byte
_ unsafe.Pointer
}
type v4l2_format struct {
_type uint32
union v4l2_format_aligned_union
}
type v4l2_pix_format struct {
Width uint32
Height uint32
Pixelformat uint32
Field uint32
Bytesperline uint32
Sizeimage uint32
Colorspace uint32
Priv uint32
Flags uint32
Ycbcr_enc uint32
Quantization uint32
Xfer_func uint32
}
type v4l2_requestbuffers struct {
count uint32
_type uint32
memory uint32
reserved [2]uint32
}
type v4l2_buffer struct {
index uint32
_type uint32
bytesused uint32
flags uint32
field uint32
timestamp unix.Timeval
timecode v4l2_timecode
sequence uint32
memory uint32
union [unsafe.Sizeof(__p)]uint8
length uint32
reserved2 uint32
reserved uint32
}
type v4l2_timecode struct {
_type uint32
flags uint32
frames uint8
seconds uint8
minutes uint8
hours uint8
userbits [4]uint8
}
type v4l2_queryctrl struct {
id uint32
_type uint32
name [32]uint8
minimum int32
maximum int32
step int32
default_value int32
flags uint32
reserved [2]uint32
}
type v4l2_control struct {
id uint32
value int32
}
type v4l2_fract struct {
Numerator uint32
Denominator uint32
}
type v4l2_streamparm_union struct {
capability uint32
output_mode uint32
time_per_frame v4l2_fract
extended_mode uint32
buffers uint32
reserved [4]uint32
data [200 - (10 * unsafe.Sizeof(uint32(0)))]byte
}
type v4l2_streamparm struct {
_type uint32
union v4l2_streamparm_union
}
func checkCapabilities(fd uintptr) (supportsVideoCapture bool, supportsVideoStreaming bool, err error) {
caps := &v4l2_capability{}
err = ioctl.Ioctl(fd, VIDIOC_QUERYCAP, uintptr(unsafe.Pointer(caps)))
if err != nil {
return
}
supportsVideoCapture = (caps.capabilities & V4L2_CAP_VIDEO_CAPTURE) != 0
supportsVideoStreaming = (caps.capabilities & V4L2_CAP_STREAMING) != 0
return
}
func getPixelFormat(fd uintptr, index uint32) (code uint32, description string, err error) {
fmtdesc := &v4l2_fmtdesc{}
fmtdesc.index = index
fmtdesc._type = V4L2_BUF_TYPE_VIDEO_CAPTURE
err = ioctl.Ioctl(fd, VIDIOC_ENUM_FMT, uintptr(unsafe.Pointer(fmtdesc)))
if err != nil {
return
}
code = fmtdesc.pixelformat
description = CToGoString(fmtdesc.description[:])
return
}
func getFrameSize(fd uintptr, index uint32, code uint32) (frameSize FrameSize, err error) {
frmsizeenum := &v4l2_frmsizeenum{}
frmsizeenum.index = index
frmsizeenum.pixel_format = code
err = ioctl.Ioctl(fd, VIDIOC_ENUM_FRAMESIZES, uintptr(unsafe.Pointer(frmsizeenum)))
if err != nil {
return
}
switch frmsizeenum._type {
case V4L2_FRMSIZE_TYPE_DISCRETE:
discrete := &v4l2_frmsize_discrete{}
err = binary.Read(bytes.NewBuffer(frmsizeenum.union[:]), NativeByteOrder, discrete)
if err != nil {
return
}
frameSize.MinWidth = discrete.Width
frameSize.MaxWidth = discrete.Width
frameSize.StepWidth = 0
frameSize.MinHeight = discrete.Height
frameSize.MaxHeight = discrete.Height
frameSize.StepHeight = 0
case V4L2_FRMSIZE_TYPE_CONTINUOUS:
case V4L2_FRMSIZE_TYPE_STEPWISE:
stepwise := &v4l2_frmsize_stepwise{}
err = binary.Read(bytes.NewBuffer(frmsizeenum.union[:]), NativeByteOrder, stepwise)
if err != nil {
return
}
frameSize.MinWidth = stepwise.Min_width
frameSize.MaxWidth = stepwise.Max_width
frameSize.StepWidth = stepwise.Step_width
frameSize.MinHeight = stepwise.Min_height
frameSize.MaxHeight = stepwise.Max_height
frameSize.StepHeight = stepwise.Step_height
}
return
}
func getName(fd uintptr) (string, error) {
var caps v4l2_capability
if err := ioctl.Ioctl(fd, VIDIOC_QUERYCAP, uintptr(unsafe.Pointer(&caps))); err != nil {
return "", err
}
return CToGoString(caps.card[:]), nil
}
func getFrameInterval(fd uintptr, index uint32, code uint32, width uint32, height uint32) (FrameRate, error) {
frmivalEnum := &v4l2_frmivalenum{
index: index,
pixel_format: code,
width: width,
height: height,
}
if err := ioctl.Ioctl(fd, VIDIOC_ENUM_FRAMEINTERVALS, uintptr(unsafe.Pointer(frmivalEnum))); err != nil {
return FrameRate{}, err
}
switch frmivalEnum._type {
case V4L2_FRMIVAL_TYPE_DISCRETE:
discrete := &v4l2_fract{}
if err := binary.Read(bytes.NewBuffer(frmivalEnum.union[:]), NativeByteOrder, discrete); err != nil {
return FrameRate{}, err
}
return FrameRate{
MinDenominator: discrete.Denominator,
MaxDenominator: discrete.Denominator,
StepDenominator: 0,
MinNumerator: discrete.Numerator,
MaxNumerator: discrete.Numerator,
StepNumerator: 0,
}, nil
case V4L2_FRMIVAL_TYPE_CONTINUOUS:
return FrameRate{}, fmt.Errorf("V4L2_FRMIVAL_TYPE_CONTINUOUS not implemented")
case V4L2_FRMIVAL_TYPE_STEPWISE:
stepwise := &v4l2_frmival_stepwise{}
if err := binary.Read(bytes.NewBuffer(frmivalEnum.union[:]), NativeByteOrder, stepwise); err != nil {
return FrameRate{}, err
}
return FrameRate{
MinDenominator: stepwise.min.Denominator,
MaxDenominator: stepwise.max.Denominator,
StepDenominator: stepwise.step.Denominator,
MinNumerator: stepwise.min.Numerator,
MaxNumerator: stepwise.max.Numerator,
StepNumerator: stepwise.step.Numerator,
}, nil
}
return FrameRate{}, fmt.Errorf("unknown frame interval type")
}
func getBusInfo(fd uintptr) (string, error) {
var caps v4l2_capability
if err := ioctl.Ioctl(fd, VIDIOC_QUERYCAP, uintptr(unsafe.Pointer(&caps))); err != nil {
return "", err
}
return CToGoString(caps.bus_info[:]), nil
}
func setImageFormat(fd uintptr, formatcode *uint32, width *uint32, height *uint32) (err error) {
format := &v4l2_format{
_type: V4L2_BUF_TYPE_VIDEO_CAPTURE,
}
pix := v4l2_pix_format{
Width: *width,
Height: *height,
Pixelformat: *formatcode,
Field: V4L2_FIELD_ANY,
}
pixbytes := &bytes.Buffer{}
err = binary.Write(pixbytes, NativeByteOrder, pix)
if err != nil {
return
}
copy(format.union.data[:], pixbytes.Bytes())
err = ioctl.Ioctl(fd, VIDIOC_S_FMT, uintptr(unsafe.Pointer(format)))
if err != nil {
return
}
pixReverse := &v4l2_pix_format{}
err = binary.Read(bytes.NewBuffer(format.union.data[:]), NativeByteOrder, pixReverse)
if err != nil {
return
}
*width = pixReverse.Width
*height = pixReverse.Height
*formatcode = pixReverse.Pixelformat
return
}
func mmapRequestBuffers(fd uintptr, buf_count *uint32) (err error) {
req := &v4l2_requestbuffers{}
req.count = *buf_count
req._type = V4L2_BUF_TYPE_VIDEO_CAPTURE
req.memory = V4L2_MEMORY_MMAP
err = ioctl.Ioctl(fd, VIDIOC_REQBUFS, uintptr(unsafe.Pointer(req)))
if err != nil {
return
}
*buf_count = req.count
return
}
func mmapQueryBuffer(fd uintptr, index uint32, length *uint32) (buffer []byte, err error) {
req := &v4l2_buffer{}
req._type = V4L2_BUF_TYPE_VIDEO_CAPTURE
req.memory = V4L2_MEMORY_MMAP
req.index = index
err = ioctl.Ioctl(fd, VIDIOC_QUERYBUF, uintptr(unsafe.Pointer(req)))
if err != nil {
return
}
var offset uint32
err = binary.Read(bytes.NewBuffer(req.union[:]), NativeByteOrder, &offset)
if err != nil {
return
}
*length = req.length
buffer, err = unix.Mmap(int(fd), int64(offset), int(req.length), unix.PROT_READ|unix.PROT_WRITE, unix.MAP_SHARED)
return
}
func mmapDequeueBuffer(fd uintptr, index *uint32, length *uint32) (err error) {
buffer := &v4l2_buffer{}
buffer._type = V4L2_BUF_TYPE_VIDEO_CAPTURE
buffer.memory = V4L2_MEMORY_MMAP
err = ioctl.Ioctl(fd, VIDIOC_DQBUF, uintptr(unsafe.Pointer(buffer)))
if err != nil {
return
}
*index = buffer.index
*length = buffer.bytesused
return
}
func mmapEnqueueBuffer(fd uintptr, index uint32) (err error) {
buffer := &v4l2_buffer{}
buffer._type = V4L2_BUF_TYPE_VIDEO_CAPTURE
buffer.memory = V4L2_MEMORY_MMAP
buffer.index = index
err = ioctl.Ioctl(fd, VIDIOC_QBUF, uintptr(unsafe.Pointer(buffer)))
return
}
func mmapReleaseBuffer(buffer []byte) (err error) {
err = unix.Munmap(buffer)
return
}
func startStreaming(fd uintptr) (err error) {
var uintPointer uint32 = V4L2_BUF_TYPE_VIDEO_CAPTURE
err = ioctl.Ioctl(fd, VIDIOC_STREAMON, uintptr(unsafe.Pointer(&uintPointer)))
return
}
func stopStreaming(fd uintptr) (err error) {
var uintPointer uint32 = V4L2_BUF_TYPE_VIDEO_CAPTURE
err = ioctl.Ioctl(fd, VIDIOC_STREAMOFF, uintptr(unsafe.Pointer(&uintPointer)))
return
}
func waitForFrame(pollFds []unix.PollFd, timeout uint32) (count int, err error) {
for {
count, err = unix.Poll(pollFds, int(timeout*1000))
if count < 0 && err == unix.EINTR {
continue
}
return
}
}
func getControl(fd uintptr, id uint32) (int32, error) {
ctrl := &v4l2_control{}
ctrl.id = id
err := ioctl.Ioctl(fd, VIDIOC_G_CTRL, uintptr(unsafe.Pointer(ctrl)))
return ctrl.value, err
}
func setControl(fd uintptr, id uint32, val int32) error {
ctrl := &v4l2_control{}
ctrl.id = id
ctrl.value = val
return ioctl.Ioctl(fd, VIDIOC_S_CTRL, uintptr(unsafe.Pointer(ctrl)))
}
func getInput(fd uintptr) (index int32, err error) {
err = ioctl.Ioctl(fd, VIDIOC_G_INPUT, uintptr(unsafe.Pointer(&index)))
return
}
func selectInput(fd uintptr, index uint32) (err error) {
err = ioctl.Ioctl(fd, VIDIOC_S_INPUT, uintptr(unsafe.Pointer(&index)))
return
}
func getFramerate(fd uintptr) (float32, error) {
param := &v4l2_streamparm{}
param._type = V4L2_BUF_TYPE_VIDEO_CAPTURE
err := ioctl.Ioctl(fd, VIDIOC_G_PARM, uintptr(unsafe.Pointer(param)))
if err != nil {
return 0, err
}
tf := param.union.time_per_frame
if tf.Denominator == 0 || tf.Numerator == 0 {
return 0, fmt.Errorf("Invalid framerate (%d/%d)", tf.Denominator, tf.Numerator)
}
return float32(tf.Denominator) / float32(tf.Numerator), nil
}
func setFramerate(fd uintptr, num, denom uint32) error {
param := &v4l2_streamparm{}
param._type = V4L2_BUF_TYPE_VIDEO_CAPTURE
param.union.time_per_frame.Numerator = num
param.union.time_per_frame.Denominator = denom
return ioctl.Ioctl(fd, VIDIOC_S_PARM, uintptr(unsafe.Pointer(param)))
}
func queryControls(fd uintptr) []control {
controls := []control{}
var err error
// Don't use V42L_CID_BASE since it is the same as brightness.
var id uint32
for err == nil {
id |= V4L2_CTRL_FLAG_NEXT_CTRL
query := &v4l2_queryctrl{}
query.id = id
err = ioctl.Ioctl(fd, VIDIOC_QUERYCTRL, uintptr(unsafe.Pointer(query)))
id = query.id
if err == nil {
if (query.flags & V4L2_CTRL_FLAG_DISABLED) != 0 {
continue
}
var c control
switch query._type {
default:
continue
case V4L2_CTRL_TYPE_INTEGER, V4L2_CTRL_TYPE_INTEGER64:
c.c_type = c_int
case V4L2_CTRL_TYPE_BOOLEAN:
c.c_type = c_bool
case V4L2_CTRL_TYPE_MENU:
c.c_type = c_menu
}
c.id = id
c.name = CToGoString(query.name[:])
c.min = query.minimum
c.max = query.maximum
c.step = query.step
controls = append(controls, c)
}
}
return controls
}
func getNativeByteOrder() binary.ByteOrder {
var i int32 = 0x01020304
u := unsafe.Pointer(&i)
pb := (*byte)(u)
b := *pb
if b == 0x04 {
return binary.LittleEndian
} else {
return binary.BigEndian
}
}
func CToGoString(c []byte) string {
n := -1
for i, b := range c {
if b == 0 {
break
}
n = i
}
return string(c[:n+1])
}
vendor/github.com/blackjack/webcam/webcam.go
Generated Vendored
+351
View File
@@ -0,0 +1,351 @@
// Library for working with webcams and other video capturing devices.
// It depends entirely on v4l2 framework, thus will compile and work
// only on Linux machine
package webcam
import (
"errors"
"reflect"
"unsafe"
"golang.org/x/sys/unix"
)
// Webcam object
type Webcam struct {
fd uintptr
bufcount uint32
buffers [][]byte
streaming bool
pollFds []unix.PollFd
}
type ControlID uint32
type Control struct {
Name string
Min int32
Max int32
Type int32
Step int32
}
// Open a webcam with a given path
// Checks if device is a v4l2 device and if it is
// capable to stream video
func Open(path string) (*Webcam, error) {
handle, err := unix.Open(path, unix.O_RDWR|unix.O_NONBLOCK, 0666)
fd := uintptr(handle)
if fd < 0 || err != nil {
return nil, err
}
supportsVideoCapture, supportsVideoStreaming, err := checkCapabilities(fd)
if err != nil {
return nil, err
}
if !supportsVideoCapture {
return nil, errors.New("Not a video capture device")
}
if !supportsVideoStreaming {
return nil, errors.New("Device does not support the streaming I/O method")
}
w := new(Webcam)
w.fd = fd
w.bufcount = 256
w.pollFds = []unix.PollFd{{Fd: int32(fd), Events: unix.POLLIN}}
return w, nil
}
// Returns image formats supported by the device alongside with
// their text description
// Note that this function is somewhat experimental. Frames are not ordered in
// any meaning, also duplicates can occur so it's up to developer to clean it up.
// See http://linuxtv.org/downloads/v4l-dvb-apis/vidioc-enum-framesizes.html
// for more information
func (w *Webcam) GetSupportedFormats() map[PixelFormat]string {
result := make(map[PixelFormat]string)
var err error
var code uint32
var desc string
var index uint32
for index = 0; err == nil; index++ {
code, desc, err = getPixelFormat(w.fd, index)
if err != nil {
break
}
result[PixelFormat(code)] = desc
}
return result
}
// GetName returns the human-readable name of the device
func (w *Webcam) GetName() (string, error) {
return getName(w.fd)
}
// GetBusInfo returns the location of the device in the system
func (w *Webcam) GetBusInfo() (string, error) {
return getBusInfo(w.fd)
}
// SelectInput selects the current video input.
func (w *Webcam) SelectInput(index uint32) error {
return selectInput(w.fd, index)
}
// GetInput queries the current video input.
func (w *Webcam) GetInput() (int32, error) {
return getInput(w.fd)
}
// Returns supported frame sizes for a given image format
func (w *Webcam) GetSupportedFrameSizes(f PixelFormat) []FrameSize {
result := make([]FrameSize, 0)
var index uint32
var err error
for index = 0; err == nil; index++ {
s, err := getFrameSize(w.fd, index, uint32(f))
if err != nil {
break
}
result = append(result, s)
}
return result
}
// GetSupportedFramerates returns supported frame rates for a given image format and frame size.
func (w *Webcam) GetSupportedFramerates(fp PixelFormat, width uint32, height uint32) []FrameRate {
var result []FrameRate
var index uint32
var err error
// keep incrementing the index value until we get an EINVAL error
index = 0
for err == nil {
r, err := getFrameInterval(w.fd, index, uint32(fp), width, height)
if err != nil {
break
}
result = append(result, r)
index++
}
return result
}
// Sets desired image format and frame size
// Note, that device driver can change that values.
// Resulting values are returned by a function
// alongside with an error if any
func (w *Webcam) SetImageFormat(f PixelFormat, width, height uint32) (PixelFormat, uint32, uint32, error) {
code := uint32(f)
cw := width
ch := height
err := setImageFormat(w.fd, &code, &width, &height)
if err != nil {
return 0, 0, 0, err
} else {
return PixelFormat(code), cw, ch, nil
}
}
// Set the number of frames to be buffered.
// Not allowed if streaming is already on.
func (w *Webcam) SetBufferCount(count uint32) error {
if w.streaming {
return errors.New("Cannot set buffer count when streaming")
}
w.bufcount = count
return nil
}
// Get a map of available controls.
func (w *Webcam) GetControls() map[ControlID]Control {
cmap := make(map[ControlID]Control)
for _, c := range queryControls(w.fd) {
cmap[ControlID(c.id)] = Control{
Name: c.name,
Min: c.min,
Max: c.max,
Type: int32(c.c_type),
Step: c.step,
}
}
return cmap
}
// Get the value of a control.
func (w *Webcam) GetControl(id ControlID) (int32, error) {
return getControl(w.fd, uint32(id))
}
// Set a control.
func (w *Webcam) SetControl(id ControlID, value int32) error {
return setControl(w.fd, uint32(id), value)
}
// Get the framerate.
func (w *Webcam) GetFramerate() (float32, error) {
return getFramerate(w.fd)
}
// Set FPS
func (w *Webcam) SetFramerate(fps float32) error {
return setFramerate(w.fd, 1000, uint32(1000*(fps)))
}
// Start streaming process
func (w *Webcam) StartStreaming() error {
if w.streaming {
return errors.New("Already streaming")
}
err := mmapRequestBuffers(w.fd, &w.bufcount)
if err != nil {
return errors.New("Failed to map request buffers: " + string(err.Error()))
}
w.buffers = make([][]byte, w.bufcount, w.bufcount)
for index, _ := range w.buffers {
var length uint32
buffer, err := mmapQueryBuffer(w.fd, uint32(index), &length)
if err != nil {
return errors.New("Failed to map memory: " + string(err.Error()))
}
w.buffers[index] = buffer
}
for index, _ := range w.buffers {
err := mmapEnqueueBuffer(w.fd, uint32(index))
if err != nil {
return errors.New("Failed to enqueue buffer: " + string(err.Error()))
}
}
err = startStreaming(w.fd)
if err != nil {
return errors.New("Failed to start streaming: " + string(err.Error()))
}
w.streaming = true
return nil
}
// Read a single frame from the webcam
// If frame cannot be read at the moment
// function will return empty slice
func (w *Webcam) ReadFrame() ([]byte, error) {
result, index, err := w.GetFrame()
if err == nil {
w.ReleaseFrame(index)
}
return result, err
}
// Get a single frame from the webcam and return the frame and
// the buffer index. To return the buffer, ReleaseFrame must be called.
// If frame cannot be read at the moment
// function will return empty slice
func (w *Webcam) GetFrame() ([]byte, uint32, error) {
var index uint32
var length uint32
err := mmapDequeueBuffer(w.fd, &index, &length)
if err != nil {
return nil, 0, err
}
return w.buffers[int(index)][:length], index, nil
}
// Release the frame buffer that was obtained via GetFrame
func (w *Webcam) ReleaseFrame(index uint32) error {
return mmapEnqueueBuffer(w.fd, index)
}
// Wait until frame could be read
func (w *Webcam) WaitForFrame(timeout uint32) error {
count, err := waitForFrame(w.pollFds, timeout)
if count < 0 || err != nil {
return err
} else if count == 0 {
return new(Timeout)
} else {
return nil
}
}
func (w *Webcam) StopStreaming() error {
if !w.streaming {
return errors.New("Request to stop streaming when not streaming")
}
w.streaming = false
for _, buffer := range w.buffers {
err := mmapReleaseBuffer(buffer)
if err != nil {
return err
}
}
return stopStreaming(w.fd)
}
// Close the device
func (w *Webcam) Close() error {
if w.streaming {
w.StopStreaming()
}
err := unix.Close(int(w.fd))
return err
}
// Sets automatic white balance correction
func (w *Webcam) SetAutoWhiteBalance(val bool) error {
v := int32(0)
if val {
v = 1
}
return setControl(w.fd, V4L2_CID_AUTO_WHITE_BALANCE, v)
}
func gobytes(p unsafe.Pointer, n int) []byte {
h := reflect.SliceHeader{uintptr(p), n, n}
s := *(*[]byte)(unsafe.Pointer(&h))
return s
}
vendor/github.com/pion/mediadevices/.clang-format
Generated Vendored
+106
View File
@@ -0,0 +1,106 @@
---
Language: Cpp
# BasedOnStyle: LLVM
AccessModifierOffset: -2
AlignAfterOpenBracket: Align
AlignConsecutiveAssignments: false
AlignConsecutiveDeclarations: false
AlignEscapedNewlines: Right
AlignOperands: true
AlignTrailingComments: true
AllowAllParametersOfDeclarationOnNextLine: true
AllowShortBlocksOnASingleLine: false
AllowShortCaseLabelsOnASingleLine: false
AllowShortFunctionsOnASingleLine: All
AllowShortIfStatementsOnASingleLine: false
AllowShortLoopsOnASingleLine: false
AlwaysBreakAfterDefinitionReturnType: None
AlwaysBreakAfterReturnType: None
AlwaysBreakBeforeMultilineStrings: false
AlwaysBreakTemplateDeclarations: false
BinPackArguments: true
BinPackParameters: true
BraceWrapping:
AfterClass: false
AfterControlStatement: false
AfterEnum: false
AfterFunction: false
AfterNamespace: false
AfterObjCDeclaration: false
AfterStruct: false
AfterUnion: false
BeforeCatch: false
BeforeElse: false
IndentBraces: false
SplitEmptyFunction: true
SplitEmptyRecord: true
SplitEmptyNamespace: true
BreakBeforeBinaryOperators: None
BreakBeforeBraces: Attach
BreakBeforeInheritanceComma: false
BreakBeforeTernaryOperators: true
BreakConstructorInitializersBeforeComma: false
BreakConstructorInitializers: BeforeColon
BreakAfterJavaFieldAnnotations: false
BreakStringLiterals: true
ColumnLimit: 120
CommentPragmas: '^ IWYU pragma:'
CompactNamespaces: false
ConstructorInitializerAllOnOneLineOrOnePerLine: false
ConstructorInitializerIndentWidth: 4
ContinuationIndentWidth: 4
Cpp11BracedListStyle: true
DerivePointerAlignment: false
DisableFormat: false
ExperimentalAutoDetectBinPacking: false
FixNamespaceComments: true
ForEachMacros:
- foreach
- Q_FOREACH
- BOOST_FOREACH
IncludeCategories:
- Regex: '^"(llvm|llvm-c|clang|clang-c)/'
Priority: 2
- Regex: '^(<|"(gtest|gmock|isl|json)/)'
Priority: 3
- Regex: '.*'
Priority: 1
IncludeIsMainRegex: '(Test)?$'
IndentCaseLabels: false
IndentWidth: 2
IndentWrappedFunctionNames: false
JavaScriptQuotes: Leave
JavaScriptWrapImports: true
KeepEmptyLinesAtTheStartOfBlocks: true
MacroBlockBegin: ''
MacroBlockEnd: ''
MaxEmptyLinesToKeep: 1
NamespaceIndentation: None
ObjCBlockIndentWidth: 2
ObjCSpaceAfterProperty: false
ObjCSpaceBeforeProtocolList: true
PenaltyBreakAssignment: 2
PenaltyBreakBeforeFirstCallParameter: 19
PenaltyBreakComment: 300
PenaltyBreakFirstLessLess: 120
PenaltyBreakString: 1000
PenaltyExcessCharacter: 1000000
PenaltyReturnTypeOnItsOwnLine: 60
PointerAlignment: Right
ReflowComments: true
SortIncludes: true
SortUsingDeclarations: true
SpaceAfterCStyleCast: false
SpaceAfterTemplateKeyword: true
SpaceBeforeAssignmentOperators: true
SpaceBeforeParens: ControlStatements
SpaceInEmptyParentheses: false
SpacesBeforeTrailingComments: 1
SpacesInAngles: false
SpacesInContainerLiterals: true
SpacesInCStyleCastParentheses: false
SpacesInParentheses: false
SpacesInSquareBrackets: false
Standard: Cpp11
TabWidth: 8
UseTab: Never
vendor/github.com/pion/mediadevices/.gitignore
Generated
+17
View File
@@ -0,0 +1,17 @@
# Binaries for programs and plugins
*.exe
*.exe~
*.dll
*.so
*.dylib
# Test binary, build with `go test -c`
*.test
# Output of the go coverage tool, specifically when used with LiteIDE
*.out
scripts/cross
coverage.txt
.idea
vendor/github.com/pion/mediadevices/.gitmodules
Generated
View File
vendor/github.com/pion/mediadevices/LICENSE
Generated Vendored
+21
View File
@@ -0,0 +1,21 @@
MIT License
Copyright (c) 2019-2020 Pion
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.
vendor/github.com/pion/mediadevices/Makefile
Generated Vendored
+83
View File
@@ -0,0 +1,83 @@
docker_owner := lherman
docker_prefix := cross
toolchain_dockerfiles := dockerfiles
script_path := $(realpath scripts)
toolchain_path := $(script_path)/$(docker_prefix)
os_list := \
linux \
windows \
darwin
arch_list := \
armv7 \
arm64 \
x64
supported_platforms := \
linux-armv7 \
linux-arm64 \
linux-x64 \
windows-x64 \
darwin-x64 \
darwin-arm64
cmd_build := build
cmd_test := test
examples_dir := examples
codec_dir := pkg/codec
codec_list := $(shell ls $(codec_dir)/*/Makefile)
codec_list := $(codec_list:$(codec_dir)/%/Makefile=%)
targets := $(foreach codec, $(codec_list), $(addprefix $(cmd_build)-$(codec)-, $(supported_platforms)))
pkgs_without_ext_device := $(shell go list ./... | grep -v mmal | grep -v vaapi)
pkgs_without_cgo := $(shell go list ./... | grep -v pkg/codec | grep -v pkg/driver | grep -v pkg/avfoundation)
define BUILD_TEMPLATE
ifneq (,$$(findstring $(2)-$(3),$$(supported_platforms)))
$$(cmd_build)-$(1)-$(2)-$(3): toolchain-$(2)-$(3)
$$(MAKE) --directory=$$(codec_dir)/$(1) \
MEDIADEVICES_TOOLCHAIN_BIN=$$(toolchain_path)/$(docker_prefix)-$(2)-$(3) \
MEDIADEVICES_TARGET_PLATFORM=$(2)-$(3) \
MEDIADEVICES_TARGET_OS=$(2) \
MEDIADEVICES_TARGET_ARCH=$(3)
endif
endef
.PHONY: all
all: $(cmd_test) $(cmd_build)
# Subcommand:
# make build[-<codec_name>-<os>-<arch>]
#
# Description:
# Build codec dependencies to multiple platforms.
#
# Examples:
# * make build: build all codecs for all supported platforms
# * make build-opus-darwin-x64: only build opus for darwin-x64 platform
$(cmd_build): $(targets)
toolchain-%: $(toolchain_dockerfiles)
$(MAKE) --directory=$< "$*" \
MEDIADEVICES_DOCKER_OWNER=$(docker_owner) \
MEDIADEVICES_DOCKER_PREFIX=$(docker_prefix)
@mkdir -p $(toolchain_path)
@docker run $(docker_owner)/$(docker_prefix)-$* > \
$(toolchain_path)/$(docker_prefix)-$*
@chmod +x $(toolchain_path)/$(docker_prefix)-$*
$(foreach codec, $(codec_list), \
$(foreach os, $(os_list), \
$(foreach arch, $(arch_list), \
$(eval $(call BUILD_TEMPLATE,$(codec),$(os),$(arch))))))
# Subcommand:
# make test
#
# Description:
# Run a series of tests
$(cmd_test):
go vet $(pkgs_without_ext_device)
go build $(pkgs_without_ext_device)
# go build without CGO
CGO_ENABLED=0 go build $(pkgs_without_cgo)
# go build with CGO
CGO_ENABLED=1 go build $(pkgs_without_ext_device)
$(MAKE) --directory=$(examples_dir)
go test -v -race -coverprofile=coverage.txt -covermode=atomic $(pkgs_without_ext_device)
vendor/github.com/pion/mediadevices/README.md
Generated Vendored
+232
View File
@@ -0,0 +1,232 @@
<h1 align="center">
<br>
Pion MediaDevices
<br>
</h1>
<h4 align="center">Go implementation of the <a href="https://developer.mozilla.org/en-US/docs/Web/API/MediaDevices">MediaDevices</a> API</h4>
<p align="center">
<a href="https://pion.ly/slack"><img src="https://img.shields.io/badge/join-us%20on%20slack-gray.svg?longCache=true&logo=slack&colorB=brightgreen" alt="Slack Widget"></a>
<img alt="GitHub Workflow Status" src="https://img.shields.io/github/actions/workflow/status/pion/mediadevices/test.yaml">
<a href="https://pkg.go.dev/github.com/pion/mediadevices"><img src="https://pkg.go.dev/badge/github.com/pion/mediadevices.svg" alt="Go Reference"></a>
<a href="https://codecov.io/gh/pion/mediadevices"><img src="https://codecov.io/gh/pion/mediadevices/branch/master/graph/badge.svg" alt="Coverage Status"></a>
<a href="LICENSE"><img src="https://img.shields.io/badge/License-MIT-yellow.svg" alt="License: MIT"></a>
</p>
<br>
`mediadevices` provides access to media input devices like cameras, microphones, and screen capture. It can also be used to encode your video/audio stream to various codec selections. `mediadevices` abstracts away the complexities of interacting with things like hardware and codecs allowing you to focus on building appilcations, interacting only with an amazingly simple, easy, and elegant API!
### Install
```bash
go get -u github.com/pion/mediadevices
```
### Usage
The following snippet shows how to capture a camera stream and store a frame as a jpeg image:
```go
package main
import (
"image/jpeg"
"os"
"github.com/pion/mediadevices"
"github.com/pion/mediadevices/pkg/prop"
// This is required to register camera adapter
_ "github.com/pion/mediadevices/pkg/driver/camera"
// Note: If you don't have a camera or your adapters are not supported,
// you can always swap your adapters with our dummy adapters below.
// _ "github.com/pion/mediadevices/pkg/driver/videotest"
)
func main() {
stream, _ := mediadevices.GetUserMedia(mediadevices.MediaStreamConstraints{
Video: func(constraint *mediadevices.MediaTrackConstraints) {
// Query for ideal resolutions
constraint.Width = prop.Int(600)
constraint.Height = prop.Int(400)
},
})
// Since track can represent audio as well, we need to cast it to
// *mediadevices.VideoTrack to get video specific functionalities
track := stream.GetVideoTracks()[0]
videoTrack := track.(*mediadevices.VideoTrack)
defer videoTrack.Close()
// Create a new video reader to get the decoded frames. Release is used
// to return the buffer to hold frame back to the source so that the buffer
// can be reused for the next frames.
videoReader := videoTrack.NewReader(false)
frame, release, _ := videoReader.Read()
defer release()
// Since frame is the standard image.Image, it's compatible with Go standard
// library. For example, capturing the first frame and store it as a jpeg image.
output, _ := os.Create("frame.jpg")
jpeg.Encode(output, frame, nil)
}
```
### More Examples
* [Webrtc](/examples/webrtc) - Use Webrtc to create a realtime peer-to-peer video call
* [Face Detection](/examples/facedetection) - Use a machine learning algorithm to detect faces in a camera stream
* [RTP Stream](examples/rtp) - Capture camera stream, encode it in H264/VP8/VP9, and send it to a RTP server
* [HTTP Broadcast](/examples/http) - Broadcast camera stream through HTTP with MJPEG
* [Archive](/examples/archive) - Archive H264 encoded video stream from a camera
### Available Media Inputs
| Input | Linux | Mac | Windows |
| :--------: | :---: | :-: | :-----: |
| Camera | ✔️ | ✔️ | ✔️ |
| Microphone | ✔️ | ✔️ | ✔️ |
| Screen | ✔️ | ✔️ | ✔️ |
By default, there's no media input registered. This decision was made to allow you to play only what you need. Therefore, you need to import the associated packages for the media inputs. For example, if you want to use a camera, you need to import the camera package as a side effect:
```go
import (
...
_ "github.com/pion/mediadevices/pkg/driver/camera"
)
```
### Available Codecs
In order to encode your video/audio, `mediadevices` needs to know what codecs that you want to use and their parameters. To do this, you need to import the associated packages for the codecs, and add them to the codec selector that you'll pass to `GetUserMedia`:
```go
package main
import (
"github.com/pion/mediadevices"
"github.com/pion/mediadevices/pkg/codec/x264" // This is required to use H264 video encoder
_ "github.com/pion/mediadevices/pkg/driver/camera" // This is required to register camera adapter
)
func main() {
// configure codec specific parameters
x264Params, _ := x264.NewParams()
x264Params.Preset = x264.PresetMedium
x264Params.BitRate = 1_000_000 // 1mbps
codecSelector := mediadevices.NewCodecSelector(
mediadevices.WithVideoEncoders(&x264Params),
)
mediaStream, _ := mediadevices.GetUserMedia(mediadevices.MediaStreamConstraints{
Video: func(c *mediadevices.MediaTrackConstraints) {},
Codec: codecSelector, // let GetUsermedia know available codecs
})
}
```
Since `mediadevices` doesn't implement the video/audio codecs, it needs to call the codec libraries from the system through cgo. Therefore, you're required to install the codec libraries before you can use them in `mediadevices`. In the next section, it shows a list of available codecs, where the packages are defined (documentation linked), and installation instructions.
Note: we do not provide recommendations on choosing one codec or another as it is very complex and can be subjective.
#### Video Codecs
##### x264
A free software library and application for encoding video streams into the H.264/MPEG-4 AVC compression format.
* Package: [github.com/pion/mediadevices/pkg/codec/x264](https://pkg.go.dev/github.com/pion/mediadevices/pkg/codec/x264)
* Installation:
* Mac: `brew install x264`
* Ubuntu: `apt install libx264-dev`
##### mmal
A framework to enable H264 hardware encoding for Raspberry Pi or boards that use VideoCore GPUs.
* Package: [github.com/pion/mediadevices/pkg/codec/mmal](https://pkg.go.dev/github.com/pion/mediadevices/pkg/codec/mmal)
* Installation: no installation needed, mmal should come built in Raspberry Pi devices
##### openh264
A codec library which supports H.264 encoding and decoding. It is suitable for use in real time applications.
* Package: [github.com/pion/mediadevices/pkg/codec/openh264](https://pkg.go.dev/github.com/pion/mediadevices/pkg/codec/openh264)
* Installation: no installation needed, included as a static binary
##### vpx
A free software video codec library from Google and the Alliance for Open Media that implements VP8/VP9 video coding formats.
* Package: [github.com/pion/mediadevices/pkg/codec/vpx](https://pkg.go.dev/github.com/pion/mediadevices/pkg/codec/vpx)
* Installation:
* Mac: `brew install libvpx`
* Ubuntu: `apt install libvpx-dev`
##### vaapi
An open source API that allows applications such as VLC media player or GStreamer to use hardware video acceleration capabilities (currently support VP8/VP9).
* Package: [github.com/pion/mediadevices/pkg/codec/vaapi](https://pkg.go.dev/github.com/pion/mediadevices/pkg/codec/vaapi)
* Installation:
* Ubuntu: `apt install libva-dev`
#### Audio Codecs
##### opus
A totally open, royalty-free, highly versatile audio codec.
* Package: [github.com/pion/mediadevices/pkg/codec/opus](https://pkg.go.dev/github.com/pion/mediadevices/pkg/codec/opus)
* Installation:
* Mac: `brew install opus`
* Ubuntu: `apt install libopus-dev`
### Benchmark
Result as of Nov 4, 2020 with Go 1.14 on a Raspberry pi 3, `mediadevices` can produce video, encode, send across network, and decode at **720p, 30 fps with < 500 ms latency**.
The test was taken by capturing a camera stream, decoding the raw frames, encoding the video stream with mmal, and sending the stream through Webrtc.
### FAQ
#### Failed to find the best driver that fits the constraints
`mediadevices` provides an automated driver discovery through `GetUserMedia` and `GetDisplayMedia`. The driver discover algorithm works something like:
1. Open all registered drivers
2. Get all properties (property describes what a driver is capable of, e.g. resolution, frame rate, etc.) from opened drivers
3. Find the best property that meets the criteria
So, when `mediadevices` returns `failed to find the best driver that fits the constraints` error, one of the following conditions might have occured:
* Driver was not imported as a side effect in your program, e.g. `import _ github.com/pion/mediadevices/pkg/driver/camera`
* Your constraint is too strict that there's no driver can fullfil your requirements. In this case, you can try to turn up the debug level by specifying the following environment variable: `export PION_LOG_DEBUG=all` to see what was too strict and tune that.
* Your driver is not supported/implemented. In this case, you can either let us know (file an issue) and wait for the maintainers to implement it. Or, you can implement it yourself and register it through `RegisterDriverAdapter`
* If trying to use `import _ github.com/pion/mediadevices/pkg/driver/screen` note that you will need to use `GetDisplayMedia` instead of `GetUserMedia`
#### Failed to find vpx/x264/mmal/opus codecs
Since `mediadevices` uses cgo to access video/audio codecs, it needs to find these libraries from the system. To accomplish this, [pkg-config](https://www.freedesktop.org/wiki/Software/pkg-config/) is used for library discovery.
If you see the following error message at compile time:
```bash
# pkg-config --cflags -- vpx
Package vpx was not found in the pkg-config search path.
Perhaps you should add the directory containing `vpx.pc'
to the PKG_CONFIG_PATH environment variable
No package 'vpx' found
pkg-config: exit status 1
```
There are 2 common problems:
* The required codec library is not installed (vpx in this example). In this case, please refer to the [available codecs](#available-codecs).
* Pkg-config fails to find the `.pc` files for this codec ([reference](https://people.freedesktop.org/~dbn/pkg-config-guide.html#using)). In this case, you need to find where the codec library's `.pc` is stored, and let pkg-config knows with: `export PKG_CONFIG_PATH=/path/to/directory`.
### Roadmap
The library can be used with our WebRTC implementation. Please refer to that [roadmap](https://github.com/pion/webrtc/issues/9) to track our major milestones.
### Community
Pion has an active community on the [Slack](https://pion.ly/slack).
Follow the [Pion Twitter](https://twitter.com/_pion) for project updates and important WebRTC news.
We are always looking to support **your projects**. Please reach out if you have something to build!
If you need commercial support or don't want to use public methods you can contact us at [team@pion.ly](mailto:team@pion.ly)
### Contributing
Check out the [contributing wiki](https://github.com/pion/webrtc/wiki/Contributing) to join the group of amazing people making this project possible
### License
MIT License - see [LICENSE](LICENSE) for full text
vendor/github.com/pion/mediadevices/codec.go
Generated Vendored
+141
View File
@@ -0,0 +1,141 @@
package mediadevices
import (
"errors"
"fmt"
"strings"
"github.com/pion/mediadevices/pkg/codec"
"github.com/pion/mediadevices/pkg/io/audio"
"github.com/pion/mediadevices/pkg/io/video"
"github.com/pion/mediadevices/pkg/prop"
"github.com/pion/webrtc/v3"
)
// CodecSelector is a container of video and audio encoder builders, which later will be used
// for codec matching.
type CodecSelector struct {
videoEncoders []codec.VideoEncoderBuilder
audioEncoders []codec.AudioEncoderBuilder
}
// CodecSelectorOption is a type for specifying CodecSelector options
type CodecSelectorOption func(*CodecSelector)
// WithVideoEncoders replace current video codecs with listed encoders
func WithVideoEncoders(encoders ...codec.VideoEncoderBuilder) CodecSelectorOption {
return func(t *CodecSelector) {
t.videoEncoders = encoders
}
}
// WithVideoEncoders replace current audio codecs with listed encoders
func WithAudioEncoders(encoders ...codec.AudioEncoderBuilder) CodecSelectorOption {
return func(t *CodecSelector) {
t.audioEncoders = encoders
}
}
// NewCodecSelector constructs CodecSelector with given variadic options
func NewCodecSelector(opts ...CodecSelectorOption) *CodecSelector {
var track CodecSelector
for _, opt := range opts {
opt(&track)
}
return &track
}
// Populate lets the webrtc engine be aware of supported codecs that are contained in CodecSelector
func (selector *CodecSelector) Populate(setting *webrtc.MediaEngine) {
for _, encoder := range selector.videoEncoders {
setting.RegisterCodec(encoder.RTPCodec().RTPCodecParameters, webrtc.RTPCodecTypeVideo)
}
for _, encoder := range selector.audioEncoders {
setting.RegisterCodec(encoder.RTPCodec().RTPCodecParameters, webrtc.RTPCodecTypeAudio)
}
}
// selectVideoCodecByNames selects a single codec that can be built and matched. codecNames can be formatted as "video/<codecName>" or "<codecName>"
func (selector *CodecSelector) selectVideoCodecByNames(reader video.Reader, inputProp prop.Media, codecNames ...string) (codec.ReadCloser, *codec.RTPCodec, error) {
var selectedEncoder codec.VideoEncoderBuilder
var encodedReader codec.ReadCloser
var errReasons []string
var err error
outer:
for _, wantCodec := range codecNames {
wantCodecLower := strings.ToLower(wantCodec)
for _, encoder := range selector.videoEncoders {
// MimeType is formated as "video/<codecName>"
if strings.HasSuffix(strings.ToLower(encoder.RTPCodec().MimeType), wantCodecLower) {
encodedReader, err = encoder.BuildVideoEncoder(reader, inputProp)
if err == nil {
selectedEncoder = encoder
break outer
}
}
errReasons = append(errReasons, fmt.Sprintf("%s: %s", encoder.RTPCodec().MimeType, err))
}
}
if selectedEncoder == nil {
return nil, nil, errors.New(strings.Join(errReasons, "\n\n"))
}
return encodedReader, selectedEncoder.RTPCodec(), nil
}
func (selector *CodecSelector) selectVideoCodec(reader video.Reader, inputProp prop.Media, codecs ...webrtc.RTPCodecParameters) (codec.ReadCloser, *codec.RTPCodec, error) {
var codecNames []string
for _, codec := range codecs {
codecNames = append(codecNames, codec.MimeType)
}
return selector.selectVideoCodecByNames(reader, inputProp, codecNames...)
}
// selectAudioCodecByNames selects a single codec that can be built and matched. codecNames can be formatted as "audio/<codecName>" or "<codecName>"
func (selector *CodecSelector) selectAudioCodecByNames(reader audio.Reader, inputProp prop.Media, codecNames ...string) (codec.ReadCloser, *codec.RTPCodec, error) {
var selectedEncoder codec.AudioEncoderBuilder
var encodedReader codec.ReadCloser
var errReasons []string
var err error
outer:
for _, wantCodec := range codecNames {
wantCodecLower := strings.ToLower(wantCodec)
for _, encoder := range selector.audioEncoders {
// MimeType is formated as "audio/<codecName>"
if strings.HasSuffix(strings.ToLower(encoder.RTPCodec().MimeType), wantCodecLower) {
encodedReader, err = encoder.BuildAudioEncoder(reader, inputProp)
if err == nil {
selectedEncoder = encoder
break outer
}
}
errReasons = append(errReasons, fmt.Sprintf("%s: %s", encoder.RTPCodec().MimeType, err))
}
}
if selectedEncoder == nil {
return nil, nil, errors.New(strings.Join(errReasons, "\n\n"))
}
return encodedReader, selectedEncoder.RTPCodec(), nil
}
func (selector *CodecSelector) selectAudioCodec(reader audio.Reader, inputProp prop.Media, codecs ...webrtc.RTPCodecParameters) (codec.ReadCloser, *codec.RTPCodec, error) {
var codecNames []string
for _, codec := range codecs {
codecNames = append(codecNames, codec.MimeType)
}
return selector.selectAudioCodecByNames(reader, inputProp, codecNames...)
}
vendor/github.com/pion/mediadevices/codecov.yml
Generated Vendored
+10
View File
@@ -0,0 +1,10 @@
coverage:
status:
project:
default:
# Allow decreasing 2% of total coverage to avoid noise.
threshold: 2%
patch: off
ignore:
- "examples/*"
vendor/github.com/pion/mediadevices/driver.go
Generated Vendored
+10
View File
@@ -0,0 +1,10 @@
package mediadevices
import (
"github.com/pion/mediadevices/pkg/driver"
)
// RegisterDriverAdapter allows user space level of driver registration
func RegisterDriverAdapter(a driver.Adapter, info driver.Info) error {
return driver.GetManager().Register(a, info)
}
vendor/github.com/pion/mediadevices/internal/logging/logging.go
Generated Vendored
+11
View File
@@ -0,0 +1,11 @@
package logging
import (
"github.com/pion/logging"
)
var loggerFactory = logging.NewDefaultLoggerFactory()
func NewLogger(scope string) logging.LeveledLogger {
return loggerFactory.NewLogger(scope)
}
vendor/github.com/pion/mediadevices/ioreader.go
Generated Vendored
+75
View File
@@ -0,0 +1,75 @@
package mediadevices
import "github.com/pion/mediadevices/pkg/codec"
type EncodedBuffer struct {
Data []byte
Samples uint32
}
type EncodedReadCloser interface {
Read() (EncodedBuffer, func(), error)
Close() error
codec.Controllable
}
type encodedReadCloserImpl struct {
readFn func() (EncodedBuffer, func(), error)
closeFn func() error
controllerFn func() codec.EncoderController
}
func (r *encodedReadCloserImpl) Read() (EncodedBuffer, func(), error) {
return r.readFn()
}
func (r *encodedReadCloserImpl) Close() error {
return r.closeFn()
}
func (r *encodedReadCloserImpl) Controller() codec.EncoderController {
return r.controllerFn()
}
type encodedIOReadCloserImpl struct {
readFn func([]byte) (int, error)
closeFn func() error
controller func() codec.EncoderController
}
func newEncodedIOReadCloserImpl(reader EncodedReadCloser) *encodedIOReadCloserImpl {
var encoded EncodedBuffer
release := func() {}
return &encodedIOReadCloserImpl{
readFn: func(b []byte) (int, error) {
var err error
if len(encoded.Data) == 0 {
release()
encoded, release, err = reader.Read()
if err != nil {
reader.Close()
return 0, err
}
}
n := copy(b, encoded.Data)
encoded.Data = encoded.Data[n:]
return n, nil
},
closeFn: reader.Close,
controller: reader.Controller,
}
}
func (r *encodedIOReadCloserImpl) Read(b []byte) (int, error) {
return r.readFn(b)
}
func (r *encodedIOReadCloserImpl) Close() error {
return r.closeFn()
}
func (r *encodedIOReadCloserImpl) Controller() codec.EncoderController {
return r.controller()
}
vendor/github.com/pion/mediadevices/logging.go
Generated Vendored
+7
View File
@@ -0,0 +1,7 @@
package mediadevices
import (
"github.com/pion/mediadevices/internal/logging"
)
var logger = logging.NewLogger("mediadevices")
vendor/github.com/pion/mediadevices/mediadeviceinfo.go
Generated Vendored
+21
View File
@@ -0,0 +1,21 @@
package mediadevices
import "github.com/pion/mediadevices/pkg/driver"
// MediaDeviceType enumerates type of media device.
type MediaDeviceType int
// MediaDeviceType definitions.
const (
VideoInput MediaDeviceType = iota + 1
AudioInput
AudioOutput
)
// MediaDeviceInfo represents https://w3c.github.io/mediacapture-main/#dom-mediadeviceinfo
type MediaDeviceInfo struct {
DeviceID string
Kind MediaDeviceType
Label string
DeviceType driver.DeviceType
}
vendor/github.com/pion/mediadevices/mediadevices.go
Generated Vendored
+222
View File
@@ -0,0 +1,222 @@
package mediadevices
import (
"fmt"
"math"
"strings"
"github.com/pion/mediadevices/pkg/driver"
"github.com/pion/mediadevices/pkg/prop"
)
var errNotFound = fmt.Errorf("failed to find the best driver that fits the constraints")
// GetDisplayMedia prompts the user to select and grant permission to capture the contents
// of a display or portion thereof (such as a window) as a MediaStream.
// Reference: https://developer.mozilla.org/en-US/docs/Web/API/MediaDevices/getDisplayMedia
func GetDisplayMedia(constraints MediaStreamConstraints) (MediaStream, error) {
trackers := make([]Track, 0)
cleanTrackers := func() {
for _, t := range trackers {
t.Close()
}
}
var videoConstraints MediaTrackConstraints
if constraints.Video != nil {
constraints.Video(&videoConstraints)
tracker, err := selectScreen(videoConstraints, constraints.Codec)
if err != nil {
cleanTrackers()
return nil, err
}
trackers = append(trackers, tracker)
}
s, err := NewMediaStream(trackers...)
if err != nil {
cleanTrackers()
return nil, err
}
return s, nil
}
// GetUserMedia prompts the user for permission to use a media input which produces a MediaStream
// with tracks containing the requested types of media.
// Reference: https://developer.mozilla.org/en-US/docs/Web/API/MediaDevices/getUserMedia
func GetUserMedia(constraints MediaStreamConstraints) (MediaStream, error) {
// TODO: It should return media stream based on constraints
trackers := make([]Track, 0)
cleanTrackers := func() {
for _, t := range trackers {
t.Close()
}
}
var videoConstraints, audioConstraints MediaTrackConstraints
if constraints.Video != nil {
constraints.Video(&videoConstraints)
tracker, err := selectVideo(videoConstraints, constraints.Codec)
if err != nil {
cleanTrackers()
return nil, err
}
trackers = append(trackers, tracker)
}
if constraints.Audio != nil {
constraints.Audio(&audioConstraints)
tracker, err := selectAudio(audioConstraints, constraints.Codec)
if err != nil {
cleanTrackers()
return nil, err
}
trackers = append(trackers, tracker)
}
s, err := NewMediaStream(trackers...)
if err != nil {
cleanTrackers()
return nil, err
}
return s, nil
}
func queryDriverProperties(filter driver.FilterFn) map[driver.Driver][]prop.Media {
var needToClose []driver.Driver
drivers := driver.GetManager().Query(filter)
m := make(map[driver.Driver][]prop.Media)
for _, d := range drivers {
if d.Status() == driver.StateClosed {
err := d.Open()
if err != nil {
// Skip this driver if we failed to open because we can't get the properties
continue
}
needToClose = append(needToClose, d)
}
m[d] = d.Properties()
}
for _, d := range needToClose {
// Since it was closed, we should close it to avoid a leak
d.Close()
}
return m
}
// select implements SelectSettings algorithm.
// Reference: https://w3c.github.io/mediacapture-main/#dfn-selectsettings
func selectBestDriver(filter driver.FilterFn, constraints MediaTrackConstraints) (driver.Driver, MediaTrackConstraints, error) {
var bestDriver driver.Driver
var bestProp prop.Media
var foundPropertiesLog []string
minFitnessDist := math.Inf(1)
foundPropertiesLog = append(foundPropertiesLog, "\n============ Found Properties ============")
driverProperties := queryDriverProperties(filter)
for d, props := range driverProperties {
priority := float64(d.Info().Priority)
for _, p := range props {
foundPropertiesLog = append(foundPropertiesLog, p.String())
fitnessDist, ok := constraints.MediaConstraints.FitnessDistance(p)
if !ok {
continue
}
fitnessDist -= priority
if fitnessDist < minFitnessDist {
minFitnessDist = fitnessDist
bestDriver = d
bestProp = p
}
}
}
foundPropertiesLog = append(foundPropertiesLog, "=============== Constraints ==============")
foundPropertiesLog = append(foundPropertiesLog, constraints.String())
foundPropertiesLog = append(foundPropertiesLog, "================ Best Fit ================")
if bestDriver == nil {
foundPropertiesLog = append(foundPropertiesLog, "Not found")
logger.Debug(strings.Join(foundPropertiesLog, "\n\n"))
return nil, MediaTrackConstraints{}, errNotFound
}
foundPropertiesLog = append(foundPropertiesLog, bestProp.String())
logger.Debug(strings.Join(foundPropertiesLog, "\n\n"))
constraints.selectedMedia = prop.Media{}
constraints.selectedMedia.MergeConstraints(constraints.MediaConstraints)
constraints.selectedMedia.Merge(bestProp)
return bestDriver, constraints, nil
}
func selectAudio(constraints MediaTrackConstraints, selector *CodecSelector) (Track, error) {
typeFilter := driver.FilterAudioRecorder()
d, c, err := selectBestDriver(typeFilter, constraints)
if err != nil {
return nil, err
}
return newTrackFromDriver(d, c, selector)
}
func selectVideo(constraints MediaTrackConstraints, selector *CodecSelector) (Track, error) {
typeFilter := driver.FilterVideoRecorder()
notScreenFilter := driver.FilterNot(driver.FilterDeviceType(driver.Screen))
filter := driver.FilterAnd(typeFilter, notScreenFilter)
d, c, err := selectBestDriver(filter, constraints)
if err != nil {
return nil, err
}
return newTrackFromDriver(d, c, selector)
}
func selectScreen(constraints MediaTrackConstraints, selector *CodecSelector) (Track, error) {
typeFilter := driver.FilterVideoRecorder()
screenFilter := driver.FilterDeviceType(driver.Screen)
filter := driver.FilterAnd(typeFilter, screenFilter)
d, c, err := selectBestDriver(filter, constraints)
if err != nil {
return nil, err
}
return newTrackFromDriver(d, c, selector)
}
func EnumerateDevices() []MediaDeviceInfo {
drivers := driver.GetManager().Query(
driver.FilterFn(func(driver.Driver) bool { return true }))
info := make([]MediaDeviceInfo, 0, len(drivers))
for _, d := range drivers {
var kind MediaDeviceType
switch {
case driver.FilterVideoRecorder()(d):
kind = VideoInput
case driver.FilterAudioRecorder()(d):
kind = AudioInput
default:
continue
}
driverInfo := d.Info()
info = append(info, MediaDeviceInfo{
DeviceID: d.ID(),
Kind: kind,
Label: driverInfo.Label,
DeviceType: driverInfo.DeviceType,
})
}
return info
}
vendor/github.com/pion/mediadevices/mediastream.go
Generated Vendored
+88
View File
@@ -0,0 +1,88 @@
package mediadevices
import (
"sync"
"github.com/pion/webrtc/v3"
)
// MediaStream is an interface that represents a collection of existing tracks.
type MediaStream interface {
// GetAudioTracks implements https://w3c.github.io/mediacapture-main/#dom-mediastream-getaudiotracks
GetAudioTracks() []Track
// GetVideoTracks implements https://w3c.github.io/mediacapture-main/#dom-mediastream-getvideotracks
GetVideoTracks() []Track
// GetTracks implements https://w3c.github.io/mediacapture-main/#dom-mediastream-gettracks
GetTracks() []Track
// AddTrack implements https://w3c.github.io/mediacapture-main/#dom-mediastream-addtrack
AddTrack(t Track)
// RemoveTrack implements https://w3c.github.io/mediacapture-main/#dom-mediastream-removetrack
RemoveTrack(t Track)
}
type mediaStream struct {
tracks map[Track]struct{}
l sync.RWMutex
}
const trackTypeDefault webrtc.RTPCodecType = 0
// NewMediaStream creates a MediaStream interface that's defined in
// https://w3c.github.io/mediacapture-main/#dom-mediastream
func NewMediaStream(tracks ...Track) (MediaStream, error) {
m := mediaStream{tracks: make(map[Track]struct{})}
for _, track := range tracks {
if _, ok := m.tracks[track]; !ok {
m.tracks[track] = struct{}{}
}
}
return &m, nil
}
func (m *mediaStream) GetAudioTracks() []Track {
return m.queryTracks(webrtc.RTPCodecTypeAudio)
}
func (m *mediaStream) GetVideoTracks() []Track {
return m.queryTracks(webrtc.RTPCodecTypeVideo)
}
func (m *mediaStream) GetTracks() []Track {
return m.queryTracks(trackTypeDefault)
}
// queryTracks returns all tracks that are the same kind as t.
// If t is 0, which is the default, queryTracks will return all the tracks.
func (m *mediaStream) queryTracks(t webrtc.RTPCodecType) []Track {
m.l.RLock()
defer m.l.RUnlock()
result := make([]Track, 0)
for track := range m.tracks {
if track.Kind() == t || t == trackTypeDefault {
result = append(result, track)
}
}
return result
}
func (m *mediaStream) AddTrack(t Track) {
m.l.Lock()
defer m.l.Unlock()
if _, ok := m.tracks[t]; ok {
return
}
m.tracks[t] = struct{}{}
}
func (m *mediaStream) RemoveTrack(t Track) {
m.l.Lock()
defer m.l.Unlock()
delete(m.tracks, t)
}
vendor/github.com/pion/mediadevices/mediastreamconstraints.go
Generated Vendored
+19
View File
@@ -0,0 +1,19 @@
package mediadevices
import (
"github.com/pion/mediadevices/pkg/prop"
)
type MediaStreamConstraints struct {
Audio MediaOption
Video MediaOption
Codec *CodecSelector
}
// MediaTrackConstraints represents https://w3c.github.io/mediacapture-main/#dom-mediatrackconstraints
type MediaTrackConstraints struct {
prop.MediaConstraints
selectedMedia prop.Media
}
type MediaOption func(*MediaTrackConstraints)
vendor/github.com/pion/mediadevices/meta.go
Generated Vendored
+35
View File
@@ -0,0 +1,35 @@
package mediadevices
import (
"github.com/pion/mediadevices/pkg/io/audio"
"github.com/pion/mediadevices/pkg/io/video"
"github.com/pion/mediadevices/pkg/prop"
)
// detectCurrentVideoProp is a small helper to get current video property
func detectCurrentVideoProp(broadcaster *video.Broadcaster) (prop.Media, error) {
var currentProp prop.Media
// Since broadcaster has a ring buffer internally, a new reader will either read the last
// buffered frame or a new frame from the source. This also implies that no frame will be lost
// in any case.
metaReader := broadcaster.NewReader(false)
metaReader = video.DetectChanges(0, 0, func(p prop.Media) { currentProp = p })(metaReader)
_, _, err := metaReader.Read()
return currentProp, err
}
// detectCurrentAudioProp is a small helper to get current audio property
func detectCurrentAudioProp(broadcaster *audio.Broadcaster) (prop.Media, error) {
var currentProp prop.Media
// Since broadcaster has a ring buffer internally, a new reader will either read the last
// buffered frame or a new frame from the source. This also implies that no frame will be lost
// in any case.
metaReader := broadcaster.NewReader(false)
metaReader = audio.DetectChanges(0, func(p prop.Media) { currentProp = p })(metaReader)
_, _, err := metaReader.Read()
return currentProp, err
}
vendor/github.com/pion/mediadevices/pkg/avfoundation/.gitignore
Generated
+25
View File
@@ -0,0 +1,25 @@
#User settings
xcuserdata/
## compatibility with Xcode 8 and earlier (ignoring not required starting Xcode 9)
*.xcscmblueprint
*.xccheckout
## compatibility with Xcode 3 and earlier (ignoring not required starting Xcode 4)
build/
DerivedData/
*.moved-aside
*.pbxuser
!default.pbxuser
*.mode1v3
!default.mode1v3
*.mode2v3
!default.mode2v3
*.perspectivev3
!default.perspectivev3
## Gcc Patch
/*.gcno
.DS_STORE
Build/
vendor/github.com/pion/mediadevices/pkg/avfoundation/avfoundation_callback_darwin.go
Generated Vendored
+54
View File
@@ -0,0 +1,54 @@
package avfoundation
import "C"
import (
"sync"
"unsafe"
)
var mu sync.Mutex
var nextID handleID
type dataCb func(data []byte)
var handles = make(map[handleID]dataCb)
type handleID int
//export onData
func onData(userData unsafe.Pointer, buf unsafe.Pointer, length C.int) {
handleNum := (*C.int)(userData)
cb, ok := lookup(handleID(*handleNum))
if ok {
data := C.GoBytes(buf, length)
cb(data)
}
}
func register(fn dataCb) handleID {
mu.Lock()
defer mu.Unlock()
nextID++
for handles[nextID] != nil {
nextID++
}
handles[nextID] = fn
return nextID
}
func lookup(i handleID) (cb dataCb, ok bool) {
mu.Lock()
defer mu.Unlock()
cb, ok = handles[i]
return
}
func unregister(i handleID) {
mu.Lock()
defer mu.Unlock()
delete(handles, i)
}
vendor/github.com/pion/mediadevices/pkg/avfoundation/avfoundation_darwin.go
Generated Vendored
+260
View File
@@ -0,0 +1,260 @@
// Package avfoundation provides AVFoundation binding for Go
package avfoundation
// #cgo CFLAGS: -x objective-c
// #cgo LDFLAGS: -framework AVFoundation -framework Foundation -framework CoreMedia -framework CoreVideo
// #include "AVFoundationBind/AVFoundationBind.h"
// #include "AVFoundationBind/AVFoundationBind.m"
// extern void onData(void*, void*, int);
// void onDataBridge(void *userData, void *buf, int len) {
// onData(userData, buf, len);
// }
import "C"
import (
"context"
"fmt"
"io"
"sync"
"unsafe"
"github.com/pion/mediadevices/pkg/frame"
"github.com/pion/mediadevices/pkg/prop"
)
type MediaType C.AVBindMediaType
const (
Video = MediaType(C.AVBindMediaTypeVideo)
Audio = MediaType(C.AVBindMediaTypeAudio)
)
// Device represents a metadata that later can be used to retrieve back the
// underlying device given by AVFoundation
type Device struct {
// UID is a unique identifier for a device
UID string
cDevice C.AVBindDevice
Name string
}
func frameFormatToAVBind(f frame.Format) (C.AVBindFrameFormat, bool) {
switch f {
case frame.FormatI420:
return C.AVBindFrameFormatI420, true
case frame.FormatNV21:
return C.AVBindFrameFormatNV21, true
case frame.FormatNV12:
return C.AVBindFrameFormatNV12, true
case frame.FormatYUYV:
return C.AVBindFrameFormatYUYV, true
case frame.FormatUYVY:
return C.AVBindFrameFormatUYVY, true
default:
return 0, false
}
}
func frameFormatFromAVBind(f C.AVBindFrameFormat) (frame.Format, bool) {
switch f {
case C.AVBindFrameFormatI420:
return frame.FormatI420, true
case C.AVBindFrameFormatNV21:
return frame.FormatNV21, true
case C.AVBindFrameFormatNV12:
return frame.FormatNV12, true
case C.AVBindFrameFormatYUYV:
return frame.FormatYUYV, true
case C.AVBindFrameFormatUYVY:
return frame.FormatUYVY, true
default:
return "", false
}
}
// Devices uses AVFoundation to query a list of devices based on the media type
func Devices(mediaType MediaType) ([]Device, error) {
var cDevicesPtr C.PAVBindDevice
var cDevicesLen C.int
status := C.AVBindDevices(C.AVBindMediaType(mediaType), &cDevicesPtr, &cDevicesLen)
if status != nil {
return nil, fmt.Errorf("%s", C.GoString(status))
}
// https://github.com/golang/go/wiki/cgo#turning-c-arrays-into-go-slices
cDevices := (*[1 << 28]C.AVBindDevice)(unsafe.Pointer(cDevicesPtr))[:cDevicesLen:cDevicesLen]
devices := make([]Device, cDevicesLen)
for i := range devices {
devices[i].UID = C.GoString(&cDevices[i].uid[0])
devices[i].cDevice = cDevices[i]
devices[i].Name = C.GoString(&cDevices[i].name[0])
}
return devices, nil
}
// ReadCloser is a wrapper around the data callback from AVFoundation. The data received from the
// the underlying callback can be retrieved by calling Read.
type ReadCloser struct {
dataChan chan []byte
id handleID
onClose func()
cancelCtx context.Context
cancelFunc func()
closeWG sync.WaitGroup
lock sync.Mutex
}
func newReadCloser(onClose func()) *ReadCloser {
var rc ReadCloser
rc.dataChan = make(chan []byte, 1)
rc.onClose = onClose
rc.id = register(rc.dataCb)
cancelCtx, cancelFunc := context.WithCancel(context.Background())
rc.cancelCtx = cancelCtx
rc.cancelFunc = cancelFunc
return &rc
}
func (rc *ReadCloser) dataCb(data []byte) {
rc.closeWG.Add(1)
defer rc.closeWG.Done()
// TODO: add a policy for slow reader
if rc.cancelCtx.Err() != nil {
return
}
select {
// Use the Done channel to avoid waiting for new data from closed camera
case <-rc.cancelCtx.Done():
case rc.dataChan <- data:
}
}
// Read reads raw data, the format is determined by the media type and property:
// - For video, each call will return a frame.
// - For audio, each call will return a chunk which its size configured by Latency
func (rc *ReadCloser) Read() ([]byte, func(), error) {
data, ok := <-rc.dataChan
if !ok {
return nil, func() {}, io.EOF
}
return data, func() {}, nil
}
// Close closes the capturing session, and no data will flow anymore
func (rc *ReadCloser) Close() {
rc.lock.Lock()
defer rc.lock.Unlock()
if rc.cancelCtx.Err() != nil {
return // already closed
}
if rc.onClose != nil {
rc.onClose()
}
rc.cancelFunc()
unregister(rc.id)
rc.closeWG.Wait()
close(rc.dataChan)
}
// Session represents a capturing session.
type Session struct {
device Device
cSession C.PAVBindSession
lock sync.Mutex
closed bool
}
// NewSession creates a new capturing session
func NewSession(device Device) (*Session, error) {
var session Session
status := C.AVBindSessionInit(device.cDevice, &session.cSession)
if status != nil {
return nil, fmt.Errorf("%s", C.GoString(status))
}
session.device = device
return &session, nil
}
// Close stops capturing session and frees up resources
func (session *Session) Close() error {
session.lock.Lock()
defer session.lock.Unlock()
if session.closed {
return nil
}
session.closed = true
if session.cSession == nil {
return nil
}
status := C.AVBindSessionFree(&session.cSession)
if status != nil {
return fmt.Errorf("%s", C.GoString(status))
}
return nil
}
// Open start capturing session. As soon as it returns successfully, the data will start
// flowing. The raw data can be retrieved by using ReadCloser's Read method.
func (session *Session) Open(property prop.Media) (*ReadCloser, error) {
frameFormat, ok := frameFormatToAVBind(property.FrameFormat)
if !ok {
return nil, fmt.Errorf("Unsupported frame format")
}
cProperty := C.AVBindMediaProperty{
width: C.int(property.Width),
height: C.int(property.Height),
frameFormat: frameFormat,
}
rc := newReadCloser(func() {
C.AVBindSessionClose(session.cSession)
})
status := C.AVBindSessionOpen(
session.cSession,
cProperty,
C.AVBindDataCallback(unsafe.Pointer(C.onDataBridge)),
unsafe.Pointer(&rc.id),
)
if status != nil {
return nil, fmt.Errorf("%s", C.GoString(status))
}
return rc, nil
}
// Properties queries a list of properties that device supports
func (session *Session) Properties() []prop.Media {
var cPropertiesPtr C.PAVBindMediaProperty
var cPropertiesLen C.int
status := C.AVBindSessionProperties(session.cSession, &cPropertiesPtr, &cPropertiesLen)
if status != nil {
return nil
}
// https://github.com/golang/go/wiki/cgo#turning-c-arrays-into-go-slices
cProperties := (*[1 << 28]C.AVBindMediaProperty)(unsafe.Pointer(cPropertiesPtr))[:cPropertiesLen:cPropertiesLen]
var properties []prop.Media
for _, cProperty := range cProperties {
frameFormat, ok := frameFormatFromAVBind(cProperty.frameFormat)
if ok {
properties = append(properties, prop.Media{
Video: prop.Video{
Width: int(cProperty.width),
Height: int(cProperty.height),
FrameFormat: frameFormat,
},
})
}
}
return properties
}
vendor/github.com/pion/mediadevices/pkg/codec/codec.go
Generated Vendored
+155
View File
@@ -0,0 +1,155 @@
package codec
import (
"time"
"github.com/pion/mediadevices/pkg/io/audio"
"github.com/pion/mediadevices/pkg/io/video"
"github.com/pion/mediadevices/pkg/prop"
"github.com/pion/rtp"
"github.com/pion/rtp/codecs"
"github.com/pion/webrtc/v3"
)
// RTPCodec wraps webrtc.RTPCodec. RTPCodec might extend webrtc.RTPCodec in the future.
type RTPCodec struct {
webrtc.RTPCodecParameters
rtp.Payloader
// Latency of static frame size codec.
Latency time.Duration
}
// NewRTPH264Codec is a helper to create an H264 codec
func NewRTPH264Codec(clockrate uint32) *RTPCodec {
return &RTPCodec{
RTPCodecParameters: webrtc.RTPCodecParameters{
RTPCodecCapability: webrtc.RTPCodecCapability{
MimeType: webrtc.MimeTypeH264,
ClockRate: 90000,
Channels: 0,
SDPFmtpLine: "level-asymmetry-allowed=1;packetization-mode=1;profile-level-id=42e01f",
RTCPFeedback: nil,
},
PayloadType: 125,
},
Payloader: &codecs.H264Payloader{},
}
}
// NewRTPVP8Codec is a helper to create an VP8 codec
func NewRTPVP8Codec(clockrate uint32) *RTPCodec {
return &RTPCodec{
RTPCodecParameters: webrtc.RTPCodecParameters{
RTPCodecCapability: webrtc.RTPCodecCapability{
MimeType: webrtc.MimeTypeVP8,
ClockRate: 90000,
Channels: 0,
SDPFmtpLine: "",
RTCPFeedback: nil,
},
PayloadType: 96,
},
Payloader: &codecs.VP8Payloader{},
}
}
// NewRTPVP9Codec is a helper to create an VP9 codec
func NewRTPVP9Codec(clockrate uint32) *RTPCodec {
return &RTPCodec{
RTPCodecParameters: webrtc.RTPCodecParameters{
RTPCodecCapability: webrtc.RTPCodecCapability{
MimeType: webrtc.MimeTypeVP9,
ClockRate: 90000,
Channels: 0,
SDPFmtpLine: "",
RTCPFeedback: nil,
},
PayloadType: 98,
},
Payloader: &codecs.VP9Payloader{},
}
}
// NewRTPOpusCodec is a helper to create an Opus codec
func NewRTPOpusCodec(clockrate uint32) *RTPCodec {
return &RTPCodec{
RTPCodecParameters: webrtc.RTPCodecParameters{
RTPCodecCapability: webrtc.RTPCodecCapability{
MimeType: webrtc.MimeTypeOpus,
ClockRate: 48000,
Channels: 2,
SDPFmtpLine: "minptime=10;useinbandfec=1",
RTCPFeedback: nil,
},
PayloadType: 111,
},
Payloader: &codecs.OpusPayloader{},
}
}
// AudioEncoderBuilder is the interface that wraps basic operations that are
// necessary to build the audio encoder.
//
// This interface is for codec implementors to provide codec specific params,
// but still giving generality for the users.
type AudioEncoderBuilder interface {
// RTPCodec represents the codec metadata
RTPCodec() *RTPCodec
// BuildAudioEncoder builds audio encoder by given media params and audio input
BuildAudioEncoder(r audio.Reader, p prop.Media) (ReadCloser, error)
}
// VideoEncoderBuilder is the interface that wraps basic operations that are
// necessary to build the video encoder.
//
// This interface is for codec implementors to provide codec specific params,
// but still giving generality for the users.
type VideoEncoderBuilder interface {
// RTPCodec represents the codec metadata
RTPCodec() *RTPCodec
// BuildVideoEncoder builds video encoder by given media params and video input
BuildVideoEncoder(r video.Reader, p prop.Media) (ReadCloser, error)
}
// ReadCloser is an io.ReadCloser with a controller
type ReadCloser interface {
Read() (b []byte, release func(), err error)
Close() error
Controllable
}
// EncoderController is the interface allowing to control the encoder behaviour after it's initialisation.
// It will possibly have common control method in the future.
// A controller can have optional methods represented by *Controller interfaces
type EncoderController interface{}
// Controllable is a interface representing a encoder which can be controlled
// after it's initialisation with an EncoderController
type Controllable interface {
Controller() EncoderController
}
// KeyFrameController is a interface representing an encoder that can be forced to produce key frame on demand
type KeyFrameController interface {
EncoderController
// ForceKeyFrame forces the next frame to be a keyframe, aka intra-frame.
ForceKeyFrame() error
}
// BitRateController is a interface representing an encoder which can have a variable bit rate
type BitRateController interface {
EncoderController
// SetBitRate sets current target bitrate, lower bitrate means smaller data will be transmitted
// but this also means that the quality will also be lower.
SetBitRate(int) error
}
// BaseParams represents an codec's encoding properties
type BaseParams struct {
// Target bitrate in bps.
BitRate int
// Expected interval of the keyframes in frames.
KeyFrameInterval int
}
vendor/github.com/pion/mediadevices/pkg/codec/x264/bridge.h
Generated Vendored
+111
View File
@@ -0,0 +1,111 @@
#include <stdint.h>
#include <stdio.h>
#include <stdlib.h>
#include <x264.h>
#define ERR_DEFAULT_PRESET -1
#define ERR_APPLY_PROFILE -2
#define ERR_ALLOC_PICTURE -3
#define ERR_OPEN_ENGINE -4
#define ERR_ENCODE -5
typedef struct Slice {
unsigned char *data;
int data_len;
} Slice;
typedef struct Encoder {
x264_t *h;
x264_picture_t pic_in;
x264_param_t param;
int force_key_frame;
} Encoder;
Encoder *enc_new(x264_param_t param, char *preset, int *rc) {
Encoder *e = (Encoder *)malloc(sizeof(Encoder));
if (x264_param_default_preset(&e->param, preset, "zerolatency") < 0) {
free(preset);
*rc = ERR_DEFAULT_PRESET;
goto fail;
}
free(preset);
/* Configure non-default params */
e->param.i_csp = param.i_csp;
e->param.i_width = param.i_width;
e->param.i_height = param.i_height;
e->param.i_fps_num = param.i_fps_num;
e->param.i_fps_den = 1;
// Intra refres:
e->param.i_keyint_max = param.i_keyint_max;
// Rate control:
e->param.rc.i_rc_method = X264_RC_ABR;
e->param.rc.i_bitrate = param.rc.i_bitrate;
e->param.rc.i_vbv_max_bitrate = param.rc.i_vbv_max_bitrate;
e->param.rc.i_vbv_buffer_size = param.rc.i_vbv_buffer_size;
// For streaming:
e->param.b_repeat_headers = 1;
e->param.b_annexb = 1;
if (x264_param_apply_profile(&e->param, "high") < 0) {
*rc = ERR_APPLY_PROFILE;
goto fail;
}
x264_picture_t pic_in;
if (x264_picture_alloc(&pic_in, param.i_csp, param.i_width, param.i_height) < 0) {
*rc = ERR_ALLOC_PICTURE;
goto fail;
}
// FIXME: we use x264_picture_alloc to set the metadata only, we don't need the allocated memory
// to store the frame. Since we free the frame memory here, we don't need to call
// x264_picture_clean later.
e->pic_in = pic_in;
x264_picture_clean(&pic_in);
e->h = x264_encoder_open(&e->param);
if (!e->h) {
*rc = ERR_OPEN_ENGINE;
goto fail;
}
return e;
fail:
free(e);
return NULL;
}
Slice enc_encode(Encoder *e, uint8_t *y, uint8_t *cb, uint8_t *cr, int *rc) {
x264_nal_t *nal;
int i_nal;
x264_picture_t pic_out;
e->pic_in.img.plane[0] = y;
e->pic_in.img.plane[1] = cb;
e->pic_in.img.plane[2] = cr;
if (e->force_key_frame) {
e->pic_in.i_type = X264_TYPE_IDR;
} else {
e->pic_in.i_type = X264_TYPE_AUTO;
}
int frame_size = x264_encoder_encode(e->h, &nal, &i_nal, &e->pic_in, &pic_out);
e->force_key_frame = 0;
Slice s = {.data_len = frame_size};
if (frame_size <= 0) {
*rc = ERR_ENCODE;
return s;
}
// e->pic_in.i_pts++;
s.data = nal->p_payload;
return s;
}
void enc_close(Encoder *e, int *rc) {
x264_encoder_close(e->h);
free(e);
}
vendor/github.com/pion/mediadevices/pkg/codec/x264/params.go
Generated Vendored
+50
View File
@@ -0,0 +1,50 @@
package x264
import (
"github.com/pion/mediadevices/pkg/codec"
"github.com/pion/mediadevices/pkg/io/video"
"github.com/pion/mediadevices/pkg/prop"
)
// Params stores libx264 specific encoding parameters.
type Params struct {
codec.BaseParams
// Faster preset has lower CPU usage but lower quality
Preset Preset
}
// Preset represents a set of default configurations from libx264
type Preset int
const (
PresetUltrafast Preset = iota
PresetSuperfast
PresetVeryfast
PresetFaster
PresetFast
PresetMedium
PresetSlow
PresetSlower
PresetVeryslow
PresetPlacebo
)
// NewParams returns default x264 codec specific parameters.
func NewParams() (Params, error) {
return Params{
BaseParams: codec.BaseParams{
KeyFrameInterval: 60,
},
}, nil
}
// RTPCodec represents the codec metadata
func (p *Params) RTPCodec() *codec.RTPCodec {
return codec.NewRTPH264Codec(90000)
}
// BuildVideoEncoder builds x264 encoder with given params
func (p *Params) BuildVideoEncoder(r video.Reader, property prop.Media) (codec.ReadCloser, error) {
return newEncoder(r, property, *p)
}
vendor/github.com/pion/mediadevices/pkg/codec/x264/x264.go
Generated Vendored
+152
View File
@@ -0,0 +1,152 @@
// Package x264 implements H264 encoder.
// This package requires libx264 headers and libraries to be built.
// Reference: https://code.videolan.org/videolan/x264/blob/master/example.c
package x264
// #cgo pkg-config: x264
// #include "bridge.h"
import "C"
import (
"fmt"
"image"
"io"
"sync"
"unsafe"
"github.com/pion/mediadevices/pkg/codec"
"github.com/pion/mediadevices/pkg/io/video"
"github.com/pion/mediadevices/pkg/prop"
)
type encoder struct {
engine *C.Encoder
r video.Reader
mu sync.Mutex
closed bool
}
type cerror int
func (e cerror) Error() string {
switch e {
case C.ERR_DEFAULT_PRESET:
return errDefaultPreset.Error()
case C.ERR_APPLY_PROFILE:
return errApplyProfile.Error()
case C.ERR_ALLOC_PICTURE:
return errAllocPicture.Error()
case C.ERR_OPEN_ENGINE:
return errOpenEngine.Error()
case C.ERR_ENCODE:
return errEncode.Error()
default:
return "unknown error"
}
}
func errFromC(rc C.int) error {
if rc == 0 {
return nil
}
return cerror(rc)
}
var (
errInitEngine = fmt.Errorf("failed to initialize x264")
errDefaultPreset = fmt.Errorf("failed to set default preset")
errApplyProfile = fmt.Errorf("failed to apply profile")
errAllocPicture = fmt.Errorf("failed to alloc picture")
errOpenEngine = fmt.Errorf("failed to open x264")
errEncode = fmt.Errorf("failed to encode")
)
func newEncoder(r video.Reader, p prop.Media, params Params) (codec.ReadCloser, error) {
if params.KeyFrameInterval == 0 {
params.KeyFrameInterval = 60
}
// Convert from bit/s to kbit/s because x264 uses kbit/s instead.
// Reference: https://code.videolan.org/videolan/x264/-/blob/7923c5818b50a3d8816eed222a7c43b418a73b36/encoder/ratecontrol.c#L657
params.BitRate /= 1000
param := C.x264_param_t{
i_csp: C.X264_CSP_I420,
i_width: C.int(p.Width),
i_height: C.int(p.Height),
i_keyint_max: C.int(params.KeyFrameInterval),
}
param.rc.i_bitrate = C.int(params.BitRate)
param.rc.i_vbv_max_bitrate = param.rc.i_bitrate
param.rc.i_vbv_buffer_size = param.rc.i_vbv_max_bitrate * 2
var rc C.int
// cPreset will be freed in C.enc_new
cPreset := C.CString(fmt.Sprint(params.Preset))
engine := C.enc_new(param, cPreset, &rc)
if err := errFromC(rc); err != nil {
return nil, err
}
e := encoder{
engine: engine,
r: video.ToI420(r),
}
return &e, nil
}
func (e *encoder) Read() ([]byte, func(), error) {
e.mu.Lock()
defer e.mu.Unlock()
if e.closed {
return nil, func() {}, io.EOF
}
img, release, err := e.r.Read()
if err != nil {
return nil, func() {}, err
}
defer release()
yuvImg := img.(*image.YCbCr)
var rc C.int
s := C.enc_encode(
e.engine,
(*C.uchar)(&yuvImg.Y[0]),
(*C.uchar)(&yuvImg.Cb[0]),
(*C.uchar)(&yuvImg.Cr[0]),
&rc,
)
if err := errFromC(rc); err != nil {
return nil, func() {}, err
}
encoded := C.GoBytes(unsafe.Pointer(s.data), s.data_len)
return encoded, func() {}, err
}
// TODO: Implement bit rate control
//var _ codec.BitRateController = (*encoder)(nil)
func (e *encoder) ForceKeyFrame() error {
e.engine.force_key_frame = C.int(1)
return nil
}
func (e *encoder) Controller() codec.EncoderController {
return e
}
func (e *encoder) Close() error {
e.mu.Lock()
defer e.mu.Unlock()
if e.closed {
return nil
}
var rc C.int
C.enc_close(e.engine, &rc)
e.closed = true
return nil
}
vendor/github.com/pion/mediadevices/pkg/driver/availability/error.go
Generated Vendored
+28
View File
@@ -0,0 +1,28 @@
package availability
import (
"errors"
)
var (
ErrUnimplemented = NewError("not implemented")
ErrBusy = NewError("device or resource busy")
ErrNoDevice = NewError("no such device")
)
type errorString struct {
s string
}
func NewError(text string) error {
return &errorString{text}
}
func IsError(err error) bool {
var target *errorString
return errors.As(err, &target)
}
func (e *errorString) Error() string {
return e.s
}
vendor/github.com/pion/mediadevices/pkg/driver/camera/camera.go
Generated Vendored
+16
View File
@@ -0,0 +1,16 @@
/*
Package camera provides a video camera driver.
Device Label Generation Rules
On Linux, the device label will be in the format of:
pci-0000:00:00.0-usb-0:0:0.0-video-index0;video0
If /dev/v4l/by-path/* is not available (for example in a docker container without
bindings in /dev/v4l/by-path/), it will be:
video0;video0
*/
package camera
// LabelSeparator is used to separate labels for a driver that
// is found from multiple locations on a host.
const LabelSeparator = ";"
vendor/github.com/pion/mediadevices/pkg/driver/camera/camera_darwin.go
Generated Vendored
+82
View File
@@ -0,0 +1,82 @@
package camera
import (
"image"
"github.com/pion/mediadevices/pkg/avfoundation"
"github.com/pion/mediadevices/pkg/driver"
"github.com/pion/mediadevices/pkg/frame"
"github.com/pion/mediadevices/pkg/io/video"
"github.com/pion/mediadevices/pkg/prop"
)
type camera struct {
device avfoundation.Device
session *avfoundation.Session
rcClose func()
}
func init() {
Initialize()
}
// Initialize finds and registers camera devices. This is part of an experimental API.
func Initialize() {
devices, err := avfoundation.Devices(avfoundation.Video)
if err != nil {
panic(err)
}
for _, device := range devices {
cam := newCamera(device)
driver.GetManager().Register(cam, driver.Info{
Label: device.UID,
DeviceType: driver.Camera,
Name: device.Name,
})
}
}
func newCamera(device avfoundation.Device) *camera {
return &camera{
device: device,
}
}
func (cam *camera) Open() error {
var err error
cam.session, err = avfoundation.NewSession(cam.device)
return err
}
func (cam *camera) Close() error {
if cam.rcClose != nil {
cam.rcClose()
}
return cam.session.Close()
}
func (cam *camera) VideoRecord(property prop.Media) (video.Reader, error) {
decoder, err := frame.NewDecoder(property.FrameFormat)
if err != nil {
return nil, err
}
rc, err := cam.session.Open(property)
if err != nil {
return nil, err
}
cam.rcClose = rc.Close
r := video.ReaderFunc(func() (image.Image, func(), error) {
frame, _, err := rc.Read()
if err != nil {
return nil, func() {}, err
}
return decoder.Decode(frame, property.Width, property.Height)
})
return r, nil
}
func (cam *camera) Properties() []prop.Media {
return cam.session.Properties()
}
vendor/github.com/pion/mediadevices/pkg/driver/camera/camera_linux.go
Generated Vendored
+464
View File
@@ -0,0 +1,464 @@
package camera
// #include <linux/videodev2.h>
import "C"
import (
"context"
"errors"
"image"
"io"
"math"
"os"
"path/filepath"
"strconv"
"sync"
"syscall"
"time"
"github.com/pion/mediadevices/pkg/driver/availability"
"github.com/blackjack/webcam"
"github.com/pion/mediadevices/pkg/driver"
"github.com/pion/mediadevices/pkg/frame"
"github.com/pion/mediadevices/pkg/io/video"
"github.com/pion/mediadevices/pkg/prop"
)
const (
maxEmptyFrameCount = 5
prioritizedDevice = "video0"
)
var (
errReadTimeout = errors.New("read timeout")
errEmptyFrame = errors.New("empty frame")
// Reference: https://commons.wikimedia.org/wiki/File:Vector_Video_Standards2.svg
supportedResolutions = [][2]int{
{320, 240},
{640, 480},
{768, 576},
{800, 600},
{1024, 768},
{1280, 854},
{1280, 960},
{1280, 1024},
{1400, 1050},
{1600, 1200},
{2048, 1536},
{320, 200},
{800, 480},
{854, 480},
{1024, 600},
{1152, 768},
{1280, 720},
{1280, 768},
{1366, 768},
{1280, 800},
{1440, 900},
{1440, 960},
{1680, 1050},
{1920, 1080},
{2048, 1080},
{1920, 1200},
{2560, 1600},
}
)
const bufCount = 2
// Camera implementation using v4l2
// Reference: https://linuxtv.org/downloads/v4l-dvb-apis/uapi/v4l/videodev.html#videodev
type camera struct {
path string
cam *webcam.Webcam
formats map[webcam.PixelFormat]frame.Format
reversedFormats map[frame.Format]webcam.PixelFormat
started bool
mutex sync.Mutex
cancel func()
prevFrameTime time.Time
}
func init() {
Initialize()
}
// Initialize finds and registers camera devices. This is part of an experimental API.
func Initialize() {
// Clear all registered camera devices to prevent duplicates.
// If first initalize call, this will be a noop.
manager := driver.GetManager()
for _, d := range manager.Query(driver.FilterVideoRecorder()) {
manager.Delete(d.ID())
}
discovered := make(map[string]struct{})
discover(discovered, "/dev/v4l/by-id/*")
discover(discovered, "/dev/v4l/by-path/*")
discover(discovered, "/dev/video*")
}
func discover(discovered map[string]struct{}, pattern string) {
devices, err := filepath.Glob(pattern)
if err != nil {
// No v4l device.
return
}
for _, device := range devices {
label := filepath.Base(device)
reallink, err := os.Readlink(device)
if err != nil {
reallink = label
} else {
reallink = filepath.Base(reallink)
}
if _, ok := discovered[reallink]; ok {
continue
}
discovered[reallink] = struct{}{}
cam := NewCamera(device)
priority := driver.PriorityNormal
if reallink == prioritizedDevice {
priority = driver.PriorityHigh
}
var name, busInfo string
if webcamCam, err := webcam.Open(cam.path); err == nil {
name, _ = webcamCam.GetName()
busInfo, _ = webcamCam.GetBusInfo()
}
driver.GetManager().Register(cam, driver.Info{
// Source: https://www.kernel.org/doc/html/v4.9/media/uapi/v4l/vidioc-querycap.html
// Name of the device, a NUL-terminated UTF-8 string. For example: “Yoyodyne TV/FM”. One driver may support
// different brands or models of video hardware. This information is intended for users, for example in a
// menu of available devices. Since multiple TV cards of the same brand may be installed which are
// supported by the same driver, this name should be combined with the character device file name
// (e.g. /dev/video2) or the bus_info string to avoid ambiguities.
Name: name + LabelSeparator + busInfo,
Label: label + LabelSeparator + reallink,
DeviceType: driver.Camera,
Priority: priority,
})
}
}
func NewCamera(path string) *camera {
formats := map[webcam.PixelFormat]frame.Format{
webcam.PixelFormat(C.V4L2_PIX_FMT_YUV420): frame.FormatI420,
webcam.PixelFormat(C.V4L2_PIX_FMT_NV21): frame.FormatNV21,
webcam.PixelFormat(C.V4L2_PIX_FMT_NV12): frame.FormatNV12,
webcam.PixelFormat(C.V4L2_PIX_FMT_YUYV): frame.FormatYUYV,
webcam.PixelFormat(C.V4L2_PIX_FMT_UYVY): frame.FormatUYVY,
webcam.PixelFormat(C.V4L2_PIX_FMT_MJPEG): frame.FormatMJPEG,
webcam.PixelFormat(C.V4L2_PIX_FMT_Z16): frame.FormatZ16,
}
reversedFormats := make(map[frame.Format]webcam.PixelFormat)
for k, v := range formats {
reversedFormats[v] = k
}
c := &camera{
path: path,
formats: formats,
reversedFormats: reversedFormats,
}
return c
}
func getCameraReadTimeout() uint32 {
// default to 5 seconds
var readTimeoutSec uint32 = 5
if val, ok := os.LookupEnv("PION_MEDIADEVICES_CAMERA_READ_TIMEOUT"); ok {
if valInt, err := strconv.Atoi(val); err == nil {
if valInt > 0 {
readTimeoutSec = uint32(valInt)
}
}
}
return readTimeoutSec
}
func (c *camera) Open() error {
cam, err := webcam.Open(c.path)
if err != nil {
return err
}
// Buffering should be handled in higher level.
err = cam.SetBufferCount(bufCount)
if err != nil {
return err
}
c.prevFrameTime = time.Now()
c.cam = cam
return nil
}
func (c *camera) Close() error {
if c.cam == nil {
return nil
}
if c.cancel != nil {
// Let the reader knows that the caller has closed the camera
c.cancel()
// Wait until the reader unref the buffer
c.mutex.Lock()
defer c.mutex.Unlock()
// Note: StopStreaming frees frame buffers even if they are still used in Go code.
// There is currently no convenient way to do this safely.
// So, consumer of this stream must close camera after unusing all images.
c.cam.StopStreaming()
c.cancel = nil
}
c.cam.Close()
return nil
}
func (c *camera) VideoRecord(p prop.Media) (video.Reader, error) {
decoder, err := frame.NewDecoder(p.FrameFormat)
if err != nil {
return nil, err
}
pf := c.reversedFormats[p.FrameFormat]
_, _, _, err = c.cam.SetImageFormat(pf, uint32(p.Width), uint32(p.Height))
if err != nil {
return nil, err
}
if p.FrameRate > 0 {
err = c.cam.SetFramerate(float32(p.FrameRate))
if err != nil {
return nil, err
}
}
if err := c.cam.StartStreaming(); err != nil {
return nil, err
}
cam := c.cam
readTimeoutSec := getCameraReadTimeout()
ctx, cancel := context.WithCancel(context.Background())
c.cancel = cancel
var buf []byte
r := video.ReaderFunc(func() (img image.Image, release func(), err error) {
// Lock to avoid accessing the buffer after StopStreaming()
c.mutex.Lock()
defer c.mutex.Unlock()
// Wait until a frame is ready
for i := 0; i < maxEmptyFrameCount; i++ {
if ctx.Err() != nil {
// Return EOF if the camera is already closed.
return nil, func() {}, io.EOF
}
if p.DiscardFramesOlderThan != 0 && time.Now().Sub(c.prevFrameTime) >= p.DiscardFramesOlderThan {
for i := 0; i < bufCount; i++ {
_ = cam.WaitForFrame(readTimeoutSec)
_, _ = cam.ReadFrame()
}
}
err := cam.WaitForFrame(readTimeoutSec)
switch err.(type) {
case nil:
case *webcam.Timeout:
return nil, func() {}, errReadTimeout
default:
// Camera has been stopped.
return nil, func() {}, err
}
b, err := cam.ReadFrame()
if err != nil {
// Camera has been stopped.
return nil, func() {}, err
}
if p.DiscardFramesOlderThan != 0 {
c.prevFrameTime = time.Now()
}
// Frame is empty.
// Retry reading and return errEmptyFrame if it exceeds maxEmptyFrameCount.
if len(b) == 0 {
continue
}
if len(b) > len(buf) {
// Grow the intermediate buffer
buf = make([]byte, len(b))
}
// move the memory from mmap to Go. This will guarantee that any data that's going out
// from this reader will be Go safe. Otherwise, it's possible that outside of this reader
// that this memory is still being used even after we close it.
n := copy(buf, b)
return decoder.Decode(buf[:n], p.Width, p.Height)
}
return nil, func() {}, errEmptyFrame
})
return r, nil
}
func (c *camera) Properties() []prop.Media {
properties := make([]prop.Media, 0)
for format := range c.cam.GetSupportedFormats() {
for _, frameSize := range c.cam.GetSupportedFrameSizes(format) {
supportedFormat, ok := c.formats[format]
if !ok {
continue
}
if frameSize.StepWidth == 0 || frameSize.StepHeight == 0 {
framerates := c.cam.GetSupportedFramerates(format, uint32(frameSize.MaxWidth), uint32(frameSize.MaxHeight))
// If the camera doesn't support framerate, we just add the resolution and format
if len(framerates) == 0 {
properties = append(properties, prop.Media{
Video: prop.Video{
Width: int(frameSize.MaxWidth),
Height: int(frameSize.MaxHeight),
FrameFormat: supportedFormat,
},
})
continue
}
for _, framerate := range framerates {
for _, fps := range enumFramerate(framerate) {
properties = append(properties, prop.Media{
Video: prop.Video{
Width: int(frameSize.MaxWidth),
Height: int(frameSize.MaxHeight),
FrameFormat: supportedFormat,
FrameRate: fps,
},
})
}
}
} else {
// FIXME: we should probably use a custom data structure to capture all of the supported resolutions
for _, supportedResolution := range supportedResolutions {
minWidth, minHeight := int(frameSize.MinWidth), int(frameSize.MinHeight)
maxWidth, maxHeight := int(frameSize.MaxWidth), int(frameSize.MaxHeight)
stepWidth, stepHeight := int(frameSize.StepWidth), int(frameSize.StepHeight)
width, height := supportedResolution[0], supportedResolution[1]
if width < minWidth || width > maxWidth ||
height < minHeight || height > maxHeight {
continue
}
if (width-minWidth)%stepWidth != 0 ||
(height-minHeight)%stepHeight != 0 {
continue
}
framerates := c.cam.GetSupportedFramerates(format, uint32(width), uint32(height))
if len(framerates) == 0 {
properties = append(properties, prop.Media{
Video: prop.Video{
Width: width,
Height: height,
FrameFormat: supportedFormat,
},
})
continue
}
for _, framerate := range framerates {
for _, fps := range enumFramerate(framerate) {
properties = append(properties, prop.Media{
Video: prop.Video{
Width: width,
Height: height,
FrameFormat: supportedFormat,
FrameRate: fps,
},
})
}
}
}
}
}
}
return properties
}
func (c *camera) IsAvailable() (bool, error) {
var err error
// close the opened file descriptor as quickly as possible and in all cases, including panics
func() {
var cam *webcam.Webcam
if cam, err = webcam.Open(c.path); err == nil {
defer cam.Close()
var index int32
// "Drivers must implement all the input ioctls when the device has one or more inputs..."
// Source: https://www.kernel.org/doc/html/latest/userspace-api/media/v4l/video.html?highlight=vidioc_enuminput
if index, err = cam.GetInput(); err == nil {
err = cam.SelectInput(uint32(index))
}
}
}()
var errno syscall.Errno
errors.As(err, &errno)
// See https://man7.org/linux/man-pages/man3/errno.3.html
switch {
case err == nil:
return true, nil
case errno == syscall.EBUSY:
return false, availability.ErrBusy
case errno == syscall.ENODEV || errno == syscall.ENOENT:
return false, availability.ErrNoDevice
default:
return false, availability.NewError(errno.Error())
}
}
// enumFramerate returns a list of fps options from a FrameRate struct.
// discrete framerates will return a list of 1 fps element.
// stepwise framerates will return a list of all possible fps options.
func enumFramerate(framerate webcam.FrameRate) []float32 {
var framerates []float32
if framerate.StepNumerator == 0 && framerate.StepDenominator == 0 {
fr, err := calcFramerate(framerate.MaxNumerator, framerate.MaxDenominator)
if err != nil {
return framerates
}
framerates = append(framerates, fr)
} else {
for n := framerate.MinNumerator; n <= framerate.MaxNumerator; n += framerate.StepNumerator {
for d := framerate.MinDenominator; d <= framerate.MaxDenominator; d += framerate.StepDenominator {
fr, err := calcFramerate(n, d)
if err != nil {
continue
}
framerates = append(framerates, fr)
}
}
}
return framerates
}
// calcFramerate turns fraction into a float32 fps value.
func calcFramerate(numerator uint32, denominator uint32) (float32, error) {
if denominator == 0 {
return 0, errors.New("framerate denominator is zero")
}
// round to three decimal places to avoid floating point precision issues
return float32(math.Round(1000.0/((float64(numerator))/float64(denominator))) / 1000), nil
}
vendor/github.com/pion/mediadevices/pkg/driver/camera/camera_windows.cpp
Generated Vendored
+502
View File
@@ -0,0 +1,502 @@
#include <iostream>
#include <unistd.h>
#include <dshow.h>
#include <qedit.h>
#include <mmsystem.h>
#include "camera_windows.hpp"
#include "_cgo_export.h"
// printErr shows string representation of HRESULT.
// This is for debugging.
void printErr(HRESULT hr)
{
char buf[128];
AMGetErrorTextA(hr, buf, 128);
fprintf(stderr, "%s\n", buf);
}
// getCameraName returns name of the device.
// returned pointer must be released by free() after use.
char* getCameraName(IMoniker* moniker)
{
LPOLESTR name;
if (FAILED(moniker->GetDisplayName(nullptr, nullptr, &name)))
return nullptr;
std::string nameStr = utf16Decode(name);
char* ret = (char*)malloc(nameStr.size() + 1);
memcpy(ret, nameStr.c_str(), nameStr.size() + 1);
LPMALLOC comalloc;
CoGetMalloc(1, &comalloc);
comalloc->Free(name);
return ret;
}
// listCamera stores information of the devices to cameraList*.
int listCamera(cameraList* list, const char** errstr)
{
ICreateDevEnum* sysDevEnum = nullptr;
IEnumMoniker* enumMon = nullptr;
if (FAILED(CoCreateInstance(
CLSID_SystemDeviceEnum, nullptr, CLSCTX_INPROC,
IID_ICreateDevEnum, (void**)&sysDevEnum)))
{
*errstr = errEnumDevice;
goto fail;
}
if (FAILED(sysDevEnum->CreateClassEnumerator(
CLSID_VideoInputDeviceCategory, &enumMon, 0)))
{
*errstr = errEnumDevice;
goto fail;
}
safeRelease(&sysDevEnum);
{
IMoniker* moniker;
list->num = 0;
while (enumMon->Next(1, &moniker, nullptr) == S_OK)
{
moniker->Release();
list->num++;
}
enumMon->Reset();
list->name = new char*[list->num];
int i = 0;
while (enumMon->Next(1, &moniker, nullptr) == S_OK)
{
list->name[i] = getCameraName(moniker);
moniker->Release();
i++;
}
}
safeRelease(&enumMon);
return 0;
fail:
safeRelease(&sysDevEnum);
safeRelease(&enumMon);
return 1;
}
// freeCameraList frees all resources stored in cameraList*.
int freeCameraList(cameraList* list, const char** errstr)
{
if (list->name != nullptr)
{
for (int i = 0; i < list->num; ++i)
{
delete list->name[i];
}
delete list->name;
}
return 1;
}
// selectCamera stores pointer to the selected device IMoniker* according to the configs in camera*.
int selectCamera(camera* cam, IMoniker** monikerSelected, const char** errstr)
{
ICreateDevEnum* sysDevEnum = nullptr;
IEnumMoniker* enumMon = nullptr;
if (FAILED(CoCreateInstance(
CLSID_SystemDeviceEnum, nullptr, CLSCTX_INPROC,
IID_ICreateDevEnum, (void**)&sysDevEnum)))
{
*errstr = errEnumDevice;
goto fail;
}
if (FAILED(sysDevEnum->CreateClassEnumerator(
CLSID_VideoInputDeviceCategory, &enumMon, 0)))
{
*errstr = errEnumDevice;
goto fail;
}
safeRelease(&sysDevEnum);
{
IMoniker* moniker;
while (enumMon->Next(1, &moniker, nullptr) == S_OK)
{
char* name = getCameraName(moniker);
if (strcmp(cam->name, name) != 0)
{
free(name);
safeRelease(&moniker);
continue;
}
free(name);
*monikerSelected = moniker;
safeRelease(&enumMon);
return 1;
}
}
safeRelease(&enumMon);
return 0;
fail:
safeRelease(&sysDevEnum);
safeRelease(&enumMon);
return 1;
}
// listResolution stores list of the device to camera*.
int listResolution(camera* cam, const char** errstr)
{
cam->props = nullptr;
IMoniker* moniker = nullptr;
IBaseFilter* captureFilter = nullptr;
ICaptureGraphBuilder2* captureGraph = nullptr;
IAMStreamConfig* config = nullptr;
IPin* src = nullptr;
LPOLESTR name;
if (!selectCamera(cam, &moniker, errstr))
{
goto fail;
}
moniker->BindToObject(0, 0, IID_IBaseFilter, (void**)&captureFilter);
safeRelease(&moniker);
src = getPin(captureFilter, PINDIR_OUTPUT);
if (src == nullptr)
{
*errstr = errGetConfig;
goto fail;
}
// Getting IAMStreamConfig is stub on Wine. Requires real Windows.
if (FAILED(src->QueryInterface(
IID_IAMStreamConfig, (void**)&config)))
{
*errstr = errGetConfig;
goto fail;
}
safeRelease(&src);
{
int count = 0, size = 0;
if (FAILED(config->GetNumberOfCapabilities(&count, &size)))
{
*errstr = errGetConfig;
goto fail;
}
cam->props = new imageProp[count];
int iProp = 0;
for (int i = 0; i < count; ++i)
{
VIDEO_STREAM_CONFIG_CAPS caps;
AM_MEDIA_TYPE* mediaType;
if (FAILED(config->GetStreamCaps(i, &mediaType, (BYTE*)&caps)))
continue;
if (mediaType->majortype != MEDIATYPE_Video ||
mediaType->formattype != FORMAT_VideoInfo ||
mediaType->pbFormat == nullptr)
continue;
VIDEOINFOHEADER* videoInfoHdr = (VIDEOINFOHEADER*)mediaType->pbFormat;
cam->props[iProp].width = videoInfoHdr->bmiHeader.biWidth;
cam->props[iProp].height = videoInfoHdr->bmiHeader.biHeight;
cam->props[iProp].fcc = videoInfoHdr->bmiHeader.biCompression;
iProp++;
}
cam->numProps = iProp;
}
safeRelease(&config);
safeRelease(&captureGraph);
safeRelease(&captureFilter);
safeRelease(&moniker);
return 0;
fail:
safeRelease(&src);
safeRelease(&config);
safeRelease(&captureGraph);
safeRelease(&captureFilter);
safeRelease(&moniker);
return 1;
}
// openCamera opens a camera and stores interface handler to camera*.
// camera* should be freed by freeCamera() after use.
int openCamera(camera* cam, const char** errstr)
{
cam->grabber = nullptr;
cam->mediaControl = nullptr;
cam->callback = nullptr;
IMoniker* moniker = nullptr;
IGraphBuilder* graphBuilder = nullptr;
IBaseFilter* captureFilter = nullptr;
IMediaControl* mediaControl = nullptr;
IBaseFilter* grabberFilter = nullptr;
ISampleGrabber* grabber = nullptr;
IBaseFilter* nullFilter = nullptr;
IPin* src = nullptr;
IPin* dst = nullptr;
IPin* end = nullptr;
IPin* nul = nullptr;
if (!selectCamera(cam, &moniker, errstr))
{
goto fail;
}
moniker->BindToObject(0, 0, IID_IBaseFilter, (void**)&captureFilter);
safeRelease(&moniker);
if (FAILED(CoCreateInstance(
CLSID_FilterGraph, nullptr, CLSCTX_INPROC,
IID_IGraphBuilder, (void**)&graphBuilder)))
{
*errstr = errGraphBuilder;
goto fail;
}
if (FAILED(graphBuilder->QueryInterface(
IID_IMediaControl, (void**)&mediaControl)))
{
*errstr = errNoControl;
goto fail;
}
if (FAILED(graphBuilder->AddFilter(captureFilter, L"capture")))
{
*errstr = errAddFilter;
goto fail;
}
if (FAILED(CoCreateInstance(
CLSID_SampleGrabber, nullptr, CLSCTX_INPROC,
IID_IBaseFilter, (void**)&grabberFilter)))
{
*errstr = errGrabber;
goto fail;
}
if (FAILED(grabberFilter->QueryInterface(IID_ISampleGrabber, (void**)&grabber)))
{
*errstr = errGrabber;
goto fail;
}
{
AM_MEDIA_TYPE mediaType;
memset(&mediaType, 0, sizeof(mediaType));
mediaType.majortype = MEDIATYPE_Video;
mediaType.subtype = MEDIASUBTYPE_YUY2;
mediaType.formattype = FORMAT_VideoInfo;
mediaType.bFixedSizeSamples = 1;
mediaType.cbFormat = sizeof(VIDEOINFOHEADER);
VIDEOINFOHEADER videoInfoHdr;
memset(&videoInfoHdr, 0, sizeof(VIDEOINFOHEADER));
videoInfoHdr.bmiHeader.biSize = sizeof(BITMAPINFOHEADER);
videoInfoHdr.bmiHeader.biWidth = cam->width;
videoInfoHdr.bmiHeader.biHeight = cam->height;
videoInfoHdr.bmiHeader.biPlanes = 1;
videoInfoHdr.bmiHeader.biBitCount = 16;
videoInfoHdr.bmiHeader.biCompression = MAKEFOURCC('Y', 'U', 'Y', '2');
mediaType.pbFormat = (BYTE*)&videoInfoHdr;
if (FAILED(grabber->SetMediaType(&mediaType)))
{
*errstr = errGrabber;
goto fail;
}
}
if (FAILED(graphBuilder->AddFilter(grabberFilter, L"grabber")))
{
*errstr = errAddFilter;
goto fail;
}
if (FAILED(CoCreateInstance(
CLSID_NullRenderer, nullptr, CLSCTX_INPROC,
IID_IBaseFilter, (void**)&nullFilter)))
{
*errstr = errTerminator;
goto fail;
}
if (FAILED(graphBuilder->AddFilter(nullFilter, L"bull")))
{
*errstr = errAddFilter;
goto fail;
}
HRESULT hr;
src = getPin(captureFilter, PINDIR_OUTPUT);
dst = getPin(grabberFilter, PINDIR_INPUT);
if (src == nullptr || dst == nullptr ||
FAILED(hr = graphBuilder->Connect(src, dst)))
{
*errstr = errConnectFilters;
goto fail;
}
safeRelease(&src);
safeRelease(&dst);
end = getPin(grabberFilter, PINDIR_OUTPUT);
nul = getPin(nullFilter, PINDIR_INPUT);
if (end == nullptr || nul == nullptr ||
FAILED(hr = graphBuilder->Connect(end, nul)))
{
*errstr = errConnectFilters;
goto fail;
}
safeRelease(&end);
safeRelease(&nul);
safeRelease(&nullFilter);
safeRelease(&captureFilter);
safeRelease(&grabberFilter);
safeRelease(&graphBuilder);
{
SampleGrabberCallback* cb = new SampleGrabberCallback(cam);
grabber->SetCallback(cb, 1);
cam->grabber = (void*)grabber;
cam->mediaControl = (void*)mediaControl;
cam->callback = (void*)cb;
grabber->SetBufferSamples(true);
mediaControl->Run();
}
return 0;
fail:
safeRelease(&src);
safeRelease(&dst);
safeRelease(&end);
safeRelease(&nul);
safeRelease(&nullFilter);
safeRelease(&grabber);
safeRelease(&grabberFilter);
safeRelease(&mediaControl);
safeRelease(&captureFilter);
safeRelease(&graphBuilder);
safeRelease(&moniker);
return 1;
}
// SampleCB is not used in this app.
HRESULT SampleGrabberCallback::SampleCB(double sampleTime, IMediaSample* sample)
{
return S_OK;
}
// BufferCB receives image from DirectShow.
HRESULT SampleGrabberCallback::BufferCB(double sampleTime, BYTE* buf, LONG len)
{
BYTE* gobuf = (BYTE*)cam_->buf;
const int nPix = cam_->width * cam_->height;
if (len > nPix * 2)
{
fprintf(stderr, "Wrong frame buffer size: %d > %d\n", len, nPix * 2);
return S_OK;
}
int yi = 0;
int cbi = cam_->width * cam_->height;
int cri = cbi + cbi / 2;
// Pack as I422
for (int y = 0; y < cam_->height; ++y)
{
int j = y * cam_->width * 2;
for (int x = 0; x < cam_->width / 2; ++x)
{
gobuf[yi] = buf[j];
gobuf[cbi] = buf[j + 1];
gobuf[yi + 1] = buf[j + 2];
gobuf[cri] = buf[j + 3];
j += 4;
yi += 2;
cbi++;
cri++;
}
}
imageCallback((size_t)cam_);
return S_OK;
}
// freeCamera closes device and frees all resources allocated by openCamera().
void freeCamera(camera* cam)
{
if (cam->mediaControl)
((IMediaControl*)cam->mediaControl)->Stop();
safeRelease((ISampleGrabber**)&cam->grabber);
safeRelease((IMediaControl**)&cam->mediaControl);
if (cam->callback)
{
((SampleGrabberCallback*)cam->callback)->Release();
delete ((SampleGrabberCallback*)cam->callback);
cam->callback = nullptr;
}
if (cam->props)
{
delete cam->props;
cam->props = nullptr;
}
}
// utf16Decode returns UTF-8 string from UTF-16 string.
std::string utf16Decode(LPOLESTR olestr)
{
std::wstring wstr(olestr);
const int len = WideCharToMultiByte(
CP_UTF8, 0,
wstr.data(), (int)wstr.size(),
nullptr, 0, nullptr, nullptr);
std::string str(len, 0);
WideCharToMultiByte(
CP_UTF8, 0,
wstr.data(), (int)wstr.size(),
(LPSTR)str.data(), len, nullptr, nullptr);
return str;
}
// getPin is a helper to get I/O pin of DirectShow filter.
IPin* getPin(IBaseFilter* filter, PIN_DIRECTION dir)
{
IEnumPins* enumPins;
if (FAILED(filter->EnumPins(&enumPins)))
return nullptr;
IPin* pin;
while (enumPins->Next(1, &pin, nullptr) == S_OK)
{
PIN_DIRECTION d;
pin->QueryDirection(&d);
if (d == dir)
{
enumPins->Release();
return pin;
}
pin->Release();
}
enumPins->Release();
return nullptr;
}
vendor/github.com/pion/mediadevices/pkg/driver/camera/camera_windows.go
Generated Vendored
+161
View File
@@ -0,0 +1,161 @@
package camera
// #cgo LDFLAGS: -lstrmiids -lole32 -lquartz
// #include <dshow.h>
// #include "camera_windows.hpp"
import "C"
import (
"fmt"
"image"
"io"
"sync"
"unsafe"
"github.com/pion/mediadevices/pkg/driver"
"github.com/pion/mediadevices/pkg/frame"
"github.com/pion/mediadevices/pkg/io/video"
"github.com/pion/mediadevices/pkg/prop"
)
var (
callbacks = make(map[uintptr]*camera)
callbacksMu sync.RWMutex
)
type camera struct {
name string
cam *C.camera
ch chan []byte
buf []byte
bufGo []byte
}
func init() {
Initialize()
}
// Initialize finds and registers camera devices. This is part of an experimental API.
func Initialize() {
C.CoInitializeEx(nil, C.COINIT_MULTITHREADED)
var list C.cameraList
var errStr *C.char
if C.listCamera(&list, &errStr) != 0 {
// Failed to list camera
fmt.Printf("Failed to list camera: %s\n", C.GoString(errStr))
return
}
for i := 0; i < int(list.num); i++ {
name := C.GoString(C.getName(&list, C.int(i)))
driver.GetManager().Register(&camera{name: name}, driver.Info{
Label: name,
DeviceType: driver.Camera,
})
}
C.freeCameraList(&list, &errStr)
}
func (c *camera) Open() error {
c.ch = make(chan []byte)
c.cam = &C.camera{
name: C.CString(c.name),
}
var errStr *C.char
if C.listResolution(c.cam, &errStr) != 0 {
return fmt.Errorf("failed to open device: %s", C.GoString(errStr))
}
return nil
}
//export imageCallback
func imageCallback(cam uintptr) {
callbacksMu.RLock()
cb, ok := callbacks[uintptr(unsafe.Pointer(cam))]
callbacksMu.RUnlock()
if !ok {
return
}
copy(cb.bufGo, cb.buf)
cb.ch <- cb.bufGo
}
func (c *camera) Close() error {
callbacksMu.Lock()
key := uintptr(unsafe.Pointer(c.cam))
if _, ok := callbacks[key]; ok {
delete(callbacks, key)
}
callbacksMu.Unlock()
close(c.ch)
if c.cam != nil {
C.free(unsafe.Pointer(c.cam.name))
C.freeCamera(c.cam)
c.cam = nil
}
return nil
}
func (c *camera) VideoRecord(p prop.Media) (video.Reader, error) {
nPix := p.Width * p.Height
c.buf = make([]byte, nPix*2) // for YUY2
c.bufGo = make([]byte, nPix*2)
c.cam.width = C.int(p.Width)
c.cam.height = C.int(p.Height)
c.cam.buf = C.size_t(uintptr(unsafe.Pointer(&c.buf[0])))
var errStr *C.char
if C.openCamera(c.cam, &errStr) != 0 {
return nil, fmt.Errorf("failed to open device: %s", C.GoString(errStr))
}
callbacksMu.Lock()
callbacks[uintptr(unsafe.Pointer(c.cam))] = c
callbacksMu.Unlock()
img := &image.YCbCr{}
r := video.ReaderFunc(func() (image.Image, func(), error) {
b, ok := <-c.ch
if !ok {
return nil, func() {}, io.EOF
}
img.Y = b[:nPix]
img.Cb = b[nPix : nPix+nPix/2]
img.Cr = b[nPix+nPix/2 : nPix*2]
img.YStride = p.Width
img.CStride = p.Width / 2
img.SubsampleRatio = image.YCbCrSubsampleRatio422
img.Rect = image.Rect(0, 0, p.Width, p.Height)
return img, func() {}, nil
})
return r, nil
}
func (c *camera) Properties() []prop.Media {
properties := []prop.Media{}
for i := 0; i < int(c.cam.numProps); i++ {
p := C.getProp(c.cam, C.int(i))
// TODO: support other FOURCC
if p.fcc == fourccYUY2 {
properties = append(properties, prop.Media{
Video: prop.Video{
Width: int(p.width),
Height: int(p.height),
FrameFormat: frame.FormatYUY2,
},
})
}
}
return properties
}
const (
fourccYUY2 = 0x32595559
)
vendor/github.com/pion/mediadevices/pkg/driver/camera/camera_windows.hpp
Generated Vendored
+120
View File
@@ -0,0 +1,120 @@
#pragma once
#include <stdint.h>
#ifdef __cplusplus
extern "C" {
#endif
typedef struct
{
int width;
int height;
uint32_t fcc;
} imageProp;
typedef struct
{
int width;
int height;
size_t buf; // uintptr
char* name;
int numProps;
imageProp* props;
void* grabber;
void* mediaControl;
void* callback;
} camera;
typedef struct
{
int num;
char** name;
} cameraList;
int openCamera(camera* cam, const char** errstr);
void freeCamera(camera* cam);
int listResolution(camera* cam, const char** errstr);
int listCamera(cameraList* list, const char** errstr);
int freeCameraList(cameraList* list, const char** errstr);
inline imageProp* getProp(camera* cam, int i)
{
return &cam->props[i];
}
inline char* getName(cameraList* list, int i)
{
return list->name[i];
}
#ifdef __cplusplus
}
#endif
#ifdef __cplusplus
#include <windows.h>
#include <string>
#include <dshow.h>
std::string utf16Decode(LPOLESTR olestr);
IPin* getPin(IBaseFilter* filter, PIN_DIRECTION dir);
char* getCameraName(IMoniker* monier);
template <class T>
void safeRelease(T** p)
{
if (*p)
{
(*p)->Release();
*p = nullptr;
}
}
class SampleGrabberCallback : public ISampleGrabberCB
{
private:
camera* cam_;
public:
inline SampleGrabberCallback(camera* cam)
: cam_(cam)
{
}
HRESULT SampleCB(double sampleTime, IMediaSample* sample) final;
HRESULT BufferCB(double sampleTime, BYTE* buffer, LONG bufferLen) final;
inline ULONG STDMETHODCALLTYPE AddRef() final
{
return 2;
}
inline ULONG STDMETHODCALLTYPE Release() final
{
return 1;
}
inline HRESULT STDMETHODCALLTYPE QueryInterface(REFIID riid, void** ppv) final
{
if (riid == IID_ISampleGrabberCB || riid == IID_IUnknown)
{
*ppv = (void*)static_cast<ISampleGrabberCB*>(this);
return NOERROR;
}
return E_NOINTERFACE;
}
};
EXTERN_C const CLSID CLSID_NullRenderer;
EXTERN_C const CLSID CLSID_SampleGrabber;
const static char* errAddFilter = "failed to add filter";
const static char* errConnectFilters = "failed to connect filters";
const static char* errEnumDevice = "failed to enum device";
const static char* errGrabber = "failed to create grabber";
const static char* errGraphBuilder = "failed to build graph";
const static char* errNoControl = "failed to control media";
const static char* errTerminator = "failed to create graph terminator";
const static char* errGetConfig = "failed to get config";
#endif
vendor/github.com/pion/mediadevices/pkg/driver/const.go
Generated Vendored
+14
View File
@@ -0,0 +1,14 @@
package driver
// DeviceType represents human readable device type. DeviceType
// can be useful to filter the drivers too.
type DeviceType string
const (
// Camera represents camera devices
Camera DeviceType = "camera"
// Microphone represents microphone devices
Microphone = "microphone"
// Screen represents screen devices
Screen = "screen"
)
vendor/github.com/pion/mediadevices/pkg/driver/driver.go
Generated Vendored
+59
View File
@@ -0,0 +1,59 @@
package driver
import (
"github.com/pion/mediadevices/pkg/driver/availability"
"github.com/pion/mediadevices/pkg/io/audio"
"github.com/pion/mediadevices/pkg/io/video"
"github.com/pion/mediadevices/pkg/prop"
)
type VideoRecorder interface {
VideoRecord(p prop.Media) (r video.Reader, err error)
}
type AudioRecorder interface {
AudioRecord(p prop.Media) (r audio.Reader, err error)
}
// Priority represents device selection priority level
type Priority float32
const (
// PriorityHigh is a value for system default devices
PriorityHigh Priority = 0.1
// PriorityNormal is a value for normal devices
PriorityNormal Priority = 0.0
// PriorityLow is a value for unrecommended devices
PriorityLow Priority = -0.1
)
type Info struct {
Label string
DeviceType DeviceType
Priority Priority
Name string
}
type Adapter interface {
Open() error
Close() error
Properties() []prop.Media
}
type Driver interface {
Adapter
ID() string
Info() Info
Status() State
}
type AvailabilityAdapter interface {
IsAvailable() (bool, error)
}
func IsAvailable(d Driver) (bool, error) {
if aa, ok := d.(AvailabilityAdapter); ok {
return aa.IsAvailable()
}
return false, availability.ErrUnimplemented
}
vendor/github.com/pion/mediadevices/pkg/driver/manager.go
Generated Vendored
+101
View File
@@ -0,0 +1,101 @@
package driver
import "sync"
// FilterFn is being used to decide if a driver should be included in the
// query result.
type FilterFn func(Driver) bool
// FilterVideoRecorder return a filter function to get a list of registered VideoRecorders
func FilterVideoRecorder() FilterFn {
return func(d Driver) bool {
_, ok := d.(VideoRecorder)
return ok
}
}
// FilterAudioRecorder return a filter function to get a list of registered AudioRecorders
func FilterAudioRecorder() FilterFn {
return func(d Driver) bool {
_, ok := d.(AudioRecorder)
return ok
}
}
// FilterID return a filter function to get registered drivers which have given ID
func FilterID(id string) FilterFn {
return func(d Driver) bool {
return d.ID() == id
}
}
// FilterDeviceType returns a filter function to get registered drivers which matches t type
func FilterDeviceType(t DeviceType) FilterFn {
return func(d Driver) bool {
return d.Info().DeviceType == t
}
}
// FilterAnd returns a filter function to take logical conjunction of given filters.
func FilterAnd(filters ...FilterFn) FilterFn {
return func(d Driver) bool {
for _, f := range filters {
if !f(d) {
return false
}
}
return true
}
}
// FilterNot returns a filter function to take logical inverse of the given filter.
func FilterNot(filter FilterFn) FilterFn {
return func(d Driver) bool {
return !filter(d)
}
}
// Manager is a singleton to manage multiple drivers and their states
type Manager struct {
mu sync.Mutex
drivers map[string]Driver
}
var manager = &Manager{
drivers: make(map[string]Driver),
}
// GetManager gets manager singleton instance
func GetManager() *Manager {
return manager
}
// Register registers adapter to be discoverable by Query
func (m *Manager) Register(a Adapter, info Info) error {
m.mu.Lock()
defer m.mu.Unlock()
d := wrapAdapter(a, info)
m.drivers[d.ID()] = d
return nil
}
// Query queries by using f to filter drivers, and simply return the filtered results.
func (m *Manager) Query(f FilterFn) []Driver {
m.mu.Lock()
defer m.mu.Unlock()
results := make([]Driver, 0)
for _, d := range m.drivers {
if ok := f(d); ok {
results = append(results, d)
}
}
return results
}
// Delete deletes a driver from manager given its ID
func (m *Manager) Delete(id string) {
m.mu.Lock()
defer m.mu.Unlock()
delete(m.drivers, id)
}
vendor/github.com/pion/mediadevices/pkg/driver/state.go
Generated Vendored
+64
View File
@@ -0,0 +1,64 @@
package driver
import "fmt"
// State represents driver's state
type State string
const (
// StateClosed means that the driver has not been opened. In this state,
// all information related to the hardware are still unknown. For example,
// if it's a video driver, the pixel format information is still unknown.
StateClosed State = "closed"
// StateOpened means that the driver is already opened and information about
// the hardware are already known and may be extracted from the driver.
StateOpened = "opened"
// StateRunning means that the driver has been sending data. The caller
// who started the driver may start reading data from the hardware.
StateRunning = "running"
)
// Update updates current state, s, to next. If f fails to execute,
// s will stay unchanged. Otherwise, s will be updated to next
func (s *State) Update(next State, f func() error) error {
type checkFunc func() error
m := map[State]checkFunc{
StateOpened: s.toOpened,
StateClosed: s.toClosed,
StateRunning: s.toRunning,
}
err := m[next]()
if err != nil {
return err
}
err = f()
if err == nil {
*s = next
}
return err
}
func (s *State) toOpened() error {
if *s != StateClosed {
return fmt.Errorf("invalid state: driver is already opened")
}
return nil
}
func (s *State) toClosed() error {
return nil
}
func (s *State) toRunning() error {
if *s == StateClosed {
return fmt.Errorf("invalid state: driver is closed")
}
if *s == StateRunning {
return fmt.Errorf("invalid state: driver is already running")
}
return nil
}
vendor/github.com/pion/mediadevices/pkg/driver/wrapper.go
Generated Vendored
+121
View File
@@ -0,0 +1,121 @@
package driver
import (
"github.com/google/uuid"
"github.com/pion/mediadevices/pkg/driver/availability"
"github.com/pion/mediadevices/pkg/io/audio"
"github.com/pion/mediadevices/pkg/io/video"
"github.com/pion/mediadevices/pkg/prop"
)
func wrapAdapter(a Adapter, info Info) Driver {
generator, err := uuid.NewRandom()
if err != nil {
panic(err)
}
id := generator.String()
d := &adapterWrapper{
Adapter: a,
id: id,
info: info,
state: StateClosed,
}
if aa, ok := a.(AvailabilityAdapter); ok {
d.isAvailable = aa.IsAvailable
}
switch v := a.(type) {
case VideoRecorder:
// Only expose Driver and VideoRecorder interfaces
d.VideoRecorder = v
r := &struct {
Driver
VideoRecorder
AvailabilityAdapter
}{d, d, d}
return r
case AudioRecorder:
// Only expose Driver and AudioRecorder interfaces
d.AudioRecorder = v
return &struct {
Driver
AudioRecorder
AvailabilityAdapter
}{d, d, d}
default:
panic("adapter has to be either VideoRecorder/AudioRecorder")
}
}
type adapterWrapper struct {
Adapter
VideoRecorder
AudioRecorder
id string
info Info
state State
isAvailable func() (bool, error)
}
func (w *adapterWrapper) ID() string {
return w.id
}
func (w *adapterWrapper) Info() Info {
return w.info
}
func (w *adapterWrapper) Status() State {
return w.state
}
func (w *adapterWrapper) Open() error {
return w.state.Update(StateOpened, w.Adapter.Open)
}
func (w *adapterWrapper) Close() error {
return w.state.Update(StateClosed, w.Adapter.Close)
}
func (w *adapterWrapper) Properties() []prop.Media {
if w.state == StateClosed {
return nil
}
p := w.Adapter.Properties()
for i := range p {
p[i].DeviceID = w.id
}
return p
}
func (w *adapterWrapper) VideoRecord(p prop.Media) (r video.Reader, err error) {
err = w.state.Update(StateRunning, func() error {
r, err = w.VideoRecorder.VideoRecord(p)
return err
})
if err != nil {
_ = w.Close()
}
return
}
func (w *adapterWrapper) AudioRecord(p prop.Media) (r audio.Reader, err error) {
err = w.state.Update(StateRunning, func() error {
r, err = w.AudioRecorder.AudioRecord(p)
return err
})
if err != nil {
_ = w.Close()
}
return
}
func (w *adapterWrapper) IsAvailable() (bool, error) {
if w.isAvailable == nil {
return false, availability.ErrUnimplemented
}
return w.isAvailable()
}
vendor/github.com/pion/mediadevices/pkg/frame/README.md
Generated Vendored
+1
View File
@@ -0,0 +1 @@
Special thanks to https://github.com/dialup-inc/ascii for I420 and NV21 decoder implementations.
vendor/github.com/pion/mediadevices/pkg/frame/compressed.go
Generated Vendored
+87
View File
@@ -0,0 +1,87 @@
package frame
import (
"bytes"
"errors"
"image"
"image/jpeg"
)
/*
Thank you to https://github.com/filiptc/gorbit/blob/fa87ff39b68a6706306f34c318e0b9a5a3c97110/image/overlay.go#L37-L40
for addMotionDht, dhtMarker, dht, and sosMarker. These are protected under the following license:
The MIT License (MIT)
Copyright (c) 2016 Philip Thomas Casado
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.
*/
var (
dhtMarker = []byte{255, 196}
dht = []byte{1, 162, 0, 0, 1, 5, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 1, 0, 3, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 16, 0, 2, 1, 3, 3, 2, 4, 3, 5, 5, 4, 4, 0, 0, 1, 125, 1, 2, 3, 0, 4, 17, 5, 18, 33, 49, 65, 6, 19, 81, 97, 7, 34, 113, 20, 50, 129, 145, 161, 8, 35, 66, 177, 193, 21, 82, 209, 240, 36, 51, 98, 114, 130, 9, 10, 22, 23, 24, 25, 26, 37, 38, 39, 40, 41, 42, 52, 53, 54, 55, 56, 57, 58, 67, 68, 69, 70, 71, 72, 73, 74, 83, 84, 85, 86, 87, 88, 89, 90, 99, 100, 101, 102, 103, 104, 105, 106, 115, 116, 117, 118, 119, 120, 121, 122, 131, 132, 133, 134, 135, 136, 137, 138, 146, 147, 148, 149, 150, 151, 152, 153, 154, 162, 163, 164, 165, 166, 167, 168, 169, 170, 178, 179, 180, 181, 182, 183, 184, 185, 186, 194, 195, 196, 197, 198, 199, 200, 201, 202, 210, 211, 212, 213, 214, 215, 216, 217, 218, 225, 226, 227, 228, 229, 230, 231, 232, 233, 234, 241, 242, 243, 244, 245, 246, 247, 248, 249, 250, 17, 0, 2, 1, 2, 4, 4, 3, 4, 7, 5, 4, 4, 0, 1, 2, 119, 0, 1, 2, 3, 17, 4, 5, 33, 49, 6, 18, 65, 81, 7, 97, 113, 19, 34, 50, 129, 8, 20, 66, 145, 161, 177, 193, 9, 35, 51, 82, 240, 21, 98, 114, 209, 10, 22, 36, 52, 225, 37, 241, 23, 24, 25, 26, 38, 39, 40, 41, 42, 53, 54, 55, 56, 57, 58, 67, 68, 69, 70, 71, 72, 73, 74, 83, 84, 85, 86, 87, 88, 89, 90, 99, 100, 101, 102, 103, 104, 105, 106, 115, 116, 117, 118, 119, 120, 121, 122, 130, 131, 132, 133, 134, 135, 136, 137, 138, 146, 147, 148, 149, 150, 151, 152, 153, 154, 162, 163, 164, 165, 166, 167, 168, 169, 170, 178, 179, 180, 181, 182, 183, 184, 185, 186, 194, 195, 196, 197, 198, 199, 200, 201, 202, 210, 211, 212, 213, 214, 215, 216, 217, 218, 226, 227, 228, 229, 230, 231, 232, 233, 234, 242, 243, 244, 245, 246, 247, 248, 249, 250}
sosMarker = []byte{255, 218}
huffmanTableInfoLength = len(dhtMarker) + len(dht) + len(sosMarker)
uninitializedHuffmanTableError jpeg.FormatError = jpeg.FormatError("uninitialized Huffman table")
)
func decodeMJPEG(frame []byte, width, height int) (image.Image, func(), error) {
img, err := jpeg.Decode(bytes.NewReader(frame))
if err == nil {
return img, func() {}, err
}
if errors.As(err, &uninitializedHuffmanTableError) {
if err.Error() == uninitializedHuffmanTableError.Error() {
img, err = jpeg.Decode(bytes.NewReader(addMotionDht(frame)))
}
}
if err != nil {
return nil, nil, err
}
return img, func() {}, err
}
func addMotionDht(frame []byte) []byte {
jpegParts := bytes.Split(frame, sosMarker)
if len(jpegParts) != 2 {
return frame
}
correctedFrame := make([]byte, len(jpegParts[0])+huffmanTableInfoLength+len(jpegParts[1]))
correctedFrameOffset := 0
copy(correctedFrame[correctedFrameOffset:], jpegParts[0])
correctedFrameOffset += len(jpegParts[0])
copy(correctedFrame[correctedFrameOffset:], dhtMarker)
correctedFrameOffset += len(dhtMarker)
copy(correctedFrame[correctedFrameOffset:], dht)
correctedFrameOffset += len(dht)
copy(correctedFrame[correctedFrameOffset:], sosMarker)
correctedFrameOffset += len(sosMarker)
copy(correctedFrame[correctedFrameOffset:], jpegParts[1])
return correctedFrame
}
vendor/github.com/pion/mediadevices/pkg/frame/decode.go
Generated Vendored
+55
View File
@@ -0,0 +1,55 @@
package frame
import (
"fmt"
)
type Format string
const (
// FormatI420 https://wiki.videolan.org/YUV#I420
FormatI420 Format = "I420"
// FormatI444 is a YUV format without sub-sampling
FormatI444 Format = "I444"
// FormatNV21 https://www.kernel.org/doc/html/v5.9/userspace-api/media/v4l/pixfmt-nv12.html
FormatNV21 = "NV21"
// FormatNV12 https://www.kernel.org/doc/html/v5.9/userspace-api/media/v4l/pixfmt-nv12.html
FormatNV12 = "NV12"
// FormatYUY2 https://www.kernel.org/doc/html/v5.9/userspace-api/media/v4l/pixfmt-yuyv.html
// YUY2 is what Windows calls YUYV
FormatYUY2 = "YUY2"
// FormatYUYV https://www.kernel.org/doc/html/v5.9/userspace-api/media/v4l/pixfmt-yuyv.html
FormatYUYV = "YUYV"
// FormatUYVY https://www.kernel.org/doc/html/v5.9/userspace-api/media/v4l/pixfmt-uyvy.html
FormatUYVY = "UYVY"
// FormatRGBA https://www.kernel.org/doc/html/v5.9/userspace-api/media/v4l/pixfmt-rgb.html
FormatRGBA Format = "RGBA"
// FormatMJPEG https://wiki.videolan.org/MJPEG
FormatMJPEG = "MJPEG"
// FormatZ16 https://www.kernel.org/doc/html/v5.9/userspace-api/media/v4l/pixfmt-z16.html
FormatZ16 = "Z16"
)
var decoderMap = map[Format]decoderFunc{
FormatI420: decodeI420,
FormatNV21: decodeNV21,
FormatNV12: decodeNV12,
FormatYUY2: decodeYUY2,
FormatYUYV: decodeYUY2,
FormatUYVY: decodeUYVY,
FormatMJPEG: decodeMJPEG,
FormatZ16: decodeZ16,
}
func NewDecoder(f Format) (Decoder, error) {
decoder, ok := decoderMap[f]
if !ok {
return nil, fmt.Errorf("%s is not supported", f)
}
return decoder, nil
}
vendor/github.com/pion/mediadevices/pkg/frame/frame.go
Generated Vendored
+14
View File
@@ -0,0 +1,14 @@
package frame
import "image"
type Decoder interface {
Decode(frame []byte, width, height int) (image.Image, func(), error)
}
// DecoderFunc is a proxy type for Decoder
type decoderFunc func(frame []byte, width, height int) (image.Image, func(), error)
func (f decoderFunc) Decode(frame []byte, width, height int) (image.Image, func(), error) {
return f(frame, width, height)
}
vendor/github.com/pion/mediadevices/pkg/frame/sampleFrames.go
Generated Vendored
+3
View File
File diff suppressed because one or more lines are too long
vendor/github.com/pion/mediadevices/pkg/frame/yuv.c
Generated Vendored
+41
View File
@@ -0,0 +1,41 @@
#include <stdint.h>
void decodeYUY2CGO(
uint8_t* y,
uint8_t* cb,
uint8_t* cr,
uint8_t* yuy2,
int width, int height)
{
const int l = width * height * 2;
int i, fast = 0, slow = 0;
for (i = 0; i < l; i += 4)
{
y[fast] = yuy2[i];
cb[slow] = yuy2[i + 1];
y[fast + 1] = yuy2[i + 2];
cr[slow] = yuy2[i + 3];
fast += 2;
++slow;
}
}
void decodeUYVYCGO(
uint8_t* y,
uint8_t* cb,
uint8_t* cr,
uint8_t* uyvy,
int width, int height)
{
const int l = width * height * 2;
int i, fast = 0, slow = 0;
for (i = 0; i < l; i += 4)
{
cb[slow] = uyvy[i];
y[fast] = uyvy[i+1];
cr[slow] = uyvy[i + 2];
y[fast + 1] = uyvy[i + 3];
fast += 2;
++slow;
}
}
vendor/github.com/pion/mediadevices/pkg/frame/yuv.go
Generated Vendored
+63
View File
@@ -0,0 +1,63 @@
package frame
import (
"fmt"
"image"
)
func decodeI420(frame []byte, width, height int) (image.Image, func(), error) {
yi := width * height
cbi := yi + width*height/4
cri := cbi + width*height/4
if cri > len(frame) {
return nil, func() {}, fmt.Errorf("frame length (%d) less than expected (%d)", len(frame), cri)
}
return &image.YCbCr{
Y: frame[:yi],
YStride: width,
Cb: frame[yi:cbi],
Cr: frame[cbi:cri],
CStride: width / 2,
SubsampleRatio: image.YCbCrSubsampleRatio420,
Rect: image.Rect(0, 0, width, height),
}, func() {}, nil
}
func decodeNV21(frame []byte, width, height int) (image.Image, func(), error) {
yi := width * height
ci := yi + width*height/2
if ci > len(frame) {
return nil, func() {}, fmt.Errorf("frame length (%d) less than expected (%d)", len(frame), ci)
}
var cb, cr []byte
for i := yi; i < ci; i += 2 {
cr = append(cr, frame[i])
cb = append(cb, frame[i+1])
}
return &image.YCbCr{
Y: frame[:yi],
YStride: width,
Cb: cb,
Cr: cr,
CStride: width / 2,
SubsampleRatio: image.YCbCrSubsampleRatio420,
Rect: image.Rect(0, 0, width, height),
}, func() {}, nil
}
func decodeNV12(frame []byte, width, height int) (image.Image, func(), error) {
img, release, err := decodeNV21(frame, width, height)
if err != nil {
return img, release, err
}
// The only difference between NV21 and NV12 is the chroma order, so simply swap them
yuv := img.(*image.YCbCr)
yuv.Cb, yuv.Cr = yuv.Cr, yuv.Cb
return yuv, release, err
}
vendor/github.com/pion/mediadevices/pkg/frame/yuv_cgo.go
Generated Vendored
+77
View File
@@ -0,0 +1,77 @@
// +build cgo
package frame
import (
"fmt"
"image"
)
// #include <stdint.h>
// void decodeYUY2CGO(uint8_t* y, uint8_t* cb, uint8_t* cr, uint8_t* yuy2, int width, int height);
// void decodeUYVYCGO(uint8_t* y, uint8_t* cb, uint8_t* cr, uint8_t* uyvy, int width, int height);
import "C"
func decodeYUY2(frame []byte, width, height int) (image.Image, func(), error) {
yi := width * height
ci := yi / 2
fi := yi + 2*ci
if len(frame) < fi {
return nil, func() {}, fmt.Errorf("frame length (%d) less than expected (%d)", len(frame), fi)
}
y := make([]byte, yi)
cb := make([]byte, ci)
cr := make([]byte, ci)
C.decodeYUY2CGO(
(*C.uchar)(&y[0]),
(*C.uchar)(&cb[0]),
(*C.uchar)(&cr[0]),
(*C.uchar)(&frame[0]),
C.int(width), C.int(height),
)
return &image.YCbCr{
Y: y,
YStride: width,
Cb: cb,
Cr: cr,
CStride: width / 2,
SubsampleRatio: image.YCbCrSubsampleRatio422,
Rect: image.Rect(0, 0, width, height),
}, func() {}, nil
}
func decodeUYVY(frame []byte, width, height int) (image.Image, func(), error) {
yi := width * height
ci := yi / 2
fi := yi + 2*ci
if len(frame) < fi {
return nil, func() {}, fmt.Errorf("frame length (%d) less than expected (%d)", len(frame), fi)
}
y := make([]byte, yi)
cb := make([]byte, ci)
cr := make([]byte, ci)
C.decodeUYVYCGO(
(*C.uchar)(&y[0]),
(*C.uchar)(&cb[0]),
(*C.uchar)(&cr[0]),
(*C.uchar)(&frame[0]),
C.int(width), C.int(height),
)
return &image.YCbCr{
Y: y,
YStride: width,
Cb: cb,
Cr: cr,
CStride: width / 2,
SubsampleRatio: image.YCbCrSubsampleRatio422,
Rect: image.Rect(0, 0, width, height),
}, func() {}, nil
}
vendor/github.com/pion/mediadevices/pkg/frame/yuv_nocgo.go
Generated Vendored
+78
View File
@@ -0,0 +1,78 @@
// +build !cgo
package frame
import (
"fmt"
"image"
)
func decodeYUY2(frame []byte, width, height int) (image.Image, func(), error) {
yi := width * height
ci := yi / 2
fi := yi + 2*ci
if len(frame) < fi {
return nil, func() {}, fmt.Errorf("frame length (%d) less than expected (%d)", len(frame), fi)
}
y := make([]byte, yi)
cb := make([]byte, ci)
cr := make([]byte, ci)
fast := 0
slow := 0
for i := 0; i < fi; i += 4 {
y[fast] = frame[i]
cb[slow] = frame[i+1]
y[fast+1] = frame[i+2]
cr[slow] = frame[i+3]
fast += 2
slow++
}
return &image.YCbCr{
Y: y,
YStride: width,
Cb: cb,
Cr: cr,
CStride: width / 2,
SubsampleRatio: image.YCbCrSubsampleRatio422,
Rect: image.Rect(0, 0, width, height),
}, func() {}, nil
}
func decodeUYVY(frame []byte, width, height int) (image.Image, func(), error) {
yi := width * height
ci := yi / 2
fi := yi + 2*ci
if len(frame) < fi {
return nil, func() {}, fmt.Errorf("frame length (%d) less than expected (%d)", len(frame), fi)
}
y := make([]byte, yi)
cb := make([]byte, ci)
cr := make([]byte, ci)
fast := 0
slow := 0
for i := 0; i < fi; i += 4 {
cb[slow] = frame[i]
y[fast] = frame[i+1]
cr[slow] = frame[i+2]
y[fast+1] = frame[i+3]
fast += 2
slow++
}
return &image.YCbCr{
Y: y,
YStride: width,
Cb: cb,
Cr: cr,
CStride: width / 2,
SubsampleRatio: image.YCbCrSubsampleRatio422,
Rect: image.Rect(0, 0, width, height),
}, func() {}, nil
}
vendor/github.com/pion/mediadevices/pkg/frame/z16.go
Generated Vendored
+36
View File
@@ -0,0 +1,36 @@
package frame
import (
"encoding/binary"
"fmt"
"image"
"image/color"
)
func decodeZ16(frame []byte, width, height int) (image.Image, func(), error) {
expectedSize := 2 * (width * height)
if expectedSize != len(frame) {
return nil, func() {}, fmt.Errorf("frame length (%d) not expected size (%d)", len(frame), expectedSize)
}
img := image.NewGray16(image.Rect(0, 0, width, height))
/*
v4l specifies images in terms of series of lines which is perplexing because the
example in https://www.kernel.org/doc/html/v5.9/userspace-api/media/v4l/pixfmt-z16.html
seems to swap the subscript of each depth value.
Clear example:
Width: 3, Height: 4
[Z_low(x_0,y_0), Z_high(x_0,y_0), Z_low(x_1,y_0), Z_high(x_1,y_0), Z_low(x_2,y_0), Z_high(x_2,y_0),
Z_low(x_0,y_1), Z_high(x_0,y_1), Z_low(x_1,y_1), Z_high(x_1,y_1), Z_low(x_2,y_1), Z_high(x_2,y_1),
Z_low(x_0,y_2), Z_high(x_0,y_2), Z_low(x_1,y_2), Z_high(x_1,y_2), Z_low(x_2,y_2), Z_high(x_2,y_2),
Z_low(x_0,y_3), Z_high(x_0,y_3), Z_low(x_1,y_3), Z_high(x_1,y_3), Z_low(x_2,y_3), Z_high(x_2,y_3)]
*/
for x := 0; x < width; x++ {
for y := 0; y < height; y++ {
idx := 2 * (x + (y * width))
z := binary.LittleEndian.Uint16(frame[idx : idx+2])
img.SetGray16(x, y, color.Gray16{Y: z})
}
}
return img, func() {}, nil
}
vendor/github.com/pion/mediadevices/pkg/io/audio/audio.go
Generated Vendored
+43
View File
@@ -0,0 +1,43 @@
package audio
import (
"github.com/pion/mediadevices/pkg/wave"
)
type Reader interface {
// Read reads data from the source. The caller is responsible to release the memory that's associated
// with data by calling the given release function. When err is not nil, the caller MUST NOT call release
// as data is going to be nil (no memory was given). Otherwise, the caller SHOULD call release after
// using the data. The caller is NOT REQUIRED to call release, as this is only a part of memory management
// optimization. If release is not called, the source is forced to allocate a new memory, which also means
// there will be new allocations during streaming, and old unused memory will become garbage. As a consequence,
// these garbage will put a lot of pressure to the garbage collector and makes it to run more often and finish
// slower as the heap memory usage increases and more garbage to collect.
Read() (chunk wave.Audio, release func(), err error)
}
type ReaderFunc func() (chunk wave.Audio, release func(), err error)
func (rf ReaderFunc) Read() (chunk wave.Audio, release func(), err error) {
chunk, release, err = rf()
return
}
// TransformFunc produces a new Reader that will produces a transformed audio
type TransformFunc func(r Reader) Reader
// Merge merges transforms and produces a new TransformFunc that will execute
// transforms in order
func Merge(transforms ...TransformFunc) TransformFunc {
return func(r Reader) Reader {
for _, transform := range transforms {
if transform == nil {
continue
}
r = transform(r)
}
return r
}
}
vendor/github.com/pion/mediadevices/pkg/io/audio/broadcast.go
Generated Vendored
+76
View File
@@ -0,0 +1,76 @@
package audio
import (
"errors"
"github.com/pion/mediadevices/pkg/io"
"github.com/pion/mediadevices/pkg/wave"
)
var errEmptySource = errors.New("Source can't be nil")
// Broadcaster is a specialized video broadcaster.
type Broadcaster struct {
ioBroadcaster *io.Broadcaster
}
type BroadcasterConfig struct {
Core *io.BroadcasterConfig
}
// NewBroadcaster creates a new broadcaster. Source is expected to drop chunks
// when any of the readers is slower than the source.
func NewBroadcaster(source Reader, config *BroadcasterConfig) *Broadcaster {
var coreConfig *io.BroadcasterConfig
if config != nil {
coreConfig = config.Core
}
broadcaster := io.NewBroadcaster(io.ReaderFunc(func() (interface{}, func(), error) {
return source.Read()
}), coreConfig)
return &Broadcaster{broadcaster}
}
// NewReader creates a new reader. Each reader will retrieve the same data from the source.
// copyFn is used to copy the data from the source to individual readers. Broadcaster uses a small ring
// buffer, this means that slow readers might miss some data if they're really late and the data is no longer
// in the ring buffer.
func (broadcaster *Broadcaster) NewReader(copyChunk bool) Reader {
copyFn := func(src interface{}) interface{} { return src }
if copyChunk {
buffer := wave.NewBuffer()
copyFn = func(src interface{}) interface{} {
realSrc, _ := src.(wave.Audio)
buffer.StoreCopy(realSrc)
return buffer.Load()
}
}
reader := broadcaster.ioBroadcaster.NewReader(copyFn)
return ReaderFunc(func() (wave.Audio, func(), error) {
data, _, err := reader.Read()
chunk, _ := data.(wave.Audio)
return chunk, func() {}, err
})
}
// ReplaceSource replaces the underlying source. This operation is thread safe.
func (broadcaster *Broadcaster) ReplaceSource(source Reader) error {
return broadcaster.ioBroadcaster.ReplaceSource(io.ReaderFunc(func() (interface{}, func(), error) {
return source.Read()
}))
}
// Source retrieves the underlying source. This operation is thread safe.
func (broadcaster *Broadcaster) Source() Reader {
source := broadcaster.ioBroadcaster.Source()
return ReaderFunc(func() (wave.Audio, func(), error) {
data, _, err := source.Read()
img, _ := data.(wave.Audio)
return img, func() {}, err
})
}
vendor/github.com/pion/mediadevices/pkg/io/audio/buffer.go
Generated Vendored
+89
View File
@@ -0,0 +1,89 @@
package audio
import (
"errors"
"github.com/pion/mediadevices/pkg/wave"
)
var errUnsupported = errors.New("unsupported audio format")
// NewBuffer creates audio transform to buffer signal to have exact nSample samples.
func NewBuffer(nSamples int) TransformFunc {
var inBuff wave.Audio
return func(r Reader) Reader {
return ReaderFunc(func() (wave.Audio, func(), error) {
for {
if inBuff != nil && inBuff.ChunkInfo().Len >= nSamples {
break
}
buff, _, err := r.Read()
if err != nil {
return nil, func() {}, err
}
switch b := buff.(type) {
case *wave.Float32Interleaved:
ib, ok := inBuff.(*wave.Float32Interleaved)
if !ok || ib.Size.Channels != b.Size.Channels {
ib = wave.NewFloat32Interleaved(
wave.ChunkInfo{
SamplingRate: b.Size.SamplingRate,
Channels: b.Size.Channels,
Len: nSamples,
},
)
ib.Data = ib.Data[:0]
ib.Size.Len = 0
inBuff = ib
}
ib.Data = append(ib.Data, b.Data...)
ib.Size.Len += b.Size.Len
case *wave.Int16Interleaved:
ib, ok := inBuff.(*wave.Int16Interleaved)
if !ok || ib.Size.Channels != b.Size.Channels {
ib = wave.NewInt16Interleaved(
wave.ChunkInfo{
SamplingRate: b.Size.SamplingRate,
Channels: b.Size.Channels,
Len: nSamples,
},
)
ib.Data = ib.Data[:0]
ib.Size.Len = 0
inBuff = ib
}
ib.Data = append(ib.Data, b.Data...)
ib.Size.Len += b.Size.Len
default:
return nil, func() {}, errUnsupported
}
}
switch ib := inBuff.(type) {
case *wave.Int16Interleaved:
ibCopy := *ib
ibCopy.Size.Len = nSamples
n := nSamples * ib.Size.Channels
ibCopy.Data = make([]int16, n)
copy(ibCopy.Data, ib.Data)
ib.Data = ib.Data[n:]
ib.Size.Len -= nSamples
return &ibCopy, func() {}, nil
case *wave.Float32Interleaved:
ibCopy := *ib
ibCopy.Size.Len = nSamples
n := nSamples * ib.Size.Channels
ibCopy.Data = make([]float32, n)
copy(ibCopy.Data, ib.Data)
ib.Data = ib.Data[n:]
ib.Size.Len -= nSamples
return &ibCopy, func() {}, nil
}
return nil, func() {}, errUnsupported
})
}
}
vendor/github.com/pion/mediadevices/pkg/io/audio/detect.go
Generated Vendored
+55
View File
@@ -0,0 +1,55 @@
package audio
import (
"time"
"github.com/pion/mediadevices/pkg/prop"
"github.com/pion/mediadevices/pkg/wave"
)
// DetectChanges will detect chunk and audio property changes. For audio property detection,
// since it's time related, interval will be used to determine the sample rate.
func DetectChanges(interval time.Duration, onChange func(prop.Media)) TransformFunc {
return func(r Reader) Reader {
var currentProp prop.Media
var chunkCount uint
return ReaderFunc(func() (wave.Audio, func(), error) {
var dirty bool
chunk, _, err := r.Read()
if err != nil {
return nil, func() {}, err
}
info := chunk.ChunkInfo()
if currentProp.ChannelCount != info.Channels {
currentProp.ChannelCount = info.Channels
dirty = true
}
if currentProp.SampleRate != info.SamplingRate {
currentProp.SampleRate = info.SamplingRate
dirty = true
}
var latency time.Duration
if currentProp.SampleRate != 0 {
latency = time.Duration(chunk.ChunkInfo().Len) * time.Second / time.Nanosecond / time.Duration(currentProp.SampleRate)
}
if currentProp.Latency != latency {
currentProp.Latency = latency
dirty = true
}
// TODO: Also detect sample format changes?
// TODO: Add audio detect changes. As of now, there's no useful property to track.
if dirty {
onChange(currentProp)
}
chunkCount++
return chunk, func() {}, nil
})
}
}
vendor/github.com/pion/mediadevices/pkg/io/audio/mixer.go
Generated Vendored
+40
View File
@@ -0,0 +1,40 @@
package audio
import (
"github.com/pion/mediadevices/pkg/wave"
"github.com/pion/mediadevices/pkg/wave/mixer"
)
// NewChannelMixer creates audio transform to mix audio channels.
func NewChannelMixer(channels int, mixer mixer.ChannelMixer) TransformFunc {
return func(r Reader) Reader {
return ReaderFunc(func() (wave.Audio, func(), error) {
buff, _, err := r.Read()
if err != nil {
return nil, func() {}, err
}
ci := buff.ChunkInfo()
if ci.Channels == channels {
return buff, func() {}, nil
}
ci.Channels = channels
var mixed wave.Audio
switch buff.(type) {
case *wave.Int16Interleaved:
mixed = wave.NewInt16Interleaved(ci)
case *wave.Int16NonInterleaved:
mixed = wave.NewInt16NonInterleaved(ci)
case *wave.Float32Interleaved:
mixed = wave.NewFloat32Interleaved(ci)
case *wave.Float32NonInterleaved:
mixed = wave.NewFloat32NonInterleaved(ci)
}
if err := mixer.Mix(mixed, buff); err != nil {
return nil, func() {}, err
}
return mixed, func() {}, nil
})
}
}
vendor/github.com/pion/mediadevices/pkg/io/broadcast.go
Generated Vendored
+164
View File
@@ -0,0 +1,164 @@
package io
import (
"fmt"
"sync/atomic"
"time"
)
const (
maskReading = 1 << 63
defaultBroadcasterRingSize = 32
// TODO: If the data source has fps greater than 30, they'll see some
// fps fluctuation. But, 30 fps should be enough for general cases.
defaultBroadcasterRingPollDuration = time.Millisecond * 33
)
var errEmptySource = fmt.Errorf("Source can't be nil")
type broadcasterData struct {
data interface{}
count uint32
err error
}
type broadcasterRing struct {
// reading (1 bit) + reserved (31 bits) + data count (32 bits)
// IMPORTANT: state has to be the first element in struct, otherwise LoadUint64 will panic in 32 bits systems
// due to unallignment
state uint64
buffer []atomic.Value
pollDuration time.Duration
}
func newBroadcasterRing(size uint, pollDuration time.Duration) *broadcasterRing {
return &broadcasterRing{buffer: make([]atomic.Value, size), pollDuration: pollDuration}
}
func (ring *broadcasterRing) index(count uint32) int {
return int(count) % len(ring.buffer)
}
func (ring *broadcasterRing) acquire(count uint32) func(*broadcasterData) {
// Reader has reached the latest data, should read from the source.
// Only allow 1 reader to read from the source. When there are more than 1 readers,
// the other readers will need to share the same data that the first reader gets from
// the source.
state := uint64(count)
if atomic.CompareAndSwapUint64(&ring.state, state, state|maskReading) {
return func(data *broadcasterData) {
i := ring.index(count)
ring.buffer[i].Store(data)
atomic.StoreUint64(&ring.state, uint64(count+1))
}
}
return nil
}
func (ring *broadcasterRing) get(count uint32) *broadcasterData {
for {
reading := uint64(count) | maskReading
// TODO: since it's lockless, it spends a lot of resources in the scheduling.
for atomic.LoadUint64(&ring.state) == reading {
// Yield current goroutine to let other goroutines to run instead
time.Sleep(ring.pollDuration)
}
i := ring.index(count)
data := ring.buffer[i].Load().(*broadcasterData)
if data.count == count {
return data
}
count++
}
}
func (ring *broadcasterRing) lastCount() uint32 {
// ring.state always keeps track the next count, so we need to subtract it by 1 to get the
// last count
return uint32(atomic.LoadUint64(&ring.state)) - 1
}
// Broadcaster is a generic pull-based broadcaster. Broadcaster is unique in a sense that
// readers can come and go at anytime, and readers don't need to close or notify broadcaster.
type Broadcaster struct {
source atomic.Value
buffer *broadcasterRing
}
// BroadcasterConfig is a config to control broadcaster behaviour
type BroadcasterConfig struct {
// BufferSize configures the underlying ring buffer size that's being used
// to avoid data lost for late readers. The default value is 32.
BufferSize uint
// PollDuration configures the sleep duration in waiting for new data to come.
// The default value is 33 ms.
PollDuration time.Duration
}
// NewBroadcaster creates a new broadcaster. Source is expected to drop frames
// when any of the readers is slower than the source.
func NewBroadcaster(source Reader, config *BroadcasterConfig) *Broadcaster {
pollDuration := defaultBroadcasterRingPollDuration
var bufferSize uint = defaultBroadcasterRingSize
if config != nil {
if config.PollDuration != 0 {
pollDuration = config.PollDuration
}
if config.BufferSize != 0 {
bufferSize = config.BufferSize
}
}
var broadcaster Broadcaster
broadcaster.buffer = newBroadcasterRing(bufferSize, pollDuration)
broadcaster.ReplaceSource(source)
return &broadcaster
}
// NewReader creates a new reader. Each reader will retrieve the same data from the source.
// copyFn is used to copy the data from the source to individual readers. Broadcaster uses a small ring
// buffer, this means that slow readers might miss some data if they're really late and the data is no longer
// in the ring buffer.
func (broadcaster *Broadcaster) NewReader(copyFn func(interface{}) interface{}) Reader {
currentCount := broadcaster.buffer.lastCount()
return ReaderFunc(func() (data interface{}, release func(), err error) {
currentCount++
if push := broadcaster.buffer.acquire(currentCount); push != nil {
data, _, err = broadcaster.source.Load().(Reader).Read()
push(&broadcasterData{
data: data,
err: err,
count: currentCount,
})
} else {
ringData := broadcaster.buffer.get(currentCount)
data, err, currentCount = ringData.data, ringData.err, ringData.count
}
if data != nil { // data is nil if an error occurred during reading
data = copyFn(data)
}
return
})
}
// ReplaceSource replaces the underlying source. This operation is thread safe.
func (broadcaster *Broadcaster) ReplaceSource(source Reader) error {
if source == nil {
return errEmptySource
}
broadcaster.source.Store(source)
return nil
}
// ReplaceSource retrieves the underlying source. This operation is thread safe.
func (broadcaster *Broadcaster) Source() Reader {
return broadcaster.source.Load().(Reader)
}
vendor/github.com/pion/mediadevices/pkg/io/error.go
Generated Vendored
+13
View File
@@ -0,0 +1,13 @@
package io
import "fmt"
// InsufficientBufferError tells the caller that the buffer provided is not sufficient/big
// enough to hold the whole data/sample.
type InsufficientBufferError struct {
RequiredSize int
}
func (e *InsufficientBufferError) Error() string {
return fmt.Sprintf("provided buffer doesn't meet the size requirement of length, %d", e.RequiredSize)
}
vendor/github.com/pion/mediadevices/pkg/io/reader.go
Generated Vendored
+23
View File
@@ -0,0 +1,23 @@
package io
// Reader is a generic data reader. In the future, interface{} should be replaced by a generic type
// to provide strong type.
type Reader interface {
// Read reads data from the source. The caller is responsible to release the memory that's associated
// with data by calling the given release function. When err is not nil, the caller MUST NOT call release
// as data is going to be nil (no memory was given). Otherwise, the caller SHOULD call release after
// using the data. The caller is NOT REQUIRED to call release, as this is only a part of memory management
// optimization. If release is not called, the source is forced to allocate a new memory, which also means
// there will be new allocations during streaming, and old unused memory will become garbage. As a consequence,
// these garbage will put a lot of pressure to the garbage collector and makes it to run more often and finish
// slower as the heap memory usage increases and more garbage to collect.
Read() (data interface{}, release func(), err error)
}
// ReaderFunc is a proxy type for Reader
type ReaderFunc func() (data interface{}, release func(), err error)
func (f ReaderFunc) Read() (data interface{}, release func(), err error) {
data, release, err = f()
return
}
vendor/github.com/pion/mediadevices/pkg/io/video/broadcast.go
Generated Vendored
+76
View File
@@ -0,0 +1,76 @@
package video
import (
"fmt"
"image"
"github.com/pion/mediadevices/pkg/io"
)
var errEmptySource = fmt.Errorf("Source can't be nil")
// Broadcaster is a specialized video broadcaster.
type Broadcaster struct {
ioBroadcaster *io.Broadcaster
}
type BroadcasterConfig struct {
Core *io.BroadcasterConfig
}
// NewBroadcaster creates a new broadcaster. Source is expected to drop frames
// when any of the readers is slower than the source.
func NewBroadcaster(source Reader, config *BroadcasterConfig) *Broadcaster {
var coreConfig *io.BroadcasterConfig
if config != nil {
coreConfig = config.Core
}
broadcaster := io.NewBroadcaster(io.ReaderFunc(func() (interface{}, func(), error) {
return source.Read()
}), coreConfig)
return &Broadcaster{broadcaster}
}
// NewReader creates a new reader. Each reader will retrieve the same data from the source.
// copyFn is used to copy the data from the source to individual readers. Broadcaster uses a small ring
// buffer, this means that slow readers might miss some data if they're really late and the data is no longer
// in the ring buffer.
func (broadcaster *Broadcaster) NewReader(copyFrame bool) Reader {
copyFn := func(src interface{}) interface{} { return src }
if copyFrame {
buffer := NewFrameBuffer(0)
copyFn = func(src interface{}) interface{} {
realSrc, _ := src.(image.Image)
buffer.StoreCopy(realSrc)
return buffer.Load()
}
}
reader := broadcaster.ioBroadcaster.NewReader(copyFn)
return ReaderFunc(func() (image.Image, func(), error) {
data, _, err := reader.Read()
img, _ := data.(image.Image)
return img, func() {}, err
})
}
// ReplaceSource replaces the underlying source. This operation is thread safe.
func (broadcaster *Broadcaster) ReplaceSource(source Reader) error {
return broadcaster.ioBroadcaster.ReplaceSource(io.ReaderFunc(func() (interface{}, func(), error) {
return source.Read()
}))
}
// Source retrieves the underlying source. This operation is thread safe.
func (broadcaster *Broadcaster) Source() Reader {
source := broadcaster.ioBroadcaster.Source()
return ReaderFunc(func() (image.Image, func(), error) {
data, _, err := source.Read()
img, _ := data.(image.Image)
return img, func() {}, err
})
}
vendor/github.com/pion/mediadevices/pkg/io/video/convert.go
Generated Vendored
+174
View File
@@ -0,0 +1,174 @@
package video
import (
"fmt"
"image"
"image/color"
"sync"
)
// imageToYCbCr converts src to *image.YCbCr and store it to dst
// Note: conversion can be lossy
func imageToYCbCr(dst *image.YCbCr, src image.Image) {
if dst == nil {
panic("dst can't be nil")
}
yuvImg, ok := src.(*image.YCbCr)
if ok {
*dst = *yuvImg
return
}
bounds := src.Bounds()
dy := bounds.Dy()
dx := bounds.Dx()
flat := dy * dx
if len(dst.Y)+len(dst.Cb)+len(dst.Cr) < 3*flat {
i0 := 1 * flat
i1 := 2 * flat
i2 := 3 * flat
if cap(dst.Y) < i2 {
dst.Y = make([]uint8, i2)
}
dst.Y = dst.Y[:i0]
dst.Cb = dst.Y[i0:i1]
dst.Cr = dst.Y[i1:i2]
}
dst.SubsampleRatio = image.YCbCrSubsampleRatio444
dst.YStride = dx
dst.CStride = dx
dst.Rect = bounds
switch s := src.(type) {
case *image.RGBA:
rgbaToI444(dst, s)
default:
i := 0
for yi := 0; yi < dy; yi++ {
for xi := 0; xi < dx; xi++ {
// TODO: probably try to get the alpha value with something like
// https://en.wikipedia.org/wiki/Alpha_compositing
r, g, b, _ := src.At(xi, yi).RGBA()
yy, cb, cr := color.RGBToYCbCr(uint8(r/256), uint8(g/256), uint8(b/256))
dst.Y[i] = yy
dst.Cb[i] = cb
dst.Cr[i] = cr
i++
}
}
}
}
// bytePool stores slices to be reused
// New method is not set as the slice size
// should be allocated according to subsample ratio
var bytesPool sync.Pool
// ToI420 converts r to a new reader that will output images in I420 format
func ToI420(r Reader) Reader {
var yuvImg image.YCbCr
getSlice := func(cLen int) []uint8 {
// Retrieve slice from pool
dst, ok := bytesPool.Get().([]byte)
// Compare value or capacity of retrieved object
// If less than expected, reallocate new object
if !ok || cap(dst) < 2*cLen {
// Allocating memory for Cb and Cr
dst = make([]byte, 2*cLen, 2*cLen)
}
return dst
}
return ReaderFunc(func() (image.Image, func(), error) {
img, _, err := r.Read()
if err != nil {
return nil, func() {}, err
}
var releaseFunc func() = func() {}
imageToYCbCr(&yuvImg, img)
// Covert pixel format to I420
switch yuvImg.SubsampleRatio {
case image.YCbCrSubsampleRatio420:
case image.YCbCrSubsampleRatio444:
cLen := yuvImg.CStride * yuvImg.Rect.Dy() / 4
dst := getSlice(cLen)
yuvImg = i444ToI420(yuvImg, dst)
releaseFunc = func() {
bytesPool.Put(dst)
}
case image.YCbCrSubsampleRatio422:
cLen := yuvImg.CStride * (yuvImg.Rect.Dy() / 2)
dst := getSlice(cLen)
yuvImg = i422ToI420(yuvImg, dst)
releaseFunc = func() {
bytesPool.Put(dst)
}
default:
return nil, releaseFunc, fmt.Errorf("unsupported pixel format: %s", yuvImg.SubsampleRatio)
}
return &yuvImg, releaseFunc, nil
})
}
// imageToRGBA converts src to *image.RGBA and store it to dst
func imageToRGBA(dst *image.RGBA, src image.Image) {
if dst == nil {
panic("dst can't be nil")
}
if srcRGBA, ok := src.(*image.RGBA); ok {
*dst = *srcRGBA
return
}
bounds := src.Bounds()
dy := bounds.Dy()
dx := bounds.Dx()
if len(dst.Pix) < 4*dx*dy {
dst.Pix = make([]uint8, 4*dx*dy)
}
dst.Stride = 4 * dx
dst.Rect = bounds
if srcYCbCr, ok := src.(*image.YCbCr); ok &&
srcYCbCr.SubsampleRatio == image.YCbCrSubsampleRatio444 {
i444ToRGBA(dst, srcYCbCr)
return
}
i := 0
for yi := 0; yi < dy; yi++ {
for xi := 0; xi < dx; xi++ {
r, g, b, a := src.At(xi, yi).RGBA()
dst.Pix[i+0] = uint8(r / 0x100)
dst.Pix[i+1] = uint8(g / 0x100)
dst.Pix[i+2] = uint8(b / 0x100)
dst.Pix[i+3] = uint8(a / 0x100)
i += 4
}
}
}
// ToRGBA converts r to a new reader that will output images in RGBA format
func ToRGBA(r Reader) Reader {
var dst image.RGBA
return ReaderFunc(func() (image.Image, func(), error) {
img, _, err := r.Read()
if err != nil {
return nil, func() {}, err
}
imageToRGBA(&dst, img)
return &dst, func() {}, nil
})
}
vendor/github.com/pion/mediadevices/pkg/io/video/convert_cgo.c
Generated Vendored
+175
View File
@@ -0,0 +1,175 @@
#include <stdint.h>
#include "_cgo_export.h"
void i444ToI420CGO(
unsigned char *cb_dst,
unsigned char *cr_dst,
unsigned char* cb,
unsigned char* cr,
const int stride, const int h)
{
int isrc0 = 0;
int isrc1 = stride;
int idst = 0;
for (int y = 0; y < h / 2; y++)
{
for (int x = 0; x < stride / 2; x++)
{
const uint8_t cb2 =
((uint16_t)cb[isrc0] + (uint16_t)cb[isrc1] +
(uint16_t)cb[isrc0 + 1] + (uint16_t)cb[isrc1 + 1]) /
4;
const uint8_t cr2 =
((uint16_t)cr[isrc0] + (uint16_t)cr[isrc1] +
(uint16_t)cr[isrc0 + 1] + (uint16_t)cr[isrc1 + 1]) /
4;
cb_dst[idst] = cb2;
cr_dst[idst] = cr2;
isrc0 += 2;
isrc1 += 2;
idst++;
}
isrc0 += stride;
isrc1 += stride;
}
}
void i422ToI420CGO(
unsigned char *cb_dst,
unsigned char *cr_dst,
unsigned char* cb,
unsigned char* cr,
const int stride, const int h)
{
int isrc = 0;
int idst = 0;
for (int y = 0; y < h / 2; y++)
{
for (int x = 0; x < stride; x++)
{
const uint8_t cb2 = ((uint16_t)cb[isrc] + (uint16_t)cb[isrc + stride]) / 2;
const uint8_t cr2 = ((uint16_t)cr[isrc] + (uint16_t)cr[isrc + stride]) / 2;
cb_dst[idst] = cb2;
cr_dst[idst] = cr2;
isrc++;
idst++;
}
isrc += stride;
}
}
void rgbToYCbCrCGO(
unsigned char* y,
unsigned char* cb,
unsigned char* cr,
const unsigned char r,
const unsigned char g,
const unsigned char b)
{
// ITU-R BT.601
const int y2 = 77 * r + 150 * g + 29 * b;
const int cb2 = -43 * r - 85 * g + 128 * b + 0x8000;
const int cr2 = 128 * r - 107 * g - 21 * b + 0x8000;
*y = y2 >> 8;
*cb = cb2 >> 8;
*cr = cr2 >> 8;
}
void repeatRGBToYCbCrCGO(
const int n,
unsigned char* y,
unsigned char* cb,
unsigned char* cr,
const unsigned char r,
const unsigned char g,
const unsigned char b)
{
int i;
for (i = 0; i < n; ++i)
{
rgbToYCbCrCGO(y, cb, cr, r, g, b);
}
}
void yCbCrToRGBCGO(
unsigned char* r,
unsigned char* g,
unsigned char* b,
const unsigned char y,
const unsigned char cb,
const unsigned char cr)
{
const int cb2 = cb - 0x80;
const int cr2 = cr - 0x80;
// ITU-R BT.601
int r2 = 256 * y + 359 * cr2;
int g2 = 256 * y - 88 * cb2 - 183 * cr2;
int b2 = 256 * y + 454 * cb2;
if (r2 < 0)
r2 = 0;
else if (r2 > 0xFFFF)
r2 = 0xFFFF;
if (g2 < 0)
g2 = 0;
else if (g2 > 0xFFFF)
g2 = 0xFFFF;
if (b2 < 0)
b2 = 0;
else if (b2 > 0xFFFF)
b2 = 0xFFFF;
*r = r2 >> 8;
*g = g2 >> 8;
*b = b2 >> 8;
}
void repeatYCbCrToRGBCGO(
const int n,
unsigned char* r,
unsigned char* g,
unsigned char* b,
const unsigned char y,
const unsigned char cb,
const unsigned char cr)
{
int i;
for (i = 0; i < n; ++i)
{
yCbCrToRGBCGO(r, g, b, y, cb, cr);
}
}
void i444ToRGBACGO(
unsigned char* rgb,
const unsigned char* y,
const unsigned char* cb,
const unsigned char* cr,
const int stride, const int h)
{
int i;
for (i = 0; i < stride * h; ++i)
{
yCbCrToRGBCGO(rgb, rgb + 1, rgb + 2, y[i], cb[i], cr[i]);
rgb[3] = 0xFF;
rgb += 4;
}
}
void rgbaToI444(
unsigned char* y,
unsigned char* cb,
unsigned char* cr,
const unsigned char* rgb,
const int stride, const int h)
{
int i;
for (i = 0; i < stride * h; ++i)
{
rgbToYCbCrCGO(&y[i], &cb[i], &cr[i], rgb[0], rgb[1], rgb[2]);
rgb += 4;
}
}
vendor/github.com/pion/mediadevices/pkg/io/video/convert_cgo.go
Generated Vendored
+105
View File
@@ -0,0 +1,105 @@
//go:build cgo
// +build cgo
package video
import (
"image"
)
// #include "convert_cgo.h"
// #cgo CFLAGS: -std=c11
import "C"
// CGO version of the functions will be selected at runtime.
// All functions switched at runtime must be declared also in convert_nocgo.go.
const hasCGOConvert = true
func i444ToI420(img image.YCbCr, dst []uint8) image.YCbCr {
h := img.Rect.Dy()
cLen := img.CStride * h / 4
// Divide preallocated memory to cbDst and crDst
// and truncate cap and len to cLen
cbDst, crDst := dst[:cLen:cLen], dst[cLen:]
crDst = crDst[:cLen:cLen]
C.i444ToI420CGO(
(*C.uchar)(&cbDst[0]), (*C.uchar)(&crDst[0]),
(*C.uchar)(&img.Cb[0]), (*C.uchar)(&img.Cr[0]),
C.int(img.CStride), C.int(h),
)
img.CStride = img.CStride / 2
img.Cb = cbDst
img.Cr = crDst
img.SubsampleRatio = image.YCbCrSubsampleRatio420
return img
}
func i422ToI420(img image.YCbCr, dst []uint8) image.YCbCr {
h := img.Rect.Dy()
cLen := img.CStride * (h / 2)
// Divide preallocated memory to cbDst and crDst
// and truncate cap and len to cLen
cbDst, crDst := dst[:cLen:cLen], dst[cLen:]
crDst = crDst[:cLen:cLen]
C.i422ToI420CGO(
(*C.uchar)(&cbDst[0]), (*C.uchar)(&crDst[0]),
(*C.uchar)(&img.Cb[0]), (*C.uchar)(&img.Cr[0]),
C.int(img.CStride), C.int(h),
)
img.Cb = cbDst
img.Cr = crDst
img.SubsampleRatio = image.YCbCrSubsampleRatio420
return img
}
func rgbToYCbCrCGO(y, cb, cr *uint8, r, g, b uint8) { // For testing
C.rgbToYCbCrCGO(
(*C.uchar)(y), (*C.uchar)(cb), (*C.uchar)(cr),
C.uchar(r), C.uchar(g), C.uchar(b),
)
}
func repeatRGBToYCbCrCGO(n int, y, cb, cr *uint8, r, g, b uint8) { // For testing
C.repeatRGBToYCbCrCGO(
C.int(n),
(*C.uchar)(y), (*C.uchar)(cb), (*C.uchar)(cr),
C.uchar(r), C.uchar(g), C.uchar(b),
)
}
func yCbCrToRGBCGO(r, g, b *uint8, y, cb, cr uint8) { // For testing
C.yCbCrToRGBCGO(
(*C.uchar)(r), (*C.uchar)(g), (*C.uchar)(b),
C.uchar(y), C.uchar(cb), C.uchar(cr),
)
}
func repeatYCbCrToRGBCGO(n int, r, g, b *uint8, y, cb, cr uint8) { // For testing
C.repeatYCbCrToRGBCGO(
C.int(n),
(*C.uchar)(r), (*C.uchar)(g), (*C.uchar)(b),
C.uchar(y), C.uchar(cb), C.uchar(cr),
)
}
func i444ToRGBA(dst *image.RGBA, src *image.YCbCr) {
C.i444ToRGBACGO(
(*C.uchar)(&dst.Pix[0]),
(*C.uchar)(&src.Y[0]),
(*C.uchar)(&src.Cb[0]),
(*C.uchar)(&src.Cr[0]),
C.int(src.Rect.Dx()),
C.int(src.Rect.Dy()),
)
}
func rgbaToI444(dst *image.YCbCr, src *image.RGBA) {
C.rgbaToI444(
(*C.uchar)(&dst.Y[0]),
(*C.uchar)(&dst.Cb[0]),
(*C.uchar)(&dst.Cr[0]),
(*C.uchar)(&src.Pix[0]),
C.int(src.Rect.Dx()),
C.int(src.Rect.Dy()),
)
}
vendor/github.com/pion/mediadevices/pkg/io/video/convert_cgo.h
Generated Vendored
+61
View File
@@ -0,0 +1,61 @@
void i444ToI420CGO(
unsigned char *cb_dst,
unsigned char *cr_dst,
unsigned char* cb,
unsigned char* cr,
const int stride, const int h);
void i422ToI420CGO(
unsigned char *cb_dst,
unsigned char *cr_dst,
unsigned char* cb,
unsigned char* cr,
const int stride, const int h);
void rgbToYCbCrCGO(
unsigned char* y,
unsigned char* cb,
unsigned char* cr,
const unsigned char r,
const unsigned char g,
const unsigned char b); // for testing
void repeatRGBToYCbCrCGO(
const int n,
unsigned char* y,
unsigned char* cb,
unsigned char* cr,
const unsigned char r,
const unsigned char g,
const unsigned char b); // for testing
void yCbCrToRGBCGO(
unsigned char* r,
unsigned char* g,
unsigned char* b,
const unsigned char y,
const unsigned char cb,
const unsigned char cr); // for testing
void repeatYCbCrToRGBCGO(
const int n,
unsigned char* r,
unsigned char* g,
unsigned char* b,
const unsigned char y,
const unsigned char cb,
const unsigned char cr); // for testing
void i444ToRGBACGO(
unsigned char* rgb,
const unsigned char* y,
const unsigned char* cb,
const unsigned char* cr,
const int stride, const int h);
void rgbaToI444(
unsigned char* y,
unsigned char* cb,
unsigned char* cr,
const unsigned char* rgb,
const int stride, const int h);
vendor/github.com/pion/mediadevices/pkg/io/video/convert_nocgo.go
Generated Vendored
+104
View File
@@ -0,0 +1,104 @@
//go:build !cgo
// +build !cgo
package video
import (
"image"
"image/color"
)
const hasCGOConvert = false
func i444ToI420(img image.YCbCr, dst []uint8) image.YCbCr {
h := img.Rect.Dy()
addrSrc0 := 0
addrSrc1 := img.CStride
cLen := img.CStride * (h / 4)
addrDst := 0
// Divide preallocated memory to cbDst and crDst
// and truncate cap and len to cLen
cbDst, crDst := dst[:cLen:cLen], dst[cLen:]
crDst = crDst[:cLen:cLen]
for i := 0; i < h/2; i++ {
for j := 0; j < img.CStride/2; j++ {
cb := uint16(img.Cb[addrSrc0]) + uint16(img.Cb[addrSrc1]) +
uint16(img.Cb[addrSrc0+1]) + uint16(img.Cb[addrSrc1+1])
cr := uint16(img.Cr[addrSrc0]) + uint16(img.Cr[addrSrc1]) +
uint16(img.Cr[addrSrc0+1]) + uint16(img.Cr[addrSrc1+1])
cbDst[addrDst] = uint8(cb / 4)
crDst[addrDst] = uint8(cr / 4)
addrSrc0 += 2
addrSrc1 += 2
addrDst++
}
addrSrc0 += img.CStride
addrSrc1 += img.CStride
}
img.CStride = img.CStride / 2
img.Cb = cbDst
img.Cr = crDst
img.SubsampleRatio = image.YCbCrSubsampleRatio420
return img
}
func i422ToI420(img image.YCbCr, dst []uint8) image.YCbCr {
h := img.Rect.Dy()
addrSrc := 0
cLen := img.CStride * (h / 2)
// Divide preallocated memory to cbDst and crDst
// and truncate cap and len to cLen
cbDst, crDst := dst[:cLen:cLen], dst[cLen:]
crDst = crDst[:cLen:cLen]
addrDst := 0
for i := 0; i < h/2; i++ {
for j := 0; j < img.CStride; j++ {
cb := uint16(img.Cb[addrSrc]) + uint16(img.Cb[addrSrc+img.CStride])
cr := uint16(img.Cr[addrSrc]) + uint16(img.Cr[addrSrc+img.CStride])
cbDst[addrDst] = uint8(cb / 2)
crDst[addrDst] = uint8(cr / 2)
addrSrc++
addrDst++
}
addrSrc += img.CStride
}
img.Cb = cbDst
img.Cr = crDst
img.SubsampleRatio = image.YCbCrSubsampleRatio420
return img
}
func i444ToRGBA(dst *image.RGBA, src *image.YCbCr) {
dx := src.Rect.Dx()
dy := src.Rect.Dy()
i := 0
j := 0
for yi := 0; yi < dy; yi++ {
for xi := 0; xi < dx; xi++ {
r, g, b := color.YCbCrToRGB(src.Y[j], src.Cb[j], src.Cr[j])
dst.Pix[i+0] = uint8(r)
dst.Pix[i+1] = uint8(g)
dst.Pix[i+2] = uint8(b)
dst.Pix[i+3] = 0xff
i += 4
}
}
}
func rgbaToI444(dst *image.YCbCr, src *image.RGBA) {
i := 0
addr := 0
dx := src.Rect.Dx()
dy := src.Rect.Dy()
for yi := 0; yi < dy; yi++ {
for xi := 0; xi < dx; xi++ {
dst.Y[i], dst.Cb[i], dst.Cr[i] = color.RGBToYCbCr(
src.Pix[addr+0], src.Pix[addr+1], src.Pix[addr+2],
)
addr += 4
i++
}
}
}
vendor/github.com/pion/mediadevices/pkg/io/video/detect.go
Generated Vendored
+60
View File
@@ -0,0 +1,60 @@
package video
import (
"image"
"math"
"time"
"github.com/pion/mediadevices/pkg/prop"
)
// DetectChanges will detect frame and video property changes. For video property detection,
// since it's time related, interval will be used to determine the sample rate.
func DetectChanges(interval time.Duration, fpsDiffTolerance float64, onChange func(prop.Media)) TransformFunc {
return func(r Reader) Reader {
var currentProp prop.Media
var lastTaken time.Time
var frames uint
return ReaderFunc(func() (image.Image, func(), error) {
var dirty bool
img, _, err := r.Read()
if err != nil {
return nil, func() {}, err
}
bounds := img.Bounds()
if currentProp.Width != bounds.Dx() {
currentProp.Width = bounds.Dx()
dirty = true
}
if currentProp.Height != bounds.Dy() {
currentProp.Height = bounds.Dy()
dirty = true
}
// TODO: maybe detect frame format? It probably doesn't make sense since some
// formats only are about memory layout, e.g. YUV2 vs NV12.
now := time.Now()
elapsed := now.Sub(lastTaken)
if elapsed >= interval {
fps := float32(float64(frames) / elapsed.Seconds())
frames = 0
lastTaken = now
if math.Abs(float64(currentProp.FrameRate-fps)) > fpsDiffTolerance {
currentProp.FrameRate = fps
dirty = true
}
}
if dirty {
onChange(currentProp)
}
frames++
return img, func() {}, nil
})
}
}
vendor/github.com/pion/mediadevices/pkg/io/video/framebuffer.go
Generated Vendored
+214
View File
@@ -0,0 +1,214 @@
package video
import (
"image"
)
// FrameBuffer is a buffer that can store any image format.
type FrameBuffer struct {
buffer []uint8
tmp image.Image
}
// NewFrameBuffer creates a new FrameBuffer instance and initialize internal buffer
// with initialSize
func NewFrameBuffer(initialSize int) *FrameBuffer {
return &FrameBuffer{
buffer: make([]uint8, initialSize),
}
}
func (buff *FrameBuffer) storeInOrder(srcs ...[]uint8) {
var neededSize int
for _, src := range srcs {
neededSize += len(src)
}
if len(buff.buffer) < neededSize {
if cap(buff.buffer) >= neededSize {
buff.buffer = buff.buffer[:neededSize]
} else {
buff.buffer = make([]uint8, neededSize)
}
}
var currentLen int
for _, src := range srcs {
copy(buff.buffer[currentLen:], src)
currentLen += len(src)
}
}
// Load loads the current owned image
func (buff *FrameBuffer) Load() image.Image {
return buff.tmp
}
// StoreCopy makes a copy of src and store its copy. StoreCopy will reuse as much memory as it can
// from the previous copies. For example, if StoreCopy is given an image that has the same resolution
// and format from the previous call, StoreCopy will not allocate extra memory and only copy the content
// from src to the previous buffer.
func (buff *FrameBuffer) StoreCopy(src image.Image) {
switch src := src.(type) {
case *image.Alpha:
clone, ok := buff.tmp.(*image.Alpha)
if ok {
*clone = *src
} else {
copied := *src
clone = &copied
}
buff.storeInOrder(src.Pix)
clone.Pix = buff.buffer[:len(src.Pix)]
buff.tmp = clone
case *image.Alpha16:
clone, ok := buff.tmp.(*image.Alpha16)
if ok {
*clone = *src
} else {
copied := *src
clone = &copied
}
buff.storeInOrder(src.Pix)
clone.Pix = buff.buffer[:len(src.Pix)]
buff.tmp = clone
case *image.CMYK:
clone, ok := buff.tmp.(*image.CMYK)
if ok {
*clone = *src
} else {
copied := *src
clone = &copied
}
buff.storeInOrder(src.Pix)
clone.Pix = buff.buffer[:len(src.Pix)]
buff.tmp = clone
case *image.Gray:
clone, ok := buff.tmp.(*image.Gray)
if ok {
*clone = *src
} else {
copied := *src
clone = &copied
}
buff.storeInOrder(src.Pix)
clone.Pix = buff.buffer[:len(src.Pix)]
buff.tmp = clone
case *image.Gray16:
clone, ok := buff.tmp.(*image.Gray16)
if ok {
*clone = *src
} else {
copied := *src
clone = &copied
}
buff.storeInOrder(src.Pix)
clone.Pix = buff.buffer[:len(src.Pix)]
buff.tmp = clone
case *image.NRGBA:
clone, ok := buff.tmp.(*image.NRGBA)
if ok {
*clone = *src
} else {
copied := *src
clone = &copied
}
buff.storeInOrder(src.Pix)
clone.Pix = buff.buffer[:len(src.Pix)]
buff.tmp = clone
case *image.NRGBA64:
clone, ok := buff.tmp.(*image.NRGBA64)
if ok {
*clone = *src
} else {
copied := *src
clone = &copied
}
buff.storeInOrder(src.Pix)
clone.Pix = buff.buffer[:len(src.Pix)]
buff.tmp = clone
case *image.RGBA:
clone, ok := buff.tmp.(*image.RGBA)
if ok {
*clone = *src
} else {
copied := *src
clone = &copied
}
buff.storeInOrder(src.Pix)
clone.Pix = buff.buffer[:len(src.Pix)]
buff.tmp = clone
case *image.RGBA64:
clone, ok := buff.tmp.(*image.RGBA64)
if ok {
*clone = *src
} else {
copied := *src
clone = &copied
}
buff.storeInOrder(src.Pix)
clone.Pix = buff.buffer[:len(src.Pix)]
buff.tmp = clone
case *image.NYCbCrA:
clone, ok := buff.tmp.(*image.NYCbCrA)
if ok {
*clone = *src
} else {
copied := *src
clone = &copied
}
var currentLen int
buff.storeInOrder(src.Y, src.Cb, src.Cr, src.A)
clone.Y = buff.buffer[currentLen : currentLen+len(src.Y) : currentLen+len(src.Y)]
currentLen += len(src.Y)
clone.Cb = buff.buffer[currentLen : currentLen+len(src.Cb) : currentLen+len(src.Cb)]
currentLen += len(src.Cb)
clone.Cr = buff.buffer[currentLen : currentLen+len(src.Cr) : currentLen+len(src.Cr)]
currentLen += len(src.Cr)
clone.A = buff.buffer[currentLen : currentLen+len(src.A) : currentLen+len(src.A)]
buff.tmp = clone
case *image.YCbCr:
clone, ok := buff.tmp.(*image.YCbCr)
if ok {
*clone = *src
} else {
copied := *src
clone = &copied
}
var currentLen int
buff.storeInOrder(src.Y, src.Cb, src.Cr)
clone.Y = buff.buffer[currentLen : currentLen+len(src.Y) : currentLen+len(src.Y)]
currentLen += len(src.Y)
clone.Cb = buff.buffer[currentLen : currentLen+len(src.Cb) : currentLen+len(src.Cb)]
currentLen += len(src.Cb)
clone.Cr = buff.buffer[currentLen : currentLen+len(src.Cr) : currentLen+len(src.Cr)]
buff.tmp = clone
default:
var converted image.RGBA
imageToRGBA(&converted, src)
buff.StoreCopy(&converted)
}
}
vendor/github.com/pion/mediadevices/pkg/io/video/scale.go
Generated Vendored
+194
View File
@@ -0,0 +1,194 @@
package video
import (
"errors"
"image"
"golang.org/x/image/draw"
)
var (
scalerTestAlgos = map[string]Scaler{
"NearestNeighbor": ScalerNearestNeighbor,
}
scalerBenchAlgos = map[string]Scaler{
"NearestNeighbor": ScalerNearestNeighbor,
"ApproxBiLinear": ScalerApproxBiLinear,
"BiLinear": ScalerBiLinear,
}
)
// Scaler represents scaling algorithm
type Scaler draw.Scaler
// List of scaling algorithms
var (
ScalerNearestNeighbor = Scaler(draw.NearestNeighbor)
ScalerApproxBiLinear = Scaler(draw.ApproxBiLinear)
ScalerBiLinear = Scaler(draw.BiLinear)
ScalerCatmullRom = Scaler(draw.CatmullRom)
)
var errUnsupportedImageType = errors.New("scaling: unsupported image type")
// Scale returns video scaling transform.
// Setting scaler=nil to use default scaler. (ScalerNearestNeighbor)
// Negative width or height value will keep the aspect ratio of incoming image.
//
// Note: computation cost to scale YCbCr format is 10 times higher than RGB
// due to the implementation in x/image/draw package.
func Scale(width, height int, scaler Scaler) TransformFunc {
return func(r Reader) Reader {
scalerCached := ScalerNearestNeighbor
if scaler != nil {
scalerCached = scaler
}
// cacheScaler makes cached version of Scaler if available.
cacheScaler := func(dRect, sRect image.Rectangle) {}
if kernel, ok := scaler.(interface {
NewScaler(int, int, int, int) draw.Scaler
}); ok {
cacheScaler = func(dRect, sRect image.Rectangle) {
scalerCached = kernel.NewScaler(dRect.Dx(), dRect.Dy(), sRect.Dx(), sRect.Dy())
}
}
var rect image.Rectangle
var imgScaled image.Image
if width > 0 && height > 0 {
rect = image.Rect(0, 0, width, height)
} else if width <= 0 && height <= 0 {
panic("Both width and height are negative!")
}
// updateRect updates output image size
updateRect := func(r image.Rectangle) {}
if height <= 0 {
updateRect = func(r image.Rectangle) {
h := r.Dy() * width / r.Dx()
rect = image.Rect(0, 0, width, h)
}
} else if width <= 0 {
updateRect = func(r image.Rectangle) {
w := r.Dx() * height / r.Dy()
rect = image.Rect(0, 0, w, height)
}
}
// fixedRect returns Rectangle of chroma plane
fixedRect := func(rect image.Rectangle, sr image.YCbCrSubsampleRatio) image.Rectangle {
switch sr {
case image.YCbCrSubsampleRatio444:
case image.YCbCrSubsampleRatio422:
rect.Max.X /= 2
case image.YCbCrSubsampleRatio420:
rect.Max.X /= 2
rect.Max.Y /= 2
}
return rect
}
src := &rgbLikeYCbCr{y: &image.Gray{}, cb: &image.Gray{}, cr: &image.Gray{}}
dst := &rgbLikeYCbCr{y: &image.Gray{}, cb: &image.Gray{}, cr: &image.Gray{}}
// ycbcrRealloc reallocs image.YCbCr if needed
ycbcrRealloc := func(i1 *image.YCbCr) {
if imgScaled == nil || imgScaled.ColorModel() != i1.ColorModel() {
updateRect(i1.Rect)
cacheScaler(rect, i1.Rect)
imgScaled = image.NewYCbCr(rect, i1.SubsampleRatio)
}
imgDst := imgScaled.(*image.YCbCr)
if imgDst.Rect.Dx() != i1.Rect.Dx() || imgDst.Rect.Dy() != i1.Rect.Dy() {
updateRect(i1.Rect)
cacheScaler(rect, i1.Rect)
}
yDx := rect.Dx()
yDy := rect.Dy()
cRect := fixedRect(rect, i1.SubsampleRatio)
cDx := cRect.Dx()
cDy := cRect.Dy()
yLen := yDx * yDy
cLen := cDx * cDy
if len(imgDst.Y) < yLen {
if cap(imgDst.Y) < yLen {
imgDst.Y = make([]uint8, yLen)
}
imgDst.Y = imgDst.Y[:yLen]
}
if len(imgDst.Cr) < cLen {
if cap(imgDst.Cr) < cLen {
imgDst.Cr = make([]uint8, cLen)
}
imgDst.Cr = imgDst.Cr[:cLen]
}
if len(imgDst.Cb) < cLen {
if cap(imgDst.Cb) < cLen {
imgDst.Cb = make([]uint8, cLen)
}
imgDst.Cb = imgDst.Cb[:cLen]
}
*dst.y = image.Gray{Pix: imgDst.Y, Stride: imgDst.YStride, Rect: rect}
*dst.cb = image.Gray{Pix: imgDst.Cb, Stride: imgDst.CStride, Rect: cRect}
*dst.cr = image.Gray{Pix: imgDst.Cr, Stride: imgDst.CStride, Rect: cRect}
}
// ycbcrRealloc reallocs image.RGBA if needed
rgbaRealloc := func(i1 *image.RGBA) {
if imgScaled == nil || imgScaled.ColorModel() != i1.ColorModel() {
updateRect(i1.Rect)
cacheScaler(rect, i1.Rect)
imgScaled = image.NewRGBA(rect)
}
imgDst := imgScaled.(*image.RGBA)
if imgDst.Rect.Dx() != i1.Rect.Dx() || imgDst.Rect.Dy() != i1.Rect.Dy() {
updateRect(i1.Rect)
cacheScaler(rect, i1.Rect)
}
l := 4 * rect.Dx() * rect.Dy()
if len(imgDst.Pix) < l {
if cap(imgDst.Pix) < l {
imgDst.Pix = make([]uint8, l)
}
imgDst.Pix = imgDst.Pix[:l]
}
}
return ReaderFunc(func() (image.Image, func(), error) {
img, _, err := r.Read()
if err != nil {
return nil, func() {}, err
}
switch v := img.(type) {
case *image.RGBA:
rgbaRealloc(v)
dst := imgScaled.(*image.RGBA)
scalerCached.Scale(dst, rect, v, v.Rect, draw.Src, nil)
cloned := *dst // clone metadata
return &cloned, func() {}, nil
case *image.YCbCr:
ycbcrRealloc(v)
// Scale each plane
*src.y = image.Gray{Pix: v.Y, Stride: v.YStride, Rect: v.Rect}
*src.cb = image.Gray{
Pix: v.Cb, Stride: v.CStride, Rect: fixedRect(v.Rect, v.SubsampleRatio),
}
*src.cr = image.Gray{
Pix: v.Cr, Stride: v.CStride, Rect: fixedRect(v.Rect, v.SubsampleRatio),
}
scalerCached.Scale(dst, dst.Bounds(), src, src.Bounds(), draw.Src, nil)
cloned := *(imgScaled.(*image.YCbCr)) // clone metadata
return &cloned, func() {}, nil
default:
return nil, func() {}, errUnsupportedImageType
}
})
}
}
vendor/github.com/pion/mediadevices/pkg/io/video/scaler_cgo.c
Generated Vendored
+61
View File
@@ -0,0 +1,61 @@
#include <stdint.h>
#include <string.h>
#include "_cgo_export.h"
void fastNearestNeighbor(
uint8_t* dst, const uint8_t* src,
const int ch,
const int dw, const int dh, const int dstride,
const int sw, const int sh, const int sstride)
{
for (int dy = 0; dy < dh; dy++)
{
const int sy = dy * sh / dh;
const uint8_t* src2 = &src[sy * sstride];
for (int dx = 0; dx < dw; dx++)
{
const int sx = ch * (dx * sw / dw);
for (int c = 0; c < ch; c++)
*(dst++) = src2[sx + c];
}
}
}
void fastBoxSampling(
uint8_t* dst, const uint8_t* src,
const int ch,
const int dw, const int dh, const int dstride,
const int sw, const int sh, const int sstride,
uint32_t* tmp)
{
memset(tmp, 0, dstride * dh * ch * sizeof(tmp[0]));
for (int sy = 0; sy < sh; sy++)
{
const uint8_t* src2 = &src[sy * sstride];
int tx = 0;
const int ty = sy * dh / sh;
uint32_t* tmp2 = &tmp[ty * dstride];
for (int sx = 0; sx < sw * ch; sx += ch)
{
for (int c = 0; c < ch; c++)
{
tmp2[tx + c] += 0x10000 | src2[sx + c];
}
if (tx * sw < sx * dw)
{
tx += ch;
}
}
}
for (int i = 0; i < dw * dh * ch; i++)
{
const uint32_t tmp2 = tmp[i];
const uint16_t sum = tmp2 & 0xFFFF;
const uint16_t num = tmp2 >> 16;
if (num > 0)
dst[i] = sum / num;
}
}
vendor/github.com/pion/mediadevices/pkg/io/video/scaler_cgo.go
Generated Vendored
+181
View File
@@ -0,0 +1,181 @@
// +build cgo
package video
import (
"golang.org/x/image/draw"
"image"
"sync"
)
// #include <stdint.h>
// void fastNearestNeighbor(
// uint8_t* dst, const uint8_t* src,
// const int ch,
// const int dw, const int dh, const int dstride,
// const int sw, const int sh, const int sstride);
// void fastBoxSampling(
// uint8_t* dst, const uint8_t* src,
// const int ch,
// const int dw, const int dh, const int dstride,
// const int sw, const int sh, const int sstride,
// uint32_t* tmp);
import "C"
// List of scaling algorithms
var (
// ScalerFastNearestNeighbor is a CGO version of NearestNeighbor scaler.
// This is roughly 4-times faster than draw.NearestNeighbor.
ScalerFastNearestNeighbor = Scaler(&FastNearestNeighbor{})
// ScalerFastBoxSampling is a CGO implementation of BoxSampling scaler.
// This is heavyer than NearestNeighbor but keeps detail on down scaling.
ScalerFastBoxSampling = Scaler(&FastBoxSampling{})
)
var poolBoxSampleBuffer = sync.Pool{
New: func() interface{} {
return &([]uint32{})
},
}
var (
scalerTestAlgosCGO = map[string]Scaler{
"FastNearestNeighbor": ScalerFastNearestNeighbor,
}
scalerBenchAlgosCGO = map[string]Scaler{
"FastNearestNeighbor": ScalerFastNearestNeighbor,
"FastBoxSampling": ScalerFastBoxSampling,
}
)
func init() {
// Append test conditions
for k, v := range scalerTestAlgosCGO {
scalerTestAlgos[k] = v
}
for k, v := range scalerBenchAlgosCGO {
scalerBenchAlgos[k] = v
}
}
// FastNearestNeighbor is a CGO version of NearestNeighbor scaler.
type FastNearestNeighbor struct {
}
// Scale implements the draw.Scaler interface.
func (f *FastNearestNeighbor) Scale(dst draw.Image, dr image.Rectangle, src image.Image, sr image.Rectangle, op draw.Op, opts *draw.Options) {
switch s := src.(type) {
case (*image.RGBA):
d := dst.(*image.RGBA)
l := d.Stride * d.Rect.Dy()
if len(d.Pix) < l {
if cap(d.Pix) < l {
d.Pix = make([]uint8, l)
}
d.Pix = d.Pix[:l]
}
C.fastNearestNeighbor(
(*C.uchar)(&d.Pix[dr.Min.X+d.Stride*dr.Min.Y]),
(*C.uchar)(&s.Pix[sr.Min.X+s.Stride*sr.Min.Y]),
4,
C.int(dr.Dx()), C.int(dr.Dy()), C.int(d.Stride),
C.int(sr.Dx()), C.int(sr.Dy()), C.int(s.Stride),
)
case (*image.Gray):
d := dst.(*image.Gray)
l := d.Stride * d.Rect.Dy()
if len(d.Pix) < l {
if cap(d.Pix) < l {
d.Pix = make([]uint8, l)
}
d.Pix = d.Pix[:l]
}
C.fastNearestNeighbor(
(*C.uchar)(&d.Pix[dr.Min.X+d.Stride*dr.Min.Y]),
(*C.uchar)(&s.Pix[sr.Min.X+s.Stride*sr.Min.Y]),
1,
C.int(dr.Dx()), C.int(dr.Dy()), C.int(d.Stride),
C.int(sr.Dx()), C.int(sr.Dy()), C.int(s.Stride),
)
case (*rgbLikeYCbCr):
d := dst.(*rgbLikeYCbCr)
f.Scale(d.y, dr, s.y, sr, op, opts)
dr2 := image.Rect(0, 0, d.cb.Stride, len(d.cb.Pix)/d.cb.Stride)
sr2 := image.Rect(0, 0, s.cb.Stride, len(s.cb.Pix)/s.cb.Stride)
f.Scale(d.cb, dr2, s.cb, sr2, op, opts)
f.Scale(d.cr, dr2, s.cr, sr2, op, opts)
default:
panic("unimplemented")
}
}
// FastBoxSampling is a CGO version of Box sampling scaler.
type FastBoxSampling struct {
}
// Scale implements the draw.Scaler interface.
func (f *FastBoxSampling) Scale(dst draw.Image, dr image.Rectangle, src image.Image, sr image.Rectangle, op draw.Op, opts *draw.Options) {
if sr.Dx() < dr.Dx() && sr.Dy() < dr.Dy() {
// Box upsampling is equivalent of NearestNeighbor
(&FastNearestNeighbor{}).Scale(dst, dr, src, sr, op, opts)
return
}
tmp := poolBoxSampleBuffer.Get().(*[]uint32)
defer poolBoxSampleBuffer.Put(tmp)
switch s := src.(type) {
case (*image.RGBA):
d := dst.(*image.RGBA)
l := d.Stride * d.Rect.Dy()
if len(d.Pix) < l {
if cap(d.Pix) < l {
d.Pix = make([]uint8, l)
}
d.Pix = d.Pix[:l]
}
if len(*tmp) < l {
*tmp = make([]uint32, l)
}
C.fastBoxSampling(
(*C.uchar)(&d.Pix[dr.Min.X+d.Stride*dr.Min.Y]),
(*C.uchar)(&s.Pix[sr.Min.X+s.Stride*sr.Min.Y]),
4,
C.int(dr.Dx()), C.int(dr.Dy()), C.int(d.Stride),
C.int(sr.Dx()), C.int(sr.Dy()), C.int(s.Stride),
(*C.uint32_t)(&(*tmp)[0]),
)
case (*image.Gray):
d := dst.(*image.Gray)
l := d.Stride * d.Rect.Dy()
if len(d.Pix) < l {
if cap(d.Pix) < l {
d.Pix = make([]uint8, l)
}
d.Pix = d.Pix[:l]
}
if len(*tmp) < l {
*tmp = make([]uint32, l)
}
C.fastBoxSampling(
(*C.uchar)(&d.Pix[dr.Min.X+d.Stride*dr.Min.Y]),
(*C.uchar)(&s.Pix[sr.Min.X+s.Stride*sr.Min.Y]),
1,
C.int(dr.Dx()), C.int(dr.Dy()), C.int(d.Stride),
C.int(sr.Dx()), C.int(sr.Dy()), C.int(s.Stride),
(*C.uint32_t)(&(*tmp)[0]),
)
case (*rgbLikeYCbCr):
d := dst.(*rgbLikeYCbCr)
f.Scale(d.y, dr, s.y, sr, op, opts)
dr2 := image.Rect(0, 0, d.cb.Stride, len(d.cb.Pix)/d.cb.Stride)
sr2 := image.Rect(0, 0, s.cb.Stride, len(s.cb.Pix)/s.cb.Stride)
f.Scale(d.cb, dr2, s.cb, sr2, op, opts)
f.Scale(d.cr, dr2, s.cr, sr2, op, opts)
default:
panic("unimplemented")
}
}
vendor/github.com/pion/mediadevices/pkg/io/video/scaler_nocgo.go
Generated Vendored
+22
View File
@@ -0,0 +1,22 @@
//go:build !cgo
// +build !cgo
package video
import (
"golang.org/x/image/draw"
"image"
)
// ScalerFastBoxSampling mock scaler for nocgo implementation to pass tests for CGO_ENABLED=0
var (
ScalerFastBoxSampling = Scaler(&FastBoxSampling{})
)
// FastBoxSampling mock implementation for nocgo implementation
// TODO implement nocgo FastBoxSampling scaling algorithm
type FastBoxSampling struct {
}
func (f *FastBoxSampling) Scale(_ draw.Image, _ image.Rectangle, _ image.Image, _ image.Rectangle, _ draw.Op, _ *draw.Options) {
}
vendor/github.com/pion/mediadevices/pkg/io/video/scaleycrcb.go
Generated Vendored
+47
View File
@@ -0,0 +1,47 @@
package video
import (
"image"
"image/color"
)
type rgbLikeYCbCr struct {
y *image.Gray
cb *image.Gray
cr *image.Gray
}
func (p *rgbLikeYCbCr) ColorModel() color.Model {
return color.RGBAModel
}
func (p *rgbLikeYCbCr) Bounds() image.Rectangle {
return p.y.Rect
}
func (p *rgbLikeYCbCr) At(x, y int) color.Color {
var yy, cb, cr uint8
yy = p.y.GrayAt(x, y).Y
if (image.Point{x, y}.In(p.cb.Rect)) {
cb = p.cb.GrayAt(x, y).Y
cr = p.cr.GrayAt(x, y).Y
}
return color.RGBA{yy, cb, cr, 255}
}
func (p *rgbLikeYCbCr) Set(x, y int, c color.Color) {
switch v := c.(type) {
case color.RGBA:
p.y.SetGray(x, y, color.Gray{v.R})
if (image.Point{x, y}.In(p.cb.Rect)) {
p.cb.SetGray(x, y, color.Gray{v.G})
p.cr.SetGray(x, y, color.Gray{v.B})
}
case *color.RGBA64:
p.y.SetGray(x, y, color.Gray{uint8(v.R / 0x100)})
if (image.Point{x, y}.In(p.cb.Rect)) {
p.cb.SetGray(x, y, color.Gray{uint8(v.G / 0x100)})
p.cr.SetGray(x, y, color.Gray{uint8(v.B / 0x100)})
}
}
}
vendor/github.com/pion/mediadevices/pkg/io/video/throttle.go
Generated Vendored
+28
View File
@@ -0,0 +1,28 @@
package video
import (
"image"
"time"
)
// Throttle returns video throttling transform.
// This transform drops some of the incoming frames to achieve given framerate in fps.
func Throttle(rate float32) TransformFunc {
return func(r Reader) Reader {
ticker := time.NewTicker(time.Duration(int64(float64(time.Second) / float64(rate))))
return ReaderFunc(func() (image.Image, func(), error) {
for {
img, _, err := r.Read()
if err != nil {
ticker.Stop()
return nil, func() {}, err
}
select {
case <-ticker.C:
return img, func() {}, nil
default:
}
}
})
}
}
vendor/github.com/pion/mediadevices/pkg/io/video/video.go
Generated Vendored
+43
View File
@@ -0,0 +1,43 @@
package video
import (
"image"
)
type Reader interface {
// Read reads data from the source. The caller is responsible to release the memory that's associated
// with data by calling the given release function. When err is not nil, the caller MUST NOT call release
// as data is going to be nil (no memory was given). Otherwise, the caller SHOULD call release after
// using the data. The caller is NOT REQUIRED to call release, as this is only a part of memory management
// optimization. If release is not called, the source is forced to allocate a new memory, which also means
// there will be new allocations during streaming, and old unused memory will become garbage. As a consequence,
// these garbage will put a lot of pressure to the garbage collector and makes it to run more often and finish
// slower as the heap memory usage increases and more garbage to collect.
Read() (img image.Image, release func(), err error)
}
type ReaderFunc func() (img image.Image, release func(), err error)
func (rf ReaderFunc) Read() (img image.Image, release func(), err error) {
img, release, err = rf()
return
}
// TransformFunc produces a new Reader that will produces a transformed video
type TransformFunc func(r Reader) Reader
// Merge merges transforms and produces a new TransformFunc that will execute
// transforms in order
func Merge(transforms ...TransformFunc) TransformFunc {
return func(r Reader) Reader {
for _, transform := range transforms {
if transform == nil {
continue
}
r = transform(r)
}
return r
}
}
vendor/github.com/pion/mediadevices/pkg/prop/bool.go
Generated Vendored
+37
View File
@@ -0,0 +1,37 @@
package prop
import "fmt"
// BoolConstraint is an interface to represent bool value constraint.
type BoolConstraint interface {
Compare(bool) (float64, bool)
Value() bool
}
// BoolExact specifies exact bool value.
type BoolExact bool
// Compare implements BoolConstraint.
func (b BoolExact) Compare(o bool) (float64, bool) {
if bool(b) == o {
return 0.0, true
}
return 1.0, false
}
// Value implements BoolConstraint.
func (b BoolExact) Value() bool { return bool(b) }
// String implements Stringify
func (b BoolExact) String() string {
return fmt.Sprintf("%t (exact)", b)
}
// Bool specifies ideal bool value.
type Bool BoolExact
// Compare implements BoolConstraint.
func (b Bool) Compare(o bool) (float64, bool) {
dist, _ := BoolExact(b).Compare(o)
return dist, true
}
vendor/github.com/pion/mediadevices/pkg/prop/duration.go
Generated Vendored
+125
View File
@@ -0,0 +1,125 @@
package prop
import (
"fmt"
"math"
"strings"
"time"
)
// DurationConstraint is an interface to represent time.Duration constraint.
type DurationConstraint interface {
Compare(time.Duration) (float64, bool)
Value() (time.Duration, bool)
}
// Duration specifies ideal duration value.
// Any value may be selected, but closest value takes priority.
type Duration time.Duration
// Compare implements DurationConstraint.
func (d Duration) Compare(a time.Duration) (float64, bool) {
return math.Abs(float64(a-time.Duration(d))) / math.Max(math.Abs(float64(a)), math.Abs(float64(d))), true
}
// Value implements DurationConstraint.
func (d Duration) Value() (time.Duration, bool) { return time.Duration(d), true }
// String implements Stringify
func (d Duration) String() string {
return fmt.Sprintf("%v (ideal)", time.Duration(d))
}
// DurationExact specifies exact duration value.
type DurationExact time.Duration
// Compare implements DurationConstraint.
func (d DurationExact) Compare(a time.Duration) (float64, bool) {
if time.Duration(d) == a {
return 0.0, true
}
return 1.0, false
}
// Value implements DurationConstraint.
func (d DurationExact) Value() (time.Duration, bool) { return time.Duration(d), true }
// String implements Stringify
func (d DurationExact) String() string {
return fmt.Sprintf("%v (exact)", time.Duration(d))
}
// DurationOneOf specifies list of expected duration values.
type DurationOneOf []time.Duration
// Compare implements DurationConstraint.
func (d DurationOneOf) Compare(a time.Duration) (float64, bool) {
for _, ii := range d {
if ii == a {
return 0.0, true
}
}
return 1.0, false
}
// Value implements DurationConstraint.
func (DurationOneOf) Value() (time.Duration, bool) { return 0, false }
// String implements Stringify
func (d DurationOneOf) String() string {
var opts []string
for _, v := range d {
opts = append(opts, fmt.Sprint(v))
}
return fmt.Sprintf("%s (one of values)", strings.Join(opts, ","))
}
// DurationRanged specifies range of expected duration value.
// If Ideal is non-zero, closest value to Ideal takes priority.
type DurationRanged struct {
Min time.Duration
Max time.Duration
Ideal time.Duration
}
// Compare implements DurationConstraint.
func (d DurationRanged) Compare(a time.Duration) (float64, bool) {
if d.Min != 0 && d.Min > a {
// Out of range
return 1.0, false
}
if d.Max != 0 && d.Max < a {
// Out of range
return 1.0, false
}
if d.Ideal == 0 {
// If the value is in the range and Ideal is not specified,
// any value is evenly acceptable.
return 0.0, true
}
switch {
case a == d.Ideal:
return 0.0, true
case a < d.Ideal:
if d.Min == 0 {
// If Min is not specified, smaller values than Ideal are even.
return 0.0, true
}
return float64(d.Ideal-a) / float64(d.Ideal-d.Min), true
default:
if d.Max == 0 {
// If Max is not specified, larger values than Ideal are even.
return 0.0, true
}
return float64(a-d.Ideal) / float64(d.Max-d.Ideal), true
}
}
// Value implements DurationConstraint.
func (DurationRanged) Value() (time.Duration, bool) { return 0, false }
// String implements Stringify
func (d DurationRanged) String() string {
return fmt.Sprintf("%s - %s (range), %s (ideal)", d.Min, d.Max, d.Ideal)
}
vendor/github.com/pion/mediadevices/pkg/prop/float.go
Generated Vendored
+124
View File
@@ -0,0 +1,124 @@
package prop
import (
"fmt"
"math"
"strings"
)
// FloatConstraint is an interface to represent float value constraint.
type FloatConstraint interface {
Compare(float32) (float64, bool)
Value() (float32, bool)
}
// Float specifies ideal float value.
// Any value may be selected, but closest value takes priority.
type Float float32
// Compare implements FloatConstraint.
func (f Float) Compare(a float32) (float64, bool) {
return math.Abs(float64(a-float32(f))) / math.Max(math.Abs(float64(a)), math.Abs(float64(f))), true
}
// Value implements FloatConstraint.
func (f Float) Value() (float32, bool) { return float32(f), true }
// String implements Stringify
func (f Float) String() string {
return fmt.Sprintf("%.2f (ideal)", f)
}
// FloatExact specifies exact float value.
type FloatExact float32
// Compare implements FloatConstraint.
func (f FloatExact) Compare(a float32) (float64, bool) {
if float32(f) == a {
return 0.0, true
}
return 1.0, false
}
// Value implements FloatConstraint.
func (f FloatExact) Value() (float32, bool) { return float32(f), true }
// String implements Stringify
func (f FloatExact) String() string {
return fmt.Sprintf("%.2f (exact)", f)
}
// FloatOneOf specifies list of expected float values.
type FloatOneOf []float32
// Compare implements FloatConstraint.
func (f FloatOneOf) Compare(a float32) (float64, bool) {
for _, ff := range f {
if ff == a {
return 0.0, true
}
}
return 1.0, false
}
// Value implements FloatConstraint.
func (FloatOneOf) Value() (float32, bool) { return 0, false }
// String implements Stringify
func (f FloatOneOf) String() string {
var opts []string
for _, v := range f {
opts = append(opts, fmt.Sprintf("%.2f", v))
}
return fmt.Sprintf("%s (one of values)", strings.Join(opts, ","))
}
// FloatRanged specifies range of expected float value.
// If Ideal is non-zero, closest value to Ideal takes priority.
type FloatRanged struct {
Min float32
Max float32
Ideal float32
}
// Compare implements FloatConstraint.
func (f FloatRanged) Compare(a float32) (float64, bool) {
if f.Min != 0 && f.Min > a {
// Out of range
return 1.0, false
}
if f.Max != 0 && f.Max < a {
// Out of range
return 1.0, false
}
if f.Ideal == 0 {
// If the value is in the range and Ideal is not specified,
// any value is evenly acceptable.
return 0.0, true
}
switch {
case a == f.Ideal:
return 0.0, true
case a < f.Ideal:
if f.Min == 0 {
// If Min is not specified, smaller values than Ideal are even.
return 0.0, true
}
return float64(f.Ideal-a) / float64(f.Ideal-f.Min), true
default:
if f.Max == 0 {
// If Max is not specified, larger values than Ideal are even.
return 0.0, true
}
return float64(a-f.Ideal) / float64(f.Max-f.Ideal), true
}
}
// Value implements FloatConstraint.
func (FloatRanged) Value() (float32, bool) { return 0, false }
// String implements Stringify
func (f FloatRanged) String() string {
return fmt.Sprintf("%.2f - %.2f (range), %.2f (ideal)", f.Min, f.Max, f.Ideal)
}
vendor/github.com/pion/mediadevices/pkg/prop/format.go
Generated Vendored
+78
View File
@@ -0,0 +1,78 @@
package prop
import (
"fmt"
"github.com/pion/mediadevices/pkg/frame"
"strings"
)
// FrameFormatConstraint is an interface to represent frame format constraint.
type FrameFormatConstraint interface {
Compare(frame.Format) (float64, bool)
Value() (frame.Format, bool)
}
// FrameFormat specifies expected frame format.
// Any value may be selected, but matched value takes priority.
type FrameFormat frame.Format
// Compare implements FrameFormatConstraint.
func (f FrameFormat) Compare(a frame.Format) (float64, bool) {
if frame.Format(f) == a {
return 0.0, true
}
return 1.0, true
}
// Value implements FrameFormatConstraint.
func (f FrameFormat) Value() (frame.Format, bool) { return frame.Format(f), true }
// String implements Stringify
func (f FrameFormat) String() string {
return fmt.Sprintf("%s (ideal)", frame.Format(f))
}
// FrameFormatExact specifies exact frame format.
type FrameFormatExact frame.Format
// Compare implements FrameFormatConstraint.
func (f FrameFormatExact) Compare(a frame.Format) (float64, bool) {
if frame.Format(f) == a {
return 0.0, true
}
return 1.0, false
}
// Value implements FrameFormatConstraint.
func (f FrameFormatExact) Value() (frame.Format, bool) { return frame.Format(f), true }
// String implements Stringify
func (f FrameFormatExact) String() string {
return fmt.Sprintf("%s (exact)", frame.Format(f))
}
// FrameFormatOneOf specifies list of expected frame format.
type FrameFormatOneOf []frame.Format
// Compare implements FrameFormatConstraint.
func (f FrameFormatOneOf) Compare(a frame.Format) (float64, bool) {
for _, ff := range f {
if ff == a {
return 0.0, true
}
}
return 1.0, false
}
// Value implements FrameFormatConstraint.
func (FrameFormatOneOf) Value() (frame.Format, bool) { return "", false }
// String implements Stringify
func (f FrameFormatOneOf) String() string {
var opts []string
for _, v := range f {
opts = append(opts, fmt.Sprint(v))
}
return fmt.Sprintf("%s (one of values)", strings.Join(opts, ","))
}
vendor/github.com/pion/mediadevices/pkg/prop/int.go
Generated Vendored
+124
View File
@@ -0,0 +1,124 @@
package prop
import (
"fmt"
"math"
"strings"
)
// IntConstraint is an interface to represent integer value constraint.
type IntConstraint interface {
Compare(int) (float64, bool)
Value() (int, bool)
}
// Int specifies ideal int value.
// Any value may be selected, but closest value takes priority.
type Int int
// Compare implements IntConstraint.
func (i Int) Compare(a int) (float64, bool) {
return math.Abs(float64(a-int(i))) / math.Max(math.Abs(float64(a)), math.Abs(float64(i))), true
}
// Value implements IntConstraint.
func (i Int) Value() (int, bool) { return int(i), true }
// String implements Stringify
func (i Int) String() string {
return fmt.Sprintf("%d (ideal)", i)
}
// IntExact specifies exact int value.
type IntExact int
// Compare implements IntConstraint.
func (i IntExact) Compare(a int) (float64, bool) {
if int(i) == a {
return 0.0, true
}
return 1.0, false
}
// String implements Stringify
func (i IntExact) String() string {
return fmt.Sprintf("%d (exact)", i)
}
// Value implements IntConstraint.
func (i IntExact) Value() (int, bool) { return int(i), true }
// IntOneOf specifies list of expected float values.
type IntOneOf []int
// Compare implements IntConstraint.
func (i IntOneOf) Compare(a int) (float64, bool) {
for _, ii := range i {
if ii == a {
return 0.0, true
}
}
return 1.0, false
}
// Value implements IntConstraint.
func (IntOneOf) Value() (int, bool) { return 0, false }
// String implements Stringify
func (i IntOneOf) String() string {
var opts []string
for _, v := range i {
opts = append(opts, fmt.Sprint(v))
}
return fmt.Sprintf("%s (one of values)", strings.Join(opts, ","))
}
// IntRanged specifies range of expected int value.
// If Ideal is non-zero, closest value to Ideal takes priority.
type IntRanged struct {
Min int
Max int
Ideal int
}
// Compare implements IntConstraint.
func (i IntRanged) Compare(a int) (float64, bool) {
if i.Min != 0 && i.Min > a {
// Out of range
return 1.0, false
}
if i.Max != 0 && i.Max < a {
// Out of range
return 1.0, false
}
if i.Ideal == 0 {
// If the value is in the range and Ideal is not specified,
// any value is evenly acceptable.
return 0.0, true
}
switch {
case a == i.Ideal:
return 0.0, true
case a < i.Ideal:
if i.Min == 0 {
// If Min is not specified, smaller values than Ideal are even.
return 0.0, true
}
return float64(i.Ideal-a) / float64(i.Ideal-i.Min), true
default:
if i.Max == 0 {
// If Max is not specified, larger values than Ideal are even.
return 0.0, true
}
return float64(a-i.Ideal) / float64(i.Max-i.Ideal), true
}
}
// Value implements IntConstraint.
func (IntRanged) Value() (int, bool) { return 0, false }
// String implements Stringify
func (i IntRanged) String() string {
return fmt.Sprintf("%d - %d (range), %d (ideal)", i.Min, i.Max, i.Ideal)
}
vendor/github.com/pion/mediadevices/pkg/prop/prop.go
Generated Vendored
+265
View File
@@ -0,0 +1,265 @@
package prop
import (
"fmt"
"reflect"
"strings"
"time"
"github.com/pion/mediadevices/pkg/frame"
)
// MediaConstraints represents set of media property constraints.
// Each field constrains property by min/ideal/max range, exact match, or oneof match.
type MediaConstraints struct {
DeviceID StringConstraint
VideoConstraints
AudioConstraints
}
func (m *MediaConstraints) String() string {
return prettifyStruct(m)
}
// Media stores single set of media propaties.
type Media struct {
DeviceID string
Video
Audio
}
func (m *Media) String() string {
return prettifyStruct(m)
}
func prettifyStruct(i interface{}) string {
var rows []string
var addRows func(int, reflect.Value)
addRows = func(level int, obj reflect.Value) {
typeOf := obj.Type()
for i := 0; i < obj.NumField(); i++ {
field := typeOf.Field(i)
value := obj.Field(i)
padding := strings.Repeat(" ", level)
switch value.Kind() {
case reflect.Struct:
rows = append(rows, fmt.Sprintf("%s%v:", padding, field.Name))
addRows(level+1, value)
case reflect.Chan, reflect.Func, reflect.Interface, reflect.Map, reflect.Ptr, reflect.Slice:
if value.IsNil() {
rows = append(rows, fmt.Sprintf("%s%v: any", padding, field.Name))
} else {
rows = append(rows, fmt.Sprintf("%s%v: %v", padding, field.Name, value))
}
default:
rows = append(rows, fmt.Sprintf("%s%v: %v", padding, field.Name, value))
}
}
}
addRows(0, reflect.ValueOf(i).Elem())
return strings.Join(rows, "\n")
}
// setterFn is a callback function to set value from fieldB to fieldA
type setterFn func(fieldA, fieldB reflect.Value)
// merge merges all the field values from o to p, except zero values. It's guaranteed that setterFn will be called
// when fieldA and fieldB are not struct.
func (p *Media) merge(o interface{}, set setterFn) {
rp := reflect.ValueOf(p).Elem()
ro := reflect.ValueOf(o)
// merge b fields to a recursively
var merge func(a, b reflect.Value)
merge = func(a, b reflect.Value) {
numFields := a.NumField()
for i := 0; i < numFields; i++ {
fieldA := a.Field(i)
fieldB := b.Field(i)
// if b is a struct, a is also a struct. Then,
// we recursively merge them
if fieldB.Kind() == reflect.Struct {
merge(fieldA, fieldB)
continue
}
// TODO: Replace this with fieldB.IsZero() when we move to go1.13
// If non-boolean or non-discrete values are zeroes we skip them
if fieldB.Interface() == reflect.Zero(fieldB.Type()).Interface() &&
fieldB.Kind() != reflect.Bool {
continue
}
set(fieldA, fieldB)
}
}
merge(rp, ro)
}
func (p *Media) Merge(o Media) {
p.merge(o, func(fieldA, fieldB reflect.Value) {
fieldA.Set(fieldB)
})
}
func (p *Media) MergeConstraints(o MediaConstraints) {
p.merge(o, func(fieldA, fieldB reflect.Value) {
switch c := fieldB.Interface().(type) {
case IntConstraint:
if v, ok := c.Value(); ok {
fieldA.Set(reflect.ValueOf(v))
}
case FloatConstraint:
if v, ok := c.Value(); ok {
fieldA.Set(reflect.ValueOf(v))
}
case DurationConstraint:
if v, ok := c.Value(); ok {
fieldA.Set(reflect.ValueOf(v))
}
case FrameFormatConstraint:
if v, ok := c.Value(); ok {
fieldA.Set(reflect.ValueOf(v))
}
case StringConstraint:
if v, ok := c.Value(); ok {
fieldA.Set(reflect.ValueOf(v))
}
case BoolConstraint:
fieldA.Set(reflect.ValueOf(c.Value()))
default:
panic("unsupported property type")
}
})
}
// FitnessDistance calculates fitness of media property and media constraints.
// If no media satisfies given constraints, second return value will be false.
func (p *MediaConstraints) FitnessDistance(o Media) (float64, bool) {
cmps := comparisons{}
cmps.add(p.DeviceID, o.DeviceID)
cmps.add(p.Width, o.Width)
cmps.add(p.Height, o.Height)
cmps.add(p.FrameFormat, o.FrameFormat)
// skip framerate if not available in media properties
if o.FrameRate > 0.0 {
cmps.add(p.FrameRate, o.FrameRate)
}
cmps.add(p.SampleRate, o.SampleRate)
cmps.add(p.Latency, o.Latency)
cmps.add(p.ChannelCount, o.ChannelCount)
cmps.add(p.IsBigEndian, o.IsBigEndian)
cmps.add(p.IsFloat, o.IsFloat)
cmps.add(p.IsInterleaved, o.IsInterleaved)
return cmps.fitnessDistance()
}
type comparisons []struct {
desired, actual interface{}
}
func (c *comparisons) add(desired, actual interface{}) {
if desired != nil {
*c = append(*c,
struct{ desired, actual interface{} }{
desired, actual,
},
)
}
}
// fitnessDistance is an implementation for https://w3c.github.io/mediacapture-main/#dfn-fitness-distance
func (c *comparisons) fitnessDistance() (float64, bool) {
var dist float64
for _, field := range *c {
var d float64
var ok bool
switch c := field.desired.(type) {
case IntConstraint:
if actual, typeOK := field.actual.(int); typeOK {
d, ok = c.Compare(actual)
} else {
panic("wrong type of actual value")
}
case FloatConstraint:
if actual, typeOK := field.actual.(float32); typeOK {
d, ok = c.Compare(actual)
} else {
panic("wrong type of actual value")
}
case DurationConstraint:
if actual, typeOK := field.actual.(time.Duration); typeOK {
d, ok = c.Compare(actual)
} else {
panic("wrong type of actual value")
}
case FrameFormatConstraint:
if actual, typeOK := field.actual.(frame.Format); typeOK {
d, ok = c.Compare(actual)
} else {
panic("wrong type of actual value")
}
case StringConstraint:
if actual, typeOK := field.actual.(string); typeOK {
d, ok = c.Compare(actual)
} else {
panic("wrong type of actual value")
}
case BoolConstraint:
if actual, typeOK := field.actual.(bool); typeOK {
d, ok = c.Compare(actual)
} else {
panic("wrong type of actual value")
}
default:
panic("unsupported constraint type")
}
dist += d
if !ok {
return 0, false
}
}
return dist, true
}
// VideoConstraints represents a video's constraints
type VideoConstraints struct {
Width, Height IntConstraint
FrameRate FloatConstraint
FrameFormat FrameFormatConstraint
DiscardFramesOlderThan time.Duration
}
// Video represents a video's constraints
type Video struct {
Width, Height int
FrameRate float32
FrameFormat frame.Format
DiscardFramesOlderThan time.Duration
}
// AudioConstraints represents an audio's constraints
type AudioConstraints struct {
ChannelCount IntConstraint
Latency DurationConstraint
SampleRate IntConstraint
SampleSize IntConstraint
IsBigEndian BoolConstraint
IsFloat BoolConstraint
IsInterleaved BoolConstraint
}
// Audio represents an audio's constraints
type Audio struct {
ChannelCount int
Latency time.Duration
SampleRate int
SampleSize int
IsBigEndian bool
IsFloat bool
IsInterleaved bool
}
vendor/github.com/pion/mediadevices/pkg/prop/string.go
Generated Vendored
+72
View File
@@ -0,0 +1,72 @@
package prop
import (
"fmt"
"strings"
)
// StringConstraint is an interface to represent string constraint.
type StringConstraint interface {
Compare(string) (float64, bool)
Value() (string, bool)
}
// String specifies expected string.
// Any value may be selected, but matched value takes priority.
type String string
// Compare implements StringConstraint.
func (f String) Compare(a string) (float64, bool) {
if string(f) == a {
return 0.0, true
}
return 1.0, true
}
// Value implements StringConstraint.
func (f String) Value() (string, bool) { return string(f), true }
// String implements Stringify
func (f String) String() string {
return fmt.Sprintf("%s (ideal)", string(f))
}
// StringExact specifies exact string.
type StringExact string
// Compare implements StringConstraint.
func (f StringExact) Compare(a string) (float64, bool) {
if string(f) == a {
return 0.0, true
}
return 1.0, false
}
// Value implements StringConstraint.
func (f StringExact) Value() (string, bool) { return string(f), true }
// String implements Stringify
func (f StringExact) String() string {
return fmt.Sprintf("%s (exact)", string(f))
}
// StringOneOf specifies list of expected string.
type StringOneOf []string
// Compare implements StringConstraint.
func (f StringOneOf) Compare(a string) (float64, bool) {
for _, ff := range f {
if ff == a {
return 0.0, true
}
}
return 1.0, false
}
// Value implements StringConstraint.
func (StringOneOf) Value() (string, bool) { return "", false }
// String implements Stringify
func (f StringOneOf) String() string {
return fmt.Sprintf("%s (one of values)", strings.Join([]string(f), ","))
}
vendor/github.com/pion/mediadevices/pkg/wave/buffer.go
Generated Vendored
+149
View File
@@ -0,0 +1,149 @@
package wave
import "fmt"
var (
errUnsupportedFormat = fmt.Errorf("Unsupported format")
)
// Buffer is a buffer that can store any audio format.
type Buffer struct {
// TODO: Probably standardize the audio formats so that we don't need to have the following different types
// and duplicated codes for each type
bufferFloat32Interleaved []float32
bufferFloat32NonInterleaved [][]float32
bufferInt16Interleaved []int16
bufferInt16NonInterleaved [][]int16
tmp Audio
}
// NewBuffer creates a new Buffer instance
func NewBuffer() *Buffer {
return &Buffer{}
}
// Load loads the current owned Audio
func (buff *Buffer) Load() Audio {
return buff.tmp
}
// StoreCopy makes a copy of src and store its copy. StoreCopy will reuse as much memory as it can
// from the previous copies. For example, if StoreCopy is given an audio that has the format from the previous call,
// StoreCopy will not allocate extra memory and only copy the content from src to the previous buffer.
func (buff *Buffer) StoreCopy(src Audio) {
switch src := src.(type) {
case *Float32Interleaved:
clone, ok := buff.tmp.(*Float32Interleaved)
if ok {
*clone = *src
} else {
copied := *src
clone = &copied
}
neededSize := len(src.Data)
if len(buff.bufferFloat32Interleaved) < neededSize {
if cap(buff.bufferFloat32Interleaved) >= neededSize {
buff.bufferFloat32Interleaved = buff.bufferFloat32Interleaved[:neededSize]
} else {
buff.bufferFloat32Interleaved = make([]float32, neededSize)
}
}
copy(buff.bufferFloat32Interleaved, src.Data)
clone.Data = buff.bufferFloat32Interleaved
buff.tmp = clone
case *Float32NonInterleaved:
clone, ok := buff.tmp.(*Float32NonInterleaved)
if ok {
*clone = *src
} else {
copied := *src
clone = &copied
}
neededSize := len(src.Data)
if len(buff.bufferFloat32NonInterleaved) < neededSize {
if cap(buff.bufferFloat32NonInterleaved) >= neededSize {
buff.bufferFloat32NonInterleaved = buff.bufferFloat32NonInterleaved[:neededSize]
} else {
buff.bufferFloat32NonInterleaved = make([][]float32, neededSize)
}
}
for i := range src.Data {
neededSize := len(src.Data[i])
if len(buff.bufferFloat32NonInterleaved[i]) < neededSize {
if cap(buff.bufferFloat32NonInterleaved[i]) >= neededSize {
buff.bufferFloat32NonInterleaved[i] = buff.bufferFloat32NonInterleaved[i][:neededSize]
} else {
buff.bufferFloat32NonInterleaved[i] = make([]float32, neededSize)
}
}
copy(buff.bufferFloat32NonInterleaved[i], src.Data[i])
}
clone.Data = buff.bufferFloat32NonInterleaved
buff.tmp = clone
case *Int16Interleaved:
clone, ok := buff.tmp.(*Int16Interleaved)
if ok {
*clone = *src
} else {
copied := *src
clone = &copied
}
neededSize := len(src.Data)
if len(buff.bufferInt16Interleaved) < neededSize {
if cap(buff.bufferInt16Interleaved) >= neededSize {
buff.bufferInt16Interleaved = buff.bufferInt16Interleaved[:neededSize]
} else {
buff.bufferInt16Interleaved = make([]int16, neededSize)
}
}
copy(buff.bufferInt16Interleaved, src.Data)
clone.Data = buff.bufferInt16Interleaved
buff.tmp = clone
case *Int16NonInterleaved:
clone, ok := buff.tmp.(*Int16NonInterleaved)
if ok {
*clone = *src
} else {
copied := *src
clone = &copied
}
neededSize := len(src.Data)
if len(buff.bufferInt16NonInterleaved) < neededSize {
if cap(buff.bufferInt16NonInterleaved) >= neededSize {
buff.bufferInt16NonInterleaved = buff.bufferInt16NonInterleaved[:neededSize]
} else {
buff.bufferInt16NonInterleaved = make([][]int16, neededSize)
}
}
for i := range src.Data {
neededSize := len(src.Data[i])
if len(buff.bufferInt16NonInterleaved[i]) < neededSize {
if cap(buff.bufferInt16NonInterleaved[i]) >= neededSize {
buff.bufferInt16NonInterleaved[i] = buff.bufferInt16NonInterleaved[i][:neededSize]
} else {
buff.bufferInt16NonInterleaved[i] = make([]int16, neededSize)
}
}
copy(buff.bufferInt16NonInterleaved[i], src.Data[i])
}
clone.Data = buff.bufferInt16NonInterleaved
buff.tmp = clone
default:
// TODO: Should have a routine to convert any format to one of the supported formats above
panic(errUnsupportedFormat)
}
}
vendor/github.com/pion/mediadevices/pkg/wave/decoder.go
Generated Vendored
+305
View File
@@ -0,0 +1,305 @@
package wave
import (
"encoding/binary"
"fmt"
"math"
"reflect"
"unsafe"
)
// Format represents how audio is formatted in memory
type Format fmt.Stringer
type RawFormat struct {
SampleSize int
IsFloat bool
Interleaved bool
}
func (f *RawFormat) String() string {
sampleSizeInBits := f.SampleSize * 8
dataTypeStr := "Int"
if f.IsFloat {
dataTypeStr = "Float"
}
interleavedStr := "NonInterleaved"
if f.Interleaved {
interleavedStr = "Interleaved"
}
return fmt.Sprintf("%s%d%s", dataTypeStr, sampleSizeInBits, interleavedStr)
}
var hostEndian binary.ByteOrder
var registeredDecoders = map[string]Decoder{}
func init() {
switch v := *(*uint16)(unsafe.Pointer(&([]byte{0x12, 0x34}[0]))); v {
case 0x1234:
hostEndian = binary.BigEndian
case 0x3412:
hostEndian = binary.LittleEndian
default:
panic(fmt.Sprintf("failed to determine host endianness: %x", v))
}
decoderBuilders := []DecoderBuilderFunc{
newInt16InterleavedDecoder,
newInt16NonInterleavedDecoder,
newFloat32InterleavedDecoder,
newFloat32NonInterleavedDecoder,
}
for _, decoderBuilder := range decoderBuilders {
err := RegisterDecoder(decoderBuilder)
if err != nil {
panic(err)
}
}
}
// Decoder decodes raw chunk to Audio
type Decoder interface {
// Decode decodes raw chunk in endian byte order
Decode(endian binary.ByteOrder, chunk []byte, channels int) (Audio, error)
}
// DecoderFunc is a proxy type for Decoder
type DecoderFunc func(endian binary.ByteOrder, chunk []byte, channels int) (Audio, error)
func (f DecoderFunc) Decode(endian binary.ByteOrder, chunk []byte, channels int) (Audio, error) {
return f(endian, chunk, channels)
}
// DecoderBuilder builds raw audio decoder
type DecoderBuilder interface {
// NewDecoder creates a new decoder for specified format
NewDecoder() (Decoder, Format)
}
// DecoderBuilderFunc is a proxy type for DecoderBuilder
type DecoderBuilderFunc func() (Decoder, Format)
func (builderFunc DecoderBuilderFunc) NewDecoder() (Decoder, Format) {
return builderFunc()
}
func RegisterDecoder(builder DecoderBuilder) error {
decoder, format := builder.NewDecoder()
formatStr := format.String()
if _, ok := registeredDecoders[formatStr]; ok {
return fmt.Errorf("%v has already been registered", format)
}
registeredDecoders[formatStr] = decoder
return nil
}
// NewDecoder creates a decoder to decode raw audio data in the given format
func NewDecoder(format Format) (Decoder, error) {
decoder, ok := registeredDecoders[format.String()]
if !ok {
return nil, fmt.Errorf("%s format is not supported", format)
}
return decoder, nil
}
func calculateChunkInfo(chunk []byte, channels int, sampleSize int) (ChunkInfo, error) {
if channels <= 0 {
return ChunkInfo{}, fmt.Errorf("channels has to be greater than 0")
}
if sampleSize <= 0 {
return ChunkInfo{}, fmt.Errorf("sample size has to be greater than 0")
}
sampleLen := channels * sampleSize
if len(chunk)%sampleLen != 0 {
expectedLen := len(chunk) + (sampleLen - len(chunk)%sampleLen)
return ChunkInfo{}, fmt.Errorf("expected chunk to have a length of %d, but got %d", expectedLen, len(chunk))
}
return ChunkInfo{
Channels: channels,
Len: len(chunk) / (channels * sampleSize),
}, nil
}
func newInt16InterleavedDecoder() (Decoder, Format) {
format := &RawFormat{
SampleSize: 2,
IsFloat: false,
Interleaved: true,
}
decoder := DecoderFunc(func(endian binary.ByteOrder, chunk []byte, channels int) (Audio, error) {
sampleSize := format.SampleSize
chunkInfo, err := calculateChunkInfo(chunk, channels, sampleSize)
if err != nil {
return nil, err
}
container := NewInt16Interleaved(chunkInfo)
if endian == hostEndian {
data := container.Data
dst := *(*[]byte)(unsafe.Pointer(&data))
hdr := (*reflect.SliceHeader)(unsafe.Pointer(&dst))
n := len(chunk)
hdr.Len, hdr.Cap = n, n
copy(dst, chunk)
return container, nil
}
sampleLen := sampleSize * channels
var i int
for offset := 0; offset+sampleLen <= len(chunk); offset += sampleLen {
for ch := 0; ch < channels; ch++ {
flatOffset := offset + ch*sampleSize
sample := endian.Uint16(chunk[flatOffset : flatOffset+sampleSize])
container.SetInt16(i, ch, Int16Sample(sample))
}
i++
}
return container, nil
})
return decoder, format
}
func newInt16NonInterleavedDecoder() (Decoder, Format) {
format := &RawFormat{
SampleSize: 2,
IsFloat: false,
Interleaved: false,
}
decoder := DecoderFunc(func(endian binary.ByteOrder, chunk []byte, channels int) (Audio, error) {
sampleSize := format.SampleSize
chunkInfo, err := calculateChunkInfo(chunk, channels, sampleSize)
if err != nil {
return nil, err
}
container := NewInt16NonInterleaved(chunkInfo)
chunkLen := len(chunk) / channels
if endian == hostEndian {
for ch := 0; ch < channels; ch++ {
data := container.Data[ch]
dst := *(*[]byte)(unsafe.Pointer(&data))
hdr := (*reflect.SliceHeader)(unsafe.Pointer(&dst))
hdr.Len, hdr.Cap = chunkLen, chunkLen
offset := ch * chunkLen
copy(dst, chunk[offset:offset+chunkLen])
}
return container, nil
}
for ch := 0; ch < channels; ch++ {
offset := ch * chunkLen
for i := 0; i < chunkInfo.Len; i++ {
flatOffset := offset + i*sampleSize
sample := endian.Uint16(chunk[flatOffset : flatOffset+sampleSize])
container.SetInt16(i, ch, Int16Sample(sample))
}
}
return container, nil
})
return decoder, format
}
func newFloat32InterleavedDecoder() (Decoder, Format) {
format := &RawFormat{
SampleSize: 4,
IsFloat: true,
Interleaved: true,
}
decoder := DecoderFunc(func(endian binary.ByteOrder, chunk []byte, channels int) (Audio, error) {
sampleSize := format.SampleSize
chunkInfo, err := calculateChunkInfo(chunk, channels, sampleSize)
if err != nil {
return nil, err
}
container := NewFloat32Interleaved(chunkInfo)
if endian == hostEndian {
data := container.Data
dst := *(*[]byte)(unsafe.Pointer(&data))
hdr := (*reflect.SliceHeader)(unsafe.Pointer(&dst))
n := len(chunk)
hdr.Len, hdr.Cap = n, n
copy(dst, chunk)
return container, nil
}
sampleLen := sampleSize * channels
var i int
for offset := 0; offset+sampleLen <= len(chunk); offset += sampleLen {
for ch := 0; ch < channels; ch++ {
flatOffset := offset + ch*sampleSize
sample := endian.Uint32(chunk[flatOffset : flatOffset+sampleSize])
sampleF := math.Float32frombits(sample)
container.SetFloat32(i, ch, Float32Sample(sampleF))
}
i++
}
return container, nil
})
return decoder, format
}
func newFloat32NonInterleavedDecoder() (Decoder, Format) {
format := &RawFormat{
SampleSize: 4,
IsFloat: true,
Interleaved: false,
}
decoder := DecoderFunc(func(endian binary.ByteOrder, chunk []byte, channels int) (Audio, error) {
sampleSize := format.SampleSize
chunkInfo, err := calculateChunkInfo(chunk, channels, sampleSize)
if err != nil {
return nil, err
}
container := NewFloat32NonInterleaved(chunkInfo)
chunkLen := len(chunk) / channels
if endian == hostEndian {
for ch := 0; ch < channels; ch++ {
data := container.Data[ch]
dst := *(*[]byte)(unsafe.Pointer(&data))
hdr := (*reflect.SliceHeader)(unsafe.Pointer(&dst))
hdr.Len, hdr.Cap = chunkLen, chunkLen
offset := ch * chunkLen
copy(dst, chunk[offset:offset+chunkLen])
}
return container, nil
}
for ch := 0; ch < channels; ch++ {
offset := ch * chunkLen
for i := 0; i < chunkInfo.Len; i++ {
flatOffset := offset + i*sampleSize
sample := endian.Uint32(chunk[flatOffset : flatOffset+sampleSize])
sampleF := math.Float32frombits(sample)
container.SetFloat32(i, ch, Float32Sample(sampleF))
}
}
return container, nil
})
return decoder, format
}
vendor/github.com/pion/mediadevices/pkg/wave/float32.go
Generated Vendored
+100
View File
@@ -0,0 +1,100 @@
package wave
// Float32Sample is a 32-bits float audio sample.
type Float32Sample float32
func (s Float32Sample) Int() int64 {
return int64(s * 0x100000000)
}
// Float32Interleaved multi-channel interlaced Audio.
type Float32Interleaved struct {
Data []float32
Size ChunkInfo
}
// ChunkInfo returns audio chunk size.
func (a *Float32Interleaved) ChunkInfo() ChunkInfo {
return a.Size
}
func (a *Float32Interleaved) SampleFormat() SampleFormat {
return Float32SampleFormat
}
func (a *Float32Interleaved) At(i, ch int) Sample {
return Float32Sample(a.Data[i*a.Size.Channels+ch])
}
func (a *Float32Interleaved) Set(i, ch int, s Sample) {
a.Data[i*a.Size.Channels+ch] = float32(Float32SampleFormat.Convert(s).(Float32Sample))
}
func (a *Float32Interleaved) SetFloat32(i, ch int, s Float32Sample) {
a.Data[i*a.Size.Channels+ch] = float32(s)
}
// SubAudio returns part of the original audio sharing the buffer.
func (a *Float32Interleaved) SubAudio(offsetSamples, nSamples int) *Float32Interleaved {
ret := *a
offset := offsetSamples * a.Size.Channels
n := nSamples * a.Size.Channels
ret.Data = ret.Data[offset : offset+n]
ret.Size.Len = nSamples
return &ret
}
func NewFloat32Interleaved(size ChunkInfo) *Float32Interleaved {
return &Float32Interleaved{
Data: make([]float32, size.Channels*size.Len),
Size: size,
}
}
// Float32NonInterleaved multi-channel interlaced Audio.
type Float32NonInterleaved struct {
Data [][]float32
Size ChunkInfo
}
// ChunkInfo returns audio chunk size.
func (a *Float32NonInterleaved) ChunkInfo() ChunkInfo {
return a.Size
}
func (a *Float32NonInterleaved) SampleFormat() SampleFormat {
return Float32SampleFormat
}
func (a *Float32NonInterleaved) At(i, ch int) Sample {
return Float32Sample(a.Data[ch][i])
}
func (a *Float32NonInterleaved) Set(i, ch int, s Sample) {
a.Data[ch][i] = float32(Float32SampleFormat.Convert(s).(Float32Sample))
}
func (a *Float32NonInterleaved) SetFloat32(i, ch int, s Float32Sample) {
a.Data[ch][i] = float32(s)
}
// SubAudio returns part of the original audio sharing the buffer.
func (a *Float32NonInterleaved) SubAudio(offsetSamples, nSamples int) *Float32NonInterleaved {
ret := *a
for i := range a.Data {
ret.Data[i] = ret.Data[i][offsetSamples : offsetSamples+nSamples]
}
ret.Size.Len = nSamples
return &ret
}
func NewFloat32NonInterleaved(size ChunkInfo) *Float32NonInterleaved {
d := make([][]float32, size.Channels)
for i := 0; i < size.Channels; i++ {
d[i] = make([]float32, size.Len)
}
return &Float32NonInterleaved{
Data: d,
Size: size,
}
}
vendor/github.com/pion/mediadevices/pkg/wave/int16.go
Generated Vendored
+100
View File
@@ -0,0 +1,100 @@
package wave
// Int16Sample is a 16-bits signed integer audio sample.
type Int16Sample int16
func (s Int16Sample) Int() int64 {
return int64(s) << 16
}
// Int16Interleaved multi-channel interlaced Audio.
type Int16Interleaved struct {
Data []int16
Size ChunkInfo
}
// ChunkInfo returns audio chunk size.
func (a *Int16Interleaved) ChunkInfo() ChunkInfo {
return a.Size
}
func (a *Int16Interleaved) SampleFormat() SampleFormat {
return Int16SampleFormat
}
func (a *Int16Interleaved) At(i, ch int) Sample {
return Int16Sample(a.Data[i*a.Size.Channels+ch])
}
func (a *Int16Interleaved) Set(i, ch int, s Sample) {
a.Data[i*a.Size.Channels+ch] = int16(Int16SampleFormat.Convert(s).(Int16Sample))
}
func (a *Int16Interleaved) SetInt16(i, ch int, s Int16Sample) {
a.Data[i*a.Size.Channels+ch] = int16(s)
}
// SubAudio returns part of the original audio sharing the buffer.
func (a *Int16Interleaved) SubAudio(offsetSamples, nSamples int) *Int16Interleaved {
ret := *a
offset := offsetSamples * a.Size.Channels
n := nSamples * a.Size.Channels
ret.Data = ret.Data[offset : offset+n]
ret.Size.Len = nSamples
return &ret
}
func NewInt16Interleaved(size ChunkInfo) *Int16Interleaved {
return &Int16Interleaved{
Data: make([]int16, size.Channels*size.Len),
Size: size,
}
}
// Int16NonInterleaved multi-channel interlaced Audio.
type Int16NonInterleaved struct {
Data [][]int16
Size ChunkInfo
}
// ChunkInfo returns audio chunk size.
func (a *Int16NonInterleaved) ChunkInfo() ChunkInfo {
return a.Size
}
func (a *Int16NonInterleaved) SampleFormat() SampleFormat {
return Int16SampleFormat
}
func (a *Int16NonInterleaved) At(i, ch int) Sample {
return Int16Sample(a.Data[ch][i])
}
func (a *Int16NonInterleaved) Set(i, ch int, s Sample) {
a.Data[ch][i] = int16(Int16SampleFormat.Convert(s).(Int16Sample))
}
func (a *Int16NonInterleaved) SetInt16(i, ch int, s Int16Sample) {
a.Data[ch][i] = int16(s)
}
// SubAudio returns part of the original audio sharing the buffer.
func (a *Int16NonInterleaved) SubAudio(offsetSamples, nSamples int) *Int16NonInterleaved {
ret := *a
for i := range a.Data {
ret.Data[i] = ret.Data[i][offsetSamples : offsetSamples+nSamples]
}
ret.Size.Len = nSamples
return &ret
}
func NewInt16NonInterleaved(size ChunkInfo) *Int16NonInterleaved {
d := make([][]int16, size.Channels)
for i := 0; i < size.Channels; i++ {
d[i] = make([]int16, size.Len)
}
return &Int16NonInterleaved{
Data: d,
Size: size,
}
}
vendor/github.com/pion/mediadevices/pkg/wave/int64.go
Generated Vendored
+8
View File
@@ -0,0 +1,8 @@
package wave
// Int64Sample is a 64-bits signed integer audio sample.
type Int64Sample int64
func (s Int64Sample) Int() int64 {
return int64(s)
}
vendor/github.com/pion/mediadevices/pkg/wave/mixer/mixer.go
Generated Vendored
+42
View File
@@ -0,0 +1,42 @@
package mixer
import (
"errors"
"github.com/pion/mediadevices/pkg/wave"
)
// ChannelMixer mixes audio into specifix channels.
type ChannelMixer interface {
Mix(dst wave.Audio, src wave.Audio) error
}
// MonoMixer mixes channels into monaural audio.
type MonoMixer struct {
}
func (m *MonoMixer) Mix(dst wave.Audio, src wave.Audio) error {
if dst.ChunkInfo().Len != src.ChunkInfo().Len {
return errors.New("buffer size mismatch")
}
dstSetter, ok := dst.(wave.EditableAudio)
if !ok {
return errors.New("destination buffer is not settable")
}
n := src.ChunkInfo().Len
channels := src.ChunkInfo().Channels
dstChannels := dst.ChunkInfo().Channels
for i := 0; i < n; i++ {
var mean int64
for ch := 0; ch < channels; ch++ {
mean += src.At(i, ch).Int()
}
mean /= int64(channels)
for ch := 0; ch < dstChannels; ch++ {
dstSetter.Set(i, ch, wave.Int64Sample(mean))
}
}
return nil
}

Some files were not shown because too many files have changed in this diff Show More