Skip to content

Commit

Permalink
Fix codec matching with different rate or channels
Browse files Browse the repository at this point in the history 
Consider clock rate and channels when matching codecs. This allows to
support codecs with the same MIME type but different sample rate or
channel count, like PCMU, PCMA, LPCM and multiopus.
  • Loading branch information
@aler9
aler9 committed Jan 25, 2025
1 parent 47bde05 commit 54a37a9
Show file tree
Hide file tree
Showing 7 changed files with 218 additions and 45 deletions.
122 changes: 94 additions & 28 deletions internal/fmtp/fmtp.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,74 @@ func parseParameters(line string) map[string]string {
return parameters
}

// ClockRateEqual checks whether two clock rates are equal.
func ClockRateEqual(mimeType string, valA, valB uint32) bool {
// Clock rate and channel checks have been introduced quite recently.
// Existing implementations often use VP8, H264 or Opus without setting clock rate or channels.
// Keep compatibility with these situations.
// It would be better to remove this exception in a future major release.
switch {
case strings.EqualFold(mimeType, "video/vp8"):
if valA == 0 {
valA = 90000
}
if valB == 0 {
valB = 90000
}

case strings.EqualFold(mimeType, "audio/opus"):
if valA == 0 {
valA = 48000
}
if valB == 0 {
valB = 48000
}
}

return valA == valB
}

// ChannelsEqual checks whether two channels are equal.
func ChannelsEqual(mimeType string, valA, valB uint16) bool {
// Clock rate and channel checks have been introduced quite recently.
// Existing implementations often use VP8, H264 or Opus without setting clock rate or channels.
// Keep compatibility with these situations.
// It would be better to remove this exception in a future major release.
if strings.EqualFold(mimeType, "audio/opus") {
if valA == 0 {
valA = 2
}
if valB == 0 {
valB = 2
}
}

if valA == 0 {
valA = 1
}
if valB == 0 {
valB = 1
}

return valA == valB
}

func paramsEqual(valA, valB map[string]string) bool {
for k, v := range valA {
if vb, ok := valB[k]; ok && !strings.EqualFold(vb, v) {
return false
}
}

for k, v := range valB {
if va, ok := valA[k]; ok && !strings.EqualFold(va, v) {
return false
}
}

return true
}

// FMTP interface for implementing custom
// FMTP parsers based on MimeType.
type FMTP interface {
Expand All @@ -39,30 +107,39 @@ type FMTP interface {
}

// Parse parses an fmtp string based on the MimeType.
func Parse(mimeType, line string) FMTP {
func Parse(mimeType string, clockRate uint32, channels uint16, line string) FMTP {
var fmtp FMTP

parameters := parseParameters(line)

switch {
case strings.EqualFold(mimeType, "video/h264"):
fmtp = &h264FMTP{
parameters: parameters,
}

case strings.EqualFold(mimeType, "video/vp9"):
fmtp = &vp9FMTP{
parameters: parameters,
case clockRate == 90000 && (channels == 0 || channels == 1):
switch {
case strings.EqualFold(mimeType, "video/vp9"):
fmtp = &vp9FMTP{
parameters: parameters,
}

case strings.EqualFold(mimeType, "video/av1"):
fmtp = &av1FMTP{
parameters: parameters,
}
}

case strings.EqualFold(mimeType, "video/av1"):
fmtp = &av1FMTP{
// Clock rate and channel checks have been introduced quite recently.
// Existing implementations often use VP8, H264 or Opus without setting clock rate or channels.
// Keep compatibility with these situations.
// It would be better to add a clock rate and channel check in a future major release.
case strings.EqualFold(mimeType, "video/h264"):
fmtp = &h264FMTP{
parameters: parameters,
}

default:
fmtp = &genericFMTP{
mimeType: mimeType,
clockRate: clockRate,
channels: channels,
parameters: parameters,
}
}
Expand All @@ -72,6 +149,8 @@ func Parse(mimeType, line string) FMTP {

type genericFMTP struct {
mimeType string
clockRate uint32
channels uint16
parameters map[string]string
}

Expand All @@ -87,23 +166,10 @@ func (g *genericFMTP) Match(b FMTP) bool {
return false
}

if !strings.EqualFold(g.mimeType, fmtp.MimeType()) {
return false
}

for k, v := range g.parameters {
if vb, ok := fmtp.parameters[k]; ok && !strings.EqualFold(vb, v) {
return false
}
}

for k, v := range fmtp.parameters {
if va, ok := g.parameters[k]; ok && !strings.EqualFold(va, v) {
return false
}
}

return true
return strings.EqualFold(g.mimeType, fmtp.MimeType()) &&
ClockRateEqual(g.mimeType, g.clockRate, fmtp.clockRate) &&
ChannelsEqual(g.mimeType, g.channels, fmtp.channels) &&
paramsEqual(g.parameters, fmtp.parameters)
}

func (g *genericFMTP) Parameter(key string) (string, bool) {
Expand Down
97 changes: 90 additions & 7 deletions internal/fmtp/fmtp_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -56,17 +56,23 @@ func TestParseParameters(t *testing.T) {

func TestParse(t *testing.T) {
for _, ca := range []struct {
name string
mimeType string
line string
expected FMTP
name string
mimeType string
clockRate uint32
channels uint16
line string
expected FMTP
}{
{
"generic",
"generic",
90000,
2,
"key-name=value",
&genericFMTP{
mimeType: "generic",
mimeType: "generic",
clockRate: 90000,
channels: 2,
parameters: map[string]string{
"key-name": "value",
},
Expand All @@ -75,9 +81,13 @@ func TestParse(t *testing.T) {
{
"generic case normalization",
"generic",
90000,
2,
"Key=value",
&genericFMTP{
mimeType: "generic",
mimeType: "generic",
clockRate: 90000,
channels: 2,
parameters: map[string]string{
"key": "value",
},
Expand All @@ -86,6 +96,8 @@ func TestParse(t *testing.T) {
{
"h264",
"video/h264",
90000,
0,
"key-name=value",
&h264FMTP{
parameters: map[string]string{
Expand All @@ -96,6 +108,8 @@ func TestParse(t *testing.T) {
{
"vp9",
"video/vp9",
90000,
0,
"key-name=value",
&vp9FMTP{
parameters: map[string]string{
Expand All @@ -106,6 +120,8 @@ func TestParse(t *testing.T) {
{
"av1",
"video/av1",
90000,
0,
"key-name=value",
&av1FMTP{
parameters: map[string]string{
Expand All @@ -115,7 +131,7 @@ func TestParse(t *testing.T) {
},
} {
t.Run(ca.name, func(t *testing.T) {
f := Parse(ca.mimeType, ca.line)
f := Parse(ca.mimeType, ca.clockRate, ca.channels, ca.line)
if !reflect.DeepEqual(ca.expected, f) {
t.Errorf("expected '%v', got '%v'", ca.expected, f)
}
Expand Down Expand Up @@ -177,6 +193,27 @@ func TestMatch(t *testing.T) { //nolint:maintidx
},
true,
},
{
"generic inferred channels",
&genericFMTP{
mimeType: "generic",
channels: 1,
parameters: map[string]string{
"key1": "value1",
"key2": "value2",
"key3": "value3",
},
},
&genericFMTP{
mimeType: "generic",
parameters: map[string]string{
"key1": "value1",
"key2": "value2",
"key3": "value3",
},
},
true,
},
{
"generic inconsistent different kind",
&genericFMTP{
Expand Down Expand Up @@ -210,6 +247,52 @@ func TestMatch(t *testing.T) { //nolint:maintidx
},
false,
},
{
"generic inconsistent different clock rate",
&genericFMTP{
mimeType: "generic",
clockRate: 90000,
parameters: map[string]string{
"key1": "value1",
"key2": "value2",
"key3": "value3",
},
},
&genericFMTP{
mimeType: "generic",
clockRate: 48000,
parameters: map[string]string{
"key1": "value1",
"key2": "value2",
"key3": "value3",
},
},
false,
},
{
"generic inconsistent different channels",
&genericFMTP{
mimeType: "generic",
clockRate: 90000,
channels: 2,
parameters: map[string]string{
"key1": "value1",
"key2": "value2",
"key3": "value3",
},
},
&genericFMTP{
mimeType: "generic",
clockRate: 90000,
channels: 1,
parameters: map[string]string{
"key1": "value1",
"key2": "value2",
"key3": "value3",
},
},
false,
},
{
"generic inconsistent different parameters",
&genericFMTP{
Expand Down
12 changes: 10 additions & 2 deletions mediaengine.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -246,7 +246,10 @@ func (m *MediaEngine) RegisterDefaultCodecs() error {
// addCodec will append codec if it not exists.
func (m *MediaEngine) addCodec(codecs []RTPCodecParameters, codec RTPCodecParameters) []RTPCodecParameters {
for _, c := range codecs {
if c.MimeType == codec.MimeType && c.PayloadType == codec.PayloadType {
if c.MimeType == codec.MimeType &&
fmtp.ClockRateEqual(c.MimeType, c.ClockRate, codec.ClockRate) &&
fmtp.ChannelsEqual(c.MimeType, c.Channels, codec.Channels) &&
c.PayloadType == codec.PayloadType {
return codecs
}
}
Expand Down Expand Up @@ -459,7 +462,12 @@ func (m *MediaEngine) matchRemoteCodec(
codecs = m.audioCodecs
}

remoteFmtp := fmtp.Parse(remoteCodec.RTPCodecCapability.MimeType, remoteCodec.RTPCodecCapability.SDPFmtpLine)
remoteFmtp := fmtp.Parse(
remoteCodec.RTPCodecCapability.MimeType,
remoteCodec.RTPCodecCapability.ClockRate,
remoteCodec.RTPCodecCapability.Channels,
remoteCodec.RTPCodecCapability.SDPFmtpLine)

if apt, hasApt := remoteFmtp.Parameter("apt"); hasApt { //nolint:nestif
payloadType, err := strconv.ParseUint(apt, 10, 8)
if err != nil {
Expand Down
3 changes: 2 additions & 1 deletion peerconnection_media_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1823,7 +1823,8 @@ func TestPeerConnection_Zero_PayloadType(t *testing.T) {
pcOffer, pcAnswer, err := newPair()
require.NoError(t, err)

audioTrack, err := NewTrackLocalStaticSample(RTPCodecCapability{MimeType: MimeTypePCMU}, "audio", "audio")
audioTrack, err := NewTrackLocalStaticSample(
RTPCodecCapability{MimeType: MimeTypePCMU, ClockRate: 8000}, "audio", "audio")
require.NoError(t, err)

_, err = pcOffer.AddTrack(audioTrack)
Expand Down
25 changes: 20 additions & 5 deletions rtpcodec.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -108,19 +108,34 @@ func codecParametersFuzzySearch(
needle RTPCodecParameters,
haystack []RTPCodecParameters,
) (RTPCodecParameters, codecMatchType) {
needleFmtp := fmtp.Parse(needle.RTPCodecCapability.MimeType, needle.RTPCodecCapability.SDPFmtpLine)
needleFmtp := fmtp.Parse(
needle.RTPCodecCapability.MimeType,
needle.RTPCodecCapability.ClockRate,
needle.RTPCodecCapability.Channels,
needle.RTPCodecCapability.SDPFmtpLine)

// First attempt to match on MimeType + SDPFmtpLine
// First attempt to match on MimeType + Channels + SDPFmtpLine
for _, c := range haystack {
cfmtp := fmtp.Parse(c.RTPCodecCapability.MimeType, c.RTPCodecCapability.SDPFmtpLine)
cfmtp := fmtp.Parse(
c.RTPCodecCapability.MimeType,
c.RTPCodecCapability.ClockRate,
c.RTPCodecCapability.Channels,
c.RTPCodecCapability.SDPFmtpLine)

if needleFmtp.Match(cfmtp) {
return c, codecMatchExact
}
}

// Fallback to just MimeType
// Fallback to just MimeType + Channels
for _, c := range haystack {
if strings.EqualFold(c.RTPCodecCapability.MimeType, needle.RTPCodecCapability.MimeType) {
if strings.EqualFold(c.RTPCodecCapability.MimeType, needle.RTPCodecCapability.MimeType) &&
fmtp.ClockRateEqual(c.RTPCodecCapability.MimeType,
c.RTPCodecCapability.ClockRate,
needle.RTPCodecCapability.ClockRate) &&
fmtp.ChannelsEqual(c.RTPCodecCapability.MimeType,
c.RTPCodecCapability.Channels,
needle.RTPCodecCapability.Channels) {
return c, codecMatchPartial
}
}
Expand Down
Loading

0 comments on commit 54a37a9

Please sign in to comment.