Align the public API with the WPT WebIDL

README.md

@@ -40,7 +40,7 @@ import webrtc
 async def main():
     pc = webrtc.RTCPeerConnection()
 
-    stream = webrtc.get_user_media()
+    stream = await webrtc.media_devices.get_user_media(webrtc.MediaStreamConstraints(audio=True))
     for track in stream.get_tracks():
         pc.add_track(track, stream)
 

docs/source/webrtc.interfaces.media_devices.rst

@@ -0,0 +1,7 @@
+webrtc.interfaces.media\_devices module
+=======================================
+
+.. automodule:: webrtc.interfaces.media_devices
+   :members:
+   :undoc-members:
+   :show-inheritance:

docs/source/webrtc.interfaces.rst

@@ -12,6 +12,7 @@ Submodules
 .. toctree::
    :maxdepth: 4
 
+   webrtc.interfaces.media_devices
    webrtc.interfaces.media_stream
    webrtc.interfaces.media_stream_track
    webrtc.interfaces.media_stream_track_processor
@@ -21,7 +22,9 @@ Submodules
    webrtc.interfaces.rtc_ice_transport
    webrtc.interfaces.rtc_peer_connection
    webrtc.interfaces.rtc_rtp_receiver
+   webrtc.interfaces.rtc_rtp_script_transform
    webrtc.interfaces.rtc_rtp_sender
    webrtc.interfaces.rtc_rtp_transceiver
    webrtc.interfaces.rtc_sctp_transport
+   webrtc.interfaces.sframe_transform
    webrtc.interfaces.track_generator

docs/source/webrtc.interfaces.rtc_rtp_script_transform.rst

@@ -0,0 +1,7 @@
+webrtc.interfaces.rtc\_rtp\_script\_transform
+=============================================
+
+.. automodule:: webrtc.interfaces.rtc_rtp_script_transform
+   :members:
+   :undoc-members:
+   :show-inheritance:

docs/source/webrtc.interfaces.sframe_transform.rst

@@ -0,0 +1,7 @@
+webrtc.interfaces.sframe\_transform
+====================================
+
+.. automodule:: webrtc.interfaces.sframe_transform
+   :members:
+   :undoc-members:
+   :show-inheritance:

docs/source/webrtc.models.rst

@@ -19,11 +19,14 @@ Submodules
    webrtc.models.media_track_constraints
    webrtc.models.rtc_certificate
    webrtc.models.rtc_configuration
+   webrtc.models.rtc_encoded_frame
    webrtc.models.rtc_ice_candidate
+   webrtc.models.rtc_offer_answer_options
    webrtc.models.rtc_rtp_transceiver_init
    webrtc.models.rtc_session_description
    webrtc.models.rtc_session_description_init
    webrtc.models.rtc_stats
    webrtc.models.rtp_parameters
    webrtc.models.rtp_source
+   webrtc.models.sframe_transform_options
    webrtc.models.video_frame

docs/source/webrtc.functions.get_user_media.rst → docs/source/webrtc.models.rtc_encoded_frame.rst

@@ -1,7 +1,7 @@
-webrtc.functions.get\_user\_media
+webrtc.models.rtc\_encoded\_frame
 =================================
 
-.. automodule:: webrtc.functions.get_user_media
+.. automodule:: webrtc.models.rtc_encoded_frame
    :members:
    :undoc-members:
    :show-inheritance:

docs/source/webrtc.models.rtc_offer_answer_options.rst

@@ -0,0 +1,7 @@
+webrtc.models.rtc\_offer\_answer\_options
+==========================================
+
+.. automodule:: webrtc.models.rtc_offer_answer_options
+   :members:
+   :undoc-members:
+   :show-inheritance:

docs/source/webrtc.models.sframe_transform_options.rst

@@ -0,0 +1,7 @@
+webrtc.models.sframe\_transform\_options
+=========================================
+
+.. automodule:: webrtc.models.sframe_transform_options
+   :members:
+   :undoc-members:
+   :show-inheritance:

docs/source/webrtc.rst

@@ -12,7 +12,6 @@ Subpackages
 .. toctree::
    :maxdepth: 4
 
-   webrtc.functions
    webrtc.interfaces
    webrtc.models
    webrtc.utils

examples/echo.py

@@ -92,7 +92,9 @@ async def main() -> None:
     """Echoes the camera of the caller back to it."""
     caller, echo = webrtc.RTCPeerConnection(), webrtc.RTCPeerConnection()
     trickle(caller, echo)
-    camera = webrtc.get_user_media(audio=False, video=True).get_video_tracks()[0]
+    camera = (await webrtc.media_devices.get_user_media(webrtc.MediaStreamConstraints(video=True))).get_video_tracks()[
+        0
+    ]
     caller.add_track(camera)
     generator = webrtc.VideoTrackGenerator()
     echoed = asyncio.get_running_loop().create_future()

examples/recorder.py

@@ -77,7 +77,7 @@ async def main() -> None:
     """Records the camera and the microphone for a few seconds."""
     sender, receiver = webrtc.RTCPeerConnection(), webrtc.RTCPeerConnection()
     trickle(sender, receiver)
-    stream = webrtc.get_user_media(audio=True, video=True)
+    stream = await webrtc.media_devices.get_user_media(webrtc.MediaStreamConstraints(audio=True, video=True))
     for track in stream.get_tracks():
         sender.add_track(track, stream)
 

pyproject.toml

@@ -141,7 +141,6 @@ ignore = [
 "python-webrtc/python/webrtc/interfaces/rtc_peer_connection.py" = ["PLR0904"]  # the W3C interface
 # user callbacks error the stream, as the specification says; handler errors go to the loop's exception handler
 "python-webrtc/python/webrtc/{streams.py,utils/events.py}" = ["BLE001"]
-"python-webrtc/python/webrtc/models/rtc_stats.py" = ["FURB189"]  # a real dict, for json and isinstance
 # native leak counters and queues have no public API; chaos replays a seeded random
 "tests/**" = ["S101", "D1", "PLR2004", "S404", "S603", "SLF001", "S311"]
 "examples/**" = ["T201"]
@@ -152,7 +151,7 @@ ignore = [
 convention = "google"
 
 [tool.ruff.lint.flake8-builtins]
-ignorelist = ["format", "id", "type"]  # W3C member names
+ignorelist = ["format", "hash", "id", "type"]  # W3C member names
 
 [tool.ruff.lint.pydoclint]
 ignore-one-line-docstrings = true

python-webrtc/cpp/src/enums/enums.h

@@ -90,7 +90,6 @@ namespace python_webrtc {
         {State::kIceConnectionFailed, "failed"},
         {State::kIceConnectionDisconnected, "disconnected"},
         {State::kIceConnectionClosed, "closed"},
-        {State::kIceConnectionMax, "max"},
     });
   };
 
@@ -129,7 +128,7 @@ namespace python_webrtc {
   template <>
   struct StrEnum<webrtc::RtpTransceiverDirection> {
     using Direction = webrtc::RtpTransceiverDirection;
-    static constexpr auto name = pybind11::detail::const_name("TransceiverDirection");
+    static constexpr auto name = pybind11::detail::const_name("RTCRtpTransceiverDirection");
     static constexpr auto values = std::to_array<std::pair<Direction, const char *>>({
         {Direction::kSendRecv, "sendrecv"},
         {Direction::kSendOnly, "sendonly"},
@@ -186,7 +185,7 @@ namespace python_webrtc {
 
   template <>
   struct StrEnum<webrtc::IceGatheringState> {
-    static constexpr auto name = pybind11::detail::const_name("CricketIceGatheringState");
+    static constexpr auto name = pybind11::detail::const_name("RTCIceGathererState");
     static constexpr auto values = std::to_array<std::pair<webrtc::IceGatheringState, const char *>>({
         {webrtc::IceGatheringState::kIceGatheringNew, "new"},
         {webrtc::IceGatheringState::kIceGatheringGathering, "gathering"},
@@ -197,7 +196,7 @@ namespace python_webrtc {
   template <>
   struct StrEnum<webrtc::DtlsTransportState> {
     using State = webrtc::DtlsTransportState;
-    static constexpr auto name = pybind11::detail::const_name("DtlsTransportState");
+    static constexpr auto name = pybind11::detail::const_name("RTCDtlsTransportState");
     static constexpr auto values = std::to_array<std::pair<State, const char *>>({
         {State::kNew, "new"},
         {State::kConnecting, "connecting"},
@@ -210,10 +209,11 @@ namespace python_webrtc {
   template <>
   struct StrEnum<webrtc::SctpTransportState> {
     using State = webrtc::SctpTransportState;
-    static constexpr auto name = pybind11::detail::const_name("SctpTransportState");
+    static constexpr auto name = pybind11::detail::const_name("RTCSctpTransportState");
     static constexpr auto values = std::to_array<std::pair<State, const char *>>({
-        {State::kNew, "new"},
         {State::kConnecting, "connecting"},
+        // libwebrtc only: the transport is connecting from its creation in the specification
+        {State::kNew, "connecting"},
         {State::kConnected, "connected"},
         {State::kClosed, "closed"},
     });

python-webrtc/cpp/src/interfaces/interfaces.cpp

@@ -7,7 +7,11 @@
 
 #include "interfaces.h"
 
+#include "../media/encoded_frame.h"
+#include "../media/frame_transformer_bridge.h"
 #include "../media/media_stream_track_processor.h"
+#include "../media/rtc_rtp_script_transform.h"
+#include "../media/sframe_transform.h"
 #include "../media/track_generator.h"
 #include "../media/video_frame_buffer.h"
 #include "../utils/alive_count.h"
@@ -65,6 +69,10 @@ namespace python_webrtc {
               {"MediaStreamTrackProcessor", AliveCount<MediaStreamTrackProcessor>::count.load()},
               {"TrackGenerator", AliveCount<TrackGenerator>::count.load()},
               {"VideoFrameBuffer", AliveCount<VideoFrameBuffer>::count.load()},
+              {"RTCRtpScriptTransform", AliveCount<RTCRtpScriptTransform>::count.load()},
+              {"SFrameTransform", AliveCount<SFrameTransform>::count.load()},
+              {"RTCEncodedFrame", AliveCount<EncodedFrame>::count.load()},
+              {"FrameTransformerBridge", AliveCount<FrameTransformerBridge>::count.load()},
           };
         },
         nogil());

python-webrtc/cpp/src/interfaces/rtc_peer_connection.cpp

@@ -128,12 +128,14 @@ namespace python_webrtc {
   void RTCPeerConnection::Init(pybind11::module &m) {
     Listeners::BindClass<RTCPeerConnection>(m, "RTCPeerConnection")
         .def(pybind11::init(nogil_factory(+[](const std::optional<ConfigurationInit> &configuration) {
-          return std::shared_ptr<RTCPeerConnection>(new RTCPeerConnection(configuration), DeleteOffLibwebrtcThread());
-        })))
+               return std::shared_ptr<RTCPeerConnection>(new RTCPeerConnection(configuration),
+                                                         DeleteOffLibwebrtcThread());
+             })),
+             pybind11::arg("configuration"))
         .def("createOffer", WithCallbacks(&RTCPeerConnection::CreateOffer), pybind11::arg("onSuccess"),
-             pybind11::arg("onFailure"), pybind11::arg("iceRestart"), pybind11::arg("voiceActivityDetection"))
+             pybind11::arg("onFailure"), pybind11::arg("iceRestart"))
         .def("createAnswer", WithCallbacks(&RTCPeerConnection::CreateAnswer), pybind11::arg("onSuccess"),
-             pybind11::arg("onFailure"), pybind11::arg("voiceActivityDetection"))
+             pybind11::arg("onFailure"))
         .def("setLocalDescription", WithCallbacks(&RTCPeerConnection::SetLocalDescription), pybind11::arg("onSuccess"),
              pybind11::arg("onFailure"), pybind11::arg("description"))
         .def("setRemoteDescription", WithCallbacks(&RTCPeerConnection::SetRemoteDescription),
@@ -499,8 +501,7 @@ namespace python_webrtc {
   }
 
   void RTCPeerConnection::CreateOffer(std::function<void(RTCSessionDescription)> &onSuccess,
-                                      std::function<void(RTCCallbackException)> &onFailure, bool iceRestart,
-                                      bool voiceActivityDetection) {
+                                      std::function<void(RTCCallbackException)> &onFailure, bool iceRestart) {
     auto pc = connection();
     auto state = pc ? pc->signaling_state() : SignalingState::kClosed;
     if (state == SignalingState::kClosed) {
@@ -518,24 +519,20 @@ namespace python_webrtc {
 
     auto options = webrtc::PeerConnectionInterface::RTCOfferAnswerOptions();
     options.ice_restart = iceRestart;
-    options.voice_activity_detection = voiceActivityDetection;
 
     pc->CreateOffer(observer.get(), options);
   }
 
   void RTCPeerConnection::CreateAnswer(std::function<void(RTCSessionDescription)> &onSuccess,
-                                       std::function<void(RTCCallbackException)> &onFailure,
-                                       bool voiceActivityDetection) {
+                                       std::function<void(RTCCallbackException)> &onFailure) {
     auto pc = connection();
     if (!pc || pc->signaling_state() == SignalingState::kClosed) {
       onFailure(RTCCallbackException(closedError("createAnswer")));
       return;
     }
 
     auto observer = webrtc::make_ref_counted<CreateSessionDescriptionObserver>(weak_from_this(), onSuccess, onFailure);
-    auto options = webrtc::PeerConnectionInterface::RTCOfferAnswerOptions();
-    options.voice_activity_detection = voiceActivityDetection;
-    pc->CreateAnswer(observer.get(), options);
+    pc->CreateAnswer(observer.get(), webrtc::PeerConnectionInterface::RTCOfferAnswerOptions());
   }
 
   void RTCPeerConnection::SaveCreatedDescription(const RTCSessionDescriptionInit &description) {
@@ -1304,6 +1301,12 @@ namespace python_webrtc {
       pc->Close();
       for (const auto &transceiver : pc->GetTransceivers()) {
         Wrap(_transceivers, transceiver)->GetReceiver()->GetTrack()->OnPeerConnectionClosed();
+        if (auto sender = RTCRtpSender::holder().Find(transceiver->sender().get())) {
+          sender->ReleaseTransform();
+        }
+        if (auto receiver = RTCRtpReceiver::holder().Find(transceiver->receiver().get())) {
+          receiver->ReleaseTransform();
+        }
       }
     }
 

python-webrtc/cpp/src/interfaces/rtc_peer_connection.h

@@ -75,11 +75,10 @@ namespace python_webrtc {
     static void ReleaseElsewhere(std::shared_ptr<RTCPeerConnection> &&connection);
 
     void CreateOffer(std::function<void(RTCSessionDescription)> &onSuccess,
-                     std::function<void(RTCCallbackException)> &onFailure, bool iceRestart,
-                     bool voiceActivityDetection);
+                     std::function<void(RTCCallbackException)> &onFailure, bool iceRestart);
 
     void CreateAnswer(std::function<void(RTCSessionDescription)> &onSuccess,
-                      std::function<void(RTCCallbackException)> &onFailure, bool voiceActivityDetection);
+                      std::function<void(RTCCallbackException)> &onFailure);
 
     // the last description createOffer or createAnswer made, which are the only ones setLocalDescription takes
     void SaveCreatedDescription(const RTCSessionDescriptionInit &description);

python-webrtc/cpp/src/interfaces/rtc_rtp_receiver.cpp

@@ -35,6 +35,7 @@ namespace python_webrtc {
 
   RTCRtpReceiver::~RTCRtpReceiver() {
     const BlockingDestructor release("RTCRtpReceiver");
+    _transform.Release();
 
     // callbacks run on the signaling thread, so after this none of them can be running or start again
     _factory->signalingThread()->BlockingCall([this]() {
@@ -63,6 +64,7 @@ namespace python_webrtc {
         .def("getStats", WithCallbacks(&RTCRtpReceiver::GetStats), pybind11::arg("onSuccess"),
              pybind11::arg("onFailure"))
         .def_static("getCapabilities", &RTCRtpReceiver::GetCapabilities, nogil(), pybind11::arg("kind"))
+        .def_property("transform", nogil_fn(&RTCRtpReceiver::GetTransform), nogil_fn(&RTCRtpReceiver::SetTransform))
         .def("_getSources", &RTCRtpReceiver::GetSources, nogil());
   }
 
@@ -157,6 +159,33 @@ namespace python_webrtc {
     return sources;
   }
 
+  FrameSource RTCRtpReceiver::TransformSource() {
+    FrameSource source;
+    source.sender = false;
+    source.video = _receiver->media_type() == webrtc::MediaType::VIDEO;
+    source.sendKeyFrameRequest = [weak = weak_from_this()]() {
+      auto self = weak.lock();
+      if (!self) {
+        return;
+      }
+      // the source of a remote video track asks the receiver's stream for a key frame (a PLI)
+      auto track = self->_receiver->track();
+      auto *video = dynamic_cast<webrtc::VideoTrackInterface *>(track.get());
+      if (video != nullptr && video->GetSource() != nullptr) {
+        video->GetSource()->GenerateKeyFrame();
+      }
+    };
+    return source;
+  }
+
+  void RTCRtpReceiver::SetTransform(const std::shared_ptr<RtpTransform> &transform) {
+    _transform.Set(
+        transform, [this]() { return TransformSource(); },
+        [this](const webrtc::scoped_refptr<FrameTransformerBridge> &bridge) {
+          _receiver->SetFrameTransformer(bridge);
+        });
+  }
+
   std::optional<webrtc::RtpCapabilities> RTCRtpReceiver::GetCapabilities(const std::string &kind) {
     auto type = mediaTypeOf(kind);
     if (!type) {