Aiortc to Create a Remote Webcam With an Android Phone

mkdir -p ~/test/android_webcam
cd ~/test/android_webcam
pdm init --non-interactive

pdm add aiohttp aiortc opencv-python aiohttp-cors

Adding packages to default dependencies: aiohttp, aiortc, opencv-python, aiohttp-cors
STATUS: Resolving dependencies
STATUS: Resolving: new pin python>=3.11,<3.12
STATUS: Resolving: new pin aiohttp 3.9.1
STATUS: Resolving: new pin aiohttp-cors 0.7.0
STATUS: Resolving: new pin aiortc 1.7.0
STATUS: Resolving: new pin opencv-python 4.9.0.80
STATUS: Resolving: new pin numpy 1.26.3
STATUS: Resolving: new pin aioice 0.9.0
STATUS: Resolving: new pin av 11.0.0
STATUS: Resolving: new pin multidict 6.0.4
STATUS: Resolving: new pin yarl 1.9.4
STATUS: Resolving: new pin aiosignal 1.3.1
STATUS: Resolving: new pin frozenlist 1.4.1
STATUS: Resolving: new pin attrs 23.2.0
STATUS: Resolving: new pin cffi 1.16.0
STATUS: Resolving: new pin cryptography 42.0.1
STATUS: Resolving: new pin google-crc32c 1.5.0
STATUS: Resolving: new pin pyee 11.1.0
STATUS: Resolving: new pin pylibsrtp 0.10.0
STATUS: Resolving: new pin pyopenssl 24.0.0
STATUS: Resolving: new pin dnspython 2.5.0
STATUS: Resolving: new pin idna 3.6
STATUS: Resolving: new pin ifaddr 0.2.0
STATUS: Resolving: new pin pycparser 2.21
STATUS: Resolving: new pin typing-extensions 4.9.0
STATUS: Fetching hashes for resolved packages...
🔒 Lock successful
Changes are written to pyproject.toml.
STATUS: Resolving packages from lockfile...
Synchronizing working set with resolved packages: 1 to add, 0 to update, 0 to remove

  ✔ Install aiohttp-cors 0.7.0 successful

🎉 All complete!

[libx264 @ 0x34b4180] non-strictly-monotonic PTS
[libx264 @ 0x34b4180] non-strictly-monotonic PTS
[mp4 @ 0x3b1bbc0] Application provided invalid, non monotonically increasing dts to muxer in stream 0: 352768 >= 352768

#!/usr/bin/env python3
# -*- coding:utf-8 -*-

import argparse
import asyncio
import json
import logging
import os
import ssl
import uuid

import numpy as np
import cv2
from aiohttp import web
from av import VideoFrame
import aiohttp_cors

from aiortc import MediaStreamTrack, RTCPeerConnection, RTCSessionDescription
from aiortc.contrib.media import MediaBlackhole, MediaPlayer, MediaRecorder, MediaRelay

from aiohttp import web
pcs = set()
relay = MediaRelay()
ROOT = os.path.dirname(__file__)

logger = logging.getLogger("pc")
logging.basicConfig(level=logging.INFO)

# width, height = 640, 480
# fps = 30

# fourcc = cv2.VideoWriter_fourcc('V','P','8','0')
# out = cv2.VideoWriter('output.webm', fourcc, fps, (width, height))

class VideoTransformTrack(MediaStreamTrack):
    """
    A video stream track that transforms frames from an another track.
    """

    kind = "video"

    def __init__(self, track):
        super().__init__()  # don't forget this!
        self.track = track
        self.transform = None

    async def recv(self):
        frame = await self.track.recv()
        # out.write(cv2.cvtColor(np.array(frame.to_image()), cv2.COLOR_RGB2BGR))
        if self.transform == "cartoon":
            img = frame.to_ndarray(format="bgr24")

            # prepare color
            img_color = cv2.pyrDown(cv2.pyrDown(img))
            for _ in range(6):
                img_color = cv2.bilateralFilter(img_color, 9, 9, 7)
                img_color = cv2.pyrUp(cv2.pyrUp(img_color))

            # prepare edges
            img_edges = cv2.cvtColor(img, cv2.COLOR_RGB2GRAY)
            img_edges = cv2.adaptiveThreshold(
                cv2.medianBlur(img_edges, 7),
                255,
                cv2.ADAPTIVE_THRESH_MEAN_C,
                cv2.THRESH_BINARY,
                9,
                2,
            )
            img_edges = cv2.cvtColor(img_edges, cv2.COLOR_GRAY2RGB)

            # combine color and edges
            img = cv2.bitwise_and(img_color, img_edges)

            # rebuild a VideoFrame, preserving timing information
            new_frame = VideoFrame.from_ndarray(img, format="bgr24")
            new_frame.pts = frame.pts
            new_frame.time_base = frame.time_base
            return new_frame
        elif self.transform == "edges":
            # perform edge detection
            img = frame.to_ndarray(format="bgr24")
            img = cv2.cvtColor(cv2.Canny(img, 100, 200), cv2.COLOR_GRAY2BGR)

            # rebuild a VideoFrame, preserving timing information
            new_frame = VideoFrame.from_ndarray(img, format="bgr24")
            new_frame.pts = frame.pts
            new_frame.time_base = frame.time_base
            return new_frame
        elif self.transform == "rotate":
            # rotate image
            img = frame.to_ndarray(format="bgr24")
            rows, cols, _ = img.shape
            M = cv2.getRotationMatrix2D((cols / 2, rows / 2), frame.time * 45, 1)
            img = cv2.warpAffine(img, M, (cols, rows))

            # rebuild a VideoFrame, preserving timing information
            new_frame = VideoFrame.from_ndarray(img, format="bgr24")
            new_frame.pts = frame.pts
            new_frame.time_base = frame.time_base
            return new_frame
        else:
            return frame


async def on_shutdown(app):
    # close peer connections
    coros = [pc.close() for pc in pcs]
    await asyncio.gather(*coros)
    pcs.clear()

async def offer(request):
    print("offer")
    params = await request.json()
    offer = RTCSessionDescription(sdp=params["sdp"], type=params["type"])

    pc = RTCPeerConnection()
    pc_id = "PeerConnection(%s)" % uuid.uuid4()
    pcs.add(pc)

    def log_info(msg, *args):
        logging.info(pc_id + " " + msg, *args)

    log_info("Created for %s", request.remote)

    # prepare local media
    recorder = MediaRecorder(
        "test.mp4",
        options={
            'vbsf': 'hevc_mp4toannexb',
            'x264opts': 'keyint=24:min-keyint=24:no-scenecut',
        }
    )

    @pc.on("datachannel")
    def on_datachannel(channel):
        @channel.on("message")
        def on_message(message):
            if isinstance(message, str) and message.startswith("ping"):
                print(f"received {message}")

    @pc.on("connectionstatechange")
    async def on_connectionstatechange():
        log_info("Connection state is %s", pc.connectionState)
        if pc.connectionState == "failed":
            await pc.close()
            pcs.discard(pc)

    @pc.on("track")
    def on_track(track):
        log_info("Track %s received", track.kind)

        # send it back
        # pc.addTrack(
        #     VideoTransformTrack(
        #         relay.subscribe(track)
        #     )
        # )
        recorder.addTrack(relay.subscribe(track))

        @track.on("ended")
        async def on_ended():
            log_info("Track %s ended", track.kind)
            # out.release()
            await recorder.stop()

    # handle offer
    await pc.setRemoteDescription(offer)
    await recorder.start()

    # send answer
    answer = await pc.createAnswer()
    await pc.setLocalDescription(answer)

    return web.Response(
        content_type="application/json",
        text=json.dumps(
            {"sdp": pc.localDescription.sdp, "type": pc.localDescription.type}
        ),
    )


def main():
    app = web.Application()
    app.on_shutdown.append(on_shutdown)
    app.router.add_post("/offer", offer)
    cors = aiohttp_cors.setup(app, defaults={
        "*": aiohttp_cors.ResourceOptions(
            allow_credentials=True,
            expose_headers="*",
            allow_headers="*",
        )
    })
    for route in list(app.router.routes()):
        cors.add(route)
        web.run_app(
            app, access_log=None, host="0.0.0.0", port=9999,
        )



if __name__ == "__main__":
    main()

<html>
  <head>
    <meta charset="UTF-8"/>
    <meta name="viewport" content="width=device-width, initial-scale=1.0" />
    <script defer src="https://konubinix.eu/ipfs/bafkreic33rowgvvugzgzajagkuwidfnit2dyqyn465iygfs67agsukk24i?orig=https://cdn.jsdelivr.net/npm/alpinejs@3.x.x/dist/cdn.min.js"></script>
    <script src="https://konubinix.eu/ipfs/bafybeihp5kzlgqt56dmy5l4z7kpymfc4kn3fnehrrtr7cid7cn7ra36yha?orig=https://cdn.tailwindcss.com/3.4.3"></script>
    <script src="https://unpkg.com/peerjs@1.5.2/dist/peerjs.min.js"></script>
    <script>
      document.addEventListener('alpine:init', () => {
          Alpine.data('app', () => ({
              pc: null,
              message: "",
              video: null,
              dc: null,
              peer: null,
              conn: null,
              peerId: null,
              current_call: null,
              wakelock: null,
              width: {min: 320, max: 5000, value: 640},
              currentwidth: null,
              currentFrameRate: null,
              configure: true,
              bodyClass: "",
              async init () {
                  this.video = document.getElementById("video")
                  this.peer = new Peer("camera", {host: "192.168.1.46", port: 9999, path: "/peerjs/"});
                  this.peer.on('open', (id) => {
                      this.message += `connected to peerjs with id ${id}`
                      this.peerId = id
                  });
                  if(! 'mediaDevices' in navigator) {
                      alert( 'Your browser does not support media devices.' );
                  }
                  await this.setupMediaStream()
                  this.peer.on('connection', async (conn) => {
                      this.message += "connected to the controller"
                      this.conn = conn
                      this.message += "calling the controller"
                      this.current_call = this.peer.call("control", this.mediaStream)
                      this.conn.on("open", async () => {
                          conn.send(JSON.stringify({type: "configure", value: this.pc === null ? true : false}))
                      })
                      conn.on('data', async (data) => {
                          this.message += `got ${data}`
                          const order = JSON.parse(data)
                          if(order.action === "start") {
                              await this.start(this.width)
                              conn.send(JSON.stringify({"type": "answer", "message": "started"}))
                          }
                          else if(order.action === "stop") {
                              await this.stop()
                              conn.send(JSON.stringify({"type": "answer", "message": "stopped"}))
                          }
                          else if(order.action === "setupMediaStream")
                          {
                              this.width.value = order.width
                              await this.setupMediaStream()
                              conn.send(JSON.stringify({"type": "answer", "message": "mediaStreamUpdate"}))
                          }
                          else {
                              this.message("Unrecognized")
                          }
                      });
                  });
                  try{
                      navigator.wakeLock.request("screen").then(
                          (w) => {
                              this.wakelock = w;
                              this.message += "wakelock acquired"
                          }
                      ).catch((err) => {
                          this.message += `${err.name}, ${err.message}`;
                      });
                  } catch (err) {
                      this.message += err.toString()
                  };
              },

              createPeerConnection() {
                  this.pc = new RTCPeerConnection();
                  this.pc.addEventListener('track', (evt) => {
                      this.video.srcObject = evt.streams[0];
                  });
              },

              async setupMediaStream() {
                  this.mediaStream = await navigator.mediaDevices.getUserMedia(
                      {
                          video: {
                              width: { ideal: this.width.value },
                              height: { ideal: this.width.value },
                              frameRate: 30,
                              facingMode: {exact: "environment"},
                          },
                          audio: true,
                      }
                  );
                  // window.ms = this.mediaStream
                  this.currentwidth = this.mediaStream.getVideoTracks()[0].getSettings().width
                  this.currentFrameRate = this.mediaStream.getVideoTracks()[0].getSettings().frameRate
                  this.video.srcObject = this.mediaStream
                  if(this.conn !== null){
                      this.current_call.close()
                      this.current_call = this.peer.call("control", this.mediaStream)
                  }
              },

              async start() {
                  this.configure = false
                  this.bodyClass = "text-slate-700 bg-black"
                  this.createPeerConnection()
                  this.mediaStream.getTracks().forEach((track) => {
                      this.pc.addTrack(track, this.mediaStream);
                  });
                  await this.negociate()
              },
              async negociate() {
                  this.message += "\nneg start"
                  var offer = await this.pc.createOffer();
                  await this.pc.setLocalDescription(offer);
                  try{
                      var resp = await fetch('http://192.168.1.94:9999/offer', {
                          body: JSON.stringify({
                              sdp: offer.sdp,
                              type: offer.type,
                          }),
                          headers: {
                              'Content-Type': 'application/json'
                          },
                          method: 'POST'
                      });
                  } catch (err) {
                      alert(err.toString())
                      alert(JSON.stringify(err));
                      throw err
                  };
                  this.message += "\nneg sent"

                  var answer = await resp.json()
                  await this.pc.setRemoteDescription(answer);
                  this.message += "\nconnected"

              },
              async stop() {
                  // if (this.wakelock != null) {
                  //     this.wakelock.release().then(() => {
                  //         this.wakelock = null;
                  //         // alert("released");
                  //     }).catch((err) => {
                  //         alert(`${err.name}, ${err.message}`);
                  //     });
                  // }
                  // this.current_call.close()
                  this.pc.getTransceivers().forEach(function(transceiver) {
                      if (transceiver.stop) {
                          transceiver.stop();
                      }
                  });
                  this.pc.getSenders().forEach(function(sender) {
                      sender.track.stop();
                  });

                  // close peer connection
                  setTimeout(async () => {
                      this.pc.close();
                      this.pc = null
                      this.message += "\nall closed"
                      this.conn.send(JSON.stringify({"type": "answer", "message": "allclosed"}))
                      this.bodyClass = ""
                      this.configure = true
                      setTimeout(async () => {
                          this.message += "connecting back"
                          await this.setupMediaStream()
                          this.current_call = this.peer.call("control", this.mediaStream)
                      }, 500)
                  }, 500);

              },
          }))
      })
    </script>
  </head>
  <body x-data="app" :class="bodyClass">
    <div>
      <!-- <button @click="start">Start</button> -->
      <!-- <button @click="stop">Stop</button> -->
      <div x-text="peerId"></div>
      <div x-text="message"></div>
      <div x-show="configure">
        <span x-text="width.value"></span>
        <span x-text="currentwidth"></span>
        <span x-text="currentFrameRate"></span>
        <div class="justify-center sm:px-20 p-15 h-screen" x-data="{show: true}">
          <button class="text-white bg-blue-700 hover:bg-blue-800 focus:ring-4 focus:ring-blue-300 font-medium rounded-lg text-sm px-5 py-2.5 me-2 mb-2 dark:bg-blue-600 dark:hover:bg-blue-700 focus:outline-none dark:focus:ring-blue-800" @click="show = !show ; $refs.video.scrollIntoView()" x-text="show ? 'Hide' : 'Show'"></button>
          <video id="video" @dblclick="$event.target.requestFullscreen()" x-show="show" class="object-scale-down max-h-full" x-ref="video" poster="https://d1tobl1u8ox4qn.cloudfront.net/2018/05/3814a83b-a2d4-4dfd-8945-f1cd003eb16f-1920x1080.jpg" autoplay="true" playsinline="true"></video>
        </div>
      </div>
    </div>
  </body>
</html>

<!DOCTYPE html>
<html>
  <head>
    <meta name="viewport" content="width=device-width, initial-scale=1.0">
    <script defer src="https://konubinix.eu/ipfs/bafkreic33rowgvvugzgzajagkuwidfnit2dyqyn465iygfs67agsukk24i?orig=https://cdn.jsdelivr.net/npm/alpinejs@3.x.x/dist/cdn.min.js"></script>
    <script src="https://konubinix.eu/ipfs/bafybeihp5kzlgqt56dmy5l4z7kpymfc4kn3fnehrrtr7cid7cn7ra36yha?orig=https://cdn.tailwindcss.com/3.4.3"></script>
    <script src="https://unpkg.com/peerjs@1.5.2/dist/peerjs.min.js"></script>
    <script>
      document.addEventListener('alpine:init', () => {
          Alpine.data('app', () => ({
              peer: null,
              peerid: null,
              conn: null,
              message: "",
              video: null,
              configure: true,
              width: {min: 320, max: 6000, value: 640},
              async init() {
                  this.video = document.getElementById('videoElement')
                  this.peer = new Peer("control", {host: "192.168.1.46", port: 9999, path: "/peerjs/"})
                  this.peer.on('open', (id) => {
                      this.message += `\nconnected to peerjs with id ${id}`
                      this.peerid = id
                  });
                  this.peer.on('call', (call) => {
                      this.message += "in call"
                      call.answer();
                      call.on('stream', (stream) => {
                          this.message += "getting stream"
                          this.video.srcObject = stream;
                      });
                  });
                  this.connect()
              },
              connect() {
                  this.conn = this.peer.connect("camera")
                  this.conn.on('data', async (data) => {
                      this.message += `\nreceived ${data}`
                      const message = JSON.parse(data)
                      if(message.type === "answer")
                      {
                          if(message.message === "started")
                          {
                              this.configure = false
                          }
                          else if(message.message === "allclosed")
                          {
                              this.configure = true
                          }
                      }
                      else if(message.type === "configure")
                      {
                          this.configure = message.value
                      }
                      else
                      {
                          this.message += `I don't deal with message of type ${message.type}`
                      }
                  })
                  this.conn.on("open", async () => {
                      this.message += "\nconnected to the camera"
                  })
              },

              async start() {
                  this.conn.send(JSON.stringify({action: "start"}))
                  this.message += "\nsent start"
              },
              async stop() {
                  this.conn.send(JSON.stringify({action: "stop"}))
                  this.message += "\nsent stop"
              },
              async setupMediaStream () {
                  this.conn.send(JSON.stringify({action: "setupMediaStream", width: this.width.value}))
              }
          }))
      })
    </script>
  </head>
  <body>

    <div x-data="app">
      <div x-text="peerid"></div>
      <div>
        <button class="text-white bg-blue-700 hover:bg-blue-800 focus:ring-4 focus:ring-blue-300 font-medium rounded-lg text-sm px-5 py-2.5 me-2 mb-2 dark:bg-blue-600 dark:hover:bg-blue-700 focus:outline-none dark:focus:ring-blue-800" @click="connect">Connect</button>
        <button class="text-white bg-blue-700 hover:bg-blue-800 focus:ring-4 focus:ring-blue-300 font-medium rounded-lg text-sm px-5 py-2.5 me-2 mb-2 dark:bg-blue-600 dark:hover:bg-blue-700 focus:outline-none dark:focus:ring-blue-800" @click="start">Start</button>
        <button class="text-white bg-blue-700 hover:bg-blue-800 focus:ring-4 focus:ring-blue-300 font-medium rounded-lg text-sm px-5 py-2.5 me-2 mb-2 dark:bg-blue-600 dark:hover:bg-blue-700 focus:outline-none dark:focus:ring-blue-800" @click="stop">Stop</button>
      </div>

      <div x-text="message"></div>


      <div x-show="configure">
        <span x-text="width.value"></span>
        <input class="w-full h-2 bg-gray-200 rounded-lg appearance-none cursor-pointer dark:bg-gray-700"
               type="range"
               :min="width.min"
               :max="width.max"
               x-model="width.value"
               @input.debounce.500ms="setupMediaStream"
               >
      </div>

      <div class="justify-center sm:px-20 p-15 h-screen" x-data="{show: true}">
        <button class="text-white bg-blue-700 hover:bg-blue-800 focus:ring-4 focus:ring-blue-300 font-medium rounded-lg text-sm px-5 py-2.5 me-2 mb-2 dark:bg-blue-600 dark:hover:bg-blue-700 focus:outline-none dark:focus:ring-blue-800" @click="show = !show ; $refs.video.scrollIntoView()" x-text="show ? 'Hide' : 'Show'"></button>
        <video x-ref="video" x-show="show" @dblclick="$event.target.requestFullscreen()" class="object-scale-down max-h-full" id="videoElement" poster="https://d1tobl1u8ox4qn.cloudfront.net/2018/05/3814a83b-a2d4-4dfd-8945-f1cd003eb16f-1920x1080.jpg" autoplay></video>
      </div>

    </div>
  </body>
</html>

pdm run ./main.py

python3 -m http.server 9904