Stopmotion Android Apps
FleetingTable of Contents
I want to be able to use 2 Android phones to make some stopmotion videos with my kids.
This is what I want to achieve.
One device, the photo takers, is left untouched on a stick. The other is used to remotely trigger photos and manage the photos.
- the controller triggers the photo taker
- the photo takers sends the picture to the server
- the server computes the thumbnail and stores the picture
- the controller looks at the photos and organize them, downloading only thumbnails to do so
All the machines are connected to a private network, in which
- creating a self signed certificate and installing it on the devices is cumbersome,
- discussing over http is not an issue
Also, I don’t want to rely on online services. The system must use local only http (not https) connections.
controlling the other device with gun.js?
Tried it. The update is not reliable. Some get send twice, some need the browser to be refreshed. Will add more work than not using it.
using webrtc to send pictures?
With webrtc, we can programatically take photos
Using this codepen as an example, this is quite easy to achieve.
The Stream
The Capture
I could write a small application in alpine.js to get the picture and send it to the server.
But getUserMedia is only available in https. That breaks one of my conditions. Too bad.
with webrtc and qr code signaling and offline mode?
I could write a PWA with
- offline mode,
- QR code for signaling the pages with each other
But how can I signal the application to the server if it needs to be installed in https and the browser won’t accept mixed content when communicating with the local server over http only?
with kivy then?
kivy provides the mean to get access to the camera
https://github.com/kivy/kivy/issues/6995
Yet, it sounds like a source of Yak shaving, having to
- find an example that actually works,
- deal with old versions of android.
webrtc + servdroid -> a first minimum viable product
Do I really need https to get access to getUserMedia? Isn’t it supposed to work with http://localhost as well?
I can
- install servDroid on the device taking pictures
- host a local index.html with the code using getUserMedia
- send the data over http to my server
stopmotion does not need real time stuff, so I don’t even need to get fancy here
- the photo taker takes photos every second
- it sends them to the server
- the server stores the last one
- the controller gets the current photo every second
- when the user clicks on a button of the controller, it post the order to the server, keeping a copy of the current photo and sending its name so that the controller can get a list of kept photos to show to the user.
The server can store scaled down version of the images to help making things smooth.
The server code may look like this
#!/usr/bin/env python3
# -*- coding:utf-8 -*-
import base64
import json
import os
import shutil
from datetime import datetime
from pathlib import Path
from shlex import split
from subprocess import check_call
from flask import Flask, make_response, redirect, render_template, request
from flask_cors import CORS
app = Flask(__name__)
CORS(app)
root = Path(".")
orig = root / "orig"
cur = root / "cur.jpg"
index = root / "index.html"
@app.route("/", methods=["POST", "GET"])
def add_or_index():
if request.method == "POST":
cur.write_bytes(
base64.b64decode(
json.loads(request.data.decode())["img"]
[len("data:image/png;base64,"):]))
print(datetime.now())
return "ok"
else:
return index.read_text()
@app.route("/click")
def click():
if not orig.exists():
os.makedirs(orig)
if not cur.exists():
return ""
now = datetime.now()
name = now.isoformat() + ".jpg"
shutil.copy(cur, orig / name)
cmd = f"convert {orig / name} -scale 64 {root / name}"
print(cmd)
check_call(split(cmd))
return f"/img/{name}"
@app.route("/img/<name>")
def img(name):
img = root / name
response = make_response(img.read_bytes())
response.headers.set('Content-Type', 'image/jpeg')
return response
if __name__ == "__main__":
app.run(host="0.0.0.0", port=9999, debug=True, threaded=True)
The controller would get the code from the server
<!DOCTYPE html>
<html>
<head>
<script defer src="https://konubinix.eu/ipfs/bafkreic33rowgvvugzgzajagkuwidfnit2dyqyn465iygfs67agsukk24i?orig=https://cdn.jsdelivr.net/npm/alpinejs@3.x.x/dist/cdn.min.js"></script>
</head>
<body>
<div x-data='app'>
<div>
<img @click="updateCur" x-bind:src="cur" width="100"/>
</div>
<button @click="click">Click</button>
<template x-for="frame in frames">
<div>
<img x-bind:src="host + frame.url"/>
</div>
</template>
</div>
<script>
document.addEventListener('alpine:init', () => {
Alpine.data('app', () => ({
cur: "",
host: window.location,
frames: [],
async click() {
try {
res = await fetch(this.host + "/click")
} catch(error) {
msg = `${error} : ${JSON.stringify(error)}`
console.log(msg)
}
res = await res.text()
if (res === "") {
alert("no photo")
} else {
this.frames = [{url: res}].concat(this.frames)
}
},
updateCur() {
this.cur = this.host + "/img/cur.jpg?" + new Date().getTime()
},
init () {
this.updateCur()
setInterval(() => {this.updateCur()}, 2000)
},
}))
})
</script>
</body>
</html>
The photo taker, hosted locally with servDroid, would take pictures and send them to the server
<!DOCTYPE html>
<html>
<head>
<script defer src="https://konubinix.eu/ipfs/bafkreic33rowgvvugzgzajagkuwidfnit2dyqyn465iygfs67agsukk24i?orig=https://cdn.jsdelivr.net/npm/alpinejs@3.x.x/dist/cdn.min.js"></script>
</head>
<body>
<script src="https://webrtc.github.io/adapter/adapter-latest.js"></script>
<div x-data>
<button @click="$store.app.click()">test</button>
</div>
<div class="button-group">
<button id="btn-start" type="button" class="button">Start Streaming</button>
</div>
<!-- Video Element & Canvas -->
<div class="play-area">
<div class="play-area-sub">
<h3>The Stream</h3>
<video id="stream" width="320" height="240"></video>
</div>
<div class="play-area-sub">
<h3>The Capture</h3>
<canvas id="capture" width="1280" height="1024"></canvas>
<div id="snapshot"></div>
</div>
</div>
<script>
async function startStreaming() {
var mediaSupport = 'mediaDevices' in navigator;
if( mediaSupport && null == cameraStream ) {
mediaStream = await navigator.mediaDevices.getUserMedia( { video: { facingMode: { exact: "environment" } }} )
cameraStream = mediaStream;
stream.srcObject = mediaStream;
stream.play();
}
else {
alert( 'Your browser does not support media devices.' );
return;
}
}
document.addEventListener(
'alpine:init', () => {
Alpine.store(
'app',
{
host: "http://192.168.1.94:9999",
first: true,
click() {
alert("clicked")
},
async captureSnapshot() {
if( null != cameraStream ) {
var ctx = capture.getContext( '2d' );
var img = new Image();
ctx.drawImage(stream, 0, 0, capture.width, capture.height);
var content = capture.toDataURL( "image/png" );
img.src = content
await fetch(
this.host,
{
method: "post",
body: JSON.stringify({
img: content,
})
}
)
img.width = 240;
snapshot.innerHTML = '';
snapshot.appendChild( img );
}
},
async init() {
await startStreaming()
setInterval(() => {this.captureSnapshot()}, 1000)
}
}
)
}
)
var btnStart = document.getElementById( "btn-start" );
// The stream & capture
var stream = document.getElementById( "stream" );
var capture = document.getElementById( "capture" );
var snapshot = document.getElementById( "snapshot" );
// The video stream
var cameraStream = null;
// Attach listeners
btnStart.addEventListener( "click", startStreaming );
// Start Streaming
</script>
</body>
</html>
peerjs + servdroid -> going one step further in term of UX
webrtc is mostly known to be a peer-to-peer technology. So far, we only used its capacity yo take pictures, but we could actually make the controller see the video stream in real time. Only getUserMedia needs https or localhost, so the other device should work fine over http, as long as it simply gets and show the stream.
So, in short:
- one device with servdroid get the video stream,
- connection with the other device over peerjs, using a local signaling server provided by peerjs,
- streaming the video realtime from a device to the other
- discussion with the server as per webrtc + servdroid + flask
This would make the UX nicer, has the video won’t give the feeling that it lags because it is refreshed every second.
code that runs on the camera
<!DOCTYPE html>
<html>
<head>
<script defer src="https://konubinix.eu/ipfs/bafkreic33rowgvvugzgzajagkuwidfnit2dyqyn465iygfs67agsukk24i?orig=https://cdn.jsdelivr.net/npm/alpinejs@3.x.x/dist/cdn.min.js"></script>
<script src="https://unpkg.com/peerjs@1.5.2/dist/peerjs.min.js"></script>
<script>
document.addEventListener('alpine:init', () => {
Alpine.data('app', () => ({
peer: null,
peerid: null,
mediaStream: null,
message: "",
wakelock: null,
current_call: null,
async init() {
this.peer = new Peer("camera", {host: "192.168.1.46", port: 9999, path: "/peerjs/", config: {
iceServers: [],
sdpSemantics: "unified-plan",
}})
this.peer.on('open', (id) => {
this.message += `connected to peerjs with id ${id}`
this.peerid = id
});
this.peer.on('connection', async (conn) => {
this.message += "connected to the controller"
conn.on('data', async (data) => {
this.message += `got ${data}, calling back`
await this.call()
});
conn.send("hello")
});
if('mediaDevices' in navigator) {
this.mediaStream = await navigator.mediaDevices.getUserMedia( { video:
// true
{ facingMode: { exact: "environment" } }
} )
}
else {
alert( 'Your browser does not support media devices.' );
}
navigator.wakeLock.request("screen").then(
(w) => {
this.wakelock = w;
this.message += "wakelock acquired"
}
).catch((err) => {
alert(`${err.name}, ${err.message}`);
});
this.peer.on('call', (call) => {
this.message += "in call"
call.answer(this.mediaStream);
call.on('stream', (stream) => {
this.message += "got stream??"
});
});
},
async call() {
this.current_call = this.peer.call("control", this.mediaStream)
},
async connect() {
this.peer.connect("control")
},
async stop () {
this.current_call.close()
if (this.wakelock != null) {
this.wakelock.release().then(() => {
this.wakelock = null;
// alert("released");
}).catch((err) => {
alert(`${err.name}, ${err.message}`);
});
}
},
}))
})
</script>
</head>
<body>
<div x-data="app">
<div x-text="peerid"></div>
<button @click="connect">Connect</button>
<button @click="call">Call</button>
<div x-text="message"></div>
</div>
</body>
</html>
code that runs on the controller
<!DOCTYPE html>
<html>
<head>
<meta name="viewport" content="width=device-width, initial-scale=1.0">
<link rel="manifest" href="./manifest.json">
<script src="https://konubinix.eu/ipfs/bafybeihp5kzlgqt56dmy5l4z7kpymfc4kn3fnehrrtr7cid7cn7ra36yha?orig=https://cdn.tailwindcss.com/3.4.3"></script>
<script defer src="https://konubinix.eu/ipfs/bafkreic33rowgvvugzgzajagkuwidfnit2dyqyn465iygfs67agsukk24i?orig=https://cdn.jsdelivr.net/npm/alpinejs@3.x.x/dist/cdn.min.js"></script>
<script src="https://unpkg.com/peerjs@1.5.2/dist/peerjs.min.js"></script>
<script>
document.addEventListener('alpine:init', () => {
Alpine.data('app', () => ({
peer: null,
peerid: null,
conn: null,
video: null,
message: "",
names: [],
wakelock: null,
btnClass: "text-white bg-blue-700 hover:bg-blue-800 focus:ring-4 focus:ring-blue-300 font-medium rounded-lg text-sm px-5 py-2.5 me-2 mb-2 dark:bg-blue-600 dark:hover:bg-blue-700 focus:outline-none dark:focus:ring-blue-800",
btnClassAnim: "text-white bg-green-700 hover:bg-green-800 focus:ring-4 focus:ring-green-300 font-medium rounded-lg text-sm px-5 py-2.5 me-2 mb-2 dark:bg-green-600 dark:hover:bg-green-700 focus:outline-none dark:focus:ring-green-800",
async init() {
this.video= document.getElementById('videoElement')
this.peer = new Peer("control", {host: "192.168.1.46", port: 9999, path: "/peerjs/", config: {
iceServers: [],
sdpSemantics: "unified-plan",
}})
this.peer.on('open', (id) => {
this.message += `connected to peerjs with id ${id}`
this.peerid = id
});
document.body.requestFullScreen()
this.peer.on('call', (call) => {
this.message += "in call"
call.answer();
call.on('stream', (stream) => {
this.message += "getting stream"
this.video.srcObject = stream;
});
});
const resp = await fetch("http://192.168.1.94:9999/list")
const text = await resp.text()
this.names = JSON.parse(text)
setTimeout(async () => {
await this.connect()
}, 1000)
navigator.wakeLock.request("screen").then(
(w) => {
this.wakelock = w;
this.message += "wakelock acquired"
}
).catch((err) => {
alert(`${err.name}, ${err.message}`);
});
},
async call() {
this.peer.call("camera")
},
async connect() {
try {
document.getElementById('app').requestFullscreen()
} catch(err) {
this.message += err.toString()
}
this.conn = this.peer.connect("camera")
this.conn.on("open", () => {
this.message += "connected to the camera"
this.conn.send("hello")
})
},
async capture() {
var canvas = document.createElement('canvas');
canvas.width = this.video.videoWidth;
canvas.height = this.video.videoHeight;
var context = canvas.getContext('2d');
context.drawImage(this.video, 0, 0, canvas.width, canvas.height);
var content = canvas.toDataURL('image/png');
let resp
try {
resp = await fetch(
"http://192.168.1.94:9999",
{
method: "post",
body: JSON.stringify({
img: content,
})
}
)
} catch(error) {
msg = `${error} : ${JSON.stringify(error)}`
alert(msg)
}
const name = await resp.text()
this.names = this.names.concat(name)
setTimeout(() => {
document.getElementById("photos").scrollLeft += document.getElementById("photos").scrollWidth //10000
}, 500)
},
async edit(name) {
names = this.names
let index = names.indexOf(name);
if (index !== -1) {
names.splice(index, 1);
}
this.names = names
try{
const resp = await fetch(
"http://192.168.1.94:9999/img/" + name + ".png",
{
method: "delete",
}
)
} catch(error) {
msg = `${error} : ${JSON.stringify(error)}`
alert(msg)
}
},
}))
})
</script>
</head>
<body>
<div x-data="app" id="app">
<div x-text="peerid"></div>
<div>
<button type="button" :class="btnClass" @click="connect">Connect</button>
<button type="button" :class="btnClass" @click="call">Call</button>
</div>
<div x-text="message"></div>
<video id="videoElement" width="640" height="480" autoplay></video>
<div class="mb-6" x-data="{anim: false}">
<button type="button" :class="anim ? btnClassAnim : btnClass" @click.throttle.500ms="anim = true; setTimeout(() => {anim = false},500)" @click="capture">capture</button>
</div>
<div class="flex justify-center h-32">
<div class="list-none flex overflow-x-auto" id="photos">
<template x-for="name in names">
<img class="m-1 rounded border-2 block" @dblclick="edit(name)" :src="'http://192.168.1.94:9999/img/' + name + '.png'"/>
</template>
</div>
</div>
</div>
</body>
</html>
code that runs on the server
mkdir -p ~/test/stopmotion
cd ~/test/stopmotion
pdm init --non-interactive
pdm add flask flask-cors watchdog
Adding packages to default dependencies: flask, flask-cors, watchdog
STATUS: Resolving dependencies
STATUS: Resolving: new pin python>=3.11,<3.12
STATUS: Resolving: new pin flask 3.0.1
STATUS: Resolving: new pin flask-cors 4.0.0
STATUS: Resolving: new pin watchdog 3.0.0
STATUS: Resolving: new pin blinker 1.7.0
STATUS: Resolving: new pin click 8.1.7
STATUS: Resolving: new pin itsdangerous 2.1.2
STATUS: Resolving: new pin jinja2 3.1.3
STATUS: Resolving: new pin werkzeug 3.0.1
STATUS: Resolving: new pin markupsafe 2.1.5
STATUS: Resolving: new pin colorama 0.4.6
STATUS: Fetching hashes for resolved packages...
🔒 Lock successful
Changes are written to pyproject.toml.
STATUS: Resolving packages from lockfile...
Synchronizing working set with resolved packages: 1 to add, 0 to update, 0 to remove
✔ Install watchdog 3.0.0 successful
🎉 All complete!
#!/usr/bin/env python3
# -*- coding:utf-8 -*-
import base64
import json
import os
import shutil
from datetime import datetime
from pathlib import Path
from shlex import split
from subprocess import check_call
from flask import Flask, make_response, redirect, render_template, request
from flask_cors import CORS
app = Flask(__name__)
CORS(app)
root = Path("imgs")
thumb = Path("thumbs")
if not thumb.exists():
os.makedirs(thumb)
def paths(name):
return root / (name + ".png"), thumb / (name + ".png")
@app.route("/", methods=["POST"])
def add():
data = json.loads(request.data.decode())
now = datetime.now()
name = now.isoformat()
img = base64.b64decode(
data["img"]
[len("data:image/png;base64,"):])
p, t = paths(name)
p.write_bytes(img)
print(datetime.now())
check_call(["convert", str(p), "-scale", "256", str(t)])
return name
@app.route("/list")
def _list():
return json.dumps([
str(child)[:-len(".png")] for child in sorted(root.iterdir())
if str(child).endswith(".png")
])
@app.route("/video")
def video():
check_call(["clk", "images", "timelapse", "--fps", "3", "--ouptut", "output.webm"])
response = make_response(t.read_bytes())
response.headers.set('Content-Type', 'application/x-matroska')
return response
@app.route("/img/<name>.png", methods=["GET", "DELETE"])
def img(name):
if request.method == "GET":
p, t = paths(name)
response = make_response(t.read_bytes())
response.headers.set('Content-Type', 'image/png')
return response
elif request.method == "DELETE":
p, t = paths(name)
p.unlink()
t.unlink()
return "ok"
if __name__ == "__main__":
app.run(host="0.0.0.0", port=9999, debug=True, threaded=True)