Merge pull request #95 from Dasfaust/hand_tracking

Pose visualization

.github/workflows/pose_vis.yml

@@ -0,0 +1,21 @@
+on: [push]
+jobs:
+ build:
+ runs-on: ubuntu-latest
+ strategy:
+ matrix:
+ python-version: [3.8]
+ steps:
+ - uses: actions/checkout@v2
+ - name: Python setup ${{ matrix.python-version }}
+ uses: actions/setup-python@v2
+ with:
+ python-version: ${{ matrix.python-version }}
+ - name: Install PoseVis
+ run: |
+ cd devices/webcam
+ python setup.py install
+ - name: Run PoseVis tests
+ run: |
+ cd devices/webcam
+ python -m unittest pose_vis/test/tests.py

.vscode/settings.json

@@ -0,0 +1,8 @@
+{
+ "buck.projectConfig": {
+ "projectPath": "c:\\Users\\das\\Desktop\\labgraph\\manylinux.buckconfig",
+ "flavor": [],
+ "target": "",
+ "platform": "linux-x86_64"
+ }
+}

devices/webcam/__init__.py

@@ -0,0 +1,2 @@
+#!/usr/bin/env python3
+# Copyright (c) Meta Platforms, Inc. and affiliates.

devices/webcam/benchmark.ipynb

@@ -0,0 +1,361 @@
+{
+ "cells": [
+ {
+ "cell_type": "markdown",
+ "id": "5a180c32",
+ "metadata": {},
+ "source": [
+ "## PoseVis Data Quality\n",
+ "\n",
+ "Runs a series of benchmarks and reports data quality metrics such as dropped frame percentage, latency, and jitter."
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "d73eadd6",
+ "metadata": {},
+ "source": [
+ "### Install py-cpuinfo for System Info"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 19,
+ "id": "470df640",
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "Collecting py-cpuinfo\n",
+ " Downloading py_cpuinfo-9.0.0-py3-none-any.whl (22 kB)\n",
+ "Installing collected packages: py-cpuinfo\n",
+ "Successfully installed py-cpuinfo-9.0.0\n"
+ ]
+ },
+ {
+ "name": "stderr",
+ "output_type": "stream",
+ "text": [
+ "\n",
+ "[notice] A new release of pip available: 22.2.2 -> 22.3\n",
+ "[notice] To update, run: python.exe -m pip install --upgrade pip\n"
+ ]
+ }
+ ],
+ "source": [
+ "!pip install py-cpuinfo"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 22,
+ "id": "c966bc06",
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "Python version: 3.10.8.final.0 (64 bit)\n",
+ "CPU: Intel(R) Core(TM) i7-6700K CPU @ 4.00GHz\n"
+ ]
+ }
+ ],
+ "source": [
+ "import cpuinfo\n",
+ "\n",
+ "info = cpuinfo.get_cpu_info()\n",
+ "print(f\"Python version: {info['python_version']}\")\n",
+ "print(f\"CPU: {info['brand_raw']}\")"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "2e33b67a",
+ "metadata": {},
+ "source": [
+ "### Run Benchmarks\n",
+ "\n",
+ "Devices tested are: (0) Logitech C270 Webcam, and (1) VUPUMER Webcam"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 12,
+ "id": "555aadb1",
+ "metadata": {
+ "scrolled": true
+ },
+ "outputs": [
+ {
+ "name": "stderr",
+ "output_type": "stream",
+ "text": [
+ "INFO:pose_vis.runner: building graph\n",
+ "INFO:pose_vis.runner: logging directory is c:\\Users\\das\\Desktop\\labgraph\\devices\\webcam\\logs\n",
+ "INFO:pose_vis.runners.benchmark_runner: benchmark output path is c:\\Users\\das\\Desktop\\labgraph\\devices\\webcam\\logs\n",
+ "INFO:pose_vis.runner: running graph\n"
+ ]
+ },
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "### Running benchmark: benchmark_1_sources_1280x720x30\n"
+ ]
+ },
+ {
+ "name": "stderr",
+ "output_type": "stream",
+ "text": [
+ "INFO:pose_vis.runner: building graph\n",
+ "INFO:pose_vis.runner: enabling extension: HandsExtension\n",
+ "INFO:pose_vis.runner: logging directory is c:\\Users\\das\\Desktop\\labgraph\\devices\\webcam\\logs\n",
+ "INFO:pose_vis.runners.benchmark_runner: benchmark output path is c:\\Users\\das\\Desktop\\labgraph\\devices\\webcam\\logs\n",
+ "INFO:pose_vis.runner: running graph\n"
+ ]
+ },
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "### Benchmark complete: benchmark_1_sources_1280x720x30\n",
+ "### Running benchmark: benchmark_1_sources_1280x720x30_HandsExtension\n"
+ ]
+ },
+ {
+ "name": "stderr",
+ "output_type": "stream",
+ "text": [
+ "INFO:pose_vis.runner: building graph\n",
+ "INFO:pose_vis.runner: logging directory is c:\\Users\\das\\Desktop\\labgraph\\devices\\webcam\\logs\n",
+ "INFO:pose_vis.runners.benchmark_runner: benchmark output path is c:\\Users\\das\\Desktop\\labgraph\\devices\\webcam\\logs\n",
+ "INFO:pose_vis.runner: running graph\n"
+ ]
+ },
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "### Benchmark complete: benchmark_1_sources_1280x720x30_HandsExtension\n",
+ "### Running benchmark: benchmark_2_sources_1280x720x30\n"
+ ]
+ },
+ {
+ "name": "stderr",
+ "output_type": "stream",
+ "text": [
+ "INFO:pose_vis.runner: building graph\n",
+ "INFO:pose_vis.runner: enabling extension: HandsExtension\n",
+ "INFO:pose_vis.runner: logging directory is c:\\Users\\das\\Desktop\\labgraph\\devices\\webcam\\logs\n",
+ "INFO:pose_vis.runners.benchmark_runner: benchmark output path is c:\\Users\\das\\Desktop\\labgraph\\devices\\webcam\\logs\n",
+ "INFO:pose_vis.runner: running graph\n"
+ ]
+ },
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "### Benchmark complete: benchmark_2_sources_1280x720x30\n",
+ "### Running benchmark: benchmark_2_sources_1280x720x30_HandsExtension\n",
+ "### Benchmark complete: benchmark_2_sources_1280x720x30_HandsExtension\n"
+ ]
+ }
+ ],
+ "source": [
+ "import os\n",
+ "import pose_vis.pose_vis\n",
+ "\n",
+ "from dataclasses import dataclass\n",
+ "from typing import List, Tuple\n",
+ "from pose_vis.extension import PoseVisExtension\n",
+ "from pose_vis.runner import PoseVisConfig\n",
+ "from pose_vis.runners.benchmark_runner import BenchmarkRunner, BenchmarkRunnerConfig\n",
+ "from pose_vis.extensions.hands import HandsExtension\n",
+ "\n",
+ "@dataclass\n",
+ "class BenchmarkConfig():\n",
+ " extensions: List[PoseVisExtension]\n",
+ " logging: bool\n",
+ " sources: List[int]\n",
+ " resolution: Tuple[int, int, int]\n",
+ " runtime: int\n",
+ "\n",
+ "benchmarks = [\n",
+ " # Single camera, 1280x720x30\n",
+ " BenchmarkConfig([], False, [0], (1280, 720, 30), 60),\n",
+ " # Single camera, 1280x720x30, hand tracking\n",
+ " BenchmarkConfig([HandsExtension()], False, [0], (1280, 720, 30), 60),\n",
+ " # Two cameras, 1280x720x30\n",
+ " BenchmarkConfig([], False, [0, 1], (1280, 720, 30), 60),\n",
+ " # Two cameras, 1280x720x30, hand tracking\n",
+ " BenchmarkConfig([HandsExtension()], False, [0, 1], (1280, 720, 30), 60),\n",
+ "]\n",
+ "\n",
+ "for benchmark in benchmarks:\n",
+ " config = PoseVisConfig(\n",
+ " extensions = benchmark.extensions,\n",
+ " log_directory = f\"webcam{os.sep}logs\",\n",
+ " log_name = \"benchmark\",\n",
+ " enable_logging = benchmark.logging,\n",
+ " display_framerate = 0,\n",
+ " stats_history_size = 0)\n",
+ "\n",
+ " resolutions = [benchmark.resolution for _ in range(len(benchmark.sources))]\n",
+ " output_name = f\"benchmark_{len(benchmark.sources)}_sources_{resolutions[0][0]}x{resolutions[0][1]}x{resolutions[0][2]}\"\n",
+ "\n",
+ " if len(benchmark.extensions) > 0:\n",
+ " ext_names = \"\"\n",
+ " for i in range(len(benchmark.extensions)):\n",
+ " sep = \"_\" if i > 0 else \"\"\n",
+ " ext_names += f\"{sep}{benchmark.extensions[i].__class__.__name__}\"\n",
+ " output_name = f\"{output_name}_{ext_names}\"\n",
+ " if benchmark.logging:\n",
+ " output_name = f\"{output_name}_logging\"\n",
+ " \n",
+ " runner_config = BenchmarkRunnerConfig(\n",
+ " sources = benchmark.sources,\n",
+ " resolutions = resolutions,\n",
+ " output_path = f\"webcam{os.sep}logs\",\n",
+ " output_name = output_name,\n",
+ " run_time = 60)\n",
+ " runner = BenchmarkRunner(config, runner_config)\n",
+ "\n",
+ " print(f\"### Running benchmark: {output_name}\")\n",
+ " runner.build()\n",
+ " runner.run()\n",
+ " print(f\"### Benchmark complete: {output_name}\")"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "3ff8a0ea",
+ "metadata": {},
+ "source": [
+ "### Display Results"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 13,
+ "id": "dc52cb95",
+ "metadata": {
+ "scrolled": true
+ },
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "## Benchmark: benchmark_1_sources_1280x720x30\n",
+ "# runtime: 60.55s\n",
+ "# dropped: 0.75%\n",
+ "# latency: 16.69ms\n",
+ "# jitter: 5.04ms\n",
+ "# [desync]\n",
+ " \n",
+ "## Benchmark: benchmark_1_sources_1280x720x30_HandsExtension\n",
+ "# runtime: 60.45s\n",
+ "# dropped: 0.64%\n",
+ "# latency: 5.17ms\n",
+ "# jitter: 4.14ms\n",
+ "# [desync]\n",
+ " \n",
+ "## Benchmark: benchmark_2_sources_1280x720x30\n",
+ "# runtime: 60.42s\n",
+ "# dropped: 0.64%\n",
+ "# latency: 13.33ms\n",
+ "# jitter: 11.09ms\n",
+ "# [desync]\n",
+ "# from source 0 to source 1: 2.43ms, jitter: 11.02ms\n",
+ " \n",
+ "## Benchmark: benchmark_2_sources_1280x720x30_HandsExtension\n",
+ "# runtime: 60.46s\n",
+ "# dropped: 28.10%\n",
+ "# latency: 4.94ms\n",
+ "# jitter: 5.80ms\n",
+ "# [desync]\n",
+ "# from source 0 to source 1: 0.11ms, jitter: 6.10ms\n",
+ " \n"
+ ]
+ }
+ ],
+ "source": [
+ "import json\n",
+ "import statistics\n",
+ "from pose_vis.utils import absolute_path\n",
+ "from typing import Dict, Union, List, Tuple\n",
+ "\n",
+ "json_files = [_file for _file in os.listdir(absolute_path(f\"webcam{os.sep}logs\")) if _file.endswith(\".json\")]\n",
+ "\n",
+ "for filename in json_files:\n",
+ " timings: Dict[str, Union[float, int, List[Tuple[float, float, float]]]] = {}\n",
+ "\n",
+ " with open(absolute_path(f\"webcam{os.sep}logs{os.sep}{filename}\")) as _file:\n",
+ " timings = json.loads(_file.read())\n",
+ "\n",
+ " print(f\"## Benchmark: {filename.removesuffix('.json')}\")\n",
+ " print(f\"# runtime: {timings['runtime']:.2f}s\")\n",
+ "\n",
+ " expected_frames = timings[\"runtime\"] * timings[\"target_fps\"]\n",
+ " # Estimate of how many frames were dropped while capturing the source\n",
+ " source_received_frames_pct = timings[\"frame_index\"] / expected_frames\n",
+ " print(f\"# dropped: {(100 - (source_received_frames_pct * 100)):.2f}%\")\n",
+ "\n",
+ " latency: List[float] = []\n",
+ " desync: List[List[float]] = []\n",
+ " # device time = system time at frame capture from device\n",
+ " # All times are captured with time.perf_counter()\n",
+ " # times: List[(device time, receive time)]\n",
+ " for i in range(1, len(timings[\"times\"])):\n",
+ " rel_device = timings[\"times\"][i][0] - timings[\"times\"][0][0]\n",
+ " rel_receive = timings[\"times\"][i][1] - timings[\"times\"][0][1]\n",
+ " # TODO: this may not be correct\n",
+ " # `rel_device` can be greater than `rel_receive`, for now we just take the absolute value\n",
+ " latency.append(abs(rel_receive - rel_device))\n",
+ "\n",
+ " if len(timings[\"sync\"]) > 0:\n",
+ " if i == 1:\n",
+ " desync = [[] for j in range(len(timings[\"sync\"][i]))]\n",
+ "\n",
+ " for j in range(len(timings[\"sync\"][i])):\n",
+ " desync[j].append(abs(timings[\"times\"][i][0] - timings[\"sync\"][i][j]))\n",
+ " \n",
+ " print(f\"# latency: {statistics.median(latency) * 1000:.2f}ms\")\n",
+ " print(f\"# jitter: {statistics.stdev(latency) * 1000:.2f}ms\")\n",
+ " print(\"# [desync]\")\n",
+ " for i, li in enumerate(desync):\n",
+ " print(f\"# from source 0 to source {i + 1}: {statistics.median(desync[i]) * 1000:.2f}ms, jitter: {statistics.stdev(desync[i]) * 1000:.2f}ms\")\n",
+ " print(\" \")"
+ ]
+ }
+ ],
+ "metadata": {
+ "kernelspec": {
+ "display_name": "Python 3.9.13 ('.venv': venv)",
+ "language": "python",
+ "name": "python3"
+ },
+ "language_info": {
+ "codemirror_mode": {
+ "name": "ipython",
+ "version": 3
+ },
+ "file_extension": ".py",
+ "mimetype": "text/x-python",
+ "name": "python",
+ "nbconvert_exporter": "python",
+ "pygments_lexer": "ipython3",
+ "version": "3.10.8"
+ },
+ "vscode": {
+ "interpreter": {
+ "hash": "23a593576959775a19d6469cad78770ba03ee1b7699646fbac2d14539ad9dcf0"
+ }
+ }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}