Csaba Bolyos
commited on
Commit
·
a31294b
1
Parent(s):
c6fb2d4
initial commit
Browse filesThis view is limited to 50 files because it contains too many changes.
See raw diff
- LICENSE +30 -0
- MCP_README.md +704 -0
- README.md +1138 -8
- app.py +48 -0
- backend/gradio_labanmovementanalysis/__init__.py +51 -0
- backend/gradio_labanmovementanalysis/__pycache__/__init__.cpython-312.pyc +0 -0
- backend/gradio_labanmovementanalysis/__pycache__/__init__.cpython-313.pyc +0 -0
- backend/gradio_labanmovementanalysis/__pycache__/agent_api.cpython-312.pyc +0 -0
- backend/gradio_labanmovementanalysis/__pycache__/json_generator.cpython-312.pyc +0 -0
- backend/gradio_labanmovementanalysis/__pycache__/labanmovementanalysis.cpython-312.pyc +0 -0
- backend/gradio_labanmovementanalysis/__pycache__/labanmovementanalysis.cpython-313.pyc +0 -0
- backend/gradio_labanmovementanalysis/__pycache__/notation_engine.cpython-312.pyc +0 -0
- backend/gradio_labanmovementanalysis/__pycache__/pose_estimation.cpython-312.pyc +0 -0
- backend/gradio_labanmovementanalysis/__pycache__/skateformer_integration.cpython-312.pyc +0 -0
- backend/gradio_labanmovementanalysis/__pycache__/video_downloader.cpython-312.pyc +0 -0
- backend/gradio_labanmovementanalysis/__pycache__/video_utils.cpython-312.pyc +0 -0
- backend/gradio_labanmovementanalysis/__pycache__/visualizer.cpython-312.pyc +0 -0
- backend/gradio_labanmovementanalysis/__pycache__/webrtc_handler.cpython-312.pyc +0 -0
- backend/gradio_labanmovementanalysis/agent_api.py +434 -0
- backend/gradio_labanmovementanalysis/json_generator.py +250 -0
- backend/gradio_labanmovementanalysis/labanmovementanalysis.py +442 -0
- backend/gradio_labanmovementanalysis/labanmovementanalysis.pyi +448 -0
- backend/gradio_labanmovementanalysis/notation_engine.py +317 -0
- backend/gradio_labanmovementanalysis/pose_estimation.py +380 -0
- backend/gradio_labanmovementanalysis/video_downloader.py +295 -0
- backend/gradio_labanmovementanalysis/video_utils.py +150 -0
- backend/gradio_labanmovementanalysis/visualizer.py +402 -0
- backend/gradio_labanmovementanalysis/webrtc_handler.py +293 -0
- backend/mcp_server.py +413 -0
- backend/requirements-mcp.txt +27 -0
- backend/requirements.txt +20 -0
- demo/__init__.py +0 -0
- demo/app.py +866 -0
- demo/css.css +157 -0
- demo/requirements.txt +1 -0
- demo/space.py +983 -0
- demo/test_component.py +69 -0
- dist/gradio_labanmovementanalysis-0.0.1-py3-none-any.whl +0 -0
- dist/gradio_labanmovementanalysis-0.0.1.tar.gz +3 -0
- dist/gradio_labanmovementanalysis-0.0.2-py3-none-any.whl +0 -0
- dist/gradio_labanmovementanalysis-0.0.2.tar.gz +3 -0
- examples/agent_example.py +142 -0
- frontend/Example.svelte +19 -0
- frontend/Index.svelte +37 -0
- frontend/gradio.config.js +9 -0
- frontend/package-lock.json +0 -0
- frontend/package.json +40 -0
- mcp.json +57 -0
- pyproject.toml +62 -0
- requirements.txt +49 -0
LICENSE
ADDED
|
@@ -0,0 +1,30 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
MIT License
|
| 2 |
+
|
| 3 |
+
Copyright (c) 2024 Csaba Bolyós (BladeSzaSza)
|
| 4 |
+
|
| 5 |
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
| 6 |
+
of this software and associated documentation files (the "Software"), to deal
|
| 7 |
+
in the Software without restriction, including without limitation the rights
|
| 8 |
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
| 9 |
+
copies of the Software, and to permit persons to whom the Software is
|
| 10 |
+
furnished to do so, subject to the following conditions:
|
| 11 |
+
|
| 12 |
+
The above copyright notice and this permission notice shall be included in all
|
| 13 |
+
copies or substantial portions of the Software.
|
| 14 |
+
|
| 15 |
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
| 16 |
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
| 17 |
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
| 18 |
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
| 19 |
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
| 20 |
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
| 21 |
+
SOFTWARE.
|
| 22 |
+
|
| 23 |
+
---
|
| 24 |
+
|
| 25 |
+
🎭 Laban Movement Analysis - Complete Suite
|
| 26 |
+
Created by: Csaba Bolyós (BladeSzaSza)
|
| 27 |
+
Contact: [email protected]
|
| 28 |
+
GitHub: https://github.com/bladeszasza
|
| 29 |
+
LinkedIn: https://www.linkedin.com/in/csaba-bolyós-00a11767/
|
| 30 |
+
Hugging Face: https://huggingface.co/BladeSzaSza
|
MCP_README.md
ADDED
|
@@ -0,0 +1,704 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# MCP & Agent Integration for Laban Movement Analysis
|
| 2 |
+
|
| 3 |
+
This project provides comprehensive MCP (Model Context Protocol) integration and agent-ready APIs for professional movement analysis with pose estimation, AI action recognition, and automation capabilities.
|
| 4 |
+
|
| 5 |
+
## 🚀 Quick Start
|
| 6 |
+
|
| 7 |
+
### 1. Install All Dependencies
|
| 8 |
+
|
| 9 |
+
```bash
|
| 10 |
+
# Clone the repository
|
| 11 |
+
git clone https://github.com/[your-repo]/labanmovementanalysis
|
| 12 |
+
cd labanmovementanalysis
|
| 13 |
+
|
| 14 |
+
# Install core dependencies
|
| 15 |
+
pip install -r backend/requirements.txt
|
| 16 |
+
|
| 17 |
+
# Install MCP and enhanced features
|
| 18 |
+
pip install -r backend/requirements-mcp.txt
|
| 19 |
+
```
|
| 20 |
+
|
| 21 |
+
### 2. Start the MCP Server
|
| 22 |
+
|
| 23 |
+
```bash
|
| 24 |
+
# Start MCP server for AI assistants
|
| 25 |
+
python -m backend.mcp_server
|
| 26 |
+
```
|
| 27 |
+
|
| 28 |
+
### 3. Configure Your AI Assistant
|
| 29 |
+
|
| 30 |
+
Add to your Claude Desktop or other MCP-compatible assistant configuration:
|
| 31 |
+
|
| 32 |
+
```json
|
| 33 |
+
{
|
| 34 |
+
"mcpServers": {
|
| 35 |
+
"laban-movement-analysis": {
|
| 36 |
+
"command": "python",
|
| 37 |
+
"args": ["-m", "backend.mcp_server"],
|
| 38 |
+
"env": {
|
| 39 |
+
"PYTHONPATH": "/path/to/labanmovementanalysis"
|
| 40 |
+
}
|
| 41 |
+
}
|
| 42 |
+
}
|
| 43 |
+
}
|
| 44 |
+
```
|
| 45 |
+
|
| 46 |
+
## 🛠️ Enhanced MCP Tools
|
| 47 |
+
|
| 48 |
+
### 1. `analyze_video`
|
| 49 |
+
Comprehensive video analysis with enhanced features including SkateFormer AI and multiple pose models.
|
| 50 |
+
|
| 51 |
+
**Parameters:**
|
| 52 |
+
- `video_path` (string): Path or URL to video (supports YouTube, Vimeo, local files)
|
| 53 |
+
- `model` (string, optional): Advanced pose model selection:
|
| 54 |
+
- **MediaPipe**: `mediapipe-lite`, `mediapipe-full`, `mediapipe-heavy`
|
| 55 |
+
- **MoveNet**: `movenet-lightning`, `movenet-thunder`
|
| 56 |
+
- **YOLO**: `yolo-v8-n/s/m/l`, `yolo-v11-n/s/m/l`
|
| 57 |
+
|
| 58 |
+
- `enable_visualization` (boolean, optional): Generate annotated video
|
| 59 |
+
- `include_keypoints` (boolean, optional): Include raw keypoint data
|
| 60 |
+
- `use_skateformer` (boolean, optional): Enable AI action recognition
|
| 61 |
+
|
| 62 |
+
**Examples:**
|
| 63 |
+
```
|
| 64 |
+
Analyze the dance video at https://youtube.com/watch?v=dQw4w9WgXcQ using SkateFormer AI
|
| 65 |
+
Analyze movement in video.mp4 using yolo-v11-s model with visualization
|
| 66 |
+
Process the exercise video with mediapipe-full and include keypoints
|
| 67 |
+
```
|
| 68 |
+
|
| 69 |
+
### 2. `get_analysis_summary`
|
| 70 |
+
Get human-readable summaries with enhanced AI insights.
|
| 71 |
+
|
| 72 |
+
**Parameters:**
|
| 73 |
+
- `analysis_id` (string): ID from previous analysis
|
| 74 |
+
|
| 75 |
+
**Enhanced Output Includes:**
|
| 76 |
+
- SkateFormer action recognition results
|
| 77 |
+
- Movement quality metrics (rhythm, complexity, symmetry)
|
| 78 |
+
- Temporal action segmentation
|
| 79 |
+
- Video source metadata (YouTube/Vimeo titles, etc.)
|
| 80 |
+
|
| 81 |
+
**Example:**
|
| 82 |
+
```
|
| 83 |
+
Get a detailed summary of analysis dance_2024-01-01T12:00:00 including AI insights
|
| 84 |
+
```
|
| 85 |
+
|
| 86 |
+
### 3. `list_available_models`
|
| 87 |
+
Comprehensive list of all 20+ pose estimation models with detailed specifications.
|
| 88 |
+
|
| 89 |
+
**Enhanced Model Information:**
|
| 90 |
+
- Performance characteristics (speed, accuracy, memory usage)
|
| 91 |
+
- Recommended use cases (real-time, research, production)
|
| 92 |
+
- Hardware requirements (CPU, GPU, memory)
|
| 93 |
+
- Keypoint specifications (17 COCO, 33 MediaPipe)
|
| 94 |
+
|
| 95 |
+
**Example:**
|
| 96 |
+
```
|
| 97 |
+
What pose estimation models are available for real-time processing?
|
| 98 |
+
List all YOLO v11 model variants with their specifications
|
| 99 |
+
```
|
| 100 |
+
|
| 101 |
+
### 4. `batch_analyze`
|
| 102 |
+
Enhanced batch processing with parallel execution and progress tracking.
|
| 103 |
+
|
| 104 |
+
**Parameters:**
|
| 105 |
+
- `video_paths` (array): List of video paths/URLs (supports mixed sources)
|
| 106 |
+
- `model` (string, optional): Pose estimation model for all videos
|
| 107 |
+
- `parallel` (boolean, optional): Enable parallel processing
|
| 108 |
+
- `use_skateformer` (boolean, optional): Enable AI analysis for all videos
|
| 109 |
+
- `output_format` (string, optional): Output format ("summary", "structured", "full")
|
| 110 |
+
|
| 111 |
+
**Enhanced Features:**
|
| 112 |
+
- Mixed source support (local files + YouTube URLs)
|
| 113 |
+
- Progress tracking and partial results
|
| 114 |
+
- Resource management and optimization
|
| 115 |
+
- Failure recovery and retry logic
|
| 116 |
+
|
| 117 |
+
**Examples:**
|
| 118 |
+
```
|
| 119 |
+
Analyze all dance videos in the playlist with SkateFormer AI
|
| 120 |
+
Batch process exercise videos using yolo-v11-s with parallel execution
|
| 121 |
+
```
|
| 122 |
+
|
| 123 |
+
### 5. `compare_movements`
|
| 124 |
+
Advanced movement comparison with AI-powered insights.
|
| 125 |
+
|
| 126 |
+
**Parameters:**
|
| 127 |
+
- `analysis_id1` (string): First analysis ID
|
| 128 |
+
- `analysis_id2` (string): Second analysis ID
|
| 129 |
+
- `comparison_type` (string, optional): Type of comparison ("basic", "detailed", "ai_enhanced")
|
| 130 |
+
|
| 131 |
+
**Enhanced Comparison Features:**
|
| 132 |
+
- SkateFormer action similarity analysis
|
| 133 |
+
- Movement quality comparisons (rhythm, complexity, symmetry)
|
| 134 |
+
- Temporal pattern matching
|
| 135 |
+
- Statistical significance testing
|
| 136 |
+
|
| 137 |
+
**Example:**
|
| 138 |
+
```
|
| 139 |
+
Compare the movement patterns between the two dance analyses with AI insights
|
| 140 |
+
Detailed comparison of exercise form between beginner and expert videos
|
| 141 |
+
```
|
| 142 |
+
|
| 143 |
+
### 6. `real_time_analysis` (New)
|
| 144 |
+
Start/stop real-time WebRTC analysis.
|
| 145 |
+
|
| 146 |
+
**Parameters:**
|
| 147 |
+
- `action` (string): "start" or "stop"
|
| 148 |
+
- `model` (string, optional): Real-time optimized model
|
| 149 |
+
- `stream_config` (object, optional): WebRTC configuration
|
| 150 |
+
|
| 151 |
+
**Example:**
|
| 152 |
+
```
|
| 153 |
+
Start real-time movement analysis using mediapipe-lite
|
| 154 |
+
```
|
| 155 |
+
|
| 156 |
+
### 7. `filter_videos_advanced` (New)
|
| 157 |
+
Advanced video filtering with AI-powered criteria.
|
| 158 |
+
|
| 159 |
+
**Parameters:**
|
| 160 |
+
- `video_paths` (array): List of video paths/URLs
|
| 161 |
+
- `criteria` (object): Enhanced filtering criteria including:
|
| 162 |
+
- Traditional LMA metrics (direction, intensity, fluidity)
|
| 163 |
+
- SkateFormer actions (dancing, jumping, etc.)
|
| 164 |
+
- Movement qualities (rhythm, complexity, symmetry)
|
| 165 |
+
- Temporal characteristics (duration, segment count)
|
| 166 |
+
|
| 167 |
+
**Example:**
|
| 168 |
+
```
|
| 169 |
+
Filter videos for high-energy dance movements with good rhythm
|
| 170 |
+
Find exercise videos with proper form (high fluidity and symmetry)
|
| 171 |
+
```
|
| 172 |
+
|
| 173 |
+
## 🤖 Enhanced Agent API
|
| 174 |
+
|
| 175 |
+
### Comprehensive Python Agent API
|
| 176 |
+
|
| 177 |
+
```python
|
| 178 |
+
from gradio_labanmovementanalysis import LabanMovementAnalysis
|
| 179 |
+
from gradio_labanmovementanalysis.agent_api import (
|
| 180 |
+
LabanAgentAPI,
|
| 181 |
+
PoseModel,
|
| 182 |
+
MovementDirection,
|
| 183 |
+
MovementIntensity,
|
| 184 |
+
analyze_and_summarize
|
| 185 |
+
)
|
| 186 |
+
|
| 187 |
+
# Initialize with all features enabled
|
| 188 |
+
analyzer = LabanMovementAnalysis(
|
| 189 |
+
enable_skateformer=True,
|
| 190 |
+
enable_webrtc=True,
|
| 191 |
+
enable_visualization=True
|
| 192 |
+
)
|
| 193 |
+
|
| 194 |
+
agent_api = LabanAgentAPI(analyzer=analyzer)
|
| 195 |
+
```
|
| 196 |
+
|
| 197 |
+
### Advanced Analysis Workflows
|
| 198 |
+
|
| 199 |
+
```python
|
| 200 |
+
# YouTube video analysis with AI
|
| 201 |
+
result = agent_api.analyze(
|
| 202 |
+
"https://youtube.com/watch?v=...",
|
| 203 |
+
model=PoseModel.YOLO_V11_S,
|
| 204 |
+
use_skateformer=True,
|
| 205 |
+
generate_visualization=True
|
| 206 |
+
)
|
| 207 |
+
|
| 208 |
+
# Enhanced batch processing
|
| 209 |
+
results = agent_api.batch_analyze(
|
| 210 |
+
["video1.mp4", "https://youtube.com/watch?v=...", "https://vimeo.com/..."],
|
| 211 |
+
model=PoseModel.YOLO_V11_S,
|
| 212 |
+
parallel=True,
|
| 213 |
+
use_skateformer=True
|
| 214 |
+
)
|
| 215 |
+
|
| 216 |
+
# AI-powered movement filtering
|
| 217 |
+
filtered = agent_api.filter_by_movement_advanced(
|
| 218 |
+
video_paths,
|
| 219 |
+
skateformer_actions=["dancing", "jumping"],
|
| 220 |
+
movement_qualities={"rhythm": 0.8, "complexity": 0.6},
|
| 221 |
+
traditional_criteria={
|
| 222 |
+
"direction": MovementDirection.UP,
|
| 223 |
+
"intensity": MovementIntensity.HIGH,
|
| 224 |
+
"min_fluidity": 0.7
|
| 225 |
+
}
|
| 226 |
+
)
|
| 227 |
+
|
| 228 |
+
# Real-time analysis control
|
| 229 |
+
agent_api.start_realtime_analysis(model=PoseModel.MEDIAPIPE_LITE)
|
| 230 |
+
live_metrics = agent_api.get_realtime_metrics()
|
| 231 |
+
agent_api.stop_realtime_analysis()
|
| 232 |
+
```
|
| 233 |
+
|
| 234 |
+
### Enhanced Quick Functions
|
| 235 |
+
|
| 236 |
+
```python
|
| 237 |
+
from gradio_labanmovementanalysis import (
|
| 238 |
+
quick_analyze_enhanced,
|
| 239 |
+
analyze_and_summarize_with_ai,
|
| 240 |
+
compare_videos_detailed
|
| 241 |
+
)
|
| 242 |
+
|
| 243 |
+
# Enhanced analysis with AI
|
| 244 |
+
data = quick_analyze_enhanced(
|
| 245 |
+
"https://youtube.com/watch?v=...",
|
| 246 |
+
model="yolo-v11-s",
|
| 247 |
+
use_skateformer=True
|
| 248 |
+
)
|
| 249 |
+
|
| 250 |
+
# AI-powered summary
|
| 251 |
+
summary = analyze_and_summarize_with_ai(
|
| 252 |
+
"dance_video.mp4",
|
| 253 |
+
include_skateformer=True,
|
| 254 |
+
detail_level="comprehensive"
|
| 255 |
+
)
|
| 256 |
+
|
| 257 |
+
# Detailed video comparison
|
| 258 |
+
comparison = compare_videos_detailed(
|
| 259 |
+
"video1.mp4",
|
| 260 |
+
"video2.mp4",
|
| 261 |
+
include_ai_analysis=True
|
| 262 |
+
)
|
| 263 |
+
```
|
| 264 |
+
|
| 265 |
+
## 🌐 Enhanced Gradio 5 Agent Features
|
| 266 |
+
|
| 267 |
+
### Comprehensive API Endpoints
|
| 268 |
+
|
| 269 |
+
The unified Gradio 5 app exposes these endpoints optimized for agents:
|
| 270 |
+
|
| 271 |
+
1. **`/analyze_standard`** - Basic LMA analysis
|
| 272 |
+
2. **`/analyze_enhanced`** - Advanced analysis with all features
|
| 273 |
+
3. **`/analyze_agent`** - Agent-optimized structured output
|
| 274 |
+
4. **`/batch_analyze`** - Efficient multiple video processing
|
| 275 |
+
5. **`/filter_videos`** - Movement-based filtering
|
| 276 |
+
6. **`/compare_models`** - Model performance comparison
|
| 277 |
+
7. **`/real_time_start`** - Start WebRTC real-time analysis
|
| 278 |
+
8. **`/real_time_stop`** - Stop WebRTC real-time analysis
|
| 279 |
+
|
| 280 |
+
### Enhanced Gradio Client Usage
|
| 281 |
+
|
| 282 |
+
```python
|
| 283 |
+
from gradio_client import Client
|
| 284 |
+
|
| 285 |
+
# Connect to unified demo
|
| 286 |
+
client = Client("http://localhost:7860")
|
| 287 |
+
|
| 288 |
+
# Enhanced single analysis
|
| 289 |
+
result = client.predict(
|
| 290 |
+
video_input="https://youtube.com/watch?v=...",
|
| 291 |
+
model="yolo-v11-s",
|
| 292 |
+
enable_viz=True,
|
| 293 |
+
use_skateformer=True,
|
| 294 |
+
include_keypoints=False,
|
| 295 |
+
api_name="/analyze_enhanced"
|
| 296 |
+
)
|
| 297 |
+
|
| 298 |
+
# Agent-optimized batch processing
|
| 299 |
+
batch_results = client.predict(
|
| 300 |
+
files=["video1.mp4", "video2.mp4"],
|
| 301 |
+
model="yolo-v11-s",
|
| 302 |
+
api_name="/batch_analyze"
|
| 303 |
+
)
|
| 304 |
+
|
| 305 |
+
# Advanced movement filtering
|
| 306 |
+
filtered_results = client.predict(
|
| 307 |
+
files=video_list,
|
| 308 |
+
direction_filter="up",
|
| 309 |
+
intensity_filter="high",
|
| 310 |
+
fluidity_threshold=0.7,
|
| 311 |
+
expansion_threshold=0.5,
|
| 312 |
+
api_name="/filter_videos"
|
| 313 |
+
)
|
| 314 |
+
|
| 315 |
+
# Model comparison analysis
|
| 316 |
+
comparison = client.predict(
|
| 317 |
+
video="test_video.mp4",
|
| 318 |
+
model1="mediapipe-full",
|
| 319 |
+
model2="yolo-v11-s",
|
| 320 |
+
api_name="/compare_models"
|
| 321 |
+
)
|
| 322 |
+
```
|
| 323 |
+
|
| 324 |
+
## 📊 Enhanced Output Formats
|
| 325 |
+
|
| 326 |
+
### AI-Enhanced Summary Format
|
| 327 |
+
```
|
| 328 |
+
🎭 Movement Analysis Summary for "Dance Performance"
|
| 329 |
+
Source: YouTube (10.5 seconds, 30fps)
|
| 330 |
+
Model: YOLO-v11-S with SkateFormer AI
|
| 331 |
+
|
| 332 |
+
📊 Traditional LMA Metrics:
|
| 333 |
+
• Primary direction: up (65% of frames)
|
| 334 |
+
• Movement intensity: high (80% of frames)
|
| 335 |
+
• Average speed: fast (2.3 units/frame)
|
| 336 |
+
• Fluidity score: 0.85/1.00 (very smooth)
|
| 337 |
+
• Expansion score: 0.72/1.00 (moderately extended)
|
| 338 |
+
|
| 339 |
+
🤖 SkateFormer AI Analysis:
|
| 340 |
+
• Detected actions: dancing (95% confidence), jumping (78% confidence)
|
| 341 |
+
• Movement qualities:
|
| 342 |
+
- Rhythm: 0.89/1.00 (highly rhythmic)
|
| 343 |
+
- Complexity: 0.76/1.00 (moderately complex)
|
| 344 |
+
- Symmetry: 0.68/1.00 (slightly asymmetric)
|
| 345 |
+
- Smoothness: 0.85/1.00 (very smooth)
|
| 346 |
+
- Energy: 0.88/1.00 (high energy)
|
| 347 |
+
|
| 348 |
+
���️ Temporal Analysis:
|
| 349 |
+
• 7 movement segments identified
|
| 350 |
+
• Average segment duration: 1.5 seconds
|
| 351 |
+
• Transition quality: smooth (0.82/1.00)
|
| 352 |
+
|
| 353 |
+
🎯 Overall Assessment: Excellent dance performance with high energy,
|
| 354 |
+
good rhythm, and smooth transitions. Slightly asymmetric but shows
|
| 355 |
+
advanced movement complexity.
|
| 356 |
+
```
|
| 357 |
+
|
| 358 |
+
### Enhanced Structured Format
|
| 359 |
+
```json
|
| 360 |
+
{
|
| 361 |
+
"success": true,
|
| 362 |
+
"video_metadata": {
|
| 363 |
+
"source": "youtube",
|
| 364 |
+
"title": "Dance Performance",
|
| 365 |
+
"duration": 10.5,
|
| 366 |
+
"platform_id": "dQw4w9WgXcQ"
|
| 367 |
+
},
|
| 368 |
+
"model_info": {
|
| 369 |
+
"pose_model": "yolo-v11-s",
|
| 370 |
+
"ai_enhanced": true,
|
| 371 |
+
"skateformer_enabled": true
|
| 372 |
+
},
|
| 373 |
+
"lma_metrics": {
|
| 374 |
+
"direction": "up",
|
| 375 |
+
"intensity": "high",
|
| 376 |
+
"speed": "fast",
|
| 377 |
+
"fluidity": 0.85,
|
| 378 |
+
"expansion": 0.72
|
| 379 |
+
},
|
| 380 |
+
"skateformer_analysis": {
|
| 381 |
+
"actions": [
|
| 382 |
+
{"type": "dancing", "confidence": 0.95, "duration": 8.2},
|
| 383 |
+
{"type": "jumping", "confidence": 0.78, "duration": 2.3}
|
| 384 |
+
],
|
| 385 |
+
"movement_qualities": {
|
| 386 |
+
"rhythm": 0.89,
|
| 387 |
+
"complexity": 0.76,
|
| 388 |
+
"symmetry": 0.68,
|
| 389 |
+
"smoothness": 0.85,
|
| 390 |
+
"energy": 0.88
|
| 391 |
+
},
|
| 392 |
+
"temporal_segments": 7,
|
| 393 |
+
"transition_quality": 0.82
|
| 394 |
+
},
|
| 395 |
+
"performance_metrics": {
|
| 396 |
+
"processing_time": 12.3,
|
| 397 |
+
"frames_analyzed": 315,
|
| 398 |
+
"keypoints_detected": 24
|
| 399 |
+
}
|
| 400 |
+
}
|
| 401 |
+
```
|
| 402 |
+
|
| 403 |
+
### Comprehensive JSON Format
|
| 404 |
+
Complete analysis including frame-by-frame data, SkateFormer attention maps, movement trajectories, and statistical summaries.
|
| 405 |
+
|
| 406 |
+
## 🏗️ Enhanced Architecture
|
| 407 |
+
|
| 408 |
+
```
|
| 409 |
+
┌─────────────────────────────────────────────────────────────┐
|
| 410 |
+
│ AI Assistant Integration │
|
| 411 |
+
│ (Claude, GPT, Local Models via MCP) │
|
| 412 |
+
└─────────────────────┬───────────────────────────────────────┘
|
| 413 |
+
│
|
| 414 |
+
┌─────────────────────▼───────────────────────────────────────┐
|
| 415 |
+
│ MCP Server │
|
| 416 |
+
│ ┌─────────────┐ ┌─────────────┐ ┌─────────────────────────┐│
|
| 417 |
+
│ │ Video │ │ Enhanced │ │ Real-time ││
|
| 418 |
+
│ │ Analysis │ │ Batch │ │ WebRTC ││
|
| 419 |
+
│ │ Tools │ │ Processing │ │ Analysis ││
|
| 420 |
+
│ └─────────────┘ └─────────────┘ └─────────────────────────┘│
|
| 421 |
+
└─────────────────────┬───────────────────────────────────────┘
|
| 422 |
+
│
|
| 423 |
+
┌─────────────────────▼───────────────────────────────────────┐
|
| 424 |
+
│ Enhanced Agent API Layer │
|
| 425 |
+
│ ┌─────────────┐ ┌─────────────┐ ┌─────────────────────────┐│
|
| 426 |
+
│ │ Movement │ │ AI-Enhanced │ │ Advanced ││
|
| 427 |
+
│ │ Filtering │ │ Comparisons │ │ Workflows ││
|
| 428 |
+
│ └─────────────┘ └─────────────┘ └─────────────────────────┘│
|
| 429 |
+
└─────────────────────┬───────────────────────────────────────┘
|
| 430 |
+
│
|
| 431 |
+
┌─────────────────────▼───────────────────────────────────────┐
|
| 432 |
+
│ Core Analysis Engine │
|
| 433 |
+
│ │
|
| 434 |
+
│ 📹 Video Input 🤖 Pose Models 🎭 SkateFormer AI │
|
| 435 |
+
│ ┌─────────────┐ ┌─────────────┐ ┌─────────────────────┐│
|
| 436 |
+
│ │Local Files │ │MediaPipe(3) │ │ Action Recognition ││
|
| 437 |
+
│ │YouTube URLs │ │MoveNet(2) │ │Movement Qualities ││
|
| 438 |
+
│ │Vimeo URLs │ │YOLO(8) │ │Temporal Segments ││
|
| 439 |
+
│ │Direct URLs │ │ │ │Attention Analysis ││
|
| 440 |
+
│ └─────────────┘ └─────────────┘ └─────────────────────┘│
|
| 441 |
+
│ │
|
| 442 |
+
│ 📊 LMA Engine 📹 WebRTC 🎨 Visualization │
|
| 443 |
+
│ ┌─────────────┐ ┌─────────────┐ ┌─────────────────────┐│
|
| 444 |
+
│ │Direction │ │Live Camera │ │ Pose Overlays ││
|
| 445 |
+
│ │Intensity │ │Real-time │ │ Motion Trails ││
|
| 446 |
+
│ │Speed/Flow │ │Sub-100ms │ │ Metric Displays ││
|
| 447 |
+
│ │Expansion │ │Adaptive FPS │ │ AI Visualizations ││
|
| 448 |
+
│ └─────────────┘ └─────────────┘ └─────────────────────┘│
|
| 449 |
+
└─────────────────────────────────────────────────────────────┘
|
| 450 |
+
```
|
| 451 |
+
|
| 452 |
+
## 📝 Advanced Agent Workflows
|
| 453 |
+
|
| 454 |
+
### 1. Comprehensive Dance Analysis Pipeline
|
| 455 |
+
```python
|
| 456 |
+
# Multi-source dance video analysis
|
| 457 |
+
videos = [
|
| 458 |
+
"local_dance.mp4",
|
| 459 |
+
"https://youtube.com/watch?v=dance1",
|
| 460 |
+
"https://vimeo.com/dance2"
|
| 461 |
+
]
|
| 462 |
+
|
| 463 |
+
# Batch analyze with AI
|
| 464 |
+
results = agent_api.batch_analyze(
|
| 465 |
+
videos,
|
| 466 |
+
model=PoseModel.YOLO_V11_S,
|
| 467 |
+
use_skateformer=True,
|
| 468 |
+
parallel=True
|
| 469 |
+
)
|
| 470 |
+
|
| 471 |
+
# Filter for high-quality performances
|
| 472 |
+
excellent_dances = agent_api.filter_by_movement_advanced(
|
| 473 |
+
videos,
|
| 474 |
+
skateformer_actions=["dancing"],
|
| 475 |
+
movement_qualities={
|
| 476 |
+
"rhythm": 0.8,
|
| 477 |
+
"complexity": 0.7,
|
| 478 |
+
"energy": 0.8
|
| 479 |
+
},
|
| 480 |
+
traditional_criteria={
|
| 481 |
+
"intensity": MovementIntensity.HIGH,
|
| 482 |
+
"min_fluidity": 0.75
|
| 483 |
+
}
|
| 484 |
+
)
|
| 485 |
+
|
| 486 |
+
# Generate comprehensive report
|
| 487 |
+
report = agent_api.generate_analysis_report(
|
| 488 |
+
results,
|
| 489 |
+
include_comparisons=True,
|
| 490 |
+
include_recommendations=True
|
| 491 |
+
)
|
| 492 |
+
```
|
| 493 |
+
|
| 494 |
+
### 2. Real-time Exercise Form Checker
|
| 495 |
+
```python
|
| 496 |
+
# Start real-time analysis
|
| 497 |
+
agent_api.start_realtime_analysis(
|
| 498 |
+
model=PoseModel.MEDIAPIPE_FULL,
|
| 499 |
+
enable_skateformer=True
|
| 500 |
+
)
|
| 501 |
+
|
| 502 |
+
# Monitor form in real-time
|
| 503 |
+
while exercise_in_progress:
|
| 504 |
+
metrics = agent_api.get_realtime_metrics()
|
| 505 |
+
|
| 506 |
+
# Check form quality
|
| 507 |
+
if metrics["fluidity"] < 0.6:
|
| 508 |
+
send_feedback("Improve movement smoothness")
|
| 509 |
+
|
| 510 |
+
if metrics["symmetry"] < 0.7:
|
| 511 |
+
send_feedback("Balance left and right movements")
|
| 512 |
+
|
| 513 |
+
time.sleep(0.1) # 10Hz monitoring
|
| 514 |
+
|
| 515 |
+
# Stop and get session summary
|
| 516 |
+
agent_api.stop_realtime_analysis()
|
| 517 |
+
session_summary = agent_api.get_session_summary()
|
| 518 |
+
```
|
| 519 |
+
|
| 520 |
+
### 3. Movement Pattern Research Workflow
|
| 521 |
+
```python
|
| 522 |
+
# Large-scale analysis for research
|
| 523 |
+
research_videos = get_research_dataset()
|
| 524 |
+
|
| 525 |
+
# Batch process with comprehensive analysis
|
| 526 |
+
results = agent_api.batch_analyze(
|
| 527 |
+
research_videos,
|
| 528 |
+
model=PoseModel.YOLO_V11_L, # High accuracy for research
|
| 529 |
+
use_skateformer=True,
|
| 530 |
+
include_keypoints=True, # Full data for research
|
| 531 |
+
parallel=True
|
| 532 |
+
)
|
| 533 |
+
|
| 534 |
+
# Statistical analysis
|
| 535 |
+
patterns = agent_api.extract_movement_patterns(
|
| 536 |
+
results,
|
| 537 |
+
pattern_types=["temporal", "spatial", "quality"],
|
| 538 |
+
clustering_method="hierarchical"
|
| 539 |
+
)
|
| 540 |
+
|
| 541 |
+
# Generate research insights
|
| 542 |
+
insights = agent_api.generate_research_insights(
|
| 543 |
+
patterns,
|
| 544 |
+
include_visualizations=True,
|
| 545 |
+
statistical_tests=True
|
| 546 |
+
)
|
| 547 |
+
```
|
| 548 |
+
|
| 549 |
+
## 🔧 Advanced Configuration & Customization
|
| 550 |
+
|
| 551 |
+
### Environment Variables
|
| 552 |
+
|
| 553 |
+
```bash
|
| 554 |
+
# Core configuration
|
| 555 |
+
export LABAN_DEFAULT_MODEL="mediapipe-full"
|
| 556 |
+
export LABAN_CACHE_DIR="/path/to/cache"
|
| 557 |
+
export LABAN_MAX_WORKERS=4
|
| 558 |
+
|
| 559 |
+
# Enhanced features
|
| 560 |
+
export LABAN_ENABLE_SKATEFORMER=true
|
| 561 |
+
export LABAN_ENABLE_WEBRTC=true
|
| 562 |
+
export LABAN_SKATEFORMER_MODEL_PATH="/path/to/skateformer"
|
| 563 |
+
|
| 564 |
+
# Performance tuning
|
| 565 |
+
export LABAN_GPU_ENABLED=true
|
| 566 |
+
export LABAN_BATCH_SIZE=8
|
| 567 |
+
export LABAN_REALTIME_FPS=30
|
| 568 |
+
|
| 569 |
+
# Video download configuration
|
| 570 |
+
export LABAN_YOUTUBE_QUALITY="720p"
|
| 571 |
+
export LABAN_MAX_DOWNLOAD_SIZE="500MB"
|
| 572 |
+
export LABAN_TEMP_DIR="/tmp/laban_downloads"
|
| 573 |
+
```
|
| 574 |
+
|
| 575 |
+
### Custom MCP Tools
|
| 576 |
+
|
| 577 |
+
```python
|
| 578 |
+
# Add custom MCP tool
|
| 579 |
+
from backend.mcp_server import server
|
| 580 |
+
|
| 581 |
+
@server.tool("custom_movement_analysis")
|
| 582 |
+
async def custom_analysis(
|
| 583 |
+
video_path: str,
|
| 584 |
+
custom_params: dict
|
| 585 |
+
) -> dict:
|
| 586 |
+
"""Custom movement analysis with specific parameters."""
|
| 587 |
+
# Your custom implementation
|
| 588 |
+
return results
|
| 589 |
+
|
| 590 |
+
# Register enhanced filters
|
| 591 |
+
@server.tool("filter_by_sport_type")
|
| 592 |
+
async def filter_by_sport(
|
| 593 |
+
videos: list,
|
| 594 |
+
sport_type: str
|
| 595 |
+
) -> dict:
|
| 596 |
+
"""Filter videos by detected sport type using SkateFormer."""
|
| 597 |
+
# Implementation using SkateFormer sport classification
|
| 598 |
+
return filtered_videos
|
| 599 |
+
```
|
| 600 |
+
|
| 601 |
+
### WebRTC Configuration
|
| 602 |
+
|
| 603 |
+
```python
|
| 604 |
+
# Custom WebRTC configuration
|
| 605 |
+
webrtc_config = {
|
| 606 |
+
"video_constraints": {
|
| 607 |
+
"width": 1280,
|
| 608 |
+
"height": 720,
|
| 609 |
+
"frameRate": 30
|
| 610 |
+
},
|
| 611 |
+
"processing_config": {
|
| 612 |
+
"max_latency_ms": 100,
|
| 613 |
+
"quality_adaptation": True,
|
| 614 |
+
"model_switching": True
|
| 615 |
+
}
|
| 616 |
+
}
|
| 617 |
+
|
| 618 |
+
agent_api.configure_webrtc(webrtc_config)
|
| 619 |
+
```
|
| 620 |
+
|
| 621 |
+
## 🤝 Contributing to Agent Features
|
| 622 |
+
|
| 623 |
+
### Adding New MCP Tools
|
| 624 |
+
|
| 625 |
+
1. Define tool in `backend/mcp_server.py`
|
| 626 |
+
2. Implement core logic in agent API
|
| 627 |
+
3. Add comprehensive documentation
|
| 628 |
+
4. Include usage examples
|
| 629 |
+
5. Write integration tests
|
| 630 |
+
|
| 631 |
+
### Extending Agent API
|
| 632 |
+
|
| 633 |
+
1. Add methods to `LabanAgentAPI` class
|
| 634 |
+
2. Ensure compatibility with existing workflows
|
| 635 |
+
3. Add structured output formats
|
| 636 |
+
4. Include error handling and validation
|
| 637 |
+
5. Update documentation
|
| 638 |
+
|
| 639 |
+
### Enhancing SkateFormer Integration
|
| 640 |
+
|
| 641 |
+
1. Extend action recognition types
|
| 642 |
+
2. Add custom movement quality metrics
|
| 643 |
+
3. Implement temporal analysis features
|
| 644 |
+
4. Add visualization components
|
| 645 |
+
5. Validate with research datasets
|
| 646 |
+
|
| 647 |
+
## 📚 Resources & References
|
| 648 |
+
|
| 649 |
+
- [MCP Specification](https://github.com/anthropics/mcp)
|
| 650 |
+
- [SkateFormer Research Paper](https://kaist-viclab.github.io/SkateFormer_site/)
|
| 651 |
+
- [Gradio 5 Documentation](https://www.gradio.app/docs)
|
| 652 |
+
- [Unified Demo Application](demo/app.py)
|
| 653 |
+
- [Core Component Code](backend/gradio_labanmovementanalysis/)
|
| 654 |
+
|
| 655 |
+
## 🎯 Production Deployment
|
| 656 |
+
|
| 657 |
+
### Docker Deployment
|
| 658 |
+
|
| 659 |
+
```dockerfile
|
| 660 |
+
FROM python:3.9-slim
|
| 661 |
+
|
| 662 |
+
COPY . /app
|
| 663 |
+
WORKDIR /app
|
| 664 |
+
|
| 665 |
+
RUN pip install -r backend/requirements.txt
|
| 666 |
+
RUN pip install -r backend/requirements-mcp.txt
|
| 667 |
+
|
| 668 |
+
EXPOSE 7860 8080
|
| 669 |
+
|
| 670 |
+
CMD ["python", "-m", "backend.mcp_server"]
|
| 671 |
+
```
|
| 672 |
+
|
| 673 |
+
### Kubernetes Configuration
|
| 674 |
+
|
| 675 |
+
```yaml
|
| 676 |
+
apiVersion: apps/v1
|
| 677 |
+
kind: Deployment
|
| 678 |
+
metadata:
|
| 679 |
+
name: laban-mcp-server
|
| 680 |
+
spec:
|
| 681 |
+
replicas: 3
|
| 682 |
+
selector:
|
| 683 |
+
matchLabels:
|
| 684 |
+
app: laban-mcp
|
| 685 |
+
template:
|
| 686 |
+
metadata:
|
| 687 |
+
labels:
|
| 688 |
+
app: laban-mcp
|
| 689 |
+
spec:
|
| 690 |
+
containers:
|
| 691 |
+
- name: mcp-server
|
| 692 |
+
image: laban-movement-analysis:latest
|
| 693 |
+
ports:
|
| 694 |
+
- containerPort: 8080
|
| 695 |
+
env:
|
| 696 |
+
- name: LABAN_MAX_WORKERS
|
| 697 |
+
value: "2"
|
| 698 |
+
- name: LABAN_ENABLE_SKATEFORMER
|
| 699 |
+
value: "true"
|
| 700 |
+
```
|
| 701 |
+
|
| 702 |
+
---
|
| 703 |
+
|
| 704 |
+
**🤖 Transform your AI assistant into a movement analysis expert with comprehensive MCP integration and agent-ready automation.**
|
README.md
CHANGED
|
@@ -1,14 +1,1144 @@
|
|
| 1 |
---
|
| 2 |
-
title: Laban Movement Analysis
|
| 3 |
-
emoji:
|
| 4 |
colorFrom: blue
|
| 5 |
-
colorTo:
|
| 6 |
sdk: gradio
|
| 7 |
-
sdk_version: 5.
|
| 8 |
-
app_file:
|
| 9 |
pinned: false
|
| 10 |
-
license:
|
| 11 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 12 |
---
|
| 13 |
|
| 14 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
---
|
| 2 |
+
title: Laban Movement Analysis - Complete Suite
|
| 3 |
+
emoji: 🎭
|
| 4 |
colorFrom: blue
|
| 5 |
+
colorTo: green
|
| 6 |
sdk: gradio
|
| 7 |
+
sdk_version: 5.0.0
|
| 8 |
+
app_file: space.py
|
| 9 |
pinned: false
|
| 10 |
+
license: mit
|
| 11 |
+
tags:
|
| 12 |
+
- laban-movement-analysis
|
| 13 |
+
- pose-estimation
|
| 14 |
+
- movement-analysis
|
| 15 |
+
- video-analysis
|
| 16 |
+
- webrtc
|
| 17 |
+
- youtube
|
| 18 |
+
- vimeo
|
| 19 |
+
- mcp
|
| 20 |
+
- agent-ready
|
| 21 |
+
- computer-vision
|
| 22 |
+
- mediapipe
|
| 23 |
+
- yolo
|
| 24 |
+
- gradio
|
| 25 |
+
short_description: Professional movement analysis with pose estimation and AI
|
| 26 |
---
|
| 27 |
|
| 28 |
+
# `gradio_labanmovementanalysis`
|
| 29 |
+
<a href="https://pypi.org/project/gradio_labanmovementanalysis/" target="_blank"><img alt="PyPI - Version" src="https://img.shields.io/pypi/v/gradio_labanmovementanalysis"></a>
|
| 30 |
+
|
| 31 |
+
A Gradio 5 component for video movement analysis using Laban Movement Analysis (LMA) with MCP support for AI agents
|
| 32 |
+
|
| 33 |
+
## Installation
|
| 34 |
+
|
| 35 |
+
```bash
|
| 36 |
+
pip install gradio_labanmovementanalysis
|
| 37 |
+
```
|
| 38 |
+
|
| 39 |
+
## Usage
|
| 40 |
+
|
| 41 |
+
```python
|
| 42 |
+
"""
|
| 43 |
+
Unified Laban Movement Analysis Demo
|
| 44 |
+
Comprehensive interface combining all features:
|
| 45 |
+
- Standard LMA analysis
|
| 46 |
+
- Enhanced features (WebRTC, YouTube/Vimeo)
|
| 47 |
+
- Agent API (batch processing, filtering)
|
| 48 |
+
- Real-time analysis
|
| 49 |
+
- Model comparison
|
| 50 |
+
|
| 51 |
+
Created by: Csaba Bolyós (BladeSzaSza)
|
| 52 |
+
Contact: [email protected]
|
| 53 |
+
GitHub: https://github.com/bladeszasza
|
| 54 |
+
LinkedIn: https://www.linkedin.com/in/csaba-bolyós-00a11767/
|
| 55 |
+
Hugging Face: https://huggingface.co/BladeSzaSza
|
| 56 |
+
|
| 57 |
+
Heavy Beta Version - Under Active Development
|
| 58 |
+
"""
|
| 59 |
+
|
| 60 |
+
import gradio as gr
|
| 61 |
+
import sys
|
| 62 |
+
from pathlib import Path
|
| 63 |
+
from typing import Dict, Any, List, Tuple
|
| 64 |
+
|
| 65 |
+
# Add parent directory to path
|
| 66 |
+
sys.path.insert(0, str(Path(__file__).parent.parent / "backend"))
|
| 67 |
+
|
| 68 |
+
from gradio_labanmovementanalysis import LabanMovementAnalysis
|
| 69 |
+
|
| 70 |
+
# Import agent API if available
|
| 71 |
+
try:
|
| 72 |
+
from gradio_labanmovementanalysis.agent_api import (
|
| 73 |
+
LabanAgentAPI,
|
| 74 |
+
PoseModel,
|
| 75 |
+
MovementDirection,
|
| 76 |
+
MovementIntensity
|
| 77 |
+
)
|
| 78 |
+
HAS_AGENT_API = True
|
| 79 |
+
except ImportError:
|
| 80 |
+
HAS_AGENT_API = False
|
| 81 |
+
|
| 82 |
+
# Import WebRTC components if available
|
| 83 |
+
try:
|
| 84 |
+
from gradio_webrtc import WebRTC
|
| 85 |
+
from gradio_labanmovementanalysis.webrtc_handler import (
|
| 86 |
+
webrtc_detection,
|
| 87 |
+
get_rtc_configuration
|
| 88 |
+
)
|
| 89 |
+
HAS_WEBRTC = True
|
| 90 |
+
except ImportError as e:
|
| 91 |
+
print(f"WebRTC import failed: {e}")
|
| 92 |
+
HAS_WEBRTC = False
|
| 93 |
+
|
| 94 |
+
# Initialize components
|
| 95 |
+
try:
|
| 96 |
+
# Initialize with WebRTC support
|
| 97 |
+
analyzer = LabanMovementAnalysis(
|
| 98 |
+
enable_webrtc=True,
|
| 99 |
+
enable_visualization=True
|
| 100 |
+
)
|
| 101 |
+
print("✅ Core features initialized successfully")
|
| 102 |
+
except Exception as e:
|
| 103 |
+
print(f"Warning: Some features may not be available: {e}")
|
| 104 |
+
analyzer = LabanMovementAnalysis(enable_webrtc=False)
|
| 105 |
+
|
| 106 |
+
# Initialize agent API if available
|
| 107 |
+
agent_api = None
|
| 108 |
+
if HAS_AGENT_API:
|
| 109 |
+
try:
|
| 110 |
+
agent_api = LabanAgentAPI()
|
| 111 |
+
except Exception as e:
|
| 112 |
+
print(f"Warning: Agent API not available: {e}")
|
| 113 |
+
agent_api = None
|
| 114 |
+
|
| 115 |
+
|
| 116 |
+
def process_video_standard(video, model, enable_viz, include_keypoints):
|
| 117 |
+
"""Standard video processing function."""
|
| 118 |
+
if video is None:
|
| 119 |
+
return None, None
|
| 120 |
+
|
| 121 |
+
try:
|
| 122 |
+
json_output, video_output = analyzer.process_video(
|
| 123 |
+
video,
|
| 124 |
+
model=model,
|
| 125 |
+
enable_visualization=enable_viz,
|
| 126 |
+
include_keypoints=include_keypoints
|
| 127 |
+
)
|
| 128 |
+
return json_output, video_output
|
| 129 |
+
except Exception as e:
|
| 130 |
+
return {"error": str(e)}, None
|
| 131 |
+
|
| 132 |
+
|
| 133 |
+
def process_video_enhanced(video_input, model, enable_viz, include_keypoints):
|
| 134 |
+
"""Enhanced video processing with all new features."""
|
| 135 |
+
if not video_input:
|
| 136 |
+
return {"error": "No video provided"}, None
|
| 137 |
+
|
| 138 |
+
try:
|
| 139 |
+
# Handle both file upload and URL input
|
| 140 |
+
video_path = video_input.name if hasattr(video_input, 'name') else video_input
|
| 141 |
+
|
| 142 |
+
json_result, viz_result = analyzer.process_video(
|
| 143 |
+
video_path,
|
| 144 |
+
model=model,
|
| 145 |
+
enable_visualization=enable_viz,
|
| 146 |
+
include_keypoints=include_keypoints
|
| 147 |
+
)
|
| 148 |
+
return json_result, viz_result
|
| 149 |
+
except Exception as e:
|
| 150 |
+
error_result = {"error": str(e)}
|
| 151 |
+
return error_result, None
|
| 152 |
+
|
| 153 |
+
|
| 154 |
+
def process_video_for_agent(video, model, output_format="summary"):
|
| 155 |
+
"""Process video with agent-friendly output format."""
|
| 156 |
+
if not HAS_AGENT_API or agent_api is None:
|
| 157 |
+
return {"error": "Agent API not available"}
|
| 158 |
+
|
| 159 |
+
if not video:
|
| 160 |
+
return {"error": "No video provided"}
|
| 161 |
+
|
| 162 |
+
try:
|
| 163 |
+
model_enum = PoseModel(model)
|
| 164 |
+
result = agent_api.analyze(video, model=model_enum, generate_visualization=False)
|
| 165 |
+
|
| 166 |
+
if output_format == "summary":
|
| 167 |
+
return {"summary": agent_api.get_movement_summary(result)}
|
| 168 |
+
elif output_format == "structured":
|
| 169 |
+
return {
|
| 170 |
+
"success": result.success,
|
| 171 |
+
"direction": result.dominant_direction.value,
|
| 172 |
+
"intensity": result.dominant_intensity.value,
|
| 173 |
+
"speed": result.dominant_speed,
|
| 174 |
+
"fluidity": result.fluidity_score,
|
| 175 |
+
"expansion": result.expansion_score,
|
| 176 |
+
"segments": len(result.movement_segments)
|
| 177 |
+
}
|
| 178 |
+
else: # json
|
| 179 |
+
return result.raw_data
|
| 180 |
+
except Exception as e:
|
| 181 |
+
return {"error": str(e)}
|
| 182 |
+
|
| 183 |
+
|
| 184 |
+
def batch_process_videos(files, model):
|
| 185 |
+
"""Process multiple videos in batch."""
|
| 186 |
+
if not HAS_AGENT_API or agent_api is None:
|
| 187 |
+
return {"error": "Agent API not available"}
|
| 188 |
+
|
| 189 |
+
if not files:
|
| 190 |
+
return {"error": "No videos provided"}
|
| 191 |
+
|
| 192 |
+
try:
|
| 193 |
+
video_paths = [f.name for f in files]
|
| 194 |
+
results = agent_api.batch_analyze(video_paths, model=PoseModel(model), parallel=True)
|
| 195 |
+
|
| 196 |
+
output = {
|
| 197 |
+
"total_videos": len(results),
|
| 198 |
+
"successful": sum(1 for r in results if r.success),
|
| 199 |
+
"failed": sum(1 for r in results if not r.success),
|
| 200 |
+
"results": []
|
| 201 |
+
}
|
| 202 |
+
|
| 203 |
+
for result in results:
|
| 204 |
+
output["results"].append({
|
| 205 |
+
"video": Path(result.video_path).name,
|
| 206 |
+
"success": result.success,
|
| 207 |
+
"summary": agent_api.get_movement_summary(result) if result.success else result.error
|
| 208 |
+
})
|
| 209 |
+
|
| 210 |
+
return output
|
| 211 |
+
except Exception as e:
|
| 212 |
+
return {"error": str(e)}
|
| 213 |
+
|
| 214 |
+
|
| 215 |
+
def filter_videos_by_movement(files, direction, intensity, min_fluidity, min_expansion):
|
| 216 |
+
"""Filter videos based on movement characteristics."""
|
| 217 |
+
if not HAS_AGENT_API or agent_api is None:
|
| 218 |
+
return {"error": "Agent API not available"}
|
| 219 |
+
|
| 220 |
+
if not files:
|
| 221 |
+
return {"error": "No videos provided"}
|
| 222 |
+
|
| 223 |
+
try:
|
| 224 |
+
video_paths = [f.name for f in files]
|
| 225 |
+
|
| 226 |
+
dir_filter = MovementDirection(direction) if direction != "any" else None
|
| 227 |
+
int_filter = MovementIntensity(intensity) if intensity != "any" else None
|
| 228 |
+
|
| 229 |
+
filtered = agent_api.filter_by_movement(
|
| 230 |
+
video_paths,
|
| 231 |
+
direction=dir_filter,
|
| 232 |
+
intensity=int_filter,
|
| 233 |
+
min_fluidity=min_fluidity if min_fluidity > 0 else None,
|
| 234 |
+
min_expansion=min_expansion if min_expansion > 0 else None
|
| 235 |
+
)
|
| 236 |
+
|
| 237 |
+
return {
|
| 238 |
+
"total_analyzed": len(video_paths),
|
| 239 |
+
"matching_videos": len(filtered),
|
| 240 |
+
"matches": [
|
| 241 |
+
{
|
| 242 |
+
"video": Path(r.video_path).name,
|
| 243 |
+
"direction": r.dominant_direction.value,
|
| 244 |
+
"intensity": r.dominant_intensity.value,
|
| 245 |
+
"fluidity": r.fluidity_score,
|
| 246 |
+
"expansion": r.expansion_score
|
| 247 |
+
}
|
| 248 |
+
for r in filtered
|
| 249 |
+
]
|
| 250 |
+
}
|
| 251 |
+
except Exception as e:
|
| 252 |
+
return {"error": str(e)}
|
| 253 |
+
|
| 254 |
+
|
| 255 |
+
def compare_models(video, model1, model2):
|
| 256 |
+
"""Compare two different pose models on the same video."""
|
| 257 |
+
if not video:
|
| 258 |
+
return "No video provided"
|
| 259 |
+
|
| 260 |
+
try:
|
| 261 |
+
# Analyze with both models
|
| 262 |
+
result1, _ = analyzer.process_video(video, model=model1, enable_visualization=False)
|
| 263 |
+
result2, _ = analyzer.process_video(video, model=model2, enable_visualization=False)
|
| 264 |
+
|
| 265 |
+
# Extract key metrics for comparison
|
| 266 |
+
def extract_metrics(result):
|
| 267 |
+
summary = result.get("movement_analysis", {}).get("summary", {})
|
| 268 |
+
return {
|
| 269 |
+
"direction": summary.get("direction", {}).get("dominant", "unknown"),
|
| 270 |
+
"intensity": summary.get("intensity", {}).get("dominant", "unknown"),
|
| 271 |
+
"speed": summary.get("speed", {}).get("dominant", "unknown"),
|
| 272 |
+
"frame_count": result.get("video_info", {}).get("frame_count", 0)
|
| 273 |
+
}
|
| 274 |
+
|
| 275 |
+
metrics1 = extract_metrics(result1)
|
| 276 |
+
metrics2 = extract_metrics(result2)
|
| 277 |
+
|
| 278 |
+
# Create comparison table data
|
| 279 |
+
comparison_data = [
|
| 280 |
+
["Direction", metrics1["direction"], metrics2["direction"],
|
| 281 |
+
"✓" if metrics1["direction"] == metrics2["direction"] else "✗"],
|
| 282 |
+
["Intensity", metrics1["intensity"], metrics2["intensity"],
|
| 283 |
+
"✓" if metrics1["intensity"] == metrics2["intensity"] else "✗"],
|
| 284 |
+
["Speed", metrics1["speed"], metrics2["speed"],
|
| 285 |
+
"✓" if metrics1["speed"] == metrics2["speed"] else "✗"],
|
| 286 |
+
["Frames Processed", str(metrics1["frame_count"]), str(metrics2["frame_count"]),
|
| 287 |
+
"✓" if metrics1["frame_count"] == metrics2["frame_count"] else "✗"]
|
| 288 |
+
]
|
| 289 |
+
|
| 290 |
+
return comparison_data
|
| 291 |
+
|
| 292 |
+
except Exception as e:
|
| 293 |
+
return [["Error", str(e), "", ""]]
|
| 294 |
+
|
| 295 |
+
|
| 296 |
+
def start_webrtc_stream(model):
|
| 297 |
+
"""Start WebRTC real-time analysis."""
|
| 298 |
+
try:
|
| 299 |
+
success = analyzer.start_webrtc_stream(model)
|
| 300 |
+
if success:
|
| 301 |
+
return "🟢 Stream Active", {"status": "streaming", "model": model}
|
| 302 |
+
else:
|
| 303 |
+
return "🔴 Failed to start", {"status": "error"}
|
| 304 |
+
except Exception as e:
|
| 305 |
+
return f"🔴 Error: {str(e)}", {"status": "error"}
|
| 306 |
+
|
| 307 |
+
|
| 308 |
+
def stop_webrtc_stream():
|
| 309 |
+
"""Stop WebRTC real-time analysis."""
|
| 310 |
+
try:
|
| 311 |
+
success = analyzer.stop_webrtc_stream()
|
| 312 |
+
if success:
|
| 313 |
+
return "🟡 Stream Stopped", {"status": "stopped"}
|
| 314 |
+
else:
|
| 315 |
+
return "🔴 Failed to stop", {"status": "error"}
|
| 316 |
+
except Exception as e:
|
| 317 |
+
return f"🔴 Error: {str(e)}", {"status": "error"}
|
| 318 |
+
|
| 319 |
+
|
| 320 |
+
def create_unified_demo():
|
| 321 |
+
"""Create the unified comprehensive demo."""
|
| 322 |
+
|
| 323 |
+
with gr.Blocks(
|
| 324 |
+
title="Laban Movement Analysis - Complete Suite by Csaba Bolyós",
|
| 325 |
+
theme=gr.themes.Soft(),
|
| 326 |
+
css="""
|
| 327 |
+
.main-header {
|
| 328 |
+
background: linear-gradient(135deg, #40826D 0%, #2E5E4A 50%, #1B3A2F 100%);
|
| 329 |
+
color: white;
|
| 330 |
+
padding: 30px;
|
| 331 |
+
border-radius: 10px;
|
| 332 |
+
margin-bottom: 20px;
|
| 333 |
+
text-align: center;
|
| 334 |
+
}
|
| 335 |
+
.feature-card {
|
| 336 |
+
border: 1px solid #e1e5e9;
|
| 337 |
+
border-radius: 8px;
|
| 338 |
+
padding: 16px;
|
| 339 |
+
margin: 8px 0;
|
| 340 |
+
background: #f8f9fa;
|
| 341 |
+
}
|
| 342 |
+
.json-output {
|
| 343 |
+
max-height: 600px;
|
| 344 |
+
overflow-y: auto;
|
| 345 |
+
font-family: monospace;
|
| 346 |
+
font-size: 12px;
|
| 347 |
+
}
|
| 348 |
+
.author-info {
|
| 349 |
+
background: linear-gradient(135deg, #40826D 0%, #2E5E4A 100%);
|
| 350 |
+
color: white;
|
| 351 |
+
padding: 15px;
|
| 352 |
+
border-radius: 8px;
|
| 353 |
+
margin: 10px 0;
|
| 354 |
+
text-align: center;
|
| 355 |
+
}
|
| 356 |
+
"""
|
| 357 |
+
) as demo:
|
| 358 |
+
|
| 359 |
+
# Main Header
|
| 360 |
+
gr.HTML("""
|
| 361 |
+
<div class="main-header">
|
| 362 |
+
<h1>🎭 Laban Movement Analysis - Complete Suite</h1>
|
| 363 |
+
<p style="font-size: 18px; margin: 10px 0;">
|
| 364 |
+
Professional movement analysis with pose estimation, AI action recognition,
|
| 365 |
+
real-time processing, and agent automation
|
| 366 |
+
</p>
|
| 367 |
+
<p style="font-size: 14px; opacity: 0.9;">
|
| 368 |
+
Supports YouTube/Vimeo URLs • WebRTC Streaming • 20+ Pose Models • MCP Integration
|
| 369 |
+
</p>
|
| 370 |
+
<p style="font-size: 12px; margin-top: 15px; opacity: 0.8;">
|
| 371 |
+
<strong>Version 0.01-beta</strong> - Heavy Beta Under Active Development
|
| 372 |
+
</p>
|
| 373 |
+
</div>
|
| 374 |
+
""")
|
| 375 |
+
|
| 376 |
+
with gr.Tabs():
|
| 377 |
+
# Tab 1: Standard Analysis
|
| 378 |
+
with gr.Tab("🎬 Standard Analysis"):
|
| 379 |
+
gr.Markdown("""
|
| 380 |
+
### Classic Laban Movement Analysis
|
| 381 |
+
Upload a video file to analyze movement using traditional LMA metrics with pose estimation.
|
| 382 |
+
""")
|
| 383 |
+
|
| 384 |
+
with gr.Row():
|
| 385 |
+
with gr.Column(scale=1):
|
| 386 |
+
video_input_std = gr.Video(
|
| 387 |
+
label="Upload Video",
|
| 388 |
+
sources=["upload"],
|
| 389 |
+
format="mp4"
|
| 390 |
+
)
|
| 391 |
+
|
| 392 |
+
model_dropdown_std = gr.Dropdown(
|
| 393 |
+
choices=["mediapipe", "movenet", "yolo"],
|
| 394 |
+
value="mediapipe",
|
| 395 |
+
label="Pose Estimation Model"
|
| 396 |
+
)
|
| 397 |
+
|
| 398 |
+
with gr.Row():
|
| 399 |
+
enable_viz_std = gr.Checkbox(
|
| 400 |
+
value=True,
|
| 401 |
+
label="Generate Visualization"
|
| 402 |
+
)
|
| 403 |
+
|
| 404 |
+
include_keypoints_std = gr.Checkbox(
|
| 405 |
+
value=False,
|
| 406 |
+
label="Include Keypoints"
|
| 407 |
+
)
|
| 408 |
+
|
| 409 |
+
process_btn_std = gr.Button("Analyze Movement", variant="primary")
|
| 410 |
+
|
| 411 |
+
gr.Examples(
|
| 412 |
+
examples=[
|
| 413 |
+
["examples/balette.mov"],
|
| 414 |
+
["examples/balette.mp4"],
|
| 415 |
+
],
|
| 416 |
+
inputs=video_input_std,
|
| 417 |
+
label="Example Videos"
|
| 418 |
+
)
|
| 419 |
+
|
| 420 |
+
with gr.Column(scale=2):
|
| 421 |
+
with gr.Tab("Analysis Results"):
|
| 422 |
+
json_output_std = gr.JSON(
|
| 423 |
+
label="Movement Analysis (JSON)",
|
| 424 |
+
elem_classes=["json-output"]
|
| 425 |
+
)
|
| 426 |
+
|
| 427 |
+
with gr.Tab("Visualization"):
|
| 428 |
+
video_output_std = gr.Video(
|
| 429 |
+
label="Annotated Video",
|
| 430 |
+
format="mp4"
|
| 431 |
+
)
|
| 432 |
+
|
| 433 |
+
gr.Markdown("""
|
| 434 |
+
**Visualization Guide:**
|
| 435 |
+
- 🦴 **Skeleton**: Pose keypoints and connections
|
| 436 |
+
- 🌊 **Trails**: Motion history (fading lines)
|
| 437 |
+
- ➡️ **Arrows**: Movement direction indicators
|
| 438 |
+
- 🎨 **Colors**: Green (low) → Orange (medium) → Red (high) intensity
|
| 439 |
+
""")
|
| 440 |
+
|
| 441 |
+
process_btn_std.click(
|
| 442 |
+
fn=process_video_standard,
|
| 443 |
+
inputs=[video_input_std, model_dropdown_std, enable_viz_std, include_keypoints_std],
|
| 444 |
+
outputs=[json_output_std, video_output_std],
|
| 445 |
+
api_name="analyze_standard"
|
| 446 |
+
)
|
| 447 |
+
|
| 448 |
+
# Tab 2: Enhanced Analysis
|
| 449 |
+
with gr.Tab("🚀 Enhanced Analysis"):
|
| 450 |
+
gr.Markdown("""
|
| 451 |
+
### Advanced Analysis with AI and URL Support
|
| 452 |
+
Analyze videos from URLs (YouTube/Vimeo), use advanced pose models, and get AI-powered insights.
|
| 453 |
+
""")
|
| 454 |
+
|
| 455 |
+
with gr.Row():
|
| 456 |
+
with gr.Column(scale=1):
|
| 457 |
+
gr.HTML('<div class="feature-card">')
|
| 458 |
+
gr.Markdown("**Video Input**")
|
| 459 |
+
|
| 460 |
+
# Changed from textbox to file upload as requested
|
| 461 |
+
video_input_enh = gr.File(
|
| 462 |
+
label="Upload Video or Drop File",
|
| 463 |
+
file_types=["video"],
|
| 464 |
+
type="filepath"
|
| 465 |
+
)
|
| 466 |
+
|
| 467 |
+
# URL input option
|
| 468 |
+
url_input_enh = gr.Textbox(
|
| 469 |
+
label="Or Enter Video URL",
|
| 470 |
+
placeholder="YouTube URL, Vimeo URL, or direct video URL",
|
| 471 |
+
info="Leave file upload empty to use URL"
|
| 472 |
+
)
|
| 473 |
+
|
| 474 |
+
gr.Examples(
|
| 475 |
+
examples=[
|
| 476 |
+
["examples/balette.mov"],
|
| 477 |
+
["https://www.youtube.com/shorts/RX9kH2l3L8U"],
|
| 478 |
+
["https://vimeo.com/815392738"]
|
| 479 |
+
],
|
| 480 |
+
inputs=url_input_enh,
|
| 481 |
+
label="Example URLs"
|
| 482 |
+
)
|
| 483 |
+
|
| 484 |
+
gr.Markdown("**Model Selection**")
|
| 485 |
+
|
| 486 |
+
model_select_enh = gr.Dropdown(
|
| 487 |
+
choices=[
|
| 488 |
+
# MediaPipe variants
|
| 489 |
+
"mediapipe-lite", "mediapipe-full", "mediapipe-heavy",
|
| 490 |
+
# MoveNet variants
|
| 491 |
+
"movenet-lightning", "movenet-thunder",
|
| 492 |
+
# YOLO variants (added X models)
|
| 493 |
+
"yolo-v8-n", "yolo-v8-s", "yolo-v8-m", "yolo-v8-l", "yolo-v8-x",
|
| 494 |
+
# YOLO v11 variants
|
| 495 |
+
"yolo-v11-n", "yolo-v11-s", "yolo-v11-m", "yolo-v11-l", "yolo-v11-x"
|
| 496 |
+
],
|
| 497 |
+
value="mediapipe-full",
|
| 498 |
+
label="Advanced Pose Models",
|
| 499 |
+
info="17+ model variants available"
|
| 500 |
+
)
|
| 501 |
+
|
| 502 |
+
gr.Markdown("**Analysis Options**")
|
| 503 |
+
|
| 504 |
+
with gr.Row():
|
| 505 |
+
enable_viz_enh = gr.Checkbox(value=True, label="Visualization")
|
| 506 |
+
|
| 507 |
+
with gr.Row():
|
| 508 |
+
include_keypoints_enh = gr.Checkbox(value=False, label="Raw Keypoints")
|
| 509 |
+
|
| 510 |
+
analyze_btn_enh = gr.Button("🚀 Enhanced Analysis", variant="primary", size="lg")
|
| 511 |
+
gr.HTML('</div>')
|
| 512 |
+
|
| 513 |
+
with gr.Column(scale=2):
|
| 514 |
+
with gr.Tab("📊 Analysis"):
|
| 515 |
+
analysis_output_enh = gr.JSON(label="Enhanced Analysis Results")
|
| 516 |
+
|
| 517 |
+
with gr.Tab("🎥 Visualization"):
|
| 518 |
+
viz_output_enh = gr.Video(label="Annotated Video")
|
| 519 |
+
|
| 520 |
+
def process_enhanced_input(file_input, url_input, model, enable_viz, include_keypoints):
|
| 521 |
+
"""Process either file upload or URL input."""
|
| 522 |
+
video_source = file_input if file_input else url_input
|
| 523 |
+
return process_video_enhanced(video_source, model, enable_viz, include_keypoints)
|
| 524 |
+
|
| 525 |
+
analyze_btn_enh.click(
|
| 526 |
+
fn=process_enhanced_input,
|
| 527 |
+
inputs=[video_input_enh, url_input_enh, model_select_enh, enable_viz_enh, include_keypoints_enh],
|
| 528 |
+
outputs=[analysis_output_enh, viz_output_enh],
|
| 529 |
+
api_name="analyze_enhanced"
|
| 530 |
+
)
|
| 531 |
+
|
| 532 |
+
# Tab 3: Agent API
|
| 533 |
+
with gr.Tab("🤖 Agent API"):
|
| 534 |
+
gr.Markdown("""
|
| 535 |
+
### AI Agent & Automation Features
|
| 536 |
+
Batch processing, filtering, and structured outputs designed for AI agents and automation.
|
| 537 |
+
""")
|
| 538 |
+
|
| 539 |
+
with gr.Tabs():
|
| 540 |
+
with gr.Tab("Single Analysis"):
|
| 541 |
+
with gr.Row():
|
| 542 |
+
with gr.Column():
|
| 543 |
+
video_input_agent = gr.Video(label="Upload Video", sources=["upload"])
|
| 544 |
+
model_select_agent = gr.Dropdown(
|
| 545 |
+
choices=["mediapipe", "movenet", "yolo"],
|
| 546 |
+
value="mediapipe",
|
| 547 |
+
label="Model"
|
| 548 |
+
)
|
| 549 |
+
output_format_agent = gr.Radio(
|
| 550 |
+
choices=["summary", "structured", "json"],
|
| 551 |
+
value="summary",
|
| 552 |
+
label="Output Format"
|
| 553 |
+
)
|
| 554 |
+
analyze_btn_agent = gr.Button("Analyze", variant="primary")
|
| 555 |
+
|
| 556 |
+
with gr.Column():
|
| 557 |
+
output_display_agent = gr.JSON(label="Agent Output")
|
| 558 |
+
|
| 559 |
+
analyze_btn_agent.click(
|
| 560 |
+
fn=process_video_for_agent,
|
| 561 |
+
inputs=[video_input_agent, model_select_agent, output_format_agent],
|
| 562 |
+
outputs=output_display_agent,
|
| 563 |
+
api_name="analyze_agent"
|
| 564 |
+
)
|
| 565 |
+
|
| 566 |
+
with gr.Tab("Batch Processing"):
|
| 567 |
+
with gr.Row():
|
| 568 |
+
with gr.Column():
|
| 569 |
+
batch_files = gr.File(
|
| 570 |
+
label="Upload Multiple Videos",
|
| 571 |
+
file_count="multiple",
|
| 572 |
+
file_types=["video"]
|
| 573 |
+
)
|
| 574 |
+
batch_model = gr.Dropdown(
|
| 575 |
+
choices=["mediapipe", "movenet", "yolo"],
|
| 576 |
+
value="mediapipe",
|
| 577 |
+
label="Model"
|
| 578 |
+
)
|
| 579 |
+
batch_btn = gr.Button("Process Batch", variant="primary")
|
| 580 |
+
|
| 581 |
+
with gr.Column():
|
| 582 |
+
batch_output = gr.JSON(label="Batch Results")
|
| 583 |
+
|
| 584 |
+
batch_btn.click(
|
| 585 |
+
fn=batch_process_videos,
|
| 586 |
+
inputs=[batch_files, batch_model],
|
| 587 |
+
outputs=batch_output,
|
| 588 |
+
api_name="batch_analyze"
|
| 589 |
+
)
|
| 590 |
+
|
| 591 |
+
with gr.Tab("Movement Filter"):
|
| 592 |
+
with gr.Row():
|
| 593 |
+
with gr.Column():
|
| 594 |
+
filter_files = gr.File(
|
| 595 |
+
label="Videos to Filter",
|
| 596 |
+
file_count="multiple",
|
| 597 |
+
file_types=["video"]
|
| 598 |
+
)
|
| 599 |
+
|
| 600 |
+
with gr.Group():
|
| 601 |
+
direction_filter = gr.Dropdown(
|
| 602 |
+
choices=["any", "up", "down", "left", "right", "stationary"],
|
| 603 |
+
value="any",
|
| 604 |
+
label="Direction Filter"
|
| 605 |
+
)
|
| 606 |
+
intensity_filter = gr.Dropdown(
|
| 607 |
+
choices=["any", "low", "medium", "high"],
|
| 608 |
+
value="any",
|
| 609 |
+
label="Intensity Filter"
|
| 610 |
+
)
|
| 611 |
+
fluidity_threshold = gr.Slider(0.0, 1.0, 0.0, label="Min Fluidity")
|
| 612 |
+
expansion_threshold = gr.Slider(0.0, 1.0, 0.0, label="Min Expansion")
|
| 613 |
+
|
| 614 |
+
filter_btn = gr.Button("Apply Filters", variant="primary")
|
| 615 |
+
|
| 616 |
+
with gr.Column():
|
| 617 |
+
filter_output = gr.JSON(label="Filtered Results")
|
| 618 |
+
|
| 619 |
+
filter_btn.click(
|
| 620 |
+
fn=filter_videos_by_movement,
|
| 621 |
+
inputs=[filter_files, direction_filter, intensity_filter,
|
| 622 |
+
fluidity_threshold, expansion_threshold],
|
| 623 |
+
outputs=filter_output,
|
| 624 |
+
api_name="filter_videos"
|
| 625 |
+
)
|
| 626 |
+
|
| 627 |
+
# Tab 4: Real-time WebRTC
|
| 628 |
+
with gr.Tab("📹 Real-time Analysis"):
|
| 629 |
+
gr.Markdown("""
|
| 630 |
+
### Live Camera Movement Analysis
|
| 631 |
+
Real-time pose detection and movement analysis from your webcam using WebRTC.
|
| 632 |
+
**Grant camera permissions when prompted for best experience.**
|
| 633 |
+
""")
|
| 634 |
+
|
| 635 |
+
# Official Gradio WebRTC approach (compatible with NumPy 1.x)
|
| 636 |
+
if HAS_WEBRTC:
|
| 637 |
+
|
| 638 |
+
# Get RTC configuration
|
| 639 |
+
rtc_config = get_rtc_configuration()
|
| 640 |
+
|
| 641 |
+
# Custom CSS following official guide
|
| 642 |
+
css_webrtc = """
|
| 643 |
+
.my-group {max-width: 480px !important; max-height: 480px !important;}
|
| 644 |
+
.my-column {display: flex !important; justify-content: center !important; align-items: center !important;}
|
| 645 |
+
"""
|
| 646 |
+
|
| 647 |
+
with gr.Column(elem_classes=["my-column"]):
|
| 648 |
+
with gr.Group(elem_classes=["my-group"]):
|
| 649 |
+
# Official WebRTC Component
|
| 650 |
+
webrtc_stream = WebRTC(
|
| 651 |
+
label="🎥 Live Camera Stream",
|
| 652 |
+
rtc_configuration=rtc_config
|
| 653 |
+
)
|
| 654 |
+
|
| 655 |
+
webrtc_model = gr.Dropdown(
|
| 656 |
+
choices=["mediapipe-lite", "movenet-lightning", "yolo-v11-n"],
|
| 657 |
+
value="mediapipe-lite",
|
| 658 |
+
label="Pose Model",
|
| 659 |
+
info="Optimized for real-time processing"
|
| 660 |
+
)
|
| 661 |
+
|
| 662 |
+
confidence_slider = gr.Slider(
|
| 663 |
+
label="Detection Confidence",
|
| 664 |
+
minimum=0.0,
|
| 665 |
+
maximum=1.0,
|
| 666 |
+
step=0.05,
|
| 667 |
+
value=0.5,
|
| 668 |
+
info="Higher = fewer false positives"
|
| 669 |
+
)
|
| 670 |
+
|
| 671 |
+
# Official WebRTC streaming setup following Gradio guide
|
| 672 |
+
webrtc_stream.stream(
|
| 673 |
+
fn=webrtc_detection,
|
| 674 |
+
inputs=[webrtc_stream, webrtc_model, confidence_slider],
|
| 675 |
+
outputs=[webrtc_stream],
|
| 676 |
+
time_limit=10 # Following official guide: 10 seconds per user
|
| 677 |
+
)
|
| 678 |
+
|
| 679 |
+
# Info display
|
| 680 |
+
gr.HTML("""
|
| 681 |
+
<div style="background: #e8f4fd; padding: 15px; border-radius: 8px; margin-top: 10px;">
|
| 682 |
+
<h4>📹 WebRTC Pose Analysis</h4>
|
| 683 |
+
<p style="margin: 5px 0;">Real-time movement analysis using your webcam</p>
|
| 684 |
+
|
| 685 |
+
<h4>🔒 Privacy</h4>
|
| 686 |
+
<p style="margin: 5px 0;">Processing happens locally - no video data stored</p>
|
| 687 |
+
|
| 688 |
+
<h4>💡 Usage</h4>
|
| 689 |
+
<ul style="margin: 5px 0; padding-left: 20px;">
|
| 690 |
+
<li>Grant camera permission when prompted</li>
|
| 691 |
+
<li>Move in front of camera to see pose detection</li>
|
| 692 |
+
<li>Adjust confidence threshold as needed</li>
|
| 693 |
+
</ul>
|
| 694 |
+
</div>
|
| 695 |
+
""")
|
| 696 |
+
|
| 697 |
+
else:
|
| 698 |
+
# Fallback if WebRTC component not available
|
| 699 |
+
gr.HTML("""
|
| 700 |
+
<div style="text-align: center; padding: 50px; border: 2px dashed #ff6b6b; border-radius: 8px; background: #ffe0e0;">
|
| 701 |
+
<h3>📦 WebRTC Component Required</h3>
|
| 702 |
+
<p><strong>To enable real-time camera analysis, install:</strong></p>
|
| 703 |
+
<code style="background: #f0f0f0; padding: 10px; border-radius: 4px; display: block; margin: 10px 0;">
|
| 704 |
+
pip install gradio-webrtc twilio
|
| 705 |
+
</code>
|
| 706 |
+
<p style="margin-top: 15px;"><em>Use Enhanced Analysis tab for video files meanwhile</em></p>
|
| 707 |
+
</div>
|
| 708 |
+
""")
|
| 709 |
+
|
| 710 |
+
# Tab 5: Model Comparison
|
| 711 |
+
with gr.Tab("⚖️ Model Comparison"):
|
| 712 |
+
gr.Markdown("""
|
| 713 |
+
### Compare Pose Estimation Models
|
| 714 |
+
Analyze the same video with different models to compare accuracy and results.
|
| 715 |
+
""")
|
| 716 |
+
|
| 717 |
+
with gr.Column():
|
| 718 |
+
comparison_video = gr.Video(
|
| 719 |
+
label="Video for Comparison",
|
| 720 |
+
sources=["upload"]
|
| 721 |
+
)
|
| 722 |
+
|
| 723 |
+
with gr.Row():
|
| 724 |
+
model1_comp = gr.Dropdown(
|
| 725 |
+
choices=["mediapipe-full", "movenet-thunder", "yolo-v11-s"],
|
| 726 |
+
value="mediapipe-full",
|
| 727 |
+
label="Model 1"
|
| 728 |
+
)
|
| 729 |
+
|
| 730 |
+
model2_comp = gr.Dropdown(
|
| 731 |
+
choices=["mediapipe-full", "movenet-thunder", "yolo-v11-s"],
|
| 732 |
+
value="yolo-v11-s",
|
| 733 |
+
label="Model 2"
|
| 734 |
+
)
|
| 735 |
+
|
| 736 |
+
compare_btn = gr.Button("🔄 Compare Models", variant="primary")
|
| 737 |
+
|
| 738 |
+
comparison_results = gr.DataFrame(
|
| 739 |
+
headers=["Metric", "Model 1", "Model 2", "Match"],
|
| 740 |
+
label="Comparison Results"
|
| 741 |
+
)
|
| 742 |
+
|
| 743 |
+
compare_btn.click(
|
| 744 |
+
fn=compare_models,
|
| 745 |
+
inputs=[comparison_video, model1_comp, model2_comp],
|
| 746 |
+
outputs=comparison_results,
|
| 747 |
+
api_name="compare_models"
|
| 748 |
+
)
|
| 749 |
+
|
| 750 |
+
# Tab 6: Documentation
|
| 751 |
+
with gr.Tab("📚 Documentation"):
|
| 752 |
+
gr.Markdown("""
|
| 753 |
+
# Complete Feature Documentation
|
| 754 |
+
|
| 755 |
+
## 🎥 Video Input Support
|
| 756 |
+
- **Local Files**: MP4, AVI, MOV, WebM formats
|
| 757 |
+
- **YouTube**: Automatic download from YouTube URLs
|
| 758 |
+
- **Vimeo**: Automatic download from Vimeo URLs
|
| 759 |
+
- **Direct URLs**: Any direct video file URL
|
| 760 |
+
|
| 761 |
+
## 🤖 Pose Estimation Models
|
| 762 |
+
|
| 763 |
+
### MediaPipe (Google) - 33 3D Landmarks
|
| 764 |
+
- **Lite**: Fastest CPU performance
|
| 765 |
+
- **Full**: Balanced accuracy/speed (recommended)
|
| 766 |
+
- **Heavy**: Highest accuracy
|
| 767 |
+
|
| 768 |
+
### MoveNet (Google) - 17 COCO Keypoints
|
| 769 |
+
- **Lightning**: Mobile-optimized, very fast
|
| 770 |
+
- **Thunder**: Higher accuracy variant
|
| 771 |
+
|
| 772 |
+
### YOLO (Ultralytics) - 17 COCO Keypoints
|
| 773 |
+
- **v8 variants**: n/s/m/l/x sizes (nano to extra-large)
|
| 774 |
+
- **v11 variants**: Latest with improved accuracy (n/s/m/l/x)
|
| 775 |
+
- **Multi-person**: Supports multiple people in frame
|
| 776 |
+
|
| 777 |
+
## 📹 Real-time WebRTC
|
| 778 |
+
|
| 779 |
+
- **Live Camera**: Direct webcam access via WebRTC
|
| 780 |
+
- **Low Latency**: Sub-100ms processing
|
| 781 |
+
- **Adaptive Quality**: Automatic performance optimization
|
| 782 |
+
- **Live Overlay**: Real-time pose and metrics display
|
| 783 |
+
|
| 784 |
+
## 🤖 Agent & MCP Integration
|
| 785 |
+
|
| 786 |
+
### API Endpoints
|
| 787 |
+
- `/analyze_standard` - Basic LMA analysis
|
| 788 |
+
- `/analyze_enhanced` - Advanced analysis with all features
|
| 789 |
+
- `/analyze_agent` - Agent-optimized output
|
| 790 |
+
- `/batch_analyze` - Multiple video processing
|
| 791 |
+
- `/filter_videos` - Movement-based filtering
|
| 792 |
+
- `/compare_models` - Model comparison
|
| 793 |
+
|
| 794 |
+
### MCP Server
|
| 795 |
+
```bash
|
| 796 |
+
# Start MCP server for AI assistants
|
| 797 |
+
python -m backend.mcp_server
|
| 798 |
+
```
|
| 799 |
+
|
| 800 |
+
### Python API
|
| 801 |
+
```python
|
| 802 |
+
from gradio_labanmovementanalysis import LabanMovementAnalysis
|
| 803 |
+
|
| 804 |
+
# Initialize with all features
|
| 805 |
+
analyzer = LabanMovementAnalysis(
|
| 806 |
+
enable_webrtc=True
|
| 807 |
+
)
|
| 808 |
+
|
| 809 |
+
# Analyze YouTube video
|
| 810 |
+
result, viz = analyzer.process_video(
|
| 811 |
+
"https://youtube.com/watch?v=...",
|
| 812 |
+
model="yolo-v11-s"
|
| 813 |
+
)
|
| 814 |
+
```
|
| 815 |
+
|
| 816 |
+
## 📊 Output Formats
|
| 817 |
+
|
| 818 |
+
### Summary Format
|
| 819 |
+
Human-readable movement analysis summary.
|
| 820 |
+
|
| 821 |
+
### Structured Format
|
| 822 |
+
```json
|
| 823 |
+
{
|
| 824 |
+
"success": true,
|
| 825 |
+
"direction": "up",
|
| 826 |
+
"intensity": "medium",
|
| 827 |
+
"fluidity": 0.85,
|
| 828 |
+
"expansion": 0.72
|
| 829 |
+
}
|
| 830 |
+
```
|
| 831 |
+
|
| 832 |
+
### Full JSON Format
|
| 833 |
+
Complete frame-by-frame analysis with all metrics.
|
| 834 |
+
|
| 835 |
+
## 🎯 Applications
|
| 836 |
+
|
| 837 |
+
- **Sports**: Technique analysis and performance tracking
|
| 838 |
+
- **Dance**: Choreography analysis and movement quality
|
| 839 |
+
- **Healthcare**: Physical therapy and rehabilitation
|
| 840 |
+
- **Research**: Large-scale movement pattern studies
|
| 841 |
+
- **Entertainment**: Interactive applications and games
|
| 842 |
+
- **Education**: Movement teaching and body awareness
|
| 843 |
+
|
| 844 |
+
## 🔗 Integration Examples
|
| 845 |
+
|
| 846 |
+
### Gradio Client
|
| 847 |
+
```python
|
| 848 |
+
from gradio_client import Client
|
| 849 |
+
|
| 850 |
+
client = Client("http://localhost:7860")
|
| 851 |
+
result = client.predict(
|
| 852 |
+
video="path/to/video.mp4",
|
| 853 |
+
model="mediapipe-full",
|
| 854 |
+
api_name="/analyze_enhanced"
|
| 855 |
+
)
|
| 856 |
+
```
|
| 857 |
+
|
| 858 |
+
### Batch Processing
|
| 859 |
+
```python
|
| 860 |
+
results = client.predict(
|
| 861 |
+
files=["video1.mp4", "video2.mp4"],
|
| 862 |
+
model="yolo-v11-s",
|
| 863 |
+
api_name="/batch_analyze"
|
| 864 |
+
)
|
| 865 |
+
```
|
| 866 |
+
""")
|
| 867 |
+
gr.HTML("""
|
| 868 |
+
<div class="author-info">
|
| 869 |
+
<p><strong>Created by:</strong> Csaba Bolyós (BladeSzaSza)</p>
|
| 870 |
+
<p style="margin: 5px 0;">
|
| 871 |
+
<a href="https://github.com/bladeszasza" style="color: #a8e6cf; text-decoration: none;">🔗 GitHub</a> •
|
| 872 |
+
<a href="https://huggingface.co/BladeSzaSza" style="color: #a8e6cf; text-decoration: none;">🤗 Hugging Face</a> •
|
| 873 |
+
<a href="https://www.linkedin.com/in/csaba-bolyós-00a11767/" style="color: #a8e6cf; text-decoration: none;">💼 LinkedIn</a>
|
| 874 |
+
</p>
|
| 875 |
+
<p style="font-size: 12px; opacity: 0.9;">Contact: [email protected]</p>
|
| 876 |
+
</div>
|
| 877 |
+
""")
|
| 878 |
+
|
| 879 |
+
# Footer with proper attribution
|
| 880 |
+
gr.HTML("""
|
| 881 |
+
<div style="text-align: center; padding: 20px; margin-top: 30px; border-top: 1px solid #eee;">
|
| 882 |
+
<p style="color: #666; margin-bottom: 10px;">
|
| 883 |
+
🎭 Laban Movement Analysis - Complete Suite | Heavy Beta Version
|
| 884 |
+
</p>
|
| 885 |
+
<p style="color: #666; font-size: 12px;">
|
| 886 |
+
Created by <strong>Csaba Bolyós</strong> | Powered by MediaPipe, MoveNet & YOLO
|
| 887 |
+
</p>
|
| 888 |
+
<p style="color: #666; font-size: 10px; margin-top: 10px;">
|
| 889 |
+
<a href="https://github.com/bladeszasza" style="color: #40826D;">GitHub</a> •
|
| 890 |
+
<a href="https://huggingface.co/BladeSzaSza" style="color: #40826D;">Hugging Face</a> •
|
| 891 |
+
<a href="https://www.linkedin.com/in/csaba-bolyós-00a11767/" style="color: #40826D;">LinkedIn</a>
|
| 892 |
+
</p>
|
| 893 |
+
</div>
|
| 894 |
+
""")
|
| 895 |
+
|
| 896 |
+
return demo
|
| 897 |
+
|
| 898 |
+
|
| 899 |
+
if __name__ == "__main__":
|
| 900 |
+
demo = create_unified_demo()
|
| 901 |
+
demo.launch(
|
| 902 |
+
server_name="0.0.0.0",
|
| 903 |
+
server_port=7860,
|
| 904 |
+
share=False,
|
| 905 |
+
show_error=True,
|
| 906 |
+
favicon_path=None
|
| 907 |
+
)
|
| 908 |
+
|
| 909 |
+
```
|
| 910 |
+
|
| 911 |
+
## `LabanMovementAnalysis`
|
| 912 |
+
|
| 913 |
+
### Initialization
|
| 914 |
+
|
| 915 |
+
<table>
|
| 916 |
+
<thead>
|
| 917 |
+
<tr>
|
| 918 |
+
<th align="left">name</th>
|
| 919 |
+
<th align="left" style="width: 25%;">type</th>
|
| 920 |
+
<th align="left">default</th>
|
| 921 |
+
<th align="left">description</th>
|
| 922 |
+
</tr>
|
| 923 |
+
</thead>
|
| 924 |
+
<tbody>
|
| 925 |
+
<tr>
|
| 926 |
+
<td align="left"><code>default_model</code></td>
|
| 927 |
+
<td align="left" style="width: 25%;">
|
| 928 |
+
|
| 929 |
+
```python
|
| 930 |
+
str
|
| 931 |
+
```
|
| 932 |
+
|
| 933 |
+
</td>
|
| 934 |
+
<td align="left"><code>"mediapipe"</code></td>
|
| 935 |
+
<td align="left">Default pose estimation model ("mediapipe", "movenet", "yolo")</td>
|
| 936 |
+
</tr>
|
| 937 |
+
|
| 938 |
+
<tr>
|
| 939 |
+
<td align="left"><code>enable_visualization</code></td>
|
| 940 |
+
<td align="left" style="width: 25%;">
|
| 941 |
+
|
| 942 |
+
```python
|
| 943 |
+
bool
|
| 944 |
+
```
|
| 945 |
+
|
| 946 |
+
</td>
|
| 947 |
+
<td align="left"><code>True</code></td>
|
| 948 |
+
<td align="left">Whether to generate visualization video by default</td>
|
| 949 |
+
</tr>
|
| 950 |
+
|
| 951 |
+
<tr>
|
| 952 |
+
<td align="left"><code>include_keypoints</code></td>
|
| 953 |
+
<td align="left" style="width: 25%;">
|
| 954 |
+
|
| 955 |
+
```python
|
| 956 |
+
bool
|
| 957 |
+
```
|
| 958 |
+
|
| 959 |
+
</td>
|
| 960 |
+
<td align="left"><code>False</code></td>
|
| 961 |
+
<td align="left">Whether to include raw keypoints in JSON output</td>
|
| 962 |
+
</tr>
|
| 963 |
+
|
| 964 |
+
<tr>
|
| 965 |
+
<td align="left"><code>enable_webrtc</code></td>
|
| 966 |
+
<td align="left" style="width: 25%;">
|
| 967 |
+
|
| 968 |
+
```python
|
| 969 |
+
bool
|
| 970 |
+
```
|
| 971 |
+
|
| 972 |
+
</td>
|
| 973 |
+
<td align="left"><code>False</code></td>
|
| 974 |
+
<td align="left">Whether to enable WebRTC real-time analysis</td>
|
| 975 |
+
</tr>
|
| 976 |
+
|
| 977 |
+
<tr>
|
| 978 |
+
<td align="left"><code>label</code></td>
|
| 979 |
+
<td align="left" style="width: 25%;">
|
| 980 |
+
|
| 981 |
+
```python
|
| 982 |
+
typing.Optional[str][str, None]
|
| 983 |
+
```
|
| 984 |
+
|
| 985 |
+
</td>
|
| 986 |
+
<td align="left"><code>None</code></td>
|
| 987 |
+
<td align="left">Component label</td>
|
| 988 |
+
</tr>
|
| 989 |
+
|
| 990 |
+
<tr>
|
| 991 |
+
<td align="left"><code>every</code></td>
|
| 992 |
+
<td align="left" style="width: 25%;">
|
| 993 |
+
|
| 994 |
+
```python
|
| 995 |
+
typing.Optional[float][float, None]
|
| 996 |
+
```
|
| 997 |
+
|
| 998 |
+
</td>
|
| 999 |
+
<td align="left"><code>None</code></td>
|
| 1000 |
+
<td align="left">None</td>
|
| 1001 |
+
</tr>
|
| 1002 |
+
|
| 1003 |
+
<tr>
|
| 1004 |
+
<td align="left"><code>show_label</code></td>
|
| 1005 |
+
<td align="left" style="width: 25%;">
|
| 1006 |
+
|
| 1007 |
+
```python
|
| 1008 |
+
typing.Optional[bool][bool, None]
|
| 1009 |
+
```
|
| 1010 |
+
|
| 1011 |
+
</td>
|
| 1012 |
+
<td align="left"><code>None</code></td>
|
| 1013 |
+
<td align="left">None</td>
|
| 1014 |
+
</tr>
|
| 1015 |
+
|
| 1016 |
+
<tr>
|
| 1017 |
+
<td align="left"><code>container</code></td>
|
| 1018 |
+
<td align="left" style="width: 25%;">
|
| 1019 |
+
|
| 1020 |
+
```python
|
| 1021 |
+
bool
|
| 1022 |
+
```
|
| 1023 |
+
|
| 1024 |
+
</td>
|
| 1025 |
+
<td align="left"><code>True</code></td>
|
| 1026 |
+
<td align="left">None</td>
|
| 1027 |
+
</tr>
|
| 1028 |
+
|
| 1029 |
+
<tr>
|
| 1030 |
+
<td align="left"><code>scale</code></td>
|
| 1031 |
+
<td align="left" style="width: 25%;">
|
| 1032 |
+
|
| 1033 |
+
```python
|
| 1034 |
+
typing.Optional[int][int, None]
|
| 1035 |
+
```
|
| 1036 |
+
|
| 1037 |
+
</td>
|
| 1038 |
+
<td align="left"><code>None</code></td>
|
| 1039 |
+
<td align="left">None</td>
|
| 1040 |
+
</tr>
|
| 1041 |
+
|
| 1042 |
+
<tr>
|
| 1043 |
+
<td align="left"><code>min_width</code></td>
|
| 1044 |
+
<td align="left" style="width: 25%;">
|
| 1045 |
+
|
| 1046 |
+
```python
|
| 1047 |
+
int
|
| 1048 |
+
```
|
| 1049 |
+
|
| 1050 |
+
</td>
|
| 1051 |
+
<td align="left"><code>160</code></td>
|
| 1052 |
+
<td align="left">None</td>
|
| 1053 |
+
</tr>
|
| 1054 |
+
|
| 1055 |
+
<tr>
|
| 1056 |
+
<td align="left"><code>interactive</code></td>
|
| 1057 |
+
<td align="left" style="width: 25%;">
|
| 1058 |
+
|
| 1059 |
+
```python
|
| 1060 |
+
typing.Optional[bool][bool, None]
|
| 1061 |
+
```
|
| 1062 |
+
|
| 1063 |
+
</td>
|
| 1064 |
+
<td align="left"><code>None</code></td>
|
| 1065 |
+
<td align="left">None</td>
|
| 1066 |
+
</tr>
|
| 1067 |
+
|
| 1068 |
+
<tr>
|
| 1069 |
+
<td align="left"><code>visible</code></td>
|
| 1070 |
+
<td align="left" style="width: 25%;">
|
| 1071 |
+
|
| 1072 |
+
```python
|
| 1073 |
+
bool
|
| 1074 |
+
```
|
| 1075 |
+
|
| 1076 |
+
</td>
|
| 1077 |
+
<td align="left"><code>True</code></td>
|
| 1078 |
+
<td align="left">None</td>
|
| 1079 |
+
</tr>
|
| 1080 |
+
|
| 1081 |
+
<tr>
|
| 1082 |
+
<td align="left"><code>elem_id</code></td>
|
| 1083 |
+
<td align="left" style="width: 25%;">
|
| 1084 |
+
|
| 1085 |
+
```python
|
| 1086 |
+
typing.Optional[str][str, None]
|
| 1087 |
+
```
|
| 1088 |
+
|
| 1089 |
+
</td>
|
| 1090 |
+
<td align="left"><code>None</code></td>
|
| 1091 |
+
<td align="left">None</td>
|
| 1092 |
+
</tr>
|
| 1093 |
+
|
| 1094 |
+
<tr>
|
| 1095 |
+
<td align="left"><code>elem_classes</code></td>
|
| 1096 |
+
<td align="left" style="width: 25%;">
|
| 1097 |
+
|
| 1098 |
+
```python
|
| 1099 |
+
typing.Optional[typing.List[str]][
|
| 1100 |
+
typing.List[str][str], None
|
| 1101 |
+
]
|
| 1102 |
+
```
|
| 1103 |
+
|
| 1104 |
+
</td>
|
| 1105 |
+
<td align="left"><code>None</code></td>
|
| 1106 |
+
<td align="left">None</td>
|
| 1107 |
+
</tr>
|
| 1108 |
+
|
| 1109 |
+
<tr>
|
| 1110 |
+
<td align="left"><code>render</code></td>
|
| 1111 |
+
<td align="left" style="width: 25%;">
|
| 1112 |
+
|
| 1113 |
+
```python
|
| 1114 |
+
bool
|
| 1115 |
+
```
|
| 1116 |
+
|
| 1117 |
+
</td>
|
| 1118 |
+
<td align="left"><code>True</code></td>
|
| 1119 |
+
<td align="left">None</td>
|
| 1120 |
+
</tr>
|
| 1121 |
+
</tbody></table>
|
| 1122 |
+
|
| 1123 |
+
|
| 1124 |
+
|
| 1125 |
+
|
| 1126 |
+
### User function
|
| 1127 |
+
|
| 1128 |
+
The impact on the users predict function varies depending on whether the component is used as an input or output for an event (or both).
|
| 1129 |
+
|
| 1130 |
+
- When used as an Input, the component only impacts the input signature of the user function.
|
| 1131 |
+
- When used as an output, the component only impacts the return signature of the user function.
|
| 1132 |
+
|
| 1133 |
+
The code snippet below is accurate in cases where the component is used as both an input and an output.
|
| 1134 |
+
|
| 1135 |
+
- **As output:** Is passed, processed data for analysis.
|
| 1136 |
+
- **As input:** Should return, analysis results.
|
| 1137 |
+
|
| 1138 |
+
```python
|
| 1139 |
+
def predict(
|
| 1140 |
+
value: typing.Dict[str, typing.Any][str, typing.Any]
|
| 1141 |
+
) -> typing.Any:
|
| 1142 |
+
return value
|
| 1143 |
+
```
|
| 1144 |
+
|
app.py
ADDED
|
@@ -0,0 +1,48 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
#!/usr/bin/env python3
|
| 2 |
+
"""
|
| 3 |
+
Laban Movement Analysis - Complete Suite
|
| 4 |
+
Hugging Face Spaces Deployment
|
| 5 |
+
|
| 6 |
+
Created by: Csaba Bolyós (BladeSzaSza)
|
| 7 |
+
Contact: [email protected]
|
| 8 |
+
GitHub: https://github.com/bladeszasza
|
| 9 |
+
LinkedIn: https://www.linkedin.com/in/csaba-bolyós-00a11767/
|
| 10 |
+
Hugging Face: https://huggingface.co/BladeSzaSza
|
| 11 |
+
|
| 12 |
+
Heavy Beta Version - Under Active Development
|
| 13 |
+
"""
|
| 14 |
+
|
| 15 |
+
import sys
|
| 16 |
+
from pathlib import Path
|
| 17 |
+
|
| 18 |
+
# Import version info
|
| 19 |
+
try:
|
| 20 |
+
from version import __version__, __author__, get_version_info
|
| 21 |
+
print(f"🎭 Laban Movement Analysis v{__version__} by {__author__}")
|
| 22 |
+
except ImportError:
|
| 23 |
+
__version__ = "0.01-beta"
|
| 24 |
+
print("🎭 Laban Movement Analysis - Version info not found")
|
| 25 |
+
|
| 26 |
+
# Add demo directory to path
|
| 27 |
+
sys.path.insert(0, str(Path(__file__).parent / "demo"))
|
| 28 |
+
|
| 29 |
+
try:
|
| 30 |
+
from app import create_unified_demo
|
| 31 |
+
|
| 32 |
+
if __name__ == "__main__":
|
| 33 |
+
demo = create_unified_demo()
|
| 34 |
+
demo.launch(
|
| 35 |
+
server_name="0.0.0.0",
|
| 36 |
+
server_port=7860,
|
| 37 |
+
share=False,
|
| 38 |
+
show_error=True,
|
| 39 |
+
favicon_path=None,
|
| 40 |
+
show_api=True
|
| 41 |
+
)
|
| 42 |
+
|
| 43 |
+
except ImportError as e:
|
| 44 |
+
print(f"Import error: {e}")
|
| 45 |
+
print("Make sure all dependencies are installed.")
|
| 46 |
+
|
| 47 |
+
except Exception as e:
|
| 48 |
+
print(f"Error launching demo: {e}")
|
backend/gradio_labanmovementanalysis/__init__.py
ADDED
|
@@ -0,0 +1,51 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
from .labanmovementanalysis import LabanMovementAnalysis
|
| 2 |
+
from . import video_utils
|
| 3 |
+
from . import pose_estimation
|
| 4 |
+
from . import notation_engine
|
| 5 |
+
from . import json_generator
|
| 6 |
+
from . import visualizer
|
| 7 |
+
|
| 8 |
+
# Import agent API if available
|
| 9 |
+
try:
|
| 10 |
+
from . import agent_api
|
| 11 |
+
from .agent_api import LabanAgentAPI, quick_analyze, analyze_and_summarize
|
| 12 |
+
_has_agent_api = True
|
| 13 |
+
except ImportError:
|
| 14 |
+
_has_agent_api = False
|
| 15 |
+
|
| 16 |
+
__all__ = [
|
| 17 |
+
'LabanMovementAnalysis',
|
| 18 |
+
'video_utils',
|
| 19 |
+
'pose_estimation',
|
| 20 |
+
'notation_engine',
|
| 21 |
+
'json_generator',
|
| 22 |
+
'visualizer'
|
| 23 |
+
]
|
| 24 |
+
|
| 25 |
+
# Add agent API to exports if available
|
| 26 |
+
if _has_agent_api:
|
| 27 |
+
__all__.extend(['agent_api', 'LabanAgentAPI', 'quick_analyze', 'analyze_and_summarize'])
|
| 28 |
+
|
| 29 |
+
# Import enhanced features if available
|
| 30 |
+
try:
|
| 31 |
+
from . import video_downloader
|
| 32 |
+
from .video_downloader import VideoDownloader, SmartVideoInput
|
| 33 |
+
__all__.extend(['video_downloader', 'VideoDownloader', 'SmartVideoInput'])
|
| 34 |
+
except ImportError:
|
| 35 |
+
pass
|
| 36 |
+
|
| 37 |
+
try:
|
| 38 |
+
from . import webrtc_handler
|
| 39 |
+
from .webrtc_handler import WebRTCMovementAnalyzer, WebRTCGradioInterface
|
| 40 |
+
__all__.extend(['webrtc_handler', 'WebRTCMovementAnalyzer', 'WebRTCGradioInterface'])
|
| 41 |
+
except ImportError:
|
| 42 |
+
pass
|
| 43 |
+
|
| 44 |
+
try:
|
| 45 |
+
# SkateFormer integration reserved for Version 2
|
| 46 |
+
# from . import skateformer_integration
|
| 47 |
+
# from .skateformer_integration import SkateFormerAnalyzer, SkateFormerConfig
|
| 48 |
+
# __all__.extend(['skateformer_integration', 'SkateFormerAnalyzer', 'SkateFormerConfig'])
|
| 49 |
+
pass
|
| 50 |
+
except ImportError:
|
| 51 |
+
pass
|
backend/gradio_labanmovementanalysis/__pycache__/__init__.cpython-312.pyc
ADDED
|
Binary file (579 Bytes). View file
|
|
|
backend/gradio_labanmovementanalysis/__pycache__/__init__.cpython-313.pyc
ADDED
|
Binary file (1.6 kB). View file
|
|
|
backend/gradio_labanmovementanalysis/__pycache__/agent_api.cpython-312.pyc
ADDED
|
Binary file (16 kB). View file
|
|
|
backend/gradio_labanmovementanalysis/__pycache__/json_generator.cpython-312.pyc
ADDED
|
Binary file (9.96 kB). View file
|
|
|
backend/gradio_labanmovementanalysis/__pycache__/labanmovementanalysis.cpython-312.pyc
ADDED
|
Binary file (14.6 kB). View file
|
|
|
backend/gradio_labanmovementanalysis/__pycache__/labanmovementanalysis.cpython-313.pyc
ADDED
|
Binary file (23.4 kB). View file
|
|
|
backend/gradio_labanmovementanalysis/__pycache__/notation_engine.cpython-312.pyc
ADDED
|
Binary file (12.7 kB). View file
|
|
|
backend/gradio_labanmovementanalysis/__pycache__/pose_estimation.cpython-312.pyc
ADDED
|
Binary file (15.5 kB). View file
|
|
|
backend/gradio_labanmovementanalysis/__pycache__/skateformer_integration.cpython-312.pyc
ADDED
|
Binary file (20.2 kB). View file
|
|
|
backend/gradio_labanmovementanalysis/__pycache__/video_downloader.cpython-312.pyc
ADDED
|
Binary file (12.8 kB). View file
|
|
|
backend/gradio_labanmovementanalysis/__pycache__/video_utils.cpython-312.pyc
ADDED
|
Binary file (6.13 kB). View file
|
|
|
backend/gradio_labanmovementanalysis/__pycache__/visualizer.cpython-312.pyc
ADDED
|
Binary file (17.4 kB). View file
|
|
|
backend/gradio_labanmovementanalysis/__pycache__/webrtc_handler.cpython-312.pyc
ADDED
|
Binary file (11.9 kB). View file
|
|
|
backend/gradio_labanmovementanalysis/agent_api.py
ADDED
|
@@ -0,0 +1,434 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""
|
| 2 |
+
Agent-friendly API for Laban Movement Analysis
|
| 3 |
+
Provides simplified interfaces for AI agents and automation
|
| 4 |
+
"""
|
| 5 |
+
|
| 6 |
+
import asyncio
|
| 7 |
+
import json
|
| 8 |
+
import logging
|
| 9 |
+
from pathlib import Path
|
| 10 |
+
from typing import Any, Dict, List, Optional, Union
|
| 11 |
+
from dataclasses import dataclass, asdict
|
| 12 |
+
from enum import Enum
|
| 13 |
+
|
| 14 |
+
# Configure logging
|
| 15 |
+
logging.basicConfig(level=logging.INFO)
|
| 16 |
+
logger = logging.getLogger(__name__)
|
| 17 |
+
|
| 18 |
+
# Import the main component
|
| 19 |
+
from .labanmovementanalysis import LabanMovementAnalysis
|
| 20 |
+
|
| 21 |
+
|
| 22 |
+
class PoseModel(str, Enum):
|
| 23 |
+
"""Available pose estimation models"""
|
| 24 |
+
MEDIAPIPE = "mediapipe"
|
| 25 |
+
MOVENET = "movenet"
|
| 26 |
+
YOLO = "yolo"
|
| 27 |
+
|
| 28 |
+
|
| 29 |
+
class MovementIntensity(str, Enum):
|
| 30 |
+
"""Movement intensity levels"""
|
| 31 |
+
LOW = "low"
|
| 32 |
+
MEDIUM = "medium"
|
| 33 |
+
HIGH = "high"
|
| 34 |
+
|
| 35 |
+
|
| 36 |
+
class MovementDirection(str, Enum):
|
| 37 |
+
"""Movement direction categories"""
|
| 38 |
+
UP = "up"
|
| 39 |
+
DOWN = "down"
|
| 40 |
+
LEFT = "left"
|
| 41 |
+
RIGHT = "right"
|
| 42 |
+
STATIONARY = "stationary"
|
| 43 |
+
|
| 44 |
+
|
| 45 |
+
@dataclass
|
| 46 |
+
class AnalysisResult:
|
| 47 |
+
"""Structured analysis result for agents"""
|
| 48 |
+
success: bool
|
| 49 |
+
video_path: str
|
| 50 |
+
duration_seconds: float
|
| 51 |
+
fps: float
|
| 52 |
+
dominant_direction: MovementDirection
|
| 53 |
+
dominant_intensity: MovementIntensity
|
| 54 |
+
dominant_speed: str
|
| 55 |
+
movement_segments: List[Dict[str, Any]]
|
| 56 |
+
fluidity_score: float
|
| 57 |
+
expansion_score: float
|
| 58 |
+
error: Optional[str] = None
|
| 59 |
+
raw_data: Optional[Dict[str, Any]] = None
|
| 60 |
+
visualization_path: Optional[str] = None
|
| 61 |
+
|
| 62 |
+
|
| 63 |
+
class LabanAgentAPI:
|
| 64 |
+
"""
|
| 65 |
+
Simplified API for AI agents to analyze movement in videos.
|
| 66 |
+
Provides high-level methods with structured outputs.
|
| 67 |
+
"""
|
| 68 |
+
|
| 69 |
+
# Gradio component compatibility
|
| 70 |
+
events = {}
|
| 71 |
+
|
| 72 |
+
def __init__(self, default_model: PoseModel = PoseModel.MEDIAPIPE):
|
| 73 |
+
"""
|
| 74 |
+
Initialize the agent API.
|
| 75 |
+
|
| 76 |
+
Args:
|
| 77 |
+
default_model: Default pose estimation model to use
|
| 78 |
+
"""
|
| 79 |
+
self.analyzer = LabanMovementAnalysis(default_model=default_model.value)
|
| 80 |
+
self.default_model = default_model
|
| 81 |
+
self._analysis_cache = {}
|
| 82 |
+
|
| 83 |
+
def analyze(
|
| 84 |
+
self,
|
| 85 |
+
video_path: Union[str, Path],
|
| 86 |
+
model: Optional[PoseModel] = None,
|
| 87 |
+
generate_visualization: bool = False,
|
| 88 |
+
cache_results: bool = True
|
| 89 |
+
) -> AnalysisResult:
|
| 90 |
+
"""
|
| 91 |
+
Analyze a video and return structured results.
|
| 92 |
+
|
| 93 |
+
Args:
|
| 94 |
+
video_path: Path to video file
|
| 95 |
+
model: Pose estimation model to use (defaults to instance default)
|
| 96 |
+
generate_visualization: Whether to create annotated video
|
| 97 |
+
cache_results: Whether to cache results for later retrieval
|
| 98 |
+
|
| 99 |
+
Returns:
|
| 100 |
+
AnalysisResult with structured movement data
|
| 101 |
+
"""
|
| 102 |
+
try:
|
| 103 |
+
# Convert path to string
|
| 104 |
+
video_path = str(video_path)
|
| 105 |
+
|
| 106 |
+
# Use default model if not specified
|
| 107 |
+
if model is None:
|
| 108 |
+
model = self.default_model
|
| 109 |
+
|
| 110 |
+
# Process video
|
| 111 |
+
json_output, viz_video = self.analyzer.process_video(
|
| 112 |
+
video_path,
|
| 113 |
+
model=model.value,
|
| 114 |
+
enable_visualization=generate_visualization,
|
| 115 |
+
include_keypoints=False
|
| 116 |
+
)
|
| 117 |
+
|
| 118 |
+
# Parse results
|
| 119 |
+
result = self._parse_analysis_output(
|
| 120 |
+
json_output,
|
| 121 |
+
video_path,
|
| 122 |
+
viz_video
|
| 123 |
+
)
|
| 124 |
+
|
| 125 |
+
# Cache if requested
|
| 126 |
+
if cache_results:
|
| 127 |
+
cache_key = f"{Path(video_path).stem}_{model.value}"
|
| 128 |
+
self._analysis_cache[cache_key] = result
|
| 129 |
+
|
| 130 |
+
return result
|
| 131 |
+
|
| 132 |
+
except Exception as e:
|
| 133 |
+
logger.error(f"Analysis failed: {str(e)}")
|
| 134 |
+
return AnalysisResult(
|
| 135 |
+
success=False,
|
| 136 |
+
video_path=str(video_path),
|
| 137 |
+
duration_seconds=0.0,
|
| 138 |
+
fps=0.0,
|
| 139 |
+
dominant_direction=MovementDirection.STATIONARY,
|
| 140 |
+
dominant_intensity=MovementIntensity.LOW,
|
| 141 |
+
dominant_speed="unknown",
|
| 142 |
+
movement_segments=[],
|
| 143 |
+
fluidity_score=0.0,
|
| 144 |
+
expansion_score=0.0,
|
| 145 |
+
error=str(e)
|
| 146 |
+
)
|
| 147 |
+
|
| 148 |
+
async def analyze_async(
|
| 149 |
+
self,
|
| 150 |
+
video_path: Union[str, Path],
|
| 151 |
+
model: Optional[PoseModel] = None,
|
| 152 |
+
generate_visualization: bool = False
|
| 153 |
+
) -> AnalysisResult:
|
| 154 |
+
"""
|
| 155 |
+
Asynchronously analyze a video.
|
| 156 |
+
|
| 157 |
+
Args:
|
| 158 |
+
video_path: Path to video file
|
| 159 |
+
model: Pose estimation model to use
|
| 160 |
+
generate_visualization: Whether to create annotated video
|
| 161 |
+
|
| 162 |
+
Returns:
|
| 163 |
+
AnalysisResult with structured movement data
|
| 164 |
+
"""
|
| 165 |
+
loop = asyncio.get_event_loop()
|
| 166 |
+
return await loop.run_in_executor(
|
| 167 |
+
None,
|
| 168 |
+
self.analyze,
|
| 169 |
+
video_path,
|
| 170 |
+
model,
|
| 171 |
+
generate_visualization
|
| 172 |
+
)
|
| 173 |
+
|
| 174 |
+
def batch_analyze(
|
| 175 |
+
self,
|
| 176 |
+
video_paths: List[Union[str, Path]],
|
| 177 |
+
model: Optional[PoseModel] = None,
|
| 178 |
+
parallel: bool = True,
|
| 179 |
+
max_workers: int = 4
|
| 180 |
+
) -> List[AnalysisResult]:
|
| 181 |
+
"""
|
| 182 |
+
Analyze multiple videos in batch.
|
| 183 |
+
|
| 184 |
+
Args:
|
| 185 |
+
video_paths: List of video file paths
|
| 186 |
+
model: Pose estimation model to use
|
| 187 |
+
parallel: Whether to process in parallel
|
| 188 |
+
max_workers: Maximum parallel workers
|
| 189 |
+
|
| 190 |
+
Returns:
|
| 191 |
+
List of AnalysisResult objects
|
| 192 |
+
"""
|
| 193 |
+
if parallel:
|
| 194 |
+
from concurrent.futures import ThreadPoolExecutor
|
| 195 |
+
|
| 196 |
+
with ThreadPoolExecutor(max_workers=max_workers) as executor:
|
| 197 |
+
futures = [
|
| 198 |
+
executor.submit(self.analyze, path, model, False)
|
| 199 |
+
for path in video_paths
|
| 200 |
+
]
|
| 201 |
+
results = [future.result() for future in futures]
|
| 202 |
+
else:
|
| 203 |
+
results = [
|
| 204 |
+
self.analyze(path, model, False)
|
| 205 |
+
for path in video_paths
|
| 206 |
+
]
|
| 207 |
+
|
| 208 |
+
return results
|
| 209 |
+
|
| 210 |
+
def get_movement_summary(self, analysis_result: AnalysisResult) -> str:
|
| 211 |
+
"""
|
| 212 |
+
Generate a natural language summary of movement analysis.
|
| 213 |
+
|
| 214 |
+
Args:
|
| 215 |
+
analysis_result: Analysis result to summarize
|
| 216 |
+
|
| 217 |
+
Returns:
|
| 218 |
+
Human-readable summary string
|
| 219 |
+
"""
|
| 220 |
+
if not analysis_result.success:
|
| 221 |
+
return f"Analysis failed: {analysis_result.error}"
|
| 222 |
+
|
| 223 |
+
summary_parts = [
|
| 224 |
+
f"Movement Analysis Summary for {Path(analysis_result.video_path).name}:",
|
| 225 |
+
f"- Duration: {analysis_result.duration_seconds:.1f} seconds",
|
| 226 |
+
f"- Primary movement direction: {analysis_result.dominant_direction.value}",
|
| 227 |
+
f"- Movement intensity: {analysis_result.dominant_intensity.value}",
|
| 228 |
+
f"- Movement speed: {analysis_result.dominant_speed}",
|
| 229 |
+
f"- Fluidity score: {analysis_result.fluidity_score:.2f}/1.00",
|
| 230 |
+
f"- Expansion score: {analysis_result.expansion_score:.2f}/1.00"
|
| 231 |
+
]
|
| 232 |
+
|
| 233 |
+
if analysis_result.movement_segments:
|
| 234 |
+
summary_parts.append(f"- Detected {len(analysis_result.movement_segments)} movement segments")
|
| 235 |
+
|
| 236 |
+
return "\n".join(summary_parts)
|
| 237 |
+
|
| 238 |
+
def compare_videos(
|
| 239 |
+
self,
|
| 240 |
+
video_path1: Union[str, Path],
|
| 241 |
+
video_path2: Union[str, Path],
|
| 242 |
+
model: Optional[PoseModel] = None
|
| 243 |
+
) -> Dict[str, Any]:
|
| 244 |
+
"""
|
| 245 |
+
Compare movement patterns between two videos.
|
| 246 |
+
|
| 247 |
+
Args:
|
| 248 |
+
video_path1: First video path
|
| 249 |
+
video_path2: Second video path
|
| 250 |
+
model: Pose estimation model to use
|
| 251 |
+
|
| 252 |
+
Returns:
|
| 253 |
+
Comparison results dictionary
|
| 254 |
+
"""
|
| 255 |
+
# Analyze both videos
|
| 256 |
+
result1 = self.analyze(video_path1, model, False)
|
| 257 |
+
result2 = self.analyze(video_path2, model, False)
|
| 258 |
+
|
| 259 |
+
if not result1.success or not result2.success:
|
| 260 |
+
return {
|
| 261 |
+
"success": False,
|
| 262 |
+
"error": "One or both analyses failed"
|
| 263 |
+
}
|
| 264 |
+
|
| 265 |
+
# Compare metrics
|
| 266 |
+
comparison = {
|
| 267 |
+
"success": True,
|
| 268 |
+
"video1": Path(video_path1).name,
|
| 269 |
+
"video2": Path(video_path2).name,
|
| 270 |
+
"metrics": {
|
| 271 |
+
"direction_match": result1.dominant_direction == result2.dominant_direction,
|
| 272 |
+
"intensity_match": result1.dominant_intensity == result2.dominant_intensity,
|
| 273 |
+
"speed_match": result1.dominant_speed == result2.dominant_speed,
|
| 274 |
+
"fluidity_difference": abs(result1.fluidity_score - result2.fluidity_score),
|
| 275 |
+
"expansion_difference": abs(result1.expansion_score - result2.expansion_score)
|
| 276 |
+
},
|
| 277 |
+
"details": {
|
| 278 |
+
"video1": {
|
| 279 |
+
"direction": result1.dominant_direction.value,
|
| 280 |
+
"intensity": result1.dominant_intensity.value,
|
| 281 |
+
"speed": result1.dominant_speed,
|
| 282 |
+
"fluidity": result1.fluidity_score,
|
| 283 |
+
"expansion": result1.expansion_score
|
| 284 |
+
},
|
| 285 |
+
"video2": {
|
| 286 |
+
"direction": result2.dominant_direction.value,
|
| 287 |
+
"intensity": result2.dominant_intensity.value,
|
| 288 |
+
"speed": result2.dominant_speed,
|
| 289 |
+
"fluidity": result2.fluidity_score,
|
| 290 |
+
"expansion": result2.expansion_score
|
| 291 |
+
}
|
| 292 |
+
}
|
| 293 |
+
}
|
| 294 |
+
|
| 295 |
+
return comparison
|
| 296 |
+
|
| 297 |
+
def filter_by_movement(
|
| 298 |
+
self,
|
| 299 |
+
video_paths: List[Union[str, Path]],
|
| 300 |
+
direction: Optional[MovementDirection] = None,
|
| 301 |
+
intensity: Optional[MovementIntensity] = None,
|
| 302 |
+
min_fluidity: Optional[float] = None,
|
| 303 |
+
min_expansion: Optional[float] = None
|
| 304 |
+
) -> List[AnalysisResult]:
|
| 305 |
+
"""
|
| 306 |
+
Filter videos based on movement characteristics.
|
| 307 |
+
|
| 308 |
+
Args:
|
| 309 |
+
video_paths: List of video paths to analyze
|
| 310 |
+
direction: Filter by movement direction
|
| 311 |
+
intensity: Filter by movement intensity
|
| 312 |
+
min_fluidity: Minimum fluidity score
|
| 313 |
+
min_expansion: Minimum expansion score
|
| 314 |
+
|
| 315 |
+
Returns:
|
| 316 |
+
List of AnalysisResults that match criteria
|
| 317 |
+
"""
|
| 318 |
+
# Analyze all videos
|
| 319 |
+
results = self.batch_analyze(video_paths)
|
| 320 |
+
|
| 321 |
+
# Apply filters
|
| 322 |
+
filtered = []
|
| 323 |
+
for result in results:
|
| 324 |
+
if not result.success:
|
| 325 |
+
continue
|
| 326 |
+
|
| 327 |
+
if direction and result.dominant_direction != direction:
|
| 328 |
+
continue
|
| 329 |
+
|
| 330 |
+
if intensity and result.dominant_intensity != intensity:
|
| 331 |
+
continue
|
| 332 |
+
|
| 333 |
+
if min_fluidity and result.fluidity_score < min_fluidity:
|
| 334 |
+
continue
|
| 335 |
+
|
| 336 |
+
if min_expansion and result.expansion_score < min_expansion:
|
| 337 |
+
continue
|
| 338 |
+
|
| 339 |
+
filtered.append(result)
|
| 340 |
+
|
| 341 |
+
return filtered
|
| 342 |
+
|
| 343 |
+
def _parse_analysis_output(
|
| 344 |
+
self,
|
| 345 |
+
json_output: Dict[str, Any],
|
| 346 |
+
video_path: str,
|
| 347 |
+
viz_path: Optional[str]
|
| 348 |
+
) -> AnalysisResult:
|
| 349 |
+
"""Parse JSON output into structured result"""
|
| 350 |
+
try:
|
| 351 |
+
# Extract video info
|
| 352 |
+
video_info = json_output.get("video_info", {})
|
| 353 |
+
duration = video_info.get("duration_seconds", 0.0)
|
| 354 |
+
fps = video_info.get("fps", 0.0)
|
| 355 |
+
|
| 356 |
+
# Extract movement summary
|
| 357 |
+
movement_analysis = json_output.get("movement_analysis", {})
|
| 358 |
+
summary = movement_analysis.get("summary", {})
|
| 359 |
+
|
| 360 |
+
# Parse dominant metrics
|
| 361 |
+
direction_data = summary.get("direction", {})
|
| 362 |
+
dominant_direction = direction_data.get("dominant", "stationary")
|
| 363 |
+
dominant_direction = MovementDirection(dominant_direction.lower())
|
| 364 |
+
|
| 365 |
+
intensity_data = summary.get("intensity", {})
|
| 366 |
+
dominant_intensity = intensity_data.get("dominant", "low")
|
| 367 |
+
dominant_intensity = MovementIntensity(dominant_intensity.lower())
|
| 368 |
+
|
| 369 |
+
speed_data = summary.get("speed", {})
|
| 370 |
+
dominant_speed = speed_data.get("dominant", "unknown")
|
| 371 |
+
|
| 372 |
+
# Get segments
|
| 373 |
+
segments = summary.get("movement_segments", [])
|
| 374 |
+
|
| 375 |
+
# Calculate aggregate scores
|
| 376 |
+
frames = movement_analysis.get("frames", [])
|
| 377 |
+
fluidity_scores = [f.get("metrics", {}).get("fluidity", 0) for f in frames]
|
| 378 |
+
expansion_scores = [f.get("metrics", {}).get("expansion", 0) for f in frames]
|
| 379 |
+
|
| 380 |
+
avg_fluidity = sum(fluidity_scores) / len(fluidity_scores) if fluidity_scores else 0.0
|
| 381 |
+
avg_expansion = sum(expansion_scores) / len(expansion_scores) if expansion_scores else 0.0
|
| 382 |
+
|
| 383 |
+
return AnalysisResult(
|
| 384 |
+
success=True,
|
| 385 |
+
video_path=video_path,
|
| 386 |
+
duration_seconds=duration,
|
| 387 |
+
fps=fps,
|
| 388 |
+
dominant_direction=dominant_direction,
|
| 389 |
+
dominant_intensity=dominant_intensity,
|
| 390 |
+
dominant_speed=dominant_speed,
|
| 391 |
+
movement_segments=segments,
|
| 392 |
+
fluidity_score=avg_fluidity,
|
| 393 |
+
expansion_score=avg_expansion,
|
| 394 |
+
raw_data=json_output,
|
| 395 |
+
visualization_path=viz_path
|
| 396 |
+
)
|
| 397 |
+
|
| 398 |
+
except Exception as e:
|
| 399 |
+
logger.error(f"Failed to parse analysis output: {str(e)}")
|
| 400 |
+
return AnalysisResult(
|
| 401 |
+
success=False,
|
| 402 |
+
video_path=video_path,
|
| 403 |
+
duration_seconds=0.0,
|
| 404 |
+
fps=0.0,
|
| 405 |
+
dominant_direction=MovementDirection.STATIONARY,
|
| 406 |
+
dominant_intensity=MovementIntensity.LOW,
|
| 407 |
+
dominant_speed="unknown",
|
| 408 |
+
movement_segments=[],
|
| 409 |
+
fluidity_score=0.0,
|
| 410 |
+
expansion_score=0.0,
|
| 411 |
+
error=f"Parse error: {str(e)}",
|
| 412 |
+
raw_data=json_output
|
| 413 |
+
)
|
| 414 |
+
|
| 415 |
+
|
| 416 |
+
# Convenience functions for quick analysis
|
| 417 |
+
def quick_analyze(video_path: Union[str, Path]) -> Dict[str, Any]:
|
| 418 |
+
"""Quick analysis with default settings, returns dict"""
|
| 419 |
+
api = LabanAgentAPI()
|
| 420 |
+
result = api.analyze(video_path)
|
| 421 |
+
return asdict(result)
|
| 422 |
+
|
| 423 |
+
# Gradio component compatibility
|
| 424 |
+
quick_analyze.events = {}
|
| 425 |
+
|
| 426 |
+
|
| 427 |
+
def analyze_and_summarize(video_path: Union[str, Path]) -> str:
|
| 428 |
+
"""Analyze video and return natural language summary"""
|
| 429 |
+
api = LabanAgentAPI()
|
| 430 |
+
result = api.analyze(video_path)
|
| 431 |
+
return api.get_movement_summary(result)
|
| 432 |
+
|
| 433 |
+
# Gradio component compatibility
|
| 434 |
+
analyze_and_summarize.events = {}
|
backend/gradio_labanmovementanalysis/json_generator.py
ADDED
|
@@ -0,0 +1,250 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""
|
| 2 |
+
JSON generator for converting movement annotations and keypoint data to structured JSON format.
|
| 3 |
+
"""
|
| 4 |
+
|
| 5 |
+
import json
|
| 6 |
+
from typing import List, Dict, Any, Optional
|
| 7 |
+
from datetime import datetime
|
| 8 |
+
|
| 9 |
+
from .pose_estimation import PoseResult, Keypoint
|
| 10 |
+
from .notation_engine import MovementMetrics, Direction, Intensity, Speed
|
| 11 |
+
|
| 12 |
+
|
| 13 |
+
def generate_json(
|
| 14 |
+
movement_metrics: List[MovementMetrics],
|
| 15 |
+
pose_results: Optional[List[List[PoseResult]]] = None,
|
| 16 |
+
video_metadata: Optional[Dict[str, Any]] = None,
|
| 17 |
+
include_keypoints: bool = False
|
| 18 |
+
) -> Dict[str, Any]:
|
| 19 |
+
"""
|
| 20 |
+
Generate structured JSON output from movement analysis results.
|
| 21 |
+
|
| 22 |
+
Args:
|
| 23 |
+
movement_metrics: List of movement metrics per frame
|
| 24 |
+
pose_results: Optional pose keypoints per frame
|
| 25 |
+
video_metadata: Optional video metadata (fps, dimensions, etc.)
|
| 26 |
+
include_keypoints: Whether to include raw keypoint data
|
| 27 |
+
|
| 28 |
+
Returns:
|
| 29 |
+
Dictionary containing formatted analysis results
|
| 30 |
+
"""
|
| 31 |
+
output = {
|
| 32 |
+
"analysis_metadata": {
|
| 33 |
+
"timestamp": datetime.now().isoformat(),
|
| 34 |
+
"version": "1.0.0",
|
| 35 |
+
"model_info": video_metadata.get("model_info", {}) if video_metadata else {}
|
| 36 |
+
},
|
| 37 |
+
"video_info": {},
|
| 38 |
+
"movement_analysis": {
|
| 39 |
+
"frame_count": len(movement_metrics),
|
| 40 |
+
"frames": []
|
| 41 |
+
}
|
| 42 |
+
}
|
| 43 |
+
|
| 44 |
+
# Add video metadata if provided
|
| 45 |
+
if video_metadata:
|
| 46 |
+
output["video_info"] = {
|
| 47 |
+
"fps": video_metadata.get("fps", 30.0),
|
| 48 |
+
"duration_seconds": len(movement_metrics) / video_metadata.get("fps", 30.0),
|
| 49 |
+
"width": video_metadata.get("width"),
|
| 50 |
+
"height": video_metadata.get("height"),
|
| 51 |
+
"frame_count": video_metadata.get("frame_count", len(movement_metrics))
|
| 52 |
+
}
|
| 53 |
+
|
| 54 |
+
# Process each frame's metrics
|
| 55 |
+
for i, metrics in enumerate(movement_metrics):
|
| 56 |
+
frame_data = {
|
| 57 |
+
"frame_index": metrics.frame_index,
|
| 58 |
+
"timestamp": metrics.timestamp,
|
| 59 |
+
"metrics": {
|
| 60 |
+
"direction": metrics.direction.value,
|
| 61 |
+
"intensity": metrics.intensity.value,
|
| 62 |
+
"speed": metrics.speed.value,
|
| 63 |
+
"velocity": round(metrics.velocity, 4),
|
| 64 |
+
"acceleration": round(metrics.acceleration, 4),
|
| 65 |
+
"fluidity": round(metrics.fluidity, 3),
|
| 66 |
+
"expansion": round(metrics.expansion, 3),
|
| 67 |
+
"total_displacement": round(metrics.total_displacement, 4)
|
| 68 |
+
}
|
| 69 |
+
}
|
| 70 |
+
|
| 71 |
+
# Add displacement if available
|
| 72 |
+
if metrics.center_displacement:
|
| 73 |
+
frame_data["metrics"]["center_displacement"] = {
|
| 74 |
+
"x": round(metrics.center_displacement[0], 4),
|
| 75 |
+
"y": round(metrics.center_displacement[1], 4)
|
| 76 |
+
}
|
| 77 |
+
|
| 78 |
+
# Add keypoints if requested and available
|
| 79 |
+
if include_keypoints and pose_results and i < len(pose_results):
|
| 80 |
+
frame_poses = pose_results[i]
|
| 81 |
+
if frame_poses:
|
| 82 |
+
frame_data["keypoints"] = []
|
| 83 |
+
for pose in frame_poses:
|
| 84 |
+
keypoint_data = {
|
| 85 |
+
"person_id": pose.person_id,
|
| 86 |
+
"points": []
|
| 87 |
+
}
|
| 88 |
+
for kp in pose.keypoints:
|
| 89 |
+
keypoint_data["points"].append({
|
| 90 |
+
"name": kp.name,
|
| 91 |
+
"x": round(kp.x, 4),
|
| 92 |
+
"y": round(kp.y, 4),
|
| 93 |
+
"confidence": round(kp.confidence, 3)
|
| 94 |
+
})
|
| 95 |
+
frame_data["keypoints"].append(keypoint_data)
|
| 96 |
+
|
| 97 |
+
output["movement_analysis"]["frames"].append(frame_data)
|
| 98 |
+
|
| 99 |
+
# Add summary statistics
|
| 100 |
+
output["movement_analysis"]["summary"] = _generate_summary(movement_metrics)
|
| 101 |
+
|
| 102 |
+
return output
|
| 103 |
+
|
| 104 |
+
|
| 105 |
+
def _generate_summary(metrics: List[MovementMetrics]) -> Dict[str, Any]:
|
| 106 |
+
"""Generate summary statistics from movement metrics."""
|
| 107 |
+
if not metrics:
|
| 108 |
+
return {}
|
| 109 |
+
|
| 110 |
+
# Count occurrences of each category
|
| 111 |
+
direction_counts = {}
|
| 112 |
+
intensity_counts = {}
|
| 113 |
+
speed_counts = {}
|
| 114 |
+
|
| 115 |
+
velocities = []
|
| 116 |
+
accelerations = []
|
| 117 |
+
fluidities = []
|
| 118 |
+
expansions = []
|
| 119 |
+
|
| 120 |
+
for m in metrics:
|
| 121 |
+
# Count categories
|
| 122 |
+
direction_counts[m.direction.value] = direction_counts.get(m.direction.value, 0) + 1
|
| 123 |
+
intensity_counts[m.intensity.value] = intensity_counts.get(m.intensity.value, 0) + 1
|
| 124 |
+
speed_counts[m.speed.value] = speed_counts.get(m.speed.value, 0) + 1
|
| 125 |
+
|
| 126 |
+
# Collect numeric values
|
| 127 |
+
velocities.append(m.velocity)
|
| 128 |
+
accelerations.append(m.acceleration)
|
| 129 |
+
fluidities.append(m.fluidity)
|
| 130 |
+
expansions.append(m.expansion)
|
| 131 |
+
|
| 132 |
+
# Calculate statistics
|
| 133 |
+
import numpy as np
|
| 134 |
+
|
| 135 |
+
summary = {
|
| 136 |
+
"direction": {
|
| 137 |
+
"distribution": direction_counts,
|
| 138 |
+
"dominant": max(direction_counts, key=direction_counts.get)
|
| 139 |
+
},
|
| 140 |
+
"intensity": {
|
| 141 |
+
"distribution": intensity_counts,
|
| 142 |
+
"dominant": max(intensity_counts, key=intensity_counts.get)
|
| 143 |
+
},
|
| 144 |
+
"speed": {
|
| 145 |
+
"distribution": speed_counts,
|
| 146 |
+
"dominant": max(speed_counts, key=speed_counts.get)
|
| 147 |
+
},
|
| 148 |
+
"velocity": {
|
| 149 |
+
"mean": round(float(np.mean(velocities)), 4),
|
| 150 |
+
"std": round(float(np.std(velocities)), 4),
|
| 151 |
+
"min": round(float(np.min(velocities)), 4),
|
| 152 |
+
"max": round(float(np.max(velocities)), 4)
|
| 153 |
+
},
|
| 154 |
+
"acceleration": {
|
| 155 |
+
"mean": round(float(np.mean(accelerations)), 4),
|
| 156 |
+
"std": round(float(np.std(accelerations)), 4),
|
| 157 |
+
"min": round(float(np.min(accelerations)), 4),
|
| 158 |
+
"max": round(float(np.max(accelerations)), 4)
|
| 159 |
+
},
|
| 160 |
+
"fluidity": {
|
| 161 |
+
"mean": round(float(np.mean(fluidities)), 3),
|
| 162 |
+
"std": round(float(np.std(fluidities)), 3)
|
| 163 |
+
},
|
| 164 |
+
"expansion": {
|
| 165 |
+
"mean": round(float(np.mean(expansions)), 3),
|
| 166 |
+
"std": round(float(np.std(expansions)), 3)
|
| 167 |
+
}
|
| 168 |
+
}
|
| 169 |
+
|
| 170 |
+
# Identify significant movement segments
|
| 171 |
+
summary["movement_segments"] = _identify_movement_segments(metrics)
|
| 172 |
+
|
| 173 |
+
return summary
|
| 174 |
+
|
| 175 |
+
|
| 176 |
+
def _identify_movement_segments(metrics: List[MovementMetrics]) -> List[Dict[str, Any]]:
|
| 177 |
+
"""Identify significant movement segments (e.g., bursts of activity)."""
|
| 178 |
+
segments = []
|
| 179 |
+
|
| 180 |
+
# Simple segmentation based on intensity changes
|
| 181 |
+
current_segment = None
|
| 182 |
+
intensity_threshold = Intensity.MEDIUM
|
| 183 |
+
|
| 184 |
+
for i, m in enumerate(metrics):
|
| 185 |
+
if m.intensity.value >= intensity_threshold.value:
|
| 186 |
+
if current_segment is None:
|
| 187 |
+
# Start new segment
|
| 188 |
+
current_segment = {
|
| 189 |
+
"start_frame": i,
|
| 190 |
+
"start_time": m.timestamp,
|
| 191 |
+
"peak_velocity": m.velocity,
|
| 192 |
+
"dominant_direction": m.direction.value
|
| 193 |
+
}
|
| 194 |
+
else:
|
| 195 |
+
# Update segment
|
| 196 |
+
if m.velocity > current_segment["peak_velocity"]:
|
| 197 |
+
current_segment["peak_velocity"] = m.velocity
|
| 198 |
+
current_segment["dominant_direction"] = m.direction.value
|
| 199 |
+
else:
|
| 200 |
+
if current_segment is not None:
|
| 201 |
+
# End segment
|
| 202 |
+
current_segment["end_frame"] = i - 1
|
| 203 |
+
current_segment["end_time"] = metrics[i-1].timestamp if i > 0 else 0
|
| 204 |
+
current_segment["duration"] = (
|
| 205 |
+
current_segment["end_time"] - current_segment["start_time"]
|
| 206 |
+
)
|
| 207 |
+
current_segment["peak_velocity"] = round(current_segment["peak_velocity"], 4)
|
| 208 |
+
segments.append(current_segment)
|
| 209 |
+
current_segment = None
|
| 210 |
+
|
| 211 |
+
# Handle segment that extends to end
|
| 212 |
+
if current_segment is not None:
|
| 213 |
+
current_segment["end_frame"] = len(metrics) - 1
|
| 214 |
+
current_segment["end_time"] = metrics[-1].timestamp
|
| 215 |
+
current_segment["duration"] = (
|
| 216 |
+
current_segment["end_time"] - current_segment["start_time"]
|
| 217 |
+
)
|
| 218 |
+
current_segment["peak_velocity"] = round(current_segment["peak_velocity"], 4)
|
| 219 |
+
segments.append(current_segment)
|
| 220 |
+
|
| 221 |
+
return segments
|
| 222 |
+
|
| 223 |
+
|
| 224 |
+
def save_json(data: Dict[str, Any], output_path: str, pretty: bool = True) -> None:
|
| 225 |
+
"""
|
| 226 |
+
Save JSON data to file.
|
| 227 |
+
|
| 228 |
+
Args:
|
| 229 |
+
data: Dictionary to save
|
| 230 |
+
output_path: Path to output file
|
| 231 |
+
pretty: Whether to format JSON with indentation
|
| 232 |
+
"""
|
| 233 |
+
with open(output_path, 'w') as f:
|
| 234 |
+
if pretty:
|
| 235 |
+
json.dump(data, f, indent=2, sort_keys=False)
|
| 236 |
+
else:
|
| 237 |
+
json.dump(data, f)
|
| 238 |
+
|
| 239 |
+
|
| 240 |
+
def format_for_display(data: Dict[str, Any]) -> str:
|
| 241 |
+
"""
|
| 242 |
+
Format JSON data for display in Gradio.
|
| 243 |
+
|
| 244 |
+
Args:
|
| 245 |
+
data: Dictionary to format
|
| 246 |
+
|
| 247 |
+
Returns:
|
| 248 |
+
Formatted JSON string
|
| 249 |
+
"""
|
| 250 |
+
return json.dumps(data, indent=2, sort_keys=False)
|
backend/gradio_labanmovementanalysis/labanmovementanalysis.py
ADDED
|
@@ -0,0 +1,442 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""
|
| 2 |
+
Custom Gradio v5 component for video-based pose analysis with LMA-inspired metrics.
|
| 3 |
+
"""
|
| 4 |
+
|
| 5 |
+
import gradio as gr
|
| 6 |
+
from gradio.components.base import Component
|
| 7 |
+
from typing import Dict, Any, Optional, Tuple, List, Union
|
| 8 |
+
import tempfile
|
| 9 |
+
import os
|
| 10 |
+
import numpy as np
|
| 11 |
+
|
| 12 |
+
from .video_utils import extract_frames, get_video_info
|
| 13 |
+
from .pose_estimation import get_pose_estimator
|
| 14 |
+
from .notation_engine import analyze_pose_sequence
|
| 15 |
+
from .json_generator import generate_json, format_for_display
|
| 16 |
+
from .visualizer import PoseVisualizer
|
| 17 |
+
from .video_downloader import SmartVideoInput
|
| 18 |
+
|
| 19 |
+
# Advanced features reserved for Version 2
|
| 20 |
+
# SkateFormer AI integration will be available in future release
|
| 21 |
+
|
| 22 |
+
try:
|
| 23 |
+
from .webrtc_handler import WebRTCMovementAnalyzer, WebRTCGradioInterface
|
| 24 |
+
HAS_WEBRTC = True
|
| 25 |
+
except ImportError:
|
| 26 |
+
HAS_WEBRTC = False
|
| 27 |
+
|
| 28 |
+
|
| 29 |
+
# SkateFormerCompatibility class removed for Version 1 stability
|
| 30 |
+
# Will be reimplemented in Version 2 with enhanced AI features
|
| 31 |
+
|
| 32 |
+
|
| 33 |
+
class LabanMovementAnalysis(Component):
|
| 34 |
+
"""
|
| 35 |
+
Gradio component for video-based pose analysis with Laban Movement Analysis metrics.
|
| 36 |
+
"""
|
| 37 |
+
|
| 38 |
+
# Component metadata
|
| 39 |
+
COMPONENT_TYPE = "composite"
|
| 40 |
+
DEFAULT_MODEL = "mediapipe"
|
| 41 |
+
|
| 42 |
+
def __init__(self,
|
| 43 |
+
default_model: str = DEFAULT_MODEL,
|
| 44 |
+
enable_visualization: bool = True,
|
| 45 |
+
include_keypoints: bool = False,
|
| 46 |
+
enable_webrtc: bool = False,
|
| 47 |
+
label: Optional[str] = None,
|
| 48 |
+
every: Optional[float] = None,
|
| 49 |
+
show_label: Optional[bool] = None,
|
| 50 |
+
container: bool = True,
|
| 51 |
+
scale: Optional[int] = None,
|
| 52 |
+
min_width: int = 160,
|
| 53 |
+
interactive: Optional[bool] = None,
|
| 54 |
+
visible: bool = True,
|
| 55 |
+
elem_id: Optional[str] = None,
|
| 56 |
+
elem_classes: Optional[List[str]] = None,
|
| 57 |
+
render: bool = True,
|
| 58 |
+
**kwargs):
|
| 59 |
+
"""
|
| 60 |
+
Initialize the Laban Movement Analysis component.
|
| 61 |
+
|
| 62 |
+
Args:
|
| 63 |
+
default_model: Default pose estimation model ("mediapipe", "movenet", "yolo")
|
| 64 |
+
enable_visualization: Whether to generate visualization video by default
|
| 65 |
+
include_keypoints: Whether to include raw keypoints in JSON output
|
| 66 |
+
enable_webrtc: Whether to enable WebRTC real-time analysis
|
| 67 |
+
label: Component label
|
| 68 |
+
... (other standard Gradio component args)
|
| 69 |
+
"""
|
| 70 |
+
super().__init__(
|
| 71 |
+
label=label,
|
| 72 |
+
every=every,
|
| 73 |
+
show_label=show_label,
|
| 74 |
+
container=container,
|
| 75 |
+
scale=scale,
|
| 76 |
+
min_width=min_width,
|
| 77 |
+
interactive=interactive,
|
| 78 |
+
visible=visible,
|
| 79 |
+
elem_id=elem_id,
|
| 80 |
+
elem_classes=elem_classes,
|
| 81 |
+
render=render,
|
| 82 |
+
**kwargs
|
| 83 |
+
)
|
| 84 |
+
|
| 85 |
+
self.default_model = default_model
|
| 86 |
+
self.enable_visualization = enable_visualization
|
| 87 |
+
self.include_keypoints = include_keypoints
|
| 88 |
+
self.enable_webrtc = enable_webrtc and HAS_WEBRTC
|
| 89 |
+
|
| 90 |
+
# Cache for pose estimators
|
| 91 |
+
self._estimators = {}
|
| 92 |
+
|
| 93 |
+
# Video input handler for URLs
|
| 94 |
+
self.video_input = SmartVideoInput()
|
| 95 |
+
|
| 96 |
+
# SkateFormer features reserved for Version 2
|
| 97 |
+
|
| 98 |
+
self.webrtc_analyzer = None
|
| 99 |
+
if self.enable_webrtc:
|
| 100 |
+
try:
|
| 101 |
+
self.webrtc_analyzer = WebRTCMovementAnalyzer(model=default_model)
|
| 102 |
+
except Exception as e:
|
| 103 |
+
print(f"Warning: Failed to initialize WebRTC: {e}")
|
| 104 |
+
self.enable_webrtc = False
|
| 105 |
+
|
| 106 |
+
def preprocess(self, payload: Dict[str, Any]) -> Dict[str, Any]:
|
| 107 |
+
"""
|
| 108 |
+
Preprocess input from the frontend.
|
| 109 |
+
|
| 110 |
+
Args:
|
| 111 |
+
payload: Input data containing video file and options
|
| 112 |
+
|
| 113 |
+
Returns:
|
| 114 |
+
Processed data for analysis
|
| 115 |
+
"""
|
| 116 |
+
if not payload:
|
| 117 |
+
return None
|
| 118 |
+
|
| 119 |
+
# Extract video file path
|
| 120 |
+
video_data = payload.get("video")
|
| 121 |
+
if not video_data:
|
| 122 |
+
return None
|
| 123 |
+
|
| 124 |
+
# Handle different input formats
|
| 125 |
+
if isinstance(video_data, str):
|
| 126 |
+
video_path = video_data
|
| 127 |
+
elif isinstance(video_data, dict):
|
| 128 |
+
video_path = video_data.get("path") or video_data.get("name")
|
| 129 |
+
else:
|
| 130 |
+
# Assume it's a file object
|
| 131 |
+
video_path = video_data.name if hasattr(video_data, "name") else str(video_data)
|
| 132 |
+
|
| 133 |
+
# Extract options
|
| 134 |
+
options = {
|
| 135 |
+
"video_path": video_path,
|
| 136 |
+
"model": payload.get("model", self.default_model),
|
| 137 |
+
"enable_visualization": payload.get("enable_visualization", self.enable_visualization),
|
| 138 |
+
"include_keypoints": payload.get("include_keypoints", self.include_keypoints)
|
| 139 |
+
}
|
| 140 |
+
|
| 141 |
+
return options
|
| 142 |
+
|
| 143 |
+
def postprocess(self, value: Any) -> Dict[str, Any]:
|
| 144 |
+
"""
|
| 145 |
+
Postprocess analysis results for the frontend.
|
| 146 |
+
|
| 147 |
+
Args:
|
| 148 |
+
value: Analysis results
|
| 149 |
+
|
| 150 |
+
Returns:
|
| 151 |
+
Formatted output for display
|
| 152 |
+
"""
|
| 153 |
+
if value is None:
|
| 154 |
+
return {"json_output": {}, "video_output": None}
|
| 155 |
+
|
| 156 |
+
# Ensure we have the expected format
|
| 157 |
+
if isinstance(value, tuple) and len(value) == 2:
|
| 158 |
+
json_data, video_path = value
|
| 159 |
+
else:
|
| 160 |
+
json_data = value
|
| 161 |
+
video_path = None
|
| 162 |
+
|
| 163 |
+
return {
|
| 164 |
+
"json_output": json_data,
|
| 165 |
+
"video_output": video_path
|
| 166 |
+
}
|
| 167 |
+
|
| 168 |
+
def process_video(self, video_input: Union[str, os.PathLike], model: str = DEFAULT_MODEL,
|
| 169 |
+
enable_visualization: bool = True,
|
| 170 |
+
include_keypoints: bool = False) -> Tuple[Dict[str, Any], Optional[str]]:
|
| 171 |
+
"""
|
| 172 |
+
Main processing function that performs pose analysis on a video.
|
| 173 |
+
|
| 174 |
+
Args:
|
| 175 |
+
video_input: Path to input video, video URL (YouTube/Vimeo), or file object
|
| 176 |
+
model: Pose estimation model to use (supports enhanced syntax like "yolo-v11-s")
|
| 177 |
+
enable_visualization: Whether to generate visualization video
|
| 178 |
+
include_keypoints: Whether to include keypoints in JSON
|
| 179 |
+
|
| 180 |
+
Returns:
|
| 181 |
+
Tuple of (analysis_json, visualization_video_path)
|
| 182 |
+
"""
|
| 183 |
+
# Handle video input (local file, URL, etc.)
|
| 184 |
+
try:
|
| 185 |
+
video_path, video_metadata = self.video_input.process_input(str(video_input))
|
| 186 |
+
print(f"Processing video: {video_metadata.get('title', 'Unknown')}")
|
| 187 |
+
if video_metadata.get('platform') in ['youtube', 'vimeo']:
|
| 188 |
+
print(f"Downloaded from {video_metadata['platform']}")
|
| 189 |
+
except Exception as e:
|
| 190 |
+
raise ValueError(f"Failed to process video input: {str(e)}")
|
| 191 |
+
# Get video metadata
|
| 192 |
+
frame_count, fps, (width, height) = get_video_info(video_path)
|
| 193 |
+
|
| 194 |
+
# Create or get pose estimator
|
| 195 |
+
if model not in self._estimators:
|
| 196 |
+
self._estimators[model] = get_pose_estimator(model)
|
| 197 |
+
estimator = self._estimators[model]
|
| 198 |
+
|
| 199 |
+
# Process video frame by frame
|
| 200 |
+
print(f"Processing {frame_count} frames with {model} model...")
|
| 201 |
+
|
| 202 |
+
all_frames = []
|
| 203 |
+
all_pose_results = []
|
| 204 |
+
|
| 205 |
+
for i, frame in enumerate(extract_frames(video_path)):
|
| 206 |
+
# Store frame if visualization is needed
|
| 207 |
+
if enable_visualization:
|
| 208 |
+
all_frames.append(frame)
|
| 209 |
+
|
| 210 |
+
# Detect poses
|
| 211 |
+
pose_results = estimator.detect(frame)
|
| 212 |
+
|
| 213 |
+
# Update frame indices
|
| 214 |
+
for result in pose_results:
|
| 215 |
+
result.frame_index = i
|
| 216 |
+
|
| 217 |
+
all_pose_results.append(pose_results)
|
| 218 |
+
|
| 219 |
+
# Progress indicator
|
| 220 |
+
if i % 30 == 0:
|
| 221 |
+
print(f"Processed {i}/{frame_count} frames...")
|
| 222 |
+
|
| 223 |
+
print("Analyzing movement patterns...")
|
| 224 |
+
|
| 225 |
+
# Analyze movement
|
| 226 |
+
movement_metrics = analyze_pose_sequence(all_pose_results, fps=fps)
|
| 227 |
+
|
| 228 |
+
# Enhanced AI analysis reserved for Version 2
|
| 229 |
+
print("LMA analysis complete - advanced AI features coming in Version 2!")
|
| 230 |
+
|
| 231 |
+
# Generate JSON output
|
| 232 |
+
video_metadata = {
|
| 233 |
+
"fps": fps,
|
| 234 |
+
"width": width,
|
| 235 |
+
"height": height,
|
| 236 |
+
"frame_count": frame_count,
|
| 237 |
+
"model_info": {
|
| 238 |
+
"name": model,
|
| 239 |
+
"type": "pose_estimation"
|
| 240 |
+
},
|
| 241 |
+
"input_metadata": video_metadata # Include video source metadata
|
| 242 |
+
}
|
| 243 |
+
|
| 244 |
+
json_output = generate_json(
|
| 245 |
+
movement_metrics,
|
| 246 |
+
all_pose_results if include_keypoints else None,
|
| 247 |
+
video_metadata,
|
| 248 |
+
include_keypoints=include_keypoints
|
| 249 |
+
)
|
| 250 |
+
|
| 251 |
+
# Enhanced AI analysis will be added in Version 2
|
| 252 |
+
|
| 253 |
+
# Generate visualization if requested
|
| 254 |
+
visualization_path = None
|
| 255 |
+
if enable_visualization:
|
| 256 |
+
print("Generating visualization video...")
|
| 257 |
+
|
| 258 |
+
# Create temporary output file
|
| 259 |
+
with tempfile.NamedTemporaryFile(suffix='.mp4', delete=False) as tmp:
|
| 260 |
+
visualization_path = tmp.name
|
| 261 |
+
|
| 262 |
+
# Create visualizer
|
| 263 |
+
visualizer = PoseVisualizer(
|
| 264 |
+
show_trails=True,
|
| 265 |
+
show_skeleton=True,
|
| 266 |
+
show_direction_arrows=True,
|
| 267 |
+
show_metrics=True
|
| 268 |
+
)
|
| 269 |
+
|
| 270 |
+
# Generate overlay video
|
| 271 |
+
visualization_path = visualizer.generate_overlay_video(
|
| 272 |
+
all_frames,
|
| 273 |
+
all_pose_results,
|
| 274 |
+
movement_metrics,
|
| 275 |
+
visualization_path,
|
| 276 |
+
fps
|
| 277 |
+
)
|
| 278 |
+
|
| 279 |
+
print(f"Visualization saved to: {visualization_path}")
|
| 280 |
+
|
| 281 |
+
return json_output, visualization_path
|
| 282 |
+
|
| 283 |
+
def __call__(self, video_path: str, **kwargs) -> Tuple[Dict[str, Any], Optional[str]]:
|
| 284 |
+
"""
|
| 285 |
+
Make the component callable for easy use.
|
| 286 |
+
|
| 287 |
+
Args:
|
| 288 |
+
video_path: Path to video file
|
| 289 |
+
**kwargs: Additional options
|
| 290 |
+
|
| 291 |
+
Returns:
|
| 292 |
+
Analysis results
|
| 293 |
+
"""
|
| 294 |
+
return self.process_video(video_path, **kwargs)
|
| 295 |
+
|
| 296 |
+
def start_webrtc_stream(self, model: str = None) -> bool:
|
| 297 |
+
"""
|
| 298 |
+
Start WebRTC real-time analysis stream.
|
| 299 |
+
|
| 300 |
+
Args:
|
| 301 |
+
model: Pose model to use for real-time analysis
|
| 302 |
+
|
| 303 |
+
Returns:
|
| 304 |
+
True if stream started successfully
|
| 305 |
+
"""
|
| 306 |
+
if not self.enable_webrtc or not self.webrtc_analyzer:
|
| 307 |
+
print("WebRTC not enabled or available")
|
| 308 |
+
return False
|
| 309 |
+
|
| 310 |
+
try:
|
| 311 |
+
if model:
|
| 312 |
+
self.webrtc_analyzer.model = model
|
| 313 |
+
self.webrtc_analyzer.pose_estimator = get_pose_estimator(model)
|
| 314 |
+
|
| 315 |
+
self.webrtc_analyzer.start_stream()
|
| 316 |
+
print(f"WebRTC stream started with {self.webrtc_analyzer.model} model")
|
| 317 |
+
return True
|
| 318 |
+
except Exception as e:
|
| 319 |
+
print(f"Failed to start WebRTC stream: {e}")
|
| 320 |
+
return False
|
| 321 |
+
|
| 322 |
+
def stop_webrtc_stream(self) -> bool:
|
| 323 |
+
"""
|
| 324 |
+
Stop WebRTC real-time analysis stream.
|
| 325 |
+
|
| 326 |
+
Returns:
|
| 327 |
+
True if stream stopped successfully
|
| 328 |
+
"""
|
| 329 |
+
if not self.webrtc_analyzer:
|
| 330 |
+
return False
|
| 331 |
+
|
| 332 |
+
try:
|
| 333 |
+
self.webrtc_analyzer.stop_stream()
|
| 334 |
+
print("WebRTC stream stopped")
|
| 335 |
+
return True
|
| 336 |
+
except Exception as e:
|
| 337 |
+
print(f"Failed to stop WebRTC stream: {e}")
|
| 338 |
+
return False
|
| 339 |
+
|
| 340 |
+
def get_webrtc_interface(self):
|
| 341 |
+
"""
|
| 342 |
+
Get WebRTC Gradio interface for real-time streaming.
|
| 343 |
+
|
| 344 |
+
Returns:
|
| 345 |
+
WebRTCGradioInterface instance or None
|
| 346 |
+
"""
|
| 347 |
+
if not self.enable_webrtc or not self.webrtc_analyzer:
|
| 348 |
+
return None
|
| 349 |
+
|
| 350 |
+
return WebRTCGradioInterface(self.webrtc_analyzer)
|
| 351 |
+
|
| 352 |
+
# SkateFormer methods moved to Version 2 development
|
| 353 |
+
# get_skateformer_compatibility() and get_skateformer_status_report()
|
| 354 |
+
# will be available in the next major release
|
| 355 |
+
|
| 356 |
+
def cleanup(self):
|
| 357 |
+
"""Clean up temporary files and resources."""
|
| 358 |
+
# Clean up video input handler
|
| 359 |
+
if hasattr(self, 'video_input'):
|
| 360 |
+
self.video_input.cleanup()
|
| 361 |
+
|
| 362 |
+
# Stop WebRTC if running
|
| 363 |
+
if self.webrtc_analyzer and self.webrtc_analyzer.is_running:
|
| 364 |
+
self.stop_webrtc_stream()
|
| 365 |
+
|
| 366 |
+
def example_payload(self) -> Dict[str, Any]:
|
| 367 |
+
"""Example input payload for documentation."""
|
| 368 |
+
return {
|
| 369 |
+
"video": {"path": "/path/to/video.mp4"},
|
| 370 |
+
"model": "mediapipe",
|
| 371 |
+
"enable_visualization": True,
|
| 372 |
+
"include_keypoints": False
|
| 373 |
+
}
|
| 374 |
+
|
| 375 |
+
def example_value(self) -> Dict[str, Any]:
|
| 376 |
+
"""Example output value for documentation."""
|
| 377 |
+
return {
|
| 378 |
+
"json_output": {
|
| 379 |
+
"analysis_metadata": {
|
| 380 |
+
"timestamp": "2024-01-01T00:00:00",
|
| 381 |
+
"version": "1.0.0",
|
| 382 |
+
"model_info": {"name": "mediapipe", "type": "pose_estimation"}
|
| 383 |
+
},
|
| 384 |
+
"video_info": {
|
| 385 |
+
"fps": 30.0,
|
| 386 |
+
"duration_seconds": 5.0,
|
| 387 |
+
"width": 1920,
|
| 388 |
+
"height": 1080,
|
| 389 |
+
"frame_count": 150
|
| 390 |
+
},
|
| 391 |
+
"movement_analysis": {
|
| 392 |
+
"frame_count": 150,
|
| 393 |
+
"frames": [
|
| 394 |
+
{
|
| 395 |
+
"frame_index": 0,
|
| 396 |
+
"timestamp": 0.0,
|
| 397 |
+
"metrics": {
|
| 398 |
+
"direction": "stationary",
|
| 399 |
+
"intensity": "low",
|
| 400 |
+
"speed": "slow",
|
| 401 |
+
"velocity": 0.0,
|
| 402 |
+
"acceleration": 0.0,
|
| 403 |
+
"fluidity": 1.0,
|
| 404 |
+
"expansion": 0.5
|
| 405 |
+
}
|
| 406 |
+
}
|
| 407 |
+
],
|
| 408 |
+
"summary": {
|
| 409 |
+
"direction": {
|
| 410 |
+
"distribution": {"stationary": 50, "up": 30, "down": 20},
|
| 411 |
+
"dominant": "stationary"
|
| 412 |
+
},
|
| 413 |
+
"intensity": {
|
| 414 |
+
"distribution": {"low": 80, "medium": 15, "high": 5},
|
| 415 |
+
"dominant": "low"
|
| 416 |
+
}
|
| 417 |
+
}
|
| 418 |
+
}
|
| 419 |
+
},
|
| 420 |
+
"video_output": "/tmp/visualization.mp4"
|
| 421 |
+
}
|
| 422 |
+
|
| 423 |
+
def api_info(self) -> Dict[str, Any]:
|
| 424 |
+
"""API information for the component."""
|
| 425 |
+
return {
|
| 426 |
+
"type": "composite",
|
| 427 |
+
"description": "Video-based pose analysis with Laban Movement Analysis metrics",
|
| 428 |
+
"parameters": {
|
| 429 |
+
"video": {"type": "file", "description": "Input video file or URL (YouTube/Vimeo)"},
|
| 430 |
+
"model": {"type": "string", "description": "Pose model: mediapipe, movenet, or yolo variants"},
|
| 431 |
+
"enable_visualization": {"type": "boolean", "description": "Generate visualization video"},
|
| 432 |
+
"include_keypoints": {"type": "boolean", "description": "Include keypoints in JSON"}
|
| 433 |
+
},
|
| 434 |
+
"returns": {
|
| 435 |
+
"json_output": {"type": "object", "description": "LMA analysis results"},
|
| 436 |
+
"video_output": {"type": "file", "description": "Visualization video (optional)"}
|
| 437 |
+
},
|
| 438 |
+
"version_2_preview": {
|
| 439 |
+
"planned_features": ["SkateFormer AI integration", "Enhanced movement recognition", "Real-time analysis"],
|
| 440 |
+
"note": "Advanced AI features coming in Version 2!"
|
| 441 |
+
}
|
| 442 |
+
}
|
backend/gradio_labanmovementanalysis/labanmovementanalysis.pyi
ADDED
|
@@ -0,0 +1,448 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""
|
| 2 |
+
Custom Gradio v5 component for video-based pose analysis with LMA-inspired metrics.
|
| 3 |
+
"""
|
| 4 |
+
|
| 5 |
+
import gradio as gr
|
| 6 |
+
from gradio.components.base import Component
|
| 7 |
+
from typing import Dict, Any, Optional, Tuple, List, Union
|
| 8 |
+
import tempfile
|
| 9 |
+
import os
|
| 10 |
+
import numpy as np
|
| 11 |
+
|
| 12 |
+
from .video_utils import extract_frames, get_video_info
|
| 13 |
+
from .pose_estimation import get_pose_estimator
|
| 14 |
+
from .notation_engine import analyze_pose_sequence
|
| 15 |
+
from .json_generator import generate_json, format_for_display
|
| 16 |
+
from .visualizer import PoseVisualizer
|
| 17 |
+
from .video_downloader import SmartVideoInput
|
| 18 |
+
|
| 19 |
+
# Optional advanced features
|
| 20 |
+
try:
|
| 21 |
+
from .skateformer_integration import SkateFormerAnalyzer
|
| 22 |
+
HAS_SKATEFORMER = True
|
| 23 |
+
except ImportError:
|
| 24 |
+
HAS_SKATEFORMER = False
|
| 25 |
+
|
| 26 |
+
try:
|
| 27 |
+
from .webrtc_handler import WebRTCMovementAnalyzer, WebRTCGradioInterface
|
| 28 |
+
HAS_WEBRTC = True
|
| 29 |
+
except ImportError:
|
| 30 |
+
HAS_WEBRTC = False
|
| 31 |
+
|
| 32 |
+
from gradio.events import Dependency
|
| 33 |
+
|
| 34 |
+
class LabanMovementAnalysis(Component):
|
| 35 |
+
"""
|
| 36 |
+
Gradio component for video-based pose analysis with Laban Movement Analysis metrics.
|
| 37 |
+
"""
|
| 38 |
+
|
| 39 |
+
# Component metadata
|
| 40 |
+
COMPONENT_TYPE = "composite"
|
| 41 |
+
DEFAULT_MODEL = "mediapipe"
|
| 42 |
+
|
| 43 |
+
def __init__(self,
|
| 44 |
+
default_model: str = DEFAULT_MODEL,
|
| 45 |
+
enable_visualization: bool = True,
|
| 46 |
+
include_keypoints: bool = False,
|
| 47 |
+
enable_webrtc: bool = False,
|
| 48 |
+
label: Optional[str] = None,
|
| 49 |
+
every: Optional[float] = None,
|
| 50 |
+
show_label: Optional[bool] = None,
|
| 51 |
+
container: bool = True,
|
| 52 |
+
scale: Optional[int] = None,
|
| 53 |
+
min_width: int = 160,
|
| 54 |
+
interactive: Optional[bool] = None,
|
| 55 |
+
visible: bool = True,
|
| 56 |
+
elem_id: Optional[str] = None,
|
| 57 |
+
elem_classes: Optional[List[str]] = None,
|
| 58 |
+
render: bool = True,
|
| 59 |
+
**kwargs):
|
| 60 |
+
"""
|
| 61 |
+
Initialize the Laban Movement Analysis component.
|
| 62 |
+
|
| 63 |
+
Args:
|
| 64 |
+
default_model: Default pose estimation model ("mediapipe", "movenet", "yolo")
|
| 65 |
+
enable_visualization: Whether to generate visualization video by default
|
| 66 |
+
include_keypoints: Whether to include raw keypoints in JSON output
|
| 67 |
+
enable_webrtc: Whether to enable WebRTC real-time analysis
|
| 68 |
+
label: Component label
|
| 69 |
+
... (other standard Gradio component args)
|
| 70 |
+
"""
|
| 71 |
+
super().__init__(
|
| 72 |
+
label=label,
|
| 73 |
+
every=every,
|
| 74 |
+
show_label=show_label,
|
| 75 |
+
container=container,
|
| 76 |
+
scale=scale,
|
| 77 |
+
min_width=min_width,
|
| 78 |
+
interactive=interactive,
|
| 79 |
+
visible=visible,
|
| 80 |
+
elem_id=elem_id,
|
| 81 |
+
elem_classes=elem_classes,
|
| 82 |
+
render=render,
|
| 83 |
+
**kwargs
|
| 84 |
+
)
|
| 85 |
+
|
| 86 |
+
self.default_model = default_model
|
| 87 |
+
self.enable_visualization = enable_visualization
|
| 88 |
+
self.include_keypoints = include_keypoints
|
| 89 |
+
self.enable_webrtc = enable_webrtc and HAS_WEBRTC
|
| 90 |
+
|
| 91 |
+
# Cache for pose estimators
|
| 92 |
+
self._estimators = {}
|
| 93 |
+
|
| 94 |
+
# Video input handler for URLs
|
| 95 |
+
self.video_input = SmartVideoInput()
|
| 96 |
+
|
| 97 |
+
# SkateFormer features reserved for Version 2
|
| 98 |
+
|
| 99 |
+
self.webrtc_analyzer = None
|
| 100 |
+
if self.enable_webrtc:
|
| 101 |
+
try:
|
| 102 |
+
self.webrtc_analyzer = WebRTCMovementAnalyzer(model=default_model)
|
| 103 |
+
except Exception as e:
|
| 104 |
+
print(f"Warning: Failed to initialize WebRTC: {e}")
|
| 105 |
+
self.enable_webrtc = False
|
| 106 |
+
|
| 107 |
+
def preprocess(self, payload: Dict[str, Any]) -> Dict[str, Any]:
|
| 108 |
+
"""
|
| 109 |
+
Preprocess input from the frontend.
|
| 110 |
+
|
| 111 |
+
Args:
|
| 112 |
+
payload: Input data containing video file and options
|
| 113 |
+
|
| 114 |
+
Returns:
|
| 115 |
+
Processed data for analysis
|
| 116 |
+
"""
|
| 117 |
+
if not payload:
|
| 118 |
+
return None
|
| 119 |
+
|
| 120 |
+
# Extract video file path
|
| 121 |
+
video_data = payload.get("video")
|
| 122 |
+
if not video_data:
|
| 123 |
+
return None
|
| 124 |
+
|
| 125 |
+
# Handle different input formats
|
| 126 |
+
if isinstance(video_data, str):
|
| 127 |
+
video_path = video_data
|
| 128 |
+
elif isinstance(video_data, dict):
|
| 129 |
+
video_path = video_data.get("path") or video_data.get("name")
|
| 130 |
+
else:
|
| 131 |
+
# Assume it's a file object
|
| 132 |
+
video_path = video_data.name if hasattr(video_data, "name") else str(video_data)
|
| 133 |
+
|
| 134 |
+
# Extract options
|
| 135 |
+
options = {
|
| 136 |
+
"video_path": video_path,
|
| 137 |
+
"model": payload.get("model", self.default_model),
|
| 138 |
+
"enable_visualization": payload.get("enable_visualization", self.enable_visualization),
|
| 139 |
+
"include_keypoints": payload.get("include_keypoints", self.include_keypoints)
|
| 140 |
+
}
|
| 141 |
+
|
| 142 |
+
return options
|
| 143 |
+
|
| 144 |
+
def postprocess(self, value: Any) -> Dict[str, Any]:
|
| 145 |
+
"""
|
| 146 |
+
Postprocess analysis results for the frontend.
|
| 147 |
+
|
| 148 |
+
Args:
|
| 149 |
+
value: Analysis results
|
| 150 |
+
|
| 151 |
+
Returns:
|
| 152 |
+
Formatted output for display
|
| 153 |
+
"""
|
| 154 |
+
if value is None:
|
| 155 |
+
return {"json_output": {}, "video_output": None}
|
| 156 |
+
|
| 157 |
+
# Ensure we have the expected format
|
| 158 |
+
if isinstance(value, tuple) and len(value) == 2:
|
| 159 |
+
json_data, video_path = value
|
| 160 |
+
else:
|
| 161 |
+
json_data = value
|
| 162 |
+
video_path = None
|
| 163 |
+
|
| 164 |
+
return {
|
| 165 |
+
"json_output": json_data,
|
| 166 |
+
"video_output": video_path
|
| 167 |
+
}
|
| 168 |
+
|
| 169 |
+
def process_video(self, video_input: Union[str, os.PathLike], model: str = DEFAULT_MODEL,
|
| 170 |
+
enable_visualization: bool = True,
|
| 171 |
+
include_keypoints: bool = False) -> Tuple[Dict[str, Any], Optional[str]]:
|
| 172 |
+
"""
|
| 173 |
+
Main processing function that performs pose analysis on a video.
|
| 174 |
+
|
| 175 |
+
Args:
|
| 176 |
+
video_input: Path to input video, video URL (YouTube/Vimeo), or file object
|
| 177 |
+
model: Pose estimation model to use (supports enhanced syntax like "yolo-v11-s")
|
| 178 |
+
enable_visualization: Whether to generate visualization video
|
| 179 |
+
include_keypoints: Whether to include keypoints in JSON
|
| 180 |
+
|
| 181 |
+
Returns:
|
| 182 |
+
Tuple of (analysis_json, visualization_video_path)
|
| 183 |
+
"""
|
| 184 |
+
# Handle video input (local file, URL, etc.)
|
| 185 |
+
try:
|
| 186 |
+
video_path, video_metadata = self.video_input.process_input(str(video_input))
|
| 187 |
+
print(f"Processing video: {video_metadata.get('title', 'Unknown')}")
|
| 188 |
+
if video_metadata.get('platform') in ['youtube', 'vimeo']:
|
| 189 |
+
print(f"Downloaded from {video_metadata['platform']}")
|
| 190 |
+
except Exception as e:
|
| 191 |
+
raise ValueError(f"Failed to process video input: {str(e)}")
|
| 192 |
+
# Get video metadata
|
| 193 |
+
frame_count, fps, (width, height) = get_video_info(video_path)
|
| 194 |
+
|
| 195 |
+
# Create or get pose estimator
|
| 196 |
+
if model not in self._estimators:
|
| 197 |
+
self._estimators[model] = get_pose_estimator(model)
|
| 198 |
+
estimator = self._estimators[model]
|
| 199 |
+
|
| 200 |
+
# Process video frame by frame
|
| 201 |
+
print(f"Processing {frame_count} frames with {model} model...")
|
| 202 |
+
|
| 203 |
+
all_frames = []
|
| 204 |
+
all_pose_results = []
|
| 205 |
+
|
| 206 |
+
for i, frame in enumerate(extract_frames(video_path)):
|
| 207 |
+
# Store frame if visualization is needed
|
| 208 |
+
if enable_visualization:
|
| 209 |
+
all_frames.append(frame)
|
| 210 |
+
|
| 211 |
+
# Detect poses
|
| 212 |
+
pose_results = estimator.detect(frame)
|
| 213 |
+
|
| 214 |
+
# Update frame indices
|
| 215 |
+
for result in pose_results:
|
| 216 |
+
result.frame_index = i
|
| 217 |
+
|
| 218 |
+
all_pose_results.append(pose_results)
|
| 219 |
+
|
| 220 |
+
# Progress indicator
|
| 221 |
+
if i % 30 == 0:
|
| 222 |
+
print(f"Processed {i}/{frame_count} frames...")
|
| 223 |
+
|
| 224 |
+
print("Analyzing movement patterns...")
|
| 225 |
+
|
| 226 |
+
# Analyze movement
|
| 227 |
+
movement_metrics = analyze_pose_sequence(all_pose_results, fps=fps)
|
| 228 |
+
|
| 229 |
+
# Enhanced AI analysis reserved for Version 2
|
| 230 |
+
print("LMA analysis complete - advanced AI features coming in Version 2!")
|
| 231 |
+
|
| 232 |
+
# Generate JSON output
|
| 233 |
+
video_metadata = {
|
| 234 |
+
"fps": fps,
|
| 235 |
+
"width": width,
|
| 236 |
+
"height": height,
|
| 237 |
+
"frame_count": frame_count,
|
| 238 |
+
"model_info": {
|
| 239 |
+
"name": model,
|
| 240 |
+
"type": "pose_estimation"
|
| 241 |
+
},
|
| 242 |
+
"input_metadata": video_metadata # Include video source metadata
|
| 243 |
+
}
|
| 244 |
+
|
| 245 |
+
json_output = generate_json(
|
| 246 |
+
movement_metrics,
|
| 247 |
+
all_pose_results if include_keypoints else None,
|
| 248 |
+
video_metadata,
|
| 249 |
+
include_keypoints=include_keypoints
|
| 250 |
+
)
|
| 251 |
+
|
| 252 |
+
# Enhanced AI analysis will be added in Version 2
|
| 253 |
+
|
| 254 |
+
# Generate visualization if requested
|
| 255 |
+
visualization_path = None
|
| 256 |
+
if enable_visualization:
|
| 257 |
+
print("Generating visualization video...")
|
| 258 |
+
|
| 259 |
+
# Create temporary output file
|
| 260 |
+
with tempfile.NamedTemporaryFile(suffix='.mp4', delete=False) as tmp:
|
| 261 |
+
visualization_path = tmp.name
|
| 262 |
+
|
| 263 |
+
# Create visualizer
|
| 264 |
+
visualizer = PoseVisualizer(
|
| 265 |
+
show_trails=True,
|
| 266 |
+
show_skeleton=True,
|
| 267 |
+
show_direction_arrows=True,
|
| 268 |
+
show_metrics=True
|
| 269 |
+
)
|
| 270 |
+
|
| 271 |
+
# Generate overlay video
|
| 272 |
+
visualization_path = visualizer.generate_overlay_video(
|
| 273 |
+
all_frames,
|
| 274 |
+
all_pose_results,
|
| 275 |
+
movement_metrics,
|
| 276 |
+
visualization_path,
|
| 277 |
+
fps
|
| 278 |
+
)
|
| 279 |
+
|
| 280 |
+
print(f"Visualization saved to: {visualization_path}")
|
| 281 |
+
|
| 282 |
+
return json_output, visualization_path
|
| 283 |
+
|
| 284 |
+
def __call__(self, video_path: str, **kwargs) -> Tuple[Dict[str, Any], Optional[str]]:
|
| 285 |
+
"""
|
| 286 |
+
Make the component callable for easy use.
|
| 287 |
+
|
| 288 |
+
Args:
|
| 289 |
+
video_path: Path to video file
|
| 290 |
+
**kwargs: Additional options
|
| 291 |
+
|
| 292 |
+
Returns:
|
| 293 |
+
Analysis results
|
| 294 |
+
"""
|
| 295 |
+
return self.process_video(video_path, **kwargs)
|
| 296 |
+
|
| 297 |
+
def start_webrtc_stream(self, model: str = None) -> bool:
|
| 298 |
+
"""
|
| 299 |
+
Start WebRTC real-time analysis stream.
|
| 300 |
+
|
| 301 |
+
Args:
|
| 302 |
+
model: Pose model to use for real-time analysis
|
| 303 |
+
|
| 304 |
+
Returns:
|
| 305 |
+
True if stream started successfully
|
| 306 |
+
"""
|
| 307 |
+
if not self.enable_webrtc or not self.webrtc_analyzer:
|
| 308 |
+
print("WebRTC not enabled or available")
|
| 309 |
+
return False
|
| 310 |
+
|
| 311 |
+
try:
|
| 312 |
+
if model:
|
| 313 |
+
self.webrtc_analyzer.model = model
|
| 314 |
+
self.webrtc_analyzer.pose_estimator = get_pose_estimator(model)
|
| 315 |
+
|
| 316 |
+
self.webrtc_analyzer.start_stream()
|
| 317 |
+
print(f"WebRTC stream started with {self.webrtc_analyzer.model} model")
|
| 318 |
+
return True
|
| 319 |
+
except Exception as e:
|
| 320 |
+
print(f"Failed to start WebRTC stream: {e}")
|
| 321 |
+
return False
|
| 322 |
+
|
| 323 |
+
def stop_webrtc_stream(self) -> bool:
|
| 324 |
+
"""
|
| 325 |
+
Stop WebRTC real-time analysis stream.
|
| 326 |
+
|
| 327 |
+
Returns:
|
| 328 |
+
True if stream stopped successfully
|
| 329 |
+
"""
|
| 330 |
+
if not self.webrtc_analyzer:
|
| 331 |
+
return False
|
| 332 |
+
|
| 333 |
+
try:
|
| 334 |
+
self.webrtc_analyzer.stop_stream()
|
| 335 |
+
print("WebRTC stream stopped")
|
| 336 |
+
return True
|
| 337 |
+
except Exception as e:
|
| 338 |
+
print(f"Failed to stop WebRTC stream: {e}")
|
| 339 |
+
return False
|
| 340 |
+
|
| 341 |
+
def get_webrtc_interface(self):
|
| 342 |
+
"""
|
| 343 |
+
Get WebRTC Gradio interface for real-time streaming.
|
| 344 |
+
|
| 345 |
+
Returns:
|
| 346 |
+
WebRTCGradioInterface instance or None
|
| 347 |
+
"""
|
| 348 |
+
if not self.enable_webrtc or not self.webrtc_analyzer:
|
| 349 |
+
return None
|
| 350 |
+
|
| 351 |
+
return WebRTCGradioInterface(self.webrtc_analyzer)
|
| 352 |
+
|
| 353 |
+
# SkateFormer methods moved to Version 2 development
|
| 354 |
+
# get_skateformer_compatibility() and get_skateformer_status_report()
|
| 355 |
+
# will be available in the next major release
|
| 356 |
+
|
| 357 |
+
def cleanup(self):
|
| 358 |
+
"""Clean up temporary files and resources."""
|
| 359 |
+
# Clean up video input handler
|
| 360 |
+
if hasattr(self, 'video_input'):
|
| 361 |
+
self.video_input.cleanup()
|
| 362 |
+
|
| 363 |
+
# Stop WebRTC if running
|
| 364 |
+
if self.webrtc_analyzer and self.webrtc_analyzer.is_running:
|
| 365 |
+
self.stop_webrtc_stream()
|
| 366 |
+
|
| 367 |
+
def example_payload(self) -> Dict[str, Any]:
|
| 368 |
+
"""Example input payload for documentation."""
|
| 369 |
+
return {
|
| 370 |
+
"video": {"path": "/path/to/video.mp4"},
|
| 371 |
+
"model": "mediapipe",
|
| 372 |
+
"enable_visualization": True,
|
| 373 |
+
"include_keypoints": False
|
| 374 |
+
}
|
| 375 |
+
|
| 376 |
+
def example_value(self) -> Dict[str, Any]:
|
| 377 |
+
"""Example output value for documentation."""
|
| 378 |
+
return {
|
| 379 |
+
"json_output": {
|
| 380 |
+
"analysis_metadata": {
|
| 381 |
+
"timestamp": "2024-01-01T00:00:00",
|
| 382 |
+
"version": "1.0.0",
|
| 383 |
+
"model_info": {"name": "mediapipe", "type": "pose_estimation"}
|
| 384 |
+
},
|
| 385 |
+
"video_info": {
|
| 386 |
+
"fps": 30.0,
|
| 387 |
+
"duration_seconds": 5.0,
|
| 388 |
+
"width": 1920,
|
| 389 |
+
"height": 1080,
|
| 390 |
+
"frame_count": 150
|
| 391 |
+
},
|
| 392 |
+
"movement_analysis": {
|
| 393 |
+
"frame_count": 150,
|
| 394 |
+
"frames": [
|
| 395 |
+
{
|
| 396 |
+
"frame_index": 0,
|
| 397 |
+
"timestamp": 0.0,
|
| 398 |
+
"metrics": {
|
| 399 |
+
"direction": "stationary",
|
| 400 |
+
"intensity": "low",
|
| 401 |
+
"speed": "slow",
|
| 402 |
+
"velocity": 0.0,
|
| 403 |
+
"acceleration": 0.0,
|
| 404 |
+
"fluidity": 1.0,
|
| 405 |
+
"expansion": 0.5
|
| 406 |
+
}
|
| 407 |
+
}
|
| 408 |
+
],
|
| 409 |
+
"summary": {
|
| 410 |
+
"direction": {
|
| 411 |
+
"distribution": {"stationary": 50, "up": 30, "down": 20},
|
| 412 |
+
"dominant": "stationary"
|
| 413 |
+
},
|
| 414 |
+
"intensity": {
|
| 415 |
+
"distribution": {"low": 80, "medium": 15, "high": 5},
|
| 416 |
+
"dominant": "low"
|
| 417 |
+
}
|
| 418 |
+
}
|
| 419 |
+
}
|
| 420 |
+
},
|
| 421 |
+
"video_output": "/tmp/visualization.mp4"
|
| 422 |
+
}
|
| 423 |
+
|
| 424 |
+
def api_info(self) -> Dict[str, Any]:
|
| 425 |
+
"""API information for the component."""
|
| 426 |
+
return {
|
| 427 |
+
"type": "composite",
|
| 428 |
+
"description": "Video-based pose analysis with Laban Movement Analysis metrics",
|
| 429 |
+
"parameters": {
|
| 430 |
+
"video": {"type": "file", "description": "Input video file or URL (YouTube/Vimeo)"},
|
| 431 |
+
"model": {"type": "string", "description": "Pose model: mediapipe, movenet, or yolo variants"},
|
| 432 |
+
"enable_visualization": {"type": "boolean", "description": "Generate visualization video"},
|
| 433 |
+
"include_keypoints": {"type": "boolean", "description": "Include keypoints in JSON"}
|
| 434 |
+
},
|
| 435 |
+
"returns": {
|
| 436 |
+
"json_output": {"type": "object", "description": "LMA analysis results"},
|
| 437 |
+
"video_output": {"type": "file", "description": "Visualization video (optional)"}
|
| 438 |
+
},
|
| 439 |
+
"version_2_preview": {
|
| 440 |
+
"planned_features": ["SkateFormer AI integration", "Enhanced movement recognition", "Real-time analysis"],
|
| 441 |
+
"note": "Advanced AI features coming in Version 2!"
|
| 442 |
+
}
|
| 443 |
+
}
|
| 444 |
+
from typing import Callable, Literal, Sequence, Any, TYPE_CHECKING
|
| 445 |
+
from gradio.blocks import Block
|
| 446 |
+
if TYPE_CHECKING:
|
| 447 |
+
from gradio.components import Timer
|
| 448 |
+
from gradio.components.base import Component
|
backend/gradio_labanmovementanalysis/notation_engine.py
ADDED
|
@@ -0,0 +1,317 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""
|
| 2 |
+
Laban Movement Analysis (LMA) inspired notation engine.
|
| 3 |
+
Computes movement metrics like direction, intensity, and speed from pose keypoints.
|
| 4 |
+
"""
|
| 5 |
+
|
| 6 |
+
import numpy as np
|
| 7 |
+
from typing import List, Dict, Optional, Tuple, Any
|
| 8 |
+
from dataclasses import dataclass
|
| 9 |
+
from enum import Enum
|
| 10 |
+
|
| 11 |
+
from .pose_estimation import PoseResult, Keypoint
|
| 12 |
+
|
| 13 |
+
|
| 14 |
+
class Direction(Enum):
|
| 15 |
+
"""Movement direction categories."""
|
| 16 |
+
UP = "up"
|
| 17 |
+
DOWN = "down"
|
| 18 |
+
LEFT = "left"
|
| 19 |
+
RIGHT = "right"
|
| 20 |
+
FORWARD = "forward"
|
| 21 |
+
BACKWARD = "backward"
|
| 22 |
+
STATIONARY = "stationary"
|
| 23 |
+
|
| 24 |
+
|
| 25 |
+
class Intensity(Enum):
|
| 26 |
+
"""Movement intensity levels."""
|
| 27 |
+
LOW = "low"
|
| 28 |
+
MEDIUM = "medium"
|
| 29 |
+
HIGH = "high"
|
| 30 |
+
|
| 31 |
+
|
| 32 |
+
class Speed(Enum):
|
| 33 |
+
"""Movement speed categories."""
|
| 34 |
+
SLOW = "slow"
|
| 35 |
+
MODERATE = "moderate"
|
| 36 |
+
FAST = "fast"
|
| 37 |
+
|
| 38 |
+
|
| 39 |
+
@dataclass
|
| 40 |
+
class MovementMetrics:
|
| 41 |
+
"""LMA-inspired movement metrics for a frame or segment."""
|
| 42 |
+
frame_index: int
|
| 43 |
+
timestamp: Optional[float] = None
|
| 44 |
+
|
| 45 |
+
# Primary metrics
|
| 46 |
+
direction: Direction = Direction.STATIONARY
|
| 47 |
+
intensity: Intensity = Intensity.LOW
|
| 48 |
+
speed: Speed = Speed.SLOW
|
| 49 |
+
|
| 50 |
+
# Numeric values
|
| 51 |
+
velocity: float = 0.0 # pixels/second or normalized units
|
| 52 |
+
acceleration: float = 0.0
|
| 53 |
+
|
| 54 |
+
# Additional qualities
|
| 55 |
+
fluidity: float = 0.0 # 0-1, smoothness of movement
|
| 56 |
+
expansion: float = 0.0 # 0-1, how spread out the pose is
|
| 57 |
+
|
| 58 |
+
# Raw displacement data
|
| 59 |
+
center_displacement: Optional[Tuple[float, float]] = None
|
| 60 |
+
total_displacement: float = 0.0
|
| 61 |
+
|
| 62 |
+
|
| 63 |
+
class MovementAnalyzer:
|
| 64 |
+
"""Analyzes pose sequences to extract LMA-style movement metrics."""
|
| 65 |
+
|
| 66 |
+
def __init__(self, fps: float = 30.0,
|
| 67 |
+
velocity_threshold_slow: float = 0.01,
|
| 68 |
+
velocity_threshold_fast: float = 0.1,
|
| 69 |
+
intensity_accel_threshold: float = 0.05):
|
| 70 |
+
"""
|
| 71 |
+
Initialize movement analyzer.
|
| 72 |
+
|
| 73 |
+
Args:
|
| 74 |
+
fps: Frames per second of the video
|
| 75 |
+
velocity_threshold_slow: Threshold for slow movement (normalized)
|
| 76 |
+
velocity_threshold_fast: Threshold for fast movement (normalized)
|
| 77 |
+
intensity_accel_threshold: Acceleration threshold for intensity
|
| 78 |
+
"""
|
| 79 |
+
self.fps = fps
|
| 80 |
+
self.frame_duration = 1.0 / fps
|
| 81 |
+
self.velocity_threshold_slow = velocity_threshold_slow
|
| 82 |
+
self.velocity_threshold_fast = velocity_threshold_fast
|
| 83 |
+
self.intensity_accel_threshold = intensity_accel_threshold
|
| 84 |
+
|
| 85 |
+
def analyze_movement(self, pose_sequence: List[List[PoseResult]]) -> List[MovementMetrics]:
|
| 86 |
+
"""
|
| 87 |
+
Analyze a sequence of poses to compute movement metrics.
|
| 88 |
+
|
| 89 |
+
Args:
|
| 90 |
+
pose_sequence: List of pose results per frame
|
| 91 |
+
|
| 92 |
+
Returns:
|
| 93 |
+
List of movement metrics per frame
|
| 94 |
+
"""
|
| 95 |
+
if not pose_sequence:
|
| 96 |
+
return []
|
| 97 |
+
|
| 98 |
+
metrics = []
|
| 99 |
+
prev_centers = None
|
| 100 |
+
prev_velocity = None
|
| 101 |
+
|
| 102 |
+
for frame_idx, frame_poses in enumerate(pose_sequence):
|
| 103 |
+
if not frame_poses:
|
| 104 |
+
# No pose detected in this frame
|
| 105 |
+
metrics.append(MovementMetrics(
|
| 106 |
+
frame_index=frame_idx,
|
| 107 |
+
timestamp=frame_idx * self.frame_duration
|
| 108 |
+
))
|
| 109 |
+
continue
|
| 110 |
+
|
| 111 |
+
# For now, analyze first person only
|
| 112 |
+
# TODO: Extend to multi-person analysis
|
| 113 |
+
pose = frame_poses[0]
|
| 114 |
+
|
| 115 |
+
# Compute body center and limb positions
|
| 116 |
+
center = self._compute_body_center(pose.keypoints)
|
| 117 |
+
limb_positions = self._get_limb_positions(pose.keypoints)
|
| 118 |
+
|
| 119 |
+
# Initialize metrics for this frame
|
| 120 |
+
frame_metrics = MovementMetrics(
|
| 121 |
+
frame_index=frame_idx,
|
| 122 |
+
timestamp=frame_idx * self.frame_duration
|
| 123 |
+
)
|
| 124 |
+
|
| 125 |
+
if prev_centers is not None and frame_idx > 0:
|
| 126 |
+
# Compute displacement and velocity
|
| 127 |
+
displacement = (
|
| 128 |
+
center[0] - prev_centers[0],
|
| 129 |
+
center[1] - prev_centers[1]
|
| 130 |
+
)
|
| 131 |
+
frame_metrics.center_displacement = displacement
|
| 132 |
+
frame_metrics.total_displacement = np.sqrt(
|
| 133 |
+
displacement[0]**2 + displacement[1]**2
|
| 134 |
+
)
|
| 135 |
+
|
| 136 |
+
# Velocity (normalized units per second)
|
| 137 |
+
frame_metrics.velocity = frame_metrics.total_displacement * self.fps
|
| 138 |
+
|
| 139 |
+
# Direction
|
| 140 |
+
frame_metrics.direction = self._compute_direction(displacement)
|
| 141 |
+
|
| 142 |
+
# Speed category
|
| 143 |
+
frame_metrics.speed = self._categorize_speed(frame_metrics.velocity)
|
| 144 |
+
|
| 145 |
+
# Acceleration and intensity
|
| 146 |
+
if prev_velocity is not None:
|
| 147 |
+
frame_metrics.acceleration = abs(
|
| 148 |
+
frame_metrics.velocity - prev_velocity
|
| 149 |
+
) * self.fps
|
| 150 |
+
frame_metrics.intensity = self._compute_intensity(
|
| 151 |
+
frame_metrics.acceleration,
|
| 152 |
+
frame_metrics.velocity
|
| 153 |
+
)
|
| 154 |
+
|
| 155 |
+
# Fluidity (based on acceleration smoothness)
|
| 156 |
+
frame_metrics.fluidity = self._compute_fluidity(
|
| 157 |
+
frame_metrics.acceleration
|
| 158 |
+
)
|
| 159 |
+
|
| 160 |
+
# Expansion (how spread out the pose is)
|
| 161 |
+
frame_metrics.expansion = self._compute_expansion(pose.keypoints)
|
| 162 |
+
|
| 163 |
+
metrics.append(frame_metrics)
|
| 164 |
+
|
| 165 |
+
# Update previous values
|
| 166 |
+
prev_centers = center
|
| 167 |
+
prev_velocity = frame_metrics.velocity
|
| 168 |
+
|
| 169 |
+
# Post-process to smooth metrics if needed
|
| 170 |
+
metrics = self._smooth_metrics(metrics)
|
| 171 |
+
|
| 172 |
+
return metrics
|
| 173 |
+
|
| 174 |
+
def _compute_body_center(self, keypoints: List[Keypoint]) -> Tuple[float, float]:
|
| 175 |
+
"""Compute the center of mass of the body."""
|
| 176 |
+
# Use major body joints for center calculation
|
| 177 |
+
major_joints = ["left_hip", "right_hip", "left_shoulder", "right_shoulder"]
|
| 178 |
+
|
| 179 |
+
x_coords = []
|
| 180 |
+
y_coords = []
|
| 181 |
+
|
| 182 |
+
for kp in keypoints:
|
| 183 |
+
if kp.name in major_joints and kp.confidence > 0.5:
|
| 184 |
+
x_coords.append(kp.x)
|
| 185 |
+
y_coords.append(kp.y)
|
| 186 |
+
|
| 187 |
+
if not x_coords:
|
| 188 |
+
# Fallback to all keypoints
|
| 189 |
+
x_coords = [kp.x for kp in keypoints if kp.confidence > 0.3]
|
| 190 |
+
y_coords = [kp.y for kp in keypoints if kp.confidence > 0.3]
|
| 191 |
+
|
| 192 |
+
if x_coords:
|
| 193 |
+
return (np.mean(x_coords), np.mean(y_coords))
|
| 194 |
+
return (0.5, 0.5) # Default center
|
| 195 |
+
|
| 196 |
+
def _get_limb_positions(self, keypoints: List[Keypoint]) -> Dict[str, Tuple[float, float]]:
|
| 197 |
+
"""Get positions of major limbs."""
|
| 198 |
+
positions = {}
|
| 199 |
+
for kp in keypoints:
|
| 200 |
+
if kp.confidence > 0.3:
|
| 201 |
+
positions[kp.name] = (kp.x, kp.y)
|
| 202 |
+
return positions
|
| 203 |
+
|
| 204 |
+
def _compute_direction(self, displacement: Tuple[float, float]) -> Direction:
|
| 205 |
+
"""Compute movement direction from displacement vector."""
|
| 206 |
+
dx, dy = displacement
|
| 207 |
+
|
| 208 |
+
# Threshold for considering movement
|
| 209 |
+
threshold = 0.005
|
| 210 |
+
|
| 211 |
+
if abs(dx) < threshold and abs(dy) < threshold:
|
| 212 |
+
return Direction.STATIONARY
|
| 213 |
+
|
| 214 |
+
# Determine primary direction
|
| 215 |
+
if abs(dx) > abs(dy):
|
| 216 |
+
return Direction.RIGHT if dx > 0 else Direction.LEFT
|
| 217 |
+
else:
|
| 218 |
+
return Direction.DOWN if dy > 0 else Direction.UP
|
| 219 |
+
|
| 220 |
+
def _categorize_speed(self, velocity: float) -> Speed:
|
| 221 |
+
"""Categorize velocity into speed levels."""
|
| 222 |
+
if velocity < self.velocity_threshold_slow:
|
| 223 |
+
return Speed.SLOW
|
| 224 |
+
elif velocity < self.velocity_threshold_fast:
|
| 225 |
+
return Speed.FAST
|
| 226 |
+
else:
|
| 227 |
+
return Speed.FAST
|
| 228 |
+
|
| 229 |
+
def _compute_intensity(self, acceleration: float, velocity: float) -> Intensity:
|
| 230 |
+
"""Compute movement intensity based on acceleration and velocity."""
|
| 231 |
+
# High acceleration or high velocity indicates high intensity
|
| 232 |
+
if acceleration > self.intensity_accel_threshold * 2 or velocity > self.velocity_threshold_fast:
|
| 233 |
+
return Intensity.HIGH
|
| 234 |
+
elif acceleration > self.intensity_accel_threshold or velocity > self.velocity_threshold_slow:
|
| 235 |
+
return Intensity.MEDIUM
|
| 236 |
+
else:
|
| 237 |
+
return Intensity.LOW
|
| 238 |
+
|
| 239 |
+
def _compute_fluidity(self, acceleration: float) -> float:
|
| 240 |
+
"""
|
| 241 |
+
Compute fluidity score (0-1) based on acceleration.
|
| 242 |
+
Lower acceleration = higher fluidity (smoother movement).
|
| 243 |
+
"""
|
| 244 |
+
# Normalize acceleration to 0-1 range
|
| 245 |
+
max_accel = 0.2 # Expected maximum acceleration
|
| 246 |
+
norm_accel = min(acceleration / max_accel, 1.0)
|
| 247 |
+
|
| 248 |
+
# Invert so low acceleration = high fluidity
|
| 249 |
+
return 1.0 - norm_accel
|
| 250 |
+
|
| 251 |
+
def _compute_expansion(self, keypoints: List[Keypoint]) -> float:
|
| 252 |
+
"""
|
| 253 |
+
Compute how expanded/contracted the pose is.
|
| 254 |
+
Returns 0-1 where 1 is fully expanded.
|
| 255 |
+
"""
|
| 256 |
+
# Calculate distances between opposite limbs
|
| 257 |
+
limb_pairs = [
|
| 258 |
+
("left_wrist", "right_wrist"),
|
| 259 |
+
("left_ankle", "right_ankle"),
|
| 260 |
+
("left_wrist", "left_ankle"),
|
| 261 |
+
("right_wrist", "right_ankle")
|
| 262 |
+
]
|
| 263 |
+
|
| 264 |
+
kp_dict = {kp.name: kp for kp in keypoints if kp.confidence > 0.3}
|
| 265 |
+
|
| 266 |
+
distances = []
|
| 267 |
+
for limb1, limb2 in limb_pairs:
|
| 268 |
+
if limb1 in kp_dict and limb2 in kp_dict:
|
| 269 |
+
kp1 = kp_dict[limb1]
|
| 270 |
+
kp2 = kp_dict[limb2]
|
| 271 |
+
dist = np.sqrt((kp1.x - kp2.x)**2 + (kp1.y - kp2.y)**2)
|
| 272 |
+
distances.append(dist)
|
| 273 |
+
|
| 274 |
+
if distances:
|
| 275 |
+
# Normalize by expected maximum distance
|
| 276 |
+
avg_dist = np.mean(distances)
|
| 277 |
+
max_expected = 1.4 # Diagonal of normalized space
|
| 278 |
+
return min(avg_dist / max_expected, 1.0)
|
| 279 |
+
|
| 280 |
+
return 0.5 # Default neutral expansion
|
| 281 |
+
|
| 282 |
+
def _smooth_metrics(self, metrics: List[MovementMetrics]) -> List[MovementMetrics]:
|
| 283 |
+
"""Apply smoothing to reduce noise in metrics."""
|
| 284 |
+
# Simple moving average for numeric values
|
| 285 |
+
window_size = 3
|
| 286 |
+
|
| 287 |
+
if len(metrics) <= window_size:
|
| 288 |
+
return metrics
|
| 289 |
+
|
| 290 |
+
# Smooth velocity and acceleration
|
| 291 |
+
for i in range(window_size, len(metrics)):
|
| 292 |
+
velocities = [m.velocity for m in metrics[i-window_size:i+1]]
|
| 293 |
+
metrics[i].velocity = np.mean(velocities)
|
| 294 |
+
|
| 295 |
+
accels = [m.acceleration for m in metrics[i-window_size:i+1]]
|
| 296 |
+
metrics[i].acceleration = np.mean(accels)
|
| 297 |
+
|
| 298 |
+
fluidities = [m.fluidity for m in metrics[i-window_size:i+1]]
|
| 299 |
+
metrics[i].fluidity = np.mean(fluidities)
|
| 300 |
+
|
| 301 |
+
return metrics
|
| 302 |
+
|
| 303 |
+
|
| 304 |
+
def analyze_pose_sequence(pose_sequence: List[List[PoseResult]],
|
| 305 |
+
fps: float = 30.0) -> List[MovementMetrics]:
|
| 306 |
+
"""
|
| 307 |
+
Convenience function to analyze a pose sequence.
|
| 308 |
+
|
| 309 |
+
Args:
|
| 310 |
+
pose_sequence: List of pose results per frame
|
| 311 |
+
fps: Video frame rate
|
| 312 |
+
|
| 313 |
+
Returns:
|
| 314 |
+
List of movement metrics
|
| 315 |
+
"""
|
| 316 |
+
analyzer = MovementAnalyzer(fps=fps)
|
| 317 |
+
return analyzer.analyze_movement(pose_sequence)
|
backend/gradio_labanmovementanalysis/pose_estimation.py
ADDED
|
@@ -0,0 +1,380 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""
|
| 2 |
+
Model-agnostic pose estimation interface with adapters for various pose detection models.
|
| 3 |
+
Each adapter provides a uniform interface for pose estimation, abstracting model-specific details.
|
| 4 |
+
"""
|
| 5 |
+
|
| 6 |
+
from abc import ABC, abstractmethod
|
| 7 |
+
import numpy as np
|
| 8 |
+
from typing import List, Dict, Optional, Tuple, Any
|
| 9 |
+
from dataclasses import dataclass
|
| 10 |
+
|
| 11 |
+
|
| 12 |
+
@dataclass
|
| 13 |
+
class Keypoint:
|
| 14 |
+
"""Represents a single pose keypoint."""
|
| 15 |
+
x: float # Normalized x coordinate (0-1)
|
| 16 |
+
y: float # Normalized y coordinate (0-1)
|
| 17 |
+
confidence: float # Confidence score (0-1)
|
| 18 |
+
name: Optional[str] = None # Joint name (e.g., "left_shoulder")
|
| 19 |
+
|
| 20 |
+
|
| 21 |
+
@dataclass
|
| 22 |
+
class PoseResult:
|
| 23 |
+
"""Result of pose estimation for a single frame."""
|
| 24 |
+
keypoints: List[Keypoint]
|
| 25 |
+
frame_index: int
|
| 26 |
+
timestamp: Optional[float] = None
|
| 27 |
+
person_id: Optional[int] = None # For multi-person tracking
|
| 28 |
+
|
| 29 |
+
|
| 30 |
+
class PoseEstimator(ABC):
|
| 31 |
+
"""Abstract base class for pose estimation models."""
|
| 32 |
+
|
| 33 |
+
@abstractmethod
|
| 34 |
+
def detect(self, frame: np.ndarray) -> List[PoseResult]:
|
| 35 |
+
"""
|
| 36 |
+
Detect poses in a single frame.
|
| 37 |
+
|
| 38 |
+
Args:
|
| 39 |
+
frame: Input frame as numpy array (BGR format)
|
| 40 |
+
|
| 41 |
+
Returns:
|
| 42 |
+
List of PoseResult objects (one per detected person)
|
| 43 |
+
"""
|
| 44 |
+
pass
|
| 45 |
+
|
| 46 |
+
@abstractmethod
|
| 47 |
+
def get_keypoint_names(self) -> List[str]:
|
| 48 |
+
"""Get the list of keypoint names this model provides."""
|
| 49 |
+
pass
|
| 50 |
+
|
| 51 |
+
def detect_batch(self, frames: List[np.ndarray]) -> List[List[PoseResult]]:
|
| 52 |
+
"""
|
| 53 |
+
Process multiple frames (default implementation processes sequentially).
|
| 54 |
+
|
| 55 |
+
Args:
|
| 56 |
+
frames: List of frames
|
| 57 |
+
|
| 58 |
+
Returns:
|
| 59 |
+
List of results per frame
|
| 60 |
+
"""
|
| 61 |
+
results = []
|
| 62 |
+
for i, frame in enumerate(frames):
|
| 63 |
+
frame_results = self.detect(frame)
|
| 64 |
+
# Update frame indices
|
| 65 |
+
for result in frame_results:
|
| 66 |
+
result.frame_index = i
|
| 67 |
+
results.append(frame_results)
|
| 68 |
+
return results
|
| 69 |
+
|
| 70 |
+
|
| 71 |
+
class MoveNetPoseEstimator(PoseEstimator):
|
| 72 |
+
"""MoveNet pose estimation adapter (TensorFlow-based)."""
|
| 73 |
+
|
| 74 |
+
# COCO keypoint names used by MoveNet
|
| 75 |
+
KEYPOINT_NAMES = [
|
| 76 |
+
"nose", "left_eye", "right_eye", "left_ear", "right_ear",
|
| 77 |
+
"left_shoulder", "right_shoulder", "left_elbow", "right_elbow",
|
| 78 |
+
"left_wrist", "right_wrist", "left_hip", "right_hip",
|
| 79 |
+
"left_knee", "right_knee", "left_ankle", "right_ankle"
|
| 80 |
+
]
|
| 81 |
+
|
| 82 |
+
def __init__(self, model_variant: str = "lightning"):
|
| 83 |
+
"""
|
| 84 |
+
Initialize MoveNet model.
|
| 85 |
+
|
| 86 |
+
Args:
|
| 87 |
+
model_variant: "lightning" (faster) or "thunder" (more accurate)
|
| 88 |
+
"""
|
| 89 |
+
self.model_variant = model_variant
|
| 90 |
+
self.model = None
|
| 91 |
+
self._load_model()
|
| 92 |
+
|
| 93 |
+
def _load_model(self):
|
| 94 |
+
"""Load MoveNet model using TensorFlow."""
|
| 95 |
+
try:
|
| 96 |
+
import tensorflow as tf
|
| 97 |
+
import tensorflow_hub as hub
|
| 98 |
+
|
| 99 |
+
# Model URLs for different variants
|
| 100 |
+
model_urls = {
|
| 101 |
+
"lightning": "https://tfhub.dev/google/movenet/singlepose/lightning/4",
|
| 102 |
+
"thunder": "https://tfhub.dev/google/movenet/singlepose/thunder/4"
|
| 103 |
+
}
|
| 104 |
+
|
| 105 |
+
self.model = hub.load(model_urls[self.model_variant])
|
| 106 |
+
self.movenet = self.model.signatures['serving_default']
|
| 107 |
+
|
| 108 |
+
except ImportError:
|
| 109 |
+
raise ImportError("TensorFlow and tensorflow_hub required for MoveNet. "
|
| 110 |
+
"Install with: pip install tensorflow tensorflow_hub")
|
| 111 |
+
|
| 112 |
+
def detect(self, frame: np.ndarray) -> List[PoseResult]:
|
| 113 |
+
"""Detect pose using MoveNet."""
|
| 114 |
+
if self.model is None:
|
| 115 |
+
self._load_model()
|
| 116 |
+
|
| 117 |
+
import tensorflow as tf
|
| 118 |
+
|
| 119 |
+
# Prepare input
|
| 120 |
+
height, width = frame.shape[:2]
|
| 121 |
+
|
| 122 |
+
# MoveNet expects RGB
|
| 123 |
+
rgb_frame = frame[:, :, ::-1] # BGR to RGB
|
| 124 |
+
|
| 125 |
+
# Resize and normalize
|
| 126 |
+
input_size = 192 if self.model_variant == "lightning" else 256
|
| 127 |
+
input_image = tf.image.resize_with_pad(
|
| 128 |
+
tf.expand_dims(rgb_frame, axis=0), input_size, input_size
|
| 129 |
+
)
|
| 130 |
+
input_image = tf.cast(input_image, dtype=tf.int32)
|
| 131 |
+
|
| 132 |
+
# Run inference
|
| 133 |
+
outputs = self.movenet(input_image)
|
| 134 |
+
keypoints_with_scores = outputs['output_0'].numpy()[0, 0, :, :]
|
| 135 |
+
|
| 136 |
+
# Convert to our format
|
| 137 |
+
keypoints = []
|
| 138 |
+
for i, (y, x, score) in enumerate(keypoints_with_scores):
|
| 139 |
+
keypoints.append(Keypoint(
|
| 140 |
+
x=float(x),
|
| 141 |
+
y=float(y),
|
| 142 |
+
confidence=float(score),
|
| 143 |
+
name=self.KEYPOINT_NAMES[i]
|
| 144 |
+
))
|
| 145 |
+
|
| 146 |
+
return [PoseResult(keypoints=keypoints, frame_index=0)]
|
| 147 |
+
|
| 148 |
+
def get_keypoint_names(self) -> List[str]:
|
| 149 |
+
return self.KEYPOINT_NAMES.copy()
|
| 150 |
+
|
| 151 |
+
|
| 152 |
+
class MediaPipePoseEstimator(PoseEstimator):
|
| 153 |
+
"""MediaPipe Pose (BlazePose) estimation adapter."""
|
| 154 |
+
|
| 155 |
+
# MediaPipe landmark names
|
| 156 |
+
LANDMARK_NAMES = [
|
| 157 |
+
"nose", "left_eye_inner", "left_eye", "left_eye_outer",
|
| 158 |
+
"right_eye_inner", "right_eye", "right_eye_outer",
|
| 159 |
+
"left_ear", "right_ear", "mouth_left", "mouth_right",
|
| 160 |
+
"left_shoulder", "right_shoulder", "left_elbow", "right_elbow",
|
| 161 |
+
"left_wrist", "right_wrist", "left_pinky", "right_pinky",
|
| 162 |
+
"left_index", "right_index", "left_thumb", "right_thumb",
|
| 163 |
+
"left_hip", "right_hip", "left_knee", "right_knee",
|
| 164 |
+
"left_ankle", "right_ankle", "left_heel", "right_heel",
|
| 165 |
+
"left_foot_index", "right_foot_index"
|
| 166 |
+
]
|
| 167 |
+
|
| 168 |
+
def __init__(self, model_complexity: int = 1, min_detection_confidence: float = 0.5):
|
| 169 |
+
"""
|
| 170 |
+
Initialize MediaPipe Pose.
|
| 171 |
+
|
| 172 |
+
Args:
|
| 173 |
+
model_complexity: 0 (lite), 1 (full), or 2 (heavy)
|
| 174 |
+
min_detection_confidence: Minimum confidence for detection
|
| 175 |
+
"""
|
| 176 |
+
self.model_complexity = model_complexity
|
| 177 |
+
self.min_detection_confidence = min_detection_confidence
|
| 178 |
+
self.pose = None
|
| 179 |
+
self._initialize()
|
| 180 |
+
|
| 181 |
+
def _initialize(self):
|
| 182 |
+
"""Initialize MediaPipe Pose."""
|
| 183 |
+
try:
|
| 184 |
+
import mediapipe as mp
|
| 185 |
+
self.mp_pose = mp.solutions.pose
|
| 186 |
+
self.pose = self.mp_pose.Pose(
|
| 187 |
+
static_image_mode=False,
|
| 188 |
+
model_complexity=self.model_complexity,
|
| 189 |
+
min_detection_confidence=self.min_detection_confidence,
|
| 190 |
+
min_tracking_confidence=0.5
|
| 191 |
+
)
|
| 192 |
+
except ImportError:
|
| 193 |
+
raise ImportError("MediaPipe required. Install with: pip install mediapipe")
|
| 194 |
+
|
| 195 |
+
def detect(self, frame: np.ndarray) -> List[PoseResult]:
|
| 196 |
+
"""Detect pose using MediaPipe."""
|
| 197 |
+
if self.pose is None:
|
| 198 |
+
self._initialize()
|
| 199 |
+
|
| 200 |
+
# MediaPipe expects RGB
|
| 201 |
+
rgb_frame = frame[:, :, ::-1] # BGR to RGB
|
| 202 |
+
height, width = frame.shape[:2]
|
| 203 |
+
|
| 204 |
+
# Process frame
|
| 205 |
+
results = self.pose.process(rgb_frame)
|
| 206 |
+
|
| 207 |
+
if not results.pose_landmarks:
|
| 208 |
+
return []
|
| 209 |
+
|
| 210 |
+
# Convert landmarks to keypoints
|
| 211 |
+
keypoints = []
|
| 212 |
+
for i, landmark in enumerate(results.pose_landmarks.landmark):
|
| 213 |
+
keypoints.append(Keypoint(
|
| 214 |
+
x=landmark.x,
|
| 215 |
+
y=landmark.y,
|
| 216 |
+
confidence=landmark.visibility if hasattr(landmark, 'visibility') else 1.0,
|
| 217 |
+
name=self.LANDMARK_NAMES[i] if i < len(self.LANDMARK_NAMES) else f"landmark_{i}"
|
| 218 |
+
))
|
| 219 |
+
|
| 220 |
+
return [PoseResult(keypoints=keypoints, frame_index=0)]
|
| 221 |
+
|
| 222 |
+
def get_keypoint_names(self) -> List[str]:
|
| 223 |
+
return self.LANDMARK_NAMES.copy()
|
| 224 |
+
|
| 225 |
+
def __del__(self):
|
| 226 |
+
"""Clean up MediaPipe resources."""
|
| 227 |
+
if self.pose:
|
| 228 |
+
self.pose.close()
|
| 229 |
+
|
| 230 |
+
|
| 231 |
+
class YOLOPoseEstimator(PoseEstimator):
|
| 232 |
+
"""YOLO-based pose estimation adapter (supports YOLOv8 and YOLOv11)."""
|
| 233 |
+
|
| 234 |
+
# COCO keypoint format used by YOLO
|
| 235 |
+
KEYPOINT_NAMES = [
|
| 236 |
+
"nose", "left_eye", "right_eye", "left_ear", "right_ear",
|
| 237 |
+
"left_shoulder", "right_shoulder", "left_elbow", "right_elbow",
|
| 238 |
+
"left_wrist", "right_wrist", "left_hip", "right_hip",
|
| 239 |
+
"left_knee", "right_knee", "left_ankle", "right_ankle"
|
| 240 |
+
]
|
| 241 |
+
|
| 242 |
+
def __init__(self, model_version: str = "v11", model_size: str = "n", confidence_threshold: float = 0.25):
|
| 243 |
+
"""
|
| 244 |
+
Initialize YOLO pose model.
|
| 245 |
+
|
| 246 |
+
Args:
|
| 247 |
+
model_version: "v8" or "v11"
|
| 248 |
+
model_size: Model size - "n" (nano), "s" (small), "m" (medium), "l" (large), "x" (xlarge)
|
| 249 |
+
confidence_threshold: Minimum confidence for detections
|
| 250 |
+
"""
|
| 251 |
+
self.model_version = model_version
|
| 252 |
+
self.model_size = model_size
|
| 253 |
+
self.confidence_threshold = confidence_threshold
|
| 254 |
+
self.model = None
|
| 255 |
+
|
| 256 |
+
# Determine model path
|
| 257 |
+
if model_version == "v8":
|
| 258 |
+
self.model_path = f"yolov8{model_size}-pose.pt"
|
| 259 |
+
else: # v11
|
| 260 |
+
self.model_path = f"yolo11{model_size}-pose.pt"
|
| 261 |
+
|
| 262 |
+
self._load_model()
|
| 263 |
+
|
| 264 |
+
def _load_model(self):
|
| 265 |
+
"""Load YOLO model."""
|
| 266 |
+
try:
|
| 267 |
+
from ultralytics import YOLO
|
| 268 |
+
self.model = YOLO(self.model_path)
|
| 269 |
+
except ImportError:
|
| 270 |
+
raise ImportError("Ultralytics required for YOLO. "
|
| 271 |
+
"Install with: pip install ultralytics")
|
| 272 |
+
|
| 273 |
+
def detect(self, frame: np.ndarray) -> List[PoseResult]:
|
| 274 |
+
"""Detect poses using YOLO."""
|
| 275 |
+
if self.model is None:
|
| 276 |
+
self._load_model()
|
| 277 |
+
|
| 278 |
+
# Run inference
|
| 279 |
+
results = self.model(frame, conf=self.confidence_threshold)
|
| 280 |
+
|
| 281 |
+
pose_results = []
|
| 282 |
+
|
| 283 |
+
# Process each detection
|
| 284 |
+
for r in results:
|
| 285 |
+
if r.keypoints is not None:
|
| 286 |
+
for person_idx, keypoints_data in enumerate(r.keypoints.data):
|
| 287 |
+
keypoints = []
|
| 288 |
+
|
| 289 |
+
# YOLO returns keypoints as [x, y, conf]
|
| 290 |
+
height, width = frame.shape[:2]
|
| 291 |
+
for i, (x, y, conf) in enumerate(keypoints_data):
|
| 292 |
+
keypoints.append(Keypoint(
|
| 293 |
+
x=float(x) / width, # Normalize to 0-1
|
| 294 |
+
y=float(y) / height, # Normalize to 0-1
|
| 295 |
+
confidence=float(conf),
|
| 296 |
+
name=self.KEYPOINT_NAMES[i] if i < len(self.KEYPOINT_NAMES) else f"joint_{i}"
|
| 297 |
+
))
|
| 298 |
+
|
| 299 |
+
pose_results.append(PoseResult(
|
| 300 |
+
keypoints=keypoints,
|
| 301 |
+
frame_index=0,
|
| 302 |
+
person_id=person_idx
|
| 303 |
+
))
|
| 304 |
+
|
| 305 |
+
return pose_results
|
| 306 |
+
|
| 307 |
+
def get_keypoint_names(self) -> List[str]:
|
| 308 |
+
return self.KEYPOINT_NAMES.copy()
|
| 309 |
+
|
| 310 |
+
|
| 311 |
+
# Note: Sapiens models removed due to complex setup requirements
|
| 312 |
+
# They require the official repository and cannot be integrated cleanly
|
| 313 |
+
# with the agent/MCP pipeline without significant complexity
|
| 314 |
+
|
| 315 |
+
|
| 316 |
+
def create_pose_estimator(model_type: str, **kwargs) -> PoseEstimator:
|
| 317 |
+
"""
|
| 318 |
+
Factory function to create pose estimator instances.
|
| 319 |
+
|
| 320 |
+
Args:
|
| 321 |
+
model_type: One of "movenet", "mediapipe", "yolo"
|
| 322 |
+
**kwargs: Model-specific parameters
|
| 323 |
+
|
| 324 |
+
Returns:
|
| 325 |
+
PoseEstimator instance
|
| 326 |
+
"""
|
| 327 |
+
estimators = {
|
| 328 |
+
"movenet": MoveNetPoseEstimator,
|
| 329 |
+
"mediapipe": MediaPipePoseEstimator,
|
| 330 |
+
"yolo": YOLOPoseEstimator,
|
| 331 |
+
}
|
| 332 |
+
|
| 333 |
+
if model_type not in estimators:
|
| 334 |
+
raise ValueError(f"Unknown model type: {model_type}. "
|
| 335 |
+
f"Available: {list(estimators.keys())}")
|
| 336 |
+
|
| 337 |
+
return estimators[model_type](**kwargs)
|
| 338 |
+
|
| 339 |
+
|
| 340 |
+
def get_pose_estimator(model_spec: str) -> PoseEstimator:
|
| 341 |
+
"""
|
| 342 |
+
Get pose estimator from model specification string.
|
| 343 |
+
|
| 344 |
+
Args:
|
| 345 |
+
model_spec: Model specification string, e.g.:
|
| 346 |
+
- "mediapipe" or "mediapipe-lite" or "mediapipe-full" or "mediapipe-heavy"
|
| 347 |
+
- "movenet-lightning" or "movenet-thunder"
|
| 348 |
+
- "yolo-v8-n" or "yolo-v11-s" etc.
|
| 349 |
+
|
| 350 |
+
Returns:
|
| 351 |
+
PoseEstimator instance
|
| 352 |
+
"""
|
| 353 |
+
model_spec = model_spec.lower()
|
| 354 |
+
|
| 355 |
+
# MediaPipe variants
|
| 356 |
+
if model_spec.startswith("mediapipe"):
|
| 357 |
+
complexity_map = {
|
| 358 |
+
"mediapipe-lite": 0,
|
| 359 |
+
"mediapipe-full": 1,
|
| 360 |
+
"mediapipe-heavy": 2,
|
| 361 |
+
"mediapipe": 1 # default
|
| 362 |
+
}
|
| 363 |
+
complexity = complexity_map.get(model_spec, 1)
|
| 364 |
+
return create_pose_estimator("mediapipe", model_complexity=complexity)
|
| 365 |
+
|
| 366 |
+
# MoveNet variants
|
| 367 |
+
elif model_spec.startswith("movenet"):
|
| 368 |
+
variant = "lightning" if "lightning" in model_spec else "thunder"
|
| 369 |
+
return create_pose_estimator("movenet", model_variant=variant)
|
| 370 |
+
|
| 371 |
+
# YOLO variants
|
| 372 |
+
elif model_spec.startswith("yolo"):
|
| 373 |
+
parts = model_spec.split("-")
|
| 374 |
+
version = "v8" if "v8" in model_spec else "v11"
|
| 375 |
+
size = parts[-1] if len(parts) > 2 else "n"
|
| 376 |
+
return create_pose_estimator("yolo", model_version=version, model_size=size)
|
| 377 |
+
|
| 378 |
+
# Legacy format support
|
| 379 |
+
else:
|
| 380 |
+
return create_pose_estimator(model_spec)
|
backend/gradio_labanmovementanalysis/video_downloader.py
ADDED
|
@@ -0,0 +1,295 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""
|
| 2 |
+
Video downloader for YouTube, Vimeo and other platforms
|
| 3 |
+
"""
|
| 4 |
+
|
| 5 |
+
import os
|
| 6 |
+
import re
|
| 7 |
+
import tempfile
|
| 8 |
+
import logging
|
| 9 |
+
from typing import Optional, Tuple, Dict, Any
|
| 10 |
+
from urllib.parse import urlparse, parse_qs
|
| 11 |
+
import subprocess
|
| 12 |
+
|
| 13 |
+
logger = logging.getLogger(__name__)
|
| 14 |
+
|
| 15 |
+
|
| 16 |
+
class VideoDownloader:
|
| 17 |
+
"""Download videos from various platforms"""
|
| 18 |
+
|
| 19 |
+
# Gradio component compatibility
|
| 20 |
+
events = {}
|
| 21 |
+
|
| 22 |
+
def __init__(self, temp_dir: Optional[str] = None):
|
| 23 |
+
"""
|
| 24 |
+
Initialize video downloader.
|
| 25 |
+
|
| 26 |
+
Args:
|
| 27 |
+
temp_dir: Directory for temporary files
|
| 28 |
+
"""
|
| 29 |
+
self.temp_dir = temp_dir or tempfile.mkdtemp(prefix="laban_video_")
|
| 30 |
+
self.supported_platforms = {
|
| 31 |
+
'youtube': self._download_youtube,
|
| 32 |
+
'vimeo': self._download_vimeo,
|
| 33 |
+
'direct': self._download_direct
|
| 34 |
+
}
|
| 35 |
+
|
| 36 |
+
def download(self, url: str) -> Tuple[str, Dict[str, Any]]:
|
| 37 |
+
"""
|
| 38 |
+
Download video from URL.
|
| 39 |
+
|
| 40 |
+
Args:
|
| 41 |
+
url: Video URL (YouTube, Vimeo, or direct video link)
|
| 42 |
+
|
| 43 |
+
Returns:
|
| 44 |
+
Tuple of (local_path, metadata)
|
| 45 |
+
"""
|
| 46 |
+
platform = self._detect_platform(url)
|
| 47 |
+
|
| 48 |
+
if platform not in self.supported_platforms:
|
| 49 |
+
raise ValueError(f"Unsupported platform: {platform}")
|
| 50 |
+
|
| 51 |
+
logger.info(f"Downloading video from {platform}: {url}")
|
| 52 |
+
|
| 53 |
+
try:
|
| 54 |
+
return self.supported_platforms[platform](url)
|
| 55 |
+
except Exception as e:
|
| 56 |
+
logger.error(f"Failed to download video: {str(e)}")
|
| 57 |
+
raise
|
| 58 |
+
|
| 59 |
+
def _detect_platform(self, url: str) -> str:
|
| 60 |
+
"""Detect video platform from URL"""
|
| 61 |
+
domain = urlparse(url).netloc.lower()
|
| 62 |
+
|
| 63 |
+
if 'youtube.com' in domain or 'youtu.be' in domain:
|
| 64 |
+
return 'youtube'
|
| 65 |
+
elif 'vimeo.com' in domain:
|
| 66 |
+
return 'vimeo'
|
| 67 |
+
elif url.endswith(('.mp4', '.avi', '.mov', '.webm')):
|
| 68 |
+
return 'direct'
|
| 69 |
+
else:
|
| 70 |
+
# Try to determine if it's a direct video link
|
| 71 |
+
return 'direct'
|
| 72 |
+
|
| 73 |
+
def _download_youtube(self, url: str) -> Tuple[str, Dict[str, Any]]:
|
| 74 |
+
"""Download video from YouTube using yt-dlp"""
|
| 75 |
+
try:
|
| 76 |
+
import yt_dlp
|
| 77 |
+
except ImportError:
|
| 78 |
+
raise ImportError("yt-dlp is required for YouTube downloads. Install with: pip install yt-dlp")
|
| 79 |
+
|
| 80 |
+
# Extract video ID
|
| 81 |
+
video_id = self._extract_youtube_id(url)
|
| 82 |
+
output_path = os.path.join(self.temp_dir, f"youtube_{video_id}.mp4")
|
| 83 |
+
|
| 84 |
+
# yt-dlp options
|
| 85 |
+
ydl_opts = {
|
| 86 |
+
'format': 'best[height<=720][ext=mp4]/best[height<=720]/best', # Limit to 720p for performance
|
| 87 |
+
'outtmpl': output_path,
|
| 88 |
+
'quiet': True,
|
| 89 |
+
'no_warnings': True,
|
| 90 |
+
'extract_flat': False,
|
| 91 |
+
}
|
| 92 |
+
|
| 93 |
+
metadata = {}
|
| 94 |
+
|
| 95 |
+
with yt_dlp.YoutubeDL(ydl_opts) as ydl:
|
| 96 |
+
try:
|
| 97 |
+
# Extract info
|
| 98 |
+
info = ydl.extract_info(url, download=True)
|
| 99 |
+
|
| 100 |
+
# Store metadata
|
| 101 |
+
metadata = {
|
| 102 |
+
'title': info.get('title', 'Unknown'),
|
| 103 |
+
'duration': info.get('duration', 0),
|
| 104 |
+
'uploader': info.get('uploader', 'Unknown'),
|
| 105 |
+
'view_count': info.get('view_count', 0),
|
| 106 |
+
'description': info.get('description', ''),
|
| 107 |
+
'platform': 'youtube',
|
| 108 |
+
'video_id': video_id
|
| 109 |
+
}
|
| 110 |
+
|
| 111 |
+
logger.info(f"Downloaded YouTube video: {metadata['title']}")
|
| 112 |
+
|
| 113 |
+
except Exception as e:
|
| 114 |
+
raise Exception(f"Failed to download YouTube video: {str(e)}")
|
| 115 |
+
|
| 116 |
+
return output_path, metadata
|
| 117 |
+
|
| 118 |
+
def _download_vimeo(self, url: str) -> Tuple[str, Dict[str, Any]]:
|
| 119 |
+
"""Download video from Vimeo using yt-dlp"""
|
| 120 |
+
try:
|
| 121 |
+
import yt_dlp
|
| 122 |
+
except ImportError:
|
| 123 |
+
raise ImportError("yt-dlp is required for Vimeo downloads. Install with: pip install yt-dlp")
|
| 124 |
+
|
| 125 |
+
# Extract video ID
|
| 126 |
+
video_id = self._extract_vimeo_id(url)
|
| 127 |
+
output_path = os.path.join(self.temp_dir, f"vimeo_{video_id}.mp4")
|
| 128 |
+
|
| 129 |
+
# yt-dlp options
|
| 130 |
+
ydl_opts = {
|
| 131 |
+
'format': 'best[height<=720][ext=mp4]/best[height<=720]/best',
|
| 132 |
+
'outtmpl': output_path,
|
| 133 |
+
'quiet': True,
|
| 134 |
+
'no_warnings': True,
|
| 135 |
+
}
|
| 136 |
+
|
| 137 |
+
metadata = {}
|
| 138 |
+
|
| 139 |
+
with yt_dlp.YoutubeDL(ydl_opts) as ydl:
|
| 140 |
+
try:
|
| 141 |
+
# Extract info
|
| 142 |
+
info = ydl.extract_info(url, download=True)
|
| 143 |
+
|
| 144 |
+
# Store metadata
|
| 145 |
+
metadata = {
|
| 146 |
+
'title': info.get('title', 'Unknown'),
|
| 147 |
+
'duration': info.get('duration', 0),
|
| 148 |
+
'uploader': info.get('uploader', 'Unknown'),
|
| 149 |
+
'description': info.get('description', ''),
|
| 150 |
+
'platform': 'vimeo',
|
| 151 |
+
'video_id': video_id
|
| 152 |
+
}
|
| 153 |
+
|
| 154 |
+
logger.info(f"Downloaded Vimeo video: {metadata['title']}")
|
| 155 |
+
|
| 156 |
+
except Exception as e:
|
| 157 |
+
raise Exception(f"Failed to download Vimeo video: {str(e)}")
|
| 158 |
+
|
| 159 |
+
return output_path, metadata
|
| 160 |
+
|
| 161 |
+
def _download_direct(self, url: str) -> Tuple[str, Dict[str, Any]]:
|
| 162 |
+
"""Download video from direct URL"""
|
| 163 |
+
import requests
|
| 164 |
+
|
| 165 |
+
# Generate filename from URL
|
| 166 |
+
filename = os.path.basename(urlparse(url).path) or "video.mp4"
|
| 167 |
+
output_path = os.path.join(self.temp_dir, filename)
|
| 168 |
+
|
| 169 |
+
try:
|
| 170 |
+
# Download with streaming
|
| 171 |
+
response = requests.get(url, stream=True)
|
| 172 |
+
response.raise_for_status()
|
| 173 |
+
|
| 174 |
+
# Get content length
|
| 175 |
+
total_size = int(response.headers.get('content-length', 0))
|
| 176 |
+
|
| 177 |
+
# Write to file
|
| 178 |
+
with open(output_path, 'wb') as f:
|
| 179 |
+
downloaded = 0
|
| 180 |
+
for chunk in response.iter_content(chunk_size=8192):
|
| 181 |
+
if chunk:
|
| 182 |
+
f.write(chunk)
|
| 183 |
+
downloaded += len(chunk)
|
| 184 |
+
|
| 185 |
+
# Progress logging
|
| 186 |
+
if total_size > 0:
|
| 187 |
+
progress = (downloaded / total_size) * 100
|
| 188 |
+
if int(progress) % 10 == 0:
|
| 189 |
+
logger.debug(f"Download progress: {progress:.1f}%")
|
| 190 |
+
|
| 191 |
+
metadata = {
|
| 192 |
+
'title': filename,
|
| 193 |
+
'platform': 'direct',
|
| 194 |
+
'url': url,
|
| 195 |
+
'size': total_size
|
| 196 |
+
}
|
| 197 |
+
|
| 198 |
+
logger.info(f"Downloaded direct video: {filename}")
|
| 199 |
+
|
| 200 |
+
except Exception as e:
|
| 201 |
+
raise Exception(f"Failed to download direct video: {str(e)}")
|
| 202 |
+
|
| 203 |
+
return output_path, metadata
|
| 204 |
+
|
| 205 |
+
def _extract_youtube_id(self, url: str) -> str:
|
| 206 |
+
"""Extract YouTube video ID from URL"""
|
| 207 |
+
patterns = [
|
| 208 |
+
r'(?:v=|\/)([0-9A-Za-z_-]{11}).*',
|
| 209 |
+
r'(?:embed\/)([0-9A-Za-z_-]{11})',
|
| 210 |
+
r'(?:watch\?v=)([0-9A-Za-z_-]{11})',
|
| 211 |
+
r'youtu\.be\/([0-9A-Za-z_-]{11})'
|
| 212 |
+
]
|
| 213 |
+
|
| 214 |
+
for pattern in patterns:
|
| 215 |
+
match = re.search(pattern, url)
|
| 216 |
+
if match:
|
| 217 |
+
return match.group(1)
|
| 218 |
+
|
| 219 |
+
raise ValueError(f"Could not extract YouTube video ID from: {url}")
|
| 220 |
+
|
| 221 |
+
def _extract_vimeo_id(self, url: str) -> str:
|
| 222 |
+
"""Extract Vimeo video ID from URL"""
|
| 223 |
+
patterns = [
|
| 224 |
+
r'vimeo\.com\/(\d+)',
|
| 225 |
+
r'player\.vimeo\.com\/video\/(\d+)'
|
| 226 |
+
]
|
| 227 |
+
|
| 228 |
+
for pattern in patterns:
|
| 229 |
+
match = re.search(pattern, url)
|
| 230 |
+
if match:
|
| 231 |
+
return match.group(1)
|
| 232 |
+
|
| 233 |
+
raise ValueError(f"Could not extract Vimeo video ID from: {url}")
|
| 234 |
+
|
| 235 |
+
def cleanup(self):
|
| 236 |
+
"""Clean up temporary files"""
|
| 237 |
+
import shutil
|
| 238 |
+
if os.path.exists(self.temp_dir):
|
| 239 |
+
try:
|
| 240 |
+
shutil.rmtree(self.temp_dir)
|
| 241 |
+
logger.info(f"Cleaned up temporary directory: {self.temp_dir}")
|
| 242 |
+
except Exception as e:
|
| 243 |
+
logger.warning(f"Failed to clean up temporary directory: {str(e)}")
|
| 244 |
+
|
| 245 |
+
|
| 246 |
+
class SmartVideoInput:
|
| 247 |
+
"""Smart video input handler that supports URLs and local files"""
|
| 248 |
+
|
| 249 |
+
events = {} # Gradio component compatibility
|
| 250 |
+
|
| 251 |
+
def __init__(self):
|
| 252 |
+
self.downloader = VideoDownloader()
|
| 253 |
+
self._temp_files = []
|
| 254 |
+
|
| 255 |
+
def process_input(self, input_path: str) -> Tuple[str, Dict[str, Any]]:
|
| 256 |
+
"""
|
| 257 |
+
Process video input - can be local file or URL.
|
| 258 |
+
|
| 259 |
+
Args:
|
| 260 |
+
input_path: Local file path or video URL
|
| 261 |
+
|
| 262 |
+
Returns:
|
| 263 |
+
Tuple of (local_path, metadata)
|
| 264 |
+
"""
|
| 265 |
+
# Check if it's a URL
|
| 266 |
+
if input_path.startswith(('http://', 'https://', 'www.')):
|
| 267 |
+
# Download video
|
| 268 |
+
local_path, metadata = self.downloader.download(input_path)
|
| 269 |
+
self._temp_files.append(local_path)
|
| 270 |
+
return local_path, metadata
|
| 271 |
+
else:
|
| 272 |
+
# Local file
|
| 273 |
+
if not os.path.exists(input_path):
|
| 274 |
+
raise FileNotFoundError(f"Video file not found: {input_path}")
|
| 275 |
+
|
| 276 |
+
metadata = {
|
| 277 |
+
'title': os.path.basename(input_path),
|
| 278 |
+
'platform': 'local',
|
| 279 |
+
'path': input_path
|
| 280 |
+
}
|
| 281 |
+
|
| 282 |
+
return input_path, metadata
|
| 283 |
+
|
| 284 |
+
def cleanup(self):
|
| 285 |
+
"""Clean up temporary files"""
|
| 286 |
+
for temp_file in self._temp_files:
|
| 287 |
+
try:
|
| 288 |
+
if os.path.exists(temp_file):
|
| 289 |
+
os.remove(temp_file)
|
| 290 |
+
logger.debug(f"Removed temporary file: {temp_file}")
|
| 291 |
+
except Exception as e:
|
| 292 |
+
logger.warning(f"Failed to remove temporary file: {str(e)}")
|
| 293 |
+
|
| 294 |
+
self._temp_files.clear()
|
| 295 |
+
self.downloader.cleanup()
|
backend/gradio_labanmovementanalysis/video_utils.py
ADDED
|
@@ -0,0 +1,150 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""
|
| 2 |
+
Video utilities for reading and writing video files, extracting frames, and assembling videos.
|
| 3 |
+
This module isolates video I/O logic from the rest of the pipeline.
|
| 4 |
+
"""
|
| 5 |
+
|
| 6 |
+
import cv2
|
| 7 |
+
import numpy as np
|
| 8 |
+
from typing import Generator, List, Tuple, Optional
|
| 9 |
+
from pathlib import Path
|
| 10 |
+
|
| 11 |
+
|
| 12 |
+
def extract_frames(video_path: str) -> Generator[np.ndarray, None, None]:
|
| 13 |
+
"""
|
| 14 |
+
Extract frames from a video file.
|
| 15 |
+
|
| 16 |
+
Args:
|
| 17 |
+
video_path: Path to the input video file
|
| 18 |
+
|
| 19 |
+
Yields:
|
| 20 |
+
numpy arrays representing each frame (BGR format)
|
| 21 |
+
"""
|
| 22 |
+
cap = cv2.VideoCapture(video_path)
|
| 23 |
+
if not cap.isOpened():
|
| 24 |
+
raise ValueError(f"Could not open video file: {video_path}")
|
| 25 |
+
|
| 26 |
+
try:
|
| 27 |
+
while True:
|
| 28 |
+
ret, frame = cap.read()
|
| 29 |
+
if not ret:
|
| 30 |
+
break
|
| 31 |
+
yield frame
|
| 32 |
+
finally:
|
| 33 |
+
cap.release()
|
| 34 |
+
|
| 35 |
+
|
| 36 |
+
def get_video_info(video_path: str) -> Tuple[int, float, Tuple[int, int]]:
|
| 37 |
+
"""
|
| 38 |
+
Get video metadata.
|
| 39 |
+
|
| 40 |
+
Args:
|
| 41 |
+
video_path: Path to the video file
|
| 42 |
+
|
| 43 |
+
Returns:
|
| 44 |
+
Tuple of (frame_count, fps, (width, height))
|
| 45 |
+
"""
|
| 46 |
+
cap = cv2.VideoCapture(video_path)
|
| 47 |
+
if not cap.isOpened():
|
| 48 |
+
raise ValueError(f"Could not open video file: {video_path}")
|
| 49 |
+
|
| 50 |
+
try:
|
| 51 |
+
frame_count = int(cap.get(cv2.CAP_PROP_FRAME_COUNT))
|
| 52 |
+
fps = cap.get(cv2.CAP_PROP_FPS)
|
| 53 |
+
width = int(cap.get(cv2.CAP_PROP_FRAME_WIDTH))
|
| 54 |
+
height = int(cap.get(cv2.CAP_PROP_FRAME_HEIGHT))
|
| 55 |
+
return frame_count, fps, (width, height)
|
| 56 |
+
finally:
|
| 57 |
+
cap.release()
|
| 58 |
+
|
| 59 |
+
|
| 60 |
+
def assemble_video(frames: List[np.ndarray], output_path: str, fps: float) -> str:
|
| 61 |
+
"""
|
| 62 |
+
Assemble frames into a video file.
|
| 63 |
+
|
| 64 |
+
Args:
|
| 65 |
+
frames: List of frame arrays (BGR format)
|
| 66 |
+
output_path: Path for the output video file
|
| 67 |
+
fps: Frames per second for the output video
|
| 68 |
+
|
| 69 |
+
Returns:
|
| 70 |
+
Path to the created video file
|
| 71 |
+
"""
|
| 72 |
+
if not frames:
|
| 73 |
+
raise ValueError("No frames provided for video assembly")
|
| 74 |
+
|
| 75 |
+
# Get frame dimensions from first frame
|
| 76 |
+
height, width = frames[0].shape[:2]
|
| 77 |
+
|
| 78 |
+
# Create video writer
|
| 79 |
+
fourcc = cv2.VideoWriter_fourcc(*'mp4v')
|
| 80 |
+
out = cv2.VideoWriter(output_path, fourcc, fps, (width, height))
|
| 81 |
+
|
| 82 |
+
if not out.isOpened():
|
| 83 |
+
raise ValueError(f"Could not create video writer for: {output_path}")
|
| 84 |
+
|
| 85 |
+
try:
|
| 86 |
+
for frame in frames:
|
| 87 |
+
out.write(frame)
|
| 88 |
+
return output_path
|
| 89 |
+
finally:
|
| 90 |
+
out.release()
|
| 91 |
+
|
| 92 |
+
|
| 93 |
+
def resize_frame(frame: np.ndarray, size: Optional[Tuple[int, int]] = None,
|
| 94 |
+
max_dimension: Optional[int] = None) -> np.ndarray:
|
| 95 |
+
"""
|
| 96 |
+
Resize a frame to specified dimensions.
|
| 97 |
+
|
| 98 |
+
Args:
|
| 99 |
+
frame: Input frame array
|
| 100 |
+
size: Target (width, height) if provided
|
| 101 |
+
max_dimension: Max dimension to constrain to while maintaining aspect ratio
|
| 102 |
+
|
| 103 |
+
Returns:
|
| 104 |
+
Resized frame
|
| 105 |
+
"""
|
| 106 |
+
if size is not None:
|
| 107 |
+
return cv2.resize(frame, size)
|
| 108 |
+
|
| 109 |
+
if max_dimension is not None:
|
| 110 |
+
h, w = frame.shape[:2]
|
| 111 |
+
if max(h, w) > max_dimension:
|
| 112 |
+
scale = max_dimension / max(h, w)
|
| 113 |
+
new_w = int(w * scale)
|
| 114 |
+
new_h = int(h * scale)
|
| 115 |
+
return cv2.resize(frame, (new_w, new_h))
|
| 116 |
+
|
| 117 |
+
return frame
|
| 118 |
+
|
| 119 |
+
|
| 120 |
+
def frames_to_video_buffer(frames: List[np.ndarray], fps: float) -> bytes:
|
| 121 |
+
"""
|
| 122 |
+
Convert frames to video buffer in memory (useful for Gradio).
|
| 123 |
+
|
| 124 |
+
Args:
|
| 125 |
+
frames: List of frame arrays
|
| 126 |
+
fps: Frames per second
|
| 127 |
+
|
| 128 |
+
Returns:
|
| 129 |
+
Video data as bytes
|
| 130 |
+
"""
|
| 131 |
+
import tempfile
|
| 132 |
+
import os
|
| 133 |
+
|
| 134 |
+
# Create temporary file
|
| 135 |
+
with tempfile.NamedTemporaryFile(suffix='.mp4', delete=False) as tmp:
|
| 136 |
+
tmp_path = tmp.name
|
| 137 |
+
|
| 138 |
+
try:
|
| 139 |
+
# Write video to temp file
|
| 140 |
+
assemble_video(frames, tmp_path, fps)
|
| 141 |
+
|
| 142 |
+
# Read back as bytes
|
| 143 |
+
with open(tmp_path, 'rb') as f:
|
| 144 |
+
video_data = f.read()
|
| 145 |
+
|
| 146 |
+
return video_data
|
| 147 |
+
finally:
|
| 148 |
+
# Clean up temp file
|
| 149 |
+
if os.path.exists(tmp_path):
|
| 150 |
+
os.unlink(tmp_path)
|
backend/gradio_labanmovementanalysis/visualizer.py
ADDED
|
@@ -0,0 +1,402 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""
|
| 2 |
+
Visualizer for creating annotated videos with pose overlays and movement indicators.
|
| 3 |
+
"""
|
| 4 |
+
|
| 5 |
+
import cv2
|
| 6 |
+
import numpy as np
|
| 7 |
+
from typing import List, Tuple, Optional, Dict, Any
|
| 8 |
+
from collections import deque
|
| 9 |
+
import colorsys
|
| 10 |
+
|
| 11 |
+
from .pose_estimation import PoseResult, Keypoint
|
| 12 |
+
from .notation_engine import MovementMetrics, Direction, Intensity, Speed
|
| 13 |
+
|
| 14 |
+
|
| 15 |
+
class PoseVisualizer:
|
| 16 |
+
"""Creates visual overlays for pose and movement analysis."""
|
| 17 |
+
|
| 18 |
+
# COCO skeleton connections for visualization
|
| 19 |
+
COCO_SKELETON = [
|
| 20 |
+
# Face
|
| 21 |
+
(0, 1), (0, 2), (1, 3), (2, 4), # nose to eyes, eyes to ears
|
| 22 |
+
# Upper body
|
| 23 |
+
(5, 6), # shoulders
|
| 24 |
+
(5, 7), (7, 9), # left arm
|
| 25 |
+
(6, 8), (8, 10), # right arm
|
| 26 |
+
(5, 11), (6, 12), # shoulders to hips
|
| 27 |
+
# Lower body
|
| 28 |
+
(11, 12), # hips
|
| 29 |
+
(11, 13), (13, 15), # left leg
|
| 30 |
+
(12, 14), (14, 16), # right leg
|
| 31 |
+
]
|
| 32 |
+
|
| 33 |
+
# MediaPipe skeleton connections (33 landmarks)
|
| 34 |
+
MEDIAPIPE_SKELETON = [
|
| 35 |
+
# Face connections
|
| 36 |
+
(0, 1), (1, 2), (2, 3), (3, 7), # left eye region
|
| 37 |
+
(0, 4), (4, 5), (5, 6), (6, 8), # right eye region
|
| 38 |
+
(9, 10), # mouth
|
| 39 |
+
# Upper body
|
| 40 |
+
(11, 12), # shoulders
|
| 41 |
+
(11, 13), (13, 15), # left arm
|
| 42 |
+
(12, 14), (14, 16), # right arm
|
| 43 |
+
(11, 23), (12, 24), # shoulders to hips
|
| 44 |
+
(23, 24), # hips
|
| 45 |
+
# Lower body
|
| 46 |
+
(23, 25), (25, 27), (27, 29), (27, 31), # left leg
|
| 47 |
+
(24, 26), (26, 28), (28, 30), (28, 32), # right leg
|
| 48 |
+
# Hands
|
| 49 |
+
(15, 17), (15, 19), (15, 21), # left hand
|
| 50 |
+
(16, 18), (16, 20), (16, 22), # right hand
|
| 51 |
+
]
|
| 52 |
+
|
| 53 |
+
def __init__(self,
|
| 54 |
+
trail_length: int = 10,
|
| 55 |
+
show_skeleton: bool = True,
|
| 56 |
+
show_trails: bool = True,
|
| 57 |
+
show_direction_arrows: bool = True,
|
| 58 |
+
show_metrics: bool = True):
|
| 59 |
+
"""
|
| 60 |
+
Initialize visualizer.
|
| 61 |
+
|
| 62 |
+
Args:
|
| 63 |
+
trail_length: Number of previous frames to show in motion trail
|
| 64 |
+
show_skeleton: Whether to draw pose skeleton
|
| 65 |
+
show_trails: Whether to draw motion trails
|
| 66 |
+
show_direction_arrows: Whether to show movement direction arrows
|
| 67 |
+
show_metrics: Whether to display text metrics on frame
|
| 68 |
+
"""
|
| 69 |
+
self.trail_length = trail_length
|
| 70 |
+
self.show_skeleton = show_skeleton
|
| 71 |
+
self.show_trails = show_trails
|
| 72 |
+
self.show_direction_arrows = show_direction_arrows
|
| 73 |
+
self.show_metrics = show_metrics
|
| 74 |
+
|
| 75 |
+
# Trail history for each keypoint
|
| 76 |
+
self.trails = {}
|
| 77 |
+
|
| 78 |
+
# Color mapping for intensity
|
| 79 |
+
self.intensity_colors = {
|
| 80 |
+
Intensity.LOW: (0, 255, 0), # Green
|
| 81 |
+
Intensity.MEDIUM: (0, 165, 255), # Orange
|
| 82 |
+
Intensity.HIGH: (0, 0, 255) # Red
|
| 83 |
+
}
|
| 84 |
+
|
| 85 |
+
def visualize_frame(self,
|
| 86 |
+
frame: np.ndarray,
|
| 87 |
+
pose_results: List[PoseResult],
|
| 88 |
+
movement_metrics: Optional[MovementMetrics] = None,
|
| 89 |
+
frame_index: int = 0) -> np.ndarray:
|
| 90 |
+
"""
|
| 91 |
+
Add visual annotations to a single frame.
|
| 92 |
+
|
| 93 |
+
Args:
|
| 94 |
+
frame: Input frame
|
| 95 |
+
pose_results: Pose detection results for this frame
|
| 96 |
+
movement_metrics: Movement analysis metrics for this frame
|
| 97 |
+
frame_index: Current frame index
|
| 98 |
+
|
| 99 |
+
Returns:
|
| 100 |
+
Annotated frame
|
| 101 |
+
"""
|
| 102 |
+
# Create a copy to avoid modifying original
|
| 103 |
+
vis_frame = frame.copy()
|
| 104 |
+
|
| 105 |
+
# Draw for each detected person
|
| 106 |
+
for person_idx, pose in enumerate(pose_results):
|
| 107 |
+
# Update trails
|
| 108 |
+
if self.show_trails:
|
| 109 |
+
self._update_trails(pose, person_idx)
|
| 110 |
+
self._draw_trails(vis_frame, person_idx)
|
| 111 |
+
|
| 112 |
+
# Draw skeleton
|
| 113 |
+
if self.show_skeleton:
|
| 114 |
+
color = self._get_color_for_metrics(movement_metrics)
|
| 115 |
+
self._draw_skeleton(vis_frame, pose, color)
|
| 116 |
+
|
| 117 |
+
# Draw keypoints
|
| 118 |
+
self._draw_keypoints(vis_frame, pose, movement_metrics)
|
| 119 |
+
|
| 120 |
+
# Draw direction arrow
|
| 121 |
+
if self.show_direction_arrows and movement_metrics:
|
| 122 |
+
self._draw_direction_arrow(vis_frame, pose, movement_metrics)
|
| 123 |
+
|
| 124 |
+
# Draw metrics overlay
|
| 125 |
+
if self.show_metrics and movement_metrics:
|
| 126 |
+
self._draw_metrics_overlay(vis_frame, movement_metrics)
|
| 127 |
+
|
| 128 |
+
return vis_frame
|
| 129 |
+
|
| 130 |
+
def generate_overlay_video(self,
|
| 131 |
+
frames: List[np.ndarray],
|
| 132 |
+
all_pose_results: List[List[PoseResult]],
|
| 133 |
+
all_movement_metrics: List[MovementMetrics],
|
| 134 |
+
output_path: str,
|
| 135 |
+
fps: float) -> str:
|
| 136 |
+
"""
|
| 137 |
+
Generate complete video with overlays.
|
| 138 |
+
|
| 139 |
+
Args:
|
| 140 |
+
frames: List of video frames
|
| 141 |
+
all_pose_results: Pose results for each frame
|
| 142 |
+
all_movement_metrics: Movement metrics for each frame
|
| 143 |
+
output_path: Path for output video
|
| 144 |
+
fps: Frames per second
|
| 145 |
+
|
| 146 |
+
Returns:
|
| 147 |
+
Path to created video
|
| 148 |
+
"""
|
| 149 |
+
if len(frames) != len(all_pose_results) or len(frames) != len(all_movement_metrics):
|
| 150 |
+
raise ValueError("Mismatched lengths between frames, poses, and metrics")
|
| 151 |
+
|
| 152 |
+
# Reset trails
|
| 153 |
+
self.trails = {}
|
| 154 |
+
|
| 155 |
+
# Process each frame
|
| 156 |
+
annotated_frames = []
|
| 157 |
+
for i, (frame, poses, metrics) in enumerate(
|
| 158 |
+
zip(frames, all_pose_results, all_movement_metrics)
|
| 159 |
+
):
|
| 160 |
+
annotated_frame = self.visualize_frame(frame, poses, metrics, i)
|
| 161 |
+
annotated_frames.append(annotated_frame)
|
| 162 |
+
|
| 163 |
+
# Import video_utils locally to avoid circular import
|
| 164 |
+
from . import video_utils
|
| 165 |
+
return video_utils.assemble_video(annotated_frames, output_path, fps)
|
| 166 |
+
|
| 167 |
+
def _update_trails(self, pose: PoseResult, person_id: int):
|
| 168 |
+
"""Update motion trails for a person."""
|
| 169 |
+
if person_id not in self.trails:
|
| 170 |
+
self.trails[person_id] = {}
|
| 171 |
+
|
| 172 |
+
for kp in pose.keypoints:
|
| 173 |
+
if kp.confidence < 0.3:
|
| 174 |
+
continue
|
| 175 |
+
|
| 176 |
+
if kp.name not in self.trails[person_id]:
|
| 177 |
+
self.trails[person_id][kp.name] = deque(maxlen=self.trail_length)
|
| 178 |
+
|
| 179 |
+
# Convert normalized coordinates to pixel coordinates
|
| 180 |
+
# This assumes we'll scale them when drawing
|
| 181 |
+
self.trails[person_id][kp.name].append((kp.x, kp.y))
|
| 182 |
+
|
| 183 |
+
def _draw_trails(self, frame: np.ndarray, person_id: int):
|
| 184 |
+
"""Draw motion trails for a person."""
|
| 185 |
+
if person_id not in self.trails:
|
| 186 |
+
return
|
| 187 |
+
|
| 188 |
+
h, w = frame.shape[:2]
|
| 189 |
+
|
| 190 |
+
for joint_name, trail in self.trails[person_id].items():
|
| 191 |
+
if len(trail) < 2:
|
| 192 |
+
continue
|
| 193 |
+
|
| 194 |
+
# Draw trail with fading effect
|
| 195 |
+
for i in range(1, len(trail)):
|
| 196 |
+
# Calculate opacity based on position in trail
|
| 197 |
+
alpha = i / len(trail)
|
| 198 |
+
color = tuple(int(c * alpha) for c in (255, 255, 255))
|
| 199 |
+
|
| 200 |
+
# Convert normalized to pixel coordinates
|
| 201 |
+
pt1 = (int(trail[i-1][0] * w), int(trail[i-1][1] * h))
|
| 202 |
+
pt2 = (int(trail[i][0] * w), int(trail[i][1] * h))
|
| 203 |
+
|
| 204 |
+
# Draw trail segment
|
| 205 |
+
cv2.line(frame, pt1, pt2, color, thickness=max(1, int(3 * alpha)))
|
| 206 |
+
|
| 207 |
+
def _draw_skeleton(self, frame: np.ndarray, pose: PoseResult, color: Tuple[int, int, int]):
|
| 208 |
+
"""Draw pose skeleton."""
|
| 209 |
+
h, w = frame.shape[:2]
|
| 210 |
+
|
| 211 |
+
# Create keypoint lookup
|
| 212 |
+
kp_dict = {kp.name: kp for kp in pose.keypoints if kp.confidence > 0.3}
|
| 213 |
+
|
| 214 |
+
# Determine which skeleton to use based on available keypoints
|
| 215 |
+
skeleton = self._get_skeleton_for_model(pose.keypoints)
|
| 216 |
+
|
| 217 |
+
# Map keypoint names to indices
|
| 218 |
+
keypoint_names = self._get_keypoint_names_for_model(pose.keypoints)
|
| 219 |
+
name_to_idx = {name: i for i, name in enumerate(keypoint_names)}
|
| 220 |
+
|
| 221 |
+
# Draw skeleton connections
|
| 222 |
+
for connection in skeleton:
|
| 223 |
+
idx1, idx2 = connection
|
| 224 |
+
if idx1 < len(keypoint_names) and idx2 < len(keypoint_names):
|
| 225 |
+
name1 = keypoint_names[idx1]
|
| 226 |
+
name2 = keypoint_names[idx2]
|
| 227 |
+
|
| 228 |
+
if name1 in kp_dict and name2 in kp_dict:
|
| 229 |
+
kp1 = kp_dict[name1]
|
| 230 |
+
kp2 = kp_dict[name2]
|
| 231 |
+
|
| 232 |
+
# Convert to pixel coordinates
|
| 233 |
+
pt1 = (int(kp1.x * w), int(kp1.y * h))
|
| 234 |
+
pt2 = (int(kp2.x * w), int(kp2.y * h))
|
| 235 |
+
|
| 236 |
+
# Draw line
|
| 237 |
+
cv2.line(frame, pt1, pt2, color, thickness=2)
|
| 238 |
+
|
| 239 |
+
def _draw_keypoints(self, frame: np.ndarray, pose: PoseResult,
|
| 240 |
+
metrics: Optional[MovementMetrics] = None):
|
| 241 |
+
"""Draw individual keypoints."""
|
| 242 |
+
h, w = frame.shape[:2]
|
| 243 |
+
|
| 244 |
+
for kp in pose.keypoints:
|
| 245 |
+
if kp.confidence < 0.3:
|
| 246 |
+
continue
|
| 247 |
+
|
| 248 |
+
# Convert to pixel coordinates
|
| 249 |
+
pt = (int(kp.x * w), int(kp.y * h))
|
| 250 |
+
|
| 251 |
+
# Color based on confidence
|
| 252 |
+
color = self._confidence_to_color(kp.confidence)
|
| 253 |
+
|
| 254 |
+
# Draw keypoint
|
| 255 |
+
cv2.circle(frame, pt, 4, color, -1)
|
| 256 |
+
cv2.circle(frame, pt, 5, (255, 255, 255), 1) # White border
|
| 257 |
+
|
| 258 |
+
def _draw_direction_arrow(self, frame: np.ndarray, pose: PoseResult,
|
| 259 |
+
metrics: MovementMetrics):
|
| 260 |
+
"""Draw arrow indicating movement direction."""
|
| 261 |
+
if metrics.direction == Direction.STATIONARY:
|
| 262 |
+
return
|
| 263 |
+
|
| 264 |
+
h, w = frame.shape[:2]
|
| 265 |
+
|
| 266 |
+
# Get body center
|
| 267 |
+
center_x = np.mean([kp.x for kp in pose.keypoints if kp.confidence > 0.3])
|
| 268 |
+
center_y = np.mean([kp.y for kp in pose.keypoints if kp.confidence > 0.3])
|
| 269 |
+
|
| 270 |
+
# Convert to pixel coordinates
|
| 271 |
+
center = (int(center_x * w), int(center_y * h))
|
| 272 |
+
|
| 273 |
+
# Calculate arrow endpoint based on direction
|
| 274 |
+
arrow_length = 50
|
| 275 |
+
direction_vectors = {
|
| 276 |
+
Direction.UP: (0, -1),
|
| 277 |
+
Direction.DOWN: (0, 1),
|
| 278 |
+
Direction.LEFT: (-1, 0),
|
| 279 |
+
Direction.RIGHT: (1, 0),
|
| 280 |
+
}
|
| 281 |
+
|
| 282 |
+
if metrics.direction in direction_vectors:
|
| 283 |
+
dx, dy = direction_vectors[metrics.direction]
|
| 284 |
+
end_point = (
|
| 285 |
+
center[0] + int(dx * arrow_length),
|
| 286 |
+
center[1] + int(dy * arrow_length)
|
| 287 |
+
)
|
| 288 |
+
|
| 289 |
+
# Color based on speed
|
| 290 |
+
color = self._get_color_for_metrics(metrics)
|
| 291 |
+
|
| 292 |
+
# Draw arrow
|
| 293 |
+
cv2.arrowedLine(frame, center, end_point, color, thickness=3, tipLength=0.3)
|
| 294 |
+
|
| 295 |
+
def _draw_metrics_overlay(self, frame: np.ndarray, metrics: MovementMetrics):
|
| 296 |
+
"""Draw text overlay with movement metrics."""
|
| 297 |
+
# Define text properties
|
| 298 |
+
font = cv2.FONT_HERSHEY_SIMPLEX
|
| 299 |
+
font_scale = 0.6
|
| 300 |
+
thickness = 2
|
| 301 |
+
|
| 302 |
+
# Create text lines
|
| 303 |
+
lines = [
|
| 304 |
+
f"Direction: {metrics.direction.value}",
|
| 305 |
+
f"Speed: {metrics.speed.value} ({metrics.velocity:.2f})",
|
| 306 |
+
f"Intensity: {metrics.intensity.value}",
|
| 307 |
+
f"Fluidity: {metrics.fluidity:.2f}",
|
| 308 |
+
f"Expansion: {metrics.expansion:.2f}"
|
| 309 |
+
]
|
| 310 |
+
|
| 311 |
+
# Draw background rectangle
|
| 312 |
+
y_offset = 30
|
| 313 |
+
max_width = max([cv2.getTextSize(line, font, font_scale, thickness)[0][0]
|
| 314 |
+
for line in lines])
|
| 315 |
+
bg_height = len(lines) * 25 + 10
|
| 316 |
+
|
| 317 |
+
cv2.rectangle(frame, (10, 10), (20 + max_width, 10 + bg_height),
|
| 318 |
+
(0, 0, 0), -1)
|
| 319 |
+
cv2.rectangle(frame, (10, 10), (20 + max_width, 10 + bg_height),
|
| 320 |
+
(255, 255, 255), 1)
|
| 321 |
+
|
| 322 |
+
# Draw text
|
| 323 |
+
for i, line in enumerate(lines):
|
| 324 |
+
color = (255, 255, 255)
|
| 325 |
+
if i == 2: # Intensity line
|
| 326 |
+
color = self.intensity_colors.get(metrics.intensity, (255, 255, 255))
|
| 327 |
+
|
| 328 |
+
cv2.putText(frame, line, (15, y_offset + i * 25),
|
| 329 |
+
font, font_scale, color, thickness)
|
| 330 |
+
|
| 331 |
+
def _get_color_for_metrics(self, metrics: Optional[MovementMetrics]) -> Tuple[int, int, int]:
|
| 332 |
+
"""Get color based on movement metrics."""
|
| 333 |
+
if metrics is None:
|
| 334 |
+
return (255, 255, 255) # White default
|
| 335 |
+
|
| 336 |
+
return self.intensity_colors.get(metrics.intensity, (255, 255, 255))
|
| 337 |
+
|
| 338 |
+
def _confidence_to_color(self, confidence: float) -> Tuple[int, int, int]:
|
| 339 |
+
"""Convert confidence score to color (green=high, red=low)."""
|
| 340 |
+
# Use HSV color space for smooth gradient
|
| 341 |
+
hue = confidence * 120 # 0=red, 120=green
|
| 342 |
+
rgb = colorsys.hsv_to_rgb(hue / 360, 1.0, 1.0)
|
| 343 |
+
return tuple(int(c * 255) for c in reversed(rgb)) # BGR for OpenCV
|
| 344 |
+
|
| 345 |
+
def _get_skeleton_for_model(self, keypoints: List[Keypoint]) -> List[Tuple[int, int]]:
|
| 346 |
+
"""Determine which skeleton definition to use based on keypoints."""
|
| 347 |
+
# Simple heuristic: if we have more than 20 keypoints, use MediaPipe skeleton
|
| 348 |
+
if len(keypoints) > 20:
|
| 349 |
+
return self.MEDIAPIPE_SKELETON
|
| 350 |
+
return self.COCO_SKELETON
|
| 351 |
+
|
| 352 |
+
def _get_keypoint_names_for_model(self, keypoints: List[Keypoint]) -> List[str]:
|
| 353 |
+
"""Get ordered list of keypoint names for the model."""
|
| 354 |
+
# If keypoints have names, use them
|
| 355 |
+
if keypoints and keypoints[0].name:
|
| 356 |
+
return [kp.name for kp in keypoints]
|
| 357 |
+
|
| 358 |
+
# Otherwise, use default COCO names
|
| 359 |
+
from .pose_estimation import MoveNetPoseEstimator
|
| 360 |
+
return MoveNetPoseEstimator.KEYPOINT_NAMES
|
| 361 |
+
|
| 362 |
+
|
| 363 |
+
def create_visualization(
|
| 364 |
+
video_path: str,
|
| 365 |
+
pose_results: List[List[PoseResult]],
|
| 366 |
+
movement_metrics: List[MovementMetrics],
|
| 367 |
+
output_path: str,
|
| 368 |
+
show_trails: bool = True,
|
| 369 |
+
show_metrics: bool = True
|
| 370 |
+
) -> str:
|
| 371 |
+
"""
|
| 372 |
+
Convenience function to create a visualization from a video file.
|
| 373 |
+
|
| 374 |
+
Args:
|
| 375 |
+
video_path: Path to input video
|
| 376 |
+
pose_results: Pose detection results
|
| 377 |
+
movement_metrics: Movement analysis results
|
| 378 |
+
output_path: Path for output video
|
| 379 |
+
show_trails: Whether to show motion trails
|
| 380 |
+
show_metrics: Whether to show metrics overlay
|
| 381 |
+
|
| 382 |
+
Returns:
|
| 383 |
+
Path to created video
|
| 384 |
+
"""
|
| 385 |
+
from . import video_utils
|
| 386 |
+
|
| 387 |
+
# Extract frames
|
| 388 |
+
frames = list(video_utils.extract_frames(video_path))
|
| 389 |
+
|
| 390 |
+
# Get video info
|
| 391 |
+
_, fps, _ = video_utils.get_video_info(video_path)
|
| 392 |
+
|
| 393 |
+
# Create visualizer
|
| 394 |
+
visualizer = PoseVisualizer(
|
| 395 |
+
show_trails=show_trails,
|
| 396 |
+
show_metrics=show_metrics
|
| 397 |
+
)
|
| 398 |
+
|
| 399 |
+
# Generate overlay video
|
| 400 |
+
return visualizer.generate_overlay_video(
|
| 401 |
+
frames, pose_results, movement_metrics, output_path, fps
|
| 402 |
+
)
|
backend/gradio_labanmovementanalysis/webrtc_handler.py
ADDED
|
@@ -0,0 +1,293 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""
|
| 2 |
+
Professional WebRTC handler for real-time video streaming and movement analysis
|
| 3 |
+
Using FastRTC (the current WebRTC standard, replaces deprecated gradio-webrtc)
|
| 4 |
+
Based on: https://fastrtc.org and https://www.gradio.app/guides/object-detection-from-webcam-with-webrtc
|
| 5 |
+
"""
|
| 6 |
+
|
| 7 |
+
import cv2
|
| 8 |
+
import numpy as np
|
| 9 |
+
from typing import Optional, Dict, Any, Tuple
|
| 10 |
+
from collections import deque
|
| 11 |
+
import time
|
| 12 |
+
import logging
|
| 13 |
+
import os
|
| 14 |
+
|
| 15 |
+
from .pose_estimation import get_pose_estimator
|
| 16 |
+
from .notation_engine import MovementAnalyzer
|
| 17 |
+
from .visualizer import PoseVisualizer
|
| 18 |
+
|
| 19 |
+
logger = logging.getLogger(__name__)
|
| 20 |
+
|
| 21 |
+
# Official Gradio WebRTC approach (compatible with NumPy 1.x)
|
| 22 |
+
try:
|
| 23 |
+
from gradio_webrtc import WebRTC
|
| 24 |
+
HAS_WEBRTC_COMPONENT = True
|
| 25 |
+
except ImportError:
|
| 26 |
+
HAS_WEBRTC_COMPONENT = False
|
| 27 |
+
|
| 28 |
+
|
| 29 |
+
class RealtimeMovementAnalyzer:
|
| 30 |
+
"""Real-time movement analyzer for WebRTC streams following Gradio 5 best practices"""
|
| 31 |
+
|
| 32 |
+
# Gradio component compatibility
|
| 33 |
+
events = {}
|
| 34 |
+
|
| 35 |
+
def __init__(self, model: str = "mediapipe-lite", buffer_size: int = 30):
|
| 36 |
+
"""
|
| 37 |
+
Initialize real-time movement analyzer.
|
| 38 |
+
|
| 39 |
+
Args:
|
| 40 |
+
model: Pose estimation model optimized for real-time processing
|
| 41 |
+
buffer_size: Number of frames to buffer for analysis
|
| 42 |
+
"""
|
| 43 |
+
self.model = model
|
| 44 |
+
self.pose_estimator = get_pose_estimator(model)
|
| 45 |
+
self.movement_analyzer = MovementAnalyzer(fps=30.0)
|
| 46 |
+
self.visualizer = PoseVisualizer(
|
| 47 |
+
trail_length=10,
|
| 48 |
+
show_skeleton=True,
|
| 49 |
+
show_trails=True,
|
| 50 |
+
show_direction_arrows=True,
|
| 51 |
+
show_metrics=True
|
| 52 |
+
)
|
| 53 |
+
|
| 54 |
+
# Real-time buffers
|
| 55 |
+
self.pose_buffer = deque(maxlen=buffer_size)
|
| 56 |
+
self.metrics_buffer = deque(maxlen=buffer_size)
|
| 57 |
+
|
| 58 |
+
# Performance tracking
|
| 59 |
+
self.frame_count = 0
|
| 60 |
+
self.last_fps_update = time.time()
|
| 61 |
+
self.current_fps = 0.0
|
| 62 |
+
|
| 63 |
+
# Current metrics for display
|
| 64 |
+
self.current_metrics = {
|
| 65 |
+
"direction": "stationary",
|
| 66 |
+
"intensity": "low",
|
| 67 |
+
"fluidity": 0.0,
|
| 68 |
+
"expansion": 0.5,
|
| 69 |
+
"fps": 0.0
|
| 70 |
+
}
|
| 71 |
+
|
| 72 |
+
def process_frame(self, image: np.ndarray, conf_threshold: float = 0.5) -> np.ndarray:
|
| 73 |
+
"""
|
| 74 |
+
Process a single frame from WebRTC stream for real-time movement analysis.
|
| 75 |
+
|
| 76 |
+
Args:
|
| 77 |
+
image: Input frame from webcam as numpy array (RGB format from WebRTC)
|
| 78 |
+
conf_threshold: Confidence threshold for pose detection
|
| 79 |
+
|
| 80 |
+
Returns:
|
| 81 |
+
Processed frame with pose overlay and movement metrics
|
| 82 |
+
"""
|
| 83 |
+
if image is None:
|
| 84 |
+
return None
|
| 85 |
+
|
| 86 |
+
# Convert RGB to BGR for OpenCV processing
|
| 87 |
+
frame_bgr = cv2.cvtColor(image, cv2.COLOR_RGB2BGR)
|
| 88 |
+
|
| 89 |
+
# Update frame count and FPS
|
| 90 |
+
self.frame_count += 1
|
| 91 |
+
current_time = time.time()
|
| 92 |
+
if current_time - self.last_fps_update >= 1.0:
|
| 93 |
+
self.current_fps = self.frame_count / (current_time - self.last_fps_update)
|
| 94 |
+
self.frame_count = 0
|
| 95 |
+
self.last_fps_update = current_time
|
| 96 |
+
self.current_metrics["fps"] = self.current_fps
|
| 97 |
+
|
| 98 |
+
# Pose detection
|
| 99 |
+
pose_results = self.pose_estimator.detect(frame_bgr)
|
| 100 |
+
|
| 101 |
+
# Store pose data
|
| 102 |
+
self.pose_buffer.append(pose_results)
|
| 103 |
+
|
| 104 |
+
# Calculate movement metrics if we have enough frames
|
| 105 |
+
if len(self.pose_buffer) >= 2:
|
| 106 |
+
recent_poses = list(self.pose_buffer)[-10:] # Last 10 frames for analysis
|
| 107 |
+
|
| 108 |
+
try:
|
| 109 |
+
# Analyze movement from recent poses
|
| 110 |
+
movement_metrics = self.movement_analyzer.analyze_movement(recent_poses)
|
| 111 |
+
|
| 112 |
+
if movement_metrics:
|
| 113 |
+
latest_metrics = movement_metrics[-1]
|
| 114 |
+
self.current_metrics.update({
|
| 115 |
+
"direction": latest_metrics.direction.value if latest_metrics.direction else "stationary",
|
| 116 |
+
"intensity": latest_metrics.intensity.value if latest_metrics.intensity else "low",
|
| 117 |
+
"fluidity": latest_metrics.fluidity if latest_metrics.fluidity is not None else 0.0,
|
| 118 |
+
"expansion": latest_metrics.expansion if latest_metrics.expansion is not None else 0.5
|
| 119 |
+
})
|
| 120 |
+
|
| 121 |
+
self.metrics_buffer.append(self.current_metrics.copy())
|
| 122 |
+
|
| 123 |
+
except Exception as e:
|
| 124 |
+
logger.warning(f"Movement analysis error: {e}")
|
| 125 |
+
|
| 126 |
+
# Apply visualization overlays
|
| 127 |
+
output_frame = self._apply_visualization(frame_bgr, pose_results, self.current_metrics)
|
| 128 |
+
|
| 129 |
+
# Convert back to RGB for WebRTC output
|
| 130 |
+
output_rgb = cv2.cvtColor(output_frame, cv2.COLOR_BGR2RGB)
|
| 131 |
+
|
| 132 |
+
return output_rgb
|
| 133 |
+
|
| 134 |
+
def _apply_visualization(self, frame: np.ndarray, pose_results: list, metrics: dict) -> np.ndarray:
|
| 135 |
+
"""Apply pose and movement visualization overlays"""
|
| 136 |
+
output_frame = frame.copy()
|
| 137 |
+
|
| 138 |
+
# Draw pose skeleton if detected
|
| 139 |
+
if pose_results:
|
| 140 |
+
for pose_result in pose_results:
|
| 141 |
+
# Draw skeleton
|
| 142 |
+
if hasattr(self.visualizer, 'draw_skeleton'):
|
| 143 |
+
output_frame = self.visualizer.draw_skeleton(output_frame, pose_result.keypoints)
|
| 144 |
+
|
| 145 |
+
# Draw keypoints
|
| 146 |
+
for keypoint in pose_result.keypoints:
|
| 147 |
+
if keypoint.confidence > 0.5:
|
| 148 |
+
x = int(keypoint.x * frame.shape[1])
|
| 149 |
+
y = int(keypoint.y * frame.shape[0])
|
| 150 |
+
cv2.circle(output_frame, (x, y), 5, (0, 255, 0), -1)
|
| 151 |
+
|
| 152 |
+
# Draw real-time metrics overlay
|
| 153 |
+
self._draw_metrics_overlay(output_frame, metrics)
|
| 154 |
+
|
| 155 |
+
return output_frame
|
| 156 |
+
|
| 157 |
+
def _draw_metrics_overlay(self, frame: np.ndarray, metrics: dict):
|
| 158 |
+
"""Draw real-time metrics overlay following professional UI standards"""
|
| 159 |
+
h, w = frame.shape[:2]
|
| 160 |
+
|
| 161 |
+
# Semi-transparent background
|
| 162 |
+
overlay = frame.copy()
|
| 163 |
+
cv2.rectangle(overlay, (10, 10), (320, 160), (0, 0, 0), -1)
|
| 164 |
+
cv2.addWeighted(overlay, 0.3, frame, 0.7, 0, frame)
|
| 165 |
+
|
| 166 |
+
# Header
|
| 167 |
+
cv2.putText(frame, "Real-time Movement Analysis", (20, 35),
|
| 168 |
+
cv2.FONT_HERSHEY_SIMPLEX, 0.6, (255, 255, 255), 2)
|
| 169 |
+
|
| 170 |
+
# Metrics
|
| 171 |
+
y_offset = 60
|
| 172 |
+
spacing = 22
|
| 173 |
+
|
| 174 |
+
cv2.putText(frame, f"Direction: {metrics['direction']}",
|
| 175 |
+
(20, y_offset), cv2.FONT_HERSHEY_SIMPLEX, 0.5, (0, 255, 0), 1)
|
| 176 |
+
y_offset += spacing
|
| 177 |
+
|
| 178 |
+
cv2.putText(frame, f"Intensity: {metrics['intensity']}",
|
| 179 |
+
(20, y_offset), cv2.FONT_HERSHEY_SIMPLEX, 0.5, (0, 255, 0), 1)
|
| 180 |
+
y_offset += spacing
|
| 181 |
+
|
| 182 |
+
cv2.putText(frame, f"Fluidity: {metrics['fluidity']:.2f}",
|
| 183 |
+
(20, y_offset), cv2.FONT_HERSHEY_SIMPLEX, 0.5, (0, 255, 0), 1)
|
| 184 |
+
y_offset += spacing
|
| 185 |
+
|
| 186 |
+
cv2.putText(frame, f"FPS: {metrics['fps']:.1f}",
|
| 187 |
+
(20, y_offset), cv2.FONT_HERSHEY_SIMPLEX, 0.5, (255, 255, 0), 1)
|
| 188 |
+
|
| 189 |
+
def get_current_metrics(self) -> dict:
|
| 190 |
+
"""Get current movement metrics for external display"""
|
| 191 |
+
return self.current_metrics.copy()
|
| 192 |
+
|
| 193 |
+
|
| 194 |
+
def get_rtc_configuration():
|
| 195 |
+
"""
|
| 196 |
+
Get RTC configuration for WebRTC.
|
| 197 |
+
Uses Twilio TURN servers if credentials are available, otherwise uses default.
|
| 198 |
+
"""
|
| 199 |
+
# For local development, no TURN servers needed
|
| 200 |
+
# For cloud deployment, set TWILIO_ACCOUNT_SID and TWILIO_AUTH_TOKEN
|
| 201 |
+
|
| 202 |
+
twilio_account_sid = os.getenv("TWILIO_ACCOUNT_SID")
|
| 203 |
+
twilio_auth_token = os.getenv("TWILIO_AUTH_TOKEN")
|
| 204 |
+
|
| 205 |
+
if twilio_account_sid and twilio_auth_token:
|
| 206 |
+
# Use Twilio TURN servers for cloud deployment
|
| 207 |
+
return {
|
| 208 |
+
"iceServers": [
|
| 209 |
+
{"urls": ["stun:global.stun.twilio.com:3478"]},
|
| 210 |
+
{
|
| 211 |
+
"urls": ["turn:global.turn.twilio.com:3478?transport=udp"],
|
| 212 |
+
"username": twilio_account_sid,
|
| 213 |
+
"credential": twilio_auth_token,
|
| 214 |
+
},
|
| 215 |
+
{
|
| 216 |
+
"urls": ["turn:global.turn.twilio.com:3478?transport=tcp"],
|
| 217 |
+
"username": twilio_account_sid,
|
| 218 |
+
"credential": twilio_auth_token,
|
| 219 |
+
},
|
| 220 |
+
]
|
| 221 |
+
}
|
| 222 |
+
else:
|
| 223 |
+
# Default configuration for local development
|
| 224 |
+
return {
|
| 225 |
+
"iceServers": [
|
| 226 |
+
{"urls": ["stun:stun.l.google.com:19302"]}
|
| 227 |
+
]
|
| 228 |
+
}
|
| 229 |
+
|
| 230 |
+
|
| 231 |
+
# Global analyzer instance for demo
|
| 232 |
+
_analyzer = None
|
| 233 |
+
|
| 234 |
+
def get_analyzer(model: str = "mediapipe-lite") -> RealtimeMovementAnalyzer:
|
| 235 |
+
"""Get or create analyzer instance"""
|
| 236 |
+
global _analyzer
|
| 237 |
+
if _analyzer is None or _analyzer.model != model:
|
| 238 |
+
_analyzer = RealtimeMovementAnalyzer(model)
|
| 239 |
+
return _analyzer
|
| 240 |
+
|
| 241 |
+
|
| 242 |
+
def webrtc_detection(image: np.ndarray, model: str, conf_threshold: float = 0.5) -> np.ndarray:
|
| 243 |
+
"""
|
| 244 |
+
Main detection function for WebRTC streaming.
|
| 245 |
+
Compatible with Gradio 5 WebRTC streaming API.
|
| 246 |
+
|
| 247 |
+
Args:
|
| 248 |
+
image: Input frame from webcam (RGB format)
|
| 249 |
+
model: Pose estimation model name
|
| 250 |
+
conf_threshold: Confidence threshold for pose detection
|
| 251 |
+
|
| 252 |
+
Returns:
|
| 253 |
+
Processed frame with pose overlay and metrics
|
| 254 |
+
"""
|
| 255 |
+
analyzer = get_analyzer(model)
|
| 256 |
+
return analyzer.process_frame(image, conf_threshold)
|
| 257 |
+
|
| 258 |
+
|
| 259 |
+
def get_webrtc_interface():
|
| 260 |
+
"""
|
| 261 |
+
Create streaming interface using built-in Gradio components.
|
| 262 |
+
Avoids NumPy 2.x dependency conflicts with FastRTC.
|
| 263 |
+
|
| 264 |
+
Returns:
|
| 265 |
+
Tuple of (streaming_config, rtc_configuration)
|
| 266 |
+
"""
|
| 267 |
+
rtc_config = get_rtc_configuration()
|
| 268 |
+
|
| 269 |
+
# Use built-in Gradio streaming capabilities
|
| 270 |
+
streaming_config = {
|
| 271 |
+
"sources": ["webcam"],
|
| 272 |
+
"streaming": True,
|
| 273 |
+
"mirror_webcam": False
|
| 274 |
+
}
|
| 275 |
+
|
| 276 |
+
return streaming_config, rtc_config
|
| 277 |
+
|
| 278 |
+
|
| 279 |
+
# Compatibility exports with Gradio component attributes
|
| 280 |
+
class WebRTCMovementAnalyzer(RealtimeMovementAnalyzer):
|
| 281 |
+
"""Real-time movement analyzer for WebRTC streams following Gradio 5 best practices"""
|
| 282 |
+
events = {} # Gradio component compatibility
|
| 283 |
+
|
| 284 |
+
|
| 285 |
+
class WebRTCGradioInterface:
|
| 286 |
+
"""Create streaming interface using built-in Gradio components.
|
| 287 |
+
Avoids NumPy 2.x dependency conflicts with FastRTC."""
|
| 288 |
+
|
| 289 |
+
events = {} # Gradio component compatibility
|
| 290 |
+
|
| 291 |
+
@staticmethod
|
| 292 |
+
def get_config():
|
| 293 |
+
return get_webrtc_interface()
|
backend/mcp_server.py
ADDED
|
@@ -0,0 +1,413 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""
|
| 2 |
+
MCP (Model Context Protocol) Server for Laban Movement Analysis
|
| 3 |
+
Provides tools for video movement analysis accessible to AI agents
|
| 4 |
+
"""
|
| 5 |
+
|
| 6 |
+
import asyncio
|
| 7 |
+
import json
|
| 8 |
+
import os
|
| 9 |
+
import tempfile
|
| 10 |
+
from datetime import datetime
|
| 11 |
+
from pathlib import Path
|
| 12 |
+
from typing import Any, Dict, List, Optional, Tuple
|
| 13 |
+
from urllib.parse import urlparse
|
| 14 |
+
import aiofiles
|
| 15 |
+
import httpx
|
| 16 |
+
|
| 17 |
+
from mcp.server import Server
|
| 18 |
+
from mcp.server.stdio import stdio_server
|
| 19 |
+
from mcp.types import (
|
| 20 |
+
Tool,
|
| 21 |
+
TextContent,
|
| 22 |
+
ImageContent,
|
| 23 |
+
EmbeddedResource,
|
| 24 |
+
ToolParameterType,
|
| 25 |
+
ToolResponse,
|
| 26 |
+
ToolResult,
|
| 27 |
+
ToolError
|
| 28 |
+
)
|
| 29 |
+
|
| 30 |
+
# Add parent directory to path for imports
|
| 31 |
+
import sys
|
| 32 |
+
sys.path.insert(0, str(Path(__file__).parent))
|
| 33 |
+
|
| 34 |
+
from gradio_labanmovementanalysis import LabanMovementAnalysis
|
| 35 |
+
|
| 36 |
+
|
| 37 |
+
class LabanMCPServer:
|
| 38 |
+
"""MCP Server for Laban Movement Analysis"""
|
| 39 |
+
|
| 40 |
+
def __init__(self):
|
| 41 |
+
self.server = Server("laban-movement-analysis")
|
| 42 |
+
self.analyzer = LabanMovementAnalysis()
|
| 43 |
+
self.analysis_cache = {}
|
| 44 |
+
self.temp_dir = tempfile.mkdtemp(prefix="laban_mcp_")
|
| 45 |
+
|
| 46 |
+
# Register tools
|
| 47 |
+
self._register_tools()
|
| 48 |
+
|
| 49 |
+
def _register_tools(self):
|
| 50 |
+
"""Register all available tools"""
|
| 51 |
+
|
| 52 |
+
@self.server.tool()
|
| 53 |
+
async def analyze_video(
|
| 54 |
+
video_path: str,
|
| 55 |
+
model: str = "mediapipe",
|
| 56 |
+
enable_visualization: bool = False,
|
| 57 |
+
include_keypoints: bool = False
|
| 58 |
+
) -> ToolResult:
|
| 59 |
+
"""
|
| 60 |
+
Analyze movement in a video file using Laban Movement Analysis.
|
| 61 |
+
|
| 62 |
+
Args:
|
| 63 |
+
video_path: Path or URL to video file
|
| 64 |
+
model: Pose estimation model ('mediapipe', 'movenet', 'yolo')
|
| 65 |
+
enable_visualization: Generate annotated video output
|
| 66 |
+
include_keypoints: Include raw keypoint data in JSON
|
| 67 |
+
|
| 68 |
+
Returns:
|
| 69 |
+
Movement analysis results and optional visualization
|
| 70 |
+
"""
|
| 71 |
+
try:
|
| 72 |
+
# Handle URL vs local path
|
| 73 |
+
if video_path.startswith(('http://', 'https://')):
|
| 74 |
+
video_path = await self._download_video(video_path)
|
| 75 |
+
|
| 76 |
+
# Process video
|
| 77 |
+
json_output, viz_video = await asyncio.to_thread(
|
| 78 |
+
self.analyzer.process_video,
|
| 79 |
+
video_path,
|
| 80 |
+
model=model,
|
| 81 |
+
enable_visualization=enable_visualization,
|
| 82 |
+
include_keypoints=include_keypoints
|
| 83 |
+
)
|
| 84 |
+
|
| 85 |
+
# Store in cache
|
| 86 |
+
analysis_id = f"{Path(video_path).stem}_{datetime.now().isoformat()}"
|
| 87 |
+
self.analysis_cache[analysis_id] = {
|
| 88 |
+
"json_output": json_output,
|
| 89 |
+
"viz_video": viz_video,
|
| 90 |
+
"timestamp": datetime.now().isoformat()
|
| 91 |
+
}
|
| 92 |
+
|
| 93 |
+
# Format response
|
| 94 |
+
response_data = {
|
| 95 |
+
"analysis_id": analysis_id,
|
| 96 |
+
"analysis": json_output,
|
| 97 |
+
"visualization_path": viz_video if viz_video else None
|
| 98 |
+
}
|
| 99 |
+
|
| 100 |
+
return ToolResult(
|
| 101 |
+
success=True,
|
| 102 |
+
content=[TextContent(text=json.dumps(response_data, indent=2))]
|
| 103 |
+
)
|
| 104 |
+
|
| 105 |
+
except Exception as e:
|
| 106 |
+
return ToolResult(
|
| 107 |
+
success=False,
|
| 108 |
+
error=ToolError(message=f"Analysis failed: {str(e)}")
|
| 109 |
+
)
|
| 110 |
+
|
| 111 |
+
@self.server.tool()
|
| 112 |
+
async def get_analysis_summary(
|
| 113 |
+
analysis_id: str
|
| 114 |
+
) -> ToolResult:
|
| 115 |
+
"""
|
| 116 |
+
Get a human-readable summary of a previous analysis.
|
| 117 |
+
|
| 118 |
+
Args:
|
| 119 |
+
analysis_id: ID of the analysis to summarize
|
| 120 |
+
|
| 121 |
+
Returns:
|
| 122 |
+
Summary of movement analysis
|
| 123 |
+
"""
|
| 124 |
+
try:
|
| 125 |
+
if analysis_id not in self.analysis_cache:
|
| 126 |
+
return ToolResult(
|
| 127 |
+
success=False,
|
| 128 |
+
error=ToolError(message=f"Analysis ID '{analysis_id}' not found")
|
| 129 |
+
)
|
| 130 |
+
|
| 131 |
+
analysis_data = self.analysis_cache[analysis_id]["json_output"]
|
| 132 |
+
|
| 133 |
+
# Extract key information
|
| 134 |
+
summary = self._generate_summary(analysis_data)
|
| 135 |
+
|
| 136 |
+
return ToolResult(
|
| 137 |
+
success=True,
|
| 138 |
+
content=[TextContent(text=summary)]
|
| 139 |
+
)
|
| 140 |
+
|
| 141 |
+
except Exception as e:
|
| 142 |
+
return ToolResult(
|
| 143 |
+
success=False,
|
| 144 |
+
error=ToolError(message=f"Summary generation failed: {str(e)}")
|
| 145 |
+
)
|
| 146 |
+
|
| 147 |
+
@self.server.tool()
|
| 148 |
+
async def list_available_models() -> ToolResult:
|
| 149 |
+
"""
|
| 150 |
+
List available pose estimation models with their characteristics.
|
| 151 |
+
|
| 152 |
+
Returns:
|
| 153 |
+
Information about available models
|
| 154 |
+
"""
|
| 155 |
+
models_info = {
|
| 156 |
+
"mediapipe": {
|
| 157 |
+
"name": "MediaPipe Pose",
|
| 158 |
+
"keypoints": 33,
|
| 159 |
+
"dimensions": "3D",
|
| 160 |
+
"optimization": "CPU",
|
| 161 |
+
"best_for": "Single person, detailed analysis",
|
| 162 |
+
"speed": "Fast"
|
| 163 |
+
},
|
| 164 |
+
"movenet": {
|
| 165 |
+
"name": "MoveNet",
|
| 166 |
+
"keypoints": 17,
|
| 167 |
+
"dimensions": "2D",
|
| 168 |
+
"optimization": "Mobile/Edge",
|
| 169 |
+
"best_for": "Real-time applications, mobile devices",
|
| 170 |
+
"speed": "Very Fast"
|
| 171 |
+
},
|
| 172 |
+
"yolo": {
|
| 173 |
+
"name": "YOLO Pose",
|
| 174 |
+
"keypoints": 17,
|
| 175 |
+
"dimensions": "2D",
|
| 176 |
+
"optimization": "GPU",
|
| 177 |
+
"best_for": "Multi-person detection",
|
| 178 |
+
"speed": "Fast (with GPU)"
|
| 179 |
+
}
|
| 180 |
+
}
|
| 181 |
+
|
| 182 |
+
return ToolResult(
|
| 183 |
+
success=True,
|
| 184 |
+
content=[TextContent(text=json.dumps(models_info, indent=2))]
|
| 185 |
+
)
|
| 186 |
+
|
| 187 |
+
@self.server.tool()
|
| 188 |
+
async def batch_analyze(
|
| 189 |
+
video_paths: List[str],
|
| 190 |
+
model: str = "mediapipe",
|
| 191 |
+
parallel: bool = True
|
| 192 |
+
) -> ToolResult:
|
| 193 |
+
"""
|
| 194 |
+
Analyze multiple videos in batch.
|
| 195 |
+
|
| 196 |
+
Args:
|
| 197 |
+
video_paths: List of video paths or URLs
|
| 198 |
+
model: Pose estimation model to use
|
| 199 |
+
parallel: Process videos in parallel
|
| 200 |
+
|
| 201 |
+
Returns:
|
| 202 |
+
Batch analysis results
|
| 203 |
+
"""
|
| 204 |
+
try:
|
| 205 |
+
results = {}
|
| 206 |
+
|
| 207 |
+
if parallel:
|
| 208 |
+
# Process in parallel
|
| 209 |
+
tasks = []
|
| 210 |
+
for path in video_paths:
|
| 211 |
+
task = self._analyze_single_video(path, model)
|
| 212 |
+
tasks.append(task)
|
| 213 |
+
|
| 214 |
+
analyses = await asyncio.gather(*tasks)
|
| 215 |
+
|
| 216 |
+
for path, analysis in zip(video_paths, analyses):
|
| 217 |
+
results[path] = analysis
|
| 218 |
+
else:
|
| 219 |
+
# Process sequentially
|
| 220 |
+
for path in video_paths:
|
| 221 |
+
results[path] = await self._analyze_single_video(path, model)
|
| 222 |
+
|
| 223 |
+
return ToolResult(
|
| 224 |
+
success=True,
|
| 225 |
+
content=[TextContent(text=json.dumps(results, indent=2))]
|
| 226 |
+
)
|
| 227 |
+
|
| 228 |
+
except Exception as e:
|
| 229 |
+
return ToolResult(
|
| 230 |
+
success=False,
|
| 231 |
+
error=ToolError(message=f"Batch analysis failed: {str(e)}")
|
| 232 |
+
)
|
| 233 |
+
|
| 234 |
+
@self.server.tool()
|
| 235 |
+
async def compare_movements(
|
| 236 |
+
analysis_id1: str,
|
| 237 |
+
analysis_id2: str
|
| 238 |
+
) -> ToolResult:
|
| 239 |
+
"""
|
| 240 |
+
Compare movement patterns between two analyzed videos.
|
| 241 |
+
|
| 242 |
+
Args:
|
| 243 |
+
analysis_id1: First analysis ID
|
| 244 |
+
analysis_id2: Second analysis ID
|
| 245 |
+
|
| 246 |
+
Returns:
|
| 247 |
+
Comparison of movement metrics
|
| 248 |
+
"""
|
| 249 |
+
try:
|
| 250 |
+
if analysis_id1 not in self.analysis_cache:
|
| 251 |
+
return ToolResult(
|
| 252 |
+
success=False,
|
| 253 |
+
error=ToolError(message=f"Analysis ID '{analysis_id1}' not found")
|
| 254 |
+
)
|
| 255 |
+
|
| 256 |
+
if analysis_id2 not in self.analysis_cache:
|
| 257 |
+
return ToolResult(
|
| 258 |
+
success=False,
|
| 259 |
+
error=ToolError(message=f"Analysis ID '{analysis_id2}' not found")
|
| 260 |
+
)
|
| 261 |
+
|
| 262 |
+
# Get analyses
|
| 263 |
+
analysis1 = self.analysis_cache[analysis_id1]["json_output"]
|
| 264 |
+
analysis2 = self.analysis_cache[analysis_id2]["json_output"]
|
| 265 |
+
|
| 266 |
+
# Compare metrics
|
| 267 |
+
comparison = self._compare_analyses(analysis1, analysis2)
|
| 268 |
+
|
| 269 |
+
return ToolResult(
|
| 270 |
+
success=True,
|
| 271 |
+
content=[TextContent(text=json.dumps(comparison, indent=2))]
|
| 272 |
+
)
|
| 273 |
+
|
| 274 |
+
except Exception as e:
|
| 275 |
+
return ToolResult(
|
| 276 |
+
success=False,
|
| 277 |
+
error=ToolError(message=f"Comparison failed: {str(e)}")
|
| 278 |
+
)
|
| 279 |
+
|
| 280 |
+
async def _download_video(self, url: str) -> str:
|
| 281 |
+
"""Download video from URL to temporary file"""
|
| 282 |
+
async with httpx.AsyncClient() as client:
|
| 283 |
+
response = await client.get(url)
|
| 284 |
+
response.raise_for_status()
|
| 285 |
+
|
| 286 |
+
# Save to temp file
|
| 287 |
+
filename = Path(urlparse(url).path).name or "video.mp4"
|
| 288 |
+
temp_path = os.path.join(self.temp_dir, filename)
|
| 289 |
+
|
| 290 |
+
async with aiofiles.open(temp_path, 'wb') as f:
|
| 291 |
+
await f.write(response.content)
|
| 292 |
+
|
| 293 |
+
return temp_path
|
| 294 |
+
|
| 295 |
+
async def _analyze_single_video(self, path: str, model: str) -> Dict[str, Any]:
|
| 296 |
+
"""Analyze a single video"""
|
| 297 |
+
try:
|
| 298 |
+
if path.startswith(('http://', 'https://')):
|
| 299 |
+
path = await self._download_video(path)
|
| 300 |
+
|
| 301 |
+
json_output, _ = await asyncio.to_thread(
|
| 302 |
+
self.analyzer.process_video,
|
| 303 |
+
path,
|
| 304 |
+
model=model,
|
| 305 |
+
enable_visualization=False
|
| 306 |
+
)
|
| 307 |
+
|
| 308 |
+
return {
|
| 309 |
+
"status": "success",
|
| 310 |
+
"analysis": json_output
|
| 311 |
+
}
|
| 312 |
+
except Exception as e:
|
| 313 |
+
return {
|
| 314 |
+
"status": "error",
|
| 315 |
+
"error": str(e)
|
| 316 |
+
}
|
| 317 |
+
|
| 318 |
+
def _generate_summary(self, analysis_data: Dict[str, Any]) -> str:
|
| 319 |
+
"""Generate human-readable summary from analysis data"""
|
| 320 |
+
summary_parts = []
|
| 321 |
+
|
| 322 |
+
# Video info
|
| 323 |
+
video_info = analysis_data.get("video_info", {})
|
| 324 |
+
summary_parts.append(f"Video Analysis Summary")
|
| 325 |
+
summary_parts.append(f"Duration: {video_info.get('duration_seconds', 0):.1f} seconds")
|
| 326 |
+
summary_parts.append(f"Resolution: {video_info.get('width', 0)}x{video_info.get('height', 0)}")
|
| 327 |
+
summary_parts.append("")
|
| 328 |
+
|
| 329 |
+
# Movement summary
|
| 330 |
+
movement_summary = analysis_data.get("movement_analysis", {}).get("summary", {})
|
| 331 |
+
|
| 332 |
+
# Direction analysis
|
| 333 |
+
direction_data = movement_summary.get("direction", {})
|
| 334 |
+
dominant_direction = direction_data.get("dominant", "unknown")
|
| 335 |
+
summary_parts.append(f"Dominant Movement Direction: {dominant_direction}")
|
| 336 |
+
|
| 337 |
+
# Intensity analysis
|
| 338 |
+
intensity_data = movement_summary.get("intensity", {})
|
| 339 |
+
dominant_intensity = intensity_data.get("dominant", "unknown")
|
| 340 |
+
summary_parts.append(f"Movement Intensity: {dominant_intensity}")
|
| 341 |
+
|
| 342 |
+
# Speed analysis
|
| 343 |
+
speed_data = movement_summary.get("speed", {})
|
| 344 |
+
dominant_speed = speed_data.get("dominant", "unknown")
|
| 345 |
+
summary_parts.append(f"Movement Speed: {dominant_speed}")
|
| 346 |
+
|
| 347 |
+
# Segments
|
| 348 |
+
segments = movement_summary.get("movement_segments", [])
|
| 349 |
+
if segments:
|
| 350 |
+
summary_parts.append(f"\nMovement Segments: {len(segments)}")
|
| 351 |
+
for i, segment in enumerate(segments[:3]): # Show first 3
|
| 352 |
+
start_time = segment.get("start_time", 0)
|
| 353 |
+
end_time = segment.get("end_time", 0)
|
| 354 |
+
movement_type = segment.get("movement_type", "unknown")
|
| 355 |
+
summary_parts.append(f" Segment {i+1}: {movement_type} ({start_time:.1f}s - {end_time:.1f}s)")
|
| 356 |
+
|
| 357 |
+
return "\n".join(summary_parts)
|
| 358 |
+
|
| 359 |
+
def _compare_analyses(self, analysis1: Dict, analysis2: Dict) -> Dict[str, Any]:
|
| 360 |
+
"""Compare two movement analyses"""
|
| 361 |
+
comparison = {
|
| 362 |
+
"video1_info": analysis1.get("video_info", {}),
|
| 363 |
+
"video2_info": analysis2.get("video_info", {}),
|
| 364 |
+
"metric_comparison": {}
|
| 365 |
+
}
|
| 366 |
+
|
| 367 |
+
# Compare summaries
|
| 368 |
+
summary1 = analysis1.get("movement_analysis", {}).get("summary", {})
|
| 369 |
+
summary2 = analysis2.get("movement_analysis", {}).get("summary", {})
|
| 370 |
+
|
| 371 |
+
# Compare directions
|
| 372 |
+
dir1 = summary1.get("direction", {})
|
| 373 |
+
dir2 = summary2.get("direction", {})
|
| 374 |
+
comparison["metric_comparison"]["direction"] = {
|
| 375 |
+
"video1_dominant": dir1.get("dominant", "unknown"),
|
| 376 |
+
"video2_dominant": dir2.get("dominant", "unknown"),
|
| 377 |
+
"match": dir1.get("dominant") == dir2.get("dominant")
|
| 378 |
+
}
|
| 379 |
+
|
| 380 |
+
# Compare intensity
|
| 381 |
+
int1 = summary1.get("intensity", {})
|
| 382 |
+
int2 = summary2.get("intensity", {})
|
| 383 |
+
comparison["metric_comparison"]["intensity"] = {
|
| 384 |
+
"video1_dominant": int1.get("dominant", "unknown"),
|
| 385 |
+
"video2_dominant": int2.get("dominant", "unknown"),
|
| 386 |
+
"match": int1.get("dominant") == int2.get("dominant")
|
| 387 |
+
}
|
| 388 |
+
|
| 389 |
+
# Compare speed
|
| 390 |
+
speed1 = summary1.get("speed", {})
|
| 391 |
+
speed2 = summary2.get("speed", {})
|
| 392 |
+
comparison["metric_comparison"]["speed"] = {
|
| 393 |
+
"video1_dominant": speed1.get("dominant", "unknown"),
|
| 394 |
+
"video2_dominant": speed2.get("dominant", "unknown"),
|
| 395 |
+
"match": speed1.get("dominant") == speed2.get("dominant")
|
| 396 |
+
}
|
| 397 |
+
|
| 398 |
+
return comparison
|
| 399 |
+
|
| 400 |
+
async def run(self):
|
| 401 |
+
"""Run the MCP server"""
|
| 402 |
+
async with stdio_server() as (read_stream, write_stream):
|
| 403 |
+
await self.server.run(read_stream, write_stream)
|
| 404 |
+
|
| 405 |
+
|
| 406 |
+
async def main():
|
| 407 |
+
"""Main entry point"""
|
| 408 |
+
server = LabanMCPServer()
|
| 409 |
+
await server.run()
|
| 410 |
+
|
| 411 |
+
|
| 412 |
+
if __name__ == "__main__":
|
| 413 |
+
asyncio.run(main())
|
backend/requirements-mcp.txt
ADDED
|
@@ -0,0 +1,27 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# MCP Server Dependencies
|
| 2 |
+
mcp>=1.0.0
|
| 3 |
+
aiofiles>=23.0.0
|
| 4 |
+
httpx>=0.24.0
|
| 5 |
+
|
| 6 |
+
# Core dependencies (include from main requirements)
|
| 7 |
+
gradio>=5.0,<6.0
|
| 8 |
+
opencv-python>=4.8.0
|
| 9 |
+
numpy>=1.24.0,<2.0.0 # Pin to 1.x for compatibility with MediaPipe/pandas
|
| 10 |
+
mediapipe>=0.10.0
|
| 11 |
+
tensorflow>=2.13.0 # For MoveNet
|
| 12 |
+
tensorflow-hub>=0.14.0 # For MoveNet models
|
| 13 |
+
ultralytics>=8.0.0 # For YOLO v8/v11
|
| 14 |
+
torch>=2.0.0
|
| 15 |
+
torchvision>=0.15.0
|
| 16 |
+
|
| 17 |
+
# Video platform support
|
| 18 |
+
yt-dlp>=2023.7.6 # YouTube/Vimeo downloads
|
| 19 |
+
requests>=2.31.0 # Direct video downloads
|
| 20 |
+
|
| 21 |
+
# Enhanced model support
|
| 22 |
+
transformers>=4.35.0
|
| 23 |
+
accelerate>=0.24.0 # For model optimization
|
| 24 |
+
|
| 25 |
+
# WebRTC support (Official Gradio approach)
|
| 26 |
+
gradio-webrtc # Official WebRTC component for Gradio
|
| 27 |
+
twilio>=8.2.0 # TURN servers for cloud deployment (optional)
|
backend/requirements.txt
ADDED
|
@@ -0,0 +1,20 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Core dependencies
|
| 2 |
+
gradio>=5.0.0
|
| 3 |
+
numpy>=1.20.0
|
| 4 |
+
opencv-python>=4.5.0
|
| 5 |
+
|
| 6 |
+
# Pose estimation model dependencies (install based on your choice)
|
| 7 |
+
# For MediaPipe:
|
| 8 |
+
mediapipe>=0.10.21
|
| 9 |
+
|
| 10 |
+
# For MoveNet (TensorFlow):
|
| 11 |
+
tensorflow>=2.8.0
|
| 12 |
+
tensorflow-hub>=0.12.0
|
| 13 |
+
|
| 14 |
+
# For YOLO:
|
| 15 |
+
ultralytics>=8.0.0
|
| 16 |
+
|
| 17 |
+
# Optional dependencies for development
|
| 18 |
+
# pytest>=7.0.0
|
| 19 |
+
# black>=22.0.0
|
| 20 |
+
# flake8>=4.0.0
|
demo/__init__.py
ADDED
|
File without changes
|
demo/app.py
ADDED
|
@@ -0,0 +1,866 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""
|
| 2 |
+
Unified Laban Movement Analysis Demo
|
| 3 |
+
Comprehensive interface combining all features:
|
| 4 |
+
- Standard LMA analysis
|
| 5 |
+
- Enhanced features (WebRTC, YouTube/Vimeo)
|
| 6 |
+
- Agent API (batch processing, filtering)
|
| 7 |
+
- Real-time analysis
|
| 8 |
+
- Model comparison
|
| 9 |
+
|
| 10 |
+
Created by: Csaba Bolyós (BladeSzaSza)
|
| 11 |
+
Contact: [email protected]
|
| 12 |
+
GitHub: https://github.com/bladeszasza
|
| 13 |
+
LinkedIn: https://www.linkedin.com/in/csaba-bolyós-00a11767/
|
| 14 |
+
Hugging Face: https://huggingface.co/BladeSzaSza
|
| 15 |
+
|
| 16 |
+
Heavy Beta Version - Under Active Development
|
| 17 |
+
"""
|
| 18 |
+
|
| 19 |
+
import gradio as gr
|
| 20 |
+
import sys
|
| 21 |
+
from pathlib import Path
|
| 22 |
+
from typing import Dict, Any, List, Tuple
|
| 23 |
+
|
| 24 |
+
# Add parent directory to path
|
| 25 |
+
sys.path.insert(0, str(Path(__file__).parent.parent / "backend"))
|
| 26 |
+
|
| 27 |
+
from gradio_labanmovementanalysis import LabanMovementAnalysis
|
| 28 |
+
|
| 29 |
+
# Import agent API if available
|
| 30 |
+
try:
|
| 31 |
+
from gradio_labanmovementanalysis.agent_api import (
|
| 32 |
+
LabanAgentAPI,
|
| 33 |
+
PoseModel,
|
| 34 |
+
MovementDirection,
|
| 35 |
+
MovementIntensity
|
| 36 |
+
)
|
| 37 |
+
HAS_AGENT_API = True
|
| 38 |
+
except ImportError:
|
| 39 |
+
HAS_AGENT_API = False
|
| 40 |
+
|
| 41 |
+
# Import WebRTC components if available
|
| 42 |
+
try:
|
| 43 |
+
from gradio_webrtc import WebRTC
|
| 44 |
+
from gradio_labanmovementanalysis.webrtc_handler import (
|
| 45 |
+
webrtc_detection,
|
| 46 |
+
get_rtc_configuration
|
| 47 |
+
)
|
| 48 |
+
HAS_WEBRTC = True
|
| 49 |
+
except ImportError as e:
|
| 50 |
+
print(f"WebRTC import failed: {e}")
|
| 51 |
+
HAS_WEBRTC = False
|
| 52 |
+
|
| 53 |
+
# Initialize components
|
| 54 |
+
try:
|
| 55 |
+
# Initialize with WebRTC support
|
| 56 |
+
analyzer = LabanMovementAnalysis(
|
| 57 |
+
enable_webrtc=True,
|
| 58 |
+
enable_visualization=True
|
| 59 |
+
)
|
| 60 |
+
print("✅ Core features initialized successfully")
|
| 61 |
+
except Exception as e:
|
| 62 |
+
print(f"Warning: Some features may not be available: {e}")
|
| 63 |
+
analyzer = LabanMovementAnalysis(enable_webrtc=False)
|
| 64 |
+
|
| 65 |
+
# Initialize agent API if available
|
| 66 |
+
agent_api = None
|
| 67 |
+
if HAS_AGENT_API:
|
| 68 |
+
try:
|
| 69 |
+
agent_api = LabanAgentAPI()
|
| 70 |
+
except Exception as e:
|
| 71 |
+
print(f"Warning: Agent API not available: {e}")
|
| 72 |
+
agent_api = None
|
| 73 |
+
|
| 74 |
+
|
| 75 |
+
def process_video_standard(video, model, enable_viz, include_keypoints):
|
| 76 |
+
"""Standard video processing function."""
|
| 77 |
+
if video is None:
|
| 78 |
+
return None, None
|
| 79 |
+
|
| 80 |
+
try:
|
| 81 |
+
json_output, video_output = analyzer.process_video(
|
| 82 |
+
video,
|
| 83 |
+
model=model,
|
| 84 |
+
enable_visualization=enable_viz,
|
| 85 |
+
include_keypoints=include_keypoints
|
| 86 |
+
)
|
| 87 |
+
return json_output, video_output
|
| 88 |
+
except Exception as e:
|
| 89 |
+
return {"error": str(e)}, None
|
| 90 |
+
|
| 91 |
+
|
| 92 |
+
def process_video_enhanced(video_input, model, enable_viz, include_keypoints):
|
| 93 |
+
"""Enhanced video processing with all new features."""
|
| 94 |
+
if not video_input:
|
| 95 |
+
return {"error": "No video provided"}, None
|
| 96 |
+
|
| 97 |
+
try:
|
| 98 |
+
# Handle both file upload and URL input
|
| 99 |
+
video_path = video_input.name if hasattr(video_input, 'name') else video_input
|
| 100 |
+
|
| 101 |
+
json_result, viz_result = analyzer.process_video(
|
| 102 |
+
video_path,
|
| 103 |
+
model=model,
|
| 104 |
+
enable_visualization=enable_viz,
|
| 105 |
+
include_keypoints=include_keypoints
|
| 106 |
+
)
|
| 107 |
+
return json_result, viz_result
|
| 108 |
+
except Exception as e:
|
| 109 |
+
error_result = {"error": str(e)}
|
| 110 |
+
return error_result, None
|
| 111 |
+
|
| 112 |
+
|
| 113 |
+
def process_video_for_agent(video, model, output_format="summary"):
|
| 114 |
+
"""Process video with agent-friendly output format."""
|
| 115 |
+
if not HAS_AGENT_API or agent_api is None:
|
| 116 |
+
return {"error": "Agent API not available"}
|
| 117 |
+
|
| 118 |
+
if not video:
|
| 119 |
+
return {"error": "No video provided"}
|
| 120 |
+
|
| 121 |
+
try:
|
| 122 |
+
model_enum = PoseModel(model)
|
| 123 |
+
result = agent_api.analyze(video, model=model_enum, generate_visualization=False)
|
| 124 |
+
|
| 125 |
+
if output_format == "summary":
|
| 126 |
+
return {"summary": agent_api.get_movement_summary(result)}
|
| 127 |
+
elif output_format == "structured":
|
| 128 |
+
return {
|
| 129 |
+
"success": result.success,
|
| 130 |
+
"direction": result.dominant_direction.value,
|
| 131 |
+
"intensity": result.dominant_intensity.value,
|
| 132 |
+
"speed": result.dominant_speed,
|
| 133 |
+
"fluidity": result.fluidity_score,
|
| 134 |
+
"expansion": result.expansion_score,
|
| 135 |
+
"segments": len(result.movement_segments)
|
| 136 |
+
}
|
| 137 |
+
else: # json
|
| 138 |
+
return result.raw_data
|
| 139 |
+
except Exception as e:
|
| 140 |
+
return {"error": str(e)}
|
| 141 |
+
|
| 142 |
+
|
| 143 |
+
def batch_process_videos(files, model):
|
| 144 |
+
"""Process multiple videos in batch."""
|
| 145 |
+
if not HAS_AGENT_API or agent_api is None:
|
| 146 |
+
return {"error": "Agent API not available"}
|
| 147 |
+
|
| 148 |
+
if not files:
|
| 149 |
+
return {"error": "No videos provided"}
|
| 150 |
+
|
| 151 |
+
try:
|
| 152 |
+
video_paths = [f.name for f in files]
|
| 153 |
+
results = agent_api.batch_analyze(video_paths, model=PoseModel(model), parallel=True)
|
| 154 |
+
|
| 155 |
+
output = {
|
| 156 |
+
"total_videos": len(results),
|
| 157 |
+
"successful": sum(1 for r in results if r.success),
|
| 158 |
+
"failed": sum(1 for r in results if not r.success),
|
| 159 |
+
"results": []
|
| 160 |
+
}
|
| 161 |
+
|
| 162 |
+
for result in results:
|
| 163 |
+
output["results"].append({
|
| 164 |
+
"video": Path(result.video_path).name,
|
| 165 |
+
"success": result.success,
|
| 166 |
+
"summary": agent_api.get_movement_summary(result) if result.success else result.error
|
| 167 |
+
})
|
| 168 |
+
|
| 169 |
+
return output
|
| 170 |
+
except Exception as e:
|
| 171 |
+
return {"error": str(e)}
|
| 172 |
+
|
| 173 |
+
|
| 174 |
+
def filter_videos_by_movement(files, direction, intensity, min_fluidity, min_expansion):
|
| 175 |
+
"""Filter videos based on movement characteristics."""
|
| 176 |
+
if not HAS_AGENT_API or agent_api is None:
|
| 177 |
+
return {"error": "Agent API not available"}
|
| 178 |
+
|
| 179 |
+
if not files:
|
| 180 |
+
return {"error": "No videos provided"}
|
| 181 |
+
|
| 182 |
+
try:
|
| 183 |
+
video_paths = [f.name for f in files]
|
| 184 |
+
|
| 185 |
+
dir_filter = MovementDirection(direction) if direction != "any" else None
|
| 186 |
+
int_filter = MovementIntensity(intensity) if intensity != "any" else None
|
| 187 |
+
|
| 188 |
+
filtered = agent_api.filter_by_movement(
|
| 189 |
+
video_paths,
|
| 190 |
+
direction=dir_filter,
|
| 191 |
+
intensity=int_filter,
|
| 192 |
+
min_fluidity=min_fluidity if min_fluidity > 0 else None,
|
| 193 |
+
min_expansion=min_expansion if min_expansion > 0 else None
|
| 194 |
+
)
|
| 195 |
+
|
| 196 |
+
return {
|
| 197 |
+
"total_analyzed": len(video_paths),
|
| 198 |
+
"matching_videos": len(filtered),
|
| 199 |
+
"matches": [
|
| 200 |
+
{
|
| 201 |
+
"video": Path(r.video_path).name,
|
| 202 |
+
"direction": r.dominant_direction.value,
|
| 203 |
+
"intensity": r.dominant_intensity.value,
|
| 204 |
+
"fluidity": r.fluidity_score,
|
| 205 |
+
"expansion": r.expansion_score
|
| 206 |
+
}
|
| 207 |
+
for r in filtered
|
| 208 |
+
]
|
| 209 |
+
}
|
| 210 |
+
except Exception as e:
|
| 211 |
+
return {"error": str(e)}
|
| 212 |
+
|
| 213 |
+
|
| 214 |
+
def compare_models(video, model1, model2):
|
| 215 |
+
"""Compare two different pose models on the same video."""
|
| 216 |
+
if not video:
|
| 217 |
+
return "No video provided"
|
| 218 |
+
|
| 219 |
+
try:
|
| 220 |
+
# Analyze with both models
|
| 221 |
+
result1, _ = analyzer.process_video(video, model=model1, enable_visualization=False)
|
| 222 |
+
result2, _ = analyzer.process_video(video, model=model2, enable_visualization=False)
|
| 223 |
+
|
| 224 |
+
# Extract key metrics for comparison
|
| 225 |
+
def extract_metrics(result):
|
| 226 |
+
summary = result.get("movement_analysis", {}).get("summary", {})
|
| 227 |
+
return {
|
| 228 |
+
"direction": summary.get("direction", {}).get("dominant", "unknown"),
|
| 229 |
+
"intensity": summary.get("intensity", {}).get("dominant", "unknown"),
|
| 230 |
+
"speed": summary.get("speed", {}).get("dominant", "unknown"),
|
| 231 |
+
"frame_count": result.get("video_info", {}).get("frame_count", 0)
|
| 232 |
+
}
|
| 233 |
+
|
| 234 |
+
metrics1 = extract_metrics(result1)
|
| 235 |
+
metrics2 = extract_metrics(result2)
|
| 236 |
+
|
| 237 |
+
# Create comparison table data
|
| 238 |
+
comparison_data = [
|
| 239 |
+
["Direction", metrics1["direction"], metrics2["direction"],
|
| 240 |
+
"✓" if metrics1["direction"] == metrics2["direction"] else "✗"],
|
| 241 |
+
["Intensity", metrics1["intensity"], metrics2["intensity"],
|
| 242 |
+
"✓" if metrics1["intensity"] == metrics2["intensity"] else "✗"],
|
| 243 |
+
["Speed", metrics1["speed"], metrics2["speed"],
|
| 244 |
+
"✓" if metrics1["speed"] == metrics2["speed"] else "✗"],
|
| 245 |
+
["Frames Processed", str(metrics1["frame_count"]), str(metrics2["frame_count"]),
|
| 246 |
+
"✓" if metrics1["frame_count"] == metrics2["frame_count"] else "✗"]
|
| 247 |
+
]
|
| 248 |
+
|
| 249 |
+
return comparison_data
|
| 250 |
+
|
| 251 |
+
except Exception as e:
|
| 252 |
+
return [["Error", str(e), "", ""]]
|
| 253 |
+
|
| 254 |
+
|
| 255 |
+
def start_webrtc_stream(model):
|
| 256 |
+
"""Start WebRTC real-time analysis."""
|
| 257 |
+
try:
|
| 258 |
+
success = analyzer.start_webrtc_stream(model)
|
| 259 |
+
if success:
|
| 260 |
+
return "🟢 Stream Active", {"status": "streaming", "model": model}
|
| 261 |
+
else:
|
| 262 |
+
return "🔴 Failed to start", {"status": "error"}
|
| 263 |
+
except Exception as e:
|
| 264 |
+
return f"🔴 Error: {str(e)}", {"status": "error"}
|
| 265 |
+
|
| 266 |
+
|
| 267 |
+
def stop_webrtc_stream():
|
| 268 |
+
"""Stop WebRTC real-time analysis."""
|
| 269 |
+
try:
|
| 270 |
+
success = analyzer.stop_webrtc_stream()
|
| 271 |
+
if success:
|
| 272 |
+
return "🟡 Stream Stopped", {"status": "stopped"}
|
| 273 |
+
else:
|
| 274 |
+
return "🔴 Failed to stop", {"status": "error"}
|
| 275 |
+
except Exception as e:
|
| 276 |
+
return f"🔴 Error: {str(e)}", {"status": "error"}
|
| 277 |
+
|
| 278 |
+
|
| 279 |
+
def create_unified_demo():
|
| 280 |
+
"""Create the unified comprehensive demo."""
|
| 281 |
+
|
| 282 |
+
with gr.Blocks(
|
| 283 |
+
title="Laban Movement Analysis - Complete Suite by Csaba Bolyós",
|
| 284 |
+
theme=gr.themes.Soft(),
|
| 285 |
+
css="""
|
| 286 |
+
.main-header {
|
| 287 |
+
background: linear-gradient(135deg, #40826D 0%, #2E5E4A 50%, #1B3A2F 100%);
|
| 288 |
+
color: white;
|
| 289 |
+
padding: 30px;
|
| 290 |
+
border-radius: 10px;
|
| 291 |
+
margin-bottom: 20px;
|
| 292 |
+
text-align: center;
|
| 293 |
+
}
|
| 294 |
+
.feature-card {
|
| 295 |
+
border: 1px solid #e1e5e9;
|
| 296 |
+
border-radius: 8px;
|
| 297 |
+
padding: 16px;
|
| 298 |
+
margin: 8px 0;
|
| 299 |
+
background: #f8f9fa;
|
| 300 |
+
}
|
| 301 |
+
.json-output {
|
| 302 |
+
max-height: 600px;
|
| 303 |
+
overflow-y: auto;
|
| 304 |
+
font-family: monospace;
|
| 305 |
+
font-size: 12px;
|
| 306 |
+
}
|
| 307 |
+
.author-info {
|
| 308 |
+
background: linear-gradient(135deg, #40826D 0%, #2E5E4A 100%);
|
| 309 |
+
color: white;
|
| 310 |
+
padding: 15px;
|
| 311 |
+
border-radius: 8px;
|
| 312 |
+
margin: 10px 0;
|
| 313 |
+
text-align: center;
|
| 314 |
+
}
|
| 315 |
+
"""
|
| 316 |
+
) as demo:
|
| 317 |
+
|
| 318 |
+
# Main Header
|
| 319 |
+
gr.HTML("""
|
| 320 |
+
<div class="main-header">
|
| 321 |
+
<h1>🎭 Laban Movement Analysis - Complete Suite</h1>
|
| 322 |
+
<p style="font-size: 18px; margin: 10px 0;">
|
| 323 |
+
Professional movement analysis with pose estimation, AI action recognition,
|
| 324 |
+
real-time processing, and agent automation
|
| 325 |
+
</p>
|
| 326 |
+
<p style="font-size: 14px; opacity: 0.9;">
|
| 327 |
+
Supports YouTube/Vimeo URLs • WebRTC Streaming • 20+ Pose Models • MCP Integration
|
| 328 |
+
</p>
|
| 329 |
+
<p style="font-size: 12px; margin-top: 15px; opacity: 0.8;">
|
| 330 |
+
<strong>Version 0.01-beta</strong> - Heavy Beta Under Active Development
|
| 331 |
+
</p>
|
| 332 |
+
</div>
|
| 333 |
+
""")
|
| 334 |
+
|
| 335 |
+
with gr.Tabs():
|
| 336 |
+
# Tab 1: Standard Analysis
|
| 337 |
+
with gr.Tab("🎬 Standard Analysis"):
|
| 338 |
+
gr.Markdown("""
|
| 339 |
+
### Classic Laban Movement Analysis
|
| 340 |
+
Upload a video file to analyze movement using traditional LMA metrics with pose estimation.
|
| 341 |
+
""")
|
| 342 |
+
|
| 343 |
+
with gr.Row():
|
| 344 |
+
with gr.Column(scale=1):
|
| 345 |
+
video_input_std = gr.Video(
|
| 346 |
+
label="Upload Video",
|
| 347 |
+
sources=["upload"],
|
| 348 |
+
format="mp4"
|
| 349 |
+
)
|
| 350 |
+
|
| 351 |
+
model_dropdown_std = gr.Dropdown(
|
| 352 |
+
choices=["mediapipe", "movenet", "yolo"],
|
| 353 |
+
value="mediapipe",
|
| 354 |
+
label="Pose Estimation Model"
|
| 355 |
+
)
|
| 356 |
+
|
| 357 |
+
with gr.Row():
|
| 358 |
+
enable_viz_std = gr.Checkbox(
|
| 359 |
+
value=True,
|
| 360 |
+
label="Generate Visualization"
|
| 361 |
+
)
|
| 362 |
+
|
| 363 |
+
include_keypoints_std = gr.Checkbox(
|
| 364 |
+
value=False,
|
| 365 |
+
label="Include Keypoints"
|
| 366 |
+
)
|
| 367 |
+
|
| 368 |
+
process_btn_std = gr.Button("Analyze Movement", variant="primary")
|
| 369 |
+
|
| 370 |
+
gr.Examples(
|
| 371 |
+
examples=[
|
| 372 |
+
["examples/balette.mov"],
|
| 373 |
+
["examples/balette.mp4"],
|
| 374 |
+
],
|
| 375 |
+
inputs=video_input_std,
|
| 376 |
+
label="Example Videos"
|
| 377 |
+
)
|
| 378 |
+
|
| 379 |
+
with gr.Column(scale=2):
|
| 380 |
+
with gr.Tab("Analysis Results"):
|
| 381 |
+
json_output_std = gr.JSON(
|
| 382 |
+
label="Movement Analysis (JSON)",
|
| 383 |
+
elem_classes=["json-output"]
|
| 384 |
+
)
|
| 385 |
+
|
| 386 |
+
with gr.Tab("Visualization"):
|
| 387 |
+
video_output_std = gr.Video(
|
| 388 |
+
label="Annotated Video",
|
| 389 |
+
format="mp4"
|
| 390 |
+
)
|
| 391 |
+
|
| 392 |
+
gr.Markdown("""
|
| 393 |
+
**Visualization Guide:**
|
| 394 |
+
- 🦴 **Skeleton**: Pose keypoints and connections
|
| 395 |
+
- 🌊 **Trails**: Motion history (fading lines)
|
| 396 |
+
- ➡️ **Arrows**: Movement direction indicators
|
| 397 |
+
- 🎨 **Colors**: Green (low) → Orange (medium) → Red (high) intensity
|
| 398 |
+
""")
|
| 399 |
+
|
| 400 |
+
process_btn_std.click(
|
| 401 |
+
fn=process_video_standard,
|
| 402 |
+
inputs=[video_input_std, model_dropdown_std, enable_viz_std, include_keypoints_std],
|
| 403 |
+
outputs=[json_output_std, video_output_std],
|
| 404 |
+
api_name="analyze_standard"
|
| 405 |
+
)
|
| 406 |
+
|
| 407 |
+
# Tab 2: Enhanced Analysis
|
| 408 |
+
with gr.Tab("🚀 Enhanced Analysis"):
|
| 409 |
+
gr.Markdown("""
|
| 410 |
+
### Advanced Analysis with AI and URL Support
|
| 411 |
+
Analyze videos from URLs (YouTube/Vimeo), use advanced pose models, and get AI-powered insights.
|
| 412 |
+
""")
|
| 413 |
+
|
| 414 |
+
with gr.Row():
|
| 415 |
+
with gr.Column(scale=1):
|
| 416 |
+
gr.HTML('<div class="feature-card">')
|
| 417 |
+
gr.Markdown("**Video Input**")
|
| 418 |
+
|
| 419 |
+
# Changed from textbox to file upload as requested
|
| 420 |
+
video_input_enh = gr.File(
|
| 421 |
+
label="Upload Video or Drop File",
|
| 422 |
+
file_types=["video"],
|
| 423 |
+
type="filepath"
|
| 424 |
+
)
|
| 425 |
+
|
| 426 |
+
# URL input option
|
| 427 |
+
url_input_enh = gr.Textbox(
|
| 428 |
+
label="Or Enter Video URL",
|
| 429 |
+
placeholder="YouTube URL, Vimeo URL, or direct video URL",
|
| 430 |
+
info="Leave file upload empty to use URL"
|
| 431 |
+
)
|
| 432 |
+
|
| 433 |
+
gr.Examples(
|
| 434 |
+
examples=[
|
| 435 |
+
["examples/balette.mov"],
|
| 436 |
+
["https://www.youtube.com/shorts/RX9kH2l3L8U"],
|
| 437 |
+
["https://vimeo.com/815392738"]
|
| 438 |
+
],
|
| 439 |
+
inputs=url_input_enh,
|
| 440 |
+
label="Example URLs"
|
| 441 |
+
)
|
| 442 |
+
|
| 443 |
+
gr.Markdown("**Model Selection**")
|
| 444 |
+
|
| 445 |
+
model_select_enh = gr.Dropdown(
|
| 446 |
+
choices=[
|
| 447 |
+
# MediaPipe variants
|
| 448 |
+
"mediapipe-lite", "mediapipe-full", "mediapipe-heavy",
|
| 449 |
+
# MoveNet variants
|
| 450 |
+
"movenet-lightning", "movenet-thunder",
|
| 451 |
+
# YOLO variants (added X models)
|
| 452 |
+
"yolo-v8-n", "yolo-v8-s", "yolo-v8-m", "yolo-v8-l", "yolo-v8-x",
|
| 453 |
+
# YOLO v11 variants
|
| 454 |
+
"yolo-v11-n", "yolo-v11-s", "yolo-v11-m", "yolo-v11-l", "yolo-v11-x"
|
| 455 |
+
],
|
| 456 |
+
value="mediapipe-full",
|
| 457 |
+
label="Advanced Pose Models",
|
| 458 |
+
info="17+ model variants available"
|
| 459 |
+
)
|
| 460 |
+
|
| 461 |
+
gr.Markdown("**Analysis Options**")
|
| 462 |
+
|
| 463 |
+
with gr.Row():
|
| 464 |
+
enable_viz_enh = gr.Checkbox(value=True, label="Visualization")
|
| 465 |
+
|
| 466 |
+
with gr.Row():
|
| 467 |
+
include_keypoints_enh = gr.Checkbox(value=False, label="Raw Keypoints")
|
| 468 |
+
|
| 469 |
+
analyze_btn_enh = gr.Button("🚀 Enhanced Analysis", variant="primary", size="lg")
|
| 470 |
+
gr.HTML('</div>')
|
| 471 |
+
|
| 472 |
+
with gr.Column(scale=2):
|
| 473 |
+
with gr.Tab("📊 Analysis"):
|
| 474 |
+
analysis_output_enh = gr.JSON(label="Enhanced Analysis Results")
|
| 475 |
+
|
| 476 |
+
with gr.Tab("🎥 Visualization"):
|
| 477 |
+
viz_output_enh = gr.Video(label="Annotated Video")
|
| 478 |
+
|
| 479 |
+
def process_enhanced_input(file_input, url_input, model, enable_viz, include_keypoints):
|
| 480 |
+
"""Process either file upload or URL input."""
|
| 481 |
+
video_source = file_input if file_input else url_input
|
| 482 |
+
return process_video_enhanced(video_source, model, enable_viz, include_keypoints)
|
| 483 |
+
|
| 484 |
+
analyze_btn_enh.click(
|
| 485 |
+
fn=process_enhanced_input,
|
| 486 |
+
inputs=[video_input_enh, url_input_enh, model_select_enh, enable_viz_enh, include_keypoints_enh],
|
| 487 |
+
outputs=[analysis_output_enh, viz_output_enh],
|
| 488 |
+
api_name="analyze_enhanced"
|
| 489 |
+
)
|
| 490 |
+
|
| 491 |
+
# Tab 3: Agent API
|
| 492 |
+
with gr.Tab("🤖 Agent API"):
|
| 493 |
+
gr.Markdown("""
|
| 494 |
+
### AI Agent & Automation Features
|
| 495 |
+
Batch processing, filtering, and structured outputs designed for AI agents and automation.
|
| 496 |
+
""")
|
| 497 |
+
|
| 498 |
+
with gr.Tabs():
|
| 499 |
+
with gr.Tab("Single Analysis"):
|
| 500 |
+
with gr.Row():
|
| 501 |
+
with gr.Column():
|
| 502 |
+
video_input_agent = gr.Video(label="Upload Video", sources=["upload"])
|
| 503 |
+
model_select_agent = gr.Dropdown(
|
| 504 |
+
choices=["mediapipe", "movenet", "yolo"],
|
| 505 |
+
value="mediapipe",
|
| 506 |
+
label="Model"
|
| 507 |
+
)
|
| 508 |
+
output_format_agent = gr.Radio(
|
| 509 |
+
choices=["summary", "structured", "json"],
|
| 510 |
+
value="summary",
|
| 511 |
+
label="Output Format"
|
| 512 |
+
)
|
| 513 |
+
analyze_btn_agent = gr.Button("Analyze", variant="primary")
|
| 514 |
+
|
| 515 |
+
with gr.Column():
|
| 516 |
+
output_display_agent = gr.JSON(label="Agent Output")
|
| 517 |
+
|
| 518 |
+
analyze_btn_agent.click(
|
| 519 |
+
fn=process_video_for_agent,
|
| 520 |
+
inputs=[video_input_agent, model_select_agent, output_format_agent],
|
| 521 |
+
outputs=output_display_agent,
|
| 522 |
+
api_name="analyze_agent"
|
| 523 |
+
)
|
| 524 |
+
|
| 525 |
+
with gr.Tab("Batch Processing"):
|
| 526 |
+
with gr.Row():
|
| 527 |
+
with gr.Column():
|
| 528 |
+
batch_files = gr.File(
|
| 529 |
+
label="Upload Multiple Videos",
|
| 530 |
+
file_count="multiple",
|
| 531 |
+
file_types=["video"]
|
| 532 |
+
)
|
| 533 |
+
batch_model = gr.Dropdown(
|
| 534 |
+
choices=["mediapipe", "movenet", "yolo"],
|
| 535 |
+
value="mediapipe",
|
| 536 |
+
label="Model"
|
| 537 |
+
)
|
| 538 |
+
batch_btn = gr.Button("Process Batch", variant="primary")
|
| 539 |
+
|
| 540 |
+
with gr.Column():
|
| 541 |
+
batch_output = gr.JSON(label="Batch Results")
|
| 542 |
+
|
| 543 |
+
batch_btn.click(
|
| 544 |
+
fn=batch_process_videos,
|
| 545 |
+
inputs=[batch_files, batch_model],
|
| 546 |
+
outputs=batch_output,
|
| 547 |
+
api_name="batch_analyze"
|
| 548 |
+
)
|
| 549 |
+
|
| 550 |
+
with gr.Tab("Movement Filter"):
|
| 551 |
+
with gr.Row():
|
| 552 |
+
with gr.Column():
|
| 553 |
+
filter_files = gr.File(
|
| 554 |
+
label="Videos to Filter",
|
| 555 |
+
file_count="multiple",
|
| 556 |
+
file_types=["video"]
|
| 557 |
+
)
|
| 558 |
+
|
| 559 |
+
with gr.Group():
|
| 560 |
+
direction_filter = gr.Dropdown(
|
| 561 |
+
choices=["any", "up", "down", "left", "right", "stationary"],
|
| 562 |
+
value="any",
|
| 563 |
+
label="Direction Filter"
|
| 564 |
+
)
|
| 565 |
+
intensity_filter = gr.Dropdown(
|
| 566 |
+
choices=["any", "low", "medium", "high"],
|
| 567 |
+
value="any",
|
| 568 |
+
label="Intensity Filter"
|
| 569 |
+
)
|
| 570 |
+
fluidity_threshold = gr.Slider(0.0, 1.0, 0.0, label="Min Fluidity")
|
| 571 |
+
expansion_threshold = gr.Slider(0.0, 1.0, 0.0, label="Min Expansion")
|
| 572 |
+
|
| 573 |
+
filter_btn = gr.Button("Apply Filters", variant="primary")
|
| 574 |
+
|
| 575 |
+
with gr.Column():
|
| 576 |
+
filter_output = gr.JSON(label="Filtered Results")
|
| 577 |
+
|
| 578 |
+
filter_btn.click(
|
| 579 |
+
fn=filter_videos_by_movement,
|
| 580 |
+
inputs=[filter_files, direction_filter, intensity_filter,
|
| 581 |
+
fluidity_threshold, expansion_threshold],
|
| 582 |
+
outputs=filter_output,
|
| 583 |
+
api_name="filter_videos"
|
| 584 |
+
)
|
| 585 |
+
|
| 586 |
+
# Tab 4: Real-time WebRTC
|
| 587 |
+
with gr.Tab("📹 Real-time Analysis"):
|
| 588 |
+
gr.Markdown("""
|
| 589 |
+
### Live Camera Movement Analysis
|
| 590 |
+
Real-time pose detection and movement analysis from your webcam using WebRTC.
|
| 591 |
+
**Grant camera permissions when prompted for best experience.**
|
| 592 |
+
""")
|
| 593 |
+
|
| 594 |
+
# Official Gradio WebRTC approach (compatible with NumPy 1.x)
|
| 595 |
+
if HAS_WEBRTC:
|
| 596 |
+
|
| 597 |
+
# Get RTC configuration
|
| 598 |
+
rtc_config = get_rtc_configuration()
|
| 599 |
+
|
| 600 |
+
# Custom CSS following official guide
|
| 601 |
+
css_webrtc = """
|
| 602 |
+
.my-group {max-width: 480px !important; max-height: 480px !important;}
|
| 603 |
+
.my-column {display: flex !important; justify-content: center !important; align-items: center !important;}
|
| 604 |
+
"""
|
| 605 |
+
|
| 606 |
+
with gr.Column(elem_classes=["my-column"]):
|
| 607 |
+
with gr.Group(elem_classes=["my-group"]):
|
| 608 |
+
# Official WebRTC Component
|
| 609 |
+
webrtc_stream = WebRTC(
|
| 610 |
+
label="🎥 Live Camera Stream",
|
| 611 |
+
rtc_configuration=rtc_config
|
| 612 |
+
)
|
| 613 |
+
|
| 614 |
+
webrtc_model = gr.Dropdown(
|
| 615 |
+
choices=["mediapipe-lite", "movenet-lightning", "yolo-v11-n"],
|
| 616 |
+
value="mediapipe-lite",
|
| 617 |
+
label="Pose Model",
|
| 618 |
+
info="Optimized for real-time processing"
|
| 619 |
+
)
|
| 620 |
+
|
| 621 |
+
confidence_slider = gr.Slider(
|
| 622 |
+
label="Detection Confidence",
|
| 623 |
+
minimum=0.0,
|
| 624 |
+
maximum=1.0,
|
| 625 |
+
step=0.05,
|
| 626 |
+
value=0.5,
|
| 627 |
+
info="Higher = fewer false positives"
|
| 628 |
+
)
|
| 629 |
+
|
| 630 |
+
# Official WebRTC streaming setup following Gradio guide
|
| 631 |
+
webrtc_stream.stream(
|
| 632 |
+
fn=webrtc_detection,
|
| 633 |
+
inputs=[webrtc_stream, webrtc_model, confidence_slider],
|
| 634 |
+
outputs=[webrtc_stream],
|
| 635 |
+
time_limit=10 # Following official guide: 10 seconds per user
|
| 636 |
+
)
|
| 637 |
+
|
| 638 |
+
# Info display
|
| 639 |
+
gr.HTML("""
|
| 640 |
+
<div style="background: #e8f4fd; padding: 15px; border-radius: 8px; margin-top: 10px;">
|
| 641 |
+
<h4>📹 WebRTC Pose Analysis</h4>
|
| 642 |
+
<p style="margin: 5px 0;">Real-time movement analysis using your webcam</p>
|
| 643 |
+
|
| 644 |
+
<h4>🔒 Privacy</h4>
|
| 645 |
+
<p style="margin: 5px 0;">Processing happens locally - no video data stored</p>
|
| 646 |
+
|
| 647 |
+
<h4>💡 Usage</h4>
|
| 648 |
+
<ul style="margin: 5px 0; padding-left: 20px;">
|
| 649 |
+
<li>Grant camera permission when prompted</li>
|
| 650 |
+
<li>Move in front of camera to see pose detection</li>
|
| 651 |
+
<li>Adjust confidence threshold as needed</li>
|
| 652 |
+
</ul>
|
| 653 |
+
</div>
|
| 654 |
+
""")
|
| 655 |
+
|
| 656 |
+
else:
|
| 657 |
+
# Fallback if WebRTC component not available
|
| 658 |
+
gr.HTML("""
|
| 659 |
+
<div style="text-align: center; padding: 50px; border: 2px dashed #ff6b6b; border-radius: 8px; background: #ffe0e0;">
|
| 660 |
+
<h3>📦 WebRTC Component Required</h3>
|
| 661 |
+
<p><strong>To enable real-time camera analysis, install:</strong></p>
|
| 662 |
+
<code style="background: #f0f0f0; padding: 10px; border-radius: 4px; display: block; margin: 10px 0;">
|
| 663 |
+
pip install gradio-webrtc twilio
|
| 664 |
+
</code>
|
| 665 |
+
<p style="margin-top: 15px;"><em>Use Enhanced Analysis tab for video files meanwhile</em></p>
|
| 666 |
+
</div>
|
| 667 |
+
""")
|
| 668 |
+
|
| 669 |
+
# Tab 5: Model Comparison
|
| 670 |
+
with gr.Tab("⚖️ Model Comparison"):
|
| 671 |
+
gr.Markdown("""
|
| 672 |
+
### Compare Pose Estimation Models
|
| 673 |
+
Analyze the same video with different models to compare accuracy and results.
|
| 674 |
+
""")
|
| 675 |
+
|
| 676 |
+
with gr.Column():
|
| 677 |
+
comparison_video = gr.Video(
|
| 678 |
+
label="Video for Comparison",
|
| 679 |
+
sources=["upload"]
|
| 680 |
+
)
|
| 681 |
+
|
| 682 |
+
with gr.Row():
|
| 683 |
+
model1_comp = gr.Dropdown(
|
| 684 |
+
choices=["mediapipe-full", "movenet-thunder", "yolo-v11-s"],
|
| 685 |
+
value="mediapipe-full",
|
| 686 |
+
label="Model 1"
|
| 687 |
+
)
|
| 688 |
+
|
| 689 |
+
model2_comp = gr.Dropdown(
|
| 690 |
+
choices=["mediapipe-full", "movenet-thunder", "yolo-v11-s"],
|
| 691 |
+
value="yolo-v11-s",
|
| 692 |
+
label="Model 2"
|
| 693 |
+
)
|
| 694 |
+
|
| 695 |
+
compare_btn = gr.Button("🔄 Compare Models", variant="primary")
|
| 696 |
+
|
| 697 |
+
comparison_results = gr.DataFrame(
|
| 698 |
+
headers=["Metric", "Model 1", "Model 2", "Match"],
|
| 699 |
+
label="Comparison Results"
|
| 700 |
+
)
|
| 701 |
+
|
| 702 |
+
compare_btn.click(
|
| 703 |
+
fn=compare_models,
|
| 704 |
+
inputs=[comparison_video, model1_comp, model2_comp],
|
| 705 |
+
outputs=comparison_results,
|
| 706 |
+
api_name="compare_models"
|
| 707 |
+
)
|
| 708 |
+
|
| 709 |
+
# Tab 6: Documentation
|
| 710 |
+
with gr.Tab("📚 Documentation"):
|
| 711 |
+
gr.Markdown("""
|
| 712 |
+
# Complete Feature Documentation
|
| 713 |
+
|
| 714 |
+
## 🎥 Video Input Support
|
| 715 |
+
- **Local Files**: MP4, AVI, MOV, WebM formats
|
| 716 |
+
- **YouTube**: Automatic download from YouTube URLs
|
| 717 |
+
- **Vimeo**: Automatic download from Vimeo URLs
|
| 718 |
+
- **Direct URLs**: Any direct video file URL
|
| 719 |
+
|
| 720 |
+
## 🤖 Pose Estimation Models
|
| 721 |
+
|
| 722 |
+
### MediaPipe (Google) - 33 3D Landmarks
|
| 723 |
+
- **Lite**: Fastest CPU performance
|
| 724 |
+
- **Full**: Balanced accuracy/speed (recommended)
|
| 725 |
+
- **Heavy**: Highest accuracy
|
| 726 |
+
|
| 727 |
+
### MoveNet (Google) - 17 COCO Keypoints
|
| 728 |
+
- **Lightning**: Mobile-optimized, very fast
|
| 729 |
+
- **Thunder**: Higher accuracy variant
|
| 730 |
+
|
| 731 |
+
### YOLO (Ultralytics) - 17 COCO Keypoints
|
| 732 |
+
- **v8 variants**: n/s/m/l/x sizes (nano to extra-large)
|
| 733 |
+
- **v11 variants**: Latest with improved accuracy (n/s/m/l/x)
|
| 734 |
+
- **Multi-person**: Supports multiple people in frame
|
| 735 |
+
|
| 736 |
+
## 📹 Real-time WebRTC
|
| 737 |
+
|
| 738 |
+
- **Live Camera**: Direct webcam access via WebRTC
|
| 739 |
+
- **Low Latency**: Sub-100ms processing
|
| 740 |
+
- **Adaptive Quality**: Automatic performance optimization
|
| 741 |
+
- **Live Overlay**: Real-time pose and metrics display
|
| 742 |
+
|
| 743 |
+
## 🤖 Agent & MCP Integration
|
| 744 |
+
|
| 745 |
+
### API Endpoints
|
| 746 |
+
- `/analyze_standard` - Basic LMA analysis
|
| 747 |
+
- `/analyze_enhanced` - Advanced analysis with all features
|
| 748 |
+
- `/analyze_agent` - Agent-optimized output
|
| 749 |
+
- `/batch_analyze` - Multiple video processing
|
| 750 |
+
- `/filter_videos` - Movement-based filtering
|
| 751 |
+
- `/compare_models` - Model comparison
|
| 752 |
+
|
| 753 |
+
### MCP Server
|
| 754 |
+
```bash
|
| 755 |
+
# Start MCP server for AI assistants
|
| 756 |
+
python -m backend.mcp_server
|
| 757 |
+
```
|
| 758 |
+
|
| 759 |
+
### Python API
|
| 760 |
+
```python
|
| 761 |
+
from gradio_labanmovementanalysis import LabanMovementAnalysis
|
| 762 |
+
|
| 763 |
+
# Initialize with all features
|
| 764 |
+
analyzer = LabanMovementAnalysis(
|
| 765 |
+
enable_webrtc=True
|
| 766 |
+
)
|
| 767 |
+
|
| 768 |
+
# Analyze YouTube video
|
| 769 |
+
result, viz = analyzer.process_video(
|
| 770 |
+
"https://youtube.com/watch?v=...",
|
| 771 |
+
model="yolo-v11-s"
|
| 772 |
+
)
|
| 773 |
+
```
|
| 774 |
+
|
| 775 |
+
## 📊 Output Formats
|
| 776 |
+
|
| 777 |
+
### Summary Format
|
| 778 |
+
Human-readable movement analysis summary.
|
| 779 |
+
|
| 780 |
+
### Structured Format
|
| 781 |
+
```json
|
| 782 |
+
{
|
| 783 |
+
"success": true,
|
| 784 |
+
"direction": "up",
|
| 785 |
+
"intensity": "medium",
|
| 786 |
+
"fluidity": 0.85,
|
| 787 |
+
"expansion": 0.72
|
| 788 |
+
}
|
| 789 |
+
```
|
| 790 |
+
|
| 791 |
+
### Full JSON Format
|
| 792 |
+
Complete frame-by-frame analysis with all metrics.
|
| 793 |
+
|
| 794 |
+
## 🎯 Applications
|
| 795 |
+
|
| 796 |
+
- **Sports**: Technique analysis and performance tracking
|
| 797 |
+
- **Dance**: Choreography analysis and movement quality
|
| 798 |
+
- **Healthcare**: Physical therapy and rehabilitation
|
| 799 |
+
- **Research**: Large-scale movement pattern studies
|
| 800 |
+
- **Entertainment**: Interactive applications and games
|
| 801 |
+
- **Education**: Movement teaching and body awareness
|
| 802 |
+
|
| 803 |
+
## 🔗 Integration Examples
|
| 804 |
+
|
| 805 |
+
### Gradio Client
|
| 806 |
+
```python
|
| 807 |
+
from gradio_client import Client
|
| 808 |
+
|
| 809 |
+
client = Client("http://localhost:7860")
|
| 810 |
+
result = client.predict(
|
| 811 |
+
video="path/to/video.mp4",
|
| 812 |
+
model="mediapipe-full",
|
| 813 |
+
api_name="/analyze_enhanced"
|
| 814 |
+
)
|
| 815 |
+
```
|
| 816 |
+
|
| 817 |
+
### Batch Processing
|
| 818 |
+
```python
|
| 819 |
+
results = client.predict(
|
| 820 |
+
files=["video1.mp4", "video2.mp4"],
|
| 821 |
+
model="yolo-v11-s",
|
| 822 |
+
api_name="/batch_analyze"
|
| 823 |
+
)
|
| 824 |
+
```
|
| 825 |
+
""")
|
| 826 |
+
gr.HTML("""
|
| 827 |
+
<div class="author-info">
|
| 828 |
+
<p><strong>Created by:</strong> Csaba Bolyós (BladeSzaSza)</p>
|
| 829 |
+
<p style="margin: 5px 0;">
|
| 830 |
+
<a href="https://github.com/bladeszasza" style="color: #a8e6cf; text-decoration: none;">🔗 GitHub</a> •
|
| 831 |
+
<a href="https://huggingface.co/BladeSzaSza" style="color: #a8e6cf; text-decoration: none;">🤗 Hugging Face</a> •
|
| 832 |
+
<a href="https://www.linkedin.com/in/csaba-bolyós-00a11767/" style="color: #a8e6cf; text-decoration: none;">💼 LinkedIn</a>
|
| 833 |
+
</p>
|
| 834 |
+
<p style="font-size: 12px; opacity: 0.9;">Contact: [email protected]</p>
|
| 835 |
+
</div>
|
| 836 |
+
""")
|
| 837 |
+
|
| 838 |
+
# Footer with proper attribution
|
| 839 |
+
gr.HTML("""
|
| 840 |
+
<div style="text-align: center; padding: 20px; margin-top: 30px; border-top: 1px solid #eee;">
|
| 841 |
+
<p style="color: #666; margin-bottom: 10px;">
|
| 842 |
+
🎭 Laban Movement Analysis - Complete Suite | Heavy Beta Version
|
| 843 |
+
</p>
|
| 844 |
+
<p style="color: #666; font-size: 12px;">
|
| 845 |
+
Created by <strong>Csaba Bolyós</strong> | Powered by MediaPipe, MoveNet & YOLO
|
| 846 |
+
</p>
|
| 847 |
+
<p style="color: #666; font-size: 10px; margin-top: 10px;">
|
| 848 |
+
<a href="https://github.com/bladeszasza" style="color: #40826D;">GitHub</a> •
|
| 849 |
+
<a href="https://huggingface.co/BladeSzaSza" style="color: #40826D;">Hugging Face</a> •
|
| 850 |
+
<a href="https://www.linkedin.com/in/csaba-bolyós-00a11767/" style="color: #40826D;">LinkedIn</a>
|
| 851 |
+
</p>
|
| 852 |
+
</div>
|
| 853 |
+
""")
|
| 854 |
+
|
| 855 |
+
return demo
|
| 856 |
+
|
| 857 |
+
|
| 858 |
+
if __name__ == "__main__":
|
| 859 |
+
demo = create_unified_demo()
|
| 860 |
+
demo.launch(
|
| 861 |
+
server_name="0.0.0.0",
|
| 862 |
+
server_port=7860,
|
| 863 |
+
share=False,
|
| 864 |
+
show_error=True,
|
| 865 |
+
favicon_path=None
|
| 866 |
+
)
|
demo/css.css
ADDED
|
@@ -0,0 +1,157 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
html {
|
| 2 |
+
font-family: Inter;
|
| 3 |
+
font-size: 16px;
|
| 4 |
+
font-weight: 400;
|
| 5 |
+
line-height: 1.5;
|
| 6 |
+
-webkit-text-size-adjust: 100%;
|
| 7 |
+
background: #fff;
|
| 8 |
+
color: #323232;
|
| 9 |
+
-webkit-font-smoothing: antialiased;
|
| 10 |
+
-moz-osx-font-smoothing: grayscale;
|
| 11 |
+
text-rendering: optimizeLegibility;
|
| 12 |
+
}
|
| 13 |
+
|
| 14 |
+
:root {
|
| 15 |
+
--space: 1;
|
| 16 |
+
--vspace: calc(var(--space) * 1rem);
|
| 17 |
+
--vspace-0: calc(3 * var(--space) * 1rem);
|
| 18 |
+
--vspace-1: calc(2 * var(--space) * 1rem);
|
| 19 |
+
--vspace-2: calc(1.5 * var(--space) * 1rem);
|
| 20 |
+
--vspace-3: calc(0.5 * var(--space) * 1rem);
|
| 21 |
+
}
|
| 22 |
+
|
| 23 |
+
.app {
|
| 24 |
+
max-width: 748px !important;
|
| 25 |
+
}
|
| 26 |
+
|
| 27 |
+
.prose p {
|
| 28 |
+
margin: var(--vspace) 0;
|
| 29 |
+
line-height: var(--vspace * 2);
|
| 30 |
+
font-size: 1rem;
|
| 31 |
+
}
|
| 32 |
+
|
| 33 |
+
code {
|
| 34 |
+
font-family: "Inconsolata", sans-serif;
|
| 35 |
+
font-size: 16px;
|
| 36 |
+
}
|
| 37 |
+
|
| 38 |
+
h1,
|
| 39 |
+
h1 code {
|
| 40 |
+
font-weight: 400;
|
| 41 |
+
line-height: calc(2.5 / var(--space) * var(--vspace));
|
| 42 |
+
}
|
| 43 |
+
|
| 44 |
+
h1 code {
|
| 45 |
+
background: none;
|
| 46 |
+
border: none;
|
| 47 |
+
letter-spacing: 0.05em;
|
| 48 |
+
padding-bottom: 5px;
|
| 49 |
+
position: relative;
|
| 50 |
+
padding: 0;
|
| 51 |
+
}
|
| 52 |
+
|
| 53 |
+
h2 {
|
| 54 |
+
margin: var(--vspace-1) 0 var(--vspace-2) 0;
|
| 55 |
+
line-height: 1em;
|
| 56 |
+
}
|
| 57 |
+
|
| 58 |
+
h3,
|
| 59 |
+
h3 code {
|
| 60 |
+
margin: var(--vspace-1) 0 var(--vspace-2) 0;
|
| 61 |
+
line-height: 1em;
|
| 62 |
+
}
|
| 63 |
+
|
| 64 |
+
h4,
|
| 65 |
+
h5,
|
| 66 |
+
h6 {
|
| 67 |
+
margin: var(--vspace-3) 0 var(--vspace-3) 0;
|
| 68 |
+
line-height: var(--vspace);
|
| 69 |
+
}
|
| 70 |
+
|
| 71 |
+
.bigtitle,
|
| 72 |
+
h1,
|
| 73 |
+
h1 code {
|
| 74 |
+
font-size: calc(8px * 4.5);
|
| 75 |
+
word-break: break-word;
|
| 76 |
+
}
|
| 77 |
+
|
| 78 |
+
.title,
|
| 79 |
+
h2,
|
| 80 |
+
h2 code {
|
| 81 |
+
font-size: calc(8px * 3.375);
|
| 82 |
+
font-weight: lighter;
|
| 83 |
+
word-break: break-word;
|
| 84 |
+
border: none;
|
| 85 |
+
background: none;
|
| 86 |
+
}
|
| 87 |
+
|
| 88 |
+
.subheading1,
|
| 89 |
+
h3,
|
| 90 |
+
h3 code {
|
| 91 |
+
font-size: calc(8px * 1.8);
|
| 92 |
+
font-weight: 600;
|
| 93 |
+
border: none;
|
| 94 |
+
background: none;
|
| 95 |
+
letter-spacing: 0.1em;
|
| 96 |
+
text-transform: uppercase;
|
| 97 |
+
}
|
| 98 |
+
|
| 99 |
+
h2 code {
|
| 100 |
+
padding: 0;
|
| 101 |
+
position: relative;
|
| 102 |
+
letter-spacing: 0.05em;
|
| 103 |
+
}
|
| 104 |
+
|
| 105 |
+
blockquote {
|
| 106 |
+
font-size: calc(8px * 1.1667);
|
| 107 |
+
font-style: italic;
|
| 108 |
+
line-height: calc(1.1667 * var(--vspace));
|
| 109 |
+
margin: var(--vspace-2) var(--vspace-2);
|
| 110 |
+
}
|
| 111 |
+
|
| 112 |
+
.subheading2,
|
| 113 |
+
h4 {
|
| 114 |
+
font-size: calc(8px * 1.4292);
|
| 115 |
+
text-transform: uppercase;
|
| 116 |
+
font-weight: 600;
|
| 117 |
+
}
|
| 118 |
+
|
| 119 |
+
.subheading3,
|
| 120 |
+
h5 {
|
| 121 |
+
font-size: calc(8px * 1.2917);
|
| 122 |
+
line-height: calc(1.2917 * var(--vspace));
|
| 123 |
+
|
| 124 |
+
font-weight: lighter;
|
| 125 |
+
text-transform: uppercase;
|
| 126 |
+
letter-spacing: 0.15em;
|
| 127 |
+
}
|
| 128 |
+
|
| 129 |
+
h6 {
|
| 130 |
+
font-size: calc(8px * 1.1667);
|
| 131 |
+
font-size: 1.1667em;
|
| 132 |
+
font-weight: normal;
|
| 133 |
+
font-style: italic;
|
| 134 |
+
font-family: "le-monde-livre-classic-byol", serif !important;
|
| 135 |
+
letter-spacing: 0px !important;
|
| 136 |
+
}
|
| 137 |
+
|
| 138 |
+
#start .md > *:first-child {
|
| 139 |
+
margin-top: 0;
|
| 140 |
+
}
|
| 141 |
+
|
| 142 |
+
h2 + h3 {
|
| 143 |
+
margin-top: 0;
|
| 144 |
+
}
|
| 145 |
+
|
| 146 |
+
.md hr {
|
| 147 |
+
border: none;
|
| 148 |
+
border-top: 1px solid var(--block-border-color);
|
| 149 |
+
margin: var(--vspace-2) 0 var(--vspace-2) 0;
|
| 150 |
+
}
|
| 151 |
+
.prose ul {
|
| 152 |
+
margin: var(--vspace-2) 0 var(--vspace-1) 0;
|
| 153 |
+
}
|
| 154 |
+
|
| 155 |
+
.gap {
|
| 156 |
+
gap: 0;
|
| 157 |
+
}
|
demo/requirements.txt
ADDED
|
@@ -0,0 +1 @@
|
|
|
|
|
|
|
| 1 |
+
gradio_labanmovementanalysis
|
demo/space.py
ADDED
|
@@ -0,0 +1,983 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
|
| 2 |
+
import gradio as gr
|
| 3 |
+
from app import demo as app
|
| 4 |
+
import os
|
| 5 |
+
|
| 6 |
+
_docs = {'LabanMovementAnalysis': {'description': 'Gradio component for video-based pose analysis with Laban Movement Analysis metrics.', 'members': {'__init__': {'default_model': {'type': 'str', 'default': '"mediapipe"', 'description': 'Default pose estimation model ("mediapipe", "movenet", "yolo")'}, 'enable_visualization': {'type': 'bool', 'default': 'True', 'description': 'Whether to generate visualization video by default'}, 'include_keypoints': {'type': 'bool', 'default': 'False', 'description': 'Whether to include raw keypoints in JSON output'}, 'enable_webrtc': {'type': 'bool', 'default': 'False', 'description': 'Whether to enable WebRTC real-time analysis'}, 'label': {'type': 'typing.Optional[str][str, None]', 'default': 'None', 'description': 'Component label'}, 'every': {'type': 'typing.Optional[float][float, None]', 'default': 'None', 'description': None}, 'show_label': {'type': 'typing.Optional[bool][bool, None]', 'default': 'None', 'description': None}, 'container': {'type': 'bool', 'default': 'True', 'description': None}, 'scale': {'type': 'typing.Optional[int][int, None]', 'default': 'None', 'description': None}, 'min_width': {'type': 'int', 'default': '160', 'description': None}, 'interactive': {'type': 'typing.Optional[bool][bool, None]', 'default': 'None', 'description': None}, 'visible': {'type': 'bool', 'default': 'True', 'description': None}, 'elem_id': {'type': 'typing.Optional[str][str, None]', 'default': 'None', 'description': None}, 'elem_classes': {'type': 'typing.Optional[typing.List[str]][\n typing.List[str][str], None\n]', 'default': 'None', 'description': None}, 'render': {'type': 'bool', 'default': 'True', 'description': None}}, 'postprocess': {'value': {'type': 'typing.Any', 'description': 'Analysis results'}}, 'preprocess': {'return': {'type': 'typing.Dict[str, typing.Any][str, typing.Any]', 'description': 'Processed data for analysis'}, 'value': None}}, 'events': {}}, '__meta__': {'additional_interfaces': {}, 'user_fn_refs': {'LabanMovementAnalysis': []}}}
|
| 7 |
+
|
| 8 |
+
abs_path = os.path.join(os.path.dirname(__file__), "css.css")
|
| 9 |
+
|
| 10 |
+
with gr.Blocks(
|
| 11 |
+
css=abs_path,
|
| 12 |
+
theme=gr.themes.Default(
|
| 13 |
+
font_mono=[
|
| 14 |
+
gr.themes.GoogleFont("Inconsolata"),
|
| 15 |
+
"monospace",
|
| 16 |
+
],
|
| 17 |
+
),
|
| 18 |
+
) as demo:
|
| 19 |
+
gr.Markdown(
|
| 20 |
+
"""
|
| 21 |
+
# `gradio_labanmovementanalysis`
|
| 22 |
+
|
| 23 |
+
<div style="display: flex; gap: 7px;">
|
| 24 |
+
<a href="https://pypi.org/project/gradio_labanmovementanalysis/" target="_blank"><img alt="PyPI - Version" src="https://img.shields.io/pypi/v/gradio_labanmovementanalysis"></a>
|
| 25 |
+
</div>
|
| 26 |
+
|
| 27 |
+
A Gradio 5 component for video movement analysis using Laban Movement Analysis (LMA) with MCP support for AI agents
|
| 28 |
+
""", elem_classes=["md-custom"], header_links=True)
|
| 29 |
+
app.render()
|
| 30 |
+
gr.Markdown(
|
| 31 |
+
"""
|
| 32 |
+
## Installation
|
| 33 |
+
|
| 34 |
+
```bash
|
| 35 |
+
pip install gradio_labanmovementanalysis
|
| 36 |
+
```
|
| 37 |
+
|
| 38 |
+
## Usage
|
| 39 |
+
|
| 40 |
+
```python
|
| 41 |
+
\"\"\"
|
| 42 |
+
Unified Laban Movement Analysis Demo
|
| 43 |
+
Comprehensive interface combining all features:
|
| 44 |
+
- Standard LMA analysis
|
| 45 |
+
- Enhanced features (WebRTC, YouTube/Vimeo)
|
| 46 |
+
- Agent API (batch processing, filtering)
|
| 47 |
+
- Real-time analysis
|
| 48 |
+
- Model comparison
|
| 49 |
+
|
| 50 |
+
Created by: Csaba Bolyós (BladeSzaSza)
|
| 51 |
+
Contact: [email protected]
|
| 52 |
+
GitHub: https://github.com/bladeszasza
|
| 53 |
+
LinkedIn: https://www.linkedin.com/in/csaba-bolyós-00a11767/
|
| 54 |
+
Hugging Face: https://huggingface.co/BladeSzaSza
|
| 55 |
+
|
| 56 |
+
Heavy Beta Version - Under Active Development
|
| 57 |
+
\"\"\"
|
| 58 |
+
|
| 59 |
+
import gradio as gr
|
| 60 |
+
import sys
|
| 61 |
+
from pathlib import Path
|
| 62 |
+
from typing import Dict, Any, List, Tuple
|
| 63 |
+
|
| 64 |
+
# Add parent directory to path
|
| 65 |
+
sys.path.insert(0, str(Path(__file__).parent.parent / "backend"))
|
| 66 |
+
|
| 67 |
+
from gradio_labanmovementanalysis import LabanMovementAnalysis
|
| 68 |
+
|
| 69 |
+
# Import agent API if available
|
| 70 |
+
try:
|
| 71 |
+
from gradio_labanmovementanalysis.agent_api import (
|
| 72 |
+
LabanAgentAPI,
|
| 73 |
+
PoseModel,
|
| 74 |
+
MovementDirection,
|
| 75 |
+
MovementIntensity
|
| 76 |
+
)
|
| 77 |
+
HAS_AGENT_API = True
|
| 78 |
+
except ImportError:
|
| 79 |
+
HAS_AGENT_API = False
|
| 80 |
+
|
| 81 |
+
# Import WebRTC components if available
|
| 82 |
+
try:
|
| 83 |
+
from gradio_webrtc import WebRTC
|
| 84 |
+
from gradio_labanmovementanalysis.webrtc_handler import (
|
| 85 |
+
webrtc_detection,
|
| 86 |
+
get_rtc_configuration
|
| 87 |
+
)
|
| 88 |
+
HAS_WEBRTC = True
|
| 89 |
+
except ImportError as e:
|
| 90 |
+
print(f"WebRTC import failed: {e}")
|
| 91 |
+
HAS_WEBRTC = False
|
| 92 |
+
|
| 93 |
+
# Initialize components
|
| 94 |
+
try:
|
| 95 |
+
# Initialize with WebRTC support
|
| 96 |
+
analyzer = LabanMovementAnalysis(
|
| 97 |
+
enable_webrtc=True,
|
| 98 |
+
enable_visualization=True
|
| 99 |
+
)
|
| 100 |
+
print("✅ Core features initialized successfully")
|
| 101 |
+
except Exception as e:
|
| 102 |
+
print(f"Warning: Some features may not be available: {e}")
|
| 103 |
+
analyzer = LabanMovementAnalysis(enable_webrtc=False)
|
| 104 |
+
|
| 105 |
+
# Initialize agent API if available
|
| 106 |
+
agent_api = None
|
| 107 |
+
if HAS_AGENT_API:
|
| 108 |
+
try:
|
| 109 |
+
agent_api = LabanAgentAPI()
|
| 110 |
+
except Exception as e:
|
| 111 |
+
print(f"Warning: Agent API not available: {e}")
|
| 112 |
+
agent_api = None
|
| 113 |
+
|
| 114 |
+
|
| 115 |
+
def process_video_standard(video, model, enable_viz, include_keypoints):
|
| 116 |
+
\"\"\"Standard video processing function.\"\"\"
|
| 117 |
+
if video is None:
|
| 118 |
+
return None, None
|
| 119 |
+
|
| 120 |
+
try:
|
| 121 |
+
json_output, video_output = analyzer.process_video(
|
| 122 |
+
video,
|
| 123 |
+
model=model,
|
| 124 |
+
enable_visualization=enable_viz,
|
| 125 |
+
include_keypoints=include_keypoints
|
| 126 |
+
)
|
| 127 |
+
return json_output, video_output
|
| 128 |
+
except Exception as e:
|
| 129 |
+
return {"error": str(e)}, None
|
| 130 |
+
|
| 131 |
+
|
| 132 |
+
def process_video_enhanced(video_input, model, enable_viz, include_keypoints):
|
| 133 |
+
\"\"\"Enhanced video processing with all new features.\"\"\"
|
| 134 |
+
if not video_input:
|
| 135 |
+
return {"error": "No video provided"}, None
|
| 136 |
+
|
| 137 |
+
try:
|
| 138 |
+
# Handle both file upload and URL input
|
| 139 |
+
video_path = video_input.name if hasattr(video_input, 'name') else video_input
|
| 140 |
+
|
| 141 |
+
json_result, viz_result = analyzer.process_video(
|
| 142 |
+
video_path,
|
| 143 |
+
model=model,
|
| 144 |
+
enable_visualization=enable_viz,
|
| 145 |
+
include_keypoints=include_keypoints
|
| 146 |
+
)
|
| 147 |
+
return json_result, viz_result
|
| 148 |
+
except Exception as e:
|
| 149 |
+
error_result = {"error": str(e)}
|
| 150 |
+
return error_result, None
|
| 151 |
+
|
| 152 |
+
|
| 153 |
+
def process_video_for_agent(video, model, output_format="summary"):
|
| 154 |
+
\"\"\"Process video with agent-friendly output format.\"\"\"
|
| 155 |
+
if not HAS_AGENT_API or agent_api is None:
|
| 156 |
+
return {"error": "Agent API not available"}
|
| 157 |
+
|
| 158 |
+
if not video:
|
| 159 |
+
return {"error": "No video provided"}
|
| 160 |
+
|
| 161 |
+
try:
|
| 162 |
+
model_enum = PoseModel(model)
|
| 163 |
+
result = agent_api.analyze(video, model=model_enum, generate_visualization=False)
|
| 164 |
+
|
| 165 |
+
if output_format == "summary":
|
| 166 |
+
return {"summary": agent_api.get_movement_summary(result)}
|
| 167 |
+
elif output_format == "structured":
|
| 168 |
+
return {
|
| 169 |
+
"success": result.success,
|
| 170 |
+
"direction": result.dominant_direction.value,
|
| 171 |
+
"intensity": result.dominant_intensity.value,
|
| 172 |
+
"speed": result.dominant_speed,
|
| 173 |
+
"fluidity": result.fluidity_score,
|
| 174 |
+
"expansion": result.expansion_score,
|
| 175 |
+
"segments": len(result.movement_segments)
|
| 176 |
+
}
|
| 177 |
+
else: # json
|
| 178 |
+
return result.raw_data
|
| 179 |
+
except Exception as e:
|
| 180 |
+
return {"error": str(e)}
|
| 181 |
+
|
| 182 |
+
|
| 183 |
+
def batch_process_videos(files, model):
|
| 184 |
+
\"\"\"Process multiple videos in batch.\"\"\"
|
| 185 |
+
if not HAS_AGENT_API or agent_api is None:
|
| 186 |
+
return {"error": "Agent API not available"}
|
| 187 |
+
|
| 188 |
+
if not files:
|
| 189 |
+
return {"error": "No videos provided"}
|
| 190 |
+
|
| 191 |
+
try:
|
| 192 |
+
video_paths = [f.name for f in files]
|
| 193 |
+
results = agent_api.batch_analyze(video_paths, model=PoseModel(model), parallel=True)
|
| 194 |
+
|
| 195 |
+
output = {
|
| 196 |
+
"total_videos": len(results),
|
| 197 |
+
"successful": sum(1 for r in results if r.success),
|
| 198 |
+
"failed": sum(1 for r in results if not r.success),
|
| 199 |
+
"results": []
|
| 200 |
+
}
|
| 201 |
+
|
| 202 |
+
for result in results:
|
| 203 |
+
output["results"].append({
|
| 204 |
+
"video": Path(result.video_path).name,
|
| 205 |
+
"success": result.success,
|
| 206 |
+
"summary": agent_api.get_movement_summary(result) if result.success else result.error
|
| 207 |
+
})
|
| 208 |
+
|
| 209 |
+
return output
|
| 210 |
+
except Exception as e:
|
| 211 |
+
return {"error": str(e)}
|
| 212 |
+
|
| 213 |
+
|
| 214 |
+
def filter_videos_by_movement(files, direction, intensity, min_fluidity, min_expansion):
|
| 215 |
+
\"\"\"Filter videos based on movement characteristics.\"\"\"
|
| 216 |
+
if not HAS_AGENT_API or agent_api is None:
|
| 217 |
+
return {"error": "Agent API not available"}
|
| 218 |
+
|
| 219 |
+
if not files:
|
| 220 |
+
return {"error": "No videos provided"}
|
| 221 |
+
|
| 222 |
+
try:
|
| 223 |
+
video_paths = [f.name for f in files]
|
| 224 |
+
|
| 225 |
+
dir_filter = MovementDirection(direction) if direction != "any" else None
|
| 226 |
+
int_filter = MovementIntensity(intensity) if intensity != "any" else None
|
| 227 |
+
|
| 228 |
+
filtered = agent_api.filter_by_movement(
|
| 229 |
+
video_paths,
|
| 230 |
+
direction=dir_filter,
|
| 231 |
+
intensity=int_filter,
|
| 232 |
+
min_fluidity=min_fluidity if min_fluidity > 0 else None,
|
| 233 |
+
min_expansion=min_expansion if min_expansion > 0 else None
|
| 234 |
+
)
|
| 235 |
+
|
| 236 |
+
return {
|
| 237 |
+
"total_analyzed": len(video_paths),
|
| 238 |
+
"matching_videos": len(filtered),
|
| 239 |
+
"matches": [
|
| 240 |
+
{
|
| 241 |
+
"video": Path(r.video_path).name,
|
| 242 |
+
"direction": r.dominant_direction.value,
|
| 243 |
+
"intensity": r.dominant_intensity.value,
|
| 244 |
+
"fluidity": r.fluidity_score,
|
| 245 |
+
"expansion": r.expansion_score
|
| 246 |
+
}
|
| 247 |
+
for r in filtered
|
| 248 |
+
]
|
| 249 |
+
}
|
| 250 |
+
except Exception as e:
|
| 251 |
+
return {"error": str(e)}
|
| 252 |
+
|
| 253 |
+
|
| 254 |
+
def compare_models(video, model1, model2):
|
| 255 |
+
\"\"\"Compare two different pose models on the same video.\"\"\"
|
| 256 |
+
if not video:
|
| 257 |
+
return "No video provided"
|
| 258 |
+
|
| 259 |
+
try:
|
| 260 |
+
# Analyze with both models
|
| 261 |
+
result1, _ = analyzer.process_video(video, model=model1, enable_visualization=False)
|
| 262 |
+
result2, _ = analyzer.process_video(video, model=model2, enable_visualization=False)
|
| 263 |
+
|
| 264 |
+
# Extract key metrics for comparison
|
| 265 |
+
def extract_metrics(result):
|
| 266 |
+
summary = result.get("movement_analysis", {}).get("summary", {})
|
| 267 |
+
return {
|
| 268 |
+
"direction": summary.get("direction", {}).get("dominant", "unknown"),
|
| 269 |
+
"intensity": summary.get("intensity", {}).get("dominant", "unknown"),
|
| 270 |
+
"speed": summary.get("speed", {}).get("dominant", "unknown"),
|
| 271 |
+
"frame_count": result.get("video_info", {}).get("frame_count", 0)
|
| 272 |
+
}
|
| 273 |
+
|
| 274 |
+
metrics1 = extract_metrics(result1)
|
| 275 |
+
metrics2 = extract_metrics(result2)
|
| 276 |
+
|
| 277 |
+
# Create comparison table data
|
| 278 |
+
comparison_data = [
|
| 279 |
+
["Direction", metrics1["direction"], metrics2["direction"],
|
| 280 |
+
"✓" if metrics1["direction"] == metrics2["direction"] else "✗"],
|
| 281 |
+
["Intensity", metrics1["intensity"], metrics2["intensity"],
|
| 282 |
+
"✓" if metrics1["intensity"] == metrics2["intensity"] else "✗"],
|
| 283 |
+
["Speed", metrics1["speed"], metrics2["speed"],
|
| 284 |
+
"✓" if metrics1["speed"] == metrics2["speed"] else "✗"],
|
| 285 |
+
["Frames Processed", str(metrics1["frame_count"]), str(metrics2["frame_count"]),
|
| 286 |
+
"✓" if metrics1["frame_count"] == metrics2["frame_count"] else "✗"]
|
| 287 |
+
]
|
| 288 |
+
|
| 289 |
+
return comparison_data
|
| 290 |
+
|
| 291 |
+
except Exception as e:
|
| 292 |
+
return [["Error", str(e), "", ""]]
|
| 293 |
+
|
| 294 |
+
|
| 295 |
+
def start_webrtc_stream(model):
|
| 296 |
+
\"\"\"Start WebRTC real-time analysis.\"\"\"
|
| 297 |
+
try:
|
| 298 |
+
success = analyzer.start_webrtc_stream(model)
|
| 299 |
+
if success:
|
| 300 |
+
return "🟢 Stream Active", {"status": "streaming", "model": model}
|
| 301 |
+
else:
|
| 302 |
+
return "🔴 Failed to start", {"status": "error"}
|
| 303 |
+
except Exception as e:
|
| 304 |
+
return f"🔴 Error: {str(e)}", {"status": "error"}
|
| 305 |
+
|
| 306 |
+
|
| 307 |
+
def stop_webrtc_stream():
|
| 308 |
+
\"\"\"Stop WebRTC real-time analysis.\"\"\"
|
| 309 |
+
try:
|
| 310 |
+
success = analyzer.stop_webrtc_stream()
|
| 311 |
+
if success:
|
| 312 |
+
return "🟡 Stream Stopped", {"status": "stopped"}
|
| 313 |
+
else:
|
| 314 |
+
return "🔴 Failed to stop", {"status": "error"}
|
| 315 |
+
except Exception as e:
|
| 316 |
+
return f"🔴 Error: {str(e)}", {"status": "error"}
|
| 317 |
+
|
| 318 |
+
|
| 319 |
+
def create_unified_demo():
|
| 320 |
+
\"\"\"Create the unified comprehensive demo.\"\"\"
|
| 321 |
+
|
| 322 |
+
with gr.Blocks(
|
| 323 |
+
title="Laban Movement Analysis - Complete Suite by Csaba Bolyós",
|
| 324 |
+
theme=gr.themes.Soft(),
|
| 325 |
+
css=\"\"\"
|
| 326 |
+
.main-header {
|
| 327 |
+
background: linear-gradient(135deg, #40826D 0%, #2E5E4A 50%, #1B3A2F 100%);
|
| 328 |
+
color: white;
|
| 329 |
+
padding: 30px;
|
| 330 |
+
border-radius: 10px;
|
| 331 |
+
margin-bottom: 20px;
|
| 332 |
+
text-align: center;
|
| 333 |
+
}
|
| 334 |
+
.feature-card {
|
| 335 |
+
border: 1px solid #e1e5e9;
|
| 336 |
+
border-radius: 8px;
|
| 337 |
+
padding: 16px;
|
| 338 |
+
margin: 8px 0;
|
| 339 |
+
background: #f8f9fa;
|
| 340 |
+
}
|
| 341 |
+
.json-output {
|
| 342 |
+
max-height: 600px;
|
| 343 |
+
overflow-y: auto;
|
| 344 |
+
font-family: monospace;
|
| 345 |
+
font-size: 12px;
|
| 346 |
+
}
|
| 347 |
+
.author-info {
|
| 348 |
+
background: linear-gradient(135deg, #40826D 0%, #2E5E4A 100%);
|
| 349 |
+
color: white;
|
| 350 |
+
padding: 15px;
|
| 351 |
+
border-radius: 8px;
|
| 352 |
+
margin: 10px 0;
|
| 353 |
+
text-align: center;
|
| 354 |
+
}
|
| 355 |
+
\"\"\"
|
| 356 |
+
) as demo:
|
| 357 |
+
|
| 358 |
+
# Main Header
|
| 359 |
+
gr.HTML(\"\"\"
|
| 360 |
+
<div class="main-header">
|
| 361 |
+
<h1>🎭 Laban Movement Analysis - Complete Suite</h1>
|
| 362 |
+
<p style="font-size: 18px; margin: 10px 0;">
|
| 363 |
+
Professional movement analysis with pose estimation, AI action recognition,
|
| 364 |
+
real-time processing, and agent automation
|
| 365 |
+
</p>
|
| 366 |
+
<p style="font-size: 14px; opacity: 0.9;">
|
| 367 |
+
Supports YouTube/Vimeo URLs • WebRTC Streaming • 20+ Pose Models • MCP Integration
|
| 368 |
+
</p>
|
| 369 |
+
<p style="font-size: 12px; margin-top: 15px; opacity: 0.8;">
|
| 370 |
+
<strong>Version 0.01-beta</strong> - Heavy Beta Under Active Development
|
| 371 |
+
</p>
|
| 372 |
+
</div>
|
| 373 |
+
\"\"\")
|
| 374 |
+
|
| 375 |
+
with gr.Tabs():
|
| 376 |
+
# Tab 1: Standard Analysis
|
| 377 |
+
with gr.Tab("🎬 Standard Analysis"):
|
| 378 |
+
gr.Markdown(\"\"\"
|
| 379 |
+
### Classic Laban Movement Analysis
|
| 380 |
+
Upload a video file to analyze movement using traditional LMA metrics with pose estimation.
|
| 381 |
+
\"\"\")
|
| 382 |
+
|
| 383 |
+
with gr.Row():
|
| 384 |
+
with gr.Column(scale=1):
|
| 385 |
+
video_input_std = gr.Video(
|
| 386 |
+
label="Upload Video",
|
| 387 |
+
sources=["upload"],
|
| 388 |
+
format="mp4"
|
| 389 |
+
)
|
| 390 |
+
|
| 391 |
+
model_dropdown_std = gr.Dropdown(
|
| 392 |
+
choices=["mediapipe", "movenet", "yolo"],
|
| 393 |
+
value="mediapipe",
|
| 394 |
+
label="Pose Estimation Model"
|
| 395 |
+
)
|
| 396 |
+
|
| 397 |
+
with gr.Row():
|
| 398 |
+
enable_viz_std = gr.Checkbox(
|
| 399 |
+
value=True,
|
| 400 |
+
label="Generate Visualization"
|
| 401 |
+
)
|
| 402 |
+
|
| 403 |
+
include_keypoints_std = gr.Checkbox(
|
| 404 |
+
value=False,
|
| 405 |
+
label="Include Keypoints"
|
| 406 |
+
)
|
| 407 |
+
|
| 408 |
+
process_btn_std = gr.Button("Analyze Movement", variant="primary")
|
| 409 |
+
|
| 410 |
+
gr.Examples(
|
| 411 |
+
examples=[
|
| 412 |
+
["examples/balette.mp4"],
|
| 413 |
+
],
|
| 414 |
+
inputs=video_input_std,
|
| 415 |
+
label="Example Videos"
|
| 416 |
+
)
|
| 417 |
+
|
| 418 |
+
with gr.Column(scale=2):
|
| 419 |
+
with gr.Tab("Analysis Results"):
|
| 420 |
+
json_output_std = gr.JSON(
|
| 421 |
+
label="Movement Analysis (JSON)",
|
| 422 |
+
elem_classes=["json-output"]
|
| 423 |
+
)
|
| 424 |
+
|
| 425 |
+
with gr.Tab("Visualization"):
|
| 426 |
+
video_output_std = gr.Video(
|
| 427 |
+
label="Annotated Video",
|
| 428 |
+
format="mp4"
|
| 429 |
+
)
|
| 430 |
+
|
| 431 |
+
gr.Markdown(\"\"\"
|
| 432 |
+
**Visualization Guide:**
|
| 433 |
+
- 🦴 **Skeleton**: Pose keypoints and connections
|
| 434 |
+
- 🌊 **Trails**: Motion history (fading lines)
|
| 435 |
+
- ➡️ **Arrows**: Movement direction indicators
|
| 436 |
+
- 🎨 **Colors**: Green (low) → Orange (medium) → Red (high) intensity
|
| 437 |
+
\"\"\")
|
| 438 |
+
|
| 439 |
+
process_btn_std.click(
|
| 440 |
+
fn=process_video_standard,
|
| 441 |
+
inputs=[video_input_std, model_dropdown_std, enable_viz_std, include_keypoints_std],
|
| 442 |
+
outputs=[json_output_std, video_output_std],
|
| 443 |
+
api_name="analyze_standard"
|
| 444 |
+
)
|
| 445 |
+
|
| 446 |
+
# Tab 2: Enhanced Analysis
|
| 447 |
+
with gr.Tab("🚀 Enhanced Analysis"):
|
| 448 |
+
gr.Markdown(\"\"\"
|
| 449 |
+
### Advanced Analysis with AI and URL Support
|
| 450 |
+
Analyze videos from URLs (YouTube/Vimeo), use advanced pose models, and get AI-powered insights.
|
| 451 |
+
\"\"\")
|
| 452 |
+
|
| 453 |
+
with gr.Row():
|
| 454 |
+
with gr.Column(scale=1):
|
| 455 |
+
gr.HTML('<div class="feature-card">')
|
| 456 |
+
gr.Markdown("**Video Input**")
|
| 457 |
+
|
| 458 |
+
# Changed from textbox to file upload as requested
|
| 459 |
+
video_input_enh = gr.File(
|
| 460 |
+
label="Upload Video or Drop File",
|
| 461 |
+
file_types=["video"],
|
| 462 |
+
type="filepath"
|
| 463 |
+
)
|
| 464 |
+
|
| 465 |
+
# URL input option
|
| 466 |
+
url_input_enh = gr.Textbox(
|
| 467 |
+
label="Or Enter Video URL",
|
| 468 |
+
placeholder="YouTube URL, Vimeo URL, or direct video URL",
|
| 469 |
+
info="Leave file upload empty to use URL"
|
| 470 |
+
)
|
| 471 |
+
|
| 472 |
+
gr.Examples(
|
| 473 |
+
examples=[
|
| 474 |
+
["examples/balette.mp4"],
|
| 475 |
+
["https://www.youtube.com/shorts/RX9kH2l3L8U"],
|
| 476 |
+
["https://vimeo.com/815392738"]
|
| 477 |
+
],
|
| 478 |
+
inputs=url_input_enh,
|
| 479 |
+
label="Example URLs"
|
| 480 |
+
)
|
| 481 |
+
|
| 482 |
+
gr.Markdown("**Model Selection**")
|
| 483 |
+
|
| 484 |
+
model_select_enh = gr.Dropdown(
|
| 485 |
+
choices=[
|
| 486 |
+
# MediaPipe variants
|
| 487 |
+
"mediapipe-lite", "mediapipe-full", "mediapipe-heavy",
|
| 488 |
+
# MoveNet variants
|
| 489 |
+
"movenet-lightning", "movenet-thunder",
|
| 490 |
+
# YOLO variants (added X models)
|
| 491 |
+
"yolo-v8-n", "yolo-v8-s", "yolo-v8-m", "yolo-v8-l", "yolo-v8-x",
|
| 492 |
+
# YOLO v11 variants
|
| 493 |
+
"yolo-v11-n", "yolo-v11-s", "yolo-v11-m", "yolo-v11-l", "yolo-v11-x"
|
| 494 |
+
],
|
| 495 |
+
value="mediapipe-full",
|
| 496 |
+
label="Advanced Pose Models",
|
| 497 |
+
info="17+ model variants available"
|
| 498 |
+
)
|
| 499 |
+
|
| 500 |
+
gr.Markdown("**Analysis Options**")
|
| 501 |
+
|
| 502 |
+
with gr.Row():
|
| 503 |
+
enable_viz_enh = gr.Checkbox(value=True, label="Visualization")
|
| 504 |
+
|
| 505 |
+
with gr.Row():
|
| 506 |
+
include_keypoints_enh = gr.Checkbox(value=False, label="Raw Keypoints")
|
| 507 |
+
|
| 508 |
+
analyze_btn_enh = gr.Button("🚀 Enhanced Analysis", variant="primary", size="lg")
|
| 509 |
+
gr.HTML('</div>')
|
| 510 |
+
|
| 511 |
+
with gr.Column(scale=2):
|
| 512 |
+
with gr.Tab("📊 Analysis"):
|
| 513 |
+
analysis_output_enh = gr.JSON(label="Enhanced Analysis Results")
|
| 514 |
+
|
| 515 |
+
with gr.Tab("🎥 Visualization"):
|
| 516 |
+
viz_output_enh = gr.Video(label="Annotated Video")
|
| 517 |
+
|
| 518 |
+
def process_enhanced_input(file_input, url_input, model, enable_viz, include_keypoints):
|
| 519 |
+
\"\"\"Process either file upload or URL input.\"\"\"
|
| 520 |
+
video_source = file_input if file_input else url_input
|
| 521 |
+
return process_video_enhanced(video_source, model, enable_viz, include_keypoints)
|
| 522 |
+
|
| 523 |
+
analyze_btn_enh.click(
|
| 524 |
+
fn=process_enhanced_input,
|
| 525 |
+
inputs=[video_input_enh, url_input_enh, model_select_enh, enable_viz_enh, include_keypoints_enh],
|
| 526 |
+
outputs=[analysis_output_enh, viz_output_enh],
|
| 527 |
+
api_name="analyze_enhanced"
|
| 528 |
+
)
|
| 529 |
+
|
| 530 |
+
# Tab 3: Agent API
|
| 531 |
+
with gr.Tab("🤖 Agent API"):
|
| 532 |
+
gr.Markdown(\"\"\"
|
| 533 |
+
### AI Agent & Automation Features
|
| 534 |
+
Batch processing, filtering, and structured outputs designed for AI agents and automation.
|
| 535 |
+
\"\"\")
|
| 536 |
+
|
| 537 |
+
with gr.Tabs():
|
| 538 |
+
with gr.Tab("Single Analysis"):
|
| 539 |
+
with gr.Row():
|
| 540 |
+
with gr.Column():
|
| 541 |
+
video_input_agent = gr.Video(label="Upload Video", sources=["upload"])
|
| 542 |
+
model_select_agent = gr.Dropdown(
|
| 543 |
+
choices=["mediapipe", "movenet", "yolo"],
|
| 544 |
+
value="mediapipe",
|
| 545 |
+
label="Model"
|
| 546 |
+
)
|
| 547 |
+
output_format_agent = gr.Radio(
|
| 548 |
+
choices=["summary", "structured", "json"],
|
| 549 |
+
value="summary",
|
| 550 |
+
label="Output Format"
|
| 551 |
+
)
|
| 552 |
+
analyze_btn_agent = gr.Button("Analyze", variant="primary")
|
| 553 |
+
|
| 554 |
+
with gr.Column():
|
| 555 |
+
output_display_agent = gr.JSON(label="Agent Output")
|
| 556 |
+
|
| 557 |
+
analyze_btn_agent.click(
|
| 558 |
+
fn=process_video_for_agent,
|
| 559 |
+
inputs=[video_input_agent, model_select_agent, output_format_agent],
|
| 560 |
+
outputs=output_display_agent,
|
| 561 |
+
api_name="analyze_agent"
|
| 562 |
+
)
|
| 563 |
+
|
| 564 |
+
with gr.Tab("Batch Processing"):
|
| 565 |
+
with gr.Row():
|
| 566 |
+
with gr.Column():
|
| 567 |
+
batch_files = gr.File(
|
| 568 |
+
label="Upload Multiple Videos",
|
| 569 |
+
file_count="multiple",
|
| 570 |
+
file_types=["video"]
|
| 571 |
+
)
|
| 572 |
+
batch_model = gr.Dropdown(
|
| 573 |
+
choices=["mediapipe", "movenet", "yolo"],
|
| 574 |
+
value="mediapipe",
|
| 575 |
+
label="Model"
|
| 576 |
+
)
|
| 577 |
+
batch_btn = gr.Button("Process Batch", variant="primary")
|
| 578 |
+
|
| 579 |
+
with gr.Column():
|
| 580 |
+
batch_output = gr.JSON(label="Batch Results")
|
| 581 |
+
|
| 582 |
+
batch_btn.click(
|
| 583 |
+
fn=batch_process_videos,
|
| 584 |
+
inputs=[batch_files, batch_model],
|
| 585 |
+
outputs=batch_output,
|
| 586 |
+
api_name="batch_analyze"
|
| 587 |
+
)
|
| 588 |
+
|
| 589 |
+
with gr.Tab("Movement Filter"):
|
| 590 |
+
with gr.Row():
|
| 591 |
+
with gr.Column():
|
| 592 |
+
filter_files = gr.File(
|
| 593 |
+
label="Videos to Filter",
|
| 594 |
+
file_count="multiple",
|
| 595 |
+
file_types=["video"]
|
| 596 |
+
)
|
| 597 |
+
|
| 598 |
+
with gr.Group():
|
| 599 |
+
direction_filter = gr.Dropdown(
|
| 600 |
+
choices=["any", "up", "down", "left", "right", "stationary"],
|
| 601 |
+
value="any",
|
| 602 |
+
label="Direction Filter"
|
| 603 |
+
)
|
| 604 |
+
intensity_filter = gr.Dropdown(
|
| 605 |
+
choices=["any", "low", "medium", "high"],
|
| 606 |
+
value="any",
|
| 607 |
+
label="Intensity Filter"
|
| 608 |
+
)
|
| 609 |
+
fluidity_threshold = gr.Slider(0.0, 1.0, 0.0, label="Min Fluidity")
|
| 610 |
+
expansion_threshold = gr.Slider(0.0, 1.0, 0.0, label="Min Expansion")
|
| 611 |
+
|
| 612 |
+
filter_btn = gr.Button("Apply Filters", variant="primary")
|
| 613 |
+
|
| 614 |
+
with gr.Column():
|
| 615 |
+
filter_output = gr.JSON(label="Filtered Results")
|
| 616 |
+
|
| 617 |
+
filter_btn.click(
|
| 618 |
+
fn=filter_videos_by_movement,
|
| 619 |
+
inputs=[filter_files, direction_filter, intensity_filter,
|
| 620 |
+
fluidity_threshold, expansion_threshold],
|
| 621 |
+
outputs=filter_output,
|
| 622 |
+
api_name="filter_videos"
|
| 623 |
+
)
|
| 624 |
+
|
| 625 |
+
# Tab 4: Real-time WebRTC
|
| 626 |
+
with gr.Tab("📹 Real-time Analysis"):
|
| 627 |
+
gr.Markdown(\"\"\"
|
| 628 |
+
### Live Camera Movement Analysis
|
| 629 |
+
Real-time pose detection and movement analysis from your webcam using WebRTC.
|
| 630 |
+
**Grant camera permissions when prompted for best experience.**
|
| 631 |
+
\"\"\")
|
| 632 |
+
|
| 633 |
+
# Official Gradio WebRTC approach (compatible with NumPy 1.x)
|
| 634 |
+
if HAS_WEBRTC:
|
| 635 |
+
|
| 636 |
+
# Get RTC configuration
|
| 637 |
+
rtc_config = get_rtc_configuration()
|
| 638 |
+
|
| 639 |
+
# Custom CSS following official guide
|
| 640 |
+
css_webrtc = \"\"\"
|
| 641 |
+
.my-group {max-width: 480px !important; max-height: 480px !important;}
|
| 642 |
+
.my-column {display: flex !important; justify-content: center !important; align-items: center !important;}
|
| 643 |
+
\"\"\"
|
| 644 |
+
|
| 645 |
+
with gr.Column(elem_classes=["my-column"]):
|
| 646 |
+
with gr.Group(elem_classes=["my-group"]):
|
| 647 |
+
# Official WebRTC Component
|
| 648 |
+
webrtc_stream = WebRTC(
|
| 649 |
+
label="🎥 Live Camera Stream",
|
| 650 |
+
rtc_configuration=rtc_config
|
| 651 |
+
)
|
| 652 |
+
|
| 653 |
+
webrtc_model = gr.Dropdown(
|
| 654 |
+
choices=["mediapipe-lite", "movenet-lightning", "yolo-v11-n"],
|
| 655 |
+
value="mediapipe-lite",
|
| 656 |
+
label="Pose Model",
|
| 657 |
+
info="Optimized for real-time processing"
|
| 658 |
+
)
|
| 659 |
+
|
| 660 |
+
confidence_slider = gr.Slider(
|
| 661 |
+
label="Detection Confidence",
|
| 662 |
+
minimum=0.0,
|
| 663 |
+
maximum=1.0,
|
| 664 |
+
step=0.05,
|
| 665 |
+
value=0.5,
|
| 666 |
+
info="Higher = fewer false positives"
|
| 667 |
+
)
|
| 668 |
+
|
| 669 |
+
# Official WebRTC streaming setup following Gradio guide
|
| 670 |
+
webrtc_stream.stream(
|
| 671 |
+
fn=webrtc_detection,
|
| 672 |
+
inputs=[webrtc_stream, webrtc_model, confidence_slider],
|
| 673 |
+
outputs=[webrtc_stream],
|
| 674 |
+
time_limit=10 # Following official guide: 10 seconds per user
|
| 675 |
+
)
|
| 676 |
+
|
| 677 |
+
# Info display
|
| 678 |
+
gr.HTML(\"\"\"
|
| 679 |
+
<div style="background: #e8f4fd; padding: 15px; border-radius: 8px; margin-top: 10px;">
|
| 680 |
+
<h4>📹 WebRTC Pose Analysis</h4>
|
| 681 |
+
<p style="margin: 5px 0;">Real-time movement analysis using your webcam</p>
|
| 682 |
+
|
| 683 |
+
<h4>🔒 Privacy</h4>
|
| 684 |
+
<p style="margin: 5px 0;">Processing happens locally - no video data stored</p>
|
| 685 |
+
|
| 686 |
+
<h4>💡 Usage</h4>
|
| 687 |
+
<ul style="margin: 5px 0; padding-left: 20px;">
|
| 688 |
+
<li>Grant camera permission when prompted</li>
|
| 689 |
+
<li>Move in front of camera to see pose detection</li>
|
| 690 |
+
<li>Adjust confidence threshold as needed</li>
|
| 691 |
+
</ul>
|
| 692 |
+
</div>
|
| 693 |
+
\"\"\")
|
| 694 |
+
|
| 695 |
+
else:
|
| 696 |
+
# Fallback if WebRTC component not available
|
| 697 |
+
gr.HTML(\"\"\"
|
| 698 |
+
<div style="text-align: center; padding: 50px; border: 2px dashed #ff6b6b; border-radius: 8px; background: #ffe0e0;">
|
| 699 |
+
<h3>📦 WebRTC Component Required</h3>
|
| 700 |
+
<p><strong>To enable real-time camera analysis, install:</strong></p>
|
| 701 |
+
<code style="background: #f0f0f0; padding: 10px; border-radius: 4px; display: block; margin: 10px 0;">
|
| 702 |
+
pip install gradio-webrtc twilio
|
| 703 |
+
</code>
|
| 704 |
+
<p style="margin-top: 15px;"><em>Use Enhanced Analysis tab for video files meanwhile</em></p>
|
| 705 |
+
</div>
|
| 706 |
+
\"\"\")
|
| 707 |
+
|
| 708 |
+
# Tab 5: Model Comparison
|
| 709 |
+
with gr.Tab("⚖️ Model Comparison"):
|
| 710 |
+
gr.Markdown(\"\"\"
|
| 711 |
+
### Compare Pose Estimation Models
|
| 712 |
+
Analyze the same video with different models to compare accuracy and results.
|
| 713 |
+
\"\"\")
|
| 714 |
+
|
| 715 |
+
with gr.Column():
|
| 716 |
+
comparison_video = gr.Video(
|
| 717 |
+
label="Video for Comparison",
|
| 718 |
+
sources=["upload"]
|
| 719 |
+
)
|
| 720 |
+
|
| 721 |
+
with gr.Row():
|
| 722 |
+
model1_comp = gr.Dropdown(
|
| 723 |
+
choices=["mediapipe-full", "movenet-thunder", "yolo-v11-s"],
|
| 724 |
+
value="mediapipe-full",
|
| 725 |
+
label="Model 1"
|
| 726 |
+
)
|
| 727 |
+
|
| 728 |
+
model2_comp = gr.Dropdown(
|
| 729 |
+
choices=["mediapipe-full", "movenet-thunder", "yolo-v11-s"],
|
| 730 |
+
value="yolo-v11-s",
|
| 731 |
+
label="Model 2"
|
| 732 |
+
)
|
| 733 |
+
|
| 734 |
+
compare_btn = gr.Button("🔄 Compare Models", variant="primary")
|
| 735 |
+
|
| 736 |
+
comparison_results = gr.DataFrame(
|
| 737 |
+
headers=["Metric", "Model 1", "Model 2", "Match"],
|
| 738 |
+
label="Comparison Results"
|
| 739 |
+
)
|
| 740 |
+
|
| 741 |
+
compare_btn.click(
|
| 742 |
+
fn=compare_models,
|
| 743 |
+
inputs=[comparison_video, model1_comp, model2_comp],
|
| 744 |
+
outputs=comparison_results,
|
| 745 |
+
api_name="compare_models"
|
| 746 |
+
)
|
| 747 |
+
|
| 748 |
+
# Tab 6: Documentation
|
| 749 |
+
with gr.Tab("📚 Documentation"):
|
| 750 |
+
gr.Markdown(\"\"\"
|
| 751 |
+
# Complete Feature Documentation
|
| 752 |
+
|
| 753 |
+
## 🎥 Video Input Support
|
| 754 |
+
- **Local Files**: MP4, AVI, MOV, WebM formats
|
| 755 |
+
- **YouTube**: Automatic download from YouTube URLs
|
| 756 |
+
- **Vimeo**: Automatic download from Vimeo URLs
|
| 757 |
+
- **Direct URLs**: Any direct video file URL
|
| 758 |
+
|
| 759 |
+
## 🤖 Pose Estimation Models
|
| 760 |
+
|
| 761 |
+
### MediaPipe (Google) - 33 3D Landmarks
|
| 762 |
+
- **Lite**: Fastest CPU performance
|
| 763 |
+
- **Full**: Balanced accuracy/speed (recommended)
|
| 764 |
+
- **Heavy**: Highest accuracy
|
| 765 |
+
|
| 766 |
+
### MoveNet (Google) - 17 COCO Keypoints
|
| 767 |
+
- **Lightning**: Mobile-optimized, very fast
|
| 768 |
+
- **Thunder**: Higher accuracy variant
|
| 769 |
+
|
| 770 |
+
### YOLO (Ultralytics) - 17 COCO Keypoints
|
| 771 |
+
- **v8 variants**: n/s/m/l/x sizes (nano to extra-large)
|
| 772 |
+
- **v11 variants**: Latest with improved accuracy (n/s/m/l/x)
|
| 773 |
+
- **Multi-person**: Supports multiple people in frame
|
| 774 |
+
|
| 775 |
+
## 📹 Real-time WebRTC
|
| 776 |
+
|
| 777 |
+
- **Live Camera**: Direct webcam access via WebRTC
|
| 778 |
+
- **Low Latency**: Sub-100ms processing
|
| 779 |
+
- **Adaptive Quality**: Automatic performance optimization
|
| 780 |
+
- **Live Overlay**: Real-time pose and metrics display
|
| 781 |
+
|
| 782 |
+
## 🤖 Agent & MCP Integration
|
| 783 |
+
|
| 784 |
+
### API Endpoints
|
| 785 |
+
- `/analyze_standard` - Basic LMA analysis
|
| 786 |
+
- `/analyze_enhanced` - Advanced analysis with all features
|
| 787 |
+
- `/analyze_agent` - Agent-optimized output
|
| 788 |
+
- `/batch_analyze` - Multiple video processing
|
| 789 |
+
- `/filter_videos` - Movement-based filtering
|
| 790 |
+
- `/compare_models` - Model comparison
|
| 791 |
+
|
| 792 |
+
### MCP Server
|
| 793 |
+
```bash
|
| 794 |
+
# Start MCP server for AI assistants
|
| 795 |
+
python -m backend.mcp_server
|
| 796 |
+
```
|
| 797 |
+
|
| 798 |
+
### Python API
|
| 799 |
+
```python
|
| 800 |
+
from gradio_labanmovementanalysis import LabanMovementAnalysis
|
| 801 |
+
|
| 802 |
+
# Initialize with all features
|
| 803 |
+
analyzer = LabanMovementAnalysis(
|
| 804 |
+
enable_webrtc=True
|
| 805 |
+
)
|
| 806 |
+
|
| 807 |
+
# Analyze YouTube video
|
| 808 |
+
result, viz = analyzer.process_video(
|
| 809 |
+
"https://youtube.com/watch?v=...",
|
| 810 |
+
model="yolo-v11-s"
|
| 811 |
+
)
|
| 812 |
+
```
|
| 813 |
+
|
| 814 |
+
## 📊 Output Formats
|
| 815 |
+
|
| 816 |
+
### Summary Format
|
| 817 |
+
Human-readable movement analysis summary.
|
| 818 |
+
|
| 819 |
+
### Structured Format
|
| 820 |
+
```json
|
| 821 |
+
{
|
| 822 |
+
"success": true,
|
| 823 |
+
"direction": "up",
|
| 824 |
+
"intensity": "medium",
|
| 825 |
+
"fluidity": 0.85,
|
| 826 |
+
"expansion": 0.72
|
| 827 |
+
}
|
| 828 |
+
```
|
| 829 |
+
|
| 830 |
+
### Full JSON Format
|
| 831 |
+
Complete frame-by-frame analysis with all metrics.
|
| 832 |
+
|
| 833 |
+
## 🎯 Applications
|
| 834 |
+
|
| 835 |
+
- **Sports**: Technique analysis and performance tracking
|
| 836 |
+
- **Dance**: Choreography analysis and movement quality
|
| 837 |
+
- **Healthcare**: Physical therapy and rehabilitation
|
| 838 |
+
- **Research**: Large-scale movement pattern studies
|
| 839 |
+
- **Entertainment**: Interactive applications and games
|
| 840 |
+
- **Education**: Movement teaching and body awareness
|
| 841 |
+
|
| 842 |
+
## 🔗 Integration Examples
|
| 843 |
+
|
| 844 |
+
### Gradio Client
|
| 845 |
+
```python
|
| 846 |
+
from gradio_client import Client
|
| 847 |
+
|
| 848 |
+
client = Client("http://localhost:7860")
|
| 849 |
+
result = client.predict(
|
| 850 |
+
video="path/to/video.mp4",
|
| 851 |
+
model="mediapipe-full",
|
| 852 |
+
api_name="/analyze_enhanced"
|
| 853 |
+
)
|
| 854 |
+
```
|
| 855 |
+
|
| 856 |
+
### Batch Processing
|
| 857 |
+
```python
|
| 858 |
+
results = client.predict(
|
| 859 |
+
files=["video1.mp4", "video2.mp4"],
|
| 860 |
+
model="yolo-v11-s",
|
| 861 |
+
api_name="/batch_analyze"
|
| 862 |
+
)
|
| 863 |
+
```
|
| 864 |
+
\"\"\")
|
| 865 |
+
gr.HTML(\"\"\"
|
| 866 |
+
<div class="author-info">
|
| 867 |
+
<p><strong>Created by:</strong> Csaba Bolyós (BladeSzaSza)</p>
|
| 868 |
+
<p style="margin: 5px 0;">
|
| 869 |
+
<a href="https://github.com/bladeszasza" style="color: #a8e6cf; text-decoration: none;">🔗 GitHub</a> •
|
| 870 |
+
<a href="https://huggingface.co/BladeSzaSza" style="color: #a8e6cf; text-decoration: none;">🤗 Hugging Face</a> •
|
| 871 |
+
<a href="https://www.linkedin.com/in/csaba-bolyós-00a11767/" style="color: #a8e6cf; text-decoration: none;">💼 LinkedIn</a>
|
| 872 |
+
</p>
|
| 873 |
+
<p style="font-size: 12px; opacity: 0.9;">Contact: [email protected]</p>
|
| 874 |
+
</div>
|
| 875 |
+
\"\"\")
|
| 876 |
+
|
| 877 |
+
# Footer with proper attribution
|
| 878 |
+
gr.HTML(\"\"\"
|
| 879 |
+
<div style="text-align: center; padding: 20px; margin-top: 30px; border-top: 1px solid #eee;">
|
| 880 |
+
<p style="color: #666; margin-bottom: 10px;">
|
| 881 |
+
🎭 Laban Movement Analysis - Complete Suite | Heavy Beta Version
|
| 882 |
+
</p>
|
| 883 |
+
<p style="color: #666; font-size: 12px;">
|
| 884 |
+
Created by <strong>Csaba Bolyós</strong> | Powered by MediaPipe, MoveNet & YOLO
|
| 885 |
+
</p>
|
| 886 |
+
<p style="color: #666; font-size: 10px; margin-top: 10px;">
|
| 887 |
+
<a href="https://github.com/bladeszasza" style="color: #40826D;">GitHub</a> •
|
| 888 |
+
<a href="https://huggingface.co/BladeSzaSza" style="color: #40826D;">Hugging Face</a> •
|
| 889 |
+
<a href="https://www.linkedin.com/in/csaba-bolyós-00a11767/" style="color: #40826D;">LinkedIn</a>
|
| 890 |
+
</p>
|
| 891 |
+
</div>
|
| 892 |
+
\"\"\")
|
| 893 |
+
|
| 894 |
+
return demo
|
| 895 |
+
|
| 896 |
+
|
| 897 |
+
if __name__ == "__main__":
|
| 898 |
+
demo = create_unified_demo()
|
| 899 |
+
demo.launch(
|
| 900 |
+
server_name="0.0.0.0",
|
| 901 |
+
server_port=7860,
|
| 902 |
+
share=False,
|
| 903 |
+
show_error=True,
|
| 904 |
+
favicon_path=None
|
| 905 |
+
)
|
| 906 |
+
|
| 907 |
+
```
|
| 908 |
+
""", elem_classes=["md-custom"], header_links=True)
|
| 909 |
+
|
| 910 |
+
|
| 911 |
+
gr.Markdown("""
|
| 912 |
+
## `LabanMovementAnalysis`
|
| 913 |
+
|
| 914 |
+
### Initialization
|
| 915 |
+
""", elem_classes=["md-custom"], header_links=True)
|
| 916 |
+
|
| 917 |
+
gr.ParamViewer(value=_docs["LabanMovementAnalysis"]["members"]["__init__"], linkify=[])
|
| 918 |
+
|
| 919 |
+
|
| 920 |
+
|
| 921 |
+
|
| 922 |
+
gr.Markdown("""
|
| 923 |
+
|
| 924 |
+
### User function
|
| 925 |
+
|
| 926 |
+
The impact on the users predict function varies depending on whether the component is used as an input or output for an event (or both).
|
| 927 |
+
|
| 928 |
+
- When used as an Input, the component only impacts the input signature of the user function.
|
| 929 |
+
- When used as an output, the component only impacts the return signature of the user function.
|
| 930 |
+
|
| 931 |
+
The code snippet below is accurate in cases where the component is used as both an input and an output.
|
| 932 |
+
|
| 933 |
+
- **As input:** Is passed, processed data for analysis.
|
| 934 |
+
- **As output:** Should return, analysis results.
|
| 935 |
+
|
| 936 |
+
```python
|
| 937 |
+
def predict(
|
| 938 |
+
value: typing.Dict[str, typing.Any][str, typing.Any]
|
| 939 |
+
) -> typing.Any:
|
| 940 |
+
return value
|
| 941 |
+
```
|
| 942 |
+
""", elem_classes=["md-custom", "LabanMovementAnalysis-user-fn"], header_links=True)
|
| 943 |
+
|
| 944 |
+
|
| 945 |
+
|
| 946 |
+
|
| 947 |
+
demo.load(None, js=r"""function() {
|
| 948 |
+
const refs = {};
|
| 949 |
+
const user_fn_refs = {
|
| 950 |
+
LabanMovementAnalysis: [], };
|
| 951 |
+
requestAnimationFrame(() => {
|
| 952 |
+
|
| 953 |
+
Object.entries(user_fn_refs).forEach(([key, refs]) => {
|
| 954 |
+
if (refs.length > 0) {
|
| 955 |
+
const el = document.querySelector(`.${key}-user-fn`);
|
| 956 |
+
if (!el) return;
|
| 957 |
+
refs.forEach(ref => {
|
| 958 |
+
el.innerHTML = el.innerHTML.replace(
|
| 959 |
+
new RegExp("\\b"+ref+"\\b", "g"),
|
| 960 |
+
`<a href="#h-${ref.toLowerCase()}">${ref}</a>`
|
| 961 |
+
);
|
| 962 |
+
})
|
| 963 |
+
}
|
| 964 |
+
})
|
| 965 |
+
|
| 966 |
+
Object.entries(refs).forEach(([key, refs]) => {
|
| 967 |
+
if (refs.length > 0) {
|
| 968 |
+
const el = document.querySelector(`.${key}`);
|
| 969 |
+
if (!el) return;
|
| 970 |
+
refs.forEach(ref => {
|
| 971 |
+
el.innerHTML = el.innerHTML.replace(
|
| 972 |
+
new RegExp("\\b"+ref+"\\b", "g"),
|
| 973 |
+
`<a href="#h-${ref.toLowerCase()}">${ref}</a>`
|
| 974 |
+
);
|
| 975 |
+
})
|
| 976 |
+
}
|
| 977 |
+
})
|
| 978 |
+
})
|
| 979 |
+
}
|
| 980 |
+
|
| 981 |
+
""")
|
| 982 |
+
|
| 983 |
+
demo.launch()
|
demo/test_component.py
ADDED
|
@@ -0,0 +1,69 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""
|
| 2 |
+
Test script to verify the Laban Movement Analysis component structure.
|
| 3 |
+
"""
|
| 4 |
+
|
| 5 |
+
import sys
|
| 6 |
+
from pathlib import Path
|
| 7 |
+
|
| 8 |
+
# Add parent directory to path
|
| 9 |
+
sys.path.insert(0, str(Path(__file__).parent.parent / "backend"))
|
| 10 |
+
|
| 11 |
+
# Test imports
|
| 12 |
+
try:
|
| 13 |
+
from gradio_labanmovementanalysis import LabanMovementAnalysis
|
| 14 |
+
print("✓ LabanMovementAnalysis component imported successfully")
|
| 15 |
+
|
| 16 |
+
from gradio_labanmovementanalysis import video_utils
|
| 17 |
+
print("✓ video_utils module imported successfully")
|
| 18 |
+
|
| 19 |
+
from gradio_labanmovementanalysis import pose_estimation
|
| 20 |
+
print("✓ pose_estimation module imported successfully")
|
| 21 |
+
|
| 22 |
+
from gradio_labanmovementanalysis import notation_engine
|
| 23 |
+
print("✓ notation_engine module imported successfully")
|
| 24 |
+
|
| 25 |
+
from gradio_labanmovementanalysis import json_generator
|
| 26 |
+
print("✓ json_generator module imported successfully")
|
| 27 |
+
|
| 28 |
+
from gradio_labanmovementanalysis import visualizer
|
| 29 |
+
print("✓ visualizer module imported successfully")
|
| 30 |
+
|
| 31 |
+
except ImportError as e:
|
| 32 |
+
print(f"✗ Import error: {e}")
|
| 33 |
+
sys.exit(1)
|
| 34 |
+
|
| 35 |
+
# Test component instantiation
|
| 36 |
+
try:
|
| 37 |
+
component = LabanMovementAnalysis()
|
| 38 |
+
print("\n✓ Component instantiated successfully")
|
| 39 |
+
|
| 40 |
+
# Test component methods
|
| 41 |
+
example_payload = component.example_payload()
|
| 42 |
+
print(f"✓ Example payload: {example_payload}")
|
| 43 |
+
|
| 44 |
+
example_value = component.example_value()
|
| 45 |
+
print(f"✓ Example value keys: {list(example_value.keys())}")
|
| 46 |
+
|
| 47 |
+
api_info = component.api_info()
|
| 48 |
+
print(f"✓ API info type: {api_info['type']}")
|
| 49 |
+
|
| 50 |
+
except Exception as e:
|
| 51 |
+
print(f"✗ Component error: {e}")
|
| 52 |
+
sys.exit(1)
|
| 53 |
+
|
| 54 |
+
# Test data structures
|
| 55 |
+
try:
|
| 56 |
+
from gradio_labanmovementanalysis.pose_estimation import Keypoint, PoseResult
|
| 57 |
+
kp = Keypoint(x=0.5, y=0.5, confidence=0.9, name="nose")
|
| 58 |
+
print(f"\n✓ Keypoint created: {kp}")
|
| 59 |
+
|
| 60 |
+
from gradio_labanmovementanalysis.notation_engine import Direction, Speed, Intensity
|
| 61 |
+
print(f"✓ Direction values: {[d.value for d in Direction]}")
|
| 62 |
+
print(f"✓ Speed values: {[s.value for s in Speed]}")
|
| 63 |
+
print(f"✓ Intensity values: {[i.value for i in Intensity]}")
|
| 64 |
+
|
| 65 |
+
except Exception as e:
|
| 66 |
+
print(f"✗ Data structure error: {e}")
|
| 67 |
+
sys.exit(1)
|
| 68 |
+
|
| 69 |
+
print("\n✅ All tests passed! The component is properly structured.")
|
dist/gradio_labanmovementanalysis-0.0.1-py3-none-any.whl
ADDED
|
Binary file (45.5 kB). View file
|
|
|
dist/gradio_labanmovementanalysis-0.0.1.tar.gz
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:c357282c4575834cf159824141a52f9d6fc99113c48438c70b24b0bcb41add51
|
| 3 |
+
size 84525422
|
dist/gradio_labanmovementanalysis-0.0.2-py3-none-any.whl
ADDED
|
Binary file (45.5 kB). View file
|
|
|
dist/gradio_labanmovementanalysis-0.0.2.tar.gz
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:e161cca0ea5443885301b3362b9b8a7553fa06622dcdef7a62a03a30d15271ef
|
| 3 |
+
size 84525440
|
examples/agent_example.py
ADDED
|
@@ -0,0 +1,142 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""
|
| 2 |
+
Example script demonstrating agent-friendly API usage
|
| 3 |
+
for Laban Movement Analysis
|
| 4 |
+
"""
|
| 5 |
+
|
| 6 |
+
import sys
|
| 7 |
+
from pathlib import Path
|
| 8 |
+
|
| 9 |
+
# Add parent to path
|
| 10 |
+
sys.path.insert(0, str(Path(__file__).parent.parent / "backend"))
|
| 11 |
+
|
| 12 |
+
from gradio_labanmovementanalysis import (
|
| 13 |
+
LabanAgentAPI,
|
| 14 |
+
PoseModel,
|
| 15 |
+
MovementDirection,
|
| 16 |
+
MovementIntensity,
|
| 17 |
+
quick_analyze,
|
| 18 |
+
analyze_and_summarize
|
| 19 |
+
)
|
| 20 |
+
|
| 21 |
+
|
| 22 |
+
def main():
|
| 23 |
+
"""Demonstrate various agent API features"""
|
| 24 |
+
|
| 25 |
+
print("🎭 Laban Movement Analysis - Agent API Examples\n")
|
| 26 |
+
|
| 27 |
+
# Example video paths (replace with your own)
|
| 28 |
+
video_path = "examples/dance.mp4"
|
| 29 |
+
|
| 30 |
+
# 1. Quick analysis with summary
|
| 31 |
+
print("1. Quick Analysis with Summary")
|
| 32 |
+
print("-" * 40)
|
| 33 |
+
summary = analyze_and_summarize(video_path)
|
| 34 |
+
print(summary)
|
| 35 |
+
print()
|
| 36 |
+
|
| 37 |
+
# 2. Detailed analysis with structured output
|
| 38 |
+
print("2. Detailed Analysis")
|
| 39 |
+
print("-" * 40)
|
| 40 |
+
api = LabanAgentAPI()
|
| 41 |
+
result = api.analyze(video_path, generate_visualization=True)
|
| 42 |
+
|
| 43 |
+
if result.success:
|
| 44 |
+
print(f"✓ Analysis successful!")
|
| 45 |
+
print(f" Direction: {result.dominant_direction.value}")
|
| 46 |
+
print(f" Intensity: {result.dominant_intensity.value}")
|
| 47 |
+
print(f" Speed: {result.dominant_speed}")
|
| 48 |
+
print(f" Fluidity: {result.fluidity_score:.2f}")
|
| 49 |
+
print(f" Expansion: {result.expansion_score:.2f}")
|
| 50 |
+
print(f" Segments: {len(result.movement_segments)}")
|
| 51 |
+
if result.visualization_path:
|
| 52 |
+
print(f" Visualization saved to: {result.visualization_path}")
|
| 53 |
+
else:
|
| 54 |
+
print(f"✗ Analysis failed: {result.error}")
|
| 55 |
+
print()
|
| 56 |
+
|
| 57 |
+
# 3. Batch processing example
|
| 58 |
+
print("3. Batch Processing")
|
| 59 |
+
print("-" * 40)
|
| 60 |
+
video_paths = [
|
| 61 |
+
"examples/dance.mp4",
|
| 62 |
+
"examples/exercise.mp4",
|
| 63 |
+
"examples/walking.mp4"
|
| 64 |
+
]
|
| 65 |
+
|
| 66 |
+
# Filter out non-existent files
|
| 67 |
+
existing_paths = [p for p in video_paths if Path(p).exists()]
|
| 68 |
+
|
| 69 |
+
if existing_paths:
|
| 70 |
+
results = api.batch_analyze(existing_paths, parallel=True)
|
| 71 |
+
for i, result in enumerate(results):
|
| 72 |
+
print(f"Video {i+1}: {Path(result.video_path).name}")
|
| 73 |
+
if result.success:
|
| 74 |
+
print(f" ✓ {result.dominant_direction.value} movement, "
|
| 75 |
+
f"{result.dominant_intensity.value} intensity")
|
| 76 |
+
else:
|
| 77 |
+
print(f" ✗ Failed: {result.error}")
|
| 78 |
+
else:
|
| 79 |
+
print(" No example videos found")
|
| 80 |
+
print()
|
| 81 |
+
|
| 82 |
+
# 4. Movement filtering example
|
| 83 |
+
print("4. Movement Filtering")
|
| 84 |
+
print("-" * 40)
|
| 85 |
+
if existing_paths and len(existing_paths) > 1:
|
| 86 |
+
# Find high-intensity movements
|
| 87 |
+
high_intensity = api.filter_by_movement(
|
| 88 |
+
existing_paths,
|
| 89 |
+
intensity=MovementIntensity.HIGH,
|
| 90 |
+
min_fluidity=0.5
|
| 91 |
+
)
|
| 92 |
+
|
| 93 |
+
print(f"Found {len(high_intensity)} high-intensity videos:")
|
| 94 |
+
for result in high_intensity:
|
| 95 |
+
print(f" - {Path(result.video_path).name}: "
|
| 96 |
+
f"fluidity={result.fluidity_score:.2f}")
|
| 97 |
+
print()
|
| 98 |
+
|
| 99 |
+
# 5. Video comparison example
|
| 100 |
+
print("5. Video Comparison")
|
| 101 |
+
print("-" * 40)
|
| 102 |
+
if len(existing_paths) >= 2:
|
| 103 |
+
comparison = api.compare_videos(existing_paths[0], existing_paths[1])
|
| 104 |
+
print(f"Comparing: {comparison['video1']} vs {comparison['video2']}")
|
| 105 |
+
print(f" Direction match: {comparison['metrics']['direction_match']}")
|
| 106 |
+
print(f" Intensity match: {comparison['metrics']['intensity_match']}")
|
| 107 |
+
print(f" Fluidity difference: {comparison['metrics']['fluidity_difference']:.2f}")
|
| 108 |
+
print()
|
| 109 |
+
|
| 110 |
+
# 6. Model comparison
|
| 111 |
+
print("6. Model Comparison")
|
| 112 |
+
print("-" * 40)
|
| 113 |
+
if existing_paths:
|
| 114 |
+
test_video = existing_paths[0]
|
| 115 |
+
models = [PoseModel.MEDIAPIPE, PoseModel.MOVENET]
|
| 116 |
+
|
| 117 |
+
for model in models:
|
| 118 |
+
result = api.analyze(test_video, model=model)
|
| 119 |
+
if result.success:
|
| 120 |
+
print(f"{model.value}: {result.dominant_direction.value} "
|
| 121 |
+
f"({result.dominant_intensity.value})")
|
| 122 |
+
|
| 123 |
+
|
| 124 |
+
def async_example():
|
| 125 |
+
"""Example of async usage"""
|
| 126 |
+
import asyncio
|
| 127 |
+
|
| 128 |
+
async def analyze_async():
|
| 129 |
+
api = LabanAgentAPI()
|
| 130 |
+
result = await api.analyze_async("examples/dance.mp4")
|
| 131 |
+
return api.get_movement_summary(result)
|
| 132 |
+
|
| 133 |
+
# Run async example
|
| 134 |
+
summary = asyncio.run(analyze_async())
|
| 135 |
+
print("Async Analysis:", summary)
|
| 136 |
+
|
| 137 |
+
|
| 138 |
+
if __name__ == "__main__":
|
| 139 |
+
main()
|
| 140 |
+
|
| 141 |
+
# Uncomment to run async example
|
| 142 |
+
# async_example()
|
frontend/Example.svelte
ADDED
|
@@ -0,0 +1,19 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
<script lang="ts">
|
| 2 |
+
export let value: string;
|
| 3 |
+
export let type: "gallery" | "table";
|
| 4 |
+
export let selected = false;
|
| 5 |
+
</script>
|
| 6 |
+
|
| 7 |
+
<div
|
| 8 |
+
class:table={type === "table"}
|
| 9 |
+
class:gallery={type === "gallery"}
|
| 10 |
+
class:selected
|
| 11 |
+
>
|
| 12 |
+
{value}
|
| 13 |
+
</div>
|
| 14 |
+
|
| 15 |
+
<style>
|
| 16 |
+
.gallery {
|
| 17 |
+
padding: var(--size-1) var(--size-2);
|
| 18 |
+
}
|
| 19 |
+
</style>
|
frontend/Index.svelte
ADDED
|
@@ -0,0 +1,37 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
<script lang="ts">
|
| 2 |
+
import { JsonView } from "@zerodevx/svelte-json-view";
|
| 3 |
+
|
| 4 |
+
import type { Gradio } from "@gradio/utils";
|
| 5 |
+
import { Block, Info } from "@gradio/atoms";
|
| 6 |
+
import { StatusTracker } from "@gradio/statustracker";
|
| 7 |
+
import type { LoadingStatus } from "@gradio/statustracker";
|
| 8 |
+
import type { SelectData } from "@gradio/utils";
|
| 9 |
+
|
| 10 |
+
export let elem_id = "";
|
| 11 |
+
export let elem_classes: string[] = [];
|
| 12 |
+
export let visible = true;
|
| 13 |
+
export let value = false;
|
| 14 |
+
export let container = true;
|
| 15 |
+
export let scale: number | null = null;
|
| 16 |
+
export let min_width: number | undefined = undefined;
|
| 17 |
+
export let loading_status: LoadingStatus;
|
| 18 |
+
export let gradio: Gradio<{
|
| 19 |
+
change: never;
|
| 20 |
+
select: SelectData;
|
| 21 |
+
input: never;
|
| 22 |
+
clear_status: LoadingStatus;
|
| 23 |
+
}>;
|
| 24 |
+
</script>
|
| 25 |
+
|
| 26 |
+
<Block {visible} {elem_id} {elem_classes} {container} {scale} {min_width}>
|
| 27 |
+
{#if loading_status}
|
| 28 |
+
<StatusTracker
|
| 29 |
+
autoscroll={gradio.autoscroll}
|
| 30 |
+
i18n={gradio.i18n}
|
| 31 |
+
{...loading_status}
|
| 32 |
+
on:clear_status={() => gradio.dispatch("clear_status", loading_status)}
|
| 33 |
+
/>
|
| 34 |
+
{/if}
|
| 35 |
+
|
| 36 |
+
<JsonView json={value} />
|
| 37 |
+
</Block>
|
frontend/gradio.config.js
ADDED
|
@@ -0,0 +1,9 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
export default {
|
| 2 |
+
plugins: [],
|
| 3 |
+
svelte: {
|
| 4 |
+
preprocess: [],
|
| 5 |
+
},
|
| 6 |
+
build: {
|
| 7 |
+
target: "modules",
|
| 8 |
+
},
|
| 9 |
+
};
|
frontend/package-lock.json
ADDED
|
The diff for this file is too large to render.
See raw diff
|
|
|
frontend/package.json
ADDED
|
@@ -0,0 +1,40 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"name": "gradio_labanmovementanalysis",
|
| 3 |
+
"version": "0.4.22",
|
| 4 |
+
"description": "Gradio UI packages",
|
| 5 |
+
"type": "module",
|
| 6 |
+
"author": "",
|
| 7 |
+
"license": "ISC",
|
| 8 |
+
"private": false,
|
| 9 |
+
"main_changeset": true,
|
| 10 |
+
"exports": {
|
| 11 |
+
".": {
|
| 12 |
+
"gradio": "./Index.svelte",
|
| 13 |
+
"svelte": "./dist/Index.svelte",
|
| 14 |
+
"types": "./dist/Index.svelte.d.ts"
|
| 15 |
+
},
|
| 16 |
+
"./example": {
|
| 17 |
+
"gradio": "./Example.svelte",
|
| 18 |
+
"svelte": "./dist/Example.svelte",
|
| 19 |
+
"types": "./dist/Example.svelte.d.ts"
|
| 20 |
+
},
|
| 21 |
+
"./package.json": "./package.json"
|
| 22 |
+
},
|
| 23 |
+
"dependencies": {
|
| 24 |
+
"@gradio/atoms": "0.16.1",
|
| 25 |
+
"@gradio/statustracker": "0.10.12",
|
| 26 |
+
"@gradio/utils": "0.10.2",
|
| 27 |
+
"@zerodevx/svelte-json-view": "^1.0.7"
|
| 28 |
+
},
|
| 29 |
+
"devDependencies": {
|
| 30 |
+
"@gradio/preview": "0.13.0"
|
| 31 |
+
},
|
| 32 |
+
"peerDependencies": {
|
| 33 |
+
"svelte": "^4.0.0"
|
| 34 |
+
},
|
| 35 |
+
"repository": {
|
| 36 |
+
"type": "git",
|
| 37 |
+
"url": "git+https://github.com/gradio-app/gradio.git",
|
| 38 |
+
"directory": "js/fallback"
|
| 39 |
+
}
|
| 40 |
+
}
|
mcp.json
ADDED
|
@@ -0,0 +1,57 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"mcpServers": {
|
| 3 |
+
"laban-movement-analysis": {
|
| 4 |
+
"command": "python",
|
| 5 |
+
"args": ["-m", "backend.mcp_server"],
|
| 6 |
+
"env": {
|
| 7 |
+
"PYTHONPATH": "."
|
| 8 |
+
},
|
| 9 |
+
"schema": {
|
| 10 |
+
"name": "Laban Movement Analysis",
|
| 11 |
+
"description": "Analyze human movement in videos using pose estimation and Laban Movement Analysis metrics",
|
| 12 |
+
"version": "1.0.0",
|
| 13 |
+
"tools": [
|
| 14 |
+
{
|
| 15 |
+
"name": "analyze_video",
|
| 16 |
+
"description": "Analyze movement in a video file",
|
| 17 |
+
"parameters": {
|
| 18 |
+
"video_path": "string",
|
| 19 |
+
"model": "string (optional)",
|
| 20 |
+
"enable_visualization": "boolean (optional)",
|
| 21 |
+
"include_keypoints": "boolean (optional)"
|
| 22 |
+
}
|
| 23 |
+
},
|
| 24 |
+
{
|
| 25 |
+
"name": "get_analysis_summary",
|
| 26 |
+
"description": "Get human-readable summary of analysis",
|
| 27 |
+
"parameters": {
|
| 28 |
+
"analysis_id": "string"
|
| 29 |
+
}
|
| 30 |
+
},
|
| 31 |
+
{
|
| 32 |
+
"name": "list_available_models",
|
| 33 |
+
"description": "List available pose estimation models",
|
| 34 |
+
"parameters": {}
|
| 35 |
+
},
|
| 36 |
+
{
|
| 37 |
+
"name": "batch_analyze",
|
| 38 |
+
"description": "Analyze multiple videos in batch",
|
| 39 |
+
"parameters": {
|
| 40 |
+
"video_paths": "array of strings",
|
| 41 |
+
"model": "string (optional)",
|
| 42 |
+
"parallel": "boolean (optional)"
|
| 43 |
+
}
|
| 44 |
+
},
|
| 45 |
+
{
|
| 46 |
+
"name": "compare_movements",
|
| 47 |
+
"description": "Compare movement patterns between videos",
|
| 48 |
+
"parameters": {
|
| 49 |
+
"analysis_id1": "string",
|
| 50 |
+
"analysis_id2": "string"
|
| 51 |
+
}
|
| 52 |
+
}
|
| 53 |
+
]
|
| 54 |
+
}
|
| 55 |
+
}
|
| 56 |
+
}
|
| 57 |
+
}
|
pyproject.toml
ADDED
|
@@ -0,0 +1,62 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
[build-system]
|
| 2 |
+
requires = [
|
| 3 |
+
"hatchling",
|
| 4 |
+
"hatch-requirements-txt",
|
| 5 |
+
"hatch-fancy-pypi-readme>=22.5.0",
|
| 6 |
+
]
|
| 7 |
+
build-backend = "hatchling.build"
|
| 8 |
+
|
| 9 |
+
[project]
|
| 10 |
+
name = "gradio_labanmovementanalysis"
|
| 11 |
+
version = "0.0.2"
|
| 12 |
+
description = "A Gradio 5 component for video movement analysis using Laban Movement Analysis (LMA) with MCP support for AI agents"
|
| 13 |
+
readme = "README.md"
|
| 14 |
+
license = "apache-2.0"
|
| 15 |
+
requires-python = ">=3.10"
|
| 16 |
+
authors = [{ name = "Csaba Bolyós", email = "[email protected]" }]
|
| 17 |
+
keywords = ["gradio-custom-component", "gradio-5", "laban-movement-analysis", "LMA", "pose-estimation", "movement-analysis", "mcp", "ai-agents", "webrtc"]
|
| 18 |
+
# Core dependencies
|
| 19 |
+
dependencies = [
|
| 20 |
+
"gradio>=5.0.0",
|
| 21 |
+
"opencv-python>=4.8.0",
|
| 22 |
+
"numpy>=1.24.0",
|
| 23 |
+
"mediapipe>=0.10.21",
|
| 24 |
+
"tensorflow>=2.13.0",
|
| 25 |
+
"ultralytics>=8.0.0",
|
| 26 |
+
"torch>=2.0.0",
|
| 27 |
+
"torchvision>=0.15.0"
|
| 28 |
+
]
|
| 29 |
+
classifiers = [
|
| 30 |
+
'Development Status :: 4 - Beta',
|
| 31 |
+
'Operating System :: OS Independent',
|
| 32 |
+
'Programming Language :: Python :: 3',
|
| 33 |
+
'Programming Language :: Python :: 3 :: Only',
|
| 34 |
+
'Programming Language :: Python :: 3.10',
|
| 35 |
+
'Programming Language :: Python :: 3.11',
|
| 36 |
+
'Programming Language :: Python :: 3.12',
|
| 37 |
+
'Topic :: Scientific/Engineering',
|
| 38 |
+
'Topic :: Scientific/Engineering :: Artificial Intelligence',
|
| 39 |
+
'Topic :: Scientific/Engineering :: Visualization',
|
| 40 |
+
]
|
| 41 |
+
|
| 42 |
+
# The repository and space URLs are optional, but recommended.
|
| 43 |
+
# Adding a repository URL will create a badge in the auto-generated README that links to the repository.
|
| 44 |
+
# Adding a space URL will create a badge in the auto-generated README that links to the space.
|
| 45 |
+
# This will make it easy for people to find your deployed demo or source code when they
|
| 46 |
+
# encounter your project in the wild.
|
| 47 |
+
|
| 48 |
+
# [project.urls]
|
| 49 |
+
# repository = "your github repository"
|
| 50 |
+
# space = "your space url"
|
| 51 |
+
|
| 52 |
+
[project.optional-dependencies]
|
| 53 |
+
dev = ["build", "twine"]
|
| 54 |
+
mcp = ["mcp>=1.0.0", "aiofiles>=23.0.0", "httpx>=0.24.0"]
|
| 55 |
+
agent = ["gradio-client>=1.0.0"]
|
| 56 |
+
all = ["gradio_labanmovementanalysis[mcp,agent]"]
|
| 57 |
+
|
| 58 |
+
[tool.hatch.build]
|
| 59 |
+
artifacts = ["/backend/gradio_labanmovementanalysis/templates", "*.pyi"]
|
| 60 |
+
|
| 61 |
+
[tool.hatch.build.targets.wheel]
|
| 62 |
+
packages = ["/backend/gradio_labanmovementanalysis"]
|
requirements.txt
ADDED
|
@@ -0,0 +1,49 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Laban Movement Analysis - Complete Suite
|
| 2 |
+
# Created by: Csaba Bolyós (BladeSzaSza)
|
| 3 |
+
# Heavy Beta Version
|
| 4 |
+
|
| 5 |
+
# Core Gradio and UI
|
| 6 |
+
gradio>=5.0.0
|
| 7 |
+
gradio-webrtc>=0.0.31
|
| 8 |
+
|
| 9 |
+
# Computer Vision and Pose Estimation
|
| 10 |
+
opencv-python>=4.8.0
|
| 11 |
+
mediapipe>=0.10.21
|
| 12 |
+
ultralytics>=8.0.0
|
| 13 |
+
|
| 14 |
+
# Scientific Computing
|
| 15 |
+
numpy>=1.21.0,<2.0.0
|
| 16 |
+
scipy>=1.7.0
|
| 17 |
+
pandas>=1.3.0
|
| 18 |
+
|
| 19 |
+
# Image and Video Processing
|
| 20 |
+
Pillow>=8.3.0
|
| 21 |
+
imageio>=2.19.0
|
| 22 |
+
imageio-ffmpeg>=0.4.7
|
| 23 |
+
moviepy>=1.0.3
|
| 24 |
+
|
| 25 |
+
# Machine Learning
|
| 26 |
+
torch>=2.0.0
|
| 27 |
+
torchvision>=0.15.0
|
| 28 |
+
tensorflow>=2.10.0
|
| 29 |
+
|
| 30 |
+
# WebRTC and Streaming
|
| 31 |
+
twilio>=8.2.0
|
| 32 |
+
aiortc>=1.4.0
|
| 33 |
+
av>=10.0.0
|
| 34 |
+
|
| 35 |
+
# Utilities
|
| 36 |
+
requests>=2.28.0
|
| 37 |
+
yt-dlp>=2023.1.6
|
| 38 |
+
tqdm>=4.64.0
|
| 39 |
+
matplotlib>=3.5.0
|
| 40 |
+
seaborn>=0.11.0
|
| 41 |
+
|
| 42 |
+
# Development and Deployment
|
| 43 |
+
python-multipart>=0.0.5
|
| 44 |
+
uvicorn>=0.18.0
|
| 45 |
+
fastapi>=0.95.0
|
| 46 |
+
|
| 47 |
+
# Optional WebRTC dependencies
|
| 48 |
+
aiohttp>=3.8.0
|
| 49 |
+
websockets>=10.0
|