|
|
<!DOCTYPE html> |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
<html lang="en"> |
|
|
|
|
|
<head></head> |
|
|
<meta charset="UTF-8"> |
|
|
<meta name="viewport" content="width=device-width, initial-scale=1.0, maximum-scale=1.0, user-scalable=0" /> |
|
|
|
|
|
<title>Sign Language Interpreter</title> |
|
|
|
|
|
|
|
|
<script> |
|
|
window.console = window.console || function (t) { }; |
|
|
</script> |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
<link rel="stylesheet" type="text/css" href="static/browser_detect.css" /> |
|
|
|
|
|
|
|
|
</head> |
|
|
|
|
|
<body translate="no" style="overflow: scroll;"> |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
<script src="https://cdn.jsdelivr.net/npm/@mediapipe/drawing_utils/drawing_utils.js" |
|
|
crossorigin="anonymous"></script> |
|
|
<script src="https://cdn.jsdelivr.net/npm/@mediapipe/hands/hands.js" crossorigin="anonymous"></script> |
|
|
<script src="https://cdn.jsdelivr.net/npm/@tensorflow/tfjs-core"></script> |
|
|
<script src="https://cdn.jsdelivr.net/npm/@tensorflow/tfjs-backend-cpu"></script> |
|
|
<script src="https://cdn.jsdelivr.net/npm/@tensorflow/tfjs-tflite/dist/tf-tflite.min.js"></script> |
|
|
<script src="https://cdn.jsdelivr.net/npm/@mediapipe/[email protected]/wasm/vision_wasm_internal.js" |
|
|
crossorigin="anonymous"></script> |
|
|
|
|
|
<section id="home-page"> |
|
|
<div class="container"> |
|
|
|
|
|
<video id="webcam" style="display:none" autoplay playsinline muted></video> |
|
|
<div class="canvas_wrapper" id="canvas_wrapper"> |
|
|
<button id="switch-camera" |
|
|
style="display:none; position: absolute; top:10px; left:10px; padding:5px; height:40px; width:40px; text-align: center; border-radius: 12.25px; font-size: 20px; font-weight: 900; border:none; background-color: #ffffff2e; backdrop-filter: blur(30px); color:black; box-shadow: 0px 4px 20px 4px rgba(0, 0, 0, 0.38); z-index:100"> |
|
|
<span>⟳</span> |
|
|
</button> |
|
|
<canvas class="output_canvas" id="output_canvas" width="100%" height="500%"></canvas> |
|
|
<center> |
|
|
<button id="webcamButton" style="font-weight: 600;"> |
|
|
<span>Enable Webcam</span> |
|
|
</button> |
|
|
<div class="mode_switch_wrapper"> |
|
|
<div class="mode_switch" id="mode-switch" style="display: none;"> |
|
|
<div class="mode_selector" id="modeSelector"></div> |
|
|
<div class="mode_switch_modeBtn">Text</div> |
|
|
<div class="mode_switch_modeBtn">Word</div> |
|
|
</div> |
|
|
</div> |
|
|
</center> |
|
|
|
|
|
</div> |
|
|
</div> |
|
|
<center> |
|
|
<img id="output_image" style="display:none"></img> |
|
|
|
|
|
<div class="wrapper_result"> |
|
|
<div id="predicted_result">></div> |
|
|
</div> |
|
|
<div class="wrapper_text"> |
|
|
<textarea id="text" onkeyup="set_output_array(this.value)"></textarea> |
|
|
<div class="textControls"> |
|
|
<button id="undoButton" onclick="undo()"> |
|
|
<svg xmlns="http://www.w3.org/2000/svg" height="16px" width="16px" |
|
|
viewBox="0 0 576 512"> |
|
|
<path fill="#fcfcfc" |
|
|
d="M576 128c0-35.3-28.7-64-64-64L205.3 64c-17 0-33.3 6.7-45.3 18.7L9.4 233.4c-6 6-9.4 14.1-9.4 22.6s3.4 16.6 9.4 22.6L160 429.3c12 12 28.3 18.7 45.3 18.7L512 448c35.3 0 64-28.7 64-64l0-256zM271 175c9.4-9.4 24.6-9.4 33.9 0l47 47 47-47c9.4-9.4 24.6-9.4 33.9 0s9.4 24.6 0 33.9l-47 47 47 47c9.4 9.4 9.4 24.6 0 33.9s-24.6 9.4-33.9 0l-47-47-47 47c-9.4 9.4-24.6 9.4-33.9 0s-9.4-24.6 0-33.9l47-47-47-47c-9.4-9.4-9.4-24.6 0-33.9z" /> |
|
|
</svg> |
|
|
</button> |
|
|
<button id="speakButton" onclick="speak(document.getElementById('text').value)"> |
|
|
<span>Listen</span> |
|
|
<svg xmlns="http://www.w3.org/2000/svg" height="16px" width="16px" |
|
|
viewBox="0 0 640 512"> |
|
|
<path fill="#fcfcfc" |
|
|
d="M533.6 32.5C598.5 85.2 640 165.8 640 256s-41.5 170.7-106.4 223.5c-10.3 8.4-25.4 6.8-33.8-3.5s-6.8-25.4 3.5-33.8C557.5 398.2 592 331.2 592 256s-34.5-142.2-88.7-186.3c-10.3-8.4-11.8-23.5-3.5-33.8s23.5-11.8 33.8-3.5zM473.1 107c43.2 35.2 70.9 88.9 70.9 149s-27.7 113.8-70.9 149c-10.3 8.4-25.4 6.8-33.8-3.5s-6.8-25.4 3.5-33.8C475.3 341.3 496 301.1 496 256s-20.7-85.3-53.2-111.8c-10.3-8.4-11.8-23.5-3.5-33.8s23.5-11.8 33.8-3.5zm-60.5 74.5C434.1 199.1 448 225.9 448 256s-13.9 56.9-35.4 74.5c-10.3 8.4-25.4 6.8-33.8-3.5s-6.8-25.4 3.5-33.8C393.1 284.4 400 271 400 256s-6.9-28.4-17.7-37.3c-10.3-8.4-11.8-23.5-3.5-33.8s23.5-11.8 33.8-3.5zM301.1 34.8C312.6 40 320 51.4 320 64l0 384c0 12.6-7.4 24-18.9 29.2s-25 3.1-34.4-5.3L131.8 352 64 352c-35.3 0-64-28.7-64-64l0-64c0-35.3 28.7-64 64-64l67.8 0L266.7 40.1c9.4-8.4 22.9-10.4 34.4-5.3z" /> |
|
|
</svg> |
|
|
</button> |
|
|
<button id="clearButton" onclick="clear_output_array()"> |
|
|
<svg xmlns="http://www.w3.org/2000/svg" height="16px" width="16px" |
|
|
viewBox="0 0 448 512"> |
|
|
<path fill="#fcfcfc" |
|
|
d="M135.2 17.7L128 32 32 32C14.3 32 0 46.3 0 64S14.3 96 32 96l384 0c17.7 0 32-14.3 32-32s-14.3-32-32-32l-96 0-7.2-14.3C307.4 6.8 296.3 0 284.2 0L163.8 0c-12.1 0-23.2 6.8-28.6 17.7zM416 128L32 128 53.2 467c1.6 25.3 22.6 45 47.9 45l245.8 0c25.3 0 46.3-19.7 47.9-45L416 128z" /> |
|
|
</svg> |
|
|
</button> |
|
|
</div> |
|
|
|
|
|
<audio id="audioPlayer">-</audio> |
|
|
</div> |
|
|
|
|
|
<div id="logUI" style="display:none"></div> |
|
|
</center> |
|
|
</section> |
|
|
<section id="info-page" style="display:none; padding: 10px;"> |
|
|
<center> |
|
|
|
|
|
<svg width="77.5" height="103" viewBox="0 0 310 412" fill="none" xmlns="http://www.w3.org/2000/svg"> |
|
|
<g filter="url(#filter0_d_21435_307)"> |
|
|
<path |
|
|
d="M186.81 198.496C176.153 230.73 130.699 230.73 120.042 198.496C118.566 194.034 117.995 189.322 118.361 184.635L129.694 27.7729C130.663 15.3457 140.999 5.75757 153.426 5.75757C165.852 5.75757 176.188 15.3457 177.158 27.7729L188.491 184.635C188.857 189.322 188.285 194.034 186.81 198.496Z" |
|
|
fill="#FF6E7D" /> |
|
|
<path |
|
|
d="M129.248 210.461C131.138 242.358 93.345 260.329 69.9261 238.67C65.7586 234.815 62.575 230.013 60.6432 224.667L21.653 116.772C17.3477 104.858 22.7777 91.6137 34.1929 86.1855C45.6081 80.7572 59.2664 84.9246 65.7351 95.8096L124.318 194.388C127.221 199.272 128.911 204.784 129.248 210.461Z" |
|
|
fill="#FF6E7D" /> |
|
|
<path |
|
|
d="M92.2772 198.028C93.9837 196.578 95.8085 195.273 97.7326 194.129L171.089 150.508C188.74 140.012 211.322 151.175 213.777 171.611C214.485 177.507 217.012 183.034 221.005 187.418L268.354 239.422C278.948 251.057 281.761 267.839 275.544 282.309L245.615 351.969C239.301 366.663 224.879 376.182 208.928 376.182H97.9241C81.9729 376.182 67.5505 366.663 61.2371 351.969L32.802 285.787C25.8462 269.597 30.2544 250.749 43.6623 239.352L92.2772 198.028Z" |
|
|
fill="#FF6E7D" /> |
|
|
<path |
|
|
d="M55.1238 304.632C26.6986 296.743 20.0237 259.356 43.9529 242.062C49.3081 238.192 55.6418 235.913 62.2284 235.486L152.085 229.664C164.75 228.844 176.01 237.693 178.247 250.224C180.485 262.756 172.986 274.975 160.824 278.614L74.5386 304.437C68.2137 306.33 61.4852 306.397 55.1238 304.632Z" |
|
|
fill="#D6505E" /> |
|
|
<path |
|
|
d="M239.546 153.001C228.433 125.986 190.518 125.338 178.413 151.957C176.038 157.178 175.042 162.922 175.519 168.637L183.682 266.336C184.68 278.282 194.526 287.521 206.477 287.725C218.427 287.929 228.609 279.033 230.048 267.128L241.82 169.769C242.509 164.074 241.726 158.3 239.546 153.001Z" |
|
|
fill="#D6505E" /> |
|
|
<path |
|
|
d="M286.006 191.73C274.39 169.568 243.251 168.174 229.541 189.203C225.491 195.415 223.646 202.808 224.305 210.184L229.849 272.21C230.925 284.253 240.697 293.621 252.745 294.16C264.794 294.699 275.429 286.245 277.666 274.35L289.188 213.088C290.559 205.803 289.437 198.277 286.006 191.73Z" |
|
|
fill="#A8404B" /> |
|
|
</g> |
|
|
<defs> |
|
|
<filter id="filter0_d_21435_307" x="0.181641" y="0.757568" width="309.576" height="410.424" |
|
|
filterUnits="userSpaceOnUse" color-interpolation-filters="sRGB"> |
|
|
<feFlood flood-opacity="0" result="BackgroundImageFix" /> |
|
|
<feColorMatrix in="SourceAlpha" type="matrix" values="0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 127 0" |
|
|
result="hardAlpha" /> |
|
|
<feOffset dy="15" /> |
|
|
<feGaussianBlur stdDeviation="10" /> |
|
|
<feComposite in2="hardAlpha" operator="out" /> |
|
|
<feColorMatrix type="matrix" values="0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0.3 0" /> |
|
|
<feBlend mode="normal" in2="BackgroundImageFix" result="effect1_dropShadow_21435_307" /> |
|
|
<feBlend mode="normal" in="SourceGraphic" in2="effect1_dropShadow_21435_307" result="shape" /> |
|
|
</filter> |
|
|
</defs> |
|
|
</svg> |
|
|
<h1>SpeakSign</h1> |
|
|
</center> |
|
|
<hr> |
|
|
<div class="about-card"> |
|
|
<span>Version</span> |
|
|
|
|
|
<span>1.2.5b</span> |
|
|
</div> |
|
|
<div class="about-card"> |
|
|
<span>Author</span> |
|
|
<span>Shantanu Khedkar</span> |
|
|
</div> |
|
|
<div class="about-card"> |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
<img src="static/ASL.png" style="height:auto; width:auto; margin: -5px"></img> |
|
|
</div> |
|
|
<div class="about-card"> |
|
|
<span>Source</span> |
|
|
<span><a href="https://github.com/Shantanu-Khedkar/silangint" |
|
|
target="_blank">https://github.com/Shantanu-Khedkar/silangint</a></span> |
|
|
</div> |
|
|
<div class="about-card"> |
|
|
<span>License</span> |
|
|
<span>GPL-v3.0 - <a href="https://github.com/Shantanu-Khedkar/silangint/blob/master/LICENSE" |
|
|
target="_blank">https://github.com/Shantanu-Khedkar/silangint/blob/master/LICENSE</a></span> |
|
|
</div> |
|
|
<div class="about-card"> |
|
|
<span>Copyright Notice</span> |
|
|
<span> |
|
|
SignSpeak - A Communicator For Signers |
|
|
<br> |
|
|
Copyright (C) 2025 Shantanu Khedkar |
|
|
<br> |
|
|
This program is free software: you can redistribute it and/or modify |
|
|
it under the terms of the GNU General Public License as published by |
|
|
the Free Software Foundation, either version 3 of the License, or |
|
|
(at your option) any later version. |
|
|
<br> |
|
|
This program is distributed in the hope that it will be useful, |
|
|
but WITHOUT ANY WARRANTY; without even the implied warranty of |
|
|
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the |
|
|
GNU General Public License for more details. |
|
|
<br> |
|
|
You should have received a copy of the GNU General Public License |
|
|
along with this program. If not, see https://www.gnu.org/licenses. |
|
|
</span> |
|
|
</div> |
|
|
|
|
|
</section> |
|
|
<section id="settings-page" style="display:none"> |
|
|
<div |
|
|
style="display:inline-flex; align-items:center; gap:10px; padding:12px; margin-bottom: 24px; margin-top: 16px;"> |
|
|
<div style="padding:10px; display: inherit;" onclick="settingsBack()"> |
|
|
<svg xmlns="http://www.w3.org/2000/svg" width="30px" height="30px" |
|
|
viewBox="0 0 320 512"> |
|
|
<path |
|
|
d="M41.4 233.4c-12.5 12.5-12.5 32.8 0 45.3l160 160c12.5 12.5 32.8 12.5 45.3 0s12.5-32.8 0-45.3L109.3 256 246.6 118.6c12.5-12.5 12.5-32.8 0-45.3s-32.8-12.5-45.3 0l-160 160z" /> |
|
|
</svg> |
|
|
</div> |
|
|
<span class="settings-header">Settings</span> |
|
|
</div> |
|
|
<form id="speechForm" style="display:block;"> |
|
|
<div class="about-card"> |
|
|
<label for="language">Select Language:</label> |
|
|
<select id="language"></select> |
|
|
</div> |
|
|
<div class="about-card"> |
|
|
<label for="voice">Select Voice:</label> |
|
|
<select id="voice"></select> |
|
|
</div> |
|
|
<div class="about-card"> |
|
|
<label for="rate">Rate:</label> |
|
|
<input type="range" id="rate" min="0.1" max="2" step="0.1" value="1"> |
|
|
<span style="display:none" id="rateValue">1</span> |
|
|
</div> |
|
|
<div class="about-card"> |
|
|
<label for="pitch">Pitch:</label> |
|
|
<input type="range" id="pitch" min="0" max="2" step="0.1" value="1"> |
|
|
<span style="display:none" id="pitchValue">1</span> |
|
|
</div> |
|
|
<div class="about-card"> |
|
|
<label for="testText">Test Voice:</label> |
|
|
<input id="testText" value="Hello, this is a test!" placeholder="Enter test sentence..." |
|
|
onkeyup="set_output_array(this.value)"></input> |
|
|
<button type="button" id="testSpeakButton" |
|
|
onclick="document.getElementById('text').value = document.getElementById('testText').value; document.getElementById('speakButton').click()"> |
|
|
<a>Listen 🔊</a> |
|
|
</button> |
|
|
</div> |
|
|
<div class="about-card"> |
|
|
<label for="duration">Input Duration <span id="durationValue"></span>(ms)</label> |
|
|
<input type="range" id="duration" min="200" max="1500" step="1" value="300" |
|
|
oninput="document.getElementById('durationValue').innerText=this.value"> |
|
|
</div> |
|
|
<div class="about-card"> |
|
|
<label for="repeat">Repeat Buffer <span id="repeatValue"></span>(ms)</label> |
|
|
<input type="range" id="repeat" min="200" max="700" step="1" value="500" |
|
|
oninput="document.getElementById('repeatValue').innerText=this.value"> |
|
|
</div> |
|
|
<div class="about-card" style="flex-direction:row; align-items: center;"> |
|
|
<label style="margin-right: 10px;" for="duration">Natural Input :</label> |
|
|
|
|
|
<label class="switch"> |
|
|
<input type="checkbox" id="natInp" unchecked> |
|
|
<span class="slider round"></span> |
|
|
</label> |
|
|
</div> |
|
|
</form> |
|
|
</section> |
|
|
<div class="bottom-nav"> |
|
|
<div id="info-" class="bottom-nav-btn"> |
|
|
<svg xmlns="http://www.w3.org/2000/svg" height="28px" width="28px" |
|
|
viewBox="0 0 512 512"> |
|
|
<path fill="#5d5d5d" |
|
|
d="M256 512A256 256 0 1 0 256 0a256 256 0 1 0 0 512zM216 336l24 0 0-64-24 0c-13.3 0-24-10.7-24-24s10.7-24 24-24l48 0c13.3 0 24 10.7 24 24l0 88 8 0c13.3 0 24 10.7 24 24s-10.7 24-24 24l-80 0c-13.3 0-24-10.7-24-24s10.7-24 24-24zm40-208a32 32 0 1 1 0 64 32 32 0 1 1 0-64z" /> |
|
|
</svg> |
|
|
</div> |
|
|
<div id="home-" class="bottom-nav-btn center-btn"> |
|
|
|
|
|
|
|
|
|
|
|
<img src="/static/logo_sil.svg" height="48" width="48px"></img> |
|
|
</div> |
|
|
<div id="settings-" class="bottom-nav-btn"> |
|
|
<svg xmlns="http://www.w3.org/2000/svg" height="28px" width="28px" |
|
|
viewBox="0 0 512 512"> |
|
|
<path fill="#5d5d5d" |
|
|
d="M495.9 166.6c3.2 8.7 .5 18.4-6.4 24.6l-43.3 39.4c1.1 8.3 1.7 16.8 1.7 25.4s-.6 17.1-1.7 25.4l43.3 39.4c6.9 6.2 9.6 15.9 6.4 24.6c-4.4 11.9-9.7 23.3-15.8 34.3l-4.7 8.1c-6.6 11-14 21.4-22.1 31.2c-5.9 7.2-15.7 9.6-24.5 6.8l-55.7-17.7c-13.4 10.3-28.2 18.9-44 25.4l-12.5 57.1c-2 9.1-9 16.3-18.2 17.8c-13.8 2.3-28 3.5-42.5 3.5s-28.7-1.2-42.5-3.5c-9.2-1.5-16.2-8.7-18.2-17.8l-12.5-57.1c-15.8-6.5-30.6-15.1-44-25.4L83.1 425.9c-8.8 2.8-18.6 .3-24.5-6.8c-8.1-9.8-15.5-20.2-22.1-31.2l-4.7-8.1c-6.1-11-11.4-22.4-15.8-34.3c-3.2-8.7-.5-18.4 6.4-24.6l43.3-39.4C64.6 273.1 64 264.6 64 256s.6-17.1 1.7-25.4L22.4 191.2c-6.9-6.2-9.6-15.9-6.4-24.6c4.4-11.9 9.7-23.3 15.8-34.3l4.7-8.1c6.6-11 14-21.4 22.1-31.2c5.9-7.2 15.7-9.6 24.5-6.8l55.7 17.7c13.4-10.3 28.2-18.9 44-25.4l12.5-57.1c2-9.1 9-16.3 18.2-17.8C227.3 1.2 241.5 0 256 0s28.7 1.2 42.5 3.5c9.2 1.5 16.2 8.7 18.2 17.8l12.5 57.1c15.8 6.5 30.6 15.1 44 25.4l55.7-17.7c8.8-2.8 18.6-.3 24.5 6.8c8.1 9.8 15.5 20.2 22.1 31.2l4.7 8.1c6.1 11 11.4 22.4 15.8 34.3zM256 336a80 80 0 1 0 0-160 80 80 0 1 0 0 160z" /> |
|
|
</svg> |
|
|
</div> |
|
|
|
|
|
</div> |
|
|
<script> |
|
|
var word_list = [] |
|
|
var speechSupported = true |
|
|
var prevSpeech = "" |
|
|
var prevSettings = "" |
|
|
|
|
|
logUI = document.getElementById("logUI") |
|
|
const speechSynthesis = window.speechSynthesis; |
|
|
const voiceSelect = document.getElementById('voice'); |
|
|
const textInput = document.getElementById('text'); |
|
|
const languageSelect = document.getElementById('language'); |
|
|
const speakButton = document.getElementById('speakButton'); |
|
|
const rateInput = document.getElementById('rate'); |
|
|
const pitchInput = document.getElementById('pitch'); |
|
|
const rateValue = document.getElementById('rateValue'); |
|
|
const pitchValue = document.getElementById('pitchValue'); |
|
|
|
|
|
let voices = []; |
|
|
|
|
|
const languageNames = { |
|
|
'en': '- English ', |
|
|
'mra': '- Marathi', |
|
|
'mr': '- Marathi', |
|
|
'hi': '- Hindi', |
|
|
'es': 'Spanish', |
|
|
'fr': 'French', |
|
|
'de': 'German', |
|
|
'it': 'Italian', |
|
|
'af': 'Afrikaans', |
|
|
'am': 'Amharic', |
|
|
'an': 'Aragonese', |
|
|
'ar': 'Arabic', |
|
|
'as': 'Assamese', |
|
|
'az': 'Azerbaijani', |
|
|
'bg': 'Bulgarian', |
|
|
'bn': 'Bengali', |
|
|
'bpy': 'Bishnupriya Manipuri', |
|
|
'bs': 'Bosnian', |
|
|
'ca': 'Catalan', |
|
|
'cs': 'Czech', |
|
|
'cy': 'Welsh', |
|
|
'da': 'Danish', |
|
|
'de': 'German', |
|
|
'el': 'Greek', |
|
|
'eo': 'Esperanto', |
|
|
'es': 'Spanish', |
|
|
'et': 'Estonian', |
|
|
'eu': 'Basque', |
|
|
'fa': 'Persian', |
|
|
'fi': 'Finnish', |
|
|
'fr': 'French', |
|
|
'ga': 'Irish', |
|
|
'gd': 'Scottish Gaelic', |
|
|
'gn': 'Guarani', |
|
|
'grc': 'Ancient Greek', |
|
|
'gu': 'Gujarati', |
|
|
'hak': 'Hakka', |
|
|
'hr': 'Croatian', |
|
|
'ht': 'Haitian Creole', |
|
|
'hu': 'Hungarian', |
|
|
'hy': 'Armenian', |
|
|
'ia': 'Interlingua', |
|
|
'id': 'Indonesian', |
|
|
'is': 'Icelandic', |
|
|
'it': 'Italian', |
|
|
'ja': 'Japanese', |
|
|
'jbo': 'Lojban', |
|
|
'ka': 'Georgian', |
|
|
'kk': 'Kazakh', |
|
|
'kl': 'Kalaallisut', |
|
|
'kn': 'Kannada', |
|
|
'ko': 'Korean', |
|
|
'kok': 'Konkani', |
|
|
'ku': 'Kurdish', |
|
|
'ky': 'Kyrgyz', |
|
|
'la': 'Latin', |
|
|
'lfn': 'Lingua Franca Nova', |
|
|
'lt': 'Lithuanian', |
|
|
'lv': 'Latvian', |
|
|
'mi': 'Māori', |
|
|
'mk': 'Macedonian', |
|
|
'ml': 'Malayalam', |
|
|
'ms': 'Malay', |
|
|
'mt': 'Maltese', |
|
|
'my': 'Burmese', |
|
|
'nci': 'Navajo', |
|
|
'ne': 'Nepali', |
|
|
'nl': 'Dutch', |
|
|
'no': 'Norwegian', |
|
|
'om': 'Oromo', |
|
|
'or': 'Odia', |
|
|
'pa': 'Punjabi', |
|
|
'pap': 'Papiamento', |
|
|
'pl': 'Polish', |
|
|
'pt': 'Portuguese', |
|
|
'quc': 'K’iche’', |
|
|
'ro': 'Romanian', |
|
|
'ru': 'Russian', |
|
|
'sd': 'Sindhi', |
|
|
'shn': 'Shan', |
|
|
'si': 'Sinhala', |
|
|
'sk': 'Slovak', |
|
|
'sl': 'Slovenian', |
|
|
'sq': 'Albanian', |
|
|
'sr': 'Serbian', |
|
|
'sv': 'Swedish', |
|
|
'sw': 'Swahili', |
|
|
'ta': 'Tamil', |
|
|
'te': 'Telugu', |
|
|
'tn': 'Tswana', |
|
|
'tr': 'Turkish', |
|
|
'tt': 'Tatar', |
|
|
'ur': 'Urdu', |
|
|
'vi': 'Vietnamese' |
|
|
|
|
|
}; |
|
|
function undo() { |
|
|
word_list.pop() |
|
|
textInput.value = word_list.join('') |
|
|
} |
|
|
function clear() { |
|
|
word_list = [] |
|
|
textInput.value = '' |
|
|
} |
|
|
function populateVoiceList() { |
|
|
voices = speechSynthesis.getVoices(); |
|
|
updateLanguageList(); |
|
|
updateVoiceList(); |
|
|
} |
|
|
async function populateTTWVoices() { |
|
|
const url = "http://127.0.0.1:8125/assets/static/voicesList.txt"; |
|
|
try { |
|
|
const response = await fetch(url); |
|
|
if (!response.ok) { |
|
|
throw new Error(`Response status: ${response.status}`); |
|
|
} |
|
|
|
|
|
var voicesList = await response.text(); |
|
|
voicesList = voicesList.split('\n') |
|
|
voicesList.pop() |
|
|
voicesList.forEach(voice => { |
|
|
const option = document.createElement('option'); |
|
|
option.value = voice; |
|
|
option.textContent = voice; |
|
|
voiceSelect.appendChild(option); |
|
|
}) |
|
|
} catch (error) { |
|
|
console.error(error.message); |
|
|
} |
|
|
} |
|
|
async function populateTTWLangs() { |
|
|
const url = "http://127.0.0.1:8125/assets/static/langList.txt"; |
|
|
try { |
|
|
const response = await fetch(url); |
|
|
if (!response.ok) { |
|
|
throw new Error(`Response status: ${response.status}`); |
|
|
} |
|
|
|
|
|
var langsList = await response.text(); |
|
|
langsList = langsList.split('\n') |
|
|
langsList.pop() |
|
|
unknownLangs = [] |
|
|
langsList.forEach(lang => { |
|
|
if (!languageNames[lang]) { |
|
|
unknownLangs.push(lang) |
|
|
} else { |
|
|
const option = document.createElement('option'); |
|
|
option.value = lang; |
|
|
option.textContent = languageNames[lang]; |
|
|
languageSelect.appendChild(option); |
|
|
} |
|
|
}) |
|
|
unknownLangs.forEach(lang => { |
|
|
const option = document.createElement('option'); |
|
|
option.value = lang; |
|
|
option.textContent = lang; |
|
|
languageSelect.appendChild(option); |
|
|
}) |
|
|
} catch (error) { |
|
|
console.error(error.message); |
|
|
} |
|
|
|
|
|
const optionNodes = Array.from(languageSelect.children); |
|
|
const comparator = new Intl.Collator('en'.slice(0, 2)).compare; |
|
|
optionNodes.sort((a, b) => comparator(a.textContent, b.textContent)); |
|
|
optionNodes.forEach((option) => languageSelect.appendChild(option)); |
|
|
languageSelect.children[0].selected = "selected" |
|
|
} |
|
|
|
|
|
function getBaseLanguageCode(lang) { |
|
|
return lang.split('-').slice(0, 2).join('-'); |
|
|
} |
|
|
function getBaseBaseLanguageCode(lang) { |
|
|
return lang.split('-').slice(0, 1).join('-'); |
|
|
} |
|
|
function getModifierLanguageCode(lang) { |
|
|
return lang.split('-').slice(1, 3).join('-'); |
|
|
} |
|
|
|
|
|
function updateLanguageList() { |
|
|
const uniqueLanguages = new Set(voices.map(voice => getBaseLanguageCode(voice.lang))); |
|
|
languageSelect.innerHTML = ''; |
|
|
|
|
|
Object.keys(languageNames).forEach(lang => { |
|
|
if (uniqueLanguages.has(lang)) { |
|
|
const option = document.createElement('option'); |
|
|
option.value = lang; |
|
|
option.textContent = languageNames[lang]; |
|
|
languageSelect.appendChild(option); |
|
|
} |
|
|
}); |
|
|
|
|
|
uniqueLanguages.forEach(lang => { |
|
|
if (!languageNames[lang]) { |
|
|
if (!languageNames[getBaseBaseLanguageCode(lang)]) { |
|
|
const option = document.createElement('option'); |
|
|
option.value = lang; |
|
|
option.textContent = lang; |
|
|
languageSelect.appendChild(option); |
|
|
} |
|
|
else { |
|
|
const option = document.createElement('option'); |
|
|
option.value = lang; |
|
|
option.textContent = `${languageNames[getBaseBaseLanguageCode(lang)]} (${getModifierLanguageCode(lang)})`; |
|
|
languageSelect.appendChild(option); |
|
|
} |
|
|
} |
|
|
}); |
|
|
|
|
|
const optionNodes = Array.from(languageSelect.children); |
|
|
const comparator = new Intl.Collator('en'.slice(0, 2)).compare; |
|
|
optionNodes.sort((a, b) => comparator(a.textContent, b.textContent)); |
|
|
optionNodes.forEach((option) => languageSelect.appendChild(option)); |
|
|
languageSelect.children[0].selected = "selected" |
|
|
} |
|
|
|
|
|
function updateVoiceList() { |
|
|
const selectedLanguage = languageSelect.value; |
|
|
voiceSelect.innerHTML = ''; |
|
|
|
|
|
voices.forEach((voice) => { |
|
|
if (getBaseLanguageCode(voice.lang) === selectedLanguage) { |
|
|
const option = document.createElement('option'); |
|
|
option.value = voice.name; |
|
|
option.textContent = `${voice.name} (${voice.lang})`; |
|
|
voiceSelect.appendChild(option); |
|
|
} |
|
|
}); |
|
|
} |
|
|
function settingsBack() { |
|
|
document.getElementById("home-").click() |
|
|
} |
|
|
function logMessage(msg) { |
|
|
const span = document.createElement('span'); |
|
|
span.textContent = msg; |
|
|
logUI.appendChild(span); |
|
|
logUI.appendChild(document.createElement('br')); |
|
|
} |
|
|
|
|
|
const originalFetch = window.fetch; |
|
|
|
|
|
|
|
|
window.fetch = async function (input, init) { |
|
|
|
|
|
const url = typeof input === 'string' ? input : input.url; |
|
|
var newUrl = url |
|
|
if (url == 'https://cdn.jsdelivr.net/npm/@mediapipe/[email protected]/wasm/vision_wasm_internal.wasm') { |
|
|
|
|
|
newUrl = 'https://cdn.jsdelivr.net/npm/@mediapipe/[email protected]/wasm/vision_wasm_internal.wasm' |
|
|
|
|
|
} |
|
|
console.log("This was FETCHED: ", newUrl) |
|
|
|
|
|
return originalFetch(newUrl, init); |
|
|
}; |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
if ('speechSynthesis' in window) { |
|
|
speechSynthesis.onvoiceschanged = () => { |
|
|
populateVoiceList(); |
|
|
}; |
|
|
|
|
|
languageSelect.addEventListener('change', updateVoiceList); |
|
|
|
|
|
rateInput.addEventListener('input', () => { |
|
|
rateValue.textContent = rateInput.value; |
|
|
}); |
|
|
|
|
|
pitchInput.addEventListener('input', () => { |
|
|
pitchValue.textContent = pitchInput.value; |
|
|
}); |
|
|
|
|
|
speakButton.addEventListener('click', () => { |
|
|
const utterance = new SpeechSynthesisUtterance(textInput.value); |
|
|
const selectedVoice = voices.find(voice => voice.name === voiceSelect.value); |
|
|
utterance.voice = selectedVoice; |
|
|
utterance.rate = rateInput.value; |
|
|
utterance.pitch = pitchInput.value; |
|
|
speechSynthesis.speak(utterance); |
|
|
}); |
|
|
|
|
|
populateVoiceList() |
|
|
|
|
|
|
|
|
} else { |
|
|
speechSupported = false; |
|
|
console.log('Text-to-speech not supported.'); |
|
|
|
|
|
populateTTWLangs() |
|
|
populateTTWVoices() |
|
|
} |
|
|
|
|
|
function speak(toSpeak) { |
|
|
console.log("speech api support", speechSupported) |
|
|
console.log("condition: ", !speechSupported) |
|
|
console.log("condition2: ", speechSupported == false) |
|
|
console.log("speech api support", speechSupported) |
|
|
if (!speechSupported) { |
|
|
const audioPlayer = document.getElementById('audioPlayer'); |
|
|
var currSettings = '&v=' + encodeURIComponent(languageSelect.value + "+" + voiceSelect.value) + '&r=' + Math.round(rateInput.value * 7.5).toString() + '&p=' + Math.round(pitchInput.value * 49.5).toString() |
|
|
if ((prevSpeech != toSpeak) || (prevSettings != currSettings)) { |
|
|
prevSpeech = toSpeak |
|
|
prevSettings = currSettings |
|
|
audioPlayer.src = 'http://127.0.0.1:8125/speech?t=' + encodeURIComponent(toSpeak) + currSettings |
|
|
console.log("Set src: ", audioPlayer.src) |
|
|
} |
|
|
|
|
|
audioPlayer.play() |
|
|
.then(() => { |
|
|
|
|
|
console.log('Audio is playing'); |
|
|
}) |
|
|
.catch(error => { |
|
|
console.error('Error playing audio:', error); |
|
|
prevSpeech = '' |
|
|
}); |
|
|
} else if ('speechSynthesis' in window) { |
|
|
console.log("probably spoken") |
|
|
} else { |
|
|
console.log("Text to speech is now not supported") |
|
|
} |
|
|
} |
|
|
|
|
|
|
|
|
|
|
|
function set_output_array(text) { |
|
|
console.log(text) |
|
|
word_list = text.split(""); |
|
|
console.log(word_list) |
|
|
} |
|
|
function clear_output_array() { |
|
|
word_list = []; |
|
|
textInput.value = "" |
|
|
} |
|
|
|
|
|
</script> |
|
|
|
|
|
<script type="module"> |
|
|
document.getElementById("info-").addEventListener("click", switchPage.bind(null, "info-")); |
|
|
document.getElementById("home-").addEventListener("click", switchPage.bind(null, "home-")); |
|
|
document.getElementById("settings-").addEventListener("click", switchPage.bind(null, "settings-")); |
|
|
|
|
|
|
|
|
import { HandLandmarker, FilesetResolver } from "https://cdn.jsdelivr.net/npm/@mediapipe/[email protected]"; |
|
|
let handLandmarker = undefined; |
|
|
let runningMode = "IMAGE"; |
|
|
let enableWebcamButton; |
|
|
let webcamRunning = false; |
|
|
var time_since_letter = 0 |
|
|
var last_letter_time = 0 |
|
|
var maxPercentage = 1 |
|
|
|
|
|
function inputFrequency() { |
|
|
return document.getElementById('duration').value |
|
|
} |
|
|
function repeatBuffer() { |
|
|
return document.getElementById('repeat').value |
|
|
} |
|
|
document.getElementById('durationValue').innerText = inputFrequency() |
|
|
document.getElementById('repeatValue').innerText = repeatBuffer() |
|
|
var is_first_run = 1 |
|
|
const letter_list = ["A", "B", "", "D", "E", "F", "G", "", "I", "J", "K", "L", "M", "N", "O", "P", "", "R", "S", "T", "U", "V", "W", "", "Y", "Z", ""] |
|
|
const phrase_list = ["Ere-ọwurọ", "Bọkọ", "Bokọ???", "Ere-Alẹ", "Mo dọkpẹ", "Ẹrẹ"] |
|
|
|
|
|
var index_list = letter_list |
|
|
|
|
|
|
|
|
|
|
|
const createHandLandmarker = async () => { |
|
|
const vision = await FilesetResolver.forVisionTasks("https://cdn.jsdelivr.net/npm/@mediapipe/[email protected]/wasm"); |
|
|
handLandmarker = await HandLandmarker.createFromOptions(vision, { |
|
|
baseOptions: { |
|
|
|
|
|
modelAssetPath: `https://storage.googleapis.com/mediapipe-models/hand_landmarker/hand_landmarker/float16/1/hand_landmarker.task`, |
|
|
delegate: "GPU" |
|
|
}, |
|
|
runningMode: runningMode, |
|
|
numHands: 1 |
|
|
}); |
|
|
}; |
|
|
createHandLandmarker(); |
|
|
|
|
|
|
|
|
|
|
|
const MODEL_PATH = "/exported" |
|
|
const WORD_MODEL = "/word" |
|
|
|
|
|
const letterDetector = tflite.loadTFLiteModel(MODEL_PATH); |
|
|
const wordDetector = tflite.loadTFLiteModel(WORD_MODEL); |
|
|
var objectDetector = letterDetector |
|
|
|
|
|
|
|
|
|
|
|
document.getElementById("mode-switch").addEventListener('click', function () { |
|
|
document.getElementById('modeSelector').classList.toggle('right') |
|
|
var modeState = Array.from(document.getElementById('modeSelector').classList).includes('right') |
|
|
console.log("swicth", modeState) |
|
|
if (modeState) { |
|
|
console.log("word") |
|
|
objectDetector = wordDetector |
|
|
index_list = phrase_list |
|
|
} else { |
|
|
console.log("letter") |
|
|
objectDetector = letterDetector |
|
|
index_list = letter_list |
|
|
} |
|
|
}) |
|
|
var global_res = 0; |
|
|
var global_cres = '' |
|
|
const video = document.getElementById("webcam"); |
|
|
const canvasElement = document.getElementById("output_canvas"); |
|
|
const canvasCtx = canvasElement.getContext("2d", { willReadFrequently: true }); |
|
|
var x_array = [] |
|
|
var y_array = [] |
|
|
var video_facing_mode = "user" |
|
|
|
|
|
const hasGetUserMedia = () => { var _a; return !!((_a = navigator.mediaDevices) === null || _a === void 0 ? void 0 : _a.getUserMedia); }; |
|
|
|
|
|
|
|
|
if (hasGetUserMedia()) { |
|
|
enableWebcamButton = document.getElementById("webcamButton"); |
|
|
enableWebcamButton.addEventListener("click", enableCam); |
|
|
document.getElementById("switch-camera").addEventListener("click", switch_camera); |
|
|
} |
|
|
else { |
|
|
console.warn("getUserMedia() is not supported by your browser"); |
|
|
} |
|
|
async function switch_camera() { |
|
|
if (video_facing_mode == 'user') { |
|
|
webcamRunning = false |
|
|
video_facing_mode = 'environment' |
|
|
await load_camera() |
|
|
webcamRunning = true |
|
|
} |
|
|
else { |
|
|
webcamRunning = false |
|
|
video_facing_mode = 'user' |
|
|
await load_camera() |
|
|
webcamRunning = true |
|
|
} |
|
|
} |
|
|
|
|
|
function enableCam(event) { |
|
|
if (!handLandmarker) { |
|
|
console.log("Wait! objectDetector not loaded yet."); |
|
|
return; |
|
|
} |
|
|
if (webcamRunning === true) { |
|
|
webcamRunning = false; |
|
|
enableWebcamButton.innerText = "ENABLE PREDICTIONS"; |
|
|
} |
|
|
else { |
|
|
webcamRunning = true; |
|
|
enableWebcamButton.style = "display:none" |
|
|
document.getElementById("switch-camera").style.display = "block" |
|
|
document.getElementById("mode-switch").style.display = "flex" |
|
|
|
|
|
} |
|
|
|
|
|
load_camera() |
|
|
} |
|
|
|
|
|
function switchPage(elem) { |
|
|
prevSpeech = "" |
|
|
|
|
|
var pH = document.getElementById("home-page") |
|
|
var pI = document.getElementById("info-page") |
|
|
var pS = document.getElementById("settings-page") |
|
|
|
|
|
pH.style.display = "none" |
|
|
pI.style.display = "none" |
|
|
pS.style.display = "none" |
|
|
|
|
|
document.getElementById(elem + "page").style.display = "block" |
|
|
if (elem != "home-") { |
|
|
webcamRunning = false |
|
|
document.getElementById("webcamButton").style = "display:block" |
|
|
document.getElementById("switch-camera").style.display = "none" |
|
|
document.getElementById("mode-switch").style.display = "none" |
|
|
var canvas = document.getElementById("output_canvas") |
|
|
const context = canvas.getContext('2d'); |
|
|
context.clearRect(0, 0, canvas.width, canvas.height); |
|
|
} else { |
|
|
textInput.value = "" |
|
|
clear_output_array() |
|
|
var canvas = document.getElementById("output_canvas") |
|
|
const context = canvas.getContext('2d'); |
|
|
context.clearRect(0, 0, canvas.width, canvas.height); |
|
|
} |
|
|
} |
|
|
|
|
|
function load_camera() { |
|
|
try { |
|
|
var stream = video.srcObject; |
|
|
|
|
|
var tracks = stream.getTracks(); |
|
|
|
|
|
tracks.forEach(function (track) { |
|
|
|
|
|
track.stop(); |
|
|
}); |
|
|
|
|
|
video.srcObject = null; |
|
|
} catch (error) { |
|
|
console.error(error.message); |
|
|
} |
|
|
|
|
|
const constraints = { |
|
|
video: { |
|
|
facingMode: video_facing_mode |
|
|
} |
|
|
}; |
|
|
|
|
|
navigator.mediaDevices.getUserMedia(constraints) |
|
|
.then((stream) => { |
|
|
video.srcObject = stream; |
|
|
video.play(); |
|
|
video.addEventListener("loadeddata", predictWebcam); |
|
|
}) |
|
|
.catch((error) => { |
|
|
console.error("Error accessing the camera: ", error.name, error.message, error.code); |
|
|
}); |
|
|
} |
|
|
let lastVideoTime = -1; |
|
|
let results = undefined; |
|
|
console.log(video); |
|
|
async function predictWebcam() { |
|
|
if (video.videoHeight == 0) { |
|
|
return |
|
|
} |
|
|
canvasElement.width = window.innerWidth; |
|
|
|
|
|
if (runningMode === "IMAGE") { |
|
|
runningMode = "VIDEO"; |
|
|
await handLandmarker.setOptions({ runningMode: "VIDEO" }); |
|
|
} |
|
|
let startTimeMs = performance.now(); |
|
|
if (lastVideoTime !== video.currentTime) { |
|
|
lastVideoTime = video.currentTime; |
|
|
results = handLandmarker.detectForVideo(video, startTimeMs); |
|
|
} |
|
|
canvasCtx.save(); |
|
|
canvasCtx.clearRect(0, 0, canvasElement.width, canvasElement.height); |
|
|
|
|
|
if (is_first_run == 1) { |
|
|
var elem_rect = document.getElementById("output_canvas").getBoundingClientRect() |
|
|
console.log(elem_rect.height | 0); |
|
|
document.getElementById("canvas_wrapper").style.height = (elem_rect.height | 0).toString() + "px" |
|
|
|
|
|
is_first_run = 0 |
|
|
} |
|
|
|
|
|
if (results.landmarks && results.handednesses[0]) { |
|
|
var current_time = Math.round(Date.now()) |
|
|
|
|
|
if (results.handednesses[0][0].categoryName == "Left") { |
|
|
if (!Array.from(document.getElementById('modeSelector').classList).includes('right')) { |
|
|
annotateImage() |
|
|
} |
|
|
canvasCtx.drawImage(video, 0, 0, canvasElement.width, (video.videoHeight / video.videoWidth) * canvasElement.width) |
|
|
annotateImage(Array.from(document.getElementById('modeSelector').classList).includes('right')) |
|
|
console.log("LEFT") |
|
|
|
|
|
} else { |
|
|
canvasCtx.drawImage(video, 0, 0, canvasElement.width, (video.videoHeight / video.videoWidth) * canvasElement.width) |
|
|
console.log("RIGHT") |
|
|
var current_result = "_" |
|
|
var previous_result = document.getElementById("predicted_result").innerText |
|
|
document.getElementById("predicted_result").innerText = current_result |
|
|
|
|
|
|
|
|
if (previous_result == current_result) { |
|
|
if (current_time - last_letter_time > getNaturalLength(" ")) { |
|
|
last_letter_time = current_time |
|
|
word_list.push(" ") |
|
|
triggerPulse() |
|
|
console.log(word_list) |
|
|
document.getElementById("text").value = word_list.join('') |
|
|
} |
|
|
} |
|
|
else { |
|
|
last_letter_time = current_time |
|
|
} |
|
|
} |
|
|
document.getElementById("predicted_result").style.width = String((current_time - last_letter_time) / getNaturalLength(global_cres) * 100) + "%" |
|
|
} |
|
|
else { |
|
|
canvasCtx.drawImage(video, 0, 0, canvasElement.width, (video.videoHeight / video.videoWidth) * canvasElement.width) |
|
|
if (30 > calculateCanvasBrightness(canvasElement)) { |
|
|
|
|
|
var current_result = "<" |
|
|
var previous_result = document.getElementById("predicted_result").innerText |
|
|
document.getElementById("predicted_result").innerText = current_result |
|
|
var current_time = Math.round(Date.now()) |
|
|
console.log(current_time - last_letter_time) |
|
|
if (previous_result == current_result) { |
|
|
if (current_time - last_letter_time > 400) { |
|
|
last_letter_time = current_time |
|
|
word_list.pop() |
|
|
console.log(word_list) |
|
|
document.getElementById("text").value = word_list.join('') |
|
|
} |
|
|
} |
|
|
else { |
|
|
last_letter_time = current_time |
|
|
} |
|
|
} else { |
|
|
last_letter_time = Math.round(Date.now()) |
|
|
|
|
|
document.getElementById("predicted_result").style.width = String(0) + "%" |
|
|
} |
|
|
} |
|
|
|
|
|
canvasCtx.restore(); |
|
|
|
|
|
if (webcamRunning === true) { |
|
|
window.requestAnimationFrame(predictWebcam); |
|
|
} |
|
|
} |
|
|
function annotateImage(firstA = true) { |
|
|
|
|
|
|
|
|
if (results.landmarks[0]) { |
|
|
x_array = [] |
|
|
y_array = [] |
|
|
results.landmarks[0].forEach(iterate) |
|
|
|
|
|
var image_height = (video.videoHeight / video.videoWidth) * canvasElement.width |
|
|
var image_width = canvasElement.width |
|
|
var min_x = Math.min(...x_array) * image_width |
|
|
var min_y = Math.min(...y_array) * image_height |
|
|
var max_x = Math.max(...x_array) * image_width |
|
|
var max_y = Math.max(...y_array) * image_height |
|
|
|
|
|
var sect_height = max_y - (min_y) |
|
|
var sect_width = max_x - (min_x) |
|
|
var center_x = (min_x + max_x) / 2 |
|
|
var center_y = (min_y + max_y) / 2 |
|
|
|
|
|
var sect_diameter = 50 |
|
|
if (sect_height > sect_width) { |
|
|
sect_diameter = sect_height |
|
|
|
|
|
} |
|
|
if (sect_height < sect_width) { |
|
|
sect_diameter = sect_width |
|
|
|
|
|
} |
|
|
|
|
|
sect_diameter = sect_diameter + 50 |
|
|
var sect_radius = sect_diameter / 2 |
|
|
var crop_top = center_y - sect_radius |
|
|
var crop_bottom = center_y + sect_radius |
|
|
var crop_left = center_x - sect_radius |
|
|
var crop_right = center_x + sect_radius |
|
|
if (crop_top < 0) { |
|
|
crop_top = 0 |
|
|
} |
|
|
if (crop_left < 0) { |
|
|
crop_left = 0 |
|
|
} |
|
|
if (crop_right > image_width) { |
|
|
crop_right = image_width |
|
|
} |
|
|
if (crop_bottom > image_height) { |
|
|
crop_bottom = image_height |
|
|
} |
|
|
if (firstA == true) { |
|
|
canvasCtx.beginPath(); |
|
|
canvasCtx.rect(crop_left, crop_top, crop_right - crop_left, crop_bottom - crop_top); |
|
|
canvasCtx.stroke(); |
|
|
if (!Array.from(document.getElementById("modeSelector").classList).includes('right')) { |
|
|
canvasCtx.fillStyle = "#ffffff" |
|
|
canvasCtx.fill() |
|
|
} |
|
|
|
|
|
} else { |
|
|
canvasCtx.beginPath(); |
|
|
canvasCtx.rect(crop_left, crop_top, crop_right - crop_left, crop_bottom - crop_top); |
|
|
canvasCtx.stroke(); |
|
|
} |
|
|
|
|
|
|
|
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
const landmarks = results.landmarks; |
|
|
if (landmarks[0]) { |
|
|
var hand = landmarks[0] |
|
|
const zValues = hand.map(node => node.z); |
|
|
const maxZ = Math.max.apply(null, zValues) |
|
|
const minZ = Math.min.apply(null, zValues) |
|
|
const zRange = maxZ - minZ |
|
|
hand.forEach(node => { |
|
|
node.z = ((node.z - minZ) / zRange) * 255 |
|
|
}); |
|
|
|
|
|
drawConnection(hand[4], hand[3], 'rgb(0, 0, 255)', 5); |
|
|
drawConnection(hand[3], hand[2], 'rgb(0, 0, 255)', 5); |
|
|
drawConnection(hand[2], hand[1], 'rgb(0, 0, 255)', 5); |
|
|
|
|
|
|
|
|
drawConnection(hand[8], hand[7], 'rgb(0, 255, 0)', 5); |
|
|
drawConnection(hand[7], hand[6], 'rgb(0, 255, 0)', 5); |
|
|
drawConnection(hand[6], hand[5], 'rgb(0, 255, 0)', 5); |
|
|
|
|
|
|
|
|
drawConnection(hand[12], hand[11], 'rgb(255, 0, 0)', 5); |
|
|
drawConnection(hand[11], hand[10], 'rgb(255, 0, 0)', 5); |
|
|
drawConnection(hand[10], hand[9], 'rgb(255, 0, 0)', 5); |
|
|
|
|
|
|
|
|
drawConnection(hand[16], hand[15], 'rgb(0, 255, 255)', 5); |
|
|
drawConnection(hand[15], hand[14], 'rgb(0, 255, 255)', 5); |
|
|
drawConnection(hand[14], hand[13], 'rgb(0, 255, 255)', 5); |
|
|
|
|
|
|
|
|
drawConnection(hand[20], hand[19], 'rgb(255, 0, 255)', 5); |
|
|
drawConnection(hand[19], hand[18], 'rgb(255, 0, 255)', 5); |
|
|
drawConnection(hand[18], hand[17], 'rgb(255, 0, 255)', 5); |
|
|
|
|
|
drawConnection(hand[0], hand[1], 'rgb(200, 200, 200)', 5); |
|
|
drawConnection(hand[0], hand[5], 'rgb(200, 200, 200)', 5); |
|
|
drawConnection(hand[0], hand[17], 'rgb(200, 200, 200)', 5); |
|
|
drawConnection(hand[5], hand[9], 'rgb(200, 200, 200)', 5); |
|
|
drawConnection(hand[9], hand[13], 'rgb(200, 200, 200)', 5); |
|
|
drawConnection(hand[13], hand[17], 'rgb(200, 200, 200)', 5); |
|
|
|
|
|
|
|
|
drawLandmarks(canvasCtx, hand[2], '#ffe5b4'); |
|
|
drawLandmarks(canvasCtx, hand[3], '#ffe5b4'); |
|
|
drawLandmarks(canvasCtx, hand[4], '#ffe5b4'); |
|
|
|
|
|
|
|
|
drawLandmarks(canvasCtx, hand[6], '#804080'); |
|
|
drawLandmarks(canvasCtx, hand[7], '#804080'); |
|
|
drawLandmarks(canvasCtx, hand[8], '#804080'); |
|
|
|
|
|
|
|
|
drawLandmarks(canvasCtx, hand[10], '#ffcc00'); |
|
|
drawLandmarks(canvasCtx, hand[11], '#ffcc00'); |
|
|
drawLandmarks(canvasCtx, hand[12], '#ffcc00'); |
|
|
|
|
|
|
|
|
drawLandmarks(canvasCtx, hand[14], '#30ff30'); |
|
|
drawLandmarks(canvasCtx, hand[15], '#30ff30'); |
|
|
drawLandmarks(canvasCtx, hand[16], '#30ff30'); |
|
|
|
|
|
|
|
|
drawLandmarks(canvasCtx, hand[18], '#1565c0'); |
|
|
drawLandmarks(canvasCtx, hand[19], '#1565c0'); |
|
|
drawLandmarks(canvasCtx, hand[20], '#1565c0'); |
|
|
|
|
|
drawLandmarks(canvasCtx, hand[0], '#ff3030'); |
|
|
|
|
|
drawLandmarks(canvasCtx, hand[1], '#ff3030'); |
|
|
|
|
|
drawLandmarks(canvasCtx, hand[5], '#ff3030'); |
|
|
|
|
|
drawLandmarks(canvasCtx, hand[9], '#ff3030'); |
|
|
|
|
|
drawLandmarks(canvasCtx, hand[13], '#ff3030'); |
|
|
|
|
|
drawLandmarks(canvasCtx, hand[17], '#ff3030'); |
|
|
|
|
|
|
|
|
if (firstA == true) { |
|
|
cropCanvas(canvasElement, crop_left, crop_top, crop_right - crop_left, crop_bottom - crop_top) |
|
|
} |
|
|
} |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
} |
|
|
|
|
|
|
|
|
function iterate(x, y) { |
|
|
x_array.push(x.x) |
|
|
y_array.push(x.y) |
|
|
} |
|
|
|
|
|
const cropCanvas = (sourceCanvas, left, top, width, height) => { |
|
|
let destCanvas = document.createElement('canvas'); |
|
|
destCanvas.width = 224; |
|
|
var cropAspectRatio = width / height; |
|
|
|
|
|
destCanvas.height = 224 / cropAspectRatio |
|
|
destCanvas.getContext("2d").drawImage( |
|
|
sourceCanvas, |
|
|
left, top, width, height, |
|
|
0, 0, 224, destCanvas.height); |
|
|
var predictionInput = tf.browser.fromPixels(destCanvas.getContext("2d").getImageData(0, 0, 224, 224)) |
|
|
|
|
|
predict(tf.expandDims(predictionInput, 0)); |
|
|
} |
|
|
function getNaturalLength(letter) { |
|
|
console.log('i got', letter, word_list[word_list.length - 1]) |
|
|
if (document.getElementById("natInp").checked) { |
|
|
console.log('naturally:') |
|
|
if (letter != word_list[word_list.length - 1]) { |
|
|
return ((inputFrequency()) * (1 / (maxPercentage / 100))) |
|
|
} else { |
|
|
console.log("penalty!") |
|
|
var rb = Number(repeatBuffer()) |
|
|
return ((inputFrequency()*(80/100))+400) |
|
|
} |
|
|
} else { |
|
|
console.log('stably:') |
|
|
if (letter != word_list[word_list.length - 1]) { |
|
|
return ((inputFrequency())) |
|
|
} else { |
|
|
console.log("penalty!") |
|
|
var rb = Number(repeatBuffer()) |
|
|
return ((inputFrequency()*(80/100))+rb) |
|
|
} |
|
|
} |
|
|
} |
|
|
function mapLogitsToPercentage(logits) { |
|
|
|
|
|
const cappedLogits = logits.map(value => Math.min(value, 260)); |
|
|
|
|
|
|
|
|
const mappedPercentages = cappedLogits.map(value => (value / 260) * 100); |
|
|
|
|
|
return mappedPercentages; |
|
|
} |
|
|
|
|
|
|
|
|
async function predict(inputTensor) { |
|
|
|
|
|
console.log(index_list[0]) |
|
|
objectDetector.then(function (res) { |
|
|
|
|
|
var prediction = res.predict(inputTensor); |
|
|
|
|
|
var outputArray = prediction.dataSync(); |
|
|
var percentages = mapLogitsToPercentage(outputArray) |
|
|
maxPercentage = Math.max(...percentages) |
|
|
document.getElementById('predicted_result').style.opacity = (maxPercentage+30)+'%' |
|
|
|
|
|
var predictedClass = percentages.indexOf(Math.max(...percentages)); |
|
|
|
|
|
var current_result = index_list[predictedClass] |
|
|
global_cres = current_result |
|
|
var previous_result = document.getElementById("predicted_result").innerText |
|
|
document.getElementById("predicted_result").innerText = current_result |
|
|
var current_time = Math.round(Date.now()) |
|
|
if (index_list[0] == " Good") { |
|
|
previous_result = " " + previous_result |
|
|
} |
|
|
console.log("p:", previous_result, "c:", current_result) |
|
|
if (previous_result == current_result) { |
|
|
|
|
|
if (current_time - last_letter_time > getNaturalLength(current_result)) { |
|
|
last_letter_time = current_time |
|
|
word_list.push(current_result) |
|
|
triggerPulse() |
|
|
console.log(word_list) |
|
|
document.getElementById("text").value = word_list.join('') |
|
|
} |
|
|
} |
|
|
else { |
|
|
last_letter_time = current_time |
|
|
} |
|
|
console.log(index_list[predictedClass]); |
|
|
}, function (err) { |
|
|
console.log(err); |
|
|
}); |
|
|
|
|
|
} |
|
|
|
|
|
function drawLandmarks(canvasCtx, landmarks, color) { |
|
|
var image_height = (video.videoHeight / video.videoWidth) * canvasElement.width |
|
|
var image_width = canvasElement.width |
|
|
|
|
|
canvasCtx.fillStyle = `rgb(${landmarks.z},${landmarks.z},${landmarks.z})`; |
|
|
canvasCtx.beginPath(); |
|
|
canvasCtx.arc(landmarks.x * image_width, landmarks.y * image_height, 8, 0, 2 * Math.PI); |
|
|
canvasCtx.fill(); |
|
|
|
|
|
} |
|
|
|
|
|
function drawConnection(startNode, endNode, strokeColor, strokeWidth) { |
|
|
|
|
|
var image_height = (video.videoHeight / video.videoWidth) * canvasElement.width |
|
|
var image_width = canvasElement.width |
|
|
|
|
|
canvasCtx.strokeStyle = strokeColor; |
|
|
canvasCtx.lineWidth = strokeWidth - 1; |
|
|
canvasCtx.beginPath(); |
|
|
canvasCtx.moveTo(startNode.x * image_width, startNode.y * image_height); |
|
|
canvasCtx.lineTo(endNode.x * image_width, endNode.y * image_height); |
|
|
canvasCtx.stroke(); |
|
|
} |
|
|
function calculateCanvasBrightness(canvas) { |
|
|
const context = canvas.getContext('2d', { willReadFrequently: true }); |
|
|
|
|
|
|
|
|
const imageData = context.getImageData(0, 0, canvas.width, canvas.height); |
|
|
const data = imageData.data; |
|
|
|
|
|
let totalBrightness = 0; |
|
|
let pixelCount = 0; |
|
|
|
|
|
|
|
|
for (let i = 0; i < data.length; i += 4) { |
|
|
const r = data[i]; |
|
|
const g = data[i + 1]; |
|
|
const b = data[i + 2]; |
|
|
|
|
|
|
|
|
const brightness = 0.299 * r + 0.587 * g + 0.114 * b; |
|
|
totalBrightness += brightness; |
|
|
pixelCount++; |
|
|
} |
|
|
|
|
|
|
|
|
const averageBrightness = totalBrightness / pixelCount; |
|
|
|
|
|
return averageBrightness; |
|
|
} |
|
|
function triggerPulse() { |
|
|
console.log('did nothing') |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
} |
|
|
</script> |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
</body> |
|
|
|
|
|
</html> |
|
|
|
