js new beginning and progress

melihonat · Dec 20, 2023 · 8bd55b8 · 8bd55b8
1 parent 5a36b74
commit 8bd55b8
Show file tree

Hide file tree

Showing 10 changed files with 8,128 additions and 0 deletions.
diff --git a/js/background.jpg b/js/background.jpg
diff --git a/js/face-api.js b/js/face-api.js
diff --git a/js/icon.png b/js/icon.png
diff --git a/js/index.html b/js/index.html
@@ -0,0 +1,26 @@
+<!DOCTYPE html>
+<html lang="en">
+<head>
+    <meta charset="UTF-8">
+    <meta name="viewport" content="width=device-width, initial-scale=1.0">
+    <link rel="stylesheet" href="styles.css">
+    <link rel="icon" href="icon.png" type="image/png">
+    <title>Whatsyourtune</title>
+    <script src="https://cdnjs.cloudflare.com/ajax/libs/tone/14.7.58/Tone.js"></script>
+    <script src="https://cdnjs.cloudflare.com/ajax/libs/tensorflow/1.2.8/tf.min.js"></script>
+    <script src="https://cdn.jsdelivr.net/npm/@magenta/music@^1.0.0/es6/core.js"></script>
+    <script src="https://cdn.jsdelivr.net/npm/@magenta/music@^1.0.0/es6/music_vae.js"></script>
+    <!-- clmtrackr library für emotion detection -->
+    <script src="https://cdnjs.cloudflare.com/ajax/libs/clmtrackr/1.1.2/clmtrackr.min.js"></script>
+
+</head>
+<body>
+    <header>
+        <img src="logo.png" alt="Whatsyourtune">
+    </header>
+    <div class="content">
+        <video id="webcam" width="640" height="480" autoplay></video>
+        <script src="index.js"></script>
+    </div>
+</body>
+</html>
diff --git a/js/index.js b/js/index.js
@@ -0,0 +1,83 @@
+// Generieren einer Melodie basierend auf der erkannten Emotion
+async function generateMelody(emotion) {
+    console.log(`Generating melody for: ${emotion}`);
+    const characteristics = getMusicCharacteristics(emotion);
+
+    try {
+        // Magenta MusicVAE Model initialisieren
+        const mvae = new music_vae.MusicVAE('https://storage.googleapis.com/magentadata/js/checkpoints/music_vae/trio_4bar');
+        await mvae.initialize();
+        console.log('Model loaded.');
+
+        // Sample-Melodie mit dem Model generieren
+        const sample = await mvae.sample(1, characteristics.temperature);
+        console.log('Melody generated.');
+
+        // Generierte Melodie abspielen
+        const player = new core.Player();
+        player.start(sample[0]);
+        console.log('Playback started.');
+    } catch (error) {
+        console.error('Error generating melody:', error);
+    }
+}
+
+// Je nach Emotion soll andere Musik abgespielt werden
+function getMusicCharacteristics(emotion) {
+    switch(emotion) {
+        case 'happy':
+            return { temperature: 1.2 };
+        case 'sad':
+            return { temperature: 0.8 };
+        default:
+            return { temperature: 1.0 };
+    }
+}
+
+// Webcam starten und clmtrackr initialisieren
+function startWebcamAndTracking() {
+    const video = document.getElementById('webcam');
+
+    // Webcam-Access
+    navigator.mediaDevices.getUserMedia({ video: true })
+        .then((stream) => {
+            video.srcObject = stream;
+            video.onloadedmetadata = function() {
+                video.play();
+                // Wenn der Webcamfeed ready ist, anfangen Gesichtszüge zu erkennen
+                startTracking(video);
+            };
+        })
+        .catch((err) => {
+            console.error("Error accessing webcam:", err);
+        });
+}
+
+// Clmtrackr initialisieren und starten
+function startTracking(videoElement) {
+    var ctracker = new clm.tracker();
+    ctracker.init();
+    ctracker.start(videoElement);
+
+    // Durchgehend nach Emotionen checken
+    trackExpression(ctracker);
+}
+
+// Emotion aus Gesichtszüge herausinterpretieren
+function trackExpression(ctracker) {
+    function drawLoop() {
+        requestAnimationFrame(drawLoop);
+        var positions = ctracker.getCurrentPosition();
+        if (positions) {
+            // Basic Logik um Emotionen zu erkennen, basierend auf der Distanz der Mundecken
+            const mouthCornerDistance = Math.abs(positions[44][1] - positions[50][1]); // Beispiel indices für Mundecken
+            const emotion = mouthCornerDistance > 15 ? 'happy' : 'sad';
+
+            // Melodie generieren
+            generateMelody(emotion);
+        }
+    }
+    drawLoop();
+}
+// Webcam und Facial Tracking starten
+startWebcamAndTracking();
diff --git a/js/logo.png b/js/logo.png