Gesture Detection & Machine Learning

#include "esp_camera.h" #include <WiFi.h> #include <Wire.h> #include <Adafruit_Sensor.h> #include <Adafruit_ADXL345_U.h> #include <PubSubClient.h> // Camera configuration #define CAMERA_MODEL_AI_THINKER #include "camera_pins.h" // WiFi credentials - DIUBAH DISINI const char* ssid = "pintar"; const char* password = "pintar2022"; // MQTT Configuration - DIUBAH DISINI const char* mqtt_server = "broker.hivemq.com"; const int mqtt_port = 1883; const char* mqtt_topic = "PerintahGestureTubuh"; // ADXL345 Sensor Adafruit_ADXL345_Unified accel = Adafruit_ADXL345_Unified(12345); // WiFi and MQTT clients WiFiClient espClient; PubSubClient client(espClient); void setupCamera() { camera_config_t config; config.ledc_channel = LEDC_CHANNEL_0; config.ledc_timer = LEDC_TIMER_0; config.pin_d0 = Y2_GPIO_NUM; config.pin_d1 = Y3_GPIO_NUM; config.pin_d2 = Y4_GPIO_NUM; config.pin_d3 = Y5_GPIO_NUM; config.pin_d4 = Y6_GPIO_NUM; config.pin_d5 = Y7_GPIO_NUM; config.pin_d6 = Y8_GPIO_NUM; config.pin_d7 = Y9_GPIO_NUM; config.pin_xclk = XCLK_GPIO_NUM; config.pin_pclk = PCLK_GPIO_NUM; config.pin_vsync = VSYNC_GPIO_NUM; config.pin_href = HREF_GPIO_NUM; config.pin_sscb_sda = SIOD_GPIO_NUM; config.pin_sscb_scl = SIOC_GPIO_NUM; config.pin_pwdn = PWDN_GPIO_NUM; config.pin_reset = RESET_GPIO_NUM; config.xclk_freq_hz = 20000000; config.pixel_format = PIXFORMAT_JPEG; if(psramFound()){ config.frame_size = FRAMESIZE_SVGA; config.jpeg_quality = 10; config.fb_count = 2; } else { config.frame_size = FRAMESIZE_CIF; config.jpeg_quality = 12; config.fb_count = 1; } esp_err_t err = esp_camera_init(&config); if (err != ESP_OK) { Serial.printf("Camera init failed with error 0x%x", err); return; } } void setupAccelerometer() { if(!accel.begin()) { Serial.println("Could not find ADXL345 sensor"); } accel.setRange(ADXL345_RANGE_16_G); } void connectWiFi() { WiFi.begin(ssid, password); while (WiFi.status() != WL_CONNECTED) { delay(1000); Serial.println("Connecting to WiFi..."); } Serial.println("WiFi connected"); Serial.print("IP Address: "); Serial.println(WiFi.localIP()); } void reconnectMQTT() { while (!client.connected()) { Serial.print("Attempting MQTT connection..."); String clientId = "ESP32-CAM-"; clientId += String(random(0xffff), HEX); if (client.connect(clientId.c_str())) { Serial.println("connected"); } else { Serial.print("failed, rc="); Serial.print(client.state()); Serial.println(" try again in 5 seconds"); delay(5000); } } } void sendMQTTCommand(String gestureName, String hexCode) { String message = "$PerintahGesture," + gestureName + "," + hexCode; client.publish(mqtt_topic, message.c_str()); Serial.println("MQTT Sent: " + message); } void setup() { Serial.begin(115200); setupCamera(); setupAccelerometer(); connectWiFi(); client.setServer(mqtt_server, mqtt_port); startCameraServer(); } void loop() { if (!client.connected()) { reconnectMQTT(); } client.loop(); // Read accelerometer data sensors_event_t event; accel.getEvent(&event); // Simulate gesture detection and send via MQTT static unsigned long lastSend = 0; if (millis() - lastSend > 5000) { // Send every 5 seconds for demo sendMQTTCommand("Standing_Neutral", "0x01"); lastSend = millis(); } delay(100); }

1. ESP32-Camera (by ESP32) - Sketch → Include Library → Manage Libraries → Search "ESP32 Camera" 2. Adafruit ADXL345 (by Adafruit) - Sketch → Include Library → Manage Libraries → Search "Adafruit ADXL345" 3. PubSubClient (by Nick O'Leary) - Sketch → Include Library → Manage Libraries → Search "PubSubClient" 4. Adafruit Unified Sensor (by Adafruit) - Sketch → Include Library → Manage Libraries → Search "Adafruit Unified Sensor"

import paho.mqtt.client as mqtt import json import time # MQTT Configuration MQTT_BROKER = "broker.hivemq.com" MQTT_PORT = 1883 MQTT_TOPIC = "PerintahGestureTubuh" # Gesture to Hex mapping GESTURE_HEX_CODES = { 0: "0x01", 1: "0x02", 2: "0x03", 3: "0x04", 4: "0x05", 5: "0x06", 6: "0x07", 7: "0x08", 8: "0x09", 9: "0x0A", 10: "0x0B", 11: "0x0C", # ... lengkapi sampai 63 63: "0x40" } GESTURE_NAMES = { 0: "Standing_Neutral", 1: "Arms_Up", 2: "Arms_Out", 3: "Right_Arm_Up", # ... lengkapi sampai 63 63: "Celebration" } class MQTTGestureClient: def __init__(self): self.client = mqtt.Client() self.client.on_connect = self.on_connect self.client.on_message = self.on_message def on_connect(self, client, userdata, flags, rc): if rc == 0: print("Connected to MQTT Broker!") client.subscribe(MQTT_TOPIC) else: print(f"Failed to connect, return code {rc}") def on_message(self, client, userdata, msg): try: payload = msg.payload.decode() print(f"Received MQTT message: {payload}") # Process gesture command if payload.startswith("$PerintahGesture"): parts = payload.split(",") if len(parts) == 3: gesture_name = parts[1] hex_code = parts[2] self.execute_gesture_command(gesture_name, hex_code) except Exception as e: print(f"Error processing MQTT message: {e}") def execute_gesture_command(self, gesture_name, hex_code): """Execute machine command based on gesture""" print(f"Executing gesture: {gesture_name} with hex code: {hex_code}") # Add your machine control logic here commands = { "0x01": "MACHINE_STANDBY", "0x02": "ARM_UP_ALL", "0x03": "ARM_EXTEND_ALL", # ... tambahkan semua command } command = commands.get(hex_code, "UNKNOWN_COMMAND") print(f"Machine Command: {command}") def send_gesture_command(self, gesture_id, confidence): """Send gesture command via MQTT""" if confidence > 0.7: # Minimum confidence threshold hex_code = GESTURE_HEX_CODES.get(gesture_id, "0x00") gesture_name = GESTURE_NAMES.get(gesture_id, "Unknown") message = f"$PerintahGesture,{gesture_name},{hex_code}" self.client.publish(MQTT_TOPIC, message) print(f"MQTT Sent: {message}") def connect(self): self.client.connect(MQTT_BROKER, MQTT_PORT, 60) self.client.loop_start() def disconnect(self): self.client.loop_stop() self.client.disconnect() # Usage example if __name__ == "__main__": mqtt_client = MQTTGestureClient() mqtt_client.connect() try: while True: # Simulate gesture detection time.sleep(5) except KeyboardInterrupt: mqtt_client.disconnect()

opencv-python==4.8.1.78 mediapipe==0.10.0 tensorflow==2.13.0 flask==2.3.3 flask-socketio==5.3.6 numpy==1.24.3 paho-mqtt==1.6.1 pandas==2.0.3 plotly==5.15.0

Format: $PerintahGesture,nama_gesture,code_gesture Contoh: $PerintahGesture,Arms_Up,0x02 $PerintahGesture,Jumping_Jack,0x08 $PerintahGesture,Thumbs_Up,0x15 $PerintahGesture,Punch_Right,0x2E