Sign_Language_Detection.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "provenance": [],
      "toc_visible": true,
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/erencankur/Sign_Language_Detection/blob/main/Sign_Language_Detection.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "# **Sign Language Detection**\n",
        "Sign language detection with artificial intelligence based model training using tensorflow.\n"
      ],
      "metadata": {
        "id": "PATXl4jPK7TI"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "# **Code Files:**\n",
        "  \n",
        "1.   First Code: Data Collection\n",
        "2.   Second Code: Model Training\n",
        "3.   Third Code: Prediction"
      ],
      "metadata": {
        "id": "0GJWHLl0Lx7O"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "# **First Code: Data Collection**\n",
        "\n"
      ],
      "metadata": {
        "id": "UeFEE0kYMsc2"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "import os\n",
        "import cv2\n",
        "import numpy as np\n",
        "import mediapipe as mp\n",
        "\n",
        "dataset_path = \"dataset\"\n",
        "\n",
        "def create_dataset_folders():\n",
        "    if not os.path.exists(dataset_path):\n",
        "        os.makedirs(dataset_path)\n",
        "\n",
        "    classes = [chr(i) for i in range(65, 91)]\n",
        "    for class_name in classes:\n",
        "        class_path = os.path.join(dataset_path, class_name)\n",
        "        if not os.path.exists(class_path):\n",
        "            os.makedirs(class_path)\n",
        "\n",
        "    print(\"All folders created.\")\n",
        "\n",
        "def initialize_hand_detection():\n",
        "    mp_hands = mp.solutions.hands\n",
        "    return mp_hands.Hands(\n",
        "        static_image_mode=False,\n",
        "        max_num_hands=1,\n",
        "        min_detection_confidence=0.7,\n",
        "        min_tracking_confidence=0.7\n",
        "    )\n",
        "\n",
        "def get_hand_boundaries(hand_landmarks, frame_shape):\n",
        "    height, width = frame_shape[:2]\n",
        "    x_min, y_min = width, height\n",
        "    x_max, y_max = 0, 0\n",
        "    points = []\n",
        "\n",
        "    for landmark in hand_landmarks.landmark:\n",
        "        x, y = int(landmark.x * width), int(landmark.y * height)\n",
        "        points.append((x, y))\n",
        "        x_min, y_min = min(x_min, x), min(y_min, y)\n",
        "        x_max, y_max = max(x_max, x), max(y_max, y)\n",
        "\n",
        "    return points, (x_min, y_min, x_max, y_max)\n",
        "\n",
        "def create_hand_mask(points, frame_shape):\n",
        "    mask = np.zeros(frame_shape[:2], dtype=np.uint8)\n",
        "\n",
        "    palm_points = np.array([points[0], points[1], points[5], points[17]])\n",
        "    cv2.fillPoly(mask, [palm_points], 255)\n",
        "\n",
        "    cv2.line(mask, points[0], points[1], 255, thickness=50)\n",
        "    cv2.line(mask, points[1], points[5], 255, thickness=50)\n",
        "    cv2.line(mask, points[5], points[17], 255, thickness=50)\n",
        "    cv2.line(mask, points[17], points[0], 255, thickness=50)\n",
        "\n",
        "    for i in range(len(points)-1):\n",
        "        if i % 4 != 0:\n",
        "            cv2.line(mask, points[i], points[i + 1], 255, thickness=20)\n",
        "\n",
        "    kernel = np.ones((25, 25), np.uint8)\n",
        "    dilated_mask = cv2.dilate(mask, kernel)\n",
        "    dilated_mask = cv2.GaussianBlur(dilated_mask, (15, 15), 0)\n",
        "\n",
        "    return dilated_mask\n",
        "\n",
        "def get_square_boundaries(boundaries, frame_shape):\n",
        "    x_min, y_min, x_max, y_max = boundaries\n",
        "    height, width = frame_shape[:2]\n",
        "\n",
        "    center_x = (x_min + x_max) // 2\n",
        "    center_y = (y_min + y_max) // 2\n",
        "\n",
        "    width_hand = x_max - x_min\n",
        "    height_hand = y_max - y_min\n",
        "\n",
        "    square_size = int(max(width_hand, height_hand) + 100)\n",
        "\n",
        "    x_min = center_x - square_size // 2\n",
        "    y_min = center_y - square_size // 2\n",
        "    x_max = center_x + square_size // 2\n",
        "    y_max = center_y + square_size // 2\n",
        "\n",
        "    if x_min < 0:\n",
        "        x_max -= x_min\n",
        "        x_min = 0\n",
        "    if y_min < 0:\n",
        "        y_max -= y_min\n",
        "        y_min = 0\n",
        "    if x_max > width:\n",
        "        x_min -= (x_max - width)\n",
        "        x_max = width\n",
        "    if y_max > height:\n",
        "        y_min -= (y_max - height)\n",
        "        y_max = height\n",
        "\n",
        "    return x_min, y_min, x_max, y_max\n",
        "\n",
        "def main():\n",
        "    create_dataset_folders()\n",
        "    hands = initialize_hand_detection()\n",
        "    cap = cv2.VideoCapture(0)\n",
        "\n",
        "    current_class = input(\"Enter the class: \")\n",
        "    class_path = os.path.join(dataset_path, current_class)\n",
        "\n",
        "    if not os.path.exists(class_path):\n",
        "        print(f\"Folder {current_class} not found.\")\n",
        "        cap.release()\n",
        "        return\n",
        "\n",
        "    print(f\"Saving images for {current_class}. Press \\\"s\\\" to take a photo.\")\n",
        "    image_count = 0\n",
        "\n",
        "    while True:\n",
        "        success, frame = cap.read()\n",
        "        if not success:\n",
        "            print(\"Frame not available.\")\n",
        "            break\n",
        "\n",
        "        frame = cv2.flip(frame, 1)\n",
        "        result = np.zeros_like(frame)\n",
        "        display_frame = frame.copy()\n",
        "\n",
        "        rgb_frame = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB)\n",
        "        results = hands.process(rgb_frame)\n",
        "\n",
        "        if results.multi_hand_landmarks:\n",
        "            for hand_landmarks in results.multi_hand_landmarks:\n",
        "                points, boundaries = get_hand_boundaries(hand_landmarks, frame.shape)\n",
        "                mask = create_hand_mask(points, frame.shape)\n",
        "                square_bounds = get_square_boundaries(boundaries, frame.shape)\n",
        "                x_min, y_min, x_max, y_max = square_bounds\n",
        "\n",
        "                cv2.rectangle(display_frame, (x_min, y_min), (x_max, y_max), (0, 255, 0), 2)\n",
        "\n",
        "                result = cv2.bitwise_and(frame, frame, mask=mask)\n",
        "\n",
        "                hand_square = result[y_min:y_max, x_min:x_max]\n",
        "                if hand_square.size > 0:\n",
        "                    if cv2.waitKey(1) & 0xFF == ord(\"s\"):\n",
        "                        square_size = max(hand_square.shape[0], hand_square.shape[1])\n",
        "                        square_img = np.zeros((square_size, square_size, 3), dtype=np.uint8)\n",
        "\n",
        "                        y_offset = (square_size - hand_square.shape[0]) // 2\n",
        "                        x_offset = (square_size - hand_square.shape[1]) // 2\n",
        "\n",
        "                        square_img[y_offset:y_offset+hand_square.shape[0],\n",
        "                                 x_offset:x_offset+hand_square.shape[1]] = hand_square\n",
        "\n",
        "                        final_img = cv2.resize(square_img, (64, 64))\n",
        "\n",
        "                        image_name = f\"{image_count + 1}.jpg\"\n",
        "                        image_path = os.path.join(class_path, image_name)\n",
        "                        cv2.imwrite(image_path, final_img)\n",
        "                        print(f\"{image_name} saved.\")\n",
        "                        image_count += 1\n",
        "\n",
        "        combined_view = cv2.addWeighted(display_frame, 0.1, result, 0.9, 0)\n",
        "        cv2.imshow(\"Data Collection\", combined_view)\n",
        "\n",
        "        if cv2.waitKey(1) & 0xFF == ord(\"q\"):\n",
        "            break\n",
        "\n",
        "    print(f\"\\n{image_count} images saved for {current_class}.\")\n",
        "    cap.release()\n",
        "    cv2.destroyAllWindows()\n",
        "\n",
        "if __name__ == \"__main__\":\n",
        "    main()"
      ],
      "metadata": {
        "id": "EbJb1rGzND9N"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "# **Second Code: Model Training**"
      ],
      "metadata": {
        "id": "6cj_saexNOVK"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "import os\n",
        "import cv2\n",
        "import numpy as np\n",
        "import tensorflow as tf\n",
        "from tensorflow.keras import Input, layers\n",
        "from tensorflow.keras.models import Sequential\n",
        "from tensorflow.keras.optimizers import Adam\n",
        "from tensorflow.keras.preprocessing.image import ImageDataGenerator\n",
        "from tensorflow.keras.callbacks import EarlyStopping\n",
        "from sklearn.model_selection import train_test_split\n",
        "import matplotlib.pyplot as plt\n",
        "\n",
        "dataset_path = \"dataset\"\n",
        "model_path = \"model.keras\"\n",
        "\n",
        "def create_model():\n",
        "    model = Sequential([\n",
        "        Input(shape=(64, 64, 3)),\n",
        "        layers.Conv2D(16, (3, 3), activation=\"relu\"),\n",
        "        layers.MaxPooling2D((2, 2)),\n",
        "        layers.Conv2D(32, (3, 3), activation=\"relu\"),\n",
        "        layers.MaxPooling2D((2, 2)),\n",
        "        layers.Flatten(),\n",
        "        layers.Dense(26, activation=\"softmax\")\n",
        "    ])\n",
        "\n",
        "    model.compile(\n",
        "        optimizer=Adam(learning_rate=0.00005),\n",
        "        loss=\"sparse_categorical_crossentropy\",\n",
        "        metrics=[\"accuracy\"]\n",
        "    )\n",
        "\n",
        "    return model\n",
        "\n",
        "def get_image_paths_and_labels(dataset_path):\n",
        "    image_paths = []\n",
        "    labels = []\n",
        "\n",
        "    for label in range(26):\n",
        "        char_label = chr(label + 65)\n",
        "\n",
        "        folder_path = os.path.join(dataset_path, char_label)\n",
        "        if os.path.exists(folder_path):\n",
        "            for file_name in os.listdir(folder_path):\n",
        "                if file_name.endswith(\".jpg\"):\n",
        "                    image_paths.append(os.path.join(folder_path, file_name))\n",
        "                    labels.append(label)\n",
        "\n",
        "    return image_paths, labels\n",
        "\n",
        "def load_images(image_paths, labels):\n",
        "    images = []\n",
        "    for img_path in image_paths:\n",
        "        img = cv2.imread(img_path)\n",
        "        img = cv2.resize(img, (64, 64))\n",
        "        images.append(img)\n",
        "\n",
        "    return np.array(images), np.array(labels)\n",
        "\n",
        "def create_data_generator():\n",
        "    return ImageDataGenerator(\n",
        "        rescale=1./255,\n",
        "        rotation_range=5,\n",
        "        shear_range=0.05,\n",
        "        zoom_range=0.05,\n",
        "        horizontal_flip=True,\n",
        "        fill_mode=\"nearest\"\n",
        "    )\n",
        "\n",
        "def plot_training_history(history):\n",
        "    fig, (ax1, ax2) = plt.subplots(1, 2, figsize=(14, 6))\n",
        "\n",
        "    ax1.plot(history.history[\"accuracy\"], label=\"Training Accuracy\", marker=\"o\", color=\"blue\")\n",
        "    ax1.plot(history.history[\"val_accuracy\"], label=\"Validation Accuracy\", marker=\"o\", color=\"cyan\")\n",
        "    ax1.set_title(\"Model Accuracy\")\n",
        "    ax1.set_xlabel(\"Epoch\")\n",
        "    ax1.set_ylabel(\"Accuracy\")\n",
        "    ax1.legend()\n",
        "    ax1.grid()\n",
        "\n",
        "    ax2.plot(history.history[\"loss\"], label=\"Training Loss\", marker=\"x\", color=\"red\")\n",
        "    ax2.plot(history.history[\"val_loss\"], label=\"Validation Loss\", marker=\"x\", color=\"orange\")\n",
        "    ax2.set_title(\"Model Loss\")\n",
        "    ax2.set_xlabel(\"Epoch\")\n",
        "    ax2.set_ylabel(\"Loss\")\n",
        "    ax2.legend()\n",
        "    ax2.grid()\n",
        "\n",
        "    plt.tight_layout()\n",
        "    plt.show()\n",
        "\n",
        "def main():\n",
        "    model = create_model()\n",
        "\n",
        "    image_paths, labels = get_image_paths_and_labels(dataset_path)\n",
        "    x_data, y_data = load_images(image_paths, labels)\n",
        "\n",
        "    x_train, x_val, y_train, y_val = train_test_split(\n",
        "        x_data, y_data, test_size=0.2, random_state=42\n",
        "    )\n",
        "\n",
        "    datagen = create_data_generator()\n",
        "    datagen.fit(x_train)\n",
        "\n",
        "    early_stopping = EarlyStopping(\n",
        "        monitor=\"val_loss\",\n",
        "        patience=5,\n",
        "        restore_best_weights=True\n",
        "    )\n",
        "\n",
        "    history = model.fit(\n",
        "        datagen.flow(x_train, y_train, batch_size=32),\n",
        "        validation_data=(x_val/255.0, y_val),\n",
        "        epochs=100,\n",
        "        callbacks=[early_stopping]\n",
        "    )\n",
        "\n",
        "    model.save(model_path)\n",
        "\n",
        "    plot_training_history(history)\n",
        "\n",
        "if __name__ == \"__main__\":\n",
        "    main()"
      ],
      "metadata": {
        "id": "pnhXAZEfNeU-"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "# **Third Code: Prediction**"
      ],
      "metadata": {
        "id": "eqyXQYpVN8ge"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "import cv2\n",
        "import numpy as np\n",
        "import mediapipe as mp\n",
        "from tensorflow.keras.models import load_model\n",
        "\n",
        "model_path = \"model.keras\"\n",
        "\n",
        "def initialize_hand_detection():\n",
        "    mp_hands = mp.solutions.hands\n",
        "    return mp_hands.Hands(\n",
        "        static_image_mode=False,\n",
        "        max_num_hands=1,\n",
        "        min_detection_confidence=0.7,\n",
        "        min_tracking_confidence=0.7\n",
        "    )\n",
        "\n",
        "def load_prediction_model():\n",
        "    return load_model(model_path)\n",
        "\n",
        "def get_class_mapping():\n",
        "    return [chr(i) for i in range(65, 91)]\n",
        "\n",
        "def process_hand_landmarks(hand_landmarks, frame_shape):\n",
        "    height, width = frame_shape[:2]\n",
        "    points = []\n",
        "    x_min, y_min = width, height\n",
        "    x_max, y_max = 0, 0\n",
        "\n",
        "    for landmark in hand_landmarks.landmark:\n",
        "        x, y = int(landmark.x * width), int(landmark.y * height)\n",
        "        points.append((x, y))\n",
        "        x_min, y_min = min(x_min, x), min(y_min, y)\n",
        "        x_max, y_max = max(x_max, x), max(y_max, y)\n",
        "\n",
        "    return points, (x_min, y_min, x_max, y_max)\n",
        "\n",
        "def create_hand_mask(points, frame_shape):\n",
        "    mask = np.zeros(frame_shape[:2], dtype=np.uint8)\n",
        "\n",
        "    palm_points = np.array([points[0], points[1], points[5], points[17]])\n",
        "    cv2.fillPoly(mask, [palm_points], 255)\n",
        "\n",
        "    cv2.line(mask, points[0], points[1], 255, thickness=50)\n",
        "    cv2.line(mask, points[1], points[5], 255, thickness=50)\n",
        "    cv2.line(mask, points[5], points[17], 255, thickness=50)\n",
        "    cv2.line(mask, points[17], points[0], 255, thickness=50)\n",
        "\n",
        "    for i in range(len(points)-1):\n",
        "        if i % 4 != 0:\n",
        "            cv2.line(mask, points[i], points[i + 1], 255, thickness=20)\n",
        "\n",
        "    kernel = np.ones((25, 25), np.uint8)\n",
        "    dilated_mask = cv2.dilate(mask, kernel)\n",
        "    dilated_mask = cv2.GaussianBlur(dilated_mask, (15, 15), 0)\n",
        "\n",
        "    return dilated_mask\n",
        "\n",
        "def get_square_boundaries(boundaries, frame_shape):\n",
        "    x_min, y_min, x_max, y_max = boundaries\n",
        "    height, width = frame_shape[:2]\n",
        "\n",
        "    center_x = (x_min + x_max) // 2\n",
        "    center_y = (y_min + y_max) // 2\n",
        "\n",
        "    width_hand = x_max - x_min\n",
        "    height_hand = y_max - y_min\n",
        "\n",
        "    square_size = int(max(width_hand, height_hand) + 100)\n",
        "\n",
        "    new_x_min = center_x - square_size // 2\n",
        "    new_y_min = center_y - square_size // 2\n",
        "    new_x_max = center_x + square_size // 2\n",
        "    new_y_max = center_y + square_size // 2\n",
        "\n",
        "    if new_x_min < 0:\n",
        "        new_x_max -= new_x_min\n",
        "        new_x_min = 0\n",
        "    if new_y_min < 0:\n",
        "        new_y_max -= new_y_min\n",
        "        new_y_min = 0\n",
        "    if new_x_max > width:\n",
        "        new_x_min -= (new_x_max - width)\n",
        "        new_x_max = width\n",
        "    if new_y_max > height:\n",
        "        new_y_min -= (new_y_max - height)\n",
        "        new_y_max = height\n",
        "\n",
        "    return new_x_min, new_y_min, new_x_max, new_y_max\n",
        "\n",
        "def predict_hand_sign(model, hand_square, class_mapping):\n",
        "    hand_square = cv2.resize(hand_square, (64, 64))\n",
        "    hand_square = hand_square / 255.0\n",
        "    hand_square = np.expand_dims(hand_square, axis=0)\n",
        "\n",
        "    predictions = model.predict(hand_square)\n",
        "    predicted_class = np.argmax(predictions)\n",
        "    return class_mapping[predicted_class]\n",
        "\n",
        "def main():\n",
        "    hands = initialize_hand_detection()\n",
        "    model = load_prediction_model()\n",
        "    class_mapping = get_class_mapping()\n",
        "    cap = cv2.VideoCapture(0)\n",
        "\n",
        "    while True:\n",
        "        success, frame = cap.read()\n",
        "        if not success:\n",
        "            print(\"Frame not available.\")\n",
        "            break\n",
        "\n",
        "        frame = cv2.flip(frame, 1)\n",
        "        result = np.zeros_like(frame)\n",
        "        display_frame = frame.copy()\n",
        "        predicted_character = \"DNE\"\n",
        "\n",
        "        rgb_frame = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB)\n",
        "        results = hands.process(rgb_frame)\n",
        "\n",
        "        if results.multi_hand_landmarks:\n",
        "            for hand_landmarks in results.multi_hand_landmarks:\n",
        "                points, boundaries = process_hand_landmarks(hand_landmarks, frame.shape)\n",
        "                mask = create_hand_mask(points, frame.shape)\n",
        "                square_bounds = get_square_boundaries(boundaries, frame.shape)\n",
        "                x_min, y_min, x_max, y_max = square_bounds\n",
        "\n",
        "                cv2.rectangle(display_frame, (x_min, y_min), (x_max, y_max), (0, 255, 0), 2)\n",
        "\n",
        "                result = cv2.bitwise_and(frame, frame, mask=mask)\n",
        "\n",
        "                hand_square = result[y_min:y_max, x_min:x_max]\n",
        "                if hand_square.size > 0:\n",
        "                    square_size = max(hand_square.shape[0], hand_square.shape[1])\n",
        "                    square_img = np.zeros((square_size, square_size, 3), dtype=np.uint8)\n",
        "\n",
        "                    y_offset = (square_size - hand_square.shape[0]) // 2\n",
        "                    x_offset = (square_size - hand_square.shape[1]) // 2\n",
        "\n",
        "                    square_img[y_offset:y_offset+hand_square.shape[0],\n",
        "                             x_offset:x_offset+hand_square.shape[1]] = hand_square\n",
        "\n",
        "                    predicted_character = predict_hand_sign(model, square_img, class_mapping)\n",
        "\n",
        "        cv2.putText(frame, f\"Predicted: {predicted_character}\", (20, 50),\n",
        "                    cv2.FONT_HERSHEY_SIMPLEX, 2, (0, 255, 0), 3)\n",
        "        cv2.imshow(\"Sign Language Detection\", frame)\n",
        "\n",
        "        if cv2.waitKey(1) & 0xFF == ord('q'):\n",
        "            break\n",
        "\n",
        "    cap.release()\n",
        "    cv2.destroyAllWindows()\n",
        "\n",
        "if __name__ == \"__main__\":\n",
        "    main()"
      ],
      "metadata": {
        "id": "VyN84Ab9OIWs"
      },
      "execution_count": null,
      "outputs": []
    }
  ]
}