Тестирование голосовых интерфейсов и функций на основе камеры

Updated Jul 2026

Тестирование голосовых интерфейсов

Голосовые функции (Siri Shortcuts, действия Google Assistant, голосовые команды в приложении) требуют иного подхода к тестированию. Вы тестируете не визуальный интерфейс — вы тестируете конвейер понимания естественного языка (NLU), который преобразует произнесённые слова в структурированные интенты и сущности.

Основная проблема: преобразование речи в текст порождает вариативные транскрипции. Одна и та же произнесённая фраза может быть транскрибирована по-разному в зависимости от акцента, фонового шума, качества микрофона и версии модели распознавания речи. Ваше приложение должно обрабатывать эту вариативность.

Тестирование обработки голосовых команд

Тестирование NLU-слоя

# Testing voice command processing (the recognition part)
def test_voice_command_parsing():
    """Test that the NLU layer correctly parses voice commands."""
    test_cases = [
        {
            "transcript": "show me my orders from last week",
            "expected_intent": "view_orders",
            "expected_entities": {"time_range": "last_week"}
        },
        {
            "transcript": "cancel order number twelve thirty four",
            "transcript_variants": [
                "cancel order number 1234",
                "cancel order #1234",
            ],
            "expected_intent": "cancel_order",
            "expected_entities": {"order_id": "1234"}
        },
        {
            "transcript": "how much is shipping to new york",
            "expected_intent": "shipping_estimate",
            "expected_entities": {"destination": "new_york"}
        },
        {
            "transcript": "add two widgets to my cart",
            "transcript_variants": [
                "add 2 widgets to my cart",
                "at two widgets to my cart",  # Common misrecognition
            ],
            "expected_intent": "add_to_cart",
            "expected_entities": {"quantity": 2, "product": "widgets"}
        },
    ]

    for case in test_cases:
        # Test primary transcript
        result = parse_voice_command(case["transcript"])
        assert result.intent == case["expected_intent"], \
            f"Intent mismatch for '{case['transcript']}': expected {case['expected_intent']}, got {result.intent}"
        for key, value in case["expected_entities"].items():
            assert result.entities[key] == value

        # Test transcript variants (speech-to-text produces different outputs)
        for variant in case.get("transcript_variants", []):
            result = parse_voice_command(variant)
            assert result.intent == case["expected_intent"], \
                f"Variant '{variant}' produced wrong intent: {result.intent}"

Неоднозначность и обработка ошибок

def test_ambiguous_voice_commands():
    """Test handling of ambiguous or incomplete commands."""
    ambiguous_cases = [
        {
            "transcript": "order",
            "expected": "clarification_needed",
            "description": "Too vague -- could be view, create, or cancel"
        },
        {
            "transcript": "cancel",
            "expected": "clarification_needed",
            "description": "Missing order ID"
        },
        {
            "transcript": "",
            "expected": "no_input",
            "description": "Empty transcript (silence)"
        },
        {
            "transcript": "asdfghjkl",
            "expected": "unrecognized",
            "description": "Nonsensical input"
        },
    ]

    for case in ambiguous_cases:
        result = parse_voice_command(case["transcript"])
        assert result.intent == case["expected"], \
            f"'{case['transcript']}' ({case['description']}): expected {case['expected']}, got {result.intent}"

def test_voice_command_confidence_threshold():
    """Commands below confidence threshold should trigger confirmation."""
    # Low confidence should prompt "Did you mean...?"
    result = parse_voice_command("cancel order maybe twelve thirty four ish")
    if result.confidence < 0.7:
        assert result.needs_confirmation is True

Сквозное тестирование голосового потока

def test_voice_to_action_flow(driver):
    """Test the complete voice command flow: mic -> speech -> action."""
    # Navigate to voice-enabled screen
    driver.find_element(AppiumBy.ACCESSIBILITY_ID, "voice-command-btn").click()

    # On emulators, inject pre-recorded audio
    driver.execute_script("mobile: shell", {
        "command": "am",
        "args": ["broadcast", "-a", "com.testapp.INJECT_AUDIO",
                 "--es", "audio_path", "/sdcard/test_audio/cancel_order.wav"]
    })

    # Verify the app processed the command
    confirmation = driver.find_element(AppiumBy.ACCESSIBILITY_ID, "voice-confirmation")
    assert "cancel" in confirmation.text.lower()
    assert "order" in confirmation.text.lower()

Тестирование функций на основе камеры

AR-функции, сканеры штрих-кодов, захват документов и детекция лиц требуют камеру, что делает их сложными для тестирования в автоматизированных пайплайнах.

Тестирование сканеров штрих-кодов

# Testing barcode scanner with synthetic camera input
def test_barcode_scanner(driver):
    """Test barcode scanning using a pre-recorded camera feed."""

    # Appium can inject camera images on supported devices
    driver.push_file(
        "/sdcard/DCIM/test_barcode.png",
        source_path="test_data/barcode_ean13.png"
    )

    # Navigate to scanner
    driver.find_element(AppiumBy.ACCESSIBILITY_ID, "scan-barcode-btn").click()

    # On emulators, use a virtual camera scene
    driver.execute_script("mobile: shell", {
        "command": "am",
        "args": ["broadcast", "-a", "com.testapp.INJECT_CAMERA_FRAME",
                 "--es", "image_path", "/sdcard/DCIM/test_barcode.png"]
    })

    # Verify barcode was decoded
    result = driver.find_element(AppiumBy.ACCESSIBILITY_ID, "scan-result")
    assert result.text == "4006381333931"  # Expected EAN-13

def test_barcode_scanner_with_various_formats(driver):
    """Test that the scanner handles multiple barcode formats."""
    test_barcodes = [
        {"file": "barcode_ean13.png", "expected": "4006381333931", "format": "EAN-13"},
        {"file": "barcode_qr.png", "expected": "https://example.com/product/123", "format": "QR"},
        {"file": "barcode_code128.png", "expected": "ABC-12345", "format": "Code 128"},
        {"file": "barcode_upc.png", "expected": "042100005264", "format": "UPC-A"},
    ]

    for barcode in test_barcodes:
        driver.push_file(
            "/sdcard/DCIM/test_barcode.png",
            source_path=f"test_data/{barcode['file']}"
        )

        driver.find_element(AppiumBy.ACCESSIBILITY_ID, "scan-barcode-btn").click()
        driver.execute_script("mobile: shell", {
            "command": "am",
            "args": ["broadcast", "-a", "com.testapp.INJECT_CAMERA_FRAME",
                     "--es", "image_path", "/sdcard/DCIM/test_barcode.png"]
        })

        result = driver.find_element(AppiumBy.ACCESSIBILITY_ID, "scan-result")
        assert result.text == barcode["expected"], \
            f"{barcode['format']} barcode: expected {barcode['expected']}, got {result.text}"

        # Go back for next test
        driver.back()

Тестирование AR-функций

def test_ar_furniture_placement(driver):
    """Test AR furniture placement feature with synthetic camera."""

    # Inject a synthetic room scene
    driver.push_file(
        "/sdcard/DCIM/ar_test_room.jpg",
        source_path="test_data/ar_room_scene.jpg"
    )

    # Navigate to AR viewer
    driver.find_element(AppiumBy.ACCESSIBILITY_ID, "ar-viewer-btn").click()

    # Select a furniture item
    driver.find_element(AppiumBy.ACCESSIBILITY_ID, "furniture-sofa").click()

    # Verify AR session started
    ar_canvas = driver.find_element(AppiumBy.ACCESSIBILITY_ID, "ar-canvas")
    assert ar_canvas.is_displayed()

    # Verify placement controls are available
    rotate_btn = driver.find_element(AppiumBy.ACCESSIBILITY_ID, "ar-rotate")
    scale_btn = driver.find_element(AppiumBy.ACCESSIBILITY_ID, "ar-scale")
    assert rotate_btn.is_displayed()
    assert scale_btn.is_displayed()

def test_camera_permission_denied_gracefully(driver):
    """App should handle camera permission denial without crashing."""
    # Deny camera permission
    driver.find_element(AppiumBy.ACCESSIBILITY_ID, "scan-barcode-btn").click()

    # Dismiss permission dialog with "Deny"
    driver.find_element(
        AppiumBy.ID, "com.android.permissioncontroller:id/permission_deny_button"
    ).click()

    # App should show a friendly message, not crash
    error_msg = driver.find_element(AppiumBy.ACCESSIBILITY_ID, "camera-denied-message")
    assert error_msg.is_displayed()
    assert "camera" in error_msg.text.lower()

Стратегии тестирования камеры/голоса в CI

Общий паттерн: разделяйте механизм захвата (камера, микрофон) и логику обработки (декодирование штрих-кодов, преобразование речи в текст, ML-инференс). Юнит-тестируйте логику обработки с известными входными данными. Интеграционно тестируйте конвейер захвата на реальных устройствах в облачных фермах устройств.