<?php
/**
 * This file is part of the Wisewand WordPress plugin.
 *
 * @package Wisewand
 * @since 1.0.0
 * @license GPL-2.0-or-later
 *
 * This program is free software; you can redistribute it and/or modify
 * it under the terms of the GNU General Public License as published by
 * the Free Software Foundation; either version 2 of the License, or
 * (at your option) any later version.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with this program. If not, see <https://www.gnu.org/licenses/>.
 */

/**
 * Similarity Calculator - Multi-Algorithm Fuzzy Matching
 *
 * Provides advanced similarity scoring for keyword deduplication
 * using multiple algorithms with weighted scoring.
 *
 * @package Wisewand
 * @subpackage Utils
 * @version 1.0.0
 */

namespace Wisewand\Utils;

// Exit if accessed directly
if ( ! defined( 'ABSPATH' ) ) {
	exit;
}


class Similarity_Calculator {

	/**
	 * Calculate similarity between two keywords using multi-algorithm approach
	 *
	 * @param string $keyword1 First keyword
	 * @param string $keyword2 Second keyword
	 * @param string $language Optional language code for context
	 * @return float Similarity score (0.0 to 1.0)
	 */
	public static function calculate_similarity( $keyword1, $keyword2, $language = '' ) {
		// CRITICAL CHECK: Number variance (age/year specificity)
		if ( ! self::are_numbers_compatible( $keyword1, $keyword2 ) ) {
			return 0.0; // Different ages/numbers = NOT duplicates
		}

		$kw1 = self::normalize_for_comparison( $keyword1 );
		$kw2 = self::normalize_for_comparison( $keyword2 );

		if ( $kw1 === $kw2 ) {
			return 1.0; // Exact match after normalization
		}

		// CRITICAL CHECK: Topic Consistency
		// Prevents clustering "telegram crypto" with "wallet crypto" just because they share "crypto"
		if ( ! self::are_topics_consistent( $kw1, $kw2 ) ) {
			return 0.0; // Different topics = NOT duplicates
		}

		$scores = array();

		// Algorithm 1: Token Overlap (35% weight - reduced to prevent single-word matches)
		$scores['token_overlap'] = self::calculate_token_overlap( $kw1, $kw2 );

		// Algorithm 2: Levenshtein Distance (35% weight - catches typos/variants)
		$scores['levenshtein'] = self::calculate_levenshtein_similarity( $kw1, $kw2 );

		// Algorithm 3: Jaro-Winkler (20% weight - good for transpositions)
		$scores['jaro_winkler'] = self::calculate_jaro_winkler( $kw1, $kw2 );

		// Algorithm 4: Phonetic Match (10% weight - catches sound-alikes)
		$scores['phonetic'] = self::calculate_phonetic_similarity( $kw1, $kw2 );

		// Weighted average
		$final_score = (
			$scores['token_overlap'] * 0.35 +
			$scores['levenshtein'] * 0.35 +
			$scores['jaro_winkler'] * 0.20 +
			$scores['phonetic'] * 0.10
		);

		return round( $final_score, 4 );
	}

	/**
	 * Check if two keywords have consistent topics (not just sharing generic words)
	 *
	 * CRITICAL: This prevents false clustering like:
	 * - "telegram crypto" with "wallet crypto" (different products, share "crypto")
	 * - "argent wallet" with "trust wallet" (different wallets, share "wallet")
	 *
	 * Requires that MAJORITY of meaningful words are shared, not just one.
	 *
	 * @param string $kw1 Normalized keyword 1
	 * @param string $kw2 Normalized keyword 2
	 * @return bool True if topics are consistent
	 */
	private static function are_topics_consistent( $kw1, $kw2 ) {
		$tokens1 = array_filter( explode( ' ', $kw1 ) );
		$tokens2 = array_filter( explode( ' ', $kw2 ) );

		if ( empty( $tokens1 ) || empty( $tokens2 ) ) {
			return false;
		}

		// Generic words that shouldn't count as "shared topic"
		$generic_words = array(
			// Crypto generic
			'crypto',
			'cryptocurrency',
			'blockchain',
			'token',
			'coin',
			'wallet',
			'exchange',
			'trading',
			'trader',
			'market',
			'price',
			'buy',
			'sell',
			// AI generic
			'ai',
			'ia',
			'intelligence',
			'artificielle',
			'artificial',
			'machine',
			'learning',
			'chatgpt',
			'gpt',
			'bot',
			'assistant',
			// Tech generic
			'app',
			'application',
			'software',
			'platform',
			'tool',
			'online',
			'free',
			'best',
			'top',
			'meilleur',
			'mejor',
			// Content generic
			'guide',
			'tutorial',
			'how',
			'what',
			'comment',
			'como',
			'pour',
			'for',
			'with',
		);

		// Filter out generic words
		$meaningful1 = array_diff( $tokens1, $generic_words );
		$meaningful2 = array_diff( $tokens2, $generic_words );

		// If all words are generic, fall back to checking if BOTH keywords are nearly identical
		if ( empty( $meaningful1 ) && empty( $meaningful2 ) ) {
			// Both are generic-only, check overall token overlap is very high
			$intersection = count( array_intersect( $tokens1, $tokens2 ) );
			$smaller_set  = min( count( $tokens1 ), count( $tokens2 ) );
			return $smaller_set > 0 && ( $intersection / $smaller_set ) >= 0.8;
		}

		// If one has meaningful words and the other doesn't, they're different topics
		if ( empty( $meaningful1 ) || empty( $meaningful2 ) ) {
			return false;
		}

		// Check if majority of meaningful words overlap
		$intersection_m     = count( array_intersect( $meaningful1, $meaningful2 ) );
		$smaller_meaningful = min( count( $meaningful1 ), count( $meaningful2 ) );

		// Require at least 60% of meaningful words to overlap
		return $smaller_meaningful > 0 && ( $intersection_m / $smaller_meaningful ) >= 0.6;
	}

	/**
	 * Get language-specific similarity threshold
	 *
	 * Romance languages (French, Spanish, Italian, Portuguese) have more
	 * morphological variation (accents, articles, gender) so use lower threshold.
	 *
	 * @param string $language_code Language code (fr, es, it, pt, en, de, etc.)
	 * @return float Threshold value (0.68 - 0.75)
	 */
	public static function get_threshold_for_language( $language_code ) {
		$language_code = strtolower( $language_code );

		$thresholds = array(
			// Romance languages - more lenient (accents, articles, gender)
			'fr'      => 0.68, // French
			'es'      => 0.68, // Spanish
			'it'      => 0.68, // Italian
			'pt'      => 0.68, // Portuguese

			// Germanic languages - moderate
			'de'      => 0.72, // German (compound words, cases)

			// English and others - standard
			'en'      => 0.75,
			'default' => 0.75,
		);

		return $thresholds[ $language_code ] ?? $thresholds['default'];
	}

	/**
	 * Check if numbers in keywords are compatible (not different ages/years/quantities)
	 *
	 * Prevents clustering:
	 * - "stories for 5 year olds" with "stories for 8 year olds" (different ages)
	 * - "top 5 plugins" with "top 10 plugins" (different quantities)
	 * - "cadeau enfant 2 ans" with "cadeau enfant 5 ans" (different ages)
	 *
	 * @param string $keyword1
	 * @param string $keyword2
	 * @return bool True if numbers are compatible (same or both absent)
	 */
	private static function are_numbers_compatible( $keyword1, $keyword2 ) {
		// Extract all numbers from both keywords
		preg_match_all( '/\d+/', $keyword1, $matches1 );
		preg_match_all( '/\d+/', $keyword2, $matches2 );

		$numbers1 = $matches1[0] ?? array();
		$numbers2 = $matches2[0] ?? array();

		// If both have no numbers, they're compatible
		if ( empty( $numbers1 ) && empty( $numbers2 ) ) {
			return true;
		}

		// If only one has numbers, they're different
		if ( empty( $numbers1 ) || empty( $numbers2 ) ) {
			return false;
		}

		// Sort numbers for comparison
		sort( $numbers1 );
		sort( $numbers2 );

		// If numbers are identical, they're compatible
		if ( $numbers1 === $numbers2 ) {
			return true;
		}

		// Numbers are different - check if they're in age/year/quantity context
		$age_keywords = array(
			// English
			'year',
			'years',
			'old',
			'olds',
			'age',
			'aged',
			'yo',
			// French
			'ans',
			'année',
			'années',
			'âge',
			'âgé',
			// German
			'jahr',
			'jahre',
			'alt',
			// Spanish
			'año',
			'años',
			'edad',
			// Italian
			'anno',
			'anni',
			'età',
			// Common quantity words
			'top',
			'best',
			'meilleur',
			'mejor',
			'migliore',
		);

		$kw1_lower = mb_strtolower( $keyword1, 'UTF-8' );
		$kw2_lower = mb_strtolower( $keyword2, 'UTF-8' );

		// Check if either keyword contains age/year context
		$has_age_context = false;
		foreach ( $age_keywords as $age_word ) {
			if ( strpos( $kw1_lower, $age_word ) !== false || strpos( $kw2_lower, $age_word ) !== false ) {
				$has_age_context = true;
				break;
			}
		}

		// If numbers differ in age/year context, they're NOT compatible
		if ( $has_age_context ) {
			return false; // Different ages = different target audiences
		}

		// Numbers differ but not in age context - check if significant difference
		// For quantities like "top 5" vs "top 10", also treat as different
		$max_diff = 3; // Allow small differences (e.g., "5-7" vs "5-6" might be ok)

		foreach ( $numbers1 as $i => $num1 ) {
			$num2 = $numbers2[ $i ] ?? null;

			if ( $num2 === null ) {
				return false; // Different number count
			}

			if ( abs( intval( $num1 ) - intval( $num2 ) ) > $max_diff ) {
				return false; // Significant number difference
			}
		}

		// Numbers are similar enough
		return true;
	}

	/**
	 * Normalize keyword for comparison with enhanced French support
	 *
	 * Features:
	 * - Accent removal (è/é/ê → e, à → a, ô → o, etc.)
	 * - Stop word filtering (articles, prepositions in 5 languages)
	 * - Synonym normalization (poème = poésie, histoire = conte)
	 * - Lemmatization (plural → singular)
	 *
	 * Examples:
	 * - "poèmes pour maman" → "poeme maman"
	 * - "poésie sur la maman" → "poeme maman"
	 * - "histoires de enfants" → "histoire enfant"
	 *
	 * @param string $keyword Original keyword
	 * @return string Normalized keyword
	 */
	private static function normalize_for_comparison( $keyword ) {
		// Step 1: Lowercase
		$keyword = mb_strtolower( $keyword, 'UTF-8' );

		// Step 2: Remove accents using WordPress built-in function
		// è/é/ê → e, à/â → a, ô → o, ç → c, ü → u, ñ → n
		if ( function_exists( 'remove_accents' ) ) {
			$keyword = remove_accents( $keyword );
		} else {
			// Fallback if WordPress function unavailable (shouldn't happen)
			$keyword = self::fallback_remove_accents( $keyword );
		}

		// Step 3: Remove special characters (keep letters, numbers, spaces)
		$keyword = preg_replace( '/[^\p{L}\p{N}\s]/u', ' ', $keyword );

		// Step 4: Normalize whitespace
		$keyword = trim( preg_replace( '/\s+/', ' ', $keyword ) );

		// Step 5: Remove stop words (articles, prepositions)
		$keyword = self::remove_stop_words( $keyword );

		// Step 6: Apply synonym normalization
		$keyword = self::normalize_synonyms( $keyword );

		// Step 7: Lemmatize remaining tokens (plural → singular)
		$tokens = explode( ' ', $keyword );
		$tokens = array_map( array( self::class, 'lemmatize_word' ), $tokens );

		// Step 8: Sort tokens alphabetically (word order independence)
		// This ensures "maman poeme" === "poeme maman"
		sort( $tokens );

		$keyword = implode( ' ', $tokens );

		// Step 9: Final whitespace cleanup
		$keyword = trim( preg_replace( '/\s+/', ' ', $keyword ) );

		return $keyword;
	}

	/**
	 * Remove stop words (articles, prepositions, common words)
	 * in multiple languages
	 *
	 * @param string $text
	 * @return string Text with stop words removed
	 */
	private static function remove_stop_words( $text ) {
		// Multi-language stop words (articles + prepositions)
		$stop_words = array(
			// French
			'le',
			'la',
			'les',
			'un',
			'une',
			'des',
			'pour',
			'sur',
			'de',
			'du',
			'en',
			'a',
			'avec',
			'sans',
			'dans',
			'par',
			'et',
			'ou',

			// English
			'the',
			'a',
			'an',
			'for',
			'on',
			'of',
			'in',
			'at',
			'with',
			'without',
			'by',
			'and',
			'or',

			// Spanish
			'el',
			'la',
			'los',
			'las',
			'uno',
			'una',
			'unos',
			'unas',
			'para',
			'sobre',
			'en',
			'con',
			'sin',
			'por',
			'y',
			'o',

			// German
			'der',
			'die',
			'das',
			'den',
			'dem',
			'ein',
			'eine',
			'fur',
			'auf',
			'von',
			'in',
			'an',
			'mit',
			'ohne',
			'und',
			'oder',

			// Italian
			'il',
			'lo',
			'la',
			'i',
			'gli',
			'le',
			'per',
			'su',
			'di',
			'da',
			'con',
			'senza',
			'e',
			'o',
		);

		$tokens   = explode( ' ', $text );
		$filtered = array_filter(
			$tokens,
			function ( $token ) use ( $stop_words ) {
				return ! in_array( $token, $stop_words, true );
			}
		);

		return implode( ' ', $filtered );
	}

	/**
	 * Normalize synonyms to canonical forms
	 *
	 * Synonym groups are treated as identical for clustering:
	 * - poème / poésie → poeme
	 * - histoire / conte / récit → histoire
	 * - maman / mère → maman
	 * - enfant / bébé / petit → enfant
	 *
	 * @param string $text
	 * @return string Text with synonyms normalized
	 */
	private static function normalize_synonyms( $text ) {
		// Synonym groups: [canonical => [synonym1, synonym2, ...]]
		// After accent removal, so use non-accented forms
		$synonym_groups = array(
			// Poem/Poetry
			'poeme'    => array( 'poeme', 'poesie', 'vers', 'rime' ),

			// Story/Tale
			'histoire' => array( 'histoire', 'conte', 'recit', 'fable' ),

			// Mother/Mom
			'maman'    => array( 'maman', 'mere', 'mère' ), // Include both for safety

			// Child/Baby
			'enfant'   => array( 'enfant', 'bebe', 'petit', 'gosse' ),

			// Story (English)
			'story'    => array( 'story', 'tale', 'narrative' ),

			// Child (English)
			'child'    => array( 'child', 'kid', 'baby', 'toddler' ),

			// Mother (English)
			'mom'      => array( 'mom', 'mother', 'mum', 'mama', 'mommy' ),

			// Father
			'papa'     => array( 'papa', 'pere', 'dad', 'father' ),
		);

		$tokens = explode( ' ', $text );

		foreach ( $tokens as $i => $token ) {
			// Check each synonym group
			foreach ( $synonym_groups as $canonical => $synonyms ) {
				if ( in_array( $token, $synonyms, true ) ) {
					$tokens[ $i ] = $canonical;
					break; // Found match, move to next token
				}
			}
		}

		return implode( ' ', $tokens );
	}

	/**
	 * Fallback accent removal if WordPress remove_accents() unavailable
	 *
	 * @param string $text
	 * @return string Text without accents
	 */
	private static function fallback_remove_accents( $text ) {
		$accent_map = array(
			'à' => 'a',
			'á' => 'a',
			'â' => 'a',
			'ã' => 'a',
			'ä' => 'a',
			'å' => 'a',
			'è' => 'e',
			'é' => 'e',
			'ê' => 'e',
			'ë' => 'e',
			'ì' => 'i',
			'í' => 'i',
			'î' => 'i',
			'ï' => 'i',
			'ò' => 'o',
			'ó' => 'o',
			'ô' => 'o',
			'õ' => 'o',
			'ö' => 'o',
			'ù' => 'u',
			'ú' => 'u',
			'û' => 'u',
			'ü' => 'u',
			'ý' => 'y',
			'ÿ' => 'y',
			'ñ' => 'n',
			'ç' => 'c',
			'À' => 'A',
			'Á' => 'A',
			'Â' => 'A',
			'Ã' => 'A',
			'Ä' => 'A',
			'Å' => 'A',
			'È' => 'E',
			'É' => 'E',
			'Ê' => 'E',
			'Ë' => 'E',
			'Ì' => 'I',
			'Í' => 'I',
			'Î' => 'I',
			'Ï' => 'I',
			'Ò' => 'O',
			'Ó' => 'O',
			'Ô' => 'O',
			'Õ' => 'O',
			'Ö' => 'O',
			'Ù' => 'U',
			'Ú' => 'U',
			'Û' => 'U',
			'Ü' => 'U',
			'Ý' => 'Y',
			'Ñ' => 'N',
			'Ç' => 'C',
		);

		return strtr( $text, $accent_map );
	}

	/**
	 * Calculate token overlap ratio (Jaccard similarity)
	 */
	private static function calculate_token_overlap( $kw1, $kw2 ) {
		$tokens1 = explode( ' ', $kw1 );
		$tokens2 = explode( ' ', $kw2 );

		if ( empty( $tokens1 ) || empty( $tokens2 ) ) {
			return 0.0;
		}

		$intersection = count( array_intersect( $tokens1, $tokens2 ) );
		$union        = count( array_unique( array_merge( $tokens1, $tokens2 ) ) );

		return $union > 0 ? ( $intersection / $union ) : 0.0;
	}

	/**
	 * Calculate Levenshtein similarity (normalized edit distance)
	 */
	private static function calculate_levenshtein_similarity( $kw1, $kw2 ) {
		$max_len = max( strlen( $kw1 ), strlen( $kw2 ) );

		if ( $max_len === 0 ) {
			return 1.0;
		}

		$distance = levenshtein( $kw1, $kw2 );

		// Normalize to 0-1 range (1 = identical, 0 = completely different)
		return 1 - ( $distance / $max_len );
	}

	/**
	 * Calculate Jaro-Winkler similarity
	 *
	 * @param string $str1
	 * @param string $str2
	 * @return float Similarity score (0.0 to 1.0)
	 */
	private static function calculate_jaro_winkler( $str1, $str2 ) {
		$jaro = self::calculate_jaro( $str1, $str2 );

		// Jaro-Winkler adds bonus for matching prefix
		$prefix_length = 0;
		$max_prefix    = min( 4, min( strlen( $str1 ), strlen( $str2 ) ) );

		for ( $i = 0; $i < $max_prefix; $i++ ) {
			if ( $str1[ $i ] === $str2[ $i ] ) {
				++$prefix_length;
			} else {
				break;
			}
		}

		$jaro_winkler = $jaro + ( $prefix_length * 0.1 * ( 1 - $jaro ) );

		return min( $jaro_winkler, 1.0 );
	}

	/**
	 * Calculate Jaro similarity
	 */
	private static function calculate_jaro( $str1, $str2 ) {
		$len1 = strlen( $str1 );
		$len2 = strlen( $str2 );

		if ( $len1 === 0 && $len2 === 0 ) {
			return 1.0;
		}

		if ( $len1 === 0 || $len2 === 0 ) {
			return 0.0;
		}

		$match_distance = max( $len1, $len2 ) / 2 - 1;
		$match_distance = max( 0, $match_distance );

		$str1_matches = array_fill( 0, $len1, false );
		$str2_matches = array_fill( 0, $len2, false );

		$matches        = 0;
		$transpositions = 0;

		// Identify matches
		for ( $i = 0; $i < $len1; $i++ ) {
			$start = max( 0, $i - $match_distance );
			$end   = min( $i + $match_distance + 1, $len2 );

			for ( $j = $start; $j < $end; $j++ ) {
				if ( $str2_matches[ $j ] || $str1[ $i ] !== $str2[ $j ] ) {
					continue;
				}

				$str1_matches[ $i ] = true;
				$str2_matches[ $j ] = true;
				++$matches;
				break;
			}
		}

		if ( $matches === 0 ) {
			return 0.0;
		}

		// Count transpositions
		$k = 0;
		for ( $i = 0; $i < $len1; $i++ ) {
			if ( ! $str1_matches[ $i ] ) {
				continue;
			}

			while ( ! $str2_matches[ $k ] ) {
				++$k;
			}

			if ( $str1[ $i ] !== $str2[ $k ] ) {
				++$transpositions;
			}

			++$k;
		}

		$jaro = (
			( $matches / $len1 ) +
			( $matches / $len2 ) +
			( ( $matches - $transpositions / 2 ) / $matches )
		) / 3;

		return $jaro;
	}

	/**
	 * Calculate phonetic similarity using metaphone
	 */
	private static function calculate_phonetic_similarity( $kw1, $kw2 ) {
		// Split into words and compare phonetics
		$words1 = explode( ' ', $kw1 );
		$words2 = explode( ' ', $kw2 );

		$phonetic1 = array_map(
			function ( $word ) {
				return metaphone( $word );
			},
			$words1
		);

		$phonetic2 = array_map(
			function ( $word ) {
				return metaphone( $word );
			},
			$words2
		);

		sort( $phonetic1 );
		sort( $phonetic2 );

		// Compare sorted phonetic arrays
		if ( $phonetic1 === $phonetic2 ) {
			return 1.0;
		}

		// Calculate partial match
		$intersection = count( array_intersect( $phonetic1, $phonetic2 ) );
		$union        = count( array_unique( array_merge( $phonetic1, $phonetic2 ) ) );

		return $union > 0 ? ( $intersection / $union ) : 0.0;
	}

	/**
	 * Generate token-sorted key for fast exact duplicate detection
	 *
	 * @param string $keyword
	 * @return string Normalized token-sorted key
	 */
	public static function generate_token_key( $keyword ) {
		$normalized = self::normalize_for_comparison( $keyword );

		// Lemmatize tokens (singular/plural normalization)
		$tokens     = explode( ' ', $normalized );
		$lemmatized = array_map( array( self::class, 'lemmatize_word' ), $tokens );

		// Sort alphabetically
		sort( $lemmatized );

		return implode( ' ', $lemmatized );
	}

	/**
	 * Lemmatize word (convert plurals to singular, etc.)
	 * Enhanced with French and multi-language support
	 *
	 * @param string $word
	 * @return string Lemmatized word
	 */
	private static function lemmatize_word( $word ) {
		// Irregular plurals (English)
		$irregulars = array(
			'children' => 'child',
			'people'   => 'person',
			'mice'     => 'mouse',
			'feet'     => 'foot',
			'teeth'    => 'tooth',
			'geese'    => 'goose',
			'men'      => 'man',
			'women'    => 'woman',
			'oxen'     => 'ox',
			'sheep'    => 'sheep',
			'fish'     => 'fish',
			'deer'     => 'deer',
			'series'   => 'series',
			'species'  => 'species',

			// French irregular plurals (after accent removal)
			'yeux'     => 'oeil',
			'chevaux'  => 'cheval',
			'travaux'  => 'travail',
		);

		if ( isset( $irregulars[ $word ] ) ) {
			return $irregulars[ $word ];
		}

		// Skip very short words
		if ( strlen( $word ) <= 3 ) {
			return $word;
		}

		// French plural rules (checked first)
		// -eaux → -eau (beaux → beau, châteaux → château)
		if ( substr( $word, -4 ) === 'eaux' ) {
			return substr( $word, 0, -1 ); // Remove just 'x'
		}

		// -aux → -al (animaux → animal, journaux → journal)
		if ( substr( $word, -3 ) === 'aux' ) {
			return substr( $word, 0, -3 ) . 'al';
		}

		// -eux → -eu (jeux → jeu, cheveux → cheveu)
		if ( substr( $word, -3 ) === 'eux' ) {
			return substr( $word, 0, -1 ); // Remove just 'x'
		}

		// English plural rules
		// -ies → -y (stories → story, poésies → poésie)
		if ( substr( $word, -3 ) === 'ies' ) {
			return substr( $word, 0, -3 ) . 'y';
		}

		// -ves → -f/-fe (wolves → wolf, knives → knife)
		if ( substr( $word, -3 ) === 'ves' ) {
			$base = substr( $word, 0, -3 );
			// Try -fe first
			if ( in_array( $base . 'fe', array( 'knife', 'wife', 'life' ) ) ) {
				return $base . 'fe';
			}
			return $base . 'f';
		}

		// -ses → -s (grasses → grass)
		if ( substr( $word, -3 ) === 'ses' ) {
			return substr( $word, 0, -2 );
		}

		// -mes → -me (poemes → poeme, but check first to avoid "games" → "game")
		if ( substr( $word, -3 ) === 'mes' && strlen( $word ) > 5 ) {
			return substr( $word, 0, -1 ); // Just remove 's'
		}

		// French double-consonant + es → just remove 's'
		// This catches: -lles, -nnes, -ttes, -rres, -sses, etc.
		if ( substr( $word, -2 ) === 'es' && strlen( $word ) > 5 ) {
			$char_before_es   = substr( $word, -3, 1 );
			$char_before_that = substr( $word, -4, 1 );

			// If double consonant before 'es', just remove 's'
			if ( $char_before_es === $char_before_that ) {
				return substr( $word, 0, -1 ); // villes → ville, personnes → personne
			}
		}

		// French adjectives ending in -tes → -te (petites → petite, courtes → courte)
		if ( substr( $word, -3 ) === 'tes' && strlen( $word ) > 5 ) {
			$before_tes = substr( $word, -4, 1 );
			// Only if NOT double 't' (already handled above)
			if ( 't' !== $before_tes ) {
				return substr( $word, 0, -1 ); // Just remove 's'
			}
		}

		// French adjectives ending in -res → -re (histoires but also autres, etc.)
		if ( substr( $word, -3 ) === 'res' && strlen( $word ) > 5 ) {
			$before_res = substr( $word, -4, 1 );
			// Only if NOT double 'r'
			if ( 'r' !== $before_res ) {
				return substr( $word, 0, -1 ); // histoires → histoire
			}
		}

		// French adjectives ending in -ges → -ge (sauvages → sauvage, rouges → rouge)
		if ( substr( $word, -3 ) === 'ges' && strlen( $word ) > 5 ) {
			return substr( $word, 0, -1 ); // Just remove 's'
		}

		// French adjectives with double vowel + s (agees → agee, idees → idee)
		if ( substr( $word, -3 ) === 'ees' && strlen( $word ) >= 5 ) {
			return substr( $word, 0, -1 ); // agees → agee (just remove 's')
		}

		// -es → remove (boxes → box for English, but avoid breaking French)
		if ( substr( $word, -2 ) === 'es' ) {
			$base             = substr( $word, 0, -2 );
			$char_before_es   = substr( $word, -3, 1 );
			$char_before_that = substr( $word, -4, 1 );

			// Don't remove if double letter before 'es' (already handled above)
			if ( $char_before_es === $char_before_that ) {
				// Skip - double consonant/vowel rules already handled this
			} elseif ( strlen( $base ) >= 3 ) {
				return $base; // boxes → box, churches → church
			}
		}

		// -s → remove (cats → cat, poemes → poeme if -es didn't catch it)
		// But don't touch -ss or -us endings, or very short words
		if ( substr( $word, -1 ) === 's' && ! in_array( substr( $word, -2 ), array( 'ss', 'us' ) ) && strlen( $word ) > 3 ) {
			return substr( $word, 0, -1 );
		}

		return $word;
	}

	/**
	 * Check if keywords should be considered duplicates based on metrics
	 *
	 * @param array $keyword1 Keyword data with metrics
	 * @param array $keyword2 Keyword data with metrics
	 * @return bool True if metrics indicate they're duplicates
	 */
	public static function are_metrics_similar( $keyword1, $keyword2 ) {
		// Must have same intent
		$intent1 = strtolower( $keyword1['search_intent'] ?? 'informational' );
		$intent2 = strtolower( $keyword2['search_intent'] ?? 'informational' );

		if ( $intent1 !== $intent2 ) {
			return false;
		}

		// Volume comparison (within ±20% OR exactly the same)
		$vol1 = intval( $keyword1['search_volume'] ?? 0 );
		$vol2 = intval( $keyword2['search_volume'] ?? 0 );

		if ( $vol1 > 0 && $vol2 > 0 ) {
			$max_volume  = max( $vol1, $vol2 );
			$volume_diff = abs( $vol1 - $vol2 ) / $max_volume;

			// Allow if volumes are very close OR identical
			if ( $volume_diff > 0.20 && $vol1 !== $vol2 ) {
				return false;
			}
		} elseif ( $vol1 === 0 && $vol2 === 0 ) {
			// Both have no data - allow clustering
		} else {
			// One has data, other doesn't - different confidence levels
			return false;
		}

		// Difficulty comparison (within ±15%)
		$diff1 = intval( $keyword1['difficulty'] ?? 0 );
		$diff2 = intval( $keyword2['difficulty'] ?? 0 );

		if ( $diff1 > 0 && $diff2 > 0 ) {
			$difficulty_diff = abs( $diff1 - $diff2 );
			if ( $difficulty_diff > 15 ) {
				return false;
			}
		}

		return true;
	}

	/**
	 * Get detailed similarity breakdown (for debugging/UI)
	 *
	 * @param string $keyword1
	 * @param string $keyword2
	 * @return array Similarity scores by algorithm
	 */
	public static function get_similarity_breakdown( $keyword1, $keyword2 ) {
		$kw1 = self::normalize_for_comparison( $keyword1 );
		$kw2 = self::normalize_for_comparison( $keyword2 );

		return array(
			'token_overlap' => round( self::calculate_token_overlap( $kw1, $kw2 ) * 100, 1 ),
			'levenshtein'   => round( self::calculate_levenshtein_similarity( $kw1, $kw2 ) * 100, 1 ),
			'jaro_winkler'  => round( self::calculate_jaro_winkler( $kw1, $kw2 ) * 100, 1 ),
			'phonetic'      => round( self::calculate_phonetic_similarity( $kw1, $kw2 ) * 100, 1 ),
			'overall'       => round( self::calculate_similarity( $keyword1, $keyword2 ) * 100, 1 ),
		);
	}

	/**
	 * Batch similarity calculation with progress tracking
	 * Uses language-specific thresholds for better accuracy
	 *
	 * @param array    $keywords Array of keyword data
	 * @param callable $progress_callback Function called with progress updates
	 * @return array Similarity matrix
	 */
	public static function calculate_similarity_matrix( $keywords, $progress_callback = null ) {
		$matrix            = array();
		$total             = count( $keywords );
		$comparisons       = 0;
		$total_comparisons = ( $total * ( $total - 1 ) ) / 2;

		for ( $i = 0; $i < $total; $i++ ) {
			for ( $j = $i + 1; $j < $total; $j++ ) {
				// Calculate similarity
				$similarity = self::calculate_similarity(
					$keywords[ $i ]['keyword'],
					$keywords[ $j ]['keyword']
				);

				// Get language-specific threshold
				$lang      = $keywords[ $i ]['language_code'] ?? 'en';
				$threshold = self::get_threshold_for_language( $lang );

				// Only store if above language-specific threshold
				if ( $similarity >= $threshold ) {
					$matrix[] = array(
						'id1'        => $keywords[ $i ]['id'],
						'id2'        => $keywords[ $j ]['id'],
						'keyword1'   => $keywords[ $i ]['keyword'],
						'keyword2'   => $keywords[ $j ]['keyword'],
						'similarity' => $similarity,
					);
				}

				++$comparisons;

				if ( $progress_callback && $comparisons % 100 === 0 ) {
					call_user_func(
						$progress_callback,
						array(
							'progress'      => round( ( $comparisons / $total_comparisons ) * 100, 1 ),
							'comparisons'   => $comparisons,
							'total'         => $total_comparisons,
							'matches_found' => count( $matrix ),
						)
					);
				}
			}
		}

		return $matrix;
	}

	/**
	 * Fast pre-filter to reduce comparison pairs
	 *
	 * Uses sorted token fingerprint instead of first letter to catch reorders
	 *
	 * @param array $keywords
	 * @return array Groups of keywords that might be similar
	 */
	public static function pre_filter_candidates( $keywords ) {
		$groups = array();

		foreach ( $keywords as $keyword ) {
			// Create filter key using sorted tokens (catches reorders!)
			$normalized = self::normalize_for_comparison( $keyword['keyword'] );
			$tokens     = explode( ' ', $normalized );
			$word_count = count( $tokens );
			$lang       = $keyword['language_code'] ?? 'en';

			// Sort tokens to create fingerprint
			// "short funny bedtime stories" → "bedtime funny short stories"
			// "funny short bedtime stories" → "bedtime funny short stories"
			// Both get SAME fingerprint!
			sort( $tokens );
			$token_fingerprint = implode( '_', array_slice( $tokens, 0, 3 ) ); // Use first 3 sorted tokens

			$filter_key = $lang . '_' . $word_count . '_' . $token_fingerprint;

			if ( ! isset( $groups[ $filter_key ] ) ) {
				$groups[ $filter_key ] = array();
			}

			$groups[ $filter_key ][] = $keyword;
		}

		return $groups;
	}
}
