<?php
# WARNING: This file is publicly viewable on the web. Do not put private data here.

# This file holds the CirrusSearch configuration which is common to all realms,
# i.e. settings that apply to both production and Beta Cluster.
#
# NOTE: Included for all wikis.
#
# Load tree:
#  |-- wmf-config/CommonSettings.php
#      |
#      `-- wmf-config/CirrusSearch-common.php
#
# If you need to reference an IP address, use the realm-specific
# files: CirrusSearch-labs.php, or CirrusSearch-production.php
#
# See: https://wikitech.wikimedia.org/wiki/Search
#
# Contact Wikimedia Operations or Wikimedia Discovery for more details.

$wgSearchType = 'CirrusSearch';

$wgCirrusSearchElasticQuirks = [];

# Turn off leading wildcard matches, they are a very slow and inefficient query
$wgCirrusSearchAllowLeadingWildcard = false;

# Turn off the more accurate but slower search mode.  It is most helpful when you
# have many small shards.  We don't do that in production and we could use the speed.
$wgCirrusSearchMoreAccurateScoringMode = false;

# Limit the number of states generated by wildcard queries (500 will allow about 20 wildcards)
$wgCirrusSearchQueryStringMaxDeterminizedStates = 500;

# Lower the timeouts - the defaults are too high and allow to scan too many
# pages. 40s shard timeout for regex allowed to deep scan 9million pages for
# insource:/the/ on commons. Keep client timeout relatively high in comparaison,
# but not higher than 60sec as it's the max time allowed for GET requests.
# we really don't want to timeout the client before the shard retrieval (we may
# release the poolcounter before the end of the query on the backend)
$wgCirrusSearchSearchShardTimeout[ 'regex' ] = '15s';
// GET requests timeout at 60s, give some room to treat request timeout (T216860)
$wgCirrusSearchClientSideSearchTimeout[ 'regex' ] = 50;
$wgCirrusSearchSearchShardTimeout[ 'default' ] = '10s';
$wgCirrusSearchClientSideSearchTimeout[ 'default' ] = 40;

# Set the backoff for Cirrus' job that reacts to template changes - slow and steady
# will help prevent spikes in Elasticsearch load.
// $wgJobBackoffThrottling['cirrusSearchLinksUpdate'] = 5;  -- disabled, Ori 3-Dec-2015
# Also engage a delay for the Cirrus job that counts incoming links to pages when
# pages are newly linked or unlinked.  Too many link count queries at once could flood
# Elasticsearch.
// $wgJobBackoffThrottling['cirrusSearchIncomingLinkCount'] = 1; -- disabled, Ori 3-Dec-2015

# Ban the hebrew plugin, it is unstable
$wgCirrusSearchBannedPlugins = [ 'elasticsearch-analysis-hebrew' ];

# Build and use an ngram index for faster regex matching
$wgCirrusSearchWikimediaExtraPlugin = [
	'regex' => [
		'build',
		'use',
	],
	'super_detect_noop' => true,
	'documentVersion' => true,
	'token_count_router' => true,
	'term_freq' => true
];

# Enable the "experimental" highlighter on all wikis
$wgCirrusSearchUseExperimentalHighlighter = true;
$wgCirrusSearchOptimizeIndexForExperimentalHighlighter = true;

// We had an incident of filling up the entire clusters redis instances after
// 6 hours, and problems with the new kafka (before we enabled compression)
// based job queue as well. Cut the time down to 2 hours, as this is not supposed
// to ride out a full-fledged outage, but paper over minor unavailabilities
// for cluster/network/etc maintenance.
$wgCirrusSearchDropDelayedJobsAfter = 60 * 60 * 2;

// Commons is special
if ( $wgDBname == 'commonswiki' ) {
	$wgCirrusSearchNamespaceMappings = [ NS_FILE => 'file' ];
} elseif ( $wgDBname == 'officewiki' || $wgDBname == 'foundationwiki' ) {
	// T94856 - makes searching difficult for locally uploaded files
	// T76957 - doesn't make sense to have Commons files on foundationwiki search
} else { // So is everyone else, for using commons
	$wgCirrusSearchExtraIndexes = [ NS_FILE => [ 'commonswiki_file' ] ];
	$wgCirrusSearchExtraIndexBoostTemplates = [
		'commonswiki_file' => [
			'wiki' => 'commonswiki',
			'boosts' => [
				// Copied from https://commons.wikimedia.org/wiki/MediaWiki:Cirrussearch-boost-templates
				'Template:Assessments/commons/featured' => 2.5,
				'Template:Picture_of_the_day' => 1.5,
				'Template:Valued_image' => 1.75,
				'Template:Assessments' => 1.5,
				'Template:Quality_image' => 1.75,
			],
		],
	];
}

// TextCat models
$wgCirrusSearchTextcatModel = [ "$IP/vendor/wikimedia/textcat/LM-query", "$IP/vendor/wikimedia/textcat/LM" ];
$wgCirrusSearchTextcatConfig = [
	'maxNgrams' => 9000,
	'maxReturnedLanguages' => 1,
	'resultsRatio' => 1.06,
	'minInputLength' => 3,
	'maxProportion' => 0.85,
	'langBoostScore' => 0.14,
	'numBoostedLangs' => 2,
];

$wgHooks['CirrusSearchMappingConfig'][] = function ( array &$config, $mappingConfigBuilder ) {
	$config['page']['properties']['popularity_score'] = [
		'type' => 'double',
	];
};

// Set the scoring method
$wgCirrusSearchCompletionDefaultScore = 'popqual';

// PoolCounter needs to be adjusted to account for additional latency when default search
// is pointed at a remote datacenter. Currently this makes the assumption that it will either
// be eqiad or codfw which have ~40ms latency between them. Multiples are chosen using
// (p75 + cross dc latency)/p75
if ( $wgCirrusSearchDefaultCluster !== $wmfDatacenter ) {
	// prefix has p75 of ~30ms
	if ( isset( $wgPoolCounterConf[ 'CirrusSearch-Prefix' ] ) ) {
		$wgPoolCounterConf['CirrusSearch-Prefix']['workers'] *= 2;
	}
	// namespace has a p75 of ~15ms
	if ( isset( $wgPoolCounterConf['CirrusSearch-NamespaceLookup' ] ) ) {
		$wgPoolCounterConf['CirrusSearch-NamespaceLookup']['workers'] *= 3;
	}
	// completion has p75 of ~30ms
	if ( isset( $wgPoolCounterConf['CirrusSearch-Completion'] ) ) {
		$wgPoolCounterConf['CirrusSearch-Completion']['workers'] *= 2;
	}
}

// Aliases for filetype: search
$wgCirrusSearchFiletypeAliases = [
	"pdf" => "office",
	"ppt" => "office",
	"doc" => "office",
	"jpg" => "bitmap",
	"image" => "bitmap",
	"webp" => "bitmap",
	"mp3" => "audio",
	"svg" => "drawing"
];

// Enable the new layout, FIXME: remove the old one
$wgCirrusSearchNewCrossProjectPage = true;
// Display X results per crossproject
$wgCirrusSearchNumCrossProjectSearchResults = 1;
// Load other project config via cirrus dump config API
$wgCirrusSearchFetchConfigFromApi = true;

// Enable the search relevance survey where configured
$wgWMESearchRelevancePages = $wmgWMESearchRelevancePages;

$wgCirrusSearchRescoreProfiles = [];
if ( $wmgCirrusSearchMLRModel ) {
	if ( !is_array( $wmgCirrusSearchMLRModel ) ) {
		$wmgCirrusSearchMLRModel = [ 'mlr-1024rs' => $wmgCirrusSearchMLRModel ];
	}
	foreach ( $wmgCirrusSearchMLRModel as $name => $mlrModel ) {
		// LTR Rescore profile
		$wgCirrusSearchRescoreProfiles[$name] = [
			'i18n_msg' => 'cirrussearch-qi-profile-wsum-inclinks-pv',
			'supported_namespaces' => 'content',
			'unsupported_syntax' => [ 'full_text_querystring', 'query_string', 'filter_only' ],
			'fallback_profile' => $wmgCirrusSearchMLRModelFallback,
			'rescore' => [
				[
					'window' => 8192,
					'window_size_override' => 'CirrusSearchFunctionRescoreWindowSize',
					'query_weight' => 1.0,
					'rescore_query_weight' => 1.0,
					'score_mode' => 'total',
					'type' => 'function_score',
					'function_chain' => 'wsum_inclinks_pv'
				],
				[
					'window' => 8192,
					'window_size_override' => 'CirrusSearchFunctionRescoreWindowSize',
					'query_weight' => 1.0,
					'rescore_query_weight' => 1.0,
					'score_mode' => 'multiply',
					'type' => 'function_score',
					'function_chain' => 'optional_chain'
				],
				[
					'window' => 1024,
					'query_weight' => 1.0,
					'rescore_query_weight' => 10000.0,
					'score_mode' => 'total',
					'type' => 'ltr',
					'model' => $mlrModel
				],
			],
		];
	}
}

// Set SPARQL endpoint for categories
$wgCirrusSearchCategoryEndpoint = 'https://query.wikidata.org/bigdata/namespace/categories/sparql';

# Load per realm specific configuration, either:
# - CirrusSearch-labs.php
# - CirrusSearch-production.php
#
require "{$wmfConfigDir}/CirrusSearch-{$wmfRealm}.php";