/**
 * Copyright (c) 2025 SoftCorporation LLC. All rights reserved.
 *
 * The Software License, Version 1.0
 *
 * SoftCorporation LLC. grants you ("Licensee") a non-exclusive, royalty free,
 * license to use, modify and redistribute this software in source and binary
 * code form, provided that the following conditions are met:
 *
 * 1. The end-user documentation included with the redistribution,
 *    if any, must include the following acknowledgment:
 *       "This product includes software developed by the
 *        SoftCorporation LLC. (http://www.softcorporation.com)."
 *    Alternately, this acknowledgment may appear in the software itself,
 *    if and wherever such third-party acknowledgments normally appear.
 *
 * 2. This software and SoftCorporation name must not be used to
 *    promote products derived from the software without prior
 *    written permission. For written permission, please contact
 *    info@softcorporation.com.
 *
 * This software is provided "AS IS," without a warranty of any kind.
 * ALL EXPRESS OR IMPLIED CONDITIONS, REPRESENTATIONS AND WARRANTIES,
 * INCLUDING ANY IMPLIED WARRANTY OF MERCHANTABILITY, FITNESS FOR A
 * PARTICULAR PURPOSE OR NON-INFRINGEMENT, ARE HEREBY EXCLUDED.
 * IN NO EVENT SHALL THE SOFTCORPORATION BE LIABLE FOR ANY DIRECT,
 * INDIRECT, INCIDENTAL OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF
 * USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION).
 *
 */
package com.softcorporation.sandi.base.service;

import java.io.IOException;
import java.math.BigDecimal;
import java.net.URLEncoder;
import java.util.ArrayList;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

import org.json.JSONArray;
import org.json.JSONObject;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.stereotype.Service;

import com.softcorporation.sandi.base.model.Client;
import com.softcorporation.sandi.base.model.Token;
import com.softcorporation.sandi.base.util.Constants;
import com.softcorporation.sandi.base.util.Utils;
import com.softcorporation.util.StringUtil;
import com.softcorporation.util.VecUtil;

@Service
public class LanguageModelService {
    
    private static final Logger LOGGER = LoggerFactory.getLogger(LanguageModelService.class);

	/**
	 * LLM service url
	 */
    @Value("${sandi.service.llm.url:}")
    private String llmUrl;

    /**
	 * LLM service timeout
	 */
    @Value("${sandi.service.llm.timeout:500}")
    private int llmTimeout;

    /**
	 * Embedding service url
	 */
    @Value("${sandi.service.emb.url:}")
    private String embUrl;

    /**
	 * Embedding service timeout
	 */
    @Value("${sandi.service.emb.timeout:100}")
    private int embTimeout;

	/**
	 * NLP service url
	 */
    @Value("${sandi.service.nlp.url:}")
    private String nlpUrl;

    /**
	 * NLP service timeout
	 */
    @Value("${sandi.service.nlp.timeout:100}")
    private int nlpTimeout;

    /**
	 * Reranking service url
	 */
    @Value("${sandi.service.rer.url:}")
    private String rerUrl;

    /**
	 * Reranking service timeout
	 */
    @Value("${sandi.service.rer.timeout:500}")
    private int rerTimeout;

    /**
	 * Max cache entries
	 */
    @Value("${sandi.service.cache.max:100}")
    private int cacheMax;

    /**
	 * Embedding byte scale
	 */
    @Value("${sandi.embedding.byte.scale:500}")
    private int embByteScale;
    
    /**
	 * Top words limit
	 */
    @Value("${sandi.search.keywords.max:4}")
    private int searchKeywordsMax;

    private String embUrl(Client client) {
        String url = client != null ? client.getEmbServiceUrl() : null;
        return (url != null && !url.isEmpty()) ? url : embUrl;
    }

    private String llmUrl(Client client) {
        String url = client != null ? client.getLlmServiceUrl() : null;
        return (url != null && !url.isEmpty()) ? url : llmUrl;
    }

    private String nlpUrl(Client client) {
        String url = client != null ? client.getNlpServiceUrl() : null;
        return (url != null && !url.isEmpty()) ? url : nlpUrl;
    }

    private String rerUrl(Client client) {
        String url = client != null ? client.getRerServiceUrl() : null;
        return (url != null && !url.isEmpty()) ? url : rerUrl;
    }

    Map<String, Object> spellCache = new LinkedHashMap<>(cacheMax, .75F, true) {
		@Override
		public boolean removeEldestEntry(Map.Entry<String, Object> eldest) {
            return size() > cacheMax;
        }
    };

	Map<String, Object> ragCache = new LinkedHashMap<>(cacheMax, .75F, true) {
		@Override
		public boolean removeEldestEntry(Map.Entry<String, Object> eldest) {
            return size() > cacheMax;
        }
    };

	Map<String, Object> expandCache = new LinkedHashMap<>(cacheMax, .75F, true) {
		@Override
		public boolean removeEldestEntry(Map.Entry<String, Object> eldest) {
            return size() > cacheMax;
        }
    };

	Map<String, Object> keywordCache = new LinkedHashMap<>(cacheMax, .75F, true) {
		@Override
		public boolean removeEldestEntry(Map.Entry<String, Object> eldest) {
            return size() > cacheMax;
        }
    };

	/**
	 * Get embedding for texts.
	 * 
	 * @param texts List of strings
	 * @return embedding
	 * @throws IOException 
	 */
	public List<double[]> getTextsEmbedding(final Client client, final List<String> texts) throws IOException {
        LOGGER.debug("Getting {} embedding(s) of text", texts.size());
		JSONObject jsonRequest = new JSONObject();
		jsonRequest.put("texts", new JSONArray(texts));

		String response = Utils.post(embUrl(client) + "/embedding", jsonRequest.toString(), embTimeout);
		JSONObject jsonResponse = new JSONObject(response);
		
		JSONArray jsonArray = jsonResponse.getJSONArray("result");
		ArrayList<double[]> embeddingList = new ArrayList<>();
        for (int n = 0; n < jsonArray.length(); n++) {
            JSONArray array = jsonArray.getJSONArray(n);
        	double[] embedding = new double[array.length()];
        	for (int i = 0; i < array.length(); i++) {
                embedding[i] = array.getDouble(i);
            }
        	embeddingList.add(embedding);
        }
		return embeddingList;
	}

	/**
	 * Get embedding of a search query.
	 * 
	 * @param text String
	 * @return embedding
	 * @throws IOException 
	 */
	public double[] getQueryEmbedding(final Client client, final String text) throws IOException {
        LOGGER.debug("Getting embedding for query: {}", text);

		String url = embUrl(client) + "/embed?isQuery=true&text=" + URLEncoder.encode(text, Constants.ENCODING);
		String response = Utils.get(url, embTimeout);
		JSONObject jsonResponse = new JSONObject(response);
		JSONArray array = jsonResponse.getJSONArray("result");
    	double[] embedding = new double[array.length()];
    	for (int i = 0; i < array.length(); i++) {
            embedding[i] = array.getDouble(i);
        }
    	return embedding;
	}

	/**
	 * Get query embedding as string.
	 * 
	 * @param text String
	 * @return embedding string
	 * @throws IOException
	 */	
	public String getQueryEmbeddingString(final Client client, final String text) throws IOException {
//        LOGGER.debug("Getting query embedding as string for text: {}", text);
		String url = embUrl(client) + "/embed?isQuery=true&text=" + URLEncoder.encode(text, Constants.ENCODING);
		String response = Utils.get(url, embTimeout);
		JSONObject jsonResponse = new JSONObject(response);
		JSONArray array = jsonResponse.getJSONArray("result");
    	double[] embedding = new double[array.length()];
    	for (int i = 0; i < array.length(); i++) {
            embedding[i] = array.getDouble(i);
        }
    	List<Byte> embByteList = VecUtil.toByteList(embedding, embByteScale);
    	StringBuilder sb = new StringBuilder();
		sb.append("[");
    	for (int i = 0; i < embByteList.size(); i++) {
    		if (i > 0) {
    			sb.append(",");
    		}
            sb.append(embByteList.get(i));
        }
		sb.append("]");
		return sb.toString();
	}
	
	/**
	 * Get main words from the search query (individually)
	 * 
	 * @param searchQuery String
	 * @param queryWords String List
	 * @return Main words list
	 * @throws IOException
	 */
	public List<String> getMainWords(final Client client, final String searchQuery, List<String> queryWords) throws IOException {
        LOGGER.debug("Extracting main words from the search query: {}", searchQuery);

        List<Double> similarities = getSimilarity(client, searchQuery, queryWords);

		Double threshold = 0.0;
		for (Double similarity : similarities) {
			threshold += similarity;
        }
		threshold = threshold / similarities.size();
		List<String> mainWords = new ArrayList<>();

		// Build scored list above threshold
		List<Map.Entry<String, Double>> scored = new ArrayList<>();
		for (int i = 0; i < queryWords.size(); i++) {
			String word = queryWords.get(i);
			Double similarity = similarities.get(i);
			if (similarity >= threshold) {
				scored.add(Map.entry(word, similarity));
			}
		}

		// Sort by score descending, take top words
		scored.sort((a, b) -> Double.compare(b.getValue(), a.getValue()));
		List<Map.Entry<String, Double>> topWords = scored.subList(0, Math.min(searchKeywordsMax, scored.size()));
		for (Map.Entry<String, Double> entry : topWords) {
			mainWords.add(entry.getKey());
			if (mainWords.size() >= searchKeywordsMax) {
				break;
			}
		}
		
        LOGGER.debug("Query Main Keywords: {}", mainWords);
		return mainWords;
	}

	/**
	 * Get keywords from the search query
	 * 
	 * @param text String
	 * @return Keywords array
	 * @throws IOException
	 */
	public JSONArray getQueryKeywords(final Client client, final String searchQuery) throws IOException {
        LOGGER.debug("Extracting keywords from the search query: {}", searchQuery);

		JSONArray queryKeywords = getKeywords(searchQuery);

		StringBuilder sb = new StringBuilder();
    	List<String> queryWords = new ArrayList<>();
        for (int i = 0; i < queryKeywords.length(); i++) {
        	JSONObject keyword = queryKeywords.getJSONObject(i);
        	String word = keyword.getString(Constants.TEXT);
        	word = StringUtil.cleanText(word);
        	if (word.length() == 0) {
        		continue;
        	}
        	String type = keyword.getString(Constants.TYPE);
        	if ((Constants.KEYWORD.equals(type) || Constants.INCLUDED.equals(type) || Constants.QUOTED.equals(type)) && !queryWords.contains(word)) {
//        		if (Constants.KEYWORD.equals(type)) {
        			queryWords.add(word);
//        		}
        		if(sb.length() > 0) {
        			sb.append(" ");
        		}
        		sb.append(word);
    		}
        }
        String positiveSearchQuery = sb.toString();
        
        // single word
        if (queryWords.size() <= 1) {
        	return queryKeywords;
        }

        JSONObject nlpObject = getNlpObject(client, positiveSearchQuery);
		if (nlpObject != null) {
			JSONArray sentences = nlpObject.getJSONArray("sentences");
			for (Object sentenceObject : sentences) {
				JSONObject sentence = (JSONObject) sentenceObject;
				JSONArray tokens = sentence.getJSONArray("tokens");
				for (int i = 0; i < tokens.length(); i++) {
					JSONObject token = tokens.getJSONObject(i);
					String pos = token.getString(Constants.POS);
					String text = token.getString(Constants.TEXT);
//					int idx = token.getInt(Constants.IDX);
					if ("PROPN".equals(pos)) {
				        for (int j = 0; j < queryKeywords.length(); j++) {
				        	JSONObject keyword = queryKeywords.getJSONObject(j);
				        	String word = keyword.getString(Constants.TEXT);
				        	String type = keyword.getString(Constants.TYPE);
				        	if (word.contains(text) && Constants.KEYWORD.equals(type)) {
								keyword.put(Constants.TYPE, Constants.ENTITY);
				    		}
				        }
    				}
				}
			}
		}

		List<String> mainWords = getMainWords(client, positiveSearchQuery, queryWords);
		String topWord = "";
		if (mainWords.size() > 0) {
			topWord = mainWords.get(0);
		}

        for (int i = 0; i < queryKeywords.length(); i++) {
        	JSONObject keyword = queryKeywords.getJSONObject(i);
        	String word = keyword.getString(Constants.TEXT);
        	String type = keyword.getString(Constants.TYPE);
        	if (Constants.KEYWORD.equals(type)) {
        		if (word.equals(topWord)) {
    				keyword.put(Constants.TYPE, Constants.ENTITY);
        		} else if (mainWords.contains(word)) {
    				if (StringUtil.isNumeric(word)) {
        				keyword.put(Constants.TYPE, Constants.NUMBER);
    				}
        		} else {
    				keyword.put(Constants.TYPE, Constants.WORD);
        		}
    		}
        }
        LOGGER.debug("Query Keywords: {}", queryKeywords);
		return queryKeywords;
	}

	/**
	 * Extract quoted tokens
	 * 
	 * @param text String
	 * @return token list
	 */
	public List<Token> extractQuotedTokens(final String text) {
		List<Token> quotedTokens = new ArrayList<>();
        if (text.length() == 0 || text.equals("*")) {
        	return quotedTokens;
        }
		boolean inQuotes = false;
		int begin = 0;
		StringBuilder sb = new StringBuilder();
		for (int i = 0; i < text.length(); i++) {
			char c = text.charAt(i);
			switch (c) {
			case '"':
				if (inQuotes) {
					if (sb.toString().trim().length() > 0) {
						quotedTokens.add(new Token(sb.toString(), begin, i));
					}
					inQuotes = false;
				} else {
					inQuotes = true;
				}
				sb.setLength(0);
				begin = i + 1;
				break;
			default:
				sb.append(c);
			}
		}
		return quotedTokens;
	}
        
	/**
	 * Extract merged tokens
	 * 
	 * @param text String
	 * @return token list
	 */
	public List<Token> extractMergedTokens(final String text, final List<Token> quotedTokens) {
		List<Token> mergedTokens = new ArrayList<>();
        if (text.length() == 0 || text.equals("*")) {
        	return mergedTokens;
        }
        
        int begin = 0;
        for (Token quotedToken : quotedTokens) {
    		int quotedBegin = quotedToken.getBegin() - 1;
    		if (quotedBegin > begin) {
    			String subText = text.substring(begin, quotedBegin);
    			Token mergedToken = new Token(subText, begin, quotedBegin);
    			mergedTokens.add(mergedToken);
    		}
    		begin = quotedToken.getEnd() + 1;
        }
		if (begin < text.length()) {
			String subText = text.substring(begin, text.length());
			Token mergedToken = new Token(subText, begin, text.length());
			mergedTokens.add(mergedToken);
		}
		return mergedTokens;
	}
    
	/**
	 * Get Token Lemma
	 * 
	 * @param Token
	 * @param nlpObject JSONObject
	 * @return lemma
	 */
	public String getTokenLemma(final Token token, JSONObject nlpObject) {
		int begin = token.getBegin();
		int end = token.getEnd();

		StringBuilder sb = new StringBuilder();

		JSONArray sentences = nlpObject.getJSONArray("sentences");
		for (Object sentenceObject : sentences) {
			JSONObject sentence = (JSONObject) sentenceObject;
			JSONArray sentenceTokens = sentence.getJSONArray("tokens");
			for (Object tokenObject : sentenceTokens) {
				JSONObject jsonToken = (JSONObject) tokenObject;
				int idx = jsonToken.getInt("idx");
				if (idx >= begin && idx < end) {
					if (sb.length() > 0) {
						sb.append(" ");
					}
					sb.append(jsonToken.getString("lem"));
				}
				if (idx >= end) {
					break;
				}
			}
		}
		return sb.toString();
	}
	 
	/**
	 * Get separate words or phrases in quotes
	 * 
	 * @param text String
	 * @return word list
	 */
	public List<String> getAllSearchWords(final String text) {
		List<String> allWords = new ArrayList<>();
        if (text.length() == 0) {
        	return allWords;
        }
        
		boolean inQuotes = false;
		StringBuilder sb = new StringBuilder();
		for (int i = 0; i < text.length(); i++) {
			char c = text.charAt(i);
			switch (c) {
			case ' ':
				if (inQuotes) {
					sb.append(' '); // replace to space
//					sb.append(c);
				} else {
					if (sb.length() > 0) {
						allWords.add(sb.toString());
						sb.setLength(0);
					}
				}
				break;
			case '"':
				if (inQuotes) {
					sb.append(c);
					if (sb.length() > 2) {
						allWords.add(sb.toString());
					}
					sb.setLength(0);
					inQuotes = false;
				} else {
					if (sb.length() > 0) {
						allWords.add(sb.toString());
						sb.setLength(0);
					}
					sb.append(c);
					inQuotes = true;
				}
				break;
			default:
				sb.append(c);
			}
		}
		if (inQuotes && sb.length() > 1) {
			String[] words = sb.toString().substring(1).split("\\s+");
			for (String word : words) {
				allWords.add(word);
			}
		} else {
			if (sb.length() > 0) {
				allWords.add(sb.toString());
			}
		}
		return allWords;
	}

	/**
	 * Get quoted search words
	 * 
	 * @param searchText String
	 * @return quoted words list
	 */
	public List<String> getQuotedSearchWords(final String text) {
		List<String> quotedWords = new ArrayList<>();
        List<String> allWords = getAllSearchWords(text);
        for (String word : allWords) {
			if (word.startsWith("\"") && word.endsWith("\"")) {
				quotedWords.add(word.substring(1, word.length() - 1));
        	}
        }
       	return quotedWords;
	}
	
    /**
	 * Get quoted words
	 * 
	 * @param text string
	 * @return Keywords in quotes
	 */
	public List<String> getQuotedWords(final String text) {
		ArrayList<String> words = new ArrayList<>();
		Pattern p = Pattern.compile("\"([^\"]*)\"");
		Matcher m = p.matcher(text);
		while (m.find()) {
			words.add("\"" + m.group(1) + "\"");
		}
		return words;
	}	
	
	/**
	 * Get quoted and numeric words
	 * 
	 * @param searchText String
	 * @return words list
	 */
	public List<String> getQuotedAndNumericWords(final String searchText) {
		List<String> words = new ArrayList<>();
        List<String> allSearchWords = getAllSearchWords(searchText);
        for (String text : allSearchWords) {
			if (Utils.containsDigits(text) || (text.startsWith("\"") && text.endsWith("\""))) {
            	words.add(text);
        	}
        }
       	return words;
	}
	
	/**
	 * Get all words
	 * 
	 * @param searchText String
	 * @return word list
	 */
	public List<String> getAllWords(final List<String> words) {
        List<String> allWords = new ArrayList<>();
        for (String word : words) {
        	word = StringUtil.cleanSpaces(StringUtil.cleanQuotes(word));
        	String[] wrds = word.split(" ");
        	for (String wrd : wrds) {
        		if (!allWords.contains(wrd)) {
                	allWords.add(wrd);
            	}
        	}
        }
       	return allWords;
	}	
	
    /**
     * Spell check the query.
     * 
     * @param searchQuery string
     * @return Spell checked query
     */
	public String spellCheck(final Client client, final String searchQuery) {
		String spelledQuery = searchQuery;
		String cached = (String) spellCache.get(searchQuery);
		if (cached == null) {
			JSONObject jsonQuery = new JSONObject();
//			jsonQuery.put("max_new_tokens", 50);
			jsonQuery.put("num_beams", 2);
			jsonQuery.put("early_stopping", true);
			jsonQuery.put("max_time", 3);
			jsonQuery.put("temperature", 0.1);
			jsonQuery.put("top_p", 0.5);
//			jsonQuery.put("task", "You are verifying the spelling. Return properly spelled query and nothing else. Don't answer the query and don't provide an explanation. Query:\n" + Util.cleanQuotes(query));
			jsonQuery.put("task", "Don't answer the query, and don't add new words to it. Instead return the same query, but properly spelled. Give correct spelling to the query: " + searchQuery);
			
			String request = jsonQuery.toString();
			LOGGER.debug("Spell check request: " + request);

			String result = "";
			try {
				String response = Utils.post(llmUrl(client) + "/run", request, llmTimeout);
				JSONObject json = new JSONObject(response);
				result = json.getString("result");
			} catch (Exception e) {
				LOGGER.error("Spellcheck Error: " + e);
			}
			String resultLC = result.toLowerCase();
			if (resultLC.contains("<s>") ||
				    resultLC.contains("<unk>") || 
				    resultLC.contains("spell ") || 
				    resultLC.contains("correct ") ||
	    		    resultLC.contains("spelled ") || 
    			    resultLC.contains("corrected ")) {
				result = "";
			}
			if (resultLC.contains("query: ")) {
				int i = resultLC.indexOf("query: ");
				result = result.substring(i + 7);
			}
			if (resultLC.length() - searchQuery.length() >= 10) {
				result = "";
			}
//			result = StringUtil.cleanText(result);
			if (result.endsWith(".")) {
				result = result.substring(0, result.length() - 1);
			}
			if (result != null && result.length() > 0) {
				spelledQuery = result;
				spellCache.put(searchQuery, spelledQuery);
			}
		} else {
			spelledQuery = cached;
		}
		
		return spelledQuery;
	}
	
    /**
     * Get RAG answer from LLM.
     * 
     * @param query string
     * @param results list
     * @return answer
     */
    public String getRAGAnswer(final Client client, final String query, final String text) {
    	String answer = "";
    	String context;
        if (text.length() > 40000) { // TODO
        	context = StringUtil.shortText(text, 40000, 35000);
        } else {
        	context = text;
        }
        
		JSONObject jsonQuery = new JSONObject();
		jsonQuery.put("max_new_tokens", 200);
//		jsonQuery.put("num_beams", 2);
//		jsonQuery.put("early_stopping", true);
//		jsonQuery.put("max_time", 10);
		String task = "Answer the query using information presented in search results below. Think and if results have no answer, return only \"No answer\" and nothing else. If the query is not a question, just describe the best result for the query. Do not give your own opinion, don't repeat the query in your answer and don't say it is search results."
		     + "\n\nQuery: " + query + "\n\nSearch Results:\n\n" + context;
        jsonQuery.put("task", task); 
		String request = jsonQuery.toString();
		LOGGER.debug("Answer JSON Query: " + request);

		try {
			String response = Utils.post(llmUrl(client) + "/run", request, llmTimeout);
			JSONObject json = new JSONObject(response);
			answer = json.getString("result");
		} catch (Exception e) {
			LOGGER.error("ERROR: " + e);
		}
		LOGGER.debug("LLM answer: " + answer);
		
		answer = StringUtil.cleanSpaces(answer);
		if (answer.length() > 0) {
			String answerL = answer.toLowerCase();
			if (answerL.contains("<s>") || 
					answerL.contains("<unk>") ||
					answerL.contains("not relevant") ||
					answerL.contains("not mentioned") ||
					answerL.contains("not specified")) {
				answer = "";
			}
		}
        return answer;
	}
    
    /**
     * Expand search query.
     * 
     * @param searchQuery string
     * @return expanded query
     */
	public String expandSearchQuery(final Client client, final String searchQuery) {
		String expandedQuery = searchQuery;
		String cached = (String) expandCache.get(searchQuery);
		if (cached == null) {
			JSONObject jsonQuery = new JSONObject();
//			jsonQuery.put("max_new_tokens", 200);
			jsonQuery.put("num_beams", 2);
			jsonQuery.put("early_stopping", true);
			jsonQuery.put("max_time", 5);
			jsonQuery.put("temperature", 0.1);
			jsonQuery.put("top_p", 0.5);
			jsonQuery.put("task", "Ask two times the same query, but using different words. Make result in json format as list of query strings. The query: " + searchQuery);
			
			String request = jsonQuery.toString();
			LOGGER.debug("Expand Search Query request: " + request);

			try {
				String response = Utils.post(llmUrl(client) + "/run", request, llmTimeout);
				JSONObject json = new JSONObject(response);
				String result = json.getString("result");
				int i1 = result.indexOf("[");
				int i2 = result.indexOf("]");
				if (i1 >= 0 && i2 > 0 && i2 > i1) {
					JSONArray queryArray = new JSONArray(result.substring(i1, i2 + 1));
					if (queryArray.length() > 0) {
						for (Object object : queryArray) {
							String query = (String) object;
							query = query.trim();
							if (query.length() > 0 && !searchQuery.equals(query)) {
								expandedQuery += ", " + query;
							}
						}
					}
				}
				if (expandedQuery.length() > searchQuery.length()) {
					expandCache.put(searchQuery, expandedQuery);
				}
			} catch (Exception e) {
				LOGGER.error("Expand Search Query Error: " + e);
			}
		} else {
			expandedQuery = cached;
		}
        LOGGER.debug("Expanded search query: {}", expandedQuery);
		return expandedQuery;
	}
	    
    /**
     * Rerank search results.
     * 
     * @param query string
     * @param texts of search results list
     * @return scores of results
     */
    public List<Double> rerankSearchResults(final Client client, final String query, final List<String> texts) {
    	JSONObject jsonQuery = new JSONObject();
		jsonQuery.put("query", query);
		jsonQuery.put("docs", texts);
		String request = jsonQuery.toString();
		LOGGER.debug("Rerank JSON Query: " + request);

		JSONArray result = null;
		try {
			String response = Utils.post(rerUrl(client) + "/rescore", request, rerTimeout);
			JSONObject json = new JSONObject(response);
			result = json.getJSONArray("result");
		} catch (Exception e) {
			LOGGER.error("ERROR: " + e);
		}
		LOGGER.debug("Rerank scores: " + result);

		if (result != null) {
			List<Double> scores = new ArrayList<>(result.length());
			for (int i = 0; i < result.length(); i++) {
				Double score = result.getDouble(i);
				scores.add(score);
			}
			if (scores.size() == texts.size()) {
				return scores;
			}
		}
        return null;
	}

	/**
	 * Get query NLP JSON Object
	 * 
	 * @param text query string
	 * @return NLP JSON Object
	 * @throws IOException 
	 */
	public JSONObject getNlpObject(Client client, String text) throws IOException {
		JSONObject jsonQuery = new JSONObject();
		jsonQuery.put("options", "all"); // TODO not all needed
		jsonQuery.put("text", text);

		String request = jsonQuery.toString();
		LOGGER.debug("NLP JSON Query: " + request);

		String response = Utils.post(nlpUrl(client) + "/nlp", request, nlpTimeout);
		JSONObject json = new JSONObject(response);
		JSONObject result = json.getJSONObject("results");
		
		LOGGER.debug("NLP JSON Response: " + result.toString());
		return result;
	}
	
	/**
	 * Get strings similarity
	 * 
	 * @param text query string
	 * @param texts List of strings
	 * @return List of similarities
	 * @throws IOException 
	 */
	public List<Double> getSimilarity(Client client, String text, List<String> texts) throws IOException {
		List<Double> output = new ArrayList<>();
	    JSONObject jsonRequest = new JSONObject();
		jsonRequest.put("text", text);
		jsonRequest.put("texts", texts);
		String request = jsonRequest.toString();
	    LOGGER.debug("Main words request: {}", request);
		
		String response = Utils.post(embUrl(client) + "/similarity", request, embTimeout);

	    JSONObject jsonResponse = new JSONObject(response);
        JSONArray similarities = jsonResponse.getJSONArray("result");
        if (similarities.length() != texts.size()) {
        	throw new IOException("Invalid similarity response");
        }			
		for (Object object : similarities) {
			BigDecimal value = (BigDecimal) object;
			output.add(value.doubleValue());
        }
		return output;
	}
	
	private void addQueryKeyword(final JSONObject jsonKeyword, final JSONArray jsonKeywords) {
		String text = jsonKeyword.getString(Constants.TEXT).toLowerCase();
		int begin = jsonKeyword.getInt(Constants.BEGIN);
		int end = jsonKeyword.getInt(Constants.END);
		boolean add = true;
    	for (Object object: jsonKeywords) {
    		JSONObject jsonKeyword2 = (JSONObject) object;
    		String text2 = jsonKeyword2.getString(Constants.TEXT).toLowerCase();
    		if (text.equals(text2) ) {
//    			add = false;
                break;
    		}
    		int begin2 = jsonKeyword2.getInt(Constants.BEGIN);
    		int end2 = jsonKeyword2.getInt(Constants.END);
    		if (end >= begin2 && end2 >= begin) {
    			add = false;
                break;
    		}
    	}
    	if (add) {
			jsonKeywords.put(jsonKeyword);
    	}
    }
	
	/**
	 * Get included / excluded words or phrases in quotes
	 * 
	 * @param text String
	 * @return keywords array
	 */
	public JSONArray getIncExcWords(final String text) {
		JSONArray  jsonKeywords = new JSONArray();
        if (text.trim().length() == 0) {
        	return jsonKeywords;
        }
        
		boolean inBegin = true;
		boolean include = false;
		boolean exclude = false;
		boolean inQuotes = false;
		int begin = -1;
		StringBuilder sb = new StringBuilder();
		for (int i = 0; i < text.length(); i++) {
			char c = text.charAt(i);
			switch (c) {
			case ' ':
				if (inQuotes) {
					sb.append(c);
				} else {
					if (include) {
						JSONObject jsonKeyword = new JSONObject();
						jsonKeyword.put(Constants.TEXT, sb.toString());
						jsonKeyword.put(Constants.TYPE, Constants.INCLUDED);
						jsonKeyword.put(Constants.BEGIN, begin);
						jsonKeyword.put(Constants.END, i);
						addQueryKeyword(jsonKeyword, jsonKeywords);
					    include = false;
					} else if (exclude) {
						JSONObject jsonKeyword = new JSONObject();
						jsonKeyword.put(Constants.TEXT, sb.toString());
						jsonKeyword.put(Constants.TYPE, Constants.EXCLUDED);
						jsonKeyword.put(Constants.BEGIN, begin);
						jsonKeyword.put(Constants.END, i);
						addQueryKeyword(jsonKeyword, jsonKeywords);
					    exclude = false;
					}
					sb.setLength(0);
					inBegin = true;
				}
				break;
			case '+':
				if (inQuotes) {
					sb.append(c);
				} else if (inBegin) {
				    include = true;
				    exclude = false;
					sb.setLength(0);
					begin = i + 1;
//				    inBegin = false;
				} else {
					sb.append(c);
				}
				break;
			case '-':
				if (inQuotes) {
					sb.append(c);
				} else if (inBegin) {
				    exclude = true;
				    include = false;
					sb.setLength(0);
					begin = i + 1;
//				    inBegin = false;
				} else {
					sb.append(c);
				}
				break;
			case '"':
				if (inQuotes) {
					if (sb.length() > 0) {
						JSONObject jsonKeyword = new JSONObject();
						jsonKeyword.put(Constants.TEXT, sb.toString());
						if (exclude) {
							jsonKeyword.put(Constants.TYPE, Constants.EXCLUDED);
						} else if (include) {
							jsonKeyword.put(Constants.TYPE, Constants.INCLUDED);
						} else {
							jsonKeyword.put(Constants.TYPE, Constants.QUOTED);
						}
						jsonKeyword.put(Constants.BEGIN, begin);
						jsonKeyword.put(Constants.END, i);
						addQueryKeyword(jsonKeyword, jsonKeywords);
					}
					sb.setLength(0);
					inQuotes = false;
				    include = false;
				    exclude = false;
				} else if (inBegin) {
					inQuotes = true;
					sb.setLength(0);
					begin = i + 1;
				    inBegin = false;
				}
				break;
			default:
				sb.append(c);
				inBegin = false;
			}
		}
		if (sb.length() > 0) {
			if (include) {
				JSONObject jsonKeyword = new JSONObject();
				jsonKeyword.put(Constants.TEXT, sb.toString());
				jsonKeyword.put(Constants.TYPE, Constants.INCLUDED);
				jsonKeyword.put(Constants.BEGIN, begin);
				jsonKeyword.put(Constants.END, text.length());
				addQueryKeyword(jsonKeyword, jsonKeywords);
			} else if (exclude) {
				JSONObject jsonKeyword = new JSONObject();
				jsonKeyword.put(Constants.TEXT, sb.toString());
				jsonKeyword.put(Constants.TYPE, Constants.EXCLUDED);
				jsonKeyword.put(Constants.BEGIN, begin);
				jsonKeyword.put(Constants.END, text.length());
				addQueryKeyword(jsonKeyword, jsonKeywords);
			}
		}
		LOGGER.debug("Included/Excluded keywords: " + jsonKeywords.toString());
		return jsonKeywords;
	}

	/**
	 * Get all keywords with included / excluded words or phrases in quotes
	 * 
	 * @param text String
	 * @return keywords array
	 */
	public JSONArray getKeywords(final String text) {
		JSONArray  jsonKeywords = new JSONArray();
        if (text.trim().length() == 0) {
        	return jsonKeywords;
        }
        
		boolean inSpace = true;
		boolean include = false;
		boolean exclude = false;
		boolean inQuotes = false;
		int begin = 0;
		StringBuilder sb = new StringBuilder();
		for (int i = 0; i < text.length(); i++) {
			char c = text.charAt(i);
			switch (c) {
			case ' ', '(', ')', '{', '}', '[', ']', '<', '>', ':', '=' :
				if (c == ' ') {
					inSpace = true;
				} else {
					inSpace = false;
				}
				if (inQuotes) {
					sb.append(c);
				} else {
					if (sb.length() > 0) {
						if (include) {
							JSONObject jsonKeyword = new JSONObject();
							jsonKeyword.put(Constants.TEXT, sb.toString());
							jsonKeyword.put(Constants.TYPE, Constants.INCLUDED);
							jsonKeyword.put(Constants.BEGIN, begin);
							jsonKeyword.put(Constants.END, i);
							addQueryKeyword(jsonKeyword, jsonKeywords);
						    include = false;
						} else if (exclude) {
							JSONObject jsonKeyword = new JSONObject();
							jsonKeyword.put(Constants.TEXT, sb.toString());
							jsonKeyword.put(Constants.TYPE, Constants.EXCLUDED);
							jsonKeyword.put(Constants.BEGIN, begin);
							jsonKeyword.put(Constants.END, i);
							addQueryKeyword(jsonKeyword, jsonKeywords);
						    exclude = false;
						} else {
							JSONObject jsonKeyword = new JSONObject();
							jsonKeyword.put(Constants.TEXT, sb.toString());
							jsonKeyword.put(Constants.TYPE, Constants.KEYWORD);
							jsonKeyword.put(Constants.BEGIN, begin);
							jsonKeyword.put(Constants.END, i);
							addQueryKeyword(jsonKeyword, jsonKeywords);
						}
					}
					sb.setLength(0);
					begin = i + 1;
				}
				break;
			case '+':
				if (inQuotes) {
					sb.append(c);
				} else if (inSpace) {
				    include = true;
				    exclude = false;
					sb.setLength(0);
					begin = i + 1;
				} else if (sb.length() == 0) {
				    include = false;
				} else {
				    sb.append(c);
				}
			    inSpace = false;
				break;
			case '-':
				if (inQuotes) {
					sb.append(c);
				} else if (inSpace) {
				    exclude = true;
				    include = false;
					sb.setLength(0);
					begin = i + 1;
				} else if (sb.length() == 0) {
				    exclude = false;
				} else {
				    sb.append(c);
				}
			    inSpace = false;
				break;
			case '"':
				if (inQuotes) {
					if (sb.length() > 0) {
						JSONObject jsonKeyword = new JSONObject();
						jsonKeyword.put(Constants.TEXT, sb.toString());
						if (exclude) {
							jsonKeyword.put(Constants.TYPE, Constants.EXCLUDED);
						} else if (include) {
							jsonKeyword.put(Constants.TYPE, Constants.INCLUDED);
						} else {
							jsonKeyword.put(Constants.TYPE, Constants.QUOTED);
						}
						jsonKeyword.put(Constants.BEGIN, begin);
						jsonKeyword.put(Constants.END, i);
						addQueryKeyword(jsonKeyword, jsonKeywords);
					}
					sb.setLength(0);
					inQuotes = false;
				    include = false;
				    exclude = false;
				} else {
					inQuotes = true;
					sb.setLength(0);
					begin = i + 1;
				}
			    inSpace = false;
				break;
			default:
				sb.append(c);
				inSpace = false;
			}
		}
		if (sb.length() > 0) {
			if (include) {
				JSONObject jsonKeyword = new JSONObject();
				jsonKeyword.put(Constants.TEXT, sb.toString());
				jsonKeyword.put(Constants.TYPE, Constants.INCLUDED);
				jsonKeyword.put(Constants.BEGIN, begin);
				jsonKeyword.put(Constants.END, text.length());
				addQueryKeyword(jsonKeyword, jsonKeywords);
			} else if (exclude) {
				JSONObject jsonKeyword = new JSONObject();
				jsonKeyword.put(Constants.TEXT, sb.toString());
				jsonKeyword.put(Constants.TYPE, Constants.EXCLUDED);
				jsonKeyword.put(Constants.BEGIN, begin);
				jsonKeyword.put(Constants.END, text.length());
				addQueryKeyword(jsonKeyword, jsonKeywords);
			} else {
				JSONObject jsonKeyword = new JSONObject();
				jsonKeyword.put(Constants.TEXT, sb.toString());
				jsonKeyword.put(Constants.TYPE, Constants.KEYWORD);
				jsonKeyword.put(Constants.BEGIN, begin);
				jsonKeyword.put(Constants.END, text.length());
				addQueryKeyword(jsonKeyword, jsonKeywords);
			}
		}
		LOGGER.debug("Keywords: " + jsonKeywords.toString());
		return jsonKeywords;
	}
	
}