/**
 * Copyright (c) 2025 SoftCorporation LLC. All rights reserved.
 *
 * The Software License, Version 1.0
 *
 * SoftCorporation LLC. grants you ("Licensee") a non-exclusive, royalty free,
 * license to use, modify and redistribute this software in source and binary
 * code form, provided that the following conditions are met:
 *
 * 1. The end-user documentation included with the redistribution,
 *    if any, must include the following acknowledgment:
 *       "This product includes software developed by the
 *        SoftCorporation LLC. (http://www.softcorporation.com)."
 *    Alternately, this acknowledgment may appear in the software itself,
 *    if and wherever such third-party acknowledgments normally appear.
 *
 * 2. This software and SoftCorporation name must not be used to
 *    promote products derived from the software without prior
 *    written permission. For written permission, please contact
 *    info@softcorporation.com.
 *
 * This software is provided "AS IS," without a warranty of any kind.
 * ALL EXPRESS OR IMPLIED CONDITIONS, REPRESENTATIONS AND WARRANTIES,
 * INCLUDING ANY IMPLIED WARRANTY OF MERCHANTABILITY, FITNESS FOR A
 * PARTICULAR PURPOSE OR NON-INFRINGEMENT, ARE HEREBY EXCLUDED.
 * IN NO EVENT SHALL THE SOFTCORPORATION BE LIABLE FOR ANY DIRECT,
 * INDIRECT, INCIDENTAL OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF
 * USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION).
 *
 */
package com.softcorporation.sandi.index.service;

import java.io.IOException;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.Iterator;
import java.util.List;
import java.util.Map;

import org.json.JSONArray;
import org.json.JSONObject;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Value;

import com.softcorporation.sandi.base.model.Client;
import com.softcorporation.sandi.base.model.TextChunk;
import com.softcorporation.sandi.base.service.TextEmbeddingService;
import com.softcorporation.sandi.base.util.Constants;
import com.softcorporation.sandi.base.util.EmbeddingException;
import com.softcorporation.sandi.base.util.Utils;
import com.softcorporation.sandi.index.util.IndexerException;
import com.softcorporation.util.StringUtil;

public abstract class IndexerService {

	private static final Logger LOGGER = LoggerFactory.getLogger(IndexerService.class);
	
    /**
     * maxDataLength int
     */
    @Value("${sandi.index.batch.data.length:10000000}")
    private int batchDataLength;
    
    /**
     * clientIndexUrl String
     */
    @Value("${sandi.service.client.index.url:}")
    private String clientIndexUrl;
    
    @Autowired
    private TextEmbeddingService textEmbeddingService;
    
	//    public void init();

	public abstract void indexDocument(String collection, JSONObject document) throws IndexerException;

	public abstract void indexDocuments(String collection, List<JSONObject> documents) throws IndexerException;

	public abstract void deleteDocument(String collection, String uniqueId) throws IndexerException;

	public abstract void deleteDocument(String collection, String ClientId, String id) throws IndexerException;

	public abstract void deleteDocuments(String collection, List<String> uniqueIds) throws IndexerException;

	public abstract void deleteDocuments(String collection, String ClientId, List<String> Ids) throws IndexerException;

	public abstract void deleteDocumentsByQuery(String collection, String ClientId, String query) throws IndexerException;

	public abstract JSONObject getDocumentById(String collection, String id, String fields) throws IndexerException;

	public abstract JSONObject getDocumentById(String collection, String clientId, String id, String fields) throws IndexerException;

	public abstract void commit(String collection) throws IndexerException;

	public int getBatchDataLength() {
		return batchDataLength;
	}

	public int processDocuments(Client client, List<Map<String, Object>> documents, boolean forceReindexing) throws IndexerException {
		List<JSONObject> jsons = new ArrayList<>();
		for (Map<String, Object> doc : documents) {
		    JSONObject json = new JSONObject(doc);
		    jsons.add(json);
		}
		return processJsonDocuments(client, jsons, forceReindexing);
	}

	public int processJsonDocument(Client client, JSONObject document, boolean forceReindexing) throws IndexerException {
		Object object = clientProcessing(client, document);
		if (object != null) {
			if (object instanceof JSONObject) {
				document = (JSONObject) object;
				document = prepareJsonDocument(client, document, forceReindexing);
		        if (document != null) {
		            indexDocument(client.getCollection(), document);
		            return 1;
		        }
			} else if (object instanceof JSONArray) {
				JSONArray array = (JSONArray) object;
				List<JSONObject> documentsToIndex = new ArrayList<>();
				for (int i = 0; i < array.length(); i++) {
					JSONObject doc = array.getJSONObject(i);
					doc = prepareJsonDocument(client, doc, forceReindexing);
		            if (doc != null) {
		            	documentsToIndex.add(doc);
		            }
				}
				if (documentsToIndex.size() > 0) {
			        indexDocuments(client.getCollection(), documentsToIndex);
				}
				return documentsToIndex.size();
			}
		}		
        return 0;
	}

	public int processJsonDocuments(Client client, List<JSONObject> documents, boolean forceReindexing) throws IndexerException {
		List<JSONObject> documentsToIndex = new ArrayList<>();
		for (JSONObject document : documents) {
			Object object = clientProcessing(client, document);
			if (object != null) {
				if (object instanceof JSONObject) {
					document = (JSONObject) object;
					document = prepareJsonDocument(client, document, forceReindexing);
		            if (document != null) {
		            	documentsToIndex.add(document);
		            }
				} else if (object instanceof JSONArray) {
					JSONArray array = (JSONArray) object;
					for (int i = 0; i < array.length(); i++) {
						JSONObject doc = array.getJSONObject(i);
						doc = prepareJsonDocument(client, doc, forceReindexing);
			            if (doc != null) {
			            	documentsToIndex.add(doc);
			            }
					}
				}
			}
        }
		if (documentsToIndex.size() > 0) {
	        indexDocuments(client.getCollection(), documentsToIndex);
		}
		return documentsToIndex.size();
	}

	public Object clientProcessing(Client client, JSONObject jsonDocument) throws IndexerException {
		if (client.isClientIndexProcessing()) {
			JSONObject request = new JSONObject();
			request.put(Constants.CLIENT, client.getClientId());
			request.put(Constants.DATA, jsonDocument);
			try {
				String response = Utils.post(clientIndexUrl, request.toString()).trim();
				if (response.length() == 0) {
			        LOGGER.info("Client {} ignore document: {}", client.getClientId(),
			        		StringUtil.formatForLogging(jsonDocument.toString()));
			        return null;
				} else if (response.startsWith("{") && response.endsWith("}")) {
					return new JSONObject(response);
				} else if (response.startsWith("[") && response.endsWith("]")) {
					return new JSONArray(response);
				} else {
			        throw new IndexerException("Invalid response from client processing:" + response);
				}
			} catch (IOException e) {
	            throw new IndexerException("Client processing failed", e);
			}
		}
		return jsonDocument;
	}

	/**
	 * Prepare JSON document
	 * 
	 * @param client
	 * @param jsonDoc
	 * @param forceReindexing
	 * @return JSON document
	 * @throws IndexerException
	 */
	public JSONObject prepareJsonDocument(final Client client, final JSONObject jsonDoc, boolean forceReindexing) 
			throws IndexerException {
		JSONObject jsonDocument = jsonDoc;
        String id = setUniqueId(jsonDocument, client.getClientId());
        LOGGER.info("Processing document ID: {}", id);
	    jsonDocument.put(Constants.CLIENT_ID, client.getClientId());

        // skip indexing if documents are identical
        String uuid = Utils.getUUID(jsonDocument.toString());
        if (!forceReindexing) {
        	String uniqueId = jsonDocument.getString(Constants._ID);
            JSONObject oldDoc = getDocumentById(client.getCollection(), uniqueId, Constants.UUID + "," + Constants.CREATED);
            if (oldDoc != null) {
                if (uuid.equals(oldDoc.optString(Constants.UUID, null))) {
                    LOGGER.info("Skipping indexing document ID: {}", id);
                    return null;
                }
                String oldCreated = oldDoc.optString(Constants.CREATED, null);
                if (oldCreated != null) {
                    jsonDocument.put(Constants.CREATED, oldCreated);
                }
            }
    	}
        jsonDocument.put(Constants.UUID, uuid);

	    if (!client.isNested()) {
	    	jsonDocument = flattenDocument(jsonDocument);
	    }

	    List<String> textsHigh = getDocumentTexts(jsonDocument, client.getFieldsHigh());
	    List<String> textsLow = getDocumentTexts(jsonDocument, client.getFieldsLow());
	    List<String> textsContent = getDocumentTexts(jsonDocument, client.getFieldsContent());

        if (textsHigh.size() > 0) {
            textsContent = StringUtil.joinStringArrays(textsHigh, textsContent);
            for (int i = 0; i < textsHigh.size(); i++) {
            	String textMatch = "ʡʡ " + textsHigh.get(i) + " ʡʡ";
            	textsHigh.set(i, textMatch);
            }
            jsonDocument.put(Constants.TEXT_HIGH, textsHigh);
        }
        if (textsLow.size() > 0) {
            textsContent = StringUtil.joinStringArrays(textsLow, textsContent);
            jsonDocument.put(Constants.TEXT_LOW, textsLow);
        }
        if (textsContent.size() > 0) {
        	jsonDocument.put(Constants.TEXT_CONTENT, textsContent);
        }
        
        // skip embedding for legacy client
        if (!client.isLegacy()) {
    	    List<TextChunk> chunksHigh = getDocumentChunks(jsonDocument, client.getFieldsHigh(), Constants.TYPE_HIGH);
    	    List<TextChunk> chunksLow = getDocumentChunks(jsonDocument, client.getFieldsLow(), Constants.TYPE_LOW);
    	    List<TextChunk> chunksContent = getDocumentChunks(jsonDocument, client.getFieldsContent(), Constants.TYPE_CONTENT);
            
    	    List<Map<String, Object>> embeddings;
			try {
				embeddings = textEmbeddingService.getDocumentEmbeddings(client, chunksHigh, chunksLow, chunksContent);
			} catch (EmbeddingException e) {
				throw new IndexerException(e.getMessage());
			}
            jsonDocument.put(Constants.EMBEDDING, embeddings);
    	}
        return jsonDocument;
	}

	/**
	 * Set unique document id
	 * @param doc
	 * @param clientId
	 * @return uniqueId
	 */
    protected String setUniqueId(JSONObject doc,  String clientId) {
    	String docId = (String) doc.optString(Constants.ID, "");
    	String id = docId.trim();
    	if (id.length() == 0) {
        	String uri = (String) doc.optString(Constants.URI, "");
        	uri = uri.trim();
        	if (uri.length() > 0) {
        	    id = uri;
        	} else {
//                id = Utils.getUUID(); // random
                id = Utils.getUUID(doc.toString()); // same id for the same document
                LOGGER.debug("Generated document ID: {}", id);
        	}
    	}
		if (!docId.equals(id)) {
	        doc.put(Constants.ID, id);
            LOGGER.debug("Modified document ID: {} for client: {}", id, clientId);
		}
    	String uniqueId;
		if (id.startsWith(clientId + "/")) {
			uniqueId = id;
		} else {
			if (id.startsWith("/")) {
				uniqueId = clientId + id;
			} else {
				uniqueId = clientId + "/" + id;
    		}
		}
        LOGGER.debug("Unique document ID: {}", uniqueId);
        doc.put(Constants._ID, uniqueId);
		return uniqueId;
    }

    protected List<String> getDocumentTexts(JSONObject doc, String[] fields) {
    	List<String> list = new ArrayList<>();
    	if (fields != null) {
        	for (String field : fields) {
        		if (doc.has(field)) {
            		Object value = doc.get(field);
            		if (value != null) {
            			if (value instanceof String) {
                			String text = (String) value;
                    		list.add(text);
            			} else if (value instanceof JSONArray) {
            				JSONArray array = (JSONArray) value;
            				for (Object object : array) {
                    			String text = String.valueOf(object);
                        		list.add(text);
            				}
            			} else {
                			String text = String.valueOf(value);
                    		list.add(text);
            			}
                	}
            	}
        	}
    	}
    	return list;
    }
	
    protected List<TextChunk> getDocumentChunks(JSONObject doc, String[] fields, String type) {
    	List<TextChunk> chunks = new ArrayList<>();
    	if (fields != null) {
        	for (String field : fields) {
        		if (doc.has(field)) {
            		Object value = doc.get(field);
            		if (value != null) {
            			String text = Utils.getTextOfObject(value);
            			List<TextChunk> textChunks = textEmbeddingService.breakTextInChunks(text, type);
            			for (TextChunk chunk : textChunks) {
            				// skip numeric chunks TODO - use word count
            				if (!StringUtil.isNumeric(chunk.getText())) {
                				chunk.setField(field);
                				chunk.setType(type);
                				chunks.add(chunk);
            				}
            			}
                	}
            	}
        	}
    	}
    	return chunks;
    }
	
    /**
     * Flatten document
     * 
     * @param doc
     * @return flat json
     */
    protected JSONObject flattenDocument(JSONObject doc) {
    	if (doc == null) {
    		return null;
    	}
    	JSONObject output = new JSONObject();
        Iterator<String> keys = doc.keys();
        while (keys.hasNext()) {
            String key = keys.next();
    		Object value = doc.get(key);
    		if (value != null) {
    			if (value instanceof JSONObject) {
    				output.put(key, value.toString());
    			} else if (value instanceof JSONArray) {
    				JSONArray array = (JSONArray) value;
    				for (int i = 0; i < array.length(); i ++) {
    					Object object = array.get(i);
    					if (object instanceof JSONObject) {
                			array.put(i, object.toString());
            			} else if (value instanceof JSONArray) {
                			array.put(i, object.toString());
            			}
    				}
    				output.put(key, array);
    			} else {
    				output.put(key, value);
    			}
        	}
    	}
    	return normalizeKeys(output);
    }
	
    /**
     * Flatten document helper. Create unique keys.
     * 
     * @param doc
     * @param path
     * @return map
     */
    protected Map<String, Object> flattenDocument(JSONObject doc, String path) {
    	Map<String, Object> map = new HashMap<String, Object>();
        Iterator<String> keys = doc.keys();
        while (keys.hasNext()) {
            String key = keys.next();
    		Object value = doc.get(key);
    		key = path + "/" + key;
    		if (value != null) {
    			if (value instanceof JSONObject) {
    				map.putAll(flattenDocument((JSONObject) value, key));
    			} else if (value instanceof JSONArray) {
    				JSONArray array = (JSONArray) value;
    				for (int i = 0; i < array.length(); i ++) {
    					Object object = array.get(i);
    					if (object instanceof JSONObject) {
                			array.put(i, object.toString());
            			} else if (value instanceof JSONArray) {
                			array.put(i, object.toString());
            			}
    				}
    				map.put(key, array);
    			} else {
    				map.put(key, value);
    			}
        	}
    	}
    	return map;
    }
	
    /**
     * Normalize keys
     * 
     * @param doc
     * @return json
     */
    private JSONObject normalizeKeys(JSONObject doc) {
    	JSONObject output = new JSONObject();
        Iterator<String> keys = doc.keys();
        while (keys.hasNext()) {
            String key = keys.next();
    		Object value = doc.get(key);
            key = key.replace('/', '_');
            key = key.replace('\\', '_');
            key = key.replace(' ', '_');
			output.put(key, value);
    	}
    	return output;
    }
	
}