#!pip install spacy -U  <<< with CPU
#!pip install pytextrank -U
#!python -m spacy download en_core_web_lg
#!python -m spacy download en_core_web_trf
#!python -m spacy validate

import os
import time 
import requests
import json
import numpy as np
import logging

import spacy
import pytextrank
from keyword_spacy import KeywordExtractor

from flask import Flask, abort, jsonify, request

from spacy.tokens import Doc, Span, Token  # @UnresolvedImport
from spacy.symbols import (
    AUX, NOUN, VERB, ADJ, ADV,
    agent, attr, aux, auxpass, csubj, csubjpass, dobj, neg, nsubj, nsubjpass, obj, pobj, xcomp,
    acl, acomp, advmod, oprd, pcomp, prep
)

logging.basicConfig(
    format="%(asctime)s - %(levelname)s - %(name)s -   %(message)s", datefmt="%m/%d/%Y %H:%M:%S", level=logging.INFO,
)
logger = logging.getLogger(__name__)

Token.set_extension('noun_chunk', default=None)
Token.set_extension('entity', default=None)

# Run on CPU only
runOnCpu = True

if runOnCpu:
#    os.environ["CUDA_DEVICE_ORDER"] = "PCI_BUS_ID"
#    os.environ['CUDA_VISIBLE_DEVICES'] = '-1'
    # Hide GPU from visible devices
    spacy.require_cpu()
else:
#    spacy.prefer_gpu(gpu_id=0)
    spacy.require_gpu(gpu_id=0)

logger.info("Spacy ver: {}".format(spacy.__version__))

#==============================
#POS    DESCRIPTION    EXAMPLES
#ADJ    adjective      big, old, green, incomprehensible, first
#ADP    adposition     in, to, during, about
#ADV    adverb         very, tomorrow, down, where, there, about
#AUX    auxiliary      is, has (done), will (do), should (do)
#CONJ   conjunction    and, or, but
#CCONJ  coordinating conjunction    and, or, but
#DET    determiner     a, an, the
#INTJ   interjection   psst, ouch, bravo, hello
#NOUN   noun           girl, cat, tree, air, beauty
#NUM    numeral        1, 2017, one, seventy-seven, IV, MMXIV
#PART   particle       ’s, not,
#PRON   pronoun        I, you, he, she, myself, themselves, somebody, my, her
#PROPN  proper noun    Mary, John, London, NATO, HBO
#PUNCT  punctuation    ., (, ), ?
#SCONJ  subordinating conjunction    if, while, that
#SYM    symbol         $, %, §, ©, +, −, ×, ÷, =, :), 
#VERB   verb           run, runs, running, eat, ate, eating
#X      other          sfpksdpsxmsa
#SPACE  space

#Options:
#sts,pos,lem,nch,ent,all
#sts  sentences
#pos  part of speech
#lem  lemma
#nch  noun chunk
#ent  entity
#all  all options
#==============================

def get_words(sent, p1, p2):
    word = sent[p1].lemma_
    p = p1 + 1
    while p <= p2:
        word += '_' + sent[p].lemma_
        p += 1
    return word

def get_tok_words(tok, sent, p):
    p1 = p
    p2 = p + 3
    if p2 >= len(sent):
        p2 = len(sent) - 1
    while p2 > p1:
        word = get_words(sent, p1, p2)
        synset_ids = wn_words.get(word)
        if synset_ids:
            break
        p2 -= 1
    n = p2 - p1    
    return word,synset_ids,n
    
def nlp_text(text, options):
    start = time.time()
    
    #options
    sts = pos = lem = idx = nch = ent = False
    if options == None:
        sts = True
    else:
        opts = options.split(',')
        for option in opts:
            if option == 'all': 
                sts = pos = lem = idx = nch = ent = True
                break
            elif option == 'sts': sts = True
            elif option == 'pos': pos = True
            elif option == 'lem': lem = True
            elif option == 'idx': idx = True
            elif option == 'nch': nch = True
            elif option == 'ent': ent = True
    
    #main processing
    doc = nlp(text)
    response = {}
    
    # noun chunks
    if nch and sts:
        for noun_chunk in doc.noun_chunks:
            l = len(noun_chunk)
            if l > 1:
#                logger.info("NP:", noun_chunk.text, noun_chunk.label_)
                for tok in noun_chunk:
                    tok._.noun_chunk = noun_chunk

    # entities
    if ent and sts:
        for ent in doc.ents:
            for tok in ent:
                tok._.entity = ent

    # sentences
    if sts:
        sentences = []
        for sent in doc.sents:
            sentence = {}
            sentence['text'] = sent.text
            tokens = []
            entities = []
            noun_chunks = []
            for tok in sent:
#                if tok.text in nlp.Defaults.stop_words:
#                    continue

                token = {}
                token['text'] = tok.text
            
                if nch and tok._.noun_chunk != None: 
                    if tok._.noun_chunk not in noun_chunks: noun_chunks.append(tok._.noun_chunk)
                if pos:
                    token['pos'] = tok.pos_
                if lem:
                    token['lem'] = tok.lemma_
                if idx:
                    token['idx'] = tok.idx
                if ent and tok._.entity != None:
                    if tok._.entity not in entities: entities.append(tok._.entity)             
                tokens.append(token)
            
            # nounChunks
            if nch:
                i = 0
                for noun_chunk in noun_chunks:
                    for tok in noun_chunk:
                        token = tokens[tok.i - sent[0].i]
                        token['nch'] = i
                    noun_chunks[i] = noun_chunk.text
                    i += 1                
                sentence['nounChunks'] = noun_chunks
            
            # entities
            if ent:
                i = 0
                for ent in entities:
                    for tok in ent:
                        token = tokens[tok.i - sent[0].i]
                        token['ent'] = i
                    entity = {'text': ent.text, 'type': ent.label_}
                    entities[i] = entity
                    i += 1                
                sentence['entities'] = entities

            sentence['tokens'] = tokens
            sentences.append(sentence)
        
        response['sentences'] = sentences

    # entities without sentences
    if ent and not sts:
        entities = []
        for ent in doc.ents:
            entities.append({'text': ent.text, 'type': ent.label_})
        response['entities'] = entities

    end = time.time()
    took = int((end-start)*1000)
    logger.info('NLP took %d (ms)' % (took))
    return response,took

#SPACY
#modelName = 'en_core_web_lg'
modelName = 'en_core_web_trf'
logger.info('Loading model: ' + modelName)
nlp = spacy.load(modelName)

#====================================
#http://localhost:8085/nlp?text=testing+in+the+state+of+New+Jersey

app = Flask(__name__)
app.config['CORS_HEADERS'] = 'Content-Type'
app.config['DEBUG'] = False

@app.route("/nlp", methods=['GET'])
def get_nlp():
    text = request.args.get('text')
    if len(text) == 0:
        abort(500)
    else:
        options = request.args.get('options')
#        logger.info(text)
        results,took = nlp_text(text, options)
        return jsonify({'results': results,'took': took})

@app.route("/nlp", methods=['POST'])
def post_nlp():
    data = request.get_json()
    if 'text' not in data or len(data['text']) == 0:
        abort(500)
    else:
        text = data['text']
        options = data['options']
#        logger.info(text)
        results,took = nlp_text(text, options)
        return jsonify({'results': results,'took': took})

if __name__ =='__main__':
    app.run(host="0.0.0.0")
