from elasticsearch import * from elasticsearch_dsl import * from os import path import os from eu.dnetlib.util import get_index_properties from elasticsearch_dsl.response import Response import logging log = logging.getLogger("scholexplorer") class ScholixConnector(object): __instance = None def __new__(cls): if ScholixConnector.__instance is None: ScholixConnector.__instance = object.__new__(cls) props = get_index_properties() index_name = props['api.index'] index_host = [x for x in props['es_index'].split(',')] #connections.create_connection(hosts=index_host, timeout=1000) ScholixConnector.__instance.connection = Elasticsearch(hosts=index_host, timeout=1000) ScholixConnector.__instance.index_host = index_host ScholixConnector.__instance.index_name = index_name return ScholixConnector.__instance def create_pidType_query(self, value, start): args = {start + '.identifier.schema': value} return Q('nested', path=start + '.identifier', query=Q('bool', must=[Q('match', **args)])) def create_pid_query(self, value, start): args = {start + '.identifier.identifier': value.lower()} return Q('nested', path=start + '.identifier', query=Q('bool', must=[Q('match', **args)])) def create_typology_query(self, value, start): args = {start + '.objectType': value} return Q('nested', path=start, query=Q('bool', must=[Q('match', **args)])) def create_dataSource_query(self, value): args = {'linkprovider.name': value} return Q('nested', path='linkprovider', query=Q('bool', must=[Q('match', **args)])) def create_publisher_query(self, value, start): args = {start + '.publisher.name': value} q = Q('nested', path=start + '.publisher', query=Q('bool', must=[Q('match', **args)])) return Q('nested', path=start, query=q) def list_datasources(self, ds_name=None): search_object = Search(using=self.connection, index=self.index_name).doc_type('scholix') if ds_name: search_object = search_object.query(self.create_dataSource_query(ds_name)) else: search_object = search_object.query() search_object.aggs.bucket('all_datasources', 'nested', path='linkprovider').bucket('all_names', 'terms', field='linkprovider.name', size=100) response = search_object.execute() if ds_name: for item in response.aggs.all_datasources.all_names.buckets: if item.key == ds_name: yield dict(name=item.key, totalRelationships=item.doc_count) else: for item in response.aggs.all_datasources.all_names.buckets: yield dict(name=item.key, totalRelationships=item.doc_count) def list_publisher(self, start, pub_name=None): log.info("Started Index from host") search_object = Search(using=self.connection, index=self.index_name).doc_type('scholix') if pub_name: search_object = search_object.query(self.create_publisher_query(pub_name, start)) else: search_object = search_object.query() search_object.aggs.bucket('all_targets', 'nested', path=start).bucket('all_t_pubs', 'nested', path=start + '.publisher').bucket( 'all_pubs', 'terms', field=start + '.publisher.name', size=1000) response = search_object.execute() for item in response.aggs.all_targets.all_t_pubs.all_pubs.buckets: if pub_name and item.key == pub_name: yield dict(name=item.key, totalRelationships=item.doc_count) else: yield dict(name=item.key, totalRelationships=item.doc_count) def links(self, provider=None, s_pid=None, t_pid=None, s_publisher=None, t_publisher=None, s_pid_type=None, t_pid_type=None, target_Type=None, source_Type=None,page=0): queries = [] if provider: log.info("PROVIDER NOT NONE: {}".format(provider)) queries.append(self.create_dataSource_query(provider)) if s_pid: log.info("S_PID NOT NONE: {}".format(s_pid)) queries.append(self.create_pid_query(s_pid, 'source')) if t_pid: queries.append(self.create_pid_query(t_pid, 'target')) if s_publisher: queries.append(self.create_publisher_query(s_publisher, 'source')) if t_publisher: queries.append(self.create_publisher_query(t_publisher, 'target')) if s_pid_type: queries.append(self.create_pidType_query(s_pid_type, 'source')) if t_pid_type: queries.append(self.create_pidType_query(t_pid_type, 'target')) if target_Type: if 'literature' == target_Type: target_Type = 'publication' queries.append(self.create_typology_query(target_Type,'target')) if source_Type: if 'literature' == source_Type: source_Type = 'publication' queries.append(self.create_typology_query(source_Type,'source')) q = None for item in queries: if not q: q = item else: q = q & item log.debug("REQUEST CREATED {}".format(q)) search_object = Search(using=self.connection, index=self.index_name).doc_type('scholix').query(q) log.debug("Page request size is {}".format(page)) if page > 9999: return [] return search_object[page:page + 100].execute() def realtionToPid(self, pid, pidType=None, datasource=None, typology=None, page=0): if pidType: query = self.create_pid_pidType_query(pidType.lower(), pid.lower()) else: query = self.create_source_pid_query(pid.lower()) filters = [] if datasource and len(datasource): filters.append(self.create_dataSource_query(datasource)) if typology and len(typology): filters.append(self.create_typology_query(typology,'target')) search_object = Search(using=self.connection, index=self.index_name).doc_type('scholix').query(query) if len(filters): search_object = search_object.filter(Q('bool', must=filters)) if page > 9999: return [] return search_object[page:page + 100].execute() def realtionToTypology(self, typology, page=0): search_object = Search(using=self.connection, index=self.index_name).doc_type('scholix').query( self.create_typology_query(typology,'target')) if page > 9999: return [] return search_object[page:page + 100].execute()