/**
 * Copyright 2008-2009 DRIVER PROJECT (ICM UW)
 * Original author: Marek Horst
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *      http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package eu.dnetlib.enabling.manager;

import java.io.File;
import java.io.FileInputStream;
import java.io.FileOutputStream;
import java.io.InputStream;
import java.io.OutputStream;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.Iterator;
import java.util.List;

import javax.jws.WebService;
import javax.xml.ws.wsaddressing.W3CEndpointReference;

import org.apache.log4j.Logger;

import sun.misc.BASE64Encoder;
import eu.dnetlib.common.utils.xml.xpath.XPathParser;
import eu.dnetlib.data.index.IIndexService;
import eu.dnetlib.data.information.similarity.ISimilarityServiceBackend;
import eu.dnetlib.data.mdstore.IMDStoreService;
import eu.dnetlib.data.mdstore.MDStoreServiceException;
import eu.dnetlib.data.sts.das.IDataAccessService;
import eu.dnetlib.data.sts.ds.IDepotService;
import eu.dnetlib.data.utility.featureextraction.Feature;
import eu.dnetlib.data.utility.featureextraction.IFeatureExtractionService;
import eu.dnetlib.data.utility.featureextraction.Feature.STORAGETYPE;
import eu.dnetlib.enabling.is.lookup.rmi.ISLookUpException;
import eu.dnetlib.enabling.is.lookup.rmi.ISLookUpService;
import eu.dnetlib.enabling.is.registry.rmi.ISRegistryService;
import eu.dnetlib.enabling.manager.rs.iterator.ResultSetFeedIterator;
import eu.dnetlib.enabling.manager.rs.iterator.ResultSetIteratorContainer;
import eu.dnetlib.enabling.resultset.rmi.ResultSetException;
import eu.dnetlib.enabling.resultset.rmi.ResultSetService;
import eu.dnetlib.enabling.tools.JaxwsServiceResolverImpl;
import eu.dnetlib.enabling.tools.ServiceResolver;


/**
 * The Class DManagerServiceFacade.
 */
@WebService(endpointInterface="eu.dnetlib.enabling.manager.IDManagerServiceFacade",
		targetNamespace = "http://schemas.xmlsoap.org/soap/envelope/")
public class DManagerServiceFacade implements IDManagerServiceFacade{

	/** The Constant log. */
	protected static final Logger log = Logger.getLogger(DManagerServiceFacade.class);

	/** IS-LU service. */
	private ISLookUpService lookUpService;

	/** Result Set service. */
	private transient ResultSetService resultSetService;
	
	/** IS Registry service. */
	private ISRegistryService registryService;
	
	/** The depot service. */
	private IDepotService depotService;
	
	/** The data access service. */
	private IDataAccessService dataAccessService;
	
	/** The mdstore service. */
	private IMDStoreService mdstoreService;
	
	/** The fes service. */
	private IFeatureExtractionService fesService;
	
	/** The index service. */
	private IIndexService indexService;
	
	/** The similarity service. */
	private ISimilarityServiceBackend similarityService;
	
	/** Current Store service version. */
	private String serviceVersion;
	
	/** Storing Iterator queue size. */
	private int rsQueueSize;

	/** Storing Iterator single call result size. */
	private int rsPackageSize;

	/** Maximum timeout (in seconds) for retrieving single result from ResultSet while storing records. */
	long maxRSIteratorTimeout;
	
	/** The rs closing timeout. */
	private int rsClosingTimeout;
	
	/** The expiry time. */
	private int expiryTime;
	
	/** The keep alive time. */
	private int keepAliveTime;
	
	/** The store backend address. */
	private String storeBackendAddress;
	
	private String mainSDOStorgePath;
	
	public void init() {}
	
	private DManagerServiceFacadeOld oldManager;
	
	public String getMainSDOStorgePath() {
		return mainSDOStorgePath;
	}

	public void setMainSDOStorgePath(String mainSDOStorgePath) {
		this.mainSDOStorgePath = mainSDOStorgePath;
	}

	/* (non-Javadoc)
	 * @see eu.dnetlib.enabling.manager.IDManagerServiceFacade#identify()
	 */
	public String identify() {
		return getServiceVersion();
	}
	
	/* (non-Javadoc)
	 * @see eu.dnetlib.enabling.manager.IDManagerServiceFacade#notifySimilarity(java.lang.String, java.lang.String)
	 */
	public void notifySimilarity(String storeDataStructureIdentifier, String repositoryName) {
		
		NotifySimilarity ns = new NotifySimilarity(storeDataStructureIdentifier, repositoryName);
		ns.start();
	}
	
	/* (non-Javadoc)
	 * @see eu.dnetlib.enabling.manager.IDManagerServiceFacade#notifyIndex(java.lang.String, java.lang.String)
	 */
	public void notifyIndex(String repositoryName, boolean fulltext, String action) {
		NotifyIndex ni = new NotifyIndex(repositoryName, fulltext, action);
		ni.start();
		log.debug("Notify Index started !!!");
	}
	
	/* (non-Javadoc)
	 * @see eu.dnetlib.enabling.manager.IDManagerServiceFacade#processSimilarityFTDirectPDF(java.lang.String)
	 */
	public void processSimilarityFTDirectPDF(String repositoryName) {
		
		SimilarityFTDirectPDF sp = new SimilarityFTDirectPDF(repositoryName);
		sp.start();
	}
	
	/* (non-Javadoc)
	 * @see eu.dnetlib.enabling.manager.IDManagerServiceFacade#processIndexFTDirectPDF(java.lang.String)
	 */
	public void processIndexFTDirectPDF(String repositoryName) {
		
		IndexFTDirectPDF ip = new IndexFTDirectPDF(repositoryName);
		ip.start();
	}
	
	
	public void feedServiceFromDirectory(String repositoryId, String service) throws Exception{
		
		if(!"INDEX".equals(service) && !"SIMILARITY".equals(service))
			throw new Exception ("No proper service parameter, please entere INDEX or SIMILARITY");
		
		FeedFromDirectory fs = new FeedFromDirectory(repositoryId, service);
		fs.start();
	}
	
	public void feedFromSingleSubDirectory(String repositoryId, String service, String subdirectory) throws Exception{
		
		if(!"INDEX".equals(service) && !"SIMILARITY".equals(service))
			throw new Exception ("No proper service parameter, please entere INDEX or SIMILARITY");
		
		FeedFromSingleSubDirectory fs = new FeedFromSingleSubDirectory(repositoryId, service, subdirectory);
		fs.start();
	}
	
	
	public void splitDirectory(String repositoryId, String subdirectory) throws Exception{
	
			File subrepodir = new File(mainSDOStorgePath+"/"+repositoryId+"/"+subdirectory);
			String[] sdoList = subrepodir.list();
		
			if (sdoList == null)
				throw new Exception ("No records in "+ subrepodir.getAbsolutePath() );
			int counter = 1;
			int dirCounter = 1;
			String newDirPath = null;
			
			for (int i=0; i<sdoList.length; i++){
					
				if (counter == 1) {
					newDirPath = mainSDOStorgePath+"/"+repositoryId+"/"+subdirectory+"."+dirCounter;
					dirCounter++;
					File dir = new File(newDirPath);
					if(!dir.mkdir())
						throw  new Exception ("can not creat directory"+ dir.getAbsolutePath());
				}	
				
				File f1 = new File(mainSDOStorgePath+"/"+repositoryId+"/"+subdirectory+"/"+sdoList[i]);
			    File f2 = new File(newDirPath+"/"+sdoList[i]);
			  
			    InputStream in = new FileInputStream(f1);
			    OutputStream out = new FileOutputStream(f2);

			      byte[] buf = new byte[1024];
			      int len;
			      while ((len = in.read(buf)) > 0){
			        out.write(buf, 0, len);
			      }
			      in.close();
			      out.close();
			      
				if(counter == 594) 
					counter = 1;
				else
					counter++;
			}
	
	}
	/**
	 * The Class NotifySimilarity.
	 */
	class NotifySimilarity extends Thread {

		/** The store data structure identifier. */
		private String storeDataStructureIdentifier;
		
		/** The repository name. */
		private String repositoryName;

		/**
		 * Instantiates a new notify similarity.
		 * 
		 * @param storeDataStructureIdentifier the store data structure identifier
		 * @param repositoryName the repository name
		 */
		public NotifySimilarity (String storeDataStructureIdentifier, String repositoryName){
			this.storeDataStructureIdentifier = storeDataStructureIdentifier;
			this.repositoryName = repositoryName;

		}

		/* (non-Javadoc)
		 * @see java.lang.Thread#run()
		 */
		public void run() {

			try {	
				/*
				log.debug("FT_SIMILARITY : rep:"+ repositoryName+ " : stid:" + storeDataStructureIdentifier);
				checkServices();
				log.debug("Get repository identifier");
				String repoId = getRepositoryIdentifier(repositoryName);
				log.debug("Get mdstore identifier");
				String mdstoreId = getDMFMDStoreIdentifier(repoId);
				log.debug("Starting EXtraction");
				W3CEndpointReference fesEPR = extraction(storeDataStructureIdentifier);

				log.debug("Processing FES ResultSet and converting to the index record format");
				ResultSetFeedIterator SDOIter = initResultSetIterator(fesEPR,"index", mdstoreId);
				log.debug("FES RS processed");
				W3CEndpointReference indexObjectsRsEPR = processRecords(SDOIter);

				BASE64Encoder encoder = new BASE64Encoder();
				String indexEPR64Enc = encoder.encode(indexObjectsRsEPR.toString().getBytes());

				log.debug("Feeding Similarity Service");
				feedSimilarity(indexEPR64Enc);
				log.debug("(notifySimilarity):READY: FTs of repository "+repositoryName+" successfully indexed");
				*/
			} catch(Exception e) {
				log.error("(notifySimilarity):ERROR: Problem by repository: "+ repositoryName + " :"+e );
			}

		}
	}
	
	/**
	 * The Class NotifyIndex.
	 */
	class NotifyIndex extends Thread {

		/** The store data structure identifier. */
		private boolean fulltext;
		
		/** The repository name. */
		private String repositoryName;

		private String action;

		/**
		 * Instantiates a new notify index.
		 * 
		 * @param storeDataStructureIdentifier the store data structure identifier
		 * @param repositoryName the repository name
		 */
		public NotifyIndex (String repositoryName, boolean fulltext, String action){
			//this.storeDataStructureIdentifier = storeDataStructureIdentifier;
			this.fulltext = fulltext;
			this.repositoryName = repositoryName;
			this.action = action;
		}

		/* (non-Javadoc)
		 * @see java.lang.Thread#run()
		 */
		public void run() {


			try {
				if (fulltext) {
					if (action.equals("both") )
						oldManager.processFullText(repositoryName);
					else if (action.equals("store") )
						oldManager.downloadAndStore(repositoryName);
					else if (action.equals("extract") )
						oldManager.extractAndStore(repositoryName);
					else if (action.equals("index") )
						oldManager.index(repositoryName);
					else 
						throw new Exception ("Action type not known! (both|store|extract)");
				} else 
					oldManager.processStandardIndexing(repositoryName);
				/*
				log.debug("FT_INDEX : rep:"+ repositoryName+ " : stid:" + storeDataStructureIdentifier);
				checkServices();
				log.debug("Get repository identifier");
				String repoId = getRepositoryIdentifier(repositoryName);
				log.debug("Get mdstore identifier");
				String mdstoreId = getDMFMDStoreIdentifier(repoId);
				log.debug("Starting EXtraction");
				W3CEndpointReference fesEPR = extraction(storeDataStructureIdentifier);

				log.debug("Processing FES ResultSet and converting to the index record format");
				ResultSetFeedIterator SDOIter = initResultSetIterator(fesEPR,"index", mdstoreId);
				log.debug("FES RS processed");
				W3CEndpointReference indexObjectsRsEPR = processRecords(SDOIter);

				BASE64Encoder encoder = new BASE64Encoder();
				String indexEPR64Enc = encoder.encode(indexObjectsRsEPR.toString().getBytes());

				feedFullText(indexEPR64Enc, mdstoreId);
				log.debug("(notifyIndex):READY: FTs of repository "+repositoryName+" successfully indexed");
				*/
			} catch(Exception e) {
				log.error("(notifyIndex):ERROR: Problem by repository: "+ repositoryName+ " :"+e);
			}
		}

	}

	/**
	 * The Class IndexFTDirectPDF.
	 */
	class IndexFTDirectPDF extends Thread {

		/** The repository name. */
		private String repositoryName;

		/**
		 * Instantiates a new index ft direct pdf.
		 * 
		 * @param repositoryName the repository name
		 */
		public IndexFTDirectPDF (String repositoryName){
			this.repositoryName = repositoryName;

		}

		/* (non-Javadoc)
		 * @see java.lang.Thread#run()
		 */
		public void run() {

			try {
				checkServices();

				String repoId = getRepositoryIdentifier(repositoryName);
				String mdstoreId = getDMFMDStoreIdentifier(repoId);

				W3CEndpointReference mdstoreEPR = deliverMDRecords(mdstoreId);

				ResultSetFeedIterator DMFIter = initResultSetIterator(mdstoreEPR,"sdo",mdstoreId);
				//String stds = createStoreDataStructure();
				//log.debug("Store Data Structure created, identifier:" + stds);
				W3CEndpointReference sdoRsEPR = processRecords(DMFIter);
				//storeObjects(storeObjectsRsEPR, stds);

				log.debug("Starting EXtraction");
				W3CEndpointReference fesEPR = extraction(sdoRsEPR);

				ResultSetFeedIterator SDOIter = initResultSetIterator(fesEPR,"index", mdstoreId);
				log.debug("EXtracted");

				W3CEndpointReference indexObjectsRsEPR = processRecords(SDOIter);
				BASE64Encoder encoder = new BASE64Encoder();
				String indexEPR64Enc = encoder.encode(indexObjectsRsEPR.toString().getBytes());

				feedFullText(indexEPR64Enc, mdstoreId);
				log.debug("(notifyIndexDPDF):READY: FTs of repository "+repositoryName+" successfully indexed");
			} catch(Exception e) {
				log.error("(notifyIndexDPDF):ERROR: Problem by repository: "+ repositoryName+ " :"+e);
			}
			
		}
			
	}

	/**
	 * The Class IndexFTDirectPDF.
	 */
	class FeedFromDirectory extends Thread {

		/** The repository name. */
		private String repositoryId;
		private String service;

		/**
		 * Instantiates a new index ft direct pdf.
		 * 
		 * @param repositoryName the repository name
		 */
		public FeedFromDirectory (String repositoryId, String service){
			this.repositoryId = repositoryId;
			this.service = service;

		}

		/* (non-Javadoc)
		 * @see java.lang.Thread#run()
		 */
		public void run() {

			try {
				//checkServices();
				String mdstoreId = getDMFMDStoreIdentifier(repositoryId);

				W3CEndpointReference indexObjectsRsEPR = processDirectory(repositoryId,mdstoreId);
				BASE64Encoder encoder = new BASE64Encoder();
				String indexEPR64Enc = encoder.encode(indexObjectsRsEPR.toString().getBytes());
				
				log.debug("Feeding " + service);
				if (service.equals("INDEX"))
					feedFullText(indexEPR64Enc, mdstoreId);
				else if (service.equals("SIMILARITY"))
					feedSimilarity(indexEPR64Enc);
				
				log.debug("(FeedFromDirectory):READY: FTs of repository "+repositoryId+" successfully indexed");
			} catch(Exception e) {
				log.error("(FeedFromDirectory):ERROR: Problem by repository: "+ repositoryId+ " :"+e);
			}
			
		}
	}
	
	class FeedFromSingleSubDirectory extends Thread {

		/** The repository name. */
		private String repositoryId;
		private String service;
		private String subdirectory;

		/**
		 * Instantiates a new index ft direct pdf.
		 * 
		 * @param repositoryName the repository name
		 */
		public FeedFromSingleSubDirectory (String repositoryId, String service, String subdirectory){
			this.repositoryId = repositoryId;
			this.service = service;
			this.subdirectory = subdirectory;

		}

		/* (non-Javadoc)
		 * @see java.lang.Thread#run()
		 */
		public void run() {

			try {
				//checkServices();
				String mdstoreId = getDMFMDStoreIdentifier(repositoryId);

				W3CEndpointReference indexObjectsRsEPR = processSubDirectory(repositoryId,mdstoreId,subdirectory);
				BASE64Encoder encoder = new BASE64Encoder();
				String indexEPR64Enc = encoder.encode(indexObjectsRsEPR.toString().getBytes());
				
				log.debug("Feeding " + service);
				if (service.equals("INDEX"))
					feedFullText(indexEPR64Enc, mdstoreId);
				else if (service.equals("SIMILARITY"))
					feedSimilarity(indexEPR64Enc);
				
				log.debug("(FeedFromSingleSubDirectory):READY: FTs of repository "+repositoryId+" successfully indexed");
			} catch(Exception e) {
				log.error("(FeedFromSingleSubDirectory):ERROR: Problem by repository: "+ repositoryId+ " :"+e);
			}
			
		}
	}
	
	/**
	 * The Class SimilarityFTDirectPDF.
	 */
	class SimilarityFTDirectPDF extends Thread {

		/** The repository name. */
		private String repositoryName;

		/**
		 * Instantiates a new similarity ft direct pdf.
		 * 
		 * @param repositoryName the repository name
		 */
		public SimilarityFTDirectPDF (String repositoryName){
			this.repositoryName = repositoryName;
		}

		/* (non-Javadoc)
		 * @see java.lang.Thread#run()
		 */
		public void run() {	

			try{
				checkServices();

				String repoId = getRepositoryIdentifier(repositoryName);
				String mdstoreId = getDMFMDStoreIdentifier(repoId);

				W3CEndpointReference mdstoreEPR = deliverMDRecords(mdstoreId);

				//ResultSetFeedIterator DMFIter = initResultSetIterator(mdstoreEPR,"store",mdstoreId);
				ResultSetFeedIterator DMFIter = initResultSetIterator(mdstoreEPR,"sdo",mdstoreId);
				
				//String stds = createStoreDataStructure();
				//log.debug("Store Data Structure created, identifier:" + stds);
				W3CEndpointReference sdoRsEPR = processRecords(DMFIter);
				//storeObjects(storeObjectsRsEPR, stds);

				log.debug("Starting EXtraction");
				W3CEndpointReference fesEPR = extraction(sdoRsEPR);

				ResultSetFeedIterator SDOIter = initResultSetIterator(fesEPR,"index", mdstoreId);
				log.debug("EXtracted");

				W3CEndpointReference indexObjectsRsEPR = processRecords(SDOIter);
				BASE64Encoder encoder = new BASE64Encoder();
				String indexEPR64Enc = encoder.encode(indexObjectsRsEPR.toString().getBytes());

				log.debug("Feeding Similarity Service");
				feedSimilarity(indexEPR64Enc);
				log.debug("(notifySimilarityDPDF):READY: FTs of repository "+repositoryName+" successfully indexed");
			} catch(Exception e) {
				log.error("(notifySimilarityDPDF):ERROR: Problem by repository: "+ repositoryName+ " :"+e);
			}
		}
	}

	/**
	 * Feed full text.
	 * 
	 * @param indexEPR64Enc the index ep r64 enc
	 * @param mdstoreId the mdstore id
	 * 
	 * @throws Exception the exception
	 */
	private void feedFullText(String indexEPR64Enc, String mdstoreId) throws Exception {
		log.debug("Get index identifier");
		String indexId = getIndexIdentifier(mdstoreId);
		log.debug("Feeding Index Service");
		indexService.feedIndex(indexId,"INCREMENTAL", indexEPR64Enc, true, true);
	}
	
	
	/**
	 * Feed similarity.
	 * 
	 * @param indexEPR64Enc the index ep r64 enc
	 * 
	 * @throws Exception the exception
	 */
	private void feedSimilarity(String indexEPR64Enc) throws Exception {	
		similarityService.create("DMF", "index", "driver");
		similarityService.feed("similarityId", indexEPR64Enc);
	}

	/**
	 * Check services.
	 * 
	 * @throws Exception the exception
	 */
	private void checkServices() throws Exception{
		try {
			log.info("Store Depot Service identified: " + depotService.identify() );
		} catch (Exception e) {
			throw new Exception ("Problem with Store Depot Service identification: "+ e);
		}
		
		try {
			log.info("Store Data Access Service identified: " + dataAccessService.identify() );
		} catch (Exception e) {
			throw new Exception ("Problem with Store Data Access Service identification: "+ e);
		}
		
		try{
			log.info("Feature Extraction Service identified: " + fesService.identify() );
		} catch (Exception e) {
			throw new Exception ("Problem with Feature Extraction Service identification: "+ e);
		}	
			
		try{
			log.info("MDStore Service identified: " + mdstoreService.identify() );
		} catch (Exception e) {
			throw new Exception ("Problem with MDStore Service identification: "+ e);
		}	
			
		try{
			log.info("Index Service identified: " + indexService.identify() );
		} catch (Exception e) {
			throw new Exception ("Problem with Index Service identification: "+ e);
		}
			
	}

	/**
	 * Extraction.
	 * 
	 * @param sdoRsEPR the stds
	 * 
	 * @return the w3 c endpoint reference
	 * 
	 * @throws Exception the exception
	 */
	private W3CEndpointReference extraction(W3CEndpointReference sdoRsEPR) throws Exception {
				
		try { 
			//W3CEndpointReference dasEPR = dataAccessService.storeLookUpSDORS(sdoRsEPR);
			//log.debug(dasEPR);
			Feature f = new Feature();
			f.setName("fulltext");
			f.setStorageType(STORAGETYPE.INCREMENTAL);
			W3CEndpointReference epr = fesService.extract(f, sdoRsEPR, null);
			final ServiceResolver serviceResolver = new JaxwsServiceResolverImpl();
			final String rsId = serviceResolver.getResourceIdentifier(epr);
			
			boolean status = true;
			while (status) {
				String extractStat = fesService.getStatusInformation(rsId);
				if (extractStat.contains("DONE"))
						status = false;
				
				log.debug("Waiting for extraction, not ready yet");
				Thread.sleep(100000);
			}
			return	epr;	
		} catch (Exception e) {
			log.error(e);
			throw new Exception(e);
		}
	}

	/**
	 * Process records.
	 * 
	 * @param resultsetIter the resultset iter
	 * 
	 * @return the w3 c endpoint reference
	 * 
	 * @throws Exception the exception
	 */
	private W3CEndpointReference processRecords(ResultSetFeedIterator resultsetIter) throws Exception {
		
		try {
			Iterator<String> iter = resultsetIter.iterate();
			
			W3CEndpointReference epr = 
				resultSetService.createPushRS(expiryTime, keepAliveTime);
			
			final ServiceResolver serviceResolver = new JaxwsServiceResolverImpl();
			final String rsId = serviceResolver.getResourceIdentifier(epr);
			
			int counter = 0;
			List<String> list = new ArrayList<String>();
			boolean send = false;
			
			while (iter.hasNext() & !send) {
				String object = iter.next();
				if ("ignorethisobject".equals(object))
					continue;
				//log.debug(object);
				if (counter<5) {
					list.add(object);
					counter++;
					send = false;
				} else {
					resultSetService.populateRS(rsId, list );
					list = new ArrayList<String>();
					counter = 0;
					send = true;
				}	
			}
			
			if (!send)
				resultSetService.populateRS(rsId,list);
			log.debug("number of element in RS for FES: "+resultSetService.getNumberOfElements(rsId));
			resultSetService.closeRS(rsId);
			return epr;
			
		} catch (ResultSetException e) {
			log.error(e);
			throw new Exception(e);
		}
			
		
	}

	
	private W3CEndpointReference processDirectory(String repoDirectory,String mdstoreId) throws Exception {
		
		try {
			
			W3CEndpointReference epr = 
				resultSetService.createPushRS(expiryTime, keepAliveTime);
			
			final ServiceResolver serviceResolver = new JaxwsServiceResolverImpl();
			final String rsId = serviceResolver.getResourceIdentifier(epr);
			
			int counter = 0;
			List<String> list = new ArrayList<String>();
			boolean populate = true;
			
			File repodir = new File(mainSDOStorgePath+"/"+repoDirectory); 
			log.debug(repodir.getAbsolutePath());
			
			String[] subdirList = repodir.list();
			
			
			if (subdirList != null & subdirList.length !=0){
				
				for (int i=0; i<subdirList.length; i++){
						
					File subrepodir = new File(mainSDOStorgePath+"/"+repoDirectory+"/"+subdirList[i]);
					if (!subrepodir.isDirectory())
						continue;
					String[] sdoList = subrepodir.list();
					
					if (sdoList != null & sdoList.length !=0){
						
						for (int k=0; k<sdoList.length; k++){
					
							//log.debug(sdoList[k]);
							if (!sdoList[k].contains(".sdo")) continue;
					
							XPathParser xPathParser = new XPathParser(
									mainSDOStorgePath+"/"+repoDirectory+"/"+subdirList[i]+"/"+sdoList[k], true);
									
									
							String uri = xPathParser.getElementValue(
								"//SDO_URI");
			
							String sdoIdentifier = xPathParser.getAttributeValue("//RESOURCE_IDENTIFIER ", 
								"value");
			
							if (uri == null || sdoIdentifier == null)
								continue;
					
							String object = createIndexRecord(uri, sdoIdentifier, mdstoreId );
					
							if (object != null) {
								if (counter<50) {
									list.add(object);
									counter++;
									populate = true;
						
								} else {
									resultSetService.populateRS(rsId, list );
									list = new ArrayList<String>();
									counter = 0;
									populate = false;
								}
					
							} else 
								log.error("Object empty, ignored");
						}
					} else
						log.error("No sdos in "+ mainSDOStorgePath+"/"+repoDirectory+"/"+subdirList[i]);	
				}
			
			} else
				log.error("No subdirectories in "+ mainSDOStorgePath+"/"+repoDirectory);
			
			if (populate)
				resultSetService.populateRS(rsId,list);
			
			log.debug("number of element in RS: "+resultSetService.getNumberOfElements(rsId));
			resultSetService.closeRS(rsId);
			return epr;
			
		} catch (ResultSetException e) {
			log.error(e);
			throw new Exception(e);
		}
			
		
	}
	
	private W3CEndpointReference processSubDirectory(String repoDirectory,String mdstoreId, String subdirectory) throws Exception {
		
		try {
			
			W3CEndpointReference epr = 
				resultSetService.createPushRS(expiryTime, keepAliveTime);
			
			final ServiceResolver serviceResolver = new JaxwsServiceResolverImpl();
			final String rsId = serviceResolver.getResourceIdentifier(epr);
			
			int counter = 0;
			List<String> list = new ArrayList<String>();
			boolean populate = true;
			
			File repodir = new File(mainSDOStorgePath+"/"+repoDirectory+"/"+subdirectory); 
			log.debug(repodir.getAbsolutePath());
			
			String[] sdoList = repodir.list();

			if (sdoList != null & sdoList.length !=0){

				for (int k=0; k<sdoList.length; k++){

					//log.debug(sdoList[k]);
					if (!sdoList[k].contains(".sdo")) continue;

					XPathParser xPathParser = new XPathParser(
							mainSDOStorgePath+"/"+repoDirectory+"/"+subdirectory+"/"+sdoList[k], true);


					String uri = xPathParser.getElementValue(
					"//SDO_URI");

					String sdoIdentifier = xPathParser.getAttributeValue("//RESOURCE_IDENTIFIER ", 
					"value");

					if (uri == null || sdoIdentifier == null)
						continue;

					String object = createIndexRecord(uri, sdoIdentifier, mdstoreId );

					if (object != null) {
						if (counter<50) {
							list.add(object);
							counter++;
							populate = true;

						} else {
							resultSetService.populateRS(rsId, list );
							list = new ArrayList<String>();
							counter = 0;
							populate = false;
						}

					} else 
						log.error("Object empty, ignored");
				}
			} else
				log.error("No sdos in "+ mainSDOStorgePath+"/"+repoDirectory+"/"+subdirectory);	
				
			
			if (populate)
				resultSetService.populateRS(rsId,list);
			
			log.debug("number of element in RS: "+resultSetService.getNumberOfElements(rsId));
			resultSetService.closeRS(rsId);
			return epr;
			
		} catch (ResultSetException e) {
			log.error(e);
			throw new Exception(e);
		}
			
		
	}

	private String createIndexRecord(String uri, String sdoIdentifier,
			String mdstoreId) {
		
		try {
		
			String dmf = mdstoreService.deliverRecord(mdstoreId, sdoIdentifier);
			
			dmf = dmf.replace("UTF-8<", "<");
			XPathParser xPathParserDMF = new XPathParser(dmf, false);
			
			// getting schema definitions from record
			HashMap<String, String> schemasDMF = xPathParserDMF.extractSchemas();
				
			String language = xPathParserDMF.getElementValue(
					"//dc:language", "dc",
					schemasDMF.get("dc"));
			
			String creator = xPathParserDMF.getElementValue(
					"//dc:creator", "dc",
					schemasDMF.get("dc"));
			
			if (creator.contains("&")){
				log.debug("WARNING:"+ creator);
				creator = creator.replace("&", "&amp;");
				log.debug("WARNING:"+ creator);
			}
			
			
			String title = xPathParserDMF.getElementValue(
					"//dc:title", "dc",
					schemasDMF.get("dc"));
			
			if (title.contains("&")){
				log.debug("WARNING:"+ title);
				title = title.replace("&", "&amp;");
				log.debug("WARNING:"+ title);
			}
			
			
			dmf = dmf.replace("<?xml version=\"1.0\" encoding=\"UTF-8\"?>", "");
			dmf = dmf.replace("&", "&amp;");
			
			StringBuffer buffer = new StringBuffer();
			buffer.append("<indexRecord><indexRecordIdentifier>");
			buffer.append(sdoIdentifier);
			buffer.append("</indexRecordIdentifier><targetFields>");
			buffer.append("<language>"+language+"</language>");
			buffer.append("<creator>"+creator+"</creator>");
			buffer.append("<title>"+title+"</title>");
			buffer.append("<fturl>");
			buffer.append(uri);
			buffer.append("</fturl></targetFields>");
			buffer.append("<result>"+dmf+"</result>");
			buffer.append("</indexRecord>");
			String result = buffer.toString();
			result = result.replace("&", "&amp;");
			
			return result;
			
		} catch (Exception e) {
			log.error(e);
			return null;
		}
		
	}

	/**
	 * Store objects.
	 * 
	 * @param epr the epr
	 * @param stds the stds
	 * 
	 * @throws Exception the exception
	 */
	private void storeObjects (W3CEndpointReference epr, String stds) throws Exception{
		try {
			
			log.debug("Sending to the Store Service!");
			String actionId = depotService.storeObjectsFromRS(stds, epr, "REFRESH");
			//log.debug("ACTION ID:" + actionId);
			
			boolean ongoing = true;
			while (ongoing) {
				log.debug("Waiting as long as the Store Service will be ready!");
				String storingProfile  = depotService.storingCallback(stds, actionId, false);
				log.debug(storingProfile);
				if (storingProfile.contains("DONE"))
						ongoing = false;
				else
					Thread.sleep(100000);
				
			}	
			//log.debug("READY");
		} catch (Exception e){
			throw new Exception(e);
		}
		
	}

	/**
	 * Gets the repository identifier.
	 * 
	 * @param repositoryName the repository name
	 * 
	 * @return the repository identifier
	 * 
	 * @throws Exception the exception
	 */
	private String getRepositoryIdentifier (String repositoryName) throws Exception {
		String repositoryIdentifier = null;
		log.info("LookUp for repository Id of "+ repositoryName);
		
		try {
			List<String> repositoryIds = lookUpService.quickSearchProfile(
					"for $x in collection(" +
					"\"/db/DRIVER/RepositoryServiceResources/RepositoryServiceResourceType\")" +
					" where $x/RESOURCE_PROFILE//ENGLISH_NAME eq " +
					"\""+repositoryName+"\" return $x/RESOURCE_PROFILE//RESOURCE_IDENTIFIER");
			
			if (repositoryIds != null && repositoryIds.size() != 0) {			
				repositoryIdentifier = repositoryIds.get(0);
				repositoryIdentifier = repositoryIdentifier.replace("<RESOURCE_IDENTIFIER value=\"", "");
				repositoryIdentifier = repositoryIdentifier.replace("\"/>", "");
				log.info("Repository identifier returned by IS : "+ repositoryIdentifier);
				return repositoryIdentifier;
			} else {
				String msg = "Could not get repository id from IS: "+"for $x in collection(" +
				"\"/db/DRIVER/RepositoryServiceResources/RepositoryServiceResourceType\")" +
				" where $x/RESOURCE_PROFILE//OFFICIAL_NAME eq " +
				"\""+repositoryName+"\" return $x/RESOURCE_PROFILE//RESOURCE_IDENTIFIER";
				log.error(msg);
				throw new Exception(msg);
			}
		} catch(ISLookUpException e) {
			throw new Exception(e);
		}
		
	}
	
	/**
	 * Gets the dMFMD store identifier.
	 * 
	 * @param repositoryId the repository id
	 * 
	 * @return the dMFMD store identifier
	 * 
	 * @throws Exception the exception
	 */
	private String getDMFMDStoreIdentifier (String repositoryId) throws Exception {
		String mdstoreId = null;

		try {
			List<String> mdstoreIds = lookUpService.quickSearchProfile(
					"for $x in collection(" +
					"\"/db/DRIVER/TransformationDSResources/TransformationDSResourceType\")" +
					" where $x/RESOURCE_PROFILE//REPOSITORY_SERVICE_IDENTIFIER eq " +
					"\""+repositoryId+"\" return $x/RESOURCE_PROFILE//DATA_SINK");

			if (mdstoreIds != null && mdstoreIds.size() != 0) {			
				mdstoreId = mdstoreIds.get(0);
				mdstoreId = mdstoreId.replace("<DATA_SINK>dnet://MDStoreDS/", "");
				mdstoreId = mdstoreId.replace("?type=REFRESH</DATA_SINK>", "");
				log.info("MDStore identifier returned by IS: "+ mdstoreId);
				return mdstoreId;
			} else {
				String msg = "Could not get mdstore id from IS: "+"for $x in collection(" +
				"\"/db/DRIVER/TransformationDSResources/TransformationDSResourceType\")" +
				" where $x/RESOURCE_PROFILE//REPOSITORY_SERVICE_IDENTIFIER eq " +
				"\""+repositoryId+"\" return $x/RESOURCE_PROFILE//DATA_SINK";
				log.error(msg);
				throw new Exception(msg);
			}
		} catch(ISLookUpException e) {
			log.error(e);
			throw new Exception(e);
		}
	}
	
	/**
	 * Gets the index identifier.
	 * 
	 * @param mdstoreId the mdstore id
	 * 
	 * @return the index identifier
	 * 
	 * @throws Exception the exception
	 */
	private String getIndexIdentifier (String mdstoreId) throws Exception {
		String indexId = null;

		try {
			List<String> indexIds = lookUpService.quickSearchProfile(
					"for $el in fn:collection("+
					"\"DRIVER/ManagerServiceMapDSResources/ManagerServiceMapDSResourceType\")"+
					" return ($el)//INDEX_MAP/INDEX[MDSTORE/@id=\""+mdstoreId+"\"]/data(@id)");

			if (indexIds != null && indexIds.size() != 0) {			
				indexId = indexIds.get(0);
				log.info("Index identifier returned by IS: "+ indexId);
				return indexId;
			} else {
				String msg = "Could not get Index id from IS: "+
				"for $el in fn:collection("+
				"\"DRIVER/ManagerServiceMapDSResources/ManagerServiceMapDSResourceType\")"+
				"return ($el)//INDEX_MAP/INDEX[MDSTORE/@id=\""+mdstoreId+"\"]/data(@id)";
				log.error(msg);
				throw new Exception(msg);
			}
		} catch(ISLookUpException e) {
			log.error(e);
			throw new Exception(e);
		}
	}
	
	/**
	 * Deliver md records.
	 * 
	 * @param mdstoreId the mdstore id
	 * 
	 * @return the w3 c endpoint reference
	 * 
	 * @throws Exception the exception
	 */
	private W3CEndpointReference deliverMDRecords(String mdstoreId) throws Exception{
		
		try {
			W3CEndpointReference mdstoreEPR = 
				mdstoreService.deliverMDRecords(mdstoreId, null, null, "valid", false);
			return mdstoreEPR;
		} catch (MDStoreServiceException e) {
			log.error(e);
			throw new Exception(e);
		}
		
	}
	
	/**
	 * Inits the result set iterator.
	 * 
	 * @param epr the epr
	 * @param objectTypes the object types
	 * @param mdstoreId the mdstore id
	 * 
	 * @return the result set feed iterator
	 * 
	 * @throws Exception the exception
	 */
	private ResultSetFeedIterator initResultSetIterator (W3CEndpointReference epr, String objectTypes,
			String mdstoreId) throws Exception {
			
			final ServiceResolver serviceResolver = new JaxwsServiceResolverImpl();
			final ResultSetService resultSetService = serviceResolver.getService(
					ResultSetService.class, epr);
			final String rsId = serviceResolver.getResourceIdentifier(epr);
			
			int DMFnumber = resultSetService.getNumberOfElements(rsId);
			log.debug("Number of objects:" + DMFnumber);
			
			ResultSetIteratorContainer rsc = new ResultSetIteratorContainer();
			rsc.setMaxTimeout(maxRSIteratorTimeout);
			rsc.setMdstoreId(mdstoreId);
			rsc.setMdstoreService(mdstoreService);
			rsc.setQueueSize(rsQueueSize);
			rsc.setPackageSize(rsPackageSize);
			rsc.setResultsetService(resultSetService);
			rsc.setRsId(rsId);
			rsc.setRsClosingTimeout(rsClosingTimeout);
			rsc.setObjectTypes(objectTypes);
			rsc.setStoreBackendAddress(storeBackendAddress);
			
			ResultSetFeedIterator rsIterator = new ResultSetFeedIterator(rsc);
			return rsIterator;
	}
	
	/**
	 * Creates the store data structure.
	 * 
	 * @return the string
	 * 
	 * @throws Exception the exception
	 */
	private String createStoreDataStructure () throws Exception {
		
		try { 
			return 	depotService.createStore(null, 0);
		} catch (Exception e) {
			log.error(e);
			throw new Exception(e);
		}
	}
	
	/**
	 * Returns IS-LU service.
	 * 
	 * @return IS-LU service
	 */
	public ISLookUpService getLookUpService() {
		return lookUpService;
	}

	/**
	 * Sets IS-LU service.
	 * 
	 * @param lookUpService the look up service
	 */
	public void setLookUpService(ISLookUpService lookUpService) {
		this.lookUpService = lookUpService;
	}
	

	/**
	 * Gets the result set service.
	 * 
	 * @return the result set service
	 */
	public ResultSetService getResultSetService() {
		return resultSetService;
	}

	
	/**
	 * Sets the result set service.
	 * 
	 * @param resultSetService the new result set service
	 */
	public void setResultSetService(ResultSetService resultSetService) {
		this.resultSetService = resultSetService;
	}
	

	/**
	 * Gets the registry service.
	 * 
	 * @return the registry service
	 */
	public ISRegistryService getRegistryService() {
		return registryService;
	}

	/**
	 * Sets the registry service.
	 * 
	 * @param registryService the new registry service
	 */
	public void setRegistryService(ISRegistryService registryService) {
		this.registryService = registryService;
	}

	/**
	 * Gets the fes service.
	 * 
	 * @return the fes service
	 */
	public IFeatureExtractionService getFesService() {
		return this.fesService;
	}

	/**
	 * Sets the fes service.
	 * 
	 * @param fesService the new fes service
	 */
	public void setFesService(IFeatureExtractionService fesService) {
		this.fesService = fesService;
	}
	
	/**
	 * Gets the data access service.
	 * 
	 * @return the data access service
	 */
	public IDataAccessService getDataAccessService() {
		return this.dataAccessService;
	}

	/**
	 * Sets the data access service.
	 * 
	 * @param dataAccessService the new data access service
	 */
	public void setDataAccessService (IDataAccessService dataAccessService) {
		this.dataAccessService = dataAccessService;
	}
	
	/**
	 * Gets the depot service.
	 * 
	 * @return the depot service
	 */
	public IDepotService getDepotService() {
		return this.depotService;
	}

	/**
	 * Sets the depot service.
	 * 
	 * @param depotService the new depot service
	 */
	public void setDepotService (IDepotService depotService) {
		this.depotService = depotService;
	}
	
	/**
	 * Gets the mdstore service.
	 * 
	 * @return the mdstore service
	 */
	public IMDStoreService getMdstoreService() {
		return this.mdstoreService;
	}

	/**
	 * Sets the mdstore service.
	 * 
	 * @param mdstoreService the new mdstore service
	 */
	public void setMdstoreService(IMDStoreService mdstoreService) {
		this.mdstoreService = mdstoreService;
	}
	
	/**
	 * Gets the index service.
	 * 
	 * @return the index service
	 */
	public IIndexService getIndexService() {
		return this.indexService;
	}

	/**
	 * Sets the index service.
	 * 
	 * @param indexService the new index service
	 */
	public void setIndexService(IIndexService indexService) {
		this.indexService = indexService;
	}
	
	/**
	 * Gets the similarity service.
	 * 
	 * @return the similarity service
	 */
	public ISimilarityServiceBackend getSimilarityService() {
		return this.similarityService;
	}

	/**
	 * Sets the similarity service.
	 * 
	 * @param similarityService the new similarity service
	 */
	public void setSimilarityService(ISimilarityServiceBackend similarityService) {
		this.similarityService = similarityService;
	}
	
	/**
	 * Returns current service version.
	 * 
	 * @return current service version
	 */
	public String getServiceVersion() {
		return this.serviceVersion;
	}

	/**
	 * Sets current service version.
	 * 
	 * @param serviceVersion the service version
	 */
	public void setServiceVersion(String serviceVersion) {
		this.serviceVersion = serviceVersion;
	}
	
	/**
	 * Returns ResultSetIterator single call result size.
	 * 
	 * @return ResultSetIterator single call result size
	 */
	public int getRsPackageSize() {
		return this.rsPackageSize;
	}

	/**
	 * Sets ResultSetIterator single call result size.
	 * 
	 * @param rsPackageSize the rs package size
	 */
	public void setRsPackageSize(int rsPackageSize) {
		this.rsPackageSize = rsPackageSize;
	}

	/**
	 * Returns ResultSetIterator queue size.
	 * 
	 * @return ResultSetIterator queue size
	 */
	public int getRsQueueSize() {
		return this.rsQueueSize;
	}

	/**
	 * Sets ResultSetIterator queue size.
	 * 
	 * @param rsQueueSize the rs queue size
	 */
	public void setRsQueueSize(int rsQueueSize) {
		this.rsQueueSize = rsQueueSize;
	}

	/**
	 * Returns maximum timeout (in seconds) for retrieving single result from
	 * ResultSet while storing records.
	 * 
	 * @return maximum timeout (in seconds) for retrieving single result from
	 * ResultSet while storing records
	 */
	public long getMaxRSIteratorTimeout() {
		return this.maxRSIteratorTimeout;
	}

	/**
	 * Sets maximum timeout (in seconds) for retrieving single result from
	 * ResultSet while storing records.
	 * 
	 * @param maxRSIteratorTimeout the max rs iterator timeout
	 */
	public void setMaxRSIteratorTimeout(long maxRSIteratorTimeout) {
		this.maxRSIteratorTimeout = maxRSIteratorTimeout;
	}
	
	/**
	 * Gets the rs closing timeout.
	 * 
	 * @return the rs closing timeout
	 */
	public int getRsClosingTimeout() {
		return this.rsClosingTimeout;
	}

	/**
	 * Sets the rs closing timeout.
	 * 
	 * @param rsClosingTimeout the new rs closing timeout
	 */
	public void setRsClosingTimeout(int rsClosingTimeout) {
		this.rsClosingTimeout = rsClosingTimeout;
	}
	
	/**
	 * Returns result set expiry time.
	 * 
	 * @return result set expiry time
	 */
	public int getExpiryTime() {
		return expiryTime;
	}

	/**
	 * Sets result set expiry time.
	 * 
	 * @param expiryTime the expiry time
	 */
	public void setExpiryTime(int expiryTime) {
		this.expiryTime = expiryTime;
	}
	
	/**
	 * Returns keep alive time.
	 * 
	 * @return keep alive time
	 */
	public int getKeepAliveTime() {
		return keepAliveTime;
	}

	/**
	 * Sets keep alive time.
	 * 
	 * @param keepAliveTime the keep alive time
	 */
	public void setKeepAliveTime(int keepAliveTime) {
		this.keepAliveTime = keepAliveTime;
	}
	
	/**
	 * Gets the store backend address.
	 * 
	 * @return the store backend address
	 */
	public String getStoreBackendAddress() {
		return storeBackendAddress;
	}

	/**
	 * Sets the store backend address.
	 * 
	 * @param storeBackendAddress the new store backend address
	 */
	public void setStoreBackendAddress(String storeBackendAddress) {
		this.storeBackendAddress = storeBackendAddress;
	}

	public void setOldManager(DManagerServiceFacadeOld oldManager) {
		this.oldManager = oldManager;
	}

	public DManagerServiceFacadeOld getOldManager() {
		return oldManager;
	}
	
}