/*
 * To change this license header, choose License Headers in Project Properties.
 * To change this template file, choose Tools | Templates
 * and open the template in the editor.
 */
package eu.dnetlib.espas.sos.client;

import eu.dnetlib.espas.pep.AuthenticationPEP;
import eu.dnetlib.espas.pep.PEPResponseMap;
import static eu.dnetlib.espas.sos.client.SOSRequestManager._logger;
import eu.dnetlib.espas.sos.client.SOSRequestStatus.RequestStatus;
import eu.dnetlib.espas.sos.client.jaxb.Data;
import eu.dnetlib.espas.sos.client.utils.DiskUtils;
import eu.dnetlib.espas.sos.client.utils.QuotaMonitor;
import eu.dnetlib.espas.sos.client.utils.RequestQuotaException;
import eu.dnetlib.espas.sos.client.utils.SOSDBUtils;
import eu.dnetlib.espas.sos.client.utils.SOSRequestStatusListener;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.net.MalformedURLException;
import java.net.URL;
import java.rmi.RemoteException;
import java.sql.SQLException;
import java.sql.Timestamp;
import java.util.Arrays;
import java.util.Collection;
import java.util.Date;
import java.util.HashMap;
import java.util.LinkedList;
import java.util.List;
import java.util.Map;
import java.util.Properties;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.Executor;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import javax.xml.bind.JAXB;
import javax.xml.transform.Transformer;
import javax.xml.transform.TransformerConfigurationException;
import javax.xml.transform.TransformerException;
import javax.xml.transform.TransformerFactory;
import javax.xml.transform.sax.SAXSource;
import javax.xml.transform.sax.SAXTransformerFactory;
import javax.xml.transform.stream.StreamResult;
import org.apache.commons.io.FileUtils;
import org.apache.log4j.Logger;
import org.springframework.beans.factory.annotation.Required;
import org.springframework.core.io.Resource;
import org.springframework.core.io.UrlResource;
import org.wso2.carbon.identity.entitlement.stub.EntitlementServiceException;
import org.xml.sax.InputSource;

/**
 * Implementation of the SOSRequest service that is responsible for managing the collection of replies from SOS result operation requests from
 * multiple SOS providers. This service provides an asynchronous request model that accepts requests from the ESPAS system and handles the collection
 * of SOS service replies in an offline manner. The provided implementation offers a getStatus operation to query the status of a pending request.
 *
 * @author gathanas
 */
public class SOSRequestManager implements SOSRequestServiceIF, OGCSensorServiceIF {

   static final Logger _logger = Logger.getLogger(SOSRequestManager.class);

   private static Executor requestPoolExecutor;
   private Map<String, Resource> nameXSLTMap;
   private int poolThreadSize;
   private SOSDBUtils dbSourceUtils;
   private AuthenticationPEP authenticationPEP = null;

   private SOSRequestStatusListener statusListener;
   private String sosStoreLocation;
   private File sosStorePath;
   private String xsltMapFileLocation = "";
   private Boolean clearTempRequestResponses = Boolean.TRUE;
   private QuotaMonitor quotaMonitor;
   private Resource ascii_csvXSLT = null;
   private Resource sosXSLT = null;
   private Resource sossweXSLT = null;
   private Resource sortResultsXSLT= null;
   private boolean processUnderQuotaFailure = true;

   public SOSRequestManager() {
   }

   
    @Override
    public String submitRequest(String offering, String observedProperty, Date startDate, Date endDate, String userId) {
      String requestKey = null;
      try {
         if (offering != null && !offering.isEmpty()) {
            SOSRequestWorker worker = new SOSRequestWorker(offering, observedProperty, startDate, endDate, 
                    dbSourceUtils, statusListener, sosStorePath, poolThreadSize, userId, clearTempRequestResponses, 
                    authenticationPEP, quotaMonitor, true);
            worker.setDefaultTransformations(nameXSLTMap);
            worker.setSortResultsTransformation(sortResultsXSLT);
            requestKey = worker.getRequestKey();
            requestPoolExecutor.execute(worker);
         }
      } catch (Exception ex) {
         _logger.error("Exception while processing SOS request", ex);
      }
      finally {
         return requestKey;
      }
    }

   @Override
   public String submitDataRequest(SOSRequestInfo requestInfo, String xsltName, InputStream xsltStream, String userId, List<Vocabulary> licenses) {
      String requestKey = null;
      try {
         if (requestInfo != null) {
            SOSRequestWorker worker = new SOSRequestWorker(requestInfo, dbSourceUtils, statusListener, sosStorePath, poolThreadSize, userId, clearTempRequestResponses, authenticationPEP, quotaMonitor, processUnderQuotaFailure);
            worker.setSortResultsTransformation(sortResultsXSLT);
            worker.setDefaultTransformations(nameXSLTMap);
            if (xsltStream != null && xsltName != null)
               worker.setTransformationDetails(xsltName, xsltStream);
            requestKey = worker.getRequestKey();
            requestPoolExecutor.execute(worker);
         }
      } catch (Exception ex) {
         _logger.error("Exception while processing SOS request", ex);
      }
      finally {
         return requestKey;
      }
   }

   @Override
   public String submitDataRequest(SOSRequestInfo requestInfo, String userId, List<Vocabulary> licenses) {
      String requestKey = null;
      try {
         if (requestInfo != null) {
            SOSRequestWorker worker = new SOSRequestWorker(requestInfo, dbSourceUtils, statusListener, sosStorePath, poolThreadSize, userId, clearTempRequestResponses, authenticationPEP, quotaMonitor, processUnderQuotaFailure);
            worker.setDefaultTransformations(nameXSLTMap);
            worker.setSortResultsTransformation(sortResultsXSLT);
            requestKey = worker.getRequestKey();
            requestPoolExecutor.execute(worker);
         }
      } catch (Exception ex) {
         _logger.error("Exception while processing SOS request", ex);
      }
      finally {
         return requestKey;
      }
   }

   @Override
   public SOSRequestStatus getDataRequestStatus(String requestId, String userId) {
      SOSRequestStatus statusResponse = new SOSRequestStatus(requestId, RequestStatus.UNKNOWN, "Failed to retrieve status information about this request. Either this has been removed from the database or there was a database connection problem.");
      try {
         Object[] requestStatus = dbSourceUtils.getSOSRequestStatus(requestId);
         List<Object[]> transformationStatusList = dbSourceUtils.getSOSTranformationStatus(requestId, userId);
         List<Object[]> providerStatusList = dbSourceUtils.getRequestProviderStatus(requestId, userId);

         statusResponse = new SOSRequestStatus(requestId,
                                               (requestStatus[0] != null || !((String) requestStatus[0]).isEmpty()) ? SOSRequestStatus.RequestStatus.
                                                  valueOf(requestStatus[0].toString()) : SOSRequestStatus.RequestStatus.UNKNOWN,
                                               (String) requestStatus[1]);

         statusResponse.setLatestUpdateTime((Date) requestStatus[2]);
         statusResponse.setExpirationDate((Date) requestStatus[3]);

         for (Object[] transformStatus : transformationStatusList)
            statusResponse.addTransaformationStatus(transformStatus[0].toString(), SOSRequestStatus.RequestStatus.valueOf(transformStatus[1].
                                                    toString()));

         for (Object[] providerStatus : providerStatusList)
            statusResponse.
               addProviderStatus((String) providerStatus[0], (RequestStatus) providerStatus[1], (String) providerStatus[2], (Date) providerStatus[3]);

      } catch (SQLException ex) {
         _logger.error("Error while retrieving status for request :" + requestId, ex);
      }
      return statusResponse;
   }

   @Override
   public InputStream getDataRequestResponseStream(String requestId, String userId) throws IOException {
      try {
         InputStream result = null;
         Object[] requestStatus = dbSourceUtils.getSOSRequestStatus(requestId);
         if (SOSRequestStatus.RequestStatus.valueOf((String) requestStatus[0]) == SOSRequestStatus.RequestStatus.COMPLETED) {
            File bundleFile = getSOSResponseBundle(requestId);
            if (bundleFile != null)
               result = new FileInputStream(bundleFile);
         }
         return result;
      } catch (SQLException ex) {
         _logger.error("Error while retrieving ", ex);
         throw new IOException(ex);
      }
   }

   @Override
   public void transformResponseIn(String requestId, String xsltName, InputStream xsltRules, String userId) throws IOException {
      SOSRequestStatus reqStatus = this.getDataRequestStatus(requestId, userId);
      if (!reqStatus.isTransformationSubmitted(xsltName))
         try {
            this.dbSourceUtils.insertTransformationRequest(requestId, userId, xsltName, SOSRequestStatus.RequestStatus.SUBMITTING, userId);
            SOSTransformationHandler transfromHndlr = new SOSTransformationHandler(requestId, userId, xsltName, xsltRules, this.statusListener, quotaMonitor);
            requestPoolExecutor.execute(transfromHndlr);
         } catch (SQLException ex) {
            _logger.error("", ex);
            throw new IOException("", ex);
         }
   }

   @Override
   public InputStream getResponseStreamIn(String requestId, String xsltName, String userId) throws IOException {
      InputStream result = null;
      SOSRequestStatus reqStatus = this.getDataRequestStatus(requestId, userId);
      if (reqStatus.getStatus() == SOSRequestStatus.RequestStatus.COMPLETED
         && reqStatus.getTransformationStatus(xsltName) == SOSRequestStatus.RequestStatus.COMPLETED) {
         File bundleFile = getSOSTransformationBundle(requestId, xsltName);
         if (bundleFile != null)
            result = new FileInputStream(bundleFile);
      }
      return result;
   }

   @Override
   public InputStream getSupportedXSLTStream(String name) {
      InputStream result = null;
      try {
         if (nameXSLTMap != null)
            result = nameXSLTMap.get(name).getInputStream();
      } catch (IOException ex) {
         _logger.error(null, ex);
      }
      return result;
   }

   @Override
   public Collection<String> getSupportedTransformations() {
      List<String> transformationNames = new LinkedList<String>();
      if (this.nameXSLTMap != null)
         return nameXSLTMap.keySet();
      return transformationNames;
   }

   @Override
   public void resubmitDataRequest(String requestId, String userId) {
//      add implementation code here
      try {
         if (SOSRequestStatus.RequestStatus.valueOf((String) dbSourceUtils.getSOSRequestStatus(requestId)[0]) == SOSRequestStatus.RequestStatus.EXPIRED)
            throw new Exception("Trying to resubmit an expired request.");

         SOSRequestWorker worker = new SOSRequestWorker(requestId, dbSourceUtils, statusListener, sosStorePath, poolThreadSize, userId, clearTempRequestResponses, authenticationPEP, quotaMonitor, processUnderQuotaFailure);
        worker.setSortResultsTransformation(sortResultsXSLT);
        worker.setDefaultTransformations(nameXSLTMap);
         requestPoolExecutor.execute(worker);
      } catch (Exception ex) {
         _logger.error("Exception while processing SOS resubmit request", ex);
      }
   }

///////////////////////////////////////////////////
//             
   private File getSOSResponseBundle(String requestId) {
      File bundleLocation = null;
      try {
         bundleLocation = this.quotaMonitor.getDmSpaceUtils().getRequestResultBundle(requestId, false);
      } catch (IOException ex) {
         _logger.error("Failed to retrieve response bundle file for request " + requestId, ex);
      }
      return bundleLocation;
   }

   private File getSOSTransformationBundle(String requestId, String transformationName) {
      File bundleLocation = null;
      try {
          String type = (transformationName.equals(SOSSweZipHandler.DEFAULT_NAME)?"zip":"xml");
         bundleLocation = this.quotaMonitor.getDmSpaceUtils().getRequestTransformationFile(requestId, transformationName, type, false);
      } catch (IOException ex) {
         _logger.error("Failed to retrieve transformation bundle file for request " + requestId + " and transformation " + transformationName, ex);
      }
      return bundleLocation;
   }

////////////////////////////////////////////////////////////////////////////////////
//          Bean initialization code
   /**
    * Bean initialization
    */
   public void init() {
      requestPoolExecutor = Executors.newFixedThreadPool(poolThreadSize);
      statusListener = new SOSRequestStatusListener(dbSourceUtils);
      this.sosStorePath = new File(sosStoreLocation);

      nameXSLTMap = new HashMap();
      nameXSLTMap.put(SOSSweZipHandler.DEFAULT_NAME, null);
      nameXSLTMap.put("ascii_csv", ascii_csvXSLT);
      nameXSLTMap.put("sos_response", sosXSLT);
      nameXSLTMap.put("sos_swe", sossweXSLT);

      if (xsltMapFileLocation != null && !xsltMapFileLocation.isEmpty())
         try {
            Properties props = new Properties();
            props.load(new FileInputStream(xsltMapFileLocation));
            for (int i = 0; i < props.keySet().size() / 2; i++)
               nameXSLTMap.
                  put(props.getProperty("xslt.mapping." + i + ".name"), new UrlResource(props.getProperty("xslt.mapping." + i + ".location")));

         } catch (IOException ex) {
            _logger.error("Exception while loading xslt mappings", ex);
         }

   }

   public int getPoolThreadSize() {
      return poolThreadSize;
   }

   public boolean getProcessUnderQuotaFailure() {
      return processUnderQuotaFailure;
   }

   public void setProcessUnderQuotaFailure(boolean processUnderQuotaFailure) {
      this.processUnderQuotaFailure = processUnderQuotaFailure;
   }

   public QuotaMonitor getQuotaMonitor() {
      return quotaMonitor;
   }

   @Required
   public void setQuotaMonitor(QuotaMonitor quotaMonitor) {
      this.quotaMonitor = quotaMonitor;
   }

   @Required
   public void setPoolThreadSize(int poolThreadSize) {
      this.poolThreadSize = poolThreadSize;
   }

   public SOSDBUtils getDbSourceUtils() {
      return dbSourceUtils;
   }

   @Required
   public void setDbSourceUtils(SOSDBUtils dbSourceUtils) {
      this.dbSourceUtils = dbSourceUtils.getDBUtilsInstance();
   }

   @Required
   public void setSosStoreLocation(String sosStoreLocation) {
      this.sosStoreLocation = sosStoreLocation;
   }

   public void setXsltMapFileLocation(String xsltMapFileLocation) {
      this.xsltMapFileLocation = xsltMapFileLocation;
   }

   public void setClearTempRequestResponses(Boolean clearTempRequestResponses) {
      if (clearTempRequestResponses == null)
         clearTempRequestResponses = Boolean.TRUE;
      this.clearTempRequestResponses = clearTempRequestResponses;
   }

   public void setAscii_csvXSLT(Resource ascii_csvXSLT) {
      this.ascii_csvXSLT = ascii_csvXSLT;
   }

   public void setSosXSLT(Resource sosXSLT) {
      this.sosXSLT = sosXSLT;
   }

    public Resource getSossweXSLT() {
        return sossweXSLT;
    }

    public void setSossweXSLT(Resource sossweXSLT) {
        this.sossweXSLT = sossweXSLT;
    }

    public Resource getSortResultsXSLT() {
        return sortResultsXSLT;
    }

    public void setSortResultsXSLT(Resource sortResultsXSLT) {
        this.sortResultsXSLT = sortResultsXSLT;
    }

   
   public void setAuthenticationPEP(AuthenticationPEP authenticationPEP) {
      this.authenticationPEP = authenticationPEP;
   }

}

/**
 * Worker threads used for monitoring the execution of SOS requests submitted from the system and their interaction with the local SOS providers.
 */
class SOSRequestWorker implements Runnable {

   private static final String SOS_REQUESTS_ARCHIVE_SUBPATH = "/sos_store/SOSRequestsArchive/";

   private boolean resubmitted = false;
   private final String userId;
   private SOSRequestInfo requestInfo;
   private String uniqueRequestID;
   private SOSDBUtils dbUtils;
   private SOSRequestStatusListener statusListener;
   private Executor providerExecutorPool;
   private Executor transformationExecutorPool;
   private File sosStorePath;
   private boolean clearTempRequestResponses = false;
   private AuthenticationPEP authenticationPEP = null;
   private String requestedTransformationName = null;
   private InputStream requestedXSLTStream = null;
   private boolean extraTransformationRequested = false;
   private QuotaMonitor quotaMonitor;
   private Map<String, Resource> defaultTransformations = null;
   private boolean processUnderQuotaFailure;
   private Resource sortResultsTransformation = null;

   public SOSRequestWorker(String requestId, SOSDBUtils dbUtils, SOSRequestStatusListener statusListener, File storePathLocation,
                           int providerThreadPoolSize, String userId, boolean clearTempResults, AuthenticationPEP authenticationPEP,
                           QuotaMonitor monitor, boolean processUnderQuotaFailure) throws SQLException {
      this.userId = userId;
      this.dbUtils = dbUtils;
      this.statusListener = statusListener;
      this.sosStorePath = storePathLocation;
      this.authenticationPEP = authenticationPEP;
      this.quotaMonitor = monitor;
      this.processUnderQuotaFailure = processUnderQuotaFailure;
      this.transformationExecutorPool = Executors.newSingleThreadExecutor();
      providerExecutorPool = Executors.newFixedThreadPool(providerThreadPoolSize);
      this.clearTempRequestResponses = clearTempResults;
      this.uniqueRequestID = requestId;
      this.resubmitted = true;
      retrieveRequestInfo();
   }

   public SOSRequestWorker(SOSRequestInfo requestInfo, SOSDBUtils dbUtils, SOSRequestStatusListener statusListener, File storePathLocation,
                           int providerThreadPoolSize, String userId, boolean clearTempResults, AuthenticationPEP authenticationPEP,
                           QuotaMonitor monitor, boolean processUnderQuotaFailure) {
      this.requestInfo = requestInfo;
      this.userId = userId;
      this.dbUtils = dbUtils;
      this.statusListener = statusListener;
      this.sosStorePath = storePathLocation;
      this.authenticationPEP = authenticationPEP;
      this.quotaMonitor = monitor;
      this.processUnderQuotaFailure = processUnderQuotaFailure;
      this.transformationExecutorPool = Executors.newSingleThreadExecutor();
      providerExecutorPool = Executors.newFixedThreadPool(providerThreadPoolSize);
      this.clearTempRequestResponses = clearTempResults;
      generateUniqueRequestKey();
   }

    SOSRequestWorker(String offering, String observedProperty, Date startDate, Date endDate, SOSDBUtils dbSourceUtils, SOSRequestStatusListener statusListener, 
            File sosStorePath, int poolThreadSize, String userId, Boolean clearTempRequestResponses, AuthenticationPEP authenticationPEP, 
            QuotaMonitor quotaMonitor, boolean processUnderQuotaFailure) throws SQLException {
      this.requestInfo = requestInfo;
      this.userId = userId;
      this.dbUtils = dbSourceUtils;
      this.statusListener = statusListener;
      this.sosStorePath = sosStorePath;
      this.authenticationPEP = authenticationPEP;
      this.quotaMonitor = quotaMonitor;
      this.processUnderQuotaFailure = processUnderQuotaFailure;
      this.transformationExecutorPool = Executors.newSingleThreadExecutor();
      providerExecutorPool = Executors.newFixedThreadPool(poolThreadSize);
      this.clearTempRequestResponses = clearTempRequestResponses;
      
      List<String> observedPropertyIdList = new LinkedList<String>();
      observedPropertyIdList.add(observedProperty);
      
          List<String> observationIdsList = dbUtils.getSOSObservationIds(offering, observedProperty, startDate, endDate);
      
      TimePeriodConstraint timeConstraint = new TimePeriodConstraint();
      timeConstraint.setFromDate(startDate);
      timeConstraint.setToDate(endDate);
      List<TimePeriodConstraint> timeConstraintList = new LinkedList<TimePeriodConstraint>();
      timeConstraintList.add(timeConstraint);
      
      this.requestInfo = new SOSRequestInfo(observationIdsList,observedPropertyIdList,timeConstraintList);
      generateUniqueRequestKey();      
    }

    public void setSortResultsTransformation(Resource sortResultsTransformation) {
        this.sortResultsTransformation = sortResultsTransformation;
    }

    
   public void setDefaultTransformations(Map<String, Resource> transformations) {
      this.defaultTransformations = transformations;
   }

   public void setTransformationDetails(String transformationName, InputStream transformationRuleStream) {
      this.requestedTransformationName = transformationName;
      this.requestedXSLTStream = transformationRuleStream;
      this.extraTransformationRequested = true;
   }

   private void generateUniqueRequestKey() {
      uniqueRequestID = requestInfo.generateUniqueRequestId();
   }

   public String getRequestKey() {
      return uniqueRequestID;
   }

   @Override
   public void run() {
      try {
         if (requestInfo != null && !requestInfo.getObservationIDList().isEmpty()) {
            List<SOSRequestInfo> providerRequestInfoList = generateProviderRequestInfo(requestInfo);
            registerRequest(providerRequestInfoList);

            if (isRequestPermited(providerRequestInfoList)) {
               CountDownLatch providerSyncLatch = new CountDownLatch(providerRequestInfoList.size());
               List<SOSProviderHandler> providerHndlrList = new LinkedList();
//            report request status

               if (statusListener != null)
                  statusListener.reportSOSRequestStatus(new SOSRequestStatus(requestInfo.getRequestId(), SOSRequestStatus.RequestStatus.RUNNING, ""));

               for (SOSRequestInfo providerInfo : providerRequestInfoList) {
                  SOSProviderHandler providerHandlr = new SOSProviderHandler(providerInfo, providerInfo.getProviderEndpoint(), authenticationPEP);
                  providerHandlr.setLatch(providerSyncLatch);

                  File filePath = getProviderRequestArchivePath(providerInfo);
                  providerHandlr.setProcessUnderQuotaFailure(processUnderQuotaFailure);
                  providerHandlr.setRequestQuotaMonitor(quotaMonitor);
                  providerHandlr.setStatusListener(statusListener);
                  providerHndlrList.add(providerHandlr);
                  providerExecutorPool.execute(providerHandlr);
               }
               providerSyncLatch.await();

               Data dataNode = new Data();
               dataNode.setRequestId(uniqueRequestID);

               for (SOSProviderHandler prHandler : providerHndlrList)
                  dataNode.getProvider().add(prHandler.getProviderNode());

               serializeRequestResults(dataNode);

               submitSortTransformation();
               
               submitDefaultTransformations();

               submitExtraTransformation(requestedTransformationName);

            }
            else
               statusListener.
                  reportSOSRequestStatus(new SOSRequestStatus(requestInfo.getRequestId(), SOSRequestStatus.RequestStatus.FAILED, " This data request has been rejected due to policy constraints."));
         }
      } catch (Exception ex) {
         SOSRequestManager._logger.
            error("Exception at thread processing request :" + this.uniqueRequestID + ". All provider processing threads will be terminated.", ex);
         if (statusListener != null)
            statusListener.
               reportSOSRequestStatus(new SOSRequestStatus(requestInfo.getRequestId(), SOSRequestStatus.RequestStatus.FAILED, "An error occured while processing the data download request. This request will be dropped."));
         ((ExecutorService) providerExecutorPool).shutdown();
      }
   }

   ////////////////////////////////////////////////////////////
   //    private methods definitions
   private void registerRequest(List<SOSRequestInfo> providerRequestInfoList) throws SQLException {
      if (!this.resubmitted)
         synchronized (this) {
            dbUtils.storeSoSRequest(providerRequestInfoList, userId);
            if (!providerRequestInfoList.isEmpty() && this.defaultTransformations != null && !defaultTransformations.isEmpty())
               for (String transformationName : defaultTransformations.keySet())
                  dbUtils.insertTransformationRequest(uniqueRequestID, userId, transformationName, SOSRequestStatus.RequestStatus.SUBMITTING, "");
         }
      else
         dbUtils.updateforResubmission(uniqueRequestID, userId);
   }

   @Deprecated
   private File getProviderRequestArchivePath(SOSRequestInfo providerInfo) {
      String locationChild = SOS_REQUESTS_ARCHIVE_SUBPATH + "/Req_" + providerInfo.getRequestId() + "/" + providerInfo.getProviderId().
         replaceAll("[^a-zA-Z0-9.-]", "_") + "/";
      File filePath = new File(this.sosStorePath, locationChild);

      if (!filePath.exists())
         filePath.mkdirs();

      return filePath;
   }

   private List<SOSRequestInfo> generateProviderRequestInfo(final SOSRequestInfo totalRequestInfo) {
      List<SOSRequestInfo> results = new LinkedList<SOSRequestInfo>();
      Map<String, SOSRequestInfo> providerRequestMap = new HashMap();
      try {
         for (String observationId : totalRequestInfo.getObservationIDList()) {
            Object[] providerObservationDetails = dbUtils.getProviderDetails(observationId);
            
            if(providerObservationDetails==null)
                continue;
            
            List<TimePeriodConstraint> _timePeriodConstraint = new LinkedList();
            for(TimePeriodConstraint _tConstraint: totalRequestInfo.getTimeConstraintList(observationId)){
                TimePeriodConstraint _ftCon = null;
                if(providerObservationDetails[2]!=null && providerObservationDetails[3]!=null) 
                    _ftCon = _tConstraint.filterConstraint(new Date(((Timestamp)providerObservationDetails[2]).getTime()), new Date(((Timestamp)providerObservationDetails[3]).getTime()));
                if(_ftCon!=null)
                    _timePeriodConstraint.add(_ftCon);                
            }

            _logger.debug("Found matchning time constraint for observation :"+observationId+" are :"+_timePeriodConstraint.size());
            
            if (providerRequestMap.containsKey(providerObservationDetails[0].toString())) {
               SOSRequestInfo providerRequestInfo = providerRequestMap.get(providerObservationDetails[0].toString());

               providerRequestInfo.addObservation(observationId, _timePeriodConstraint);
            }
            else {
               SOSRequestInfo provRequestInfo = new SOSRequestInfo(uniqueRequestID, Arrays.asList(observationId.toString()), totalRequestInfo.
                                                                   getPropertyIDList(),
                                                                   _timePeriodConstraint);
               provRequestInfo.setProviderId(providerObservationDetails[0].toString());
               provRequestInfo.setUserId(this.userId);
               try {
                   if(providerObservationDetails[1]==null || ((String)providerObservationDetails[1]).isEmpty())
                       continue;
                   else
                       provRequestInfo.setProviderEndpoint(new URL(providerObservationDetails[1].toString()));
               } catch (MalformedURLException ex) {
                  SOSRequestManager._logger.error("Exception while processing url for provider :" + providerObservationDetails[1].toString(), ex);
                  continue;
               }

               providerRequestMap.put(provRequestInfo.getProviderId(), provRequestInfo);
            }

         }

         for (String providerId : providerRequestMap.keySet()){
            SOSRequestInfo reqInfo = providerRequestMap.get(providerId);
            reqInfo.setObservationOfferingMappings(dbUtils.getObservationOfferings(reqInfo.getObservationIDList(),reqInfo.getPropertyIDList()));
            results.add(reqInfo);
         }

      } catch (SQLException ex) {
         SOSRequestManager._logger.error(null, ex);
      }
      finally {
         return results;
      }
   }

   private void serializeRequestResults(Data dataNode) throws IOException {
      File bundleFile = quotaMonitor.getDmSpaceUtils().getRequestResultBundle(uniqueRequestID, true);
      JAXB.marshal(dataNode, bundleFile);

//        clean all temporary result files!!!
      if (clearTempRequestResponses)
         this.quotaMonitor.cleanupTempRequestSpace(uniqueRequestID);
   }

   private void submitDefaultTransformations() {
      File bundleFile = null;
      try {
         bundleFile = quotaMonitor.getDmSpaceUtils().getRequestResultBundle(uniqueRequestID, false);
      } catch (IOException ex) {
         SOSRequestManager._logger.error("Failed to retrieve bundle file for request :" + uniqueRequestID
            + ". Please make sure that the default sos request archive store is accessible.", ex);
      }

      RequestStatus requestStatus = RequestStatus.UNKNOWN;
      try {
         Object statusValue = dbUtils.getSOSRequestStatus(uniqueRequestID)[0];
         requestStatus = (statusValue != null ? RequestStatus.valueOf((String)statusValue) : RequestStatus.UNKNOWN);
      } catch (SQLException ex) {
         SOSRequestManager._logger.error("Failed to retrieve sos request status for request " + uniqueRequestID, ex);
      }

      if (this.defaultTransformations != null && !defaultTransformations.isEmpty())
         if (bundleFile != null && bundleFile.exists() && requestStatus != RequestStatus.FAILED && requestStatus != RequestStatus.UNKNOWN)
            for (String transformationName : defaultTransformations.keySet())
               try {
                   SOSTransformationHandler transformationHandler;
                   if(transformationName.equals(SOSSweZipHandler.DEFAULT_NAME))
                       transformationHandler = new SOSSweZipHandler(uniqueRequestID, userId, transformationName,
                                                                                                null, statusListener, quotaMonitor);
                   else
                      transformationHandler = new SOSTransformationHandler(uniqueRequestID, userId, transformationName,
                                                                                                this.defaultTransformations.get(transformationName).
                                                                                                getInputStream(), statusListener, quotaMonitor);
                  transformationExecutorPool.execute(transformationHandler);
               } catch (Exception ex) {
                  SOSRequestManager._logger.error("Exception while submitting transformation requests", ex);
                  continue;
               }
         else
            this.statusListener.
               reportSOSRequestStatus(new SOSRequestStatus(uniqueRequestID, RequestStatus.FAILED, "SOS request processing has failed. "
                                                           + "The system was undable to collect valid results from the associated providers."));
      else
         this.statusListener.reportSOSRequestStatus(new SOSRequestStatus(uniqueRequestID, RequestStatus.COMPLETED, ""));
   }

   private void submitExtraTransformation(String transformationName) throws SQLException {
      File bundleFile = null;
      try {
         bundleFile = quotaMonitor.getDmSpaceUtils().getRequestResultBundle(uniqueRequestID, false);
      } catch (IOException ex) {
         SOSRequestManager._logger.error("Failed to retrieve bundle file for request :" + uniqueRequestID
            + ". Please make sure that the default sos request archive store is accessible.", ex);
      }

      RequestStatus requestStatus = RequestStatus.UNKNOWN;
      try {
         Object statusValue = dbUtils.getSOSRequestStatus(uniqueRequestID)[0];
         requestStatus = statusValue != null ? RequestStatus.valueOf(statusValue.toString()) : RequestStatus.UNKNOWN;
      } catch (SQLException ex) {
         SOSRequestManager._logger.error("Failed to retrieve sos request status for request " + uniqueRequestID, ex);
      }

      if (bundleFile != null && bundleFile.exists() && requestStatus != RequestStatus.FAILED && requestStatus == RequestStatus.COMPLETED && requestStatus != RequestStatus.UNKNOWN
         && extraTransformationRequested && !dbUtils.isTransformationSubmitted(uniqueRequestID, userId, transformationName)) {
         dbUtils.insertTransformationRequest(uniqueRequestID, userId, transformationName, SOSRequestStatus.RequestStatus.SUBMITTING, "");
         SOSTransformationHandler transformationHandler = new SOSTransformationHandler(uniqueRequestID, userId, transformationName, requestedXSLTStream, statusListener, quotaMonitor);
         transformationExecutorPool.execute(transformationHandler);
      }
   }

   private void retrieveRequestInfo() throws SQLException {
//        retrieve all required information from db
      List<String> observationIds = this.dbUtils.getSOSRequestObservationIds(uniqueRequestID, userId);
      List<String> propertyIds = this.dbUtils.getSOSRequestPropertyIds(uniqueRequestID, userId);
      List<String> timeConstraintDescriptions = this.dbUtils.getSOSRequestTimeConstraints(uniqueRequestID, userId);

      List<TimePeriodConstraint> timeConstraints = new LinkedList<TimePeriodConstraint>();
      for (String timeConstraintDescription : timeConstraintDescriptions)
         timeConstraints.add(new TimePeriodConstraint(timeConstraintDescription));

      this.requestInfo = new SOSRequestInfo(this.uniqueRequestID, observationIds, propertyIds, timeConstraints);
   }

   private boolean isRequestPermited(List<SOSRequestInfo> providerRequestInfoList) throws RemoteException, EntitlementServiceException, Exception {
      boolean totalRequestPermited = false;
      for (SOSRequestInfo providerRequestInfo : providerRequestInfoList) {
         boolean providerRequestPermited = false;
         PEPResponseMap pepResponseMap = authenticationPEP.isPermitedRequest(providerRequestInfo.getObservationIDList(), userId, "access", null);
         for (String providerObservation : providerRequestInfo.getObservationIDList())
            if (pepResponseMap.isResourcePermited(providerObservation)) {
               providerRequestPermited = true;
               break;
            }

         if (providerRequestPermited) {
            totalRequestPermited = true;
            break;
         }
         else
            statusListener.
               reportSOSRequestProviderStatus(providerRequestInfo.getProviderId(), new SOSRequestStatus(uniqueRequestID, RequestStatus.FAILED, "This data request has been denied due to policy constraints."));
      }

      return totalRequestPermited;
   }

    private void submitSortTransformation() throws IOException, TransformerException, TransformerConfigurationException, RequestQuotaException {
        ResultSorter sorter = new ResultSorter(this.uniqueRequestID,this.quotaMonitor.getDmSpaceUtils());
        FileInputStream xmlBundleStream = new FileInputStream(this.quotaMonitor.getDmSpaceUtils().getRequestResultBundle(this.uniqueRequestID, false));
        sorter.sort(xmlBundleStream, this.sortResultsTransformation.getInputStream());
    }
}

class ResultSorter{
    
    private String requestId;
    private DiskUtils diskUtils;

    public ResultSorter(String requestId, DiskUtils diskUtils) {
        this.requestId = requestId;
        this.diskUtils = diskUtils;
    }

    
    private File getSOSSortedBundle(String requestId) throws IOException {
        File sortedBundle = diskUtils.getSortedRequestResultBundle(requestId, true);
        return sortedBundle;
    }

    protected void sort(InputStream contentStream, InputStream transformationRulesStream) throws TransformerConfigurationException, TransformerException, IOException, RequestQuotaException {

        if(contentStream==null || transformationRulesStream==null)
            throw new IOException("Either of the Content or Transformation rules stream is null!");
        
        TransformerFactory transformFactory = TransformerFactory.newInstance();
        SAXTransformerFactory saxTFactory = ((SAXTransformerFactory) transformFactory);
        Transformer saxTransfomer = saxTFactory.newTransformer(new SAXSource(new InputSource(transformationRulesStream)));
        File transformationOutputFile = getSOSSortedBundle(requestId);
        FileOutputStream outputStream = new FileOutputStream(transformationOutputFile);
        saxTransfomer.transform(new SAXSource(new InputSource(contentStream)), new StreamResult(outputStream));
        saxTransfomer.reset();

        File originalResultBundle= diskUtils.getRequestResultBundle(requestId, false);
        
        if(FileUtils.sizeOf(transformationOutputFile)>0){
            FileUtils.forceDelete(originalResultBundle);
            FileUtils.moveFile(transformationOutputFile, originalResultBundle);
        }
         
    }
}