package eu.dnetlib.data.claims.migration; import eu.dnetlib.data.claims.migration.entity.*; import eu.dnetlib.data.claims.migration.handler.*; import eu.dnetlib.data.claimsDemo.ClaimUtils; import eu.dnetlib.data.claimsDemo.SQLStoreException; import org.apache.log4j.Logger; import org.springframework.context.ApplicationContext; import org.springframework.context.support.ClassPathXmlApplicationContext; import org.springframework.context.support.PropertySourcesPlaceholderConfigurer; import java.io.*; import java.math.BigInteger; import java.security.MessageDigest; import java.sql.Timestamp; import java.util.ArrayList; import java.util.Date; import java.util.List; /** * Created by kiatrop on 4/2/2016. */ public class Migration { private static final Logger logger = Logger.getLogger(Migration.class); RelationHandler relationHandler = null; ContextRelationHandler contextRelationHandler = null; ProjectHandler projectHandler = null; DMFContextHandler dmfContextHandler = null; DMFResultHandler dmfResultHandler = null; IndexResultHandler indexResultHandler = null; ExternalRecordHandler externalRecordHandler = null; ResultHandler resultHandler = null; ClaimHandler claimHandler = null; ClaimValidation claimValidation = new ClaimValidation(); public Migration() { ApplicationContext context = new ClassPathXmlApplicationContext("eu/dnetlib/data/claims/migration/springContext-claimsDemo.xml"); relationHandler = context.getBean(RelationHandler.class); contextRelationHandler = context.getBean(ContextRelationHandler.class); projectHandler = context.getBean(ProjectHandler.class); dmfContextHandler = context.getBean(DMFContextHandler.class); dmfResultHandler = context.getBean(DMFResultHandler.class); externalRecordHandler = context.getBean(ExternalRecordHandler.class); indexResultHandler = context.getBean(IndexResultHandler.class); resultHandler = context.getBean(ResultHandler.class); claimHandler = context.getBean(ClaimHandler.class); claimValidation = context.getBean(ClaimValidation.class); } public List createRelationsClaims() throws Exception, SQLStoreException { List relations = relationHandler.fetchAllRelations(); List claims = new ArrayList(); for (Relation relation : relations) { Claim claim= new Claim(); claim.setUserMail(relation.getClaimedBy()); claim.setDate(relation.getClaimDate()); claim.setId(relation.getClaimId()); System.out.println("Claim id:" + claim.getId()); OpenaireEntity source = null; //first work with the Source if (relation.getSourceType().equals(ClaimUtils.PROJECT)) { source= buildProject(relation.getSourceId(),relation.getClaimId()); claim.setSourceType(ClaimUtils.PROJECT); } else if (relation.getSourceType().equals(ClaimUtils.CONTEXT)) { Context context = dmfContextHandler.fetchContextByIdFromDmf(relation.getSourceId()); source = context; claim.setSourceType(ClaimUtils.CONTEXT); } else if (relation.getSourceType().equals(ClaimUtils.DATASET) || relation.getSourceType().equals(ClaimUtils.PUBLICATION)) { Result result = buildResult(relation.getDmf(), relation.getCollectedFrom(), relation.getSourceId(), relation.getClaimId()); source = result; claim.setSourceType(result.getResultType()); } claim.setSource(source); Result target = null; if (relation.getTargetType().equals(ClaimUtils.DATASET) || relation.getTargetType().equals(ClaimUtils.PUBLICATION)) { target = buildResult(relation.getDmf(), relation.getCollectedFrom(), relation.getTargetId(), relation.getClaimId()); } if( target != null) { claim.setTarget(target); claim.setTargetType(target.getResultType()); } if(claimValidation.validateClaim(claim)){ // is valid // System.out.println(claim); claims.add(claim); } } return claims; } public List createContextRelationClaims() throws Exception, SQLStoreException { List relations = contextRelationHandler.fetchAllConceptRelations(); List claims = new ArrayList(); for (ContextRelation relation : relations) { Claim claim= new Claim(); claim.setId(relation.getClaimId()); claim.setUserMail(relation.getClaimedBy()); claim.setDate(relation.getClaimDate()); System.out.println("Claim id:" + claim.getId()); //first work with the Source Context context = dmfContextHandler.extractContextFromDMF(relation.getContextDmf()); claim.setSourceType(ClaimUtils.CONTEXT); claim.setSource(context); Result target = buildResult(relation.getResultDmf(), relation.getCollectedFrom(), relation.getResultId(), relation.getClaimId()); if( target != null){ claim.setTarget(target); claim.setTargetType(target.getResultType()); } if(claimValidation.validateClaim(claim)){ // is valid claims.add(claim); } } return claims; } /** *Returns a project from Openaire. * In case the project id has a "welcometrust" prefix changes it to "wt__________", before search for it in the index. * * @param projectId * @param claimId * @return Project * @throws Exception */ public Project buildProject(String projectId, String claimId) throws Exception { String id = projectId; if(projectId != null && projectId.contains("welcometrust")){ id=projectId.replace("welcometrust","wt__________"); } Project project = projectHandler.fetchProjectByID(id); if (project == null) { try{ PrintWriter out = new PrintWriter(new BufferedWriter(new FileWriter("missing_projects.txt", true))); Date date= new java.util.Date(); out.println(new Timestamp(date.getTime())+" ProjectId: "+id+ " ClaimId: "+claimId); out.close(); }catch (IOException e) { logger.error("Couldn't write to file " + "missing_projects.txt",e); } } return project; } /** * *If there is a dmf, the result is extracted from the external sources (crossRef, Datacite, Orcid) * Otherwise Result is extracted from Openaire * @param dmf * @param collectedFrom * @param resultId * @param claimId * @return Result or null */ private Result buildResult(String dmf, String collectedFrom, String resultId, String claimId){ Result result = null; String external_id= null; // testing if((dmf !=null && collectedFrom != null)&& !collectedFrom.equals(ClaimUtils.COLLECTED_FROM_OPENAIRE)){ if (collectedFrom.equals(ClaimUtils.COLLECTED_FROM_CROSSREF)) { try { String doi= dmfResultHandler.fetchDoiByDMF(dmf); external_id=doi; if(doi != null) { result = externalRecordHandler.fetchResultfromCrossref(doi); } } catch (Exception e) { logger.error("Error fetching result from Crossref",e); } } else if (collectedFrom.equals(ClaimUtils.COLLECTED_FROM_ORCID)) { try { String orcidwork= dmfResultHandler.fetchOrcidWorkByDMF(dmf); external_id=orcidwork; if(orcidwork != null) { result = externalRecordHandler.fetchResultfromOrcid(orcidwork); } } catch (Exception e) { logger.error("Error fetching result from Orcid",e); } } else if (collectedFrom.equals(ClaimUtils.COLLECTED_FROM_DATACITE)) { try { String doi= dmfResultHandler.fetchDoiByDMF(dmf); external_id=doi; if(doi != null) { result = externalRecordHandler.fetchResultfromDatacite(doi); } } catch (Exception e) { logger.error("Error fetching result from Datacite",e); } } if(result == null){ //report missing external results try{ PrintWriter out = new PrintWriter(new BufferedWriter(new FileWriter(claimValidation.getPathToSaveReport()+"missing_results_external_sources.txt", true))); Date date= new java.util.Date(); out.println(new Timestamp(date.getTime())+" OpenaireId: "+resultId+ " collectedFrom: "+collectedFrom +" externalId: "+external_id+" claimId: "+claimId); out.close(); }catch (IOException e) { logger.error("Couldn't write to file " + "missing_results_external_sources.txt",e); } //give a second chance - search index result = buildOpenaireResult(resultId,claimId); if(result!=null) { try { PrintWriter out = new PrintWriter(new BufferedWriter(new FileWriter(claimValidation.getPathToSaveReport()+"external_results_found_inOpenaire.txt", true))); Date date = new java.util.Date(); out.println(new Timestamp(date.getTime()) + " OpenaireId: " + resultId + " collectedFrom: " + collectedFrom + " externalId: " + external_id + " claimId: " + claimId); out.close(); } catch (IOException e) { logger.error("Couldn't write to file " + "external_results_found_inOpenaire.txt",e); } } } if(result!=null){ // if result found enriched with access rights/ embargo date from DMF try { result.setAccessRights(dmfResultHandler.fetchAccessRights(dmf)); result.setEmbargoEndDate(dmfResultHandler.fetchEmbargoEndDateByDMF(dmf)); } catch (Exception e) { logger.error("Error fetching Access or Embargo end date from DMF",e); } } } else { //If dmf/ collecteFrom is null or is collected from openaire search in the index result = buildOpenaireResult(resultId,claimId); } return result; } /** * * @param resultId * @param claimId * @return target Result from Openaire or null */ public Result buildOpenaireResult(String resultId, String claimId ){ Result result = null; try { result = indexResultHandler.fetchResultById(resultId); } catch (Exception e) { logger.error("Error fetching result from Openaire",e); } //If it is not found in the index if (result == null) { //give a second chance as dedup try { result = indexResultHandler.fetchDedupResultById(resultId); } catch (Exception e) { logger.error("Error fetching dedup Result",e); } if (result != null) { try { PrintWriter out = new PrintWriter(new BufferedWriter(new FileWriter(claimValidation.getPathToSaveReport()+"dedup_results_found.txt", true))); Date date = new java.util.Date(); out.println(new Timestamp(date.getTime()) + " OpenaireId: " + resultId + " claimId: " + claimId); out.close(); } catch (IOException e) { logger.error("Couldn't write to file " + "external_results_found_inOpenaire.txt",e); } }else { //report it as missing try { PrintWriter out = new PrintWriter(new BufferedWriter(new FileWriter(claimValidation.getPathToSaveReport()+"missing_results.txt", true))); Date date = new java.util.Date(); out.println(new Timestamp(date.getTime()) + " OpenaireId: " + resultId + " ClaimId: " + claimId); out.close(); } catch (IOException e) { logger.error("Couldn't write to file " + "missing_results.txt",e); } } }else{ //FOUND report results that came from datacite but are publications //TODO deal with them as externals? if(result.getProvenanceaction() != null && result.getProvenanceaction().equals("user:claim:datacite")&&result.getOai()!=null){ try{ PrintWriter out = new PrintWriter(new BufferedWriter(new FileWriter(claimValidation.getPathToSaveReport()+"datacite_claim_results.txt", true))); Date date= new java.util.Date(); out.println(new Timestamp(date.getTime())+" openaireId:"+resultId+" aoi: " +result.getOai() +" provenanceactions: "+result.getProvenanceaction()+" type: "+result.getResultType()+ " ClaimId: "+claimId); out.close(); try { result= externalRecordHandler.fetchResultfromDatacite(result.getOai()); result.setOpenaireId(resultId); }catch(Exception e){ logger.error("Couldn't get Result from Datacite "+result.getOai(),e); } }catch (IOException e) { logger.error("Couldn't write to file " + "datacite_claim_results.txt",e); } } } return result; } /** * For each claim of the list * if there are results in the relation exports their metadaa file * Save the claim in DB * @param claims * @throws Exception */ private void saveClaims(List claims) throws Exception, SQLStoreException { for(Claim claim : claims){ /* if(claim.getTargetType().equals(ClaimUtils.DATASET)||claim.getTargetType().equals(ClaimUtils.PUBLICATION)){ String path = resultHandler.exportMetadataFileForResult((Result)claim.getTarget()); ((Result) claim.getTarget()).setRecordPath(path); } if(claim.getSourceType().equals(ClaimUtils.DATASET)||claim.getSourceType().equals(ClaimUtils.PUBLICATION)){ String path = resultHandler.exportMetadataFileForResult((Result)claim.getSource()); ((Result) claim.getSource()).setRecordPath(path); }*/ claim = claimHandler.exportMedatataForClaim(claim); claimHandler.saveClaim(claim); } } public static void main(String[] args) throws IOException { Migration migration = new Migration(); try { migration.claimHandler.getQueryGenerator().setMigrationTable("claims_view"); List claims = null; claims = migration.createContextRelationClaims(); // claims = migration.createRelationsClaims(); migration.printStatistics(claims); migration.saveClaims(claims); } catch (Exception e) { logger.error("Error in migration",e); } catch (SQLStoreException e) { e.printStackTrace(); } } public void printStatistics(List claims){ //testing List claimsOp= new ArrayList(); List claimsCr= new ArrayList(); List claimsDat= new ArrayList(); List claimsOrc= new ArrayList(); Integer targetOp=0; Integer targetCr=0; Integer targetDat=0; Integer targetOrc=0; Integer sourceRes=0; Integer sourceOp=0; Integer sourceCr=0; Integer sourceDat=0; Integer sourceOrc=0; Integer projects=0; Integer contexts=0; Integer totalClaims=0; for(Claim claim: claims){ // System.out.println(claim.toString()); if(((Result)claim.getTarget()).getCollectedFrom().equals(ClaimUtils.COLLECTED_FROM_CROSSREF)){ targetCr++; }else if(((Result)claim.getTarget()).getCollectedFrom().equals(ClaimUtils.COLLECTED_FROM_DATACITE)){ targetDat++; }else if(((Result)claim.getTarget()).getCollectedFrom().equals(ClaimUtils.COLLECTED_FROM_ORCID)){ targetOrc++; }else { targetOp++; } if(claim.getSourceType().equals(ClaimUtils.PUBLICATION)||claim.getSourceType().equals(ClaimUtils.DATASET)){ sourceRes++; if(((Result)claim.getSource()).getCollectedFrom().equals(ClaimUtils.COLLECTED_FROM_CROSSREF)){ sourceCr++; }else if(((Result)claim.getSource()).getCollectedFrom().equals(ClaimUtils.COLLECTED_FROM_DATACITE)){ sourceDat++; }else if(((Result)claim.getSource()).getCollectedFrom().equals(ClaimUtils.COLLECTED_FROM_ORCID)){ sourceOrc++; }else { sourceOp++; } }else if(claim.getSourceType().equals(ClaimUtils.PROJECT)) { projects++; }else if(claim.getSourceType().equals(ClaimUtils.CONTEXT)){ contexts++; } } totalClaims=claims.size(); System.out.println("\n\nTargets:\n\nCrossref : "+targetCr+ " DataCite :"+targetDat+" Orcid : "+targetOrc+" OpenAire : "+targetOp); System.out.println("\n\nSources:\n\nAll : "+sourceRes+" Crossref : "+sourceCr+ " DataCite :"+sourceDat+" Orcid : "+sourceOrc+" OpenAire : "+sourceOp); System.out.println("\n\nTotalClaims: "+totalClaims+" Projects: "+projects+ " Contexts :"+contexts); } }