package eu.dnetlib.data.mapreduce.actions; import static eu.dnetlib.data.mapreduce.hbase.dataimport.DumpToActionsUtility.getStringValue; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertTrue; import java.io.BufferedReader; import java.io.IOException; import java.io.InputStream; import java.io.InputStreamReader; import java.util.List; import com.google.protobuf.InvalidProtocolBufferException; import org.apache.commons.lang3.StringUtils; import org.junit.Before; import org.junit.Test; import com.google.gson.JsonArray; import com.google.gson.JsonObject; import com.google.gson.JsonParser; import com.googlecode.protobuf.format.JsonFormat; import com.googlecode.protobuf.format.JsonFormat.ParseException; import eu.dnetlib.actionmanager.actions.ActionFactory; import eu.dnetlib.actionmanager.actions.AtomicAction; import eu.dnetlib.actionmanager.common.Agent; import eu.dnetlib.data.mapreduce.hbase.Reporter; import eu.dnetlib.data.mapreduce.hbase.dataimport.DumpToActionsUtility; import eu.dnetlib.data.mapreduce.hbase.dataimport.OrcidToActions; import eu.dnetlib.data.proto.OafProtos.Oaf; public class OrcidToActionsTest { private String setName; private Agent agent; private Reporter reporter; private final String generatedJson = "{\"kind\": \"entity\",\"entity\": {\"type\": \"result\",\"result\": {\"metadata\": {\"title\": [{\"value\": \"Cryptogenic fibrosing alveolitis in Bergen Hospital district\",\"qualifier\": {\"classid\": \"main title\",\"classname\": \"main title\",\"schemeid\": \"dnet:dataCite_title\",\"schemename\": \"dnet:dataCite_title\"}}],\"relevantdate\": [{\"value\": \"1999-01-01\",\"qualifier\": {\"classid\": \"issued\",\"classname\": \"issued\",\"schemeid\": \"dnet:dataCite_date\",\"schemename\": \"dnet:dataCite_date\"}}],\"dateofacceptance\": {\"value\": \"1999-01-01\"},\"resulttype\": {\"classid\": \"publication\",\"classname\": \"publication\",\"schemeid\": \"dnet:result_typologies\",\"schemename\": \"dnet:result_typologies\"},\"resourcetype\": {\"classid\": \"conference-abstract\",\"classname\": \"conference-abstract\",\"schemeid\": \"dnet:dataCite_resource\",\"schemename\": \"dnet:dataCite_resource\"},\"source\": [{\"value\": \"University of Southern Denmark\"}],\"author\": [{\"fullname\": \"Christian von Plessen\",\"name\": \"Christian\",\"surname\": \"von Plessen\",\"rank\": 1,\"pid\": [{\"key\": \"ORCID\",\"value\": \"0000-0002-6134-6780\"}]},{\"fullname\": \"Grinde, Ø.\",\"name\": \"Ø.\",\"surname\": \"Grinde\",\"rank\": 2},{\"fullname\": \"Gulsvik, A.\",\"name\": \"A.\",\"surname\": \"Gulsvik\",\"rank\": 3}]},\"instance\": [{\"accessright\": {\"classid\": \"UNKNOWN\",\"classname\": \"UNKNOWN\",\"schemeid\": \"dnet:access_modes\",\"schemename\": \"dnet:access_modes\"},\"instancetype\": {\"classid\": \"0004\",\"classname\": \"Conference object\",\"schemeid\": \"dnet:publication_resource\",\"schemename\": \"dnet:publication_resource\"},\"hostedby\": {\"key\": \"10|openaire____::55045bd2a65019fd8e6741a755395c8c\",\"value\": \"Unknown Repository\"},\"url\": [\"http://findresearcher.sdu.dk/portal/en/publications/cryptogenic-fibrosing-alveolitis-in-bergen-hospital-district(14a8eea6-ddf6-4055-9e7e-a82525ae36c2).html\"],\"collectedfrom\": {\"key\": \"10|openaire____::806360c771262b4d6770e7cdf04b5c5a\",\"value\": \"ORCID\"},\"dateofacceptance\": {\"value\": \"1999-01-01\"}}]},\"originalId\": [\"29721246\"],\"collectedfrom\": [{\"key\": \"10|openaire____::806360c771262b4d6770e7cdf04b5c5a\",\"value\": \"ORCID\"}],\"pid\": [{\"value\": \"14a8eea6-ddf6-4055-9e7e-a82525ae36c2\",\"qualifier\": {\"classid\": \"orcidworkid\",\"classname\": \"orcidworkid\",\"schemeid\": \"dnet:pid_types\",\"schemename\": \"dnet:pid_types\"}}],\"dateofcollection\": \"2018-10-22\",\"id\": \"50|orcid_______::01d661c1f7824215e5150d59b30f7cf7\",\"dateoftransformation\": \"2019-10-04T12:14:24+02:00\"},\"dataInfo\": {\"inferred\": false,\"deletedbyinference\": false,\"trust\": \"0.9\",\"provenanceaction\": {\"classid\": \"sysimport:actionset:orcidworks-no-doi\",\"classname\": \"sysimport:actionset:orcidworks-no-doi\",\"schemeid\": \"dnet:provenanceActions\",\"schemename\": \"dnet:provenanceActions\"}},\"lastupdatetimestamp\": 1570184064208}\n" + ""; private boolean printProto = false; @Before public void setup() { setName = "ORCID"; agent = new Agent("agentId", "agentName", Agent.AGENT_TYPE.service); reporter = (Reporter) (counterGroup, counterName, delta) -> System.out.println(String.format("COUNTER: %s - %s : %d", counterGroup, counterName, delta)); } @Test public void testUnicodeAction() throws IOException { doTestSingleAction("/eu/dnetlib/data/mapreduce/actions/OrcidAction_1.json", printProto); } @Test public void testSourceWorkIdAction() throws IOException { doTestSingleAction("/eu/dnetlib/data/mapreduce/actions/OrcidAction_2.json", printProto); } @Test public void testAuthorsRank() throws IOException { doTestSingleAction("/eu/dnetlib/data/mapreduce/actions/OrcidAction_3.json", printProto); doTestSingleAction("/eu/dnetlib/data/mapreduce/actions/OrcidAction_4.json", printProto); doTestSingleAction("/eu/dnetlib/data/mapreduce/actions/OrcidAction_5.json", printProto); } @Test public void testUrl() throws IOException { doTestSingleAction("/eu/dnetlib/data/mapreduce/actions/OrcidAction_6.json", printProto); } @Test public void testNullUrlAction() throws IOException { doTestSingleAction("/eu/dnetlib/data/mapreduce/actions/OrcidAction_7.json", printProto); } @Test public void testMassiveOrcidAction() throws IOException { doTestAllOrcidAction("/eu/dnetlib/data/mapreduce/actions/part-100"); } @Test public void testJsonToProto() throws ParseException { final Oaf.Builder builder = Oaf.newBuilder(); JsonFormat.merge(generatedJson, builder); System.out.println(builder.build()); } private void doTestSingleAction(final String filePath, boolean print) throws IOException { final InputStream is = this.getClass().getResourceAsStream(filePath); final BufferedReader in = new BufferedReader(new InputStreamReader(is)); final String line = in.readLine(); final JsonParser parser = new JsonParser(); final JsonObject root = parser.parse(line).getAsJsonObject(); final List actions = OrcidToActions.generatePublicationActionsFromDump(root, new ActionFactory(), setName, agent, reporter); if(print) { if (actions!= null) { actions.forEach(it -> { try { System.out.println( String.format(" RowKey:%s TargetColumnFamily:%s TargetColumn: %s\n value:\n%s", it.getTargetRowKey(), it.getTargetColumnFamily(), it.getTargetColumn(), Oaf.parseFrom(it.getTargetValue()))); } catch (InvalidProtocolBufferException e) { e.printStackTrace(); } }); } } } private void doTestAllOrcidAction(final String filePath) throws IOException { final InputStream is = this.getClass().getResourceAsStream(filePath); final BufferedReader in = new BufferedReader(new InputStreamReader(is)); String line = in.readLine(); int i = 0; while (StringUtils.isNotBlank(line)) { final JsonParser parser = new JsonParser(); final JsonObject root = parser.parse(line).getAsJsonObject(); try { OrcidToActions.generatePublicationActionsFromDump(root, new ActionFactory(), setName, agent, reporter); } catch (final Throwable e) { System.out.println("Exception parsing: " + line); throw new RuntimeException(e); } line = in.readLine(); i += 1; } } @Test public void testGeneratedJson() throws ParseException { final JsonParser parser = new JsonParser(); final JsonObject root = parser.parse(generatedJson).getAsJsonObject(); assertEquals(getStringValue(root, "kind"), "entity"); final JsonObject entity = root.getAsJsonObject("entity"); final List pids = DumpToActionsUtility.getArrayObjects(entity, "pid"); pids.forEach(pid -> { final JsonObject qualifier = pid.getAsJsonObject("qualifier"); assertTrue(qualifier.get("classid").getAsString().matches("(ark|arxiv|pmc|pmid|orcidworkid|urn)")); }); assertEquals(getStringValue(entity, "type"), "result"); assertNotNull(getStringValue(entity, "id")); assertNotNull(getStringValue(entity, "dateofcollection")); assertTrue(DumpToActionsUtility.isValidDate(getStringValue(entity, "dateofcollection"))); final JsonObject metadata = entity.getAsJsonObject("result").getAsJsonObject("metadata"); assertNotNull(metadata.getAsJsonArray("title")); assertNotNull(metadata.getAsJsonArray("relevantdate")); assertNotNull(metadata.getAsJsonObject("dateofacceptance")); assertTrue(DumpToActionsUtility.isValidDate(metadata.getAsJsonObject("dateofacceptance").get("value").getAsString())); assertNotNull(metadata.getAsJsonObject("resulttype")); assertNotNull(metadata.getAsJsonArray("author")); final JsonArray instance = entity.getAsJsonObject("result").getAsJsonArray("instance"); instance.forEach(i -> { assertNotNull(i.getAsJsonObject().getAsJsonObject("accessright")); assertNotNull(i.getAsJsonObject().getAsJsonObject("hostedby")); assertNotNull(i.getAsJsonObject().getAsJsonObject("collectedfrom")); assertNotNull(i.getAsJsonObject().getAsJsonObject("instancetype")); }); } }