Need help to build correct schema to represent my data in netflixgraph
Shyam Jena
Hi Team ,
Thanks for this awesome library.I am trying to see if this can help reduce the memory footprints for one of my application.I need some help to model my data model to represent the my existing data using netflixgraph .Below is the data model that stores info about the things running on different physical hosts for a specific DC
{
"DC2" : {
"Databases" : {
"Oracle" : [ "host4.domain.com", "host5.domain.com" ]
}
},
"DC1" : {
"Databases" : {
"Mongo" : [ "host1.domain.com", "host3.domain.com" ],
"Oracle" : [ "host1.domain.com", "host2.domain.com" ]
},
"Caches" : {
"Hazelcast" : [ "host1.domain.com", "host3.domain.com" ],
"Redis" : [ "host2.domain.com" ]
}
}
}
Below are the queries I need to make
- Find out all hosts running cache = redis in dc1
- Find out all hosts running database = oracle in dc2
- Find out all hosts running any databases etc ...
- Find out infrastructure types running on any dc.
Below is the test that I built to build the schema, populate and query my data .The issue is when I run a query to find how many hosts are running Oracle in DC1 I am getting wrong results .
public class TestMain {
OrdinalMap<String> hostOrdinals = new OrdinalMap<String>();
OrdinalMap<String> infraTypeOrdinals = new OrdinalMap<String>();
OrdinalMap<String> infraTypeOptionOrdinals = new OrdinalMap<String>();
OrdinalMap<String> dcOrdinals = new OrdinalMap<String>();
NFGraphSpec spec = new NFGraphSpec(
new NFNodeSpec(
"DC",
new NFPropertySpec("hasInfrastructure", "Infrastructure", MULTIPLE | COMPACT)
),
new NFNodeSpec("Infrastructure", new NFPropertySpec("hasOptions", "InfrastructureOptions", MULTIPLE | COMPACT)),
new NFNodeSpec("InfrastructureOptions", new NFPropertySpec("runsOnHost", "Host", MULTIPLE | COMPACT)),
new NFNodeSpec("Host")
);
public NFCompressedGraph initGraph() throws JsonProcessingException {
NFBuildGraph infraGraph = new NFBuildGraph(spec);
Map<String, Map<String, Map<String, List<String>>>> infraMapByDC = getInfraDataMap();
infraMapByDC.forEach((dc, infraOptionsMap) -> {
int dcOrdinal = dcOrdinals.add(dc);
infraOptionsMap.forEach((infraType, infraTypeOptionsWithHosts) -> {
int infraTypeOrdinal = infraTypeOrdinals.add(infraType);
infraTypeOptionsWithHosts.forEach((infraTypeOption, hostList) -> {
int infraTypeOptionOrdinal = infraTypeOptionOrdinals.add(infraTypeOption);
infraGraph.addConnection("Infrastructure", infraTypeOrdinal, "hasOptions", infraTypeOptionOrdinal);
hostList.forEach(host -> {
int hostOrdinal = hostOrdinals.add(host);
infraGraph.addConnection("InfrastructureOptions", infraTypeOptionOrdinal, "runsOnHost", hostOrdinal);
});
});
infraGraph.addConnection("DC", dcOrdinal, "hasInfrastructure", infraTypeOrdinal);
});
});
return infraGraph.compress();
}
public Map<String, Map<String, Map<String, List<String>>>> getInfraDataMap() throws JsonProcessingException {
Map<String, Map<String, Map<String, List<String>>>> infraMapByDC = new HashMap<>(2);
Map<String, Map<String, List<String>>> dc1InfraMap = new HashMap<>(2);
Map<String, List<String>> dc1DatabaseOptions = new HashMap<>();
dc1DatabaseOptions.put("Oracle", Arrays.asList("host1.domain.com", "host2.domain.com"));
dc1DatabaseOptions.put("Mongo", Arrays.asList("host1.domain.com", "host3.domain.com"));
Map<String, List<String>> dc1CacheOptions = new HashMap<>();
dc1CacheOptions.put("Redis", Collections.singletonList("host2.domain.com"));
dc1CacheOptions.put("Hazelcast", Arrays.asList("host1.domain.com", "host3.domain.com"));
dc1InfraMap.put("Databases", dc1DatabaseOptions);
dc1InfraMap.put("Caches", dc1CacheOptions);
Map<String, Map<String, List<String>>> dc2InfraMap = new HashMap<>(2);
Map<String, List<String>> dc2DatabaseOptions = new HashMap<>();
dc2DatabaseOptions.put("Oracle", Arrays.asList("host4.domain.com", "host5.domain.com"));
dc2InfraMap.put("Databases", dc2DatabaseOptions);
infraMapByDC.put("DC1", dc1InfraMap);
infraMapByDC.put("DC2", dc2InfraMap);
ObjectMapper om = new ObjectMapper();
final String json = om.writerWithDefaultPrettyPrinter().writeValueAsString(infraMapByDC);
System.out.println(json);
return infraMapByDC;
}
public Set<String> findHostsForInfraTypeAndOptionAndDC(final NFCompressedGraph infraGraph, final String dc, final String infraType, final String infraOption) {
int dcOrdinal = dcOrdinals.get(dc);
int infraTypeOrdinal = infraTypeOrdinals.get(infraType);
int infraTypOptionOrdinal = infraTypeOptionOrdinals.get(infraOption);
Set<String> hostList = new HashSet<>();
OrdinalSet infraTypesForDC = infraGraph.getConnectionSet("DC", dcOrdinal, "hasInfrastructure");
if (infraTypesForDC.contains(infraTypeOrdinal)) {
OrdinalSet infraTypeOptionsForType = infraGraph.getConnectionSet("Infrastructure", infraTypeOrdinal, "hasOptions");
if (infraTypeOptionsForType.contains(infraTypOptionOrdinal)) {
OrdinalIterator hostListIterator = infraGraph.getConnectionIterator("InfrastructureOptions", infraTypOptionOrdinal, "runsOnHost");
int hostOrdinal = hostListIterator.nextOrdinal();
while (hostOrdinal != OrdinalIterator.NO_MORE_ORDINALS) {
hostList.add(hostOrdinals.get(hostOrdinal));
hostOrdinal = hostListIterator.nextOrdinal();
}
}
}
return hostList;
}
public static void main(String[] args) throws JsonProcessingException {
TestMain testMain = new TestMain();
NFCompressedGraph infraGraph = testMain.initGraph();
Set<String> hostList = testMain.findHostsForInfraTypeAndOptionAndDC(infraGraph,"DC1","Databases","Oracle");
System.out.println(hostList);
//Expected hostList is "host1.domain.com", "host2.domain.com"
//Found hostList is host2.domain.com, host4.domain.com, host1.domain.com, host5.domain.com
}
}