Neo4j Cypher query execution plan optimization - neo4j

Question

I have the following Cypher query:
MATCH (dg:DecisionGroup {id: -2})-[rdgd:CONTAINS]->(childD:Decision:Profile )
MATCH (childD)-[:EMPLOYMENT_AS]->(root2:Employment )
WHERE root2.id IN ([1]) WITH DISTINCT childD, dg, rdgd
MATCH path3=(root3:Location )-[:CONTAINS*0..]->(descendant3:Location)
WHERE (descendant3.id IN ([1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35]) OR root3.id IN ([1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35]))
UNWIND nodes(path3) AS pathNode3 WITH childD, dg, rdgd, COLLECT(DISTINCT pathNode3) AS pathNodes3
MATCH (childD)-[:LOCATED_IN]->(pathNode3) WHERE pathNode3 IN pathNodes3 WITH DISTINCT childD, dg, rdgd WHERE (childD.`active` = true) AND (childD.`experienceMonths` >= 129) AND ( (childD.`minSalaryUsd` <= 8883) OR (childD.`minHourlyRateUsd` <= 126) )
MATCH (childD)-[criterionRelationship8:HAS_VOTE_ON]->(c:Criterion {id: 2}) WHERE (criterionRelationship8.`properties.experienceMonths` >= 1) WITH DISTINCT childD, dg, rdgd
MATCH (childD)-[criterionRelationship10:HAS_VOTE_ON]->(c:Criterion {id: 36}) WHERE (criterionRelationship10.`avgVotesWeight` >= 1.0) AND (criterionRelationship10.`properties.experienceMonths` >= 1) WITH DISTINCT childD, dg, rdgd
MATCH (childD)-[criterionRelationship13:HAS_VOTE_ON]->(c:Criterion {id: 4}) WHERE (criterionRelationship13.`properties.experienceMonths` >= 0) WITH DISTINCT childD, dg, rdgd
MATCH (childD)-[criterionRelationship15:HAS_VOTE_ON]->(c:Criterion {id: 22}) WHERE (criterionRelationship15.`avgVotesWeight` >= 1.0) AND (criterionRelationship15.`properties.experienceMonths` >= 1) WITH DISTINCT childD, dg, rdgd
OPTIONAL MATCH (childD)-[ru:CREATED_BY]->(u:User) WITH childD, u, ru, dg, rdgd
OPTIONAL MATCH (childD)-[vg:HAS_VOTE_ON]->(c:Criterion) WHERE c.id IN [2, 36, 4, 22] WITH c, childD, u, ru, dg, rdgd, (vg.avgVotesWeight * (CASE WHEN c IS NOT NULL THEN coalesce({`22`:1.2236918603185925, `2`:2.9245935245152226, `36`:0.2288013749943646, `4`:3.9599506966378435}[toString(c.id)], 1.0) ELSE 1.0 END)) as weight, vg.totalVotes as totalVotes
WITH childD, u, ru , dg, rdgd , toFloat(sum(weight)) as weight, toInteger(sum(totalVotes)) as totalVotes
ORDER BY weight DESC , childD.createdAt DESC
SKIP 0 LIMIT 20
WITH * OPTIONAL MATCH (childD)-[rup:UPDATED_BY]->(up:User)
RETURN rdgd, ru, u, rup, up, childD AS decision, weight, totalVotes, [ (c1)<-[vg1:HAS_VOTE_ON]-(childD) WHERE c1.id IN [2, 36, 4, 22] | {criterion: c1, relationship: vg1} ] AS weightedCriteria
This query is automatically generated by my Cypher query builder. Right now on 1000 Profiles the query executes ~8 seconds.
Looks like this part of the query causes most of the issues:
MATCH (childD)-[:EMPLOYMENT_AS]->(root2:Employment )
WHERE root2.id IN ([1]) WITH DISTINCT childD, dg, rdgd
MATCH path3=(root3:Location )-[:CONTAINS*0..]->(descendant3:Location)
WHERE (descendant3.id IN ([1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35]) OR root3.id IN ([1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35]))
UNWIND nodes(path3) AS pathNode3 WITH childD, dg, rdgd, COLLECT(DISTINCT pathNode3) AS pathNodes3
MATCH (childD)-[:LOCATED_IN]->(pathNode3) WHERE pathNode3 IN pathNodes3 WITH DISTINCT childD, dg, rdgd
Is there a way to optimize this?
This is PROFILE output:
UPDATED
I reimplemented initial part of the query to the following:
WITH [] as ceNodeList MATCH (root2:Employment )
WHERE root2.id IN ([1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16])
WITH ceNodeList, root2, COLLECT(root2) AS listRoot2
WITH apoc.coll.unionAll(ceNodeList, listRoot2) AS ceNodeList
WITH apoc.coll.toSet(ceNodeList) as ceNodeList
MATCH (root3:Location )
WHERE root3.id IN ([1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 64, 65, 66, 67, 68, 69, 70, 71, 72, 73])
WITH ceNodeList, root3, COLLECT(root3) AS listRoot3
OPTIONAL MATCH (root3)-[:CONTAINS*0..]->(descendant3:Location)
OPTIONAL MATCH (ascendant3:Location)-[:CONTAINS*0..]->(root3)
WITH ceNodeList, listRoot3, COLLECT( DISTINCT ascendant3) AS listAscendant3, COLLECT( DISTINCT descendant3) AS listDescendant3
WITH listRoot3, listAscendant3, apoc.coll.unionAll(ceNodeList, apoc.coll.unionAll(listDescendant3, apoc.coll.unionAll(listRoot3, listAscendant3))) AS ceNodeList
WITH apoc.coll.toSet(ceNodeList) as ceNodeList
UNWIND ceNodeList AS ceNode
WITH DISTINCT ceNode MATCH (dg:DecisionGroup {id: -2})-[rdgd:CONTAINS]->(childD:Decision:Profile ) -[:REQUIRES]->(ceNode)
WITH DISTINCT childD, dg, rdgd, collect(ceNode) as ceNodes
WITH childD, dg, rdgd, ceNodes, reduce(ceNodeLabels = [], n IN ceNodes | ceNodeLabels + labels(n)) as ceNodeLabels
WHERE all(x IN ['Employment', 'Location']
WHERE x IN ceNodeLabels) WITH childD, dg, rdgd return count(childD)
Now it works several times faster, but still not perfect. Is there something I may do in order to improve this?
UPDATED1
WITH [] as ceNodeList
MATCH (root2:Location )
WHERE root2.id IN ([1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 64, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, 79, 80, 81, 82, 83, 84, 85, 86, 87, 88, 89, 90, 91, 92, 93, 94, 95, 96, 97, 98, 99, 100])
WITH ceNodeList, root2
OPTIONAL MATCH (root2)-[:CONTAINS*0..]->(descendant2:Location)
OPTIONAL MATCH (ascendant2:Location)-[:CONTAINS*0..]->(root2)
WITH ceNodeList, COLLECT(root2) AS listRoot2, COLLECT( DISTINCT ascendant2) AS listAscendant2, COLLECT( DISTINCT descendant2) AS listDescendant2
WITH apoc.coll.union(ceNodeList, apoc.coll.union(listDescendant2, apoc.coll.union(listRoot2, listAscendant2))) AS ceNodeList
WITH ceNodeList MATCH (root3:Employment )
WHERE root3.id IN ([101, 102, 103, 104, 105])
WITH ceNodeList, COLLECT(root3) AS listRoot3
WITH apoc.coll.union(ceNodeList, listRoot3) AS ceNodeList
WITH ceNodeList
UNWIND ceNodeList as seNode
WITH collect(seNode.id) as seNodeIds with apoc.coll.toSet(seNodeIds) as seNodeIds
MATCH (dg:DecisionGroup {id: -2})-[rdgd:CONTAINS]->(childD:Profile ) -[:REQUIRES]->(ceNode)
WHERE ceNode.id in seNodeIds
WITH DISTINCT childD, dg, rdgd, collect(ceNode) as ceNodes
WITH childD, dg, rdgd, ceNodes, reduce(ceNodeLabels = [], n IN ceNodes | ceNodeLabels + labels(n)) as ceNodeLabels
WHERE all(x IN ['Employment', 'Location']
WHERE x IN ceNodeLabels)
WITH childD, dg, rdgd

Answer 1

Try this:
WITH [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35] AS ids
WITH reduce(idsMap = {}, x IN ids | apoc.map.setEntry(idsMap, toString(x), true))
MATCH (dg:DecisionGroup {id: -2})-[rdgd:CONTAINS]->(childD:Decision:Profile )
MATCH (childD)-[:EMPLOYMENT_AS]->(root2:Employment )
WHERE root2.id = 1
WITH DISTINCT childD, dg, rdgd, idsMap
MATCH (descendant3:Location) WHERE apoc.map.get(idsMap, toString(descendant3.id), false) = true
MATCH path3=(root3:Location )-[:CONTAINS*0..]->(descendant3)
WHERE apoc.map.get(idsMap, toString(root3.id), false) = true
UNWIND nodes(path3) AS pathNode3 WITH childD, dg, rdgd, COLLECT(DISTINCT pathNode3) AS pathNodes3
MATCH (childD)-[:LOCATED_IN]->(pathNode3) WHERE pathNode3 IN pathNodes3 WITH DISTINCT childD, dg, rdgd WHERE (childD.`active` = true) AND (childD.`experienceMonths` >= 129) AND ( (childD.`minSalaryUsd` <= 8883) OR (childD.`minHourlyRateUsd` <= 126) )
MATCH (childD)-[criterionRelationship8:HAS_VOTE_ON]->(c:Criterion {id: 2}) WHERE (criterionRelationship8.`properties.experienceMonths` >= 1) WITH DISTINCT childD, dg, rdgd
MATCH (childD)-[criterionRelationship10:HAS_VOTE_ON]->(c:Criterion {id: 36}) WHERE (criterionRelationship10.`avgVotesWeight` >= 1.0) AND (criterionRelationship10.`properties.experienceMonths` >= 1) WITH DISTINCT childD, dg, rdgd
MATCH (childD)-[criterionRelationship13:HAS_VOTE_ON]->(c:Criterion {id: 4}) WHERE (criterionRelationship13.`properties.experienceMonths` >= 0) WITH DISTINCT childD, dg, rdgd
MATCH (childD)-[criterionRelationship15:HAS_VOTE_ON]->(c:Criterion {id: 22}) WHERE (criterionRelationship15.`avgVotesWeight` >= 1.0) AND (criterionRelationship15.`properties.experienceMonths` >= 1) WITH DISTINCT childD, dg, rdgd
OPTIONAL MATCH (childD)-[ru:CREATED_BY]->(u:User) WITH childD, u, ru, dg, rdgd
OPTIONAL MATCH (childD)-[vg:HAS_VOTE_ON]->(c:Criterion) WHERE c.id IN [2, 36, 4, 22] WITH c, childD, u, ru, dg, rdgd, (vg.avgVotesWeight * (CASE WHEN c IS NOT NULL THEN coalesce({`22`:1.2236918603185925, `2`:2.9245935245152226, `36`:0.2288013749943646, `4`:3.9599506966378435}[toString(c.id)], 1.0) ELSE 1.0 END)) as weight, vg.totalVotes as totalVotes
WITH childD, u, ru , dg, rdgd , toFloat(sum(weight)) as weight, toInteger(sum(totalVotes)) as totalVotes
ORDER BY weight DESC , childD.createdAt DESC
SKIP 0 LIMIT 20
WITH * OPTIONAL MATCH (childD)-[rup:UPDATED_BY]->(up:User)
RETURN rdgd, ru, u, rup, up, childD AS decision, weight, totalVotes, [ (c1)<-[vg1:HAS_VOTE_ON]-(childD) WHERE c1.id IN [2, 36, 4, 22] | {criterion: c1, relationship: vg1} ] AS weightedCriteria
Here, I have created a map from the ids given and then used it instead of IN operator.
Update:
I think your new query can be simplified a bit. We can combine apoc.coll.unionAll and apoc.coll.toSet, with a single call to apoc.coll.union, try this:
MATCH (root2:Employment)
WHERE root2.id IN ([1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16])
WITH COLLECT(root2) AS ceNodeList
MATCH (root3:Location)
WHERE root3.id IN ([1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 64, 65, 66, 67, 68, 69, 70, 71, 72, 73])
WITH ceNodeList, root3, COLLECT(root3) AS listRoot3
OPTIONAL MATCH (root3)-[:CONTAINS*0..]-(descendants:Location)
WITH ceNodeList, listRoot3, COLLECT(DISTINCT descendant3) AS listDescendant3
WITH apoc.coll.union(ceNodeList, apoc.coll.union(listDescendant3, listRoot3)) AS ceNodeList
UNWIND ceNodeList AS ceNode
WITH DISTINCT ceNode MATCH (dg:DecisionGroup {id: -2})-[rdgd:CONTAINS]->(childD:Decision:Profile)-[:REQUIRES]->(ceNode)
WITH DISTINCT childD, dg, rdgd, collect(ceNode) as ceNodes
WITH childD, dg, rdgd, ceNodes, reduce(ceNodeLabels = [], n IN ceNodes | ceNodeLabels + labels(n)) as ceNodeLabels
WHERE all(x IN ['Employment', 'Location']
WHERE x IN ceNodeLabels) WITH childD, dg, rdgd return count(childD)