Open dianaoa22 opened 2 months ago
Do you have indexes on related fields used to create edge relationship? I tried accessing linked file to look over but apparently couldn't get to the file.
Code.txt Yes, I have created GIN index as the MATCH query used id inside the properties. CREATE INDEX load_item_gin_idx ON ms_graph."Item" USING gin (properties); CREATE INDEX
{ "id": 844424930132046, "properties": { "id": "3bd6197582434a01b340b889b357f3a4", "label": "xxxx", "properties": "{path: "/c/e", name: "test", s-id: "21977", description: "test", seattr: "2000", state: "enable", customer: "13"}" } }
Here is index structure I found helpful:
CREATE INDEX [CONCURRENTLY] <index_name> ON "<schema/graph_name>"."<vertice_table_name>"
(ag_catalog.agtype_access_operator(properties, '"<property_name>"'::agtype));
Please, replace
This would take some time, I will let you know once I have the data, thanks for the suggestions.
Np at all. One thing I was looking for is sample config.json file as it would help me to get to some test data faster.
import json
data = { "root": { "child1": { "name": "test", "location": "test" }, "child2": [ { "child2-class": "classname", "child2-number": 1 } ], "child3": [ { "child4": [ { "child4-pref": 1, "child4-type": "type-schild5" }, { "child4-type": "type-schild5", "child4-pref": 2 } ], "child5": [ { "child5-number": 1, "entryp": { "econtrol": { "ploc": "ploc-10001", "state": "enable" } }, "inside": { "mgmt": { "state": "enable" } } } ], "child3-type": "new-child53", "pref-number": 1 }, { "child3-type": "test-schild5-c", "pref-number": 2, "child4": [ { "child4-pref": 1 }, { "child4-pref": 2 } ], "child5": [ { "entryp": { "econtrol": { "ploc": "ploc-10001", "state": "enable" } }, "child5-number": 1, "inside": { "mgmt": { "state": "enable" } } } ] }, { "child3-type": "new-child53", "pref-number": 3, "child4": [ { "child4-type": "type-schild5", "child4-pref": 1 }, { "child4-type": "type-schild5", "child4-pref": 2 } ], "child5": [ { "inside": { "mgmt": { "state": "enable" } }, "child5-number": 1 } ] }, { "child4": [ { "child4-type": "type-schild5", "child4-pref": 1 }, { "child4-pref": 2, "child4-type": "type-schild5" } ], "child5": [ { "inside": { "mgmt": { "state": "enable" } }, "child5-number": 1 } ], "child3-type": "new-child53", "pref-number": 4 }, { "child3-type": "new-child53", "pref-number": 5, "child4": [ { "child4-pref": 1, "child4-type": "type-schild5" } ], "child5": [ { "inside": { "mgmt": { "state": "enable" }, "band": { "band-no": "NQP-10001" } }, "child5-number": 1 } ] }, { "child4": [ { "child4-type": "type-schild5", "child4-pref": 1 }, { "child4-type": "type-schild5", "child4-pref": 2 } ], "child5": [ { "inside": { "mgmt": { "state": "enable" } }, "child5-number": 1, "entryp": { "econtrol": { "state": "enable", "ploc": "ploc-10001" } } } ], "child3-type": "new-child53", "pref-number": 6 }, { "child3-type": "new-child53", "pref-number": 8, "child4": [ { "child4-type": "type-schild5", "child4-pref": 1 } ], "child5": [ { "inside": { "mgmt": { "state": "enable" }, "net": { "test": "FRL-10001" } }, "child5-number": 1 } ] } ] } }
def add_entries(data, num_entries): base_child3 = { "child3-type": "new-child53", "child4": [], "child5": [], "pref-number": None # Will be set dynamically }
base_child4 = {
"child4-pref": None, # Will be set dynamically
"child4-type": "type-schild5"
}
base_child5 = {
"child5-number": None, # Will be set dynamically
"entryp": {
"econtrol": {
"ploc": "ploc-10001",
"state": "enable"
}
},
"inside": {
"mgmt": {
"state": "enable"
}
}
}
# Generate new child3 entries
new_child3s = []
for i in range(1, num_entries + 1):
child3 = base_child3.copy()
child3['pref-number'] = i
# Generate child4 entries for the child3
child4_entries = []
for j in range(1, 5): # 4 child4 entries per child3 for example
child4 = base_child4.copy()
child4['child4-pref'] = j
child4_entries.append(child4)
child3['child4'] = child4_entries
# Generate child5 entries for the child3
child5_entries = []
for j in range(1, 5): # 4 child5 entries per child3 for example
child5 = base_child5.copy()
child5['child5-number'] = j
child5_entries.append(child5)
child3['child5'] = child5_entries
new_child3s.append(child3)
# Add new child3s to the data
data['root']['child3'].extend(new_child3s)
num_entries = 100000
add_entries(data, num_entries)
print(f"Total child3 entries: {len(data['root']['child3'])}")
with open('big_data.json', 'w') as file: json.dump(data, file, indent=4)
print("Data added successfully and saved to updated_data.json")
Run this python program to get a big data, I am unable to imprt the json due to size restriction of 25MB
Thank you
Main.txt Describe the bug I am creating 83K vertices and 83K edges by parsing a json file with JAVA. The vertex creation takes about 1 min however for edges it takes around 1hr+.
How are you accessing AGE (Command line, driver, etc.)?
What data setup do we need to do?