|
|
| (8 intermediate revisions by the same user not shown) |
| Line 1: |
Line 1: |
| import re, json, os, sys
| | Kotoba |
| import instructor
| |
| import openai
| |
| import requests
| |
| from graphviz import Digraph
| |
| from langchain_community.graphs import Neo4jGraph
| |
| from neo4j import GraphDatabase
| |
| | |
| import matplotlib.pyplot as plt
| |
| from sklearn.datasets import make_blobs
| |
| from sklearn.cluster import KMeans
| |
| from sklearn.metrics import silhouette_score
| |
| from sklearn.preprocessing import StandardScaler
| |
| from sklearn import metrics
| |
| from collections import defaultdict
| |
| import kotoba.knowledge_structure as k_s
| |
| import kotoba.chatbot_utils as c_t
| |
| import importlib
| |
| import networkx as nx
| |
| # import nxneo4j as nx
| |
| from graphdatascience import GraphDataScience
| |
| from langchain.chains import GraphCypherQAChain
| |
| from langchain_openai import ChatOpenAI
| |
| | |
| llm = c_t.get_llm()
| |
| chain = GraphCypherQAChain.from_llm(graph=graph, llm=llm, verbose=True)
| |
| response = chain.invoke({"query": "What was the cast of the Casino?"})
| |
| | |
| fUrl = "https://www.olympus-ims.com/en/rvi-products/iplex-nx/#!cms[focus]=cmsContent13653"
| |
| driver = GraphDatabase.driver("bolt://localhost:7687", auth=("neo4j", os.environ['NEO4J_PASS']))
| |
| graph = Neo4jGraph("bolt://localhost:7687", "neo4j", os.environ['NEO4J_PASS'])
| |
| gds = GraphDataScience("bolt://localhost:7687", auth=("neo4j", os.environ['NEO4J_PASS']))
| |
| | |
| def neo4j_node(driver,G):
| |
| nodeL = G.nodes
| |
| nodeType = "Section {name: STRING, id: STRING}"
| |
| queryS = "CREATE IF NOT EXISTS\n"
| |
| for n in nodeL:
| |
| g = G.nodes[n]
| |
| sectS = ""
| |
| for i in ['Chapter','Section','Subsection']:
| |
| try:
| |
| sectS += "%s: %s | " % (i,g[i])
| |
| except:
| |
| pass
| |
|
| |
| s = '(sum_' + str(n) + ': Section {name :"' + sectS + '"}),' + "\n"
| |
| queryS += s
| |
| queryS = queryS[:-2]
| |
| driver.execute_query(queryS)
| |
| gds.run_cypher(queryS)
| |
| | |
| def neo4j_edge(driver,G):
| |
| #n = G.edges[(k,h)]
| |
| edgeL = G.edges
| |
| for e in edgeL:
| |
| edge = edgeL[e]
| |
| #'MATCH ('+str(k)+':Instruction {name: 'Charlie Sheen'}), (oliver:Person {name: 'Oliver Stone'})'
| |
| driver.execute_query('('+str(k)+')-[r:CONTAINS '+str(n)+']->('+str(h)')')
| |
|
| |
| def neo4j_graph(driver,collN):
| |
| driver.execute_query("CREATE OR REPLACE DATABASE " + collN )
| |
| | |
| | |
|
| |
| | |
| gds.run_cypher("""
| |
| CREATE
| |
| (m: City {name: "Malmö"}),
| |
| (l: City {name: "London"}),
| |
| (s: City {name: "San Mateo"}),
| |
| (m)-[:FLY_TO]->(l),
| |
| (l)-[:FLY_TO]->(m),
| |
| (l)-[:FLY_TO]->(s),
| |
| (s)-[:FLY_TO]->(l)
| |
| """)
| |
| res = gds.graph.project.estimate(["City"],"FLY_TO",readConcurrency=4)
| |
| G, result = gds.graph.project("offices",["City"],"FLY_TO",readConcurrency=4)
| |
| G = gds.graph.get("offices")
| |
| G.drop()
| |
| query = """MATCH (n)-->(m)
| |
| RETURN gds.graph.project($graph_name, n, m, {sourceNodeLabels: $label,targetNodeLabels: $label,relationshipType: $rel_type})"""
| |
| G, result = gds.graph.cypher.project(query,database="neo4j",graph_name="offices",label="City",rel_type="FLY_TO")
| |
| n = G.node_count()
| |
| props = G.node_properties("City")
| |
| result = gds.degree.mutate(G, mutateProperty="degree")
| |
| | |
| nodeL = pd.read_csv(os.environ['HOME'] + '/lav/soft/raw/got-s1-nodes.csv')
| |
| edgeL = pd.read_csv(os.environ['HOME'] + '/lav/soft/raw/got-s1-edges.csv')
| |
| | |
| | |
| def pd2ndeo(nodeL=None,linkL=None):
| |
| if nodeL == None:
| |
| nodeL = pd.DataFrame({"nodeId": [0, 1, 2, 3],"labels": ["A", "B", "C", "A"],"prop1": [42, 1337, 8, 0],"otherProperty": [0.1, 0.2, 0.3, 0.4]})
| |
| if linkL == None:
| |
| linkL = pd.DataFrame({"sourceNodeId": [0, 1, 2, 3],"targetNodeId": [1, 2, 3, 0],"relationshipType": ["REL", "REL", "REL", "REL"],"weight": [0.0, 0.0, 0.1, 42.0]})
| |
| G = gds.graph.construct("grid",nodeL,linkL)
| |
| return G
| |
| | |
| | |
| def netx2neo(nx_G = None):
| |
| if nx_G == None:
| |
| nx_G = nx.DiGraph()
| |
| nx_G.add_node(1, labels=["Person"], age=52)
| |
| nx_G.add_node(42, labels=["Product", "Item"], cost=17.2)
| |
| nx_G.add_edge(1, 42, relationshipType="BUYS", quantity=4)
| |
| G = gds.graph.networkx.load(nx_G, "purchases")
| |
| return G
| |
| | |
| | |
| importlib.reload(c_t)
| |
| def build_document_graph(summL,collN,baseDir):
| |
| from collections import defaultdict
| |
| def tree(): return defaultdict(tree)
| |
| sL = ['Chapter', 'Section', 'Subsection','id']
| |
| treeD = tree()
| |
| for i in summL:
| |
| d = dict(i.metadata)
| |
| for s in sL:
| |
| if s not in d:
| |
| d[s] = ''
| |
| m = {"page_content":i.page_content,"metadata":i.metadata}
| |
| treeD[d[sL[0]]][d[sL[1]]][d[sL[2]]] = m
| |
|
| |
| G = nx.DiGraph(name="document_graph")
| |
| # G = nx.DiGraph(driver)
| |
| G.add_node(0,type="document")
| |
| for k1 in treeD.keys():
| |
| G.add_node(k1,type=sL[0])
| |
| for k2 in treeD[k1].keys():
| |
| G.add_node(k2,type=sL[1])
| |
| G.add_edge(k1,k2)
| |
| for k3 in treeD[k1][k2].keys():
| |
| G.add_node(k3,type=sL[2],text=treeD[k1][k2][k3]['page_content'])
| |
| G.add_edge(k2,k3)
| |
|
| |
| if False:
| |
| #nx.draw_kamada_kawai(G,with_labels = True)
| |
| nx.draw_spring(G,with_labels = True)
| |
| plt.show()
| |
| nx.write_graphml(G,baseDir + collN + ".graphml")
| |
| nx.pagerank(G)
| |
| nx.betweenness_centrality(G)
| |
| nx.closeness_centrality(G)
| |
| | |
| some_dict = {'a': 1, 'b': 2}
| |
| session = driver.session()
| |
| session.run(query="CREATE (x) SET x = {dict_param}",parameters={'dict_param': some_dict})
| |
| | |
| def build_knowledge_graph(summL,collN,baseDir):
| |
| embdL = c_t.embed_text(summL)
| |
| kmeans = KMeans(init="random",n_clusters=15,n_init=10,max_iter=300,random_state=42)
| |
| kmeans.fit(embdL)
| |
| clustL = kmeans.labels_
| |
| treeD = defaultdict(list)
| |
| for i,j in enumerate(clustL):
| |
| treeD[j].append(summL[i])
| |
| | |
| print([len(treeD[x]) for x in treeD.keys()])
| |
| treeL = []
| |
| G = nx.DiGraph(name="knowledge_graph")
| |
| # G = nx.DiGraph(driver)
| |
| G.add_node("0",name="document",id="0",Chapter=collN)
| |
| for k in treeD.keys():
| |
| treeL.append("\n".join([x.page_content for x in treeD[k]]))
| |
| G.add_node(k,**x.metadata)
| |
| G.add_edge('0',k)
| |
| for x in treeD[k]:
| |
| i = x.metadata['id']
| |
| G.add_node(i,**x.metadata)
| |
| G.add_edge(k,i)
| |
| | |
| if False:
| |
| nx.draw_kamada_kawai(G,with_labels = True)
| |
| plt.show()
| |
| nx.write_graphml(G,baseDir + collN + ".graphml")
| |
| nx.pagerank(G)
| |
| nx.betweenness_centrality(G)
| |
| nx.closeness_centrality(G)
| |
| | |
| | |
| if False: #categorical metrics
| |
| scores = defaultdict(list)
| |
| scores["Homogeneity"].append(metrics.homogeneity_score(labels, kmeans.labels_))
| |
| scores["Completeness"].append(metrics.completeness_score(labels, kmeans.labels_))
| |
| scores["V-measure"].append(metrics.v_measure_score(labels, kmeans.labels_))
| |
| scores["Adjusted Rand-Index"].append(metrics.adjusted_rand_score(labels, km.labels_))
| |
| scores["Silhouette Coefficient"].append(metrics.silhouette_score(X, km.labels_, sample_size=2000))
| |
| | |
| | |
| | |
| | |
| # Import movie information
| |
| | |
| movies_query = """
| |
| LOAD CSV WITH HEADERS FROM
| |
| 'https://raw.githubusercontent.com/tomasonjo/blog-datasets/main/movies/movies_small.csv'
| |
| AS row
| |
| MERGE (m:Movie {id:row.movieId})
| |
| SET m.released = date(row.released),
| |
| m.title = row.title,
| |
| m.imdbRating = toFloat(row.imdbRating)
| |
| FOREACH (director in split(row.director, '|') |
| |
| MERGE (p:Person {name:trim(director)})
| |
| MERGE (p)-[:DIRECTED]->(m))
| |
| FOREACH (actor in split(row.actors, '|') |
| |
| MERGE (p:Person {name:trim(actor)})
| |
| MERGE (p)-[:ACTED_IN]->(m))
| |
| FOREACH (genre in split(row.genres, '|') |
| |
| MERGE (g:Genre {name:trim(genre)})
| |
| MERGE (m)-[:IN_GENRE]->(g))
| |
| """
| |
| | |
| graph.query(movies_query)
| |
| | |
| | |
| with open(baseDir + fName + '.html') as fByte:
| |
| fString = fByte.read()
| |
| response = requests.get(fUrl)
| |
| soup = BeautifulSoup(response.text, "html.parser")
| |
| paragraphs = soup.find_all("p")
| |
| text = " ".join([p.get_text() for p in paragraphs])
| |
| | |
| user_input = "spark"
| |
| openai.api_key = os.environ['OPENAI_API_KEY']
| |
| prompt = f"Help me understand following by describing as a detailed knowledge graph: {user_input}"
| |
| completion: KnowledgeGraph = openai.ChatCompletion.create(model="gpt-3.5-turbo-16k",messages=[{"role": "user","content": prompt,}],response_model=KnowledgeGraph,)
| |
| response_data = completion.model_dump()
| |
| edges = response_data["edges"]
| |
| def _restore(e):
| |
| e["from"] = e["from_"]
| |
| return e
| |
| | |
| response_data["edges"] = [_restore(e) for e in edges]
| |
| results = driver.get_response_data(response_data)
| |
| | |
| dot = Digraph(comment="Knowledge Graph")
| |
| response_dict = response_data
| |
| for node in response_dict.get("nodes", []):
| |
| dot.node(node["id"], f"{node['label']} ({node['type']})")
| |
| | |
| for edge in response_dict.get("edges", []):
| |
| dot.edge(edge["from"], edge["to"], label=edge["relationship"])
| |
| | |
| dot.render("knowledge_graph.gv", view=False)
| |
| dot.format = "png"
| |
| dot.render("static/knowledge_graph", view=False)
| |
| png_url = f"{request.url_root}static/knowledge_graph.png"
| |
| | |
| (nodes, edges) = driver.get_graph_data()
| |
| response_dict = response_data
| |
| nodes = [
| |
| {
| |
| "data": {
| |
| "id": node["id"],
| |
| "label": node["label"],
| |
| "color": node.get("color", "defaultColor"),
| |
| }
| |
| }
| |
| for node in response_dict["nodes"]
| |
| ]
| |
| edges = [
| |
| {
| |
| "data": {
| |
| "source": edge["from"],
| |
| "target": edge["to"],
| |
| "label": edge["relationship"],
| |
| "color": edge.get("color", "defaultColor"),
| |
| }
| |
| }
| |
| for edge in response_dict["edges"]
| |
| ]
| |
| graphD = jsonify({"elements": {"nodes": nodes, "edges": edges}})
| |