Upload 6 files

src/chains.py

@@ -0,0 +1,80 @@
+from langchain.document_loaders import PyPDFLoader, PyPDFDirectoryLoader
+from langchain.indexes import VectorstoreIndexCreator
+from langchain.llms import AzureOpenAI, OpenAI
+from langchain.text_splitter import CharacterTextSplitter
+from langchain.embeddings import OpenAIEmbeddings
+from langchain.vectorstores import Chroma
+from langchain.chains import RetrievalQA, ConversationalRetrievalChain, RetrievalQAWithSourcesChain
+from langchain.chains.question_answering import load_qa_chain
+from langchain.memory import ConversationBufferMemory
+
+from langchain.chat_models import AzureChatOpenAI
+
+
+import os
+import openai
+os.environ['CWD'] = os.getcwd()
+
+# for testing
+import src.constants as constants
+# import constants 
+os.environ['OPENAI_API_KEY'] = constants.AZURE_OPENAI_KEY_FR
+os.environ['OPENAI_API_BASE'] = constants.AZURE_OPENAI_ENDPOINT_FR
+os.environ['OPENAI_API_VERSION'] = "2023-05-15"
+os.environ['OPENAI_API_TYPE'] = "azure"
+# openai.api_type = "azure"
+# openai.api_base = constants.AZURE_OPENAI_ENDPOINT_FR
+# openai.api_version = "2023-05-15"
+openai.api_key = constants.OPEN_AI_KEY
+
+def get_document_key(doc):
+    return doc.metadata['source'] + '_page_' + str(doc.metadata['page'])
+
+
+import os
+from typing import Optional
+
+class PDFEmbeddings():
+    def __init__(self, path: Optional[str] = None):
+        self.path = path or os.path.join(os.environ['CWD'], 'archive')
+        self.text_splitter = CharacterTextSplitter(separator="\n", chunk_size=1000, chunk_overlap=200)
+        self.embeddings = OpenAIEmbeddings(deployment= constants.AZURE_ENGINE_NAME_US, chunk_size=1,
+                                           openai_api_key= constants.AZURE_OPENAI_KEY_US,
+                                           openai_api_base= constants.AZURE_OPENAI_ENDPOINT_US,
+                                           openai_api_version= "2023-05-15",
+                                           openai_api_type= "azure",)
+        self.vectorstore = Chroma(persist_directory=constants.persistent_dir, embedding_function=self.embeddings)
+        self.retriever = self.vectorstore.as_retriever(search_type = "similarity", search_kwags= {"k": 5})
+        self.memory = ConversationBufferMemory(memory_key='pdf_memory', return_messages=True)
+
+    def process_documents(self):
+        # Load the documents and process them
+        loader = PyPDFDirectoryLoader(self.path)
+        documents = loader.load()
+        chunks = self.text_splitter.split_documents(documents)
+        self.vectorstore.add_documents(chunks)
+
+    def search(self, query: str, chain_type: str = "stuff"):
+        chain = RetrievalQA.from_chain_type(llm= AzureChatOpenAI(deployment_name= constants.AZURE_ENGINE_NAME_FR, temperature=0),
+                                            retriever= self.retriever, chain_type= chain_type, return_source_documents= True)
+        result = chain({"query": query})
+        return result
+
+    def conversational_search(self, query: str, chain_type: str = "stuff"):
+        chain = ConversationalRetrievalChain.from_llm(llm= AzureChatOpenAI(deployment_name= constants.AZURE_ENGINE_NAME_FR),
+                                                      retriever= self.retriever, memory= self.memory, chain_type= chain_type)
+        result = chain({"question": query})
+        return result['answer']
+
+    def load_and_run_chain(self, query: str, chain_type: str = "stuff"):
+        chain = load_qa_chain(llm= AzureChatOpenAI(deployment_name= constants.AZURE_ENGINE_NAME_FR), chain_type= chain_type)
+        return chain.run(input_documents = self.retriever, question = query)
+
+if __name__ == '__main__':
+    pdf_embed = PDFEmbeddings()
+    # pdf_embed.process_documents() # This takes a while, so we only do it once
+    result = pdf_embed.search("Give me a list of short relevant queries to look for papers related to the topics of the papers in the source documents.")
+    print("\n\n", result['result'], "\n")
+    print("Source documents:")
+    for doc in result['source_documents']:
+        print(doc.metadata['source'])

src/constants.py

@@ -0,0 +1,167 @@
+AZURE_OPENAI_KEY_FR = "d296db50d1ad471caf944d1b148ae6c7"
+AZURE_OPENAI_ENDPOINT_FR = "https://openai-resource-team-11-france.openai.azure.com/"
+AZURE_ENGINE_NAME_FR = "gpt35-team-11"
+
+AZURE_OPENAI_KEY_US = "af7e76186d2c46458bcb6d1ecef91d68"
+AZURE_OPENAI_ENDPOINT_US = "https://openai-resource-team-11-east-us.openai.azure.com/"
+AZURE_ENGINE_NAME_US = "ada002-team-11"
+
+OPEN_AI_KEY = "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"
+
+persistent_dir = "db"
+
+# Taxonomy dictionary arxiv
+arxiv_taxonomy = {
+    "astro-ph.CO": "Cosmology and Nongalactic Astrophysics",
+    "astro-ph.EP": "Earth and Planetary Astrophysics",
+    "astro-ph.GA": "Astrophysics of Galaxies",
+    "astro-ph.HE": "High Energy Astrophysical Phenomena",
+    "astro-ph.IM": "Instrumentation and Methods for Astrophysics",
+    "astro-ph.SR": "Solar and Stellar Astrophysics",
+    "cond-mat.dis-nn": "Disordered Systems and Neural Networks",
+    "cond-mat.mes-hall": "Mesoscale and Nanoscale Physics",
+    "cond-mat.mtrl-sci": "Materials Science",
+    "cond-mat.other": "Other Condensed Matter",
+    "cond-mat.quant-gas": "Quantum Gases",
+    "cond-mat.soft": "Soft Condensed Matter",
+    "cond-mat.stat-mech": "Statistical Mechanics",
+    "cond-mat.str-el": "Strongly Correlated Electrons",
+    "cond-mat.supr-con": "Superconductivity",
+    "cs.AI": "Artificial Intelligence",
+    "cs.AR": "Hardware Architecture",
+    "cs.CC": "Computational Complexity",
+    "cs.CE": "Computational Engineering, Finance, and Science",
+    "cs.CG": "Computational Geometry",
+    "cs.CL": "Computation and Language",
+    "cs.CR": "Cryptography and Security",
+    "cs.CV": "Computer Vision and Pattern Recognition",
+    "cs.CY": "Computers and Society",
+    "cs.DB": "Databases",
+    "cs.DC": "Distributed, Parallel, and Cluster Computing",
+    "cs.DL": "Digital Libraries",
+    "cs.DM": "Discrete Mathematics",
+    "cs.DS": "Data Structures and Algorithms",
+    "cs.ET": "Emerging Technologies",
+    "cs.FL": "Formal Languages and Automata Theory",
+    "cs.GL": "General Literature",
+    "cs.GR": "Graphics",
+    "cs.GT": "Computer Science and Game Theory",
+    "cs.HC": "Human-Computer Interaction",
+    "cs.IR": "Information Retrieval",
+    "cs.IT": "Information Theory",
+    "cs.LG": "Machine Learning",
+    "cs.LO": "Logic in Computer Science",
+    "cs.MA": "Multiagent Systems",
+    "cs.MM": "Multimedia",
+    "cs.MS": "Mathematical Software",
+    "cs.NA": "Numerical Analysis",
+    "cs.NE": "Neural and Evolutionary Computing",
+    "cs.NI": "Networking and Internet Architecture",
+    "cs.OH": "Other Computer Science",
+    "cs.OS": "Operating Systems",
+    "cs.PF": "Performance",
+    "cs.PL": "Programming Languages",
+    "cs.RO": "Robotics",
+    "cs.SC": "Symbolic Computation",
+    "cs.SD": "Sound",
+    "cs.SE": "Software Engineering",
+    "cs.SI": "Social and Information Networks",
+    "cs.SY": "Systems and Control",
+    "econ.EM": "Econometrics",
+    "eess.AS": "Audio and Speech Processing",
+    "eess.IV": "Image and Video Processing",
+    "eess.SP": "Signal Processing",
+    "gr-qc": "General Relativity and Quantum Cosmology",
+    "hep-ex": "High Energy Physics - Experiment",
+    "hep-lat": "High Energy Physics - Lattice",
+    "hep-ph": "High Energy Physics - Phenomenology",
+    "hep-th": "High Energy Physics - Theory",
+    "math.AC": "Commutative Algebra",
+    "math.AG": "Algebraic Geometry",
+    "math.AP": "Analysis of PDEs",
+    "math.AT": "Algebraic Topology",
+    "math.CA": "Classical Analysis and ODEs",
+    "math.CO": "Combinatorics",
+    "math.CT": "Category Theory",
+    "math.CV": "Complex Variables",
+    "math.DG": "Differential Geometry",
+    "math.DS": "Dynamical Systems",
+    "math.FA": "Functional Analysis",
+    "math.GM": "General Mathematics",
+    "math.GN": "General Topology",
+    "math.GR": "Group Theory",
+    "math.GT": "Geometric Topology",
+    "math.HO": "History and Overview",
+    "math.IT": "Information Theory",
+    "math.KT": "K-Theory and Homology",
+    "math.LO": "Logic",
+    "math.MG": "Metric Geometry",
+    "math.MP": "Mathematical Physics",
+    "math.NA": "Numerical Analysis",
+    "math.NT": "Number Theory",
+    "math.OA": "Operator Algebras",
+    "math.OC": "Optimization and Control",
+    "math.PR": "Probability",
+    "math.QA": "Quantum Algebra",
+    "math.RA": "Rings and Algebras",
+    "math.RT": "Representation Theory",
+    "math.SG": "Symplectic Geometry",
+    "math.SP": "Spectral Theory",
+    "math.ST": "Statistics Theory",
+    "math-ph": "Mathematical Physics",
+    "nlin.AO": "Adaptation and Self-Organizing Systems",
+    "nlin.CD": "Chaotic Dynamics",
+    "nlin.CG": "Cellular Automata and Lattice Gases",
+    "nlin.PS": "Pattern Formation and Solitons",
+    "nlin.SI": "Exactly Solvable and Integrable Systems",
+    "nucl-ex": "Nuclear Experiment",
+    "nucl-th": "Nuclear Theory",
+    "physics.acc-ph": "Accelerator Physics",
+    "physics.ao-ph": "Atmospheric and Oceanic Physics",
+    "physics.app-ph": "Applied Physics",
+    "physics.atom-ph": "Atomic Physics",
+    "physics.atm-clus": "Atomic and Molecular Clusters",
+    "physics.bio-ph": "Biological Physics",
+    "physics.chem-ph": "Chemical Physics",
+    "physics.class-ph": "Classical Physics",
+    "physics.comp-ph": "Computational Physics",
+    "physics.data-an": "Data Analysis, Statistics and Probability",
+    "physics.ed-ph": "Physics Education",
+    "physics.flu-dyn": "Fluid Dynamics",
+    "physics.gen-ph": "General Physics",
+    "physics.geo-ph": "Geophysics",
+    "physics.hist-ph": "History and Philosophy of Physics",
+    "physics.ins-det": "Instrumentation and Detectors",
+    "physics.med-ph": "Medical Physics",
+    "physics.optics": "Optics",
+    "physics.plasm-ph": "Plasma Physics",
+    "physics.pop-ph": "Popular Physics",
+    "physics.soc-ph": "Physics and Society",
+    "physics.space-ph": "Space Physics",
+    "q-bio.BM": "Biomolecules",
+    "q-bio.CB": "Cell Behavior",
+    "q-bio.GN": "Genomics",
+    "q-bio.MN": "Molecular Networks",
+    "q-bio.NC": "Neurons and Cognition",
+    "q-bio.OT": "Other Quantitative Biology",
+    "q-bio.PE": "Populations and Evolution",
+    "q-bio.QM": "Quantitative Methods",
+    "q-bio.SC": "Subcellular Processes",
+    "q-bio.TO": "Tissues and Organs",
+    "q-fin.CP": "Computational Finance",
+    "q-fin.EC": "Economics",
+    "q-fin.GN": "General Finance",
+    "q-fin.MF": "Mathematical Finance",
+    "q-fin.PM": "Portfolio Management",
+    "q-fin.PR": "Pricing of Securities",
+    "q-fin.RM": "Risk Management",
+    "q-fin.ST": "Statistical Finance",
+    "q-fin.TR": "Trading and Market Microstructure",
+    "quant-ph": "Quantum Physics",
+    "stat.AP": "Applications",
+    "stat.CO": "Computation",
+    "stat.ME": "Methodology",
+    "stat.ML": "Machine Learning",
+    "stat.OT": "Other Statistics",
+    "stat.TH": "Statistics Theory"
+}

src/search.py

@@ -0,0 +1,146 @@
+import arxiv
+from crossref.restful import Works
+import pytz
+from datetime import date
+from datetime import datetime
+
+
+class Search_Papers():
+    def __init__(self, query, search_by, search_by_query, sort_by, sort_order):
+
+        self.query = query
+        self.search_mode = search_by
+        self.search_mode_query = search_by_query
+
+        self.sort_by = sort_by
+
+        self.sort_order = sort_order
+
+        self.time_search = datetime.now(pytz.utc).replace(microsecond=0)
+
+        return
+
+    
+    def search_arxiv_NResults(self, query, max_results, sort_by, sort_order):
+        search_results = arxiv.Search(
+        query = query,
+        max_results = max_results,
+        sort_by = sort_by,
+        sort_order = sort_order
+        )
+
+        return search_results.results()
+
+
+
+    def search_arxiv_Timeframe(self, query, timeframe, sort_by, sort_order):
+
+        collection = []
+        exit_flag = 0
+        n_iter = 0
+        max_results = 10
+
+        while True:
+            search_list = arxiv.Search(
+                query = query,
+                max_results = (n_iter+1)*max_results,
+                sort_by = sort_by,
+                sort_order = sort_order
+                )
+
+
+            results = list(search_list.results())
+
+            for i in range(n_iter*max_results, len(results)):
+                #print(timeframe, results[i].published, results[i].published < timeframe)
+
+                if results[i].published > timeframe:
+                    collection.append(results[i])
+                else:
+                    exit_flag = 1
+                    break
+          
+            if exit_flag:
+                break
+            else:
+                n_iter += 1
+          
+
+        return collection
+
+    
+    
+    def search_arxiv(self):
+
+        if self.sort_by == "PublishDate":
+            sort_by = arxiv.SortCriterion.SubmittedDate
+        elif self.sort_by == "LastUpdatedDate":
+            sort_by = arxiv.SortCriterion.LastUpdatedDate
+        else:
+            sort_by = arxiv.SortCriterion.Relevance
+
+
+        if self.sort_order == "Ascending":
+            sort_order = arxiv.SortOrder.Ascending
+        else:
+            sort_order = arxiv.SortOrder.Descending
+
+
+
+        if self.search_mode == "NumberResults":
+            search_results = self.search_arxiv_NResults(self.query, self.search_mode_query, sort_by, sort_order)
+        else:
+            search_results = self.search_arxiv_Timeframe(self.query, self.search_mode_query, sort_by, sort_order)
+
+        return search_results
+    
+    
+    def search_general_NResults(self, query, max_results, sort_by, sort_order):
+        works = Works()
+        # search_results = works.filter(category_name = query).sort(sort_by).order(sort_order).sample(max_results)
+        search_results = works.query(bibliographic = query).sort(sort_by).order(sort_order).sample(max_results)
+        return search_results
+
+
+    def search_general_Timeframe(self, query, timeframe, sort_by, sort_order):
+        works = Works()
+        today = (datetime.combine(date.today(), datetime.min.time())).strftime("%Y-%m-%d")
+      
+        # search_results = works.filter(category_name = query, from_created_date=timeframe, until_created_date = today).sort(sort_by).order(sort_order)
+        search_results = works.query(bibliographic = query).filter(from_created_date=timeframe, until_created_date = today).sort(sort_by).order(sort_order)
+         
+        return search_results
+
+    #'Magnetic Field Conditions Upstream of Ganymede'
+
+    def search_general(self):
+
+        if self.search_mode == 'Timeframe':
+            timeframe = self.search_mode_query.strftime("%Y-%m-%d")
+            
+
+
+
+        if self.sort_by == "PublishDate":
+            sort_by = 'created'
+        elif self.sort_by == "LastUpdatedDate":
+            sort_by = 'updated'
+        else:
+            sort_by = 'relevance'
+
+
+        if self.sort_order == "Ascending":
+            sort_order = "asc"
+        else:
+            sort_order = "desc"
+
+
+
+        if self.search_mode == "NumberResults":
+            search_results = self.search_general_NResults(self.query, self.search_mode_query, sort_by, sort_order)
+        else:
+            search_results = self.search_general_Timeframe(self.query, timeframe, sort_by, sort_order)
+
+        return search_results
+    
+