Upload 3 files

App.py

@@ -0,0 +1,448 @@
+from langchain.llms import LlamaCpp
+from langchain.vectorstores import FAISS
+from langchain.prompts import PromptTemplate
+from langchain.docstore.document import Document
+from langchain.memory import ConversationBufferMemory
+from langchain.embeddings import HuggingFaceEmbeddings
+from langchain.chains import ConversationalRetrievalChain
+from langchain.chains.summarize import load_summarize_chain
+from langchain.document_loaders import YoutubeLoader, TextLoader
+from langchain.llms.huggingface_pipeline import HuggingFacePipeline
+from langchain.text_splitter import CharacterTextSplitter, RecursiveCharacterTextSplitter
+import os
+import sys
+import torch
+import tempfile
+import transformers
+from PIL import Image
+from datetime import datetime
+from torch import cuda, bfloat16
+from transformers import pipeline
+import streamlit as st
+from streamlit_chat import message
+from streamlit_js_eval import streamlit_js_eval
+
+def save_feedback(feedback):
+    feedback_dir = "feedback"
+    if not os.path.exists(feedback_dir):
+        os.makedirs(feedback_dir)
+    timestamp = datetime.now().strftime("%Y-%m-%d_%H-%M-%S")
+    filename = f"feedback_{timestamp}.txt"
+    filepath = os.path.join(feedback_dir, filename)
+    with open(filepath, "w") as file:
+        file.write(feedback)
+
+def transcribe(tmp_audio_path):
+
+    device = "cuda:1" if torch.cuda.is_available() else "cpu"
+
+    pipe = pipeline(
+    "automatic-speech-recognition",
+    model="openai/whisper-small.en",
+    chunk_length_s=30,
+    device=device,
+    )
+
+    audio = tmp_audio_path
+    prediction = pipe(audio, batch_size=8)["text"]
+
+    return prediction.strip()
+
+def summarize(transcript, option):
+    # Laod model
+    llm = LlamaCpp(
+        streaming = True,
+        model_path="mistral-7b-instruct-v0.2.Q4_K_M.gguf",
+        #model_path="mistral-7b-openorca.Q4_K_M.gguf",
+        #model_path="starling-lm-7b-alpha.Q4_K_M.gguf",
+        #model_path="zephyr-7b-beta.Q4_K_M.gguf",
+        #model_path="dolphin-2.1-mistral-7b.Q4_K_M.gguf",
+        #model_path="llama-2-7b.Q4_K_M.gguf",
+        temperature=0,
+        top_p=1, 
+        n_ctx=4096,
+        max_tokens=-1,#output word count limitation
+        verbose=True,
+        #n_gpu_layers=35,
+        #n_gpu_layers=-1,#all move to gpu
+        #n_batch=256,
+        context_length = 6000
+        )
+
+    ### Text preprocessing
+    target_len = 600
+    chunk_size = 3000
+    chunk_overlap = 200
+    text_splitter = CharacterTextSplitter(
+            chunk_size=chunk_size,
+            chunk_overlap=chunk_overlap,
+            length_function=len,
+        )
+    texts = text_splitter.split_text(transcript,)
+    docs = [Document(page_content=t) for t in texts[:]]
+    
+
+
+
+    general_prompt_template = """
+                  Do not explain what you are doing. Do not self reference. You are a professional summary writer. 
+                  Write a concise summary of the text that cover the key points of the text. and present the results as follows: 
+                    - Serveral paragraphs with the following content: Topic, Outline, Description
+                    - A key point list in the format of one key point in one paragraph
+                    - A markdown list with the definition of the important key terms mentioned
+                    ```{text}```
+                    SUMMARY:
+
+               """
+
+
+    lecture_prompt_template = """
+                 Do not explain what you are doing. Do not self reference. You are tasked with summarizing a lecture. Write a concise summary covering the lecture's key points and organize the results as follows:
+                    - Lecture Topic: Provide a brief overview of the main subject discussed in the lecture.
+                    - Lecture Description: Summarize the content and purpose of the lecture in a few sentences.
+                    - Outline: Present an outline of the lecture's structure, including main sections and subtopics.
+                    - Key Points: List the most important points discussed in the lecture, each presented in a separate paragraph.
+                    - Formulas and Equations: Include any significant formulas or equations introduced in the lecture.
+                    - Markdown Table: Create a markdown table to define and explain important terms and concepts mentioned in the lecture.
+                    ```{text}```
+                    SUMMARY:
+                    
+                      """
+
+    tutorial_prompt_template = """
+                  Do not explain what you are doing. Do not self reference. You have been assigned to summarize a tutorial video. Your task is to provide a concise summary covering the tutorial's main points and organize the results as follows:
+                    - Tutorial Topic: Briefly introduce the main subject matter covered in the tutorial.
+                    - Tutorial Description: Summarize the purpose and objectives of the tutorial in a few sentences.
+                    - Tutorial Structure: Outline the tutorial's structure, including main sections, steps, or modules.
+                    - Key Points: List the essential concepts or techniques explained in the tutorial, with each concept presented in its paragraph.
+                    - Practical Examples: Include any practical examples or demonstrations provided in the tutorial.
+                    - Tips and Tricks: Highlight any useful tips or tricks shared by the tutorial presenter.
+                    - Markdown Table: Create a markdown table to define and explain important terms and concepts introduced in the tutorial.
+                    ```{text}```
+                    SUMMARY:
+                      """
+
+    speech_prompt_template = """
+                  Do not explain what you are doing. Do not self reference. Your task is to summarize a speech. Write a concise summary covering the key points of the speech and organize the results as follows:
+                    - Speech Topic: Provide a brief introduction to the main subject matter addressed in the speech.
+                    - Speaker Introduction: Briefly introduce the speaker, including their background and credentials.
+                    - Speech Overview: Summarize the main themes or objectives of the speech in a few sentences.
+                    - Key Messages: List the key messages or arguments conveyed in the speech, with each message presented in its paragraph.
+                    - Examples and Illustrations: Include any relevant examples or illustrations provided by the speaker to support their points.
+                    - Closing Remarks: Summarize any concluding remarks or calls to action made by the speaker.
+                    - Markdown Table: Create a markdown table to define and explain important terms or concepts mentioned in the speech.
+                    ```{text}```
+                    SUMMARY:
+                      """
+
+    documentary_prompt_template = """
+                  Do not explain what you are doing. Do not self reference. Your task is to summarize a documentary. Write a concise summary covering the main points of the documentary and organize the results as follows:
+                    - Documentary Title: Provide the title of the documentary.
+                    - Documentary Overview: Briefly introduce the subject matter and purpose of the documentary in a few sentences.
+                    - Director's Background: Provide background information about the director or creators of the documentary.
+                    - Key Themes: List the key themes or topics explored in the documentary, with each theme presented in its paragraph.
+                    - Interviews and Testimonials: Highlight any interviews or testimonials featured in the documentary.
+                    - Footage and Visuals: Describe any significant footage or visual elements used to convey the documentary's message.
+                    - Conclusion: Summarize the main takeaways or conclusions drawn from the documentary.
+                    - Markdown Table: Create a markdown table to define and explain important terms or concepts mentioned in the documentary.
+                    ```{text}```
+                    SUMMARY:
+                      """
+    
+
+
+    if option == 'Default':
+        prompt_template = general_prompt_template
+    elif option == 'Lecture':
+        prompt_template = lecture_prompt_template
+    elif option == 'Speech':
+        prompt_template = speech_prompt_template
+    elif option == 'Tutorial':
+        prompt_template = tutorial_prompt_template
+    elif option == 'Documentary':
+        prompt_template = documentary_prompt_template
+
+
+
+
+    PROMPT = PromptTemplate(template=prompt_template, input_variables=["text"])
+    refine_template = (
+            "Your job is to produce a final summary\n"
+            "We have provided an existing summary up to a certain point: {existing_answer}\n"
+            "We have the opportunity to refine the existing summary"
+            "with some more context below.\n"
+            "------------\n"
+            "{text}\n"
+            "------------\n"
+            f"Given the new context, refine the original summary in English within {target_len} words and do not mention the summary is refined."
+        )
+    refine_prompt = PromptTemplate(
+            input_variables=["existing_answer", "text"],
+            template=refine_template,
+        )
+    chain = load_summarize_chain(
+            llm,
+            chain_type="refine",
+            return_intermediate_steps=True,
+            question_prompt=PROMPT,
+            refine_prompt=refine_prompt,
+        )
+    
+    
+    resp = chain(docs)
+    output_text = resp["output_text"]
+    
+    return output_text
+
+def initialize_session_state():
+    if 'history' not in st.session_state:
+        st.session_state['history'] = []
+
+    if 'generated' not in st.session_state:
+        st.session_state['generated'] = ["Hello! Ask me anything about your video!"]
+
+    if 'past' not in st.session_state:
+        st.session_state['past'] = ["Hey!"]
+        
+def conversation_chat(query, chain, history):
+    result = chain({"question": query, "chat_history": history})
+    history.append((query, result["answer"]))
+    return result["answer"]
+
+def display_chat_history(chain):
+    reply_container = st.container()
+    container = st.container()
+
+    with container:
+        with st.form(key='my_form', clear_on_submit=True):
+            user_input = st.text_input("Question:", placeholder="Ask about your video transcript", key='input')
+            submit_button = st.form_submit_button(label='Send')
+
+        if submit_button and user_input:
+            with st.spinner('Generating response...'):
+                output = conversation_chat(user_input, chain, st.session_state['history'])
+
+            st.session_state['past'].append(user_input)
+            st.session_state['generated'].append(output)
+
+    if st.session_state['generated']:
+        with reply_container:
+            for i in range(len(st.session_state['generated'])):
+                message(st.session_state["past"][i], is_user=True, key=str(i) + '_user', avatar_style="thumbs")
+                message(st.session_state["generated"][i], key=str(i), avatar_style="fun-emoji")
+
+def create_conversational_chain(vector_store, option):
+    chatbot_general_prompt = PromptTemplate(input_variables=["history", "context", "question"], template="""
+    You are a knowledgeable chatbot, here to help with questions of the user. Your tone should be professional and informative.
+    
+    Context: {context}
+    History: {history}
+
+    User: {question}
+    Chatbot:"" 
+    """)
+    chatbot_lecture_prompt = PromptTemplate(input_variables=["history", "context", "question"], template="""
+    You are a knowledgeable chatbot, you already have the knowledge of a lecture video transcript. Help with questions of the user 
+        with use of this lecture video transcript. Your tone should be professional and informative.
+    
+    Context: {context}
+    History: {history}
+
+    User: {question}
+    Chatbot:"" 
+    """)
+    chatbot_speech_prompt = PromptTemplate(input_variables=["history", "context", "question"], template="""
+    You are a knowledgeable chatbot, you already have the knowledge of a speech video transcript. Help with questions of the user 
+        with use of this speech video transcript. Your tone should be professional and informative.
+    
+    Context: {context}
+    History: {history}
+
+    User: {question}
+    Chatbot:"" 
+    """)
+    chatbot_tutorial_prompt = PromptTemplate(input_variables=["history", "context", "question"], template="""
+    You are a knowledgeable chatbot, you already have the knowledge of a tutorial video transcript. Help with questions of the user 
+        with use of this tutorial video transcript. Your tone should be professional and informative.
+    
+    Context: {context}
+    History: {history}
+
+    User: {question}
+    Chatbot:"" 
+    """)
+
+    chatbot_documentary_prompt = PromptTemplate(input_variables=["history", "context", "question"], template="""
+    You are a knowledgeable chatbot, you already have the knowledge of a documentary video transcript. Help with questions of the user 
+        with use of this documentary video transcript. Your tone should be professional and informative.
+    
+    Context: {context}
+    History: {history}
+
+    User: {question}
+    Chatbot:"" 
+    """)
+
+    if option == 'Default':
+        chatbot_prompt = chatbot_general_prompt
+    elif option == 'Lecture':
+        chatbot_prompt = chatbot_lecture_prompt
+    elif option == 'Speech':
+        chatbot_prompt = chatbot_speech_prompt
+    elif option == 'Tutorial':
+        chatbot_prompt = chatbot_tutorial_prompt
+    elif option == 'Documentary':
+        chatbot_prompt = chatbot_documentary_prompt
+
+    
+    # Create llm
+    llm = LlamaCpp(
+        streaming = True,
+        #model_path="mistral-7b-instruct-v0.1.Q4_K_M.gguf",
+        #model_path="mistral-7b-openorca.Q4_K_M.gguf",
+        #model_path="starling-lm-7b-alpha.Q4_K_M.gguf",
+        model_path="zephyr-7b-beta.Q4_K_M.gguf",
+        #model_path="dolphin-2.1-mistral-7b.Q4_K_M.gguf",
+        #model_path="llama-2-7b.Q4_K_M.gguf",
+        temperature=0,
+        top_p=1, 
+        n_ctx=4096,
+        max_tokens=-1,#output word count limitation
+        verbose=True,
+        #n_gpu_layers=35,
+        #n_gpu_layers=-1,#all move to gpu
+        #n_batch=256,
+        )
+    
+    memory = ConversationBufferMemory(memory_key="chat_history", return_messages=True)
+
+    chain = ConversationalRetrievalChain.from_llm(llm=llm, chain_type='stuff',
+                                                 retriever=vector_store.as_retriever(search_kwargs={"k": 2}),
+                                                 memory=memory, condense_question_prompt=chatbot_prompt)
+    return chain
+
+def main():
+    # Initialize session state
+    initialize_session_state()
+    im = Image.open('sricon.png')
+    st.set_page_config(page_title=' 🤖Automatic Video Assistant🔗', layout='wide', page_icon = im)
+
+    # Set up the Streamlit app layout
+    st.title("🤖 Automatic Video Assistant 🔗")
+    st.subheader(" Powered by LangChain + Streamlit")
+
+    hide_default_format = """
+        <style>
+        #MainMenu {visibility: hidden; }
+        footer {visibility: hidden;}
+        </style>
+        """
+    st.markdown(hide_default_format, unsafe_allow_html=True)
+
+    #sidebar
+    with st.sidebar:
+        st.markdown("# Introduction")
+        st.markdown(
+        "Automatic Video Assistant is able to summarize videos and answer related questions.")
+        st.markdown("You can select specific video types to enhance the assistant's performance.")
+        st.markdown("You can input local video or YouTube video link.")
+        st.markdown("# Input your video to start!")
+        st.markdown("---")
+        st.markdown("# Feedback")
+        txt = st.text_area(
+                "We will continue to improve💪?",
+                "Please share your feedback... ",
+            )
+        if st.button('Submit'):
+            save_feedback(txt)
+            st.write('Your feedback is submitted!')
+
+        
+    option = st.selectbox(
+    'Please indicate you video type for better interaction😀',
+    ('Default', 'Lecture', 'Speech', 'Tutorial', 'Documentary'))
+
+    st.write('Selected video type:', option)
+    
+
+    #User Input File 
+    audio_file = st.file_uploader("Upload Video", type=["mp4", "wav","mp3","mov","avi","wmv"])
+
+    with st.form('myform', clear_on_submit=True):
+        youtube_url = st.text_input("Or enter a YouTube URL")
+        submitted = st.form_submit_button('Submit')
+
+    # Check if either YouTube URL or file uploaded
+    if (submitted and youtube_url) or audio_file:
+        if youtube_url:
+            loader = YoutubeLoader.from_youtube_url(youtube_url, add_video_info=False)
+        else:
+            #transcript = transcribe(audio_file)
+            with tempfile.NamedTemporaryFile(delete=False) as tmp_audio_file:
+                tmp_audio_file.write(audio_file.read())
+                tmp_audio_path = tmp_audio_file.name
+            transcript = transcribe(tmp_audio_path)
+
+        with st.expander("See Transcript"):
+            if youtube_url:
+                transcript = loader.load()
+                # Save the transcript to a text file
+                with open("transcript.txt", "w", encoding="utf-8") as file:
+                    transcript_text = '\n'.join([document.page_content for document in transcript])
+                    file.write(transcript_text)
+                with open("transcript.txt", "r", encoding="utf-8") as file:
+                    transcript = file.read()
+            else:
+                with open("transcript.txt", "w") as f:
+                    f.write(transcript)
+            # Display the transcript
+            st.write(transcript)
+            # Provide a download button for the transcript
+            st.download_button("Download Transcript", transcript, key='transcript_download_button')
+
+        st.subheader("Do you want a summary for this video?")
+        if 'clicked' not in st.session_state:
+            st.session_state.clicked = False
+        def click_button():
+            st.session_state.clicked = True
+        st.button('Generate summary', on_click=click_button)
+        #Summarize
+        if st.session_state.clicked:
+                with st.expander("See Summary", expanded=True):
+                        st.header("Summary")
+                        summary = summarize(transcript, option)
+                        with open("summary.txt", "w") as f:
+                            f.write(summary)
+                        with open("summary.txt", "r") as f:
+                            for line in f:
+                                st.write(line)
+                        #st.write_stream(summary)
+                        #st.download_button("Download Summary", summary, key='summary_download_button')
+    
+
+       
+        loader = TextLoader("transcript.txt")
+        documents = loader.load()
+
+        st.header("Chatbot🤖")
+        text_splitter = RecursiveCharacterTextSplitter(chunk_size=1000, chunk_overlap=20)
+        text_chunks = text_splitter.split_documents(documents)
+
+        # Create embeddings
+        embeddings = HuggingFaceEmbeddings(model_name="sentence-transformers/all-MiniLM-L6-v2", 
+                                        model_kwargs={'device': 'cuda:1'})
+                                        #model_kwargs={'device': 'cpu'}) #Almost the same speed
+
+        # Create vector store
+        vector_store = FAISS.from_documents(text_chunks, embedding=embeddings)
+
+        # Create the chain object
+        chain = create_conversational_chain(vector_store, option)
+
+        display_chat_history(chain)
+
+        if st.button("Click to start with a new video"):
+            streamlit_js_eval(js_expressions="parent.window.location.reload()")
+
+if __name__ == "__main__":
+    main()

transcript.txt

@@ -0,0 +1 @@
+Today on The Stay At Home Chef I'm showing you how to cook steak perfectly every time. This method works for any kind of steak and will help you achieve the perfect steak every single time. There's no need to be intimidated by cooking steak because it's actually super easy and I'm going to show you how. The first thing you need to do is pull your steaks out and let them sit at room temperature for at least 30 minutes. If you're using frozen steaks you'll want to thaw them in the fridge first but you do want them to come to room temperature. And if you'll notice we're using nice thick steaks. Thick steaks are actually easier to cook perfectly. Thin steaks will cook too fast and may even dry out so choose a good thick steak. Today we're using a couple of New York strip steaks but this same method works with ribeye or t-bone or even filet so choose a cut of steak that you like. You can see this steak has some nice marbling in it and some fat around the outside. There are actually three different grades of beef, select, choice, and prime and it's all based on the marbling and flavor in the beef. Now the difference between select and choice is actually pretty big so I always recommend choosing at least choice grade or prime grade beef when you're making steak. This is actually a choice grade New York steak. Once your steaks have had a chance to sit out at room temperature for 30 minutes, preheat your oven to 375 degrees Fahrenheit and heat a large heavy skillet on the stove until it's smoking hot. Then we'll prep our steaks. Drizzle just a little bit of olive oil onto each steak and rub it into the top and the sides as well as the bottom. Then we're going to put together our rub. Today I'm going to keep it super simple and just use a mixture of salt and pepper. Use a teaspoon of salt, half a teaspoon of black pepper, and half a teaspoon of white pepper. Mix these together. You can add other spices and herbs but I like to keep it simple and let the meat shine. We're going to take this mixture, sprinkle it over our steaks, and rub it right in. You want to make sure you do both sides. Now since these steaks are really thick, you want to go a little more heavy-handed than you might think. Before we go over and cook our steaks, there's one last thing we want to do. I like to finish off my steaks with butter, so I'm going to make a little garlic compound butter here by mixing 2 tablespoons of softened butter with 1 to 2 cloves of crushed garlic and you just mash that together with a fork. You can also add in fresh herbs to this butter to make it an herb compound butter but I'm keeping it simple with just a little bit of garlic. Next we're going to head on over to the stove and I'm pretty sure our skillet is smoking hot which is exactly what we need. Place the steaks down directly into the pan. You don't need oil or anything. And we're going to sear each side of the steak for 2-3 minutes per side. And then since these have a little fat strip that runs along the side, you want to render that fat down. So just turn it onto its side and hold it there with tongs. You always want to render the fat so you don't end up with a chewy piece of fat around the edges. Once it's seared on both sides and you've rendered the fat then just slide them in the oven to finish cooking. The amount of time that you need to leave your steaks in the oven varies depending on how long you seared your steaks and how thick they are. A good rule of thumb is about five minutes per inch, but if you want 100% accurate results, use a meat thermometer. I pull my steak out when it reaches 130 degrees Fahrenheit for a good medium rare. The steaks will need to rest for at least 10 minutes before serving, so you want to dollop on a tablespoon of butter onto each one so it'll melt over the steak while it rests. As you let your steaks rest the internal temperature will continue to rise by about 5 degrees. It's important to let your steaks rest because it gives them a chance for the juices to reabsorb into the meat, giving you the juiciest, most perfect steak possible. After 10 minutes of resting, you are ready to eat! Thanks for watching! You can find the full written recipe in the video description. Be sure to subscribe, like, and follow, and check out the rest of my videos, where you can find hundreds of restaurant quality recipes you can easily make at home. See you later!