import plotly.graph_objects as go import streamlit as st import yfinance as yf import pandas as pd import numpy as np from urllib.request import urlopen, Request from bs4 import BeautifulSoup import plotly import plotly.express as px import nltk nltk.downloader.download('vader_lexicon') from nltk.sentiment.vader import SentimentIntensityAnalyzer from datetime import datetime from sklearn.preprocessing import MinMaxScaler finviz_url = 'https://finviz.com/quote.ashx?t=' sector_etf_mapping = { 'Consumer Durables': 'XLY', 'Consumer Discretionary': 'XLY', 'Consumer Staples': 'XLP', 'Energy': 'XLE', 'Financials': 'XLF', 'Health Care': 'XLV', 'Industrials': 'XLI', 'Materials': 'XLB', 'Real Estate': 'XLRE', 'Technology': 'XLK', 'Utilities': 'XLU', 'Communication Services': 'XLC' } # Create two functions to calculate if a level is SUPPORT or a RESISTANCE level through fractal identification def is_Suppport_Level(df, i): support = df['Low'][i] < df['Low'][i - 1] and df['Low'][i] < df['Low'][i + 1] and df['Low'][i + 1] < df['Low'][i + 2] and df['Low'][i - 1] < df['Low'][i - 2] return support def is_Resistance_Level(df, i): resistance = df['High'][i] > df['High'][i - 1] and df['High'][i] > df['High'][i + 1] and df['High'][i + 1] > df['High'][i + 2] and df['High'][i - 1] > df['High'][i - 2] return resistance # This function, given a price value, returns True or False depending on if it is too near to some previously discovered key level. def distance_from_mean(level, levels, mean): return np.sum([abs(level - y) < mean for y in levels]) == 0 # Function to get the trend of a given stock or ETF def get_trend(ticker, period='1mo'): stock = yf.Ticker(ticker) hist = stock.history(period=period) return hist['Close'] # Function to normalize a pandas series def normalize(series): if len(series.unique()) > 1: # Check if the series has more than one unique value scaler = MinMaxScaler(feature_range=(0, 1)) scaled_values = scaler.fit_transform(series.values.reshape(-1,1)) return pd.Series(scaled_values.flatten(), index=series.index) else: return series # If the series has only one unique value, return it as is # Function to get the trend of a given sector or industry def get_trend_from_csv(df, column, value, period='1mo'): # Filter the dataframe by the given sector or industry df_filtered = df[df[column] == value] # Calculate the mean of the closing prices of all stocks in the sector or industry trend = df_filtered.resample(period).mean() return trend def get_news(ticker): url = finviz_url + ticker req = Request(url=url,headers={'User-Agent': 'Mozilla/5.0 (Windows NT 6.1; WOW64; rv:20.0) Gecko/20100101 Firefox/20.0'}) response = urlopen(req) # Read the contents of the file into 'html' html = BeautifulSoup(response) # Find 'news-table' in the Soup and load it into 'news_table' news_table = html.find(id='news-table') return news_table # parse news into dataframe def parse_news(news_table): parsed_news = [] for x in news_table.findAll('tr'): # occasionally x (below) may be None when the html table is poorly formatted, skip it in try except instead of throwing an error and exiting # may also use an if loop here to check if x is None first try: # read the text from each tr tag into text # get text from a only text = x.a.get_text() # splite text in the td tag into a list date_scrape = x.td.text.split() # if the length of 'date_scrape' is 1, load 'time' as the only element if len(date_scrape) == 1: time = date_scrape[0] # else load 'date' as the 1st element and 'time' as the second else: date = date_scrape[0] time = date_scrape[1] # Append ticker, date, time and headline as a list to the 'parsed_news' list parsed_news.append([date, time, text]) except: pass # Set column names columns = ['date', 'time', 'headline'] # Convert the parsed_news list into a DataFrame called 'parsed_and_scored_news' parsed_news_df = pd.DataFrame(parsed_news, columns=columns) # Create a pandas datetime object from the strings in 'date' and 'time' column parsed_news_df['datetime'] = pd.to_datetime(parsed_news_df['date'] + ' ' + parsed_news_df['time']) return parsed_news_df def score_news(parsed_news_df): # Instantiate the sentiment intensity analyzer vader = SentimentIntensityAnalyzer() # Iterate through the headlines and get the polarity scores using vader scores = parsed_news_df['headline'].apply(vader.polarity_scores).tolist() # Convert the 'scores' list of dicts into a DataFrame scores_df = pd.DataFrame(scores) # Join the DataFrames of the news and the list of dicts parsed_and_scored_news = parsed_news_df.join(scores_df, rsuffix='_right') parsed_and_scored_news = parsed_and_scored_news.set_index('datetime') parsed_and_scored_news = parsed_and_scored_news.drop(['date', 'time'], 1) parsed_and_scored_news = parsed_and_scored_news.rename(columns={"compound": "sentiment_score"}) return parsed_and_scored_news def plot_hourly_sentiment(parsed_and_scored_news, ticker): # Group by date and ticker columns from scored_news and calculate the mean mean_scores = parsed_and_scored_news.resample('H').mean() # Plot a bar chart with plotly fig = px.bar(mean_scores, x=mean_scores.index, y='sentiment_score', title = ticker + ' Hourly Sentiment Scores') return fig # instead of using fig.show(), we return fig and turn it into a graphjson object for displaying in web page later def plot_daily_sentiment(parsed_and_scored_news, ticker): # Group by date and ticker columns from scored_news and calculate the mean mean_scores = parsed_and_scored_news.resample('D').mean() # Plot a bar chart with plotly fig = px.bar(mean_scores, x=mean_scores.index, y='sentiment_score', title = ticker + ' Daily Sentiment Scores') return fig # instead of using fig.show(), we return fig and turn it into a graphjson object for displaying in web page later # Streamlit app st.set_page_config(page_title = "Stock Trend and News Sentiment Analyzer", layout = "wide") st.header("Stock Trend & News Sentiment Analyzer") st.markdown('Enter a stock symbol / ticker, then upload your CSV (from TradingView) with the following columns present: Ticker,Description,Price,Sector. Select the matching sector afterwards and run the analysis.') ticker = st.text_input('Enter Stock Ticker', '').upper() uploaded_file = st.file_uploader("Choose a CSV file", type="csv") if uploaded_file is not None: data = pd.read_csv(uploaded_file) try: # Display company name, last close, and sector from CSV company_name = data.loc[data['Ticker'] == ticker, 'Description'].values[0] last_close = data.loc[data['Ticker'] == ticker, 'Price'].values[0] csv_sector = data.loc[data['Ticker'] == ticker, 'Sector'].values[0] st.write(f"Company: {company_name} ({csv_sector})") st.write(f"Last Close: {last_close}") # Define a list of all possible sectors and their corresponding ETFs all_sectors = ['XLY', 'XLP', 'XLE', 'XLF', 'XLV', 'XLI', 'XLB', 'XLRE', 'XLK', 'XLU', 'XLC'] sector_mapping = { 'XLY': 'Consumer Discretionary', 'XLP': 'Consumer Staples', 'XLE': 'Energy', 'XLF': 'Financials', 'XLV': 'Health Care', 'XLI': 'Industrials', 'XLB': 'Materials', 'XLRE': 'Real Estate', 'XLK': 'Technology', 'XLU': 'Utilities', 'XLC': 'Communication Services' } # Get sector from CSV sector = data.loc[data['Ticker'] == ticker, 'Sector'].values[0] # If the sector is not found in the list of all sectors, set it to '--Not Found--' if sector not in sector_mapping.values(): sector = '--Not Found--' else: # Convert the sector to its corresponding ETF sector = list(sector_mapping.keys())[list(sector_mapping.values()).index(sector)] # Display a dropdown menu for the sector with the current sector selected by default sector = st.selectbox('Sector', options=all_sectors, index=all_sectors.index(sector) if sector in all_sectors else 0) # Display the sector mapping table st.table(pd.DataFrame(list(sector_mapping.items()), columns=['ETF', 'Sector'])) if st.button('Submit'): # Get trends stock_trend = get_trend(ticker) sector_trend = get_trend(sector) # Normalize trends scaler = MinMaxScaler() stock_trend = pd.DataFrame(scaler.fit_transform(stock_trend.values.reshape(-1,1)), index=stock_trend.index, columns=['Close']) sector_trend = pd.DataFrame(scaler.fit_transform(sector_trend.values.reshape(-1,1)), index=sector_trend.index, columns=['Close']) # Plot trends fig = go.Figure() fig.add_trace(go.Scatter(x=stock_trend.index, y=stock_trend['Close'], mode='lines', name=ticker)) fig.add_trace(go.Scatter(x=sector_trend.index, y=sector_trend['Close'], mode='lines', name=sector)) fig.update_layout(title='Stock and Sector Trend over the past 30 days', xaxis_title='Date', yaxis_title='Normalized Close Price', autosize=False, width=2000, height=1200) st.plotly_chart(fig) ticker2 = yf.Ticker(ticker) df = ticker2.history(period="1mo", actions=False) # Creating a list and feeding it the identified support and resistance levels via the Support and Resistance functions levels = [] level_types = [] mean = np.mean(df['High'] - df['Low']) for i in range(2, df.shape[0] - 2): if is_Suppport_Level(df, i): level = df['Low'][i].round(2) if distance_from_mean(level, levels, mean): levels.append(level) level_types.append('Support') elif is_Resistance_Level(df, i): level = df['High'][i].round(2) if distance_from_mean(level, levels, mean): levels.append(level) level_types.append('Resistance') # Plotting the data fig2 = go.Figure(data=[go.Candlestick(x=df.index, open=df['Open'], high=df['High'], low=df['Low'], close=df['Close'])]) for level, level_type in zip(levels, level_types): fig2.add_trace(go.Scatter(x=df.index, y=[level]*len(df.index), mode='lines', name=level_type)) fig2.update_layout(title='Support and Resistance levels for ' + ticker, xaxis_title='Date', yaxis_title='Price', autosize=False, width=2000, height=1200) st.plotly_chart(fig2) st.subheader("Hourly and Daily Sentiment of {} Stock".format(ticker)) news_table = get_news(ticker) parsed_news_df = parse_news(news_table) parsed_and_scored_news = score_news(parsed_news_df) fig_hourly = plot_hourly_sentiment(parsed_and_scored_news, ticker) fig_daily = plot_daily_sentiment(parsed_and_scored_news, ticker) st.plotly_chart(fig_hourly) st.plotly_chart(fig_daily) description = """ The above chart averages the sentiment scores of {} stock hourly and daily. The table below gives each of the most recent headlines of the stock and the negative, neutral, positive and an aggregated sentiment score. The news headlines are obtained from the FinViz website. Sentiments are given by the nltk.sentiment.vader Python library. Adapted from https://github.com/damianboh/stock_sentiment_streamlit """.format(ticker) st.write(description) st.table(parsed_and_scored_news) except Exception as e: print(str(e)) st.write("Enter a stock ticker, e.g. 'AAPL' above and hit Enter.") else: st.write("Please upload a CSV file.") hide_streamlit_style = """ """ st.markdown(hide_streamlit_style, unsafe_allow_html=True) # streamlit run "c:/Dropbox/BIMWERX/_work/YouTube/elephant/230701-10 - ML Predict/src/stock_sentiment2.py"