Create inference_data_pipeline.py

inference_data_pipeline.py

@@ -0,0 +1,92 @@
+#Class to fetch news and stock data from the web for a specific ticker and combine them into a dataframe.
+
+import pandas as pd
+import yfinance as yf
+from pygooglenews import GoogleNews
+from transformers import AutoTokenizer, AutoModelForSequenceClassification
+from transformers import pipeline
+class InferenceDataPipeline:
+    def __init__(self, ticker, time_period_news, time_period_stock):
+        self.ticker = ticker
+        self.time_period_news = time_period_news
+        self.time_period_stock = time_period_stock
+
+    def get_data(self):
+        stock_data = self.get_stock_data()
+        news_data = self.get_news_data()
+        news_sentiment = self.get_sentiment(news_data)
+        combined_data = self.combine_data(stock_data, news_sentiment)
+
+        return combined_data
+
+
+    def get_stock_data(self):
+        data = yf.download(self.ticker , period = self.time_period_stock)
+        df = pd.DataFrame()
+        df['Open'] = data['Open']
+        df['Close'] = data['Close']
+        df['High'] = data['High']
+        df['Low'] = data['Low']
+        df['Volume'] = data['Volume']
+
+        return df
+
+    def get_news_data(self):
+        googlenews = GoogleNews()
+        news_data = googlenews.search(self.ticker, when=self.time_period_news)
+        news_data = pd.DataFrame(news_data['entries'])
+        return news_data
+
+    def get_sentiment(self, news_data):
+        tokenizer = AutoTokenizer.from_pretrained("ProsusAI/finbert")
+        model = AutoModelForSequenceClassification.from_pretrained("ProsusAI/finbert")
+        classifier = pipeline('sentiment-analysis', model=model, tokenizer=tokenizer)
+
+        news_sentiment = []
+        for i in range(len(news_data)):
+            sentiment = classifier(news_data['title'][i], top_k=None)
+            postive_score = sentiment[0]['score']
+            negative_score = sentiment[1]['score']
+            neutral_score = sentiment[2]['score']
+            reformmated_time_stamp = pd.to_datetime(news_data['published'][i]).date()
+            news_sentiment.append({'Date': reformmated_time_stamp, 'positive_score': postive_score, 'negative_score': negative_score, 'neutral_score': neutral_score})
+        return pd.DataFrame(news_sentiment)
+
+    def combine_data(self, stock_data, news_sentiment):
+        news_sentiment = (
+            news_sentiment
+            .groupby('Date')
+            .mean()
+            .fillna(0)
+            .reset_index()
+            .set_index('Date')
+            .sort_index()
+        )
+
+        common_index = pd.date_range(
+            start=pd.Timestamp(min(pd.Timestamp(stock_data.index[0]), pd.Timestamp(news_sentiment.index[0]))),
+            end=pd.Timestamp(max(pd.Timestamp(stock_data.index[-1]), pd.Timestamp(news_sentiment.index[-1]))),
+            freq='D'
+        )
+        stock_data = stock_data.reindex(common_index).fillna(-1)
+
+        news_sentiment = news_sentiment.reindex(common_index).fillna(0)
+
+        #Ensure stock_data and news_sentiment have combatile indices
+        stock_data.index = pd.to_datetime(stock_data.index).normalize()
+        news_sentiment.index = pd.to_datetime(news_sentiment.index).normalize()
+
+        combined_data = pd.merge(
+            stock_data,
+            news_sentiment,
+            how='left',
+            left_index=True,
+            right_index=True
+        )
+
+        #Drop all close values that are -1
+        combined_data = combined_data[combined_data['Close'] != -1]
+        #fill all missing values with 0
+        combined_data = combined_data.fillna(0)
+
+        return combined_data