Upload app.py

app.py

@@ -0,0 +1,319 @@
+# -*- coding: utf-8 -*-
+"""Gradio_Demo.ipynb
+
+Automatically generated by Colaboratory.
+
+Original file is located at
+    https://colab.research.google.com/drive/1eXu8deuh8Jl5Mmm0DxX0XlR4-2E6IN0l
+"""
+
+!pip install gradio
+
+!git clone https://github.com/omnabill/NLP_Task
+
+import pandas as pd
+import seaborn as sns 
+import matplotlib.pyplot as plt
+import numpy as np
+
+# import the necessary libraries
+import nltk
+import string
+import re
+
+from time import time
+
+from gensim.parsing.preprocessing import STOPWORDS
+from gensim.utils import simple_preprocess
+from sklearn.model_selection import GridSearchCV
+
+from sklearn.model_selection import train_test_split
+from sklearn.utils.class_weight import compute_sample_weight
+from sklearn.feature_extraction.text import CountVectorizer, TfidfVectorizer
+from sklearn.metrics import accuracy_score, confusion_matrix
+from sklearn.naive_bayes import MultinomialNB
+from sklearn.pipeline import Pipeline
+from sklearn.feature_extraction.text import TfidfTransformer
+from sklearn.linear_model import SGDClassifier,LogisticRegression
+from sklearn.ensemble import ExtraTreesRegressor
+from sklearn import model_selection, svm
+import gradio as gr
+
+import warnings
+warnings.filterwarnings('ignore')
+
+"""### First Let's read our data"""
+
+df = pd.read_csv('/content/NLP_Task/flask-app/Job titles and industries.csv')
+
+df.head()
+
+df.tail()
+
+"""### Let's Explore our data to understand it .."""
+
+df.info()
+
+df.describe()
+
+df_count = df['industry'].value_counts().rename_axis('Category').reset_index(name='Count')
+
+df_count.head()
+
+plt.figure(figsize=(10,7))
+sns.barplot(x=df_count.Category,y=df_count.Count/len(df))
+plt.show()
+
+df_count.Count/len(df)
+
+"""Here there is a Class Less than 5% of existance which is "Accountancy" , So this can be a Way of imbalanced data that can disturb our model .
+
+This is a Problem That should be dealed with before passing our data to a model .
+
+### Let's complete First our EDA for our data .
+
+#### Let's check first for duplicated rows
+"""
+
+df[df.duplicated()==True]
+
+"""There are many duplicate rows let's start deal with them through keep one copy of duplicated rows"""
+
+df.drop_duplicates(subset="job title",inplace=True)
+
+df.info()
+
+plt.figure(figsize=(10,6))
+df.groupby('industry').count().plot.bar()
+
+df_count = df['industry'].value_counts().rename_axis('Category').reset_index(name='Count')
+
+plt.figure(figsize=(10,7))
+sns.barplot(x=df_count.Category,y=df_count.Count/len(df))
+plt.show()
+
+df_count.Count/len(df)
+
+"""### Let's Deal With text (Text Preprocessing)
+
+This Through : 
+
+1- Remove Stop words Like 'The','and','of',etc...
+
+2- change numerical numbers to text notation
+
+3- Remove Punctiuations:  We remove punctuations so that we don’t have different forms of the same word. If we don’t remove the punctuation, then been. been, been! will be treated separately.
+
+4- Text Lowercase:  to reduce the size of the vocabulary of our text data.
+
+5- Remove White spaces
+"""
+
+import inflect
+p = inflect.engine()
+
+def Text_Cleaner(text):
+    """
+        text: a string 
+        return: modified clean string
+    """
+    result = ""
+    
+    for token in text.split(' '):
+        
+        if token not in STOPWORDS and len(token) >= 1:
+            
+            if token.isdigit():
+                
+                temp = p.number_to_words(token)  #change numerical number to text
+                result+=temp+" "
+            else:
+                
+                if not re.match(r'£[0-9]+',  token):  
+                    
+                    token = token.lower()  #lower case string
+                    result+=token+" "     
+    translator = str.maketrans('', '', string.punctuation)  #remove punctiuation signs
+    return " ".join(result.translate(translator).split())
+
+
+df['job title'] = df['job title'].map(Text_Cleaner)
+df['job title'].head(20)
+
+"""### Deal With imbalanced data and pass it to model .. 
+
+Train More Than 1 Model For Text Classification
+
+1- SGD Classifier with Hyper Parameter Tuning 
+
+2- Multi nominal Naive Bayes 
+
+3- Logistic Regression
+
+4- SVM (Support vector machine calssifier)
+
+### hyper parameter tuning for SGDClassifier .
+"""
+
+X = df['job title']
+y = df['industry']
+X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state = 42)
+sample_weight = compute_sample_weight("balanced",y_train)
+
+
+
+pipeline = Pipeline([
+    ('vect', CountVectorizer()),
+    ('tfidf', TfidfTransformer()),
+    ('clf', SGDClassifier()),
+])
+
+# uncommenting more parameters will give better exploring power but will
+# increase processing time in a combinatorial way
+parameters = {
+    'vect__max_df': (0.5, 0.75, 1.0),
+    'vect__max_features': (None, 5000, 10000, 50000),
+    'vect__ngram_range': ((1, 1), (1, 2)),  # unigrams or bigrams
+    # 'tfidf__use_idf': (True, False),
+    # 'tfidf__norm': ('l1', 'l2'),
+    'clf__max_iter': (20,),
+    'clf__alpha': (0.00001, 0.000001),
+    'clf__penalty': ('l2', 'elasticnet'),
+    'clf__loss': ['log','hinge'],
+    'clf__max_iter': (10, 50, 80),
+}
+
+
+# multiprocessing requires the fork to happen in a __main__ protected
+# block
+
+# find the best parameters for both the feature extraction and the
+# classifier
+grid_search = GridSearchCV(pipeline, parameters, n_jobs=-1, verbose=1)
+
+print("Performing grid search...")
+print("pipeline:", [name for name, _ in pipeline.steps])
+print("parameters:")
+print(parameters)
+t0 = time()
+grid_search.fit(X_train, y_train)
+print("done in %0.3fs" % (time() - t0))
+print()
+
+print("Best score: %0.3f" % grid_search.best_score_)
+print("Best parameters set:")
+best_parameters = grid_search.best_estimator_.get_params()
+for param_name in sorted(parameters.keys()):
+        print("\t%s: %r" % (param_name, best_parameters[param_name]))
+
+make_pipline = Pipeline([('vect', CountVectorizer(max_df=0.5,max_features= 50000,ngram_range= (1, 1))),
+               ('tfidf', TfidfTransformer()),
+               ('SGD' , SGDClassifier(loss='log', penalty='l2',alpha=1e-05, random_state=42, max_iter=80)),])
+
+make_pipline.fit(X_train, y_train, **{'SGD__sample_weight': sample_weight})
+
+y_pred = make_pipline.predict(X_test)
+print('accuracy = {}'.format(accuracy_score(y_pred, y_test)) )
+
+
+
+"""### Naive Bayes"""
+
+make_pipline = Pipeline([('vect', CountVectorizer()),
+               ('tfidf', TfidfTransformer()),
+               ('clf', MultinomialNB()),])
+
+make_pipline.fit(X_train, y_train, **{'clf__sample_weight': sample_weight})
+
+y_pred = make_pipline.predict(X_test)
+print('accuracy %s' % accuracy_score(y_pred, y_test))
+
+y_pred
+
+"""### Logistic Regression"""
+
+make_pipline = Pipeline([('vect', CountVectorizer()),
+               ('tfidf', TfidfTransformer()),
+               ('log' , LogisticRegression(n_jobs=1, C=1e5)),])
+
+make_pipline.fit(X_train, y_train, **{'log__sample_weight': sample_weight})
+
+y_pred = make_pipline.predict(X_test)
+print('accuracy %s' % accuracy_score(y_pred, y_test))
+
+
+
+"""### Support Vector Machine"""
+
+make_pipline = Pipeline([('vect', CountVectorizer()),
+               ('tfidf', TfidfTransformer()),
+               ('clf', svm.SVC(C=1.0, kernel='linear', degree=3, gamma='auto',probability=True)),])
+
+make_pipline.fit(X_train, y_train, **{'clf__sample_weight': sample_weight})
+
+y_pred = make_pipline.predict(X_test)
+print('accuracy %s' % accuracy_score(y_pred, y_test))
+
+X_test.to_csv('out.csv', index=False)
+
+
+
+"""### Confusion Matrix for the final model Used SVM """
+
+a = np.flip(df['industry'].unique())
+
+from sklearn.metrics import confusion_matrix
+mat = confusion_matrix(y_test, y_pred)
+plt.figure(figsize=(10,8))
+ax = sns.heatmap(mat.T, square=True, annot=True, fmt='d', cbar=False,
+            xticklabels=a, yticklabels=a)
+bottom, top = ax.get_ylim()
+ax.set_ylim(bottom + 0.5, top - 0.5)
+
+plt.title('Confussion Matrix')
+plt.xlabel('true label')
+plt.ylabel('predicted label');
+
+"""IT is the highest class which missed it's labels , while it appeares that Marketing and education missed the leaset"""
+
+from sklearn import metrics 
+print(metrics.classification_report(y_test, y_pred, target_names=a))
+
+"""The performance measurment prove our thoughts from confusion matrix as the marketing and education are the highest classes with true positive rate precision that for classes that are marketing or education and they are classified as marketing or education
+while Accountancy have the leaset true positive rate 
+"""
+
+metrics.roc_auc_score(y_test, make_pipline.predict_proba(X_test), multi_class='ovr')
+
+"""The Area Under the Receiver Operating Characteristic Curve (AUC) is between 0.5 to 1 which 1 denates the out standing performance of the model, 0.5 the opposite , here 0.97 is a very good score that indicates that our model performance is very good .
+
+### Deploy model as Gradio Api Service
+"""
+
+def Job_Class(text):
+    # img = img.reshape(1, 100, 100, 1)
+    prediction = make_pipline.predict([text])
+    # class_names = ["Accountancy", "Education","IT","Marketing"]
+    return prediction[0]
+
+#set the user uploaded image as the input array
+#match same shape as the input shape in the model
+# im = gr.inputs.Image(shape=(100, 100), image_mode='L', invert_colors=False, source="upload")
+
+#setup the interface
+iface = gr.Interface(
+  fn=Job_Class, 
+  inputs=gr.inputs.Textbox(lines=2, placeholder="Write here the job descrebtion ..."), 
+  outputs="text")
+iface.launch(share=True)
+
+# prediction = make_pipline.predict(['senior technical support engineer'])
+
+# type(prediction[0])
+
+!pip install ipython
+
+pip install nbconvert
+
+!ipython nbconvert — to script Gradio_Demo.ipynb
+