This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
class Prediction(): | |
def __init__(self): | |
self.model = model_obj.model | |
def predict_validation(self): | |
valid_sentences = load_data_obj.validation_data_frame["query"].tolist() | |
valid_labels = load_data_obj.validation_data_frame["category"].tolist() | |
preprocess_bert_data_obj = PreprocessingBertData() | |
val_x = preprocess_bert_data_obj.prepare_data_x(valid_sentences) |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
class Evaluation(): | |
def get_accuracy(self,actuals, predictions): | |
acc = accuracy_score(actuals, predictions) | |
return acc | |
eval_obj = Evaluation() | |
ytest,ypred = pred_obj.predict_validation() | |
#print(ytest,ypred) | |
acc = eval_obj.get_accuracy(ytest,ypred) | |
print("Auc: {:.2%}".format(acc)) |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
class DesignModel(): | |
def __init__(self): | |
self.model = None | |
self.train_data = [train_input_ids, train_input_masks, train_segment_ids] | |
self.train_labels = train_labels | |
def bert_model(self,max_seq_length): | |
in_id = Input(shape=(max_seq_length,), dtype=tf.int32, name="input_ids") | |
in_mask = Input(shape=(max_seq_length,), dtype=tf.int32, name="input_masks") | |
in_segment = Input(shape=(max_seq_length,), dtype=tf.int32, name="segment_ids") |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
class PreprocessingBertData(): | |
def prepare_data_x(self,train_sentences): | |
x = bert_model_obj.get_input_array(train_sentences) | |
return x | |
def prepare_data_y(self,train_labels): | |
y = list() | |
for item in train_labels: | |
label = item |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
class BertModel(object): | |
def __init__(self): | |
self.max_len = 128 | |
bert_path = "https://tfhub.dev/tensorflow/bert_en_uncased_L-12_H-768_A-12/1" | |
FullTokenizer=bert.bert_tokenization.FullTokenizer | |
self.bert_module = hub.KerasLayer(bert_path,trainable=True) |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
class LoadingData(): | |
def __init__(self): | |
train_file_path = os.path.join("benchmarking_data","Train") | |
validation_file_path = os.path.join("benchmarking_data","Validate") | |
category_id = 0 | |
self.cat_to_intent = {} | |
self.intent_to_cat = {} | |
for dirname, _, filenames in os.walk(train_file_path): |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
100%|██████████| 1/1 [00:00<00:00, 1014.83it/s] | |
100%|██████████| 1/1 [00:00<00:00, 506.37it/s] | |
100%|██████████| 1/1 [00:00<00:00, 972.71it/s] | |
100%|██████████| 1/1 [00:00<00:00, 514.95it/s] | |
100%|██████████| 1/1 [00:00<00:00, 489.42it/s] | |
Predicted Intent: SearchCreativeWork Actual Intent: SearchScreeningEvent Query: I want to see Medal for the General | |
Predicted Intent: BookRestaurant Actual Intent: BookRestaurant Query: Book a reservation for 5 people at the top-rated brasserie restaurant | |
Predicted Intent: AddToPlaylist Actual Intent: AddToPlaylist Query: Can I put this tune onto my sin estres playlist? | |
Predicted Intent: AddToPlaylist Actual Intent: AddToPlaylist Query: add the artist Pete Murray to my relaxing playlist | |
Predicted Intent: BookRestaurant Actual Intent: BookRestaurant Query: Book me a reservation for a party of 3 at a pub in Northern Mariana Islands |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
from numpy import array | |
import tensorflow as tf | |
from tensorflow.keras.preprocessing.text import Tokenizer | |
from tensorflow.keras.utils import to_categorical | |
from tensorflow.keras.preprocessing.sequence import pad_sequences | |
from tensorflow.keras.models import Sequential | |
from tensorflow.keras.layers import Dense | |
from tensorflow.keras.layers import LSTM | |
from tensorflow.keras.layers import Dropout | |
from tensorflow.keras.layers import Embedding |