semantic_similarity_with_bert.py

# -*- coding: utf-8 -*-
"""semantic_similarity_with_bert

Automatically generated by Colaboratory.

Original file is located at
    https://colab.research.google.com/github/keras-team/keras-io/blob/master/examples/nlp/ipynb/semantic_similarity_with_bert.ipynb
"""

!pip install transformers
import numpy as np
import pandas as pd
import tensorflow as tf
import transformers

"""## Configuration"""

max_length = 512  # Maximum length of input sentence to the model.
batch_size = 8
epochs = 2

# Labels in our dataset.
labels = ["contradiction", "entailment", "neutral"]

"""## Load the Data"""

# !curl -LO https://raw.githubusercontent.com/MohamadMerchant/SNLI/master/data.tar.gz
# !tar -xvzf data.tar.gz

# There are more than 550k samples in total; we will use 100k for this example.
train_df = pd.read_csv("SNLI_Corpus/snli_1.0_train.csv", nrows=100000)
valid_df = pd.read_csv("SNLI_Corpus/snli_1.0_dev.csv")
test_df = pd.read_csv("SNLI_Corpus/snli_1.0_test.csv")

# Shape of the data
print(f"Total train samples : {train_df.shape[0]}")
print(f"Total validation samples: {valid_df.shape[0]}")
print(f"Total test samples: {valid_df.shape[0]}")

test=test_df.drop(test_df[test_df['label'] == 2].index) 
test

"""Let's look at one sample from the dataset:

## Preprocessing
"""

# We have some NaN entries in our train data, we will simply drop them.
print("Number of missing values")
print(train_df.isnull().sum())
train_df.dropna(axis=0, inplace=True)

train_df["label"] = train_df["similarity"].apply(
    lambda x: 0 if x == "contradiction" else 1 if x == "entailment" else 2
)
y_train = tf.keras.utils.to_categorical(train_df.label, num_classes=3)

valid_df["label"] = valid_df["similarity"].apply(
    lambda x: 0 if x == "contradiction" else 1 if x == "entailment" else 2
)
y_val = tf.keras.utils.to_categorical(valid_df.label, num_classes=3)

test["label"] = test["similarity"].apply(
    lambda x: 0 if x == "contradiction" else 1
)
y_test = tf.keras.utils.to_categorical(test.label, num_classes=2)

import pandas as pd
from sklearn.model_selection import train_test_split
df=pd.read_csv("/content/sample_data/combined_data_paraphrase.csv", delimiter=',', header=None, names=['sentence1', 'sentence2', 'label', 'is_sts'])
df=df.iloc[1:]

df.label=df.label.astype(int)

y=df.label
x=df.drop(['label','is_sts'],axis=1)
x_train,x_test,y_train1,y_test1=train_test_split(x,y,test_size=0.2)

length=0
for index,row in x.iterrows():
    if len(row['sentence2'])>length:
      length=len(row['sentence2'])
print(length)

# Commented out IPython magic to ensure Python compatibility.
# %reset_selective -f len

y_train2=tf.keras.utils.to_categorical(y_train1, num_classes=2)
y_train2

y_test2=tf.keras.utils.to_categorical(y_test1, num_classes=2)

"""## Bert Custom Data Generator"""

class BertSemanticDataGenerator(tf.keras.utils.Sequence):
    """Generates batches of data.

    Args:
        sentence_pairs: Array of premise and hypothesis input sentences.
        labels: Array of labels.
        batch_size: Integer batch size.
        shuffle: boolean, whether to shuffle the data.
        include_targets: boolean, whether to incude the labels.

    Returns:
        Tuples `([input_ids, attention_mask, `token_type_ids], labels)`
        (or just `[input_ids, attention_mask, `token_type_ids]`
         if `include_targets=False`)
    """

    def __init__(
        self,
        sentence_pairs,
        labels,
        batch_size=batch_size,
        shuffle=True,
        include_targets=True,
    ):
        self.sentence_pairs = sentence_pairs
        self.labels = labels
        self.shuffle = shuffle
        self.batch_size = batch_size
        self.include_targets = include_targets
        # Load our BERT Tokenizer to encode the text.
        # We will use base-base-uncased pretrained model.
        self.tokenizer = transformers.BertTokenizer.from_pretrained(
            "bert-base-uncased", do_lower_case=True
        )
        self.indexes = np.arange(len(self.sentence_pairs))
        self.on_epoch_end()

    def __len__(self):
        # Denotes the number of batches per epoch.
        return len(self.sentence_pairs) // self.batch_size

    def __getitem__(self, idx):
        # Retrieves the batch of index.
        indexes = self.indexes[idx * self.batch_size : (idx + 1) * self.batch_size]
        sentence_pairs = self.sentence_pairs[indexes]

        # With BERT tokenizer's batch_encode_plus batch of both the sentences are
        # encoded together and separated by [SEP] token.
        encoded = self.tokenizer.batch_encode_plus(
            sentence_pairs.tolist(),
            add_special_tokens=True,
            max_length=max_length,
            truncation=True,
            return_attention_mask=True,
            return_token_type_ids=True,
            pad_to_max_length=True,
            return_tensors="tf",
        )

        # Convert batch of encoded features to numpy array.
        input_ids = np.array(encoded["input_ids"], dtype="int32")
        attention_masks = np.array(encoded["attention_mask"], dtype="int32")
        token_type_ids = np.array(encoded["token_type_ids"], dtype="int32")

        # Set to true if data generator is used for training/validation.
        if self.include_targets:
            labels = np.array(self.labels[indexes], dtype="int32")
            return [input_ids, attention_masks, token_type_ids], labels
        else:
            return [input_ids, attention_masks, token_type_ids]

    def on_epoch_end(self):
        # Shuffle indexes after each epoch if shuffle is set to True.
        if self.shuffle:
            np.random.RandomState(42).shuffle(self.indexes)

"""## Build the model."""

# Create the model under a distribution strategy scope.
strategy = tf.distribute.MirroredStrategy()

with strategy.scope():
    # Encoded token ids from BERT tokenizer.
    input_ids = tf.keras.layers.Input(
        shape=(max_length,), dtype=tf.int32, name="input_ids"
    )
    # Attention masks indicates to the model which tokens should be attended to.
    attention_masks = tf.keras.layers.Input(
        shape=(max_length,), dtype=tf.int32, name="attention_masks"
    )
    # Token type ids are binary masks identifying different sequences in the model.
    token_type_ids = tf.keras.layers.Input(
        shape=(max_length,), dtype=tf.int32, name="token_type_ids"
    )
    # Loading pretrained BERT model.
    bert_model = transformers.TFBertModel.from_pretrained("bert-base-uncased")
    # Freeze the BERT model to reuse the pretrained features without modifying them.
    bert_model.trainable = False

    sequence_output, pooled_output = bert_model(
        input_ids, attention_mask=attention_masks, token_type_ids=token_type_ids
    )
    # Add trainable layers on top of frozen layers to adapt the pretrained features on the new data.
    bi_lstm = tf.keras.layers.Bidirectional(
        tf.keras.layers.LSTM(64, return_sequences=True)
    )(sequence_output)
    # Applying hybrid pooling approach to bi_lstm sequence output.
    avg_pool = tf.keras.layers.GlobalAveragePooling1D()(bi_lstm)
    max_pool = tf.keras.layers.GlobalMaxPooling1D()(bi_lstm)
    concat = tf.keras.layers.concatenate([avg_pool, max_pool])
    dropout = tf.keras.layers.Dropout(0.3)(concat)
    output = tf.keras.layers.Dense(2, activation="softmax")(dropout)
    model = tf.keras.models.Model(
        inputs=[input_ids, attention_masks, token_type_ids], outputs=output
    )

    model.compile(
        optimizer=tf.keras.optimizers.Adam(),
        loss="categorical_crossentropy",
        metrics=["acc"],
    )


print(f"Strategy: {strategy}")
model.summary()

"""Create train and validation data generators"""

train_data = BertSemanticDataGenerator(
    x_train[["sentence1", "sentence2"]].values.astype("str"),
    y_train2,
    batch_size=batch_size,
    shuffle=True,
)
valid_data = BertSemanticDataGenerator(
    x_test[["sentence1", "sentence2"]].values.astype("str"),
    y_test2,
    batch_size=batch_size,
    shuffle=False,
)

"""## Train the Model

Training is done only for the top layers to perform "feature extraction",
which will allow the model to use the representations of the pretrained model.
"""

history = model.fit(
    train_data,
    validation_data=valid_data,
    epochs=epochs,
    use_multiprocessing=True,
    workers=-1,
)

"""## Fine-tuning

This step must only be performed after the feature extraction model has
been trained to convergence on the new data.

This is an optional last step where `bert_model` is unfreezed and retrained
with a very low learning rate. This can deliver meaningful improvement by
incrementally adapting the pretrained features to the new data.
"""

# Unfreeze the bert_model.
bert_model.trainable = True
# Recompile the model to make the change effective.
model.compile(
    optimizer=tf.keras.optimizers.Adam(1e-5),
    loss="categorical_crossentropy",
    metrics=["accuracy"],
)
model.summary()

"""# Train the entire model end-to-end."""

history = model.fit(
    train_data,
    validation_data=valid_data,
    epochs=epochs,
    use_multiprocessing=True,
    workers=-1,
)

"""## Evaluate model on the test set"""

test_data = BertSemanticDataGenerator(
    test[["sentence1", "sentence2"]].values.astype("str"),
    y_test,
    batch_size=batch_size,
    shuffle=False,
)
model.evaluate(test_data, verbose=1)

test_data1 = BertSemanticDataGenerator(
    x_test[["sentence1", "sentence2"]].values.astype("str"),
    y_test2,
    batch_size=batch_size,
    shuffle=False,
)
model.evaluate(test_data1, verbose=1)

import csv
data_msr=pd.read_csv("/content/sample_data/msr_paraphrase_data.txt", sep = '\t', quoting=csv.QUOTE_NONE,error_bad_lines=False);
data_msr.head()

df_msr=pd.DataFrame()
df_msr

syn, ant, raw,ind = [],[],[],[]
index=0
for i,row in data_msr.iterrows():
  if i%2==0:
    syn.append(row['String'])
  else:
    ant.append(row['String'])
    raw.append(1)
    index+=1
    ind.append(index)
# print(len(syn))
# print(len(ant))
# print(len(raw))
# print(index)
df_msr['sentence1']=syn
df_msr['sentence2']=ant
df_msr['label']=raw
df_msr['index']=ind

testlabel_msr=df_msr.label
trainfeatures_msr=df_msr.drop(['label','index'],axis=1)
test_msr = tf.keras.utils.to_categorical(testlabel_msr, num_classes=2)