Sentiment Analysis pre-processing added.py

short_pos = open("C:\\Users\\Raora\\Downloads\\pos-1.txt","r").read()
short_neg = open("C:\\Users\\Raora\\Downloads\\neg-1.txt","r").read()


documents = []

for r in short_pos.split('\n'):
    documents.append( (r, "pos") )

for r in short_neg.split('\n'):
    documents.append( (r, "neg") )


##print(documents[:5])
print("Length of documents: " + str(len(documents)))


import nltk
##nltk.download('punkt')
# !pip install autocorrect
from nltk.tokenize import word_tokenize
import spacy
nlp = spacy.load("en_core_web_sm")
all_words = []

def preprocessing(text):
    doc = nlp(text)
    tokens = [token.text.lower() for token in doc if not (token.is_stop or not token.is_alpha)]
    lemmas=[token.lemma_ for token in nlp(' '.join(tokens))]
    return lemmas

short_pos_words = preprocessing(short_pos)
short_neg_words = preprocessing(short_neg)
print(len(short_pos_words))
print(len(short_neg_words))

for w in short_pos_words:
    all_words.append(w.lower())

for w in short_neg_words:
    all_words.append(w.lower())

all_words = nltk.FreqDist(all_words)

print("Length of BOW: " + str(len(all_words)))

lof = 500

word_features = list(all_words.keys())[:lof]
print("Length of features: " + str(len(word_features)))


def find_features(document):
    words = word_tokenize(document)
    features = {}
    for w in word_features:
        features[w] = (w in words)

    return features


featuresets = [(find_features(rev), category) for (rev, category) in documents]

print("Length of feature sets: " + str(len(featuresets)))

import random
import pickle

random.shuffle(featuresets)

##print(featuresets[0])



from nltk.classify.scikitlearn import SklearnClassifier

from sklearn.naive_bayes import MultinomialNB,BernoulliNB

from sklearn.linear_model import LogisticRegression,SGDClassifier
from sklearn.svm import SVC, LinearSVC, NuSVC

ratio1 = 80
ratio2 = 20

training_set = featuresets[:int((lof * ratio1) / 100)]
testing_set =  featuresets[int((lof * ratio2) / 100):]

classifier = nltk.NaiveBayesClassifier.train(training_set)
print("Original Naive Bayes Algo accuracy percent:", (nltk.classify.accuracy(classifier, testing_set))*100)

##print(classifier.show_most_informative_features(15))

##save_classifier = open("naivebayes.pickle","wb")
##pickle.dump(classifier, save_classifier)
##save_classifier.close()


##classifier_f = open("naivebayes.pickle", "rb")
##classifier = pickle.load(classifier_f)
##classifier_f.close()
##
##
##print("Original Naive Bayes Algo accuracy percent:", (nltk.classify.accuracy(classifier, testing_set))*100)
##classifier.show_most_informative_features(15)



MNB_classifier = SklearnClassifier(MultinomialNB())
MNB_classifier.train(training_set)
print("MNB_classifier accuracy percent:", (nltk.classify.accuracy(MNB_classifier, testing_set))*100)

BernoulliNB_classifier = SklearnClassifier(BernoulliNB())
BernoulliNB_classifier.train(training_set)
print("BernoulliNB_classifier accuracy percent:", (nltk.classify.accuracy(BernoulliNB_classifier, testing_set))*100)

LogisticRegression_classifier = SklearnClassifier(LogisticRegression())
LogisticRegression_classifier.train(training_set)
print("LogisticRegression_classifier accuracy percent:", (nltk.classify.accuracy(LogisticRegression_classifier, testing_set))*100)

SGDClassifier_classifier = SklearnClassifier(SGDClassifier())
SGDClassifier_classifier.train(training_set)
print("SGDClassifier_classifier accuracy percent:", (nltk.classify.accuracy(SGDClassifier_classifier, testing_set))*100)

SVC_classifier = SklearnClassifier(SVC())
SVC_classifier.train(training_set)
print("SVC_classifier accuracy percent:", (nltk.classify.accuracy(SVC_classifier, testing_set))*100)

LinearSVC_classifier = SklearnClassifier(LinearSVC())
LinearSVC_classifier.train(training_set)
print("LinearSVC_classifier accuracy percent:", (nltk.classify.accuracy(LinearSVC_classifier, testing_set))*100)

NuSVC_classifier = SklearnClassifier(NuSVC())
NuSVC_classifier.train(training_set)
print("NuSVC_classifier accuracy percent:", (nltk.classify.accuracy(NuSVC_classifier, testing_set))*100)