Stanford NER Implementation in Python

Import and Setup

import os
java_path = "C:/Program Files/Java/jdk-16.0.1/bin/java.exe"
os.environ['JAVAHOME'] = java_path

import urllib.request
import zipfile

from nltk.tag.stanford import StanfordNERTagger
# Set the direct path to the NER Tagger.
# Use english.all.3class.distsim (three class classifier) to find three classes of named entities.
_model_filename = r'C:\Delphi\Web\Marketing-Analytics\py\data\stanford-ner-2015-04-20\classifiers\english.all.3class.distsim.crf.ser.gz'
_path_to_jar = r'C:\Delphi\Web\Marketing-Analytics\py\data\stanford-ner-2015-04-20\stanford-ner.jar'

# Initialize the NLTK's Stanford NER Tagger API with the DIRECT PATH to the model and .jar file.
st = StanfordNERTagger(model_filename=_model_filename, path_to_jar=_path_to_jar)

entities = []

Read Data

# read the tweets
import pandas as pd
df = pd.read_json('data/tweets-statuses.json')  
df[:5]

Tag Named Entities

'''
Iterate through tweets list and 
(1) tag the named entities, and 
(2) extract and store only entities related to three classes – organization, person, location.
'''
for tweet in df['text']:
    # split the tweet by whitespace, into words, and tag them (see o/p lst_tags)
    lst_tags = st.tag(tweet.split())  
    for tup in lst_tags:  # for each (tag, word) tuple in lst_tags
        if(tup[1] != 'O'):  # exclude 'O' (Outside - do not belong to named entity class)
            entities.append(tup)
lst_tags # list of tags for the last tweet

# Print the first 20 entities
entities[:20]

Dataframe df_entities

# Load entity tuples to dataframe df_entities and name the columns “word” and “ner”
df_entities = pd.DataFrame(entities) 
df_entities.columns = ["word","ner"]
df_entities 

Extract Organizations

from collections import Counter
'''
Counter is a class from the collections module that counts the frequency of elements 
in a collection, like a list. It returns a dictionary-like object where keys are the 
elements, and values are their counts.
'''

# Filter df_entities to extract rows with NER = Organisations.
organizations = df_entities[df_entities['ner'].str.contains("ORGANIZATION")]

# top 10 Organisations: Get the top 10 most mentioned organizations.
cnt = Counter(organizations['word'])
cnt.most_common(10)

Extract People

# Filter df_entities to extract rows with NER = PERSON.
people = df_entities[df_entities['ner'].str.contains("PERSON")]

# top 10 Persons: Get the top 10 most mentioned persons.
cnt = Counter(people['word'])
cnt.most_common(10)

Extract Locations

# Filter df_entities to extract rows with NER = LOCATION.
locations = df_entities[df_entities['ner'].str.contains("LOCATION")]

# top 5 Locations: Get the top 10 most mentioned locations.
cnt = Counter(locations['word'])
cnt.most_common(5)

Liverpool Tweets

# Extract the tweets containing the organization ‘Liverpool’.
liverpool_tweets = df[df['text'].str.contains('Liverpool')]
print(liverpool_tweets['text'])