NLP Based Analysis of Literary Texts (M 5.2.3)

cd D:\DKPro

java -Xmx4g -jar ddw-0.4.7.jar -input file.txt -output folder

java -Xmx4g -jar ddw-0.4.7.jar -language de -input C:\EffiBriestKurz.txt -output D:\DKPro\Workspace

java -Xmx4g -jar ddw-0.4.7.jar -language de -input .\EffiBriestKurz.txt -output .

File written, DONE 

java -Xmx4g -jar ddw-0.4.7.jar -language de -input file.txt -output folder

Exception in thread "main" java.lang.OutOfMemoryError: Java heap space or The specified size exceeds the maximum representable size. Error: Could not create the Java Virtual Machine

java –Xms -jar ddw-0.4.7.jar -input file.txt -output folder

java -Xms4g -jar ddw-0.4.7.jar -input EffiBriestKurz.txt -output D:\DKPro\Workspace

###################################
# Segmentation
###################################
useSegmenter = true # line 6
segmenter = de.tudarmstadt.ukp.dkpro.core.opennlp.OpenNlpSegmenter # line 7

# Possible values for segmenter:
# - de.tudarmstadt.ukp.dkpro.core.tokit.BreakIteratorSegmenter
# - de.tudarmstadt.ukp.dkpro.core.clearnlp.ClearNlpSegmenter
# - de.tudarmstadt.ukp.dkpro.core.opennlp.OpenNlpSegmenter (default)
# - de.tudarmstadt.ukp.dkpro.core.stanfordnlp.StanfordSegmenter

java -Xmx4g -jar ddw-0.4.7.jar -config /path/to/my/config/myconfigfile.properties -input file.txt -output folder

java -Xmx4g -jar ddw-0.4.7.jar -config myconfigfile.properties -input file.txt -output folder

java -Xmx4g -jar ddw-0.4.7.jar -config myfile1.properties,myconfig2.properties,myfile3.properties -input file.txt -output folder

java -Xmx4g -jar ddw-0.4.7.jar -config myFullVersion.properties,myPOSTagger.properties -input file.txt -output folder

columns_input = ['SentenceId', 'TokenId', 'Token', 'CPOS']

df = df[columns_input]                                     # use only the selected columns

sentences = df.groupby('SentenceId')                        # sort by sentence id

sent = sentences.get_group(10)                              # get sentence no. 10, returns a smaller dataframe

tags = df.groupby('CPOS')                                   # sort by CPOS values

adj = tags.get_group('ADJ')                                 # get all adjectives

nn = sent[sent['CPOS'] == 'NN']                             # get nouns from the sentence

for sent_id, sent in sentences:                             # iterate through sentences

    for tok_id, tok, pos in zip(sent['TokenId'], sent['Token'], sent['CPOS']):  # go through each token in the sentence

        print(tok_id, tok, pos)

setwd("~/DKPro/")

getwd()

# Extract file names
files = list.files(pattern = "*.csv")
 
# Create directories
dir.create("dv/")
dir.create("pos/")
dir.create("dv/corpus/")
dir.create("pos/corpus/")
 
for(file in files)
{
  # Read file
  df = read.table(file, header = T, fill = T)
 
  # Prepare filename
  shortfile = sub(".csv", "", file)
 
  # Write Adjectives and Adverbes to analyse the author's inventaar of descriptive vocabulary
  dv = df$Lemma[df$CPOS == "ADJ" | df$CPOS == "ADV"]
  filename = paste("./dv/corpus/", shortfile, sep = "")
  write(paste(dv, collapse = " "), file = filename)
 
  # Write POS tags to compare sentence structure
  filename = paste("./pos/corpus/", shortfile, sep = "")
  write(paste(df$CPOS, collapse=" "), file = filename)
}

install.packages("stylo")

library(stylo)

setwd("./dv/")

setwd("./pos/")

stylo()

pip3 install gensim pandas numpy pyLDAvis

sudo apt-get install python-matplotlib

pip3 install matplotlib

#!/usr/bin/env python

from gensim.corpora import MmCorpus, Dictionary
from gensim.models import LdaMulticore
import pandas as pd
import numpy as np
import os
import sys
import csv

# input
columns = ['ParagraphId', 'TokenId', 'Lemma', 'CPOS', 'NamedEntity']   # columns to read from csv file
pos_tags = ['ADJ', 'NN']                        # parts-of-speech to include into the model
 
# stopwords
stopwordlist = "stopwords.txt"                  # path to text file, e.g. stopwords.txt in the same directory as the script
 
# document size (in words)
#doc_size = 1000000 # set to arbitrarily large value to use original doc size
doc_size = 1000                                 # the document size for LDA commonly ranges from 500-2000 words
doc_split = 0                                   # uses the pipeline's ParagraphId to split text into documents, overrides doc_size - 1: on, 0: off 
 
# model parameters, cf. https://radimrehurek.com/gensim/models/ldamodel.html
no_of_topics = 20                               # no. of topics to be generated
no_of_passes = 200                              # no. of lda iterations - usually, the more the better, but increases computing time
 
eval = 1                                        # perplexity estimation every n chunks - the smaller the better, but increases computing time
chunk = 10                                      # documents to process at once
 
alpha = "symmetric" # "symmetric", "asymmetric", "auto", or array (default: a symmetric 1.0/num_topics prior)
                                                # affects sparsity of the document-topic (theta) distribution


# custom alpha may increase topic coherence, but may also produce more topics with zero probability
#alpha = np.array([ 0.02, 0.02, 0.02, 0.03, 0.03, 0.03, 0.04, 0.04, 0.04, 0.05,
# 0.05, 0.04, 0.04, 0.04, 0.03, 0.03, 0.03, 0.02, 0.02, 0.02])

eta = None                                      # can be a number (int/float), an array, or None
                                                # affects topic-word (lambda) distribution - not necessarily beneficial to topic coherence

def preprocessing(path, columns, pos_tags, doc_size, doc_split, stopwordlist):
    docs = []
    doc_labels = []
    stopwords = ""
 
    print("reading files ...\n")
 
    try:
        with open(stopwordlist, 'r') as f: stopwords = f.read()
    except OSError:
        pass
    stopwords = sorted(set(stopwords.split("\n")))
 
    for file in os.listdir(path=path):
        if not file.startswith("."):
            filepath = path+"/"+file
            print(filepath)
 
            df = pd.read_csv(filepath, sep="\t", quoting=csv.QUOTE_NONE)
            df = df[columns]
            df = df.groupby('CPOS')
 
            doc = pd.DataFrame()
            for p in pos_tags:                          # collect only the specified parts-of-speech
                doc = doc.append(df.get_group(p))
 
            names = df.get_group('NP')['Lemma'].values.astype(str)  # add proper nouns to stopword list
            stopwords += names.tolist()
 
            # construct documents
            if doc_split:                               # size according to paragraph id
                doc = doc.groupby('ParagraphId')
                for para_id, para in doc:
                    docs.append(para['Lemma'].values.astype(str))
                    doc_labels.append(file.split(".")[0]+" #"+str(para_id))     # use filename + doc id as plot label
            else:                                       # size according to doc_size
                doc = doc.sort(columns='TokenId')
                i = 1
                while(doc_size < doc.shape[0]):
                    docs.append(doc[:doc_size]['Lemma'].values.astype(str))
                    doc_labels.append(file.split(".")[0]+" #"+str(i))
                    doc = doc.drop(doc.index[:doc_size])        # drop doc_size rows
                    i += 1
                docs.append(doc['Lemma'].values.astype(str))    # add the rest
                doc_labels.append(file.split(".")[0]+" #"+str(i))
 
    print("\nnormalizing and vectorizing ...\n")        # cf. https://radimrehurek.com/gensim/tut1.html
 
    texts = [[word for word in doc if word not in stopwords] for doc in docs]       # remove stopwords
 
    all_tokens = sum(texts, [])                                                     # remove words that appear only once
    tokens_once = set(word for word in set(all_tokens) if all_tokens.count(word) == 1)
    texts = [[word for word in text if word not in tokens_once] for text in texts]
 
    dictionary = Dictionary(texts)                      # vectorize
    corpus = [dictionary.doc2bow(text) for text in texts]
 
    return dictionary, corpus, doc_labels

df = df.groupby('NamedEntity')

names = df.get_group('B-PER')['Lemma'].values.astype(str)

names += df.get_group('I-PER')['Lemma'].values.astype(str)

stopwords += names.tolist()

df = df.groupby('CPOS')

names = df.get_group('NP')['Lemma'].values.astype(str)

stopwords += names.tolist()

if len(sys.argv) < 2:
    print("usage: {0} [folder containing csv files]\n"
          "parameters are set inside the script.".format(sys.argv[0]))
    sys.exit(1)
 
path = sys.argv[1]
foldername = path.split("/")[-1]
 
dictionary, corpus, doc_labels = preprocessing(path, columns, pos_tags, doc_size, doc_split, stopwordlist)

print("fitting the model ...\n")
 
model = LdaMulticore(corpus=corpus, id2word=dictionary, num_topics=no_of_topics, passes=no_of_passes,
                 eval_every=eval, chunksize=chunk, alpha=alpha, eta=eta)
 
print(model, "\n")
 
topics = model.show_topics(num_topics=no_of_topics)
 
for item, i in zip(topics, enumerate(topics)):
    print("topic #"+str(i[0])+": "+str(item)+"\n")

topic #0: 0.012*instant + 0.011*universe + 0.010*mad + 0.008*way + 0.008*everyone + 0.007*ship + 0.007*whilst + 0.007*other + 0.007*poor + 0.007*moment
topic #1: 0.008*world + 0.007*horror + 0.006*years + 0.006*body + 0.006*other + 0.006*terrible + 0.004*woman + 0.004*tree + 0.004*family + 0.004*baronet
topic #2: 0.009*corridor + 0.009*foot + 0.009*hand + 0.008*woman + 0.007*eyes + 0.007*lover + 0.007*floor + 0.006*chamber + 0.006*shape + 0.006*estate
topic #3: 0.012*point + 0.012*foot + 0.011*specimen + 0.011*inch + 0.009*print + 0.008*tube + 0.008*vegetable + 0.008*animal + 0.008*camp + 0.008*diameter
topic #4: 0.012*other + 0.012*way + 0.012*face + 0.010*case + 0.010*last + 0.010*eyes + 0.009*hand + 0.009*moor + 0.007*nothing + 0.006*anything
topic #5: 0.013*arms + 0.008*shape + 0.006*human + 0.005*tree + 0.005*lip + 0.005*neck + 0.005*face + 0.005*loam + 0.005*pave + 0.005*preferable
topic #6: 0.000*incoherent + 0.000*reality + 0.000*riches + 0.000*fearful + 0.000*neighbor + 0.000*oriental + 0.000*liking + 0.000*tentacle + 0.000*prize-fighter + 0.000*bristle
topic #7: 0.016*eyes + 0.012*poor + 0.011*anything + 0.010*hot + 0.009*punkah + 0.009*chap + 0.009*cooly + 0.008*face + 0.008*native + 0.006*sort
topic #8: 0.017*stain + 0.015*chemical + 0.012*test + 0.009*file + 0.009*rooms + 0.008*wagonette + 0.007*text + 0.007*eccentric + 0.007*fare + 0.006*misfortune
topic #9: 0.017*buffalo + 0.016*foot + 0.015*child + 0.015*herd + 0.014*things + 0.013*branch + 0.011*boy + 0.010*eyes + 0.010*moon + 0.009*skin
topic #10: 0.000*incoherent + 0.000*reality + 0.000*riches + 0.000*fearful + 0.000*neighbor + 0.000*oriental + 0.000*liking + 0.000*tentacle + 0.000*prize-fighter + 0.000*bristle
topic #11: 0.017*eyes + 0.013*tree + 0.013*foot + 0.009*hand + 0.008*cliff + 0.008*fire + 0.007*hands + 0.007*shoulder + 0.007*figure + 0.007*ruin
topic #12: 0.026*things + 0.020*dretful + 0.017*home + 0.016*while + 0.013*fine + 0.011*legs + 0.010*round + 0.010*afraid + 0.009*loud + 0.008*bit
topic #13: 0.000*incoherent + 0.000*reality + 0.000*riches + 0.000*fearful + 0.000*neighbor + 0.000*oriental + 0.000*liking + 0.000*tentacle + 0.000*prize-fighter + 0.000*bristle
topic #14: 0.013*desert + 0.008*palm + 0.008*human + 0.007*hand + 0.006*hut + 0.006*other + 0.006*lamp + 0.005*shadow + 0.005*eyes + 0.005*foot
topic #15: 0.009*case + 0.009*other + 0.008*family + 0.006*cellar + 0.005*manuscript + 0.005*record + 0.005*account + 0.005*much + 0.005*years + 0.005*interest
topic #16: 0.015*wind + 0.015*plane + 0.013*camp + 0.012*snow + 0.010*wireless + 0.010*world + 0.009*other + 0.009*antarctic + 0.008*whole + 0.008*seal
topic #17: 0.000*incoherent + 0.000*reality + 0.000*riches + 0.000*fearful + 0.000*neighbor + 0.000*oriental + 0.000*liking + 0.000*tentacle + 0.000*prize-fighter + 0.000*bristle
topic #18: 0.011*foot + 0.009*base + 0.008*plane + 0.008*world + 0.008*camp + 0.007*crew + 0.007*trip + 0.007*peak + 0.007*years + 0.006*unknown
topic #19: 0.003*cleanliness + 0.003*hawk-like + 0.003*luncheon + 0.000*readiness + 0.000*channels + 0.000*brigade + 0.000*enthusiast + 0.000*exactness + 0.000*edition + 0.000*politics

import logging
logging.basicConfig(format='%(asctime)s : %(levelname)s : %(message)s', level=logging.INFO)

print("saving ...\n")
 
if not os.path.exists("out"): os.makedirs("out")
 
with open("out/"+foldername+"_doclabels.txt", "w") as f:
    for item in doc_labels: f.write(item+"\n")
 
with open("out/"+foldername+"_topics.txt", "w") as f:
    for item, i in zip(topics, enumerate(topics)):
        f.write("topic #"+str(i[0])+": "+str(item)+"\n")
 
dictionary.save("out/"+foldername+".dict")
MmCorpus.serialize("out/"+foldername+".mm", corpus)
model.save("out/"+foldername+".lda")

pip3 install pandas scikit-learn

import pandas as pd
import numpy as np
import os, sys
from collections import Counter
from sklearn.feature_extraction import DictVectorizer
from sklearn.preprocessing import Imputer
from sklearn.ensemble import RandomForestClassifier
from sklearn.cross_validation import cross_val_score, ShuffleSplit

   1. Current token
   2. Previous token
   3. Next token
   4. Concatenation of the wordclass tags of these three tokens (as
       assigned by an automatic WOTANlite tagger (van Halteren et al.., 2001)
   5. Concatenation of
       a. length of the sentence (in 7 classes: 1, 2, 3, 4, 5-10,11-20 or 21+
           tokens)
       b. position in the sentence (in 3 classes: first three tokens, last
           three tokens, other)
   6. Concatenation of
       a. part of speech of the current token, i.e. the initial part of the
           wordclass tag
       b. frequency of the current token in the text (in 5 classes: 1, 2-5,
           6-10,11-20 or 21+)
       c. number of blocks (consisting of 1/7th of the text) in which the
           current token is found (in 4 classes: 1, 2-3,4-6,7)
       d. distance in sentences to the previous occurrence of the current token
           (in 7 classes: NONE, SAME, 1, 2-3,4-7,8-15,16+)

def wordcount(wordlist):
    dict = {}
    for word in wordlist:
        if word not in dict: dict[word] = 1
        else: dict[word] += 1
    return dict
 
def token_in_textblock(text, token):        # returns number of blocks (consisting of 1/7th of the text)
    blocks = []                             # in which the current token is found, in 4 classes: 1, 2-3,4-6,7
    block_size = len(text)/7
    last = no_of_blocks = 0
 
    while last < len(text):
        blocks.append(text[int(last):int(last + block_size)])
        last += block_size
 
    for block in blocks:
        if token in block: no_of_blocks += 1
 
    if no_of_blocks == 1: occur_class = 1
    elif 2 <= no_of_blocks <= 3: occur_class = 2
    elif 4 <= no_of_blocks <= 6: occur_class = 3
    else: occur_class = 4
 
    return occur_class
 
def distance_to_previous(curr_tok_id, curr_sent_id, occurrences):
    # returns distance in sentences to the previous occurrence
    # of the current token (in 7 classes: NONE, SAME, 1, 2-3,4-7,8-15,16+
 
    occurrences = occurrences.reset_index()                             # add new index from 0 .. len(occurrences.index)
 
    current_key = occurrences[occurrences['TokenId'] == curr_tok_id].index[0]   # get row corresponding to curr_tok_id + its new index value
 
    if current_key > 0:                                                 # there is more than one && its not the first occurrence
        prev_sent_id = int(occurrences.iloc[current_key-1, 1])          # get previous sentence id based on that index
 
        dist = curr_sent_id - prev_sent_id
 
        if dist == 0: d_class = 2
        elif dist == 1: d_class = 3
        elif 2 <= dist <= 3: d_class = 4
        elif 4 <= dist <= 7: d_class = 5
        elif 8 <= dist <= 15: d_class = 6
        elif 16 <= dist: d_class = 7
    else:
        d_class = 1
 
    return d_class
 


def featureselect(text):
    columns = ['SentenceId', 'TokenId', 'Token', 'CPOS']
    columns_features = ['CurrToken', 'PrevToken', 'NextToken', 'TokenTags', 'LengthPosition', 'TagFreqOccur']
 
    csv = pd.read_csv(text, sep="\t")
    df = csv[columns]                               # create copy containing only the specified columns
 
    sent_max = df["SentenceId"].max()               # number of sentences in the text
    token_max = df["TokenId"].max()                 # number of tokens in the text
 
    text = list(df["Token"])
    word_freq = wordcount(text)                     # word frequencies
 
    features = pd.DataFrame(columns=columns_features, index=range(token_max+1))       # dataframe to hold the results
 
    for sent_id in range(sent_max+1):               # iterate through sentences
        sentence = df[df['SentenceId'] == sent_id]  # return rows corresponding to sent_id
 
        s_len = len(sentence.index)                 # length of the sentence
        if s_len == 1: s_class = 1                  # in 7 classes: 1, 2, 3, 4, 5-10,11-20 or 21+ tokens
        elif s_len == 2: s_class = 2
        elif s_len == 3: s_class = 3
        elif s_len == 4: s_class = 4
        elif 5 <= s_len <= 10: s_class = 5
        elif 11 <= s_len <= 20: s_class = 6
        elif 21 <= s_len: s_class = 7
 
        tok_count = 1
        for row in sentence.iterrows():
            tok_id = row[0]                         # row/dataframe index is the same as TokenId
 
            features.iat[tok_id, 0] = current_tok = row[1].get("Token")             # save current token
            tokentags = current_pos = row[1].get("CPOS")                            # get current pos tag
 
            if tok_id > 0:
                features.iat[tok_id, 1] = df.iloc[tok_id-1, 2]                      # save previous token
                tokentags += "-" + df.iloc[tok_id-1, 3]                             # get previous pos tag
            else:
                tokentags += "-NaN"
 
            if tok_id < token_max:
                features.iat[tok_id, 2] = df.iloc[tok_id+1, 2]                      # save next token
                tokentags += "-" + df.iloc[tok_id+1, 3]                             # get next pos tag
            else:
                tokentags += "-NaN"
 
            features.iat[tok_id, 3] = tokentags                         # save pos tags
 
            if tok_count <= 3: t_class = 1                              # position in the sentence
            elif (s_len-3) < tok_count <= s_len: t_class = 2            # in 3 classes: first three tokens, last three tokens, other
            else: t_class = 3
 
            features.iat[tok_id, 4] = str(s_class) + "-" + str(t_class) # save sentence length + token position
 
            tok_freq = word_freq[current_tok]                           # frequency of the current token in the text
            if tok_freq == 1: f_class = 1                               # in 5 classes: 1, 2-5, 6-10,11-20 or 21+
            elif 2 <= tok_freq <= 5: f_class = 2
            elif 6 <= tok_freq <= 10: f_class = 3
            elif 11 <= tok_freq <= 20: f_class = 4
            elif 21 <= tok_freq: f_class = 5
 
            block_occur = token_in_textblock(text, current_tok)
 
            occurrences = df[df['Token'] == current_tok]                # new dataframe containing all of curr_token's occurrences
            previous_distance = distance_to_previous(tok_id, sent_id, occurrences)
 
            features.iat[tok_id, 5] = current_pos + "-" + str(f_class) + "-" + str(block_occur) + "-" + str(previous_distance)
 
            tok_count += 1
 
    return features

           CurrToken       PrevToken       NextToken     TokenTags LengthPosition TagFreqOccur
0                Den             NaN     Mittelgrund    ART-NaN-NN            6-1    ART-2-1-1
1        Mittelgrund             Den          bilden      NN-ART-V            6-1     NN-1-1-1
2             bilden     Mittelgrund          Säulen       V-NN-NN            6-1      V-1-1-1
3             Säulen          bilden             mit       NN-V-PP            6-3     NN-1-1-1
4                mit          Säulen          weiten     PP-NN-ADJ            6-3     PP-3-3-1
5             weiten             mit  Zwischenräumen     ADJ-PP-NN            6-3    ADJ-1-1-1
6     Zwischenräumen          weiten               ,   NN-ADJ-PUNC            6-3     NN-1-1-1
7                  ,  Zwischenräumen             das   PUNC-NN-ART            6-3   PUNC-5-4-1
8                das               ,        Peristyl   ART-PUNC-NN            6-3    ART-3-3-1
9           Peristyl             das     bezeichnend    NN-ART-ADJ            6-2     NN-1-1-1
10       bezeichnend        Peristyl               .   ADJ-NN-PUNC            6-2    ADJ-1-1-1
11                 .     bezeichnend              Im   PUNC-ADJ-PP            6-2   PUNC-5-4-1
12                Im               .    Hintergrunde    PP-PUNC-NN            6-1     PP-2-2-1
13      Hintergrunde              Im             der     NN-PP-ART            6-1     NN-1-1-1
14               der    Hintergrunde          Tempel     ART-NN-NN            6-1    ART-5-4-1
15            Tempel             der               ,   NN-ART-PUNC            6-3     NN-3-3-1
16                 ,          Tempel              zu    PUNC-NN-PP            6-3   PUNC-5-4-3
17                zu               ,             dem    PP-PUNC-PR            6-3     PP-4-4-1
18               dem              zu         mehrere      PR-PP-PR            6-3     PR-3-3-1
19           mehrere             dem          Stufen      PR-PR-NN            6-3     PR-2-2-1
20            Stufen         mehrere     emporführen       NN-PR-V            6-2     NN-2-1-1
21       emporführen          Stufen               .     V-NN-PUNC            6-2      V-1-1-1
22                 .     emporführen            Nach     PUNC-V-PP            6-2   PUNC-5-4-3
23              Nach               .           vorne   PP-PUNC-ADV            6-1     PP-2-2-1
24             vorne            Nach               ,   ADV-PP-PUNC            6-1    ADV-1-1-1
25                 ,           vorne          rechts  PUNC-ADV-ADV            6-1   PUNC-5-4-3
26            rechts               ,             die  ADV-PUNC-ART            6-3    ADV-1-1-1
27               die          rechts          Statue    ART-ADV-NN            6-3    ART-5-4-1
28            Statue             die           Amors     NN-ART-NP            6-3     NN-1-1-1
29             Amors          Statue               ,    NP-NN-PUNC            6-3     NP-1-1-1
...              ...             ...             ...           ...            ...          ...

def preprocessing(path, n):
    feats = []
    y = []
 
    print("processing files and randomly selecting {0} features each ...\n".format(n))
 
    for file in os.listdir(path=path):
        if not file.startswith("."):
            author = file.split("-")[0].replace("%20", " ")
            filepath = path+"/"+file
            print(filepath)
 
            for i in range(n): y.append(author)                     # add n labels to y
 
            with open(filepath, "r") as f:
                feat = featureselect(f)                             # perform feature selection
                rows = np.random.choice(feat.index.values, n)       # randomly select n observations
                feat_rand = feat.ix[rows]
 
                feats.append(feat_rand)
                f.close()
 
    data = pd.concat(feats, ignore_index=True)                      # merge into one dataframe
 
    print("\ndimensions of X: {0}".format(data.shape))
    print("dimensions of y: {0}\n".format(len(y)))
 
    print("vectorizing ...\n")
 
    vec = DictVectorizer(sparse=False)
    X = vec.fit_transform(data.T.to_dict().values())
    print("dimensions of X after vectorization: {0}\n".format(X.shape))
 
    imp = Imputer(missing_values='NaN', strategy='median', axis=0)    # replace NaN
    X = imp.fit_transform(X)
 
    return X, y, vec

n_obs = 700                                                         # no. of observations to select
n_trees = 30                                                        # no. of estimators in RandomForestClassifier
 
if len(sys.argv) < 3:
    print("usage: {0} [folder containing csv files for training] [csv file for testing]".format(sys.argv[0]))
    sys.exit(1)
 
# do feature selection, normalization, and vectorization
X, y, vec = preprocessing(sys.argv[1], n_obs)
 
# model training
print("training classifier ...\n")
clf = RandomForestClassifier(n_estimators=n_trees, n_jobs=-1).fit(X, y) # -1 sets n_jobs to the number of CPU cores
print(clf)
 
# evaluation
print("\nperforming cross validation (n_iter=5, test_size=0.125) ...")
cv = ShuffleSplit(X.shape[0], n_iter=5, test_size=0.125, random_state=4)
scores = cross_val_score(clf, X, y, cv=cv, n_jobs=-1)
print(scores)
print("mean accuracy: %0.2f (+/- %0.2f)\n" % (scores.mean(), scores.std() * 2))

processing files and randomly selecting 700 features each ...

train/Grillparzer%20-%20Das%20goldene%20Vließ%20(Anfang)%20(Drama).txt.csv
train/Grillparzer%20-%20Das%20Kloster%20bei%20Sendomir%20(Anfang)%20(Prosa).txt.csv
train/Grillparzer%20-%20Der%20arme%20Spielmann%20(Anfang)%20(Prosa).txt.csv
train/Grillparzer%20-%20Der%20Traum%20ein%20Leben%20(Anfang)%20(Drama).txt.csv
train/Grillparzer%20-%20Ein%20Erlebnis%20(Prosa).txt.csv
train/Grillparzer%20-%20Gedichte%201%20(Lyrik).txt.csv
train/Grillparzer%20-%20Gedichte%202%20(Lyrik).txt.csv
train/Grillparzer%20-%20Gedichte%203%20(Lyrik).txt.csv
train/von%20Kleist%20-%20Amphitryon%20(Anfang)%20(Drama).txt.csv
train/von%20Kleist%20-%20An%20Wilhelmine%20(Lyrik).txt.csv
train/von%20Kleist%20-%20Das%20Bettelweib%20von%20Locarno%20(Prosa).txt.csv
train/von%20Kleist%20-%20Das%20Erdbeben%20in%20Chili%20(Prosa).txt.csv
train/von%20Kleist%20-%20Das%20Käthchen%20von%20Heilbronn%20(Anfang)%20(Drama).txt.csv
train/von%20Kleist%20-%20Der%20Welt%20Lauf%20(Lyrik).txt.csv
train/von%20Kleist%20-%20Der%20zerbrochne%20Krug%20(Anfang)%20(Drama).txt.csv
train/von%20Kleist%20-%20Die%20beiden%20Tauben%20(Lyrik).txt.csv

dimensions of X: (11200, 6)
dimensions of y: 11200

vectorizing ...

dimensions of X after vectorization: (11200, 9692)

training classifier ...

RandomForestClassifier(bootstrap=True, class_weight=None, criterion='gini',
            max_depth=None, max_features='auto', max_leaf_nodes=None,
            min_samples_leaf=1, min_samples_split=2,
            min_weight_fraction_leaf=0.0, n_estimators=30, n_jobs=-1,
            oob_score=False, random_state=None, verbose=0,
            warm_start=False)

performing cross validation (n_iter=5, test_size=0.125) ...
[ 0.76214286  0.75928571  0.76714286  0.75142857  0.75785714]
mean accuracy: 0.76 (+/- 0.01)

print("predicting author for {0} ...\n".format(sys.argv[2]))
 
# feature selection and preprocessing for testfile
with open(sys.argv[2], "r") as f:
    feat = featureselect(f)                             # perform feature selection
    rows = np.random.choice(feat.index.values, n_obs)       # randomly select n observations
    feat = feat.ix[rows]
 
print("dimensions of X_test: {0}".format(feat.shape))
 
X_test = vec.transform(feat.T.to_dict().values())       # vec must be the same DictVectorizer object as generated by preprocessing()
 
print("dimensions of X_test after vectorization: {0}\n".format(X_test.shape))
 
imp = Imputer(missing_values='NaN', strategy='median', axis=0)    # replace NaN
X_test = imp.fit_transform(X_test)
 
# prediction
y_pred = clf.predict(X_test)
 
c = Counter(y_pred)
c_key = list(c.keys())
c_val = list(c.values())
print(c_key[0], c_val[0]/(sum(c.values())/100), "% - ",
      c_key[1], c_val[1]/(sum(c.values())/100), "%")

predicting author for test/von%20Kleist%20-%20Der%20Findling%20(Prosa).txt.csv ...

dimensions of X_test: (700, 6)
dimensions of X_test after vectorization: (672, 9692)

von Kleist  77.52976190476191 % -  Grillparzer  22.470238095238095 %

import warnings
warnings.filterwarnings("ignore")

pip3 install wikipedia
pip3 install networkx

import wikipedia
import re

def create_authors(working_directory, wiki_page, wiki_section):
    """Gathers names from Wikipedia"""

    print("\nCreating authors.txt ...")
    with open(working_directory + "/authors.txt", "w", encoding='utf-8') as authors:
        full_content = wikipedia.page(wiki_page)
        selected_content = full_content.section(wiki_section)
        only_name = re.sub("[ \t\r\n\f]+[\(\[].*?[\]\)]","", selected_content)  # erases characters after full name
        authors.write(only_name)
        print(only_name)

Creating authors.txt ...
Dietmar von Aist
Friedrich von Hausen
Heinrich von Rugge
Heinrich von Veldeke
Herger
Der von Kürenberg
Meinloh von Sevelingen
Rudolf von Fenis
Spervogel

def crawl_wikipedia(authors_file, output_directory):
    """Crawls Wikipedia with authors.txt"""

    print("\nCrawling Wikipedia ...")
    with open(authors_file, "r", encoding="utf-8") as authors:
        for author in authors.read().splitlines():
            try:
                page_title = wikipedia.page(author)
                if page_title:
                    with open(output_directory + "/" + author + ".txt", "w", encoding='utf-8') as new_author:
                        new_author.write(page_title.content)
                        print(author + ": saved")

                else:
                    print("Error: Cannot create variable for wikipedia.page")

            except wikipedia.exceptions.DisambiguationError:
                pass
            except wikipedia.exceptions.HTTPTimeoutError:
                pass
            except wikipedia.exceptions.RedirectError:
                pass
            except wikipedia.exceptions.PageError:
                pass

Crawling Wikipedia ...
Dietmar von Aist: saved
Friedrich von Hausen: saved
Heinrich von Rugge: saved
Heinrich von Veldeke: saved
Herger: saved
Der von Kürenberg: saved
Meinloh von Sevelingen: saved
Rudolf von Fenis: saved
Spervogel: saved

def main(working_directory, output_directory, wiki_page, wiki_section):
    """
    :param working_directory: e.g. /users/networks
    :param output_directory: e.g. /users/networks/wikis
    :param wiki_page: e.g. "Liste deutschsprachiger Lyriker"
    :param wiki_section: e.g. "12. Jahrhundert"
    """

    wikipedia.set_lang("de")    # change language
    create_authors(working_directory, wiki_page, wiki_section)
    crawl_wikipedia(sys.argv[1] + "/authors.txt", output_directory)

if __name__ == "__main__":
    import sys
    main(sys.argv[1], sys.argv[2], sys.argv[3], sys.argv[4])

def main(authors_file, output_directory):
    """
    :param authors_file: e.g. /users/networks/my_own_file.txt
    :param output_directory: e.g. /users/networks/wikis
    """

    wikipedia.set_lang("de")    # change language
    crawl_wikipedia(authors_file, output_directory)

if __name__ == "__main__":
    import sys
    main(sys.argv[1], sys.argv[2])

import csv
from collections import defaultdict
import itertools
import glob
import os
import networkx as nx
import matplotlib.pyplot as plt
import re

def ne_count(input_file):
    """Extracts only Named Entities"""

    ne_counter = defaultdict(int)
    with open(input_file, encoding='utf-8') as csv_file:
        read_csv = csv.DictReader(csv_file, delimiter='\t', quoting=csv.QUOTE_NONE)
        lemma = []

        for row in read_csv:
            if row['NamedEntity'] != "_" and row['CPOS'] != "PUNC":
                lemma.append(row['Lemma'])
            else:
                if lemma:
                    joined_lemma = ' '.join(lemma)
                    ne_counter[joined_lemma] += 1
                    lemma = []
    return ne_counter

def compare_ne_counter(ne_dict1, ne_dict2):
    """Compares two dictionaries"""

    weight = 0
    for key in ne_dict1.keys():
        if key in ne_dict2.keys():
            weight += 1
    print("this is the weight: " + str(weight))
    return weight

def extract_basename(file_path):
    """Extracts names from file names"""

    file_name_txt_csv = os.path.basename(file_path)
    file_name_txt = os.path.splitext(file_name_txt_csv)
    file_name = os.path.splitext(file_name_txt[0])
    return file_name[0]

def create_graph(input_folder):
    """Creates graph including nodes and edges"""

    G = nx.Graph()
    file_list = glob.glob(input_folder)

    for item in file_list:
        G.add_node(extract_basename(item))

    for a, b in itertools.combinations(file_list, 2):
        weight = compare_ne_counter(ne_count(a), ne_count(b))
        if weight > 10:
            G.add_edge(extract_basename(a), extract_basename(b), {'weight': weight})
            # create edges a->b (weight)

    print("Number of nodes:", G.number_of_nodes(), "  Number of edges: ", G.number_of_edges())
    return G

this is the weight: 20
this is the weight: 11
this is the weight: 15
this is the weight: 7
this is the weight: 5
this is the weight: 9
this is the weight: 12
this is the weight: 18
this is the weight: 16
this is the weight: 10
this is the weight: 10
this is the weight: 14
this is the weight: 7
this is the weight: 8
this is the weight: 11
this is the weight: 9
this is the weight: 9
this is the weight: 9
this is the weight: 9
this is the weight: 8
this is the weight: 8
Number of nodes: 7   Number of edges:  8

def main(input_folder, output_folder):
    """
    :param input_folder: e.g. /users/networks/csv
    :param output_folder: e.g. /users/networks
    """

    G = create_graph(input_folder + "/*")
    # If you want to create a circular graph, add '#' in front of every line of the following block,
    # erase the '#' of the three lines after 'Circular drawing', and run the script (again)
    pos = nx.spring_layout(G)
    nx.draw_networkx_labels(G, pos, font_size='8', font_color='r')
    nx.draw_networkx_edges(G, pos, alpha=0.1)
    plt.axis('off')
    plt.savefig(output_folder + "/graph.png")

    # Circular drawing:
    # nx.draw_circular(G, with_labels=True, alpha=0.3, font_size='8')
    # plt.axis('off')
    # plt.savefig(output_folder + "/circular.png")


if __name__ == "__main__":
    import sys
    main(sys.argv[1], sys.argv[2])