---
title: "Basic Text analysis in R"
author: "Till Bergmann"
date: "May 18, 2016"
output: html_document
---

```{r}
library(gutenbergr)
library(dplyr)
```


```{r}
frankenstein = gutenberg_download(84)
head(frankenstein$text)
```

```{r}
text = paste0(frankenstein$text, collapse = " ")

```

[Basic Text Mining in R](https://rstudio-pubs-static.s3.amazonaws.com/31867_8236987cf0a8444e962ccd2aec46d9c3.html)

## Preprocessing text

```{r}
library(tm)
text = Corpus(VectorSource(text))

text = tm_map(text, removePunctuation)   
# text[[1]]$content %>% head()
```

```{r}
text = tm_map(text, removeNumbers)      # *Removing numbers:*    
text = tm_map(text, content_transformer(tolower))   # *Converting to lowercase:*    
text = tm_map(text, removeWords, c('thou',stopwords("english")))   # *Removing "stopwords"
text = tm_map(text, stripWhitespace)

```

## Document Term Matrix

```{r}
dtm = DocumentTermMatrix(text)   
dtm   
```

```{r}
inspect(dtm[1, 1:20])
```

## Frequent terms

```{r}
freq = colSums(as.matrix(dtm))  %>% data.frame() %>% 
                                    select(n=1) %>%
                                    add_rownames('word') %>%
                                    mutate(rank=rank(-n, ties.method='random'))
``` 

```{r}
freq %>% arrange(desc(n)) %>% head()
```

```{r}
freq %>% arrange(n) %>% head()
```

## Freqency of frequencies


```{r}
with(freq,
     plot(rank, n, type='p'))
```

```{r}
with(ff,
     plot(log(rank), log(count), type='p'))
```

"the most frequent word will occur approximately twice as often as the second most frequent word, three times as often as the third most frequent word"

[Zipf's law](https://upload.wikimedia.org/wikipedia/commons/a/ac/Zipf_30wiki_en_labels.png)


## Word Cloud

```{r}
library(wordcloud)   
```

```{r}
wordcloud(freq$word, freq$n, min.freq=55,colors=brewer.pal(9, "Reds")   )
# display.brewer.all()
```





## Comparing texts to each other

```{r}

carroll_books = gutenberg_works(author == "Carroll, Lewis") %>%   gutenberg_download(meta_fields = "title")

text = carroll_books %>% group_by(title) %>% summarise(text = paste0(text, collapse = " "))
titles = text$title
docs = Corpus(VectorSource(text$text))


## Preprocessing      
docs <- tm_map(docs, removePunctuation)   # *Removing punctuation:*    
docs <- tm_map(docs, removeNumbers)      # *Removing numbers:*    
docs <- tm_map(docs, content_transformer(tolower))   # *Converting to lowercase:*    
# docs <- tm_map(docs, removeWords, stopwords("english"))   # *Removing "stopwords" 
docs <- tm_map(docs, stripWhitespace)   # *Stripping whitespace   
docs <- tm_map(docs, PlainTextDocument)   


dtm <- DocumentTermMatrix(docs)   
freq <- rowSums(as.matrix(dtm))   
dtm = dtm[!is.na(freq),]
row.names(dtm) = titles[!is.na(freq)]

dtmss <- removeSparseTerms(dtm, 0.15) # This makes a matrix that is only 15% empty space, maximum.   

```

```{r}
library(cluster)   
# d = dist(dtmss)
c = cor(dtmss %>% as.matrix() %>% t())

fit <- princomp(c, cor=TRUE)
df = fit$scores[,1:2] %>% as.data.frame() %>% add_rownames('title')
library(ggplot2)
library(cowplot)


ggplot(df, aes(x=Comp.1, y=Comp.2)) +
  geom_text(aes(label=title), size=5)
```