dylan markov.Rmd

---
title: "Untitled"
author: "Thomas Rosenthal"
date: "14/08/2021"
output: html_document
---

```{r}
library(rvest) # to extract text from sites
library(stringr) # for easier string manipulation
library(readr) # to read text files
library(tidytext) # for natural language processing
library(dplyr) # for easier data manipulation
library(tidyr) # to make data wide and long
library(jsonlite) # to deal with json files

```

```{r}
return_third_word <- function( woord1, woord2){
        woord <- trigrams %>%
                filter_(~word1 == woord1, ~word2 == woord2) %>%
                sample_n(1, weight = n) %>%
                .[["word3"]]
        if(length(woord) == 0){
                bleh <- filter_(bigrams_picard, ~word1 == woord2) %>%
                        sample_n(1, weight = n)
                warning("no word found, adding ", bleh, "to", woord1 , woord2)
                woord <- bleh
        }
        woord
}
```

```{r}
generate_lyric <- function(word1, word2, sentencelength =5, debug =TRUE){
        #input validation
        if(sentencelength <3)stop("I need more to work with")
        sentencelength <- sentencelength -2
        # starting
        sentence <- c(word1, word2)
        woord1 <- word1
        woord2 <- word2
        for(i in seq_len(sentencelength)){
                if(debug == TRUE)print(i)
                word <- return_third_word( woord1, woord2)
                sentence <- c(sentence, word)
                woord1 <- woord2
                woord2 <- word
        }
        output <-paste(sentence, collapse = " ")
        output
}
```

```{r}
full_lyrics <- read_csv(paste0('/Users/thomas/Documents/mrpotatocode/','Dylan Lyrics.csv'))


full_lyrics <- readxl::read_excel(paste0('/Users/thomas/Documents/mrpotatocode/','Bo Burnham Inside Lyrics.xlsx'))
```

```{r}
clean_lyrics <- full_lyrics %>% mutate(lyrics = str_replace_all(lyrics,"[\r\n\t]" , " ")) %>% select(lyrics)
```


```{r}
clean_lyrics <- clean_lyrics %>% mutate(lyrics = str_replace_all(lyrics,"[\\[\\]]", ''))
```


```{r}
token <- clean_lyrics %>%
  unnest_tokens(word, lyrics)

bigram <- clean_lyrics %>%
  unnest_tokens(word, lyrics,token = "ngrams", n = 2)
```

```{r}
bigram <- na.omit(bigram)
bigram_sep <- separate(bigram, word, c("word1", "word2"), sep = " ") 
filter1 <- filter(bigram_sep,  !word1 %in% stop_words$word) 
filter2 <- filter(filter1, !word2 %in% stop_words$word) 
```

```{r}
trigrams <- clean_lyrics %>%
        unnest_tokens(trigram, lyrics, token = "ngrams",to_lower = TRUE, n= 3) %>%
        separate(trigram, c("word1", "word2", "word3"), sep = " ") %>%
        count(word1, word2,word3, sort = TRUE)
```


```{r}
one <- sample(bigram_sep$word1, 1, replace = FALSE)
two <- sample(bigram_sep$word2, 1, replace = FALSE)
#generate_lyric(one,two,9)
```



### MARKOV PART, FAST
```{r}
library(markovifyR)
markov_model <-  generate_markovify_model(
input_text = clean_lyrics$lyrics,
markov_state_size = 2L,
max_overlap_total = 25,
max_overlap_ratio = .85#,
#tries = 100
)
```

```{r}
markovify_text(
markov_model = markov_model,
maximum_sentence_length = NULL,
output_column_name = 'quote',
count = 10,
tries = 6,
only_distinct = TRUE,
return_message = TRUE
)
```


### TENSORFLOW PART, SLOW -- LIKE ACTUALLY SO SLOW
```{r}
library(keras)
library(tokenizers)
max_length <- 40
text <- clean_lyrics %>%
    pull(lyrics) %>%
    str_c(collapse = " ") %>%
    tokenize_words(lowercase = FALSE, strip_numeric = FALSE, simplify = TRUE) %>% 
    str_to_title()
    #tokenize_characters(lowercase = FALSE, strip_non_alphanum = FALSE, simplify = TRUE)

chars <- text %>%
    unique() %>%
    sort()
```

```{r}
dataset <- map(
    seq(1, length(text) - max_length - 1, by = 3), 
    ~list(sentence = text[.x:(.x + max_length - 1)], 
          next_char = text[.x + max_length])
)
dataset <- transpose(dataset)
```


```{r}
vectorize <- function(data, chars, max_length){
    x <- array(0, dim = c(length(data$sentence), max_length, length(chars)))
    y <- array(0, dim = c(length(data$sentence), length(chars)))
    
    for(i in 1:length(data$sentence)){
        x[i,,] <- sapply(chars, function(x){
            as.integer(x == data$sentence[[i]])
        })
        y[i,] <- as.integer(chars == data$next_char[[i]])
    }
    
    list(y = y,
         x = x)
}
vectors <- vectorize(dataset, chars, max_length)
```

```{r}
create_model <- function(chars, max_length){
    keras_model_sequential() %>%
        layer_lstm(128, input_shape = c(max_length, length(chars))) %>%
        layer_dense(length(chars)) %>%
        layer_activation("softmax") %>% 
        compile(
            loss = "categorical_crossentropy", 
            optimizer = optimizer_rmsprop(lr = 0.01)
        )
}
```

```{r}
fit_model <- function(model, vectors, epochs = 1){
    model %>% fit(
        vectors$x, vectors$y,
        batch_size = 128,
        epochs = epochs
    )
    NULL
}
```

```{r}
generate_phrase <- function(model, text, chars, max_length, diversity){
    
    # this function chooses the next character for the phrase
    choose_next_char <- function(preds, chars, temperature){
        preds <- log(preds) / temperature
        exp_preds <- exp(preds)
        preds <- exp_preds / sum(exp(preds))
        
        next_index <- rmultinom(1, 1, preds) %>% 
            as.integer() %>%
            which.max()
        chars[next_index]
    }
    
    # this function takes a sequence of characters and turns it into a numeric array for the model
    convert_sentence_to_data <- function(sentence, chars){
        x <- sapply(chars, function(x){
            as.integer(x == sentence)
        })
        array_reshape(x, c(1, dim(x)))
    }
    
    # the inital sentence is from the text
    start_index <- sample(1:(length(text) - max_length), size = 1)
    sentence <- text[start_index:(start_index + max_length - 1)]
    generated <- ""
    
    # while we still need characters for the phrase
    for(i in 1:(max_length * 20)){
        
        sentence_data <- convert_sentence_to_data(sentence, chars)
        
        # get the predictions for each next character
        preds <- predict(model, sentence_data)
        
        # choose the character
        next_char <- choose_next_char(preds, chars, diversity)
        
        # add it to the text and continue
        generated <- str_c(generated, next_char, collapse = " ")
        sentence <- c(sentence[-1], next_char)
    }
    
    generated
}
```

```{r}
iterate_model <- function(model, text, chars, max_length, 
                          diversity, vectors, iterations){
    for(iteration in 1:iterations){
        
        message(sprintf("iteration: %02d ---------------\n\n", iteration))
        
        fit_model(model, vectors)
        
        for(diversity in c(0.2, 0.5, 1)){
            
            message(sprintf("diversity: %f ---------------\n\n", diversity))
            
            current_phrase <- 1:10 %>% 
                map_chr(function(x) generate_phrase(model,
                                                    text,
                                                    chars,
                                                    max_length, 
                                                    diversity))
            
            message(current_phrase, sep="\n")
            message("\n\n")
            
        }
    }
    NULL
}
```

```{r}
model <- create_model(chars, max_length)
```

```{r}
iterate_model(model, text, chars, max_length, diversity, vectors, 40)
```

```{r}
#result <- data_frame(diversity = rep(c(0.4, 0.5, 0.8), 17)) %>%
          data_frame(diversity = rep(c(.5, .75, 1), 5)) %>% 
    mutate(phrase = map_chr(diversity,
                            ~ generate_phrase(model, text, chars, max_length, .x))) %>%
    arrange(diversity) -> result

result %>% mutate(phrase = gsub('([[:upper:]])', ' \\1', phrase)) %>%
    sample_n(10) %>%
    arrange(diversity) %>%
    kableExtra::kable()

#%>% mutate(ab = gsub('([[:upper:]])', ' \\1', a))

```