-
Notifications
You must be signed in to change notification settings - Fork 0
/
dylan scrape.Rmd
79 lines (55 loc) · 1.28 KB
/
dylan scrape.Rmd
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
---
title: "Untitled"
author: "Thomas Rosenthal"
date: "14/08/2021"
output: html_document
---
```{r setup, include=FALSE}
knitr::opts_chunk$set(echo = TRUE)
```
```{r}
library(rvest) # to extract text from sites
library(stringr) # for easier string manipulation
library(readr) # to read text files
library(tidytext) # for natural language processing
library(dplyr) # for easier data manipulation
library(tidyr) # to make data wide and long
library(jsonlite) # to deal with json files
```
```{r}
raw_data <- read_html("http://www.bobdylan.com/songs/")
```
```{r}
songs <- raw_data %>%
html_nodes('div [id="item-list"]') %>%
html_nodes('a') %>%
html_attr("href")
```
```{r}
URLs_generation <- tibble(raw_text = songs)
```
```{r}
URLs_generation <- URLs_generation %>% filter(str_detect(raw_text, 'songs')) %>% distinct()
```
```{r}
scrape <- function(url) {
url_html <- read_html(url)
lyrics <-
url_html %>%
html_node('div [class="article-content lyrics"]') %>%
html_text()
title <-
url_html %>%
html_node('h2') %>%
html_text()
Sys.sleep(2.5)
tibble(title = title, lyrics = lyrics)
}
```
```{r}
all_lyrics <- purrr::map_dfr(URLs_generation[["raw_text"]], scrape)
```
```{r}
library()
write_csv(all_lyrics, 'Dylan Lyrics.csv')
```