class: center, middle, inverse, title-slide # Reproducible workflows with the <a href="https://github.com/ropensci/drake">drake</a> R package ## make for R ### Kirill Müller ### 2019-04-14 --- # Why *drake*? 1. too many slow R scripts? 1. *rmarkdown* code chunks too heavy? &nbsp; .right[ # Save time and ensure reproducibility! ## data analysis → set of data transformations ] --- <div id="htmlwidget-ec897849c5c50a6f7846" style="width:100%;height:504px;" class="visNetwork html-widget"></div> <script type="application/json" data-for="htmlwidget-ec897849c5c50a6f7846">{"x":{"nodes":{"id":["data","fit","hist","p-OJQXOLLEMF2GCLTYNRZXQ","p-OJSXA33SOQXHAZDG","raw_data","report"],"imported":[false,false,false,true,false,false,false],"label":["data","fit","hist","file raw-data.xlsx","file report.pdf","raw_data","report"],"status":["outdated","outdated","outdated","imported","outdated","outdated","outdated"],"type":["object","object","object","file","file","object","object"],"font.size":[20,20,20,20,20,20,20],"color":["#000000","#000000","#000000","#1874CD","#000000","#000000","#000000"],"shape":["dot","dot","dot","square","square","dot","dot"],"level":[3,4,4,1,6,2,5],"x":[0,-1,1,0,0,0,0],"y":[-0.2,0.2,0.2,-1,1,-0.6,0.6]},"edges":{"from":["p-OJQXOLLEMF2GCLTYNRZXQ","raw_data","data","data","fit","hist","report"],"to":["raw_data","data","fit","hist","report","report","p-OJSXA33SOQXHAZDG"],"arrows":["to","to","to","to","to","to","to"],"smooth":[true,true,true,true,true,true,true]},"nodesToDataframe":true,"edgesToDataframe":true,"options":{"width":"100%","height":"100%","nodes":{"shape":"dot","physics":false},"manipulation":{"enabled":false},"layout":{"hierarchical":{"enabled":true,"direction":"LR"}},"edges":{"smooth":false},"physics":{"stabilization":false},"interaction":{"navigationButtons":true}},"groups":null,"width":"100%","height":null,"idselection":{"enabled":false,"style":"width: 150px; height: 26px","useLabels":true,"main":"Select by id"},"byselection":{"enabled":false,"style":"width: 150px; height: 26px","multiple":false,"hideColor":"rgba(200,200,200,0.5)"},"main":{"text":"Dependency graph","style":"font-family:Georgia, Times New Roman, Times, serif;font-weight:bold;font-size:20px;text-align:center;"},"submain":null,"footer":null,"background":"rgba(0, 0, 0, 0)","highlight":{"enabled":false,"hoverNearest":false,"degree":1,"algorithm":"all","hideColor":"rgba(200,200,200,0.5)","labelOnly":true},"collapse":{"enabled":true,"fit":false,"resetHighlight":true,"clusterOptions":null,"keepCoord":true,"labelSuffix":"(cluster)"},"igraphlayout":{"type":"square"},"tooltipStay":300,"tooltipStyle":"position: fixed;visibility:hidden;padding: 5px;white-space: nowrap;font-family: verdana;font-size:14px;font-color:#000000;background-color: #f5f4ed;-moz-border-radius: 3px;-webkit-border-radius: 3px;border-radius: 3px;border: 1px solid #808074;box-shadow: 3px 3px 10px rgba(0, 0, 0, 0.2);"},"evals":[],"jsHooks":[]}</script> --- ```r library(drake) library(tidyverse) create_plot <- function(data) { ggplot(data, aes(x = Petal.Width, fill = Species)) + geom_histogram() } plan <- drake_plan( raw_data = readxl::read_xlsx(file_in("raw-data.xlsx")), data = raw_data %>% mutate(Species = forcats::fct_inorder(Species)), hist = create_plot(data), fit = lm(Sepal.Width ~ Petal.Width + Species, data), report = rmarkdown::render( knitr_in("report.Rmd"), output_file = file_out("report.pdf"), quiet = TRUE ) ) ``` --- ```r plan ## # A tibble: 5 x 2 ## target command ## <chr> <expr> ## 1 raw_data readxl::read_xlsx(file_in("raw-data.xlsx")) … ## 2 data raw_data %>% mutate(Species = forcats::fct_inorder(Sp… ## 3 hist create_plot(data) … ## 4 fit lm(Sepal.Width ~ Petal.Width + Species, data) … ## 5 report rmarkdown::render(knitr_in("report.Rmd"), output_file… ``` --- .pull-left[ ```r make(plan) ## target raw_data ## target data ## target fit ## target hist ## target report ``` ] .pull-right[ <img src="index_files/figure-html/make-report-1.png" width="1727" /> ] --- ```r summary(readd(fit)) ## ## Call: ## lm(formula = Sepal.Width ~ Petal.Width + Species, data = data) ## ## Residuals: ## Min 1Q Median 3Q Max ## -1.17017 -0.19105 0.00793 0.19173 0.85172 ## ## Coefficients: ## Estimate Std. Error t value Pr(>|t|) ## (Intercept) 3.23587 0.05194 62.295 < 2e-16 *** ## Petal.Width 0.78102 0.12121 6.443 1.59e-09 *** ## Speciesversicolor -1.50150 0.14407 -10.422 < 2e-16 *** ## Speciesvirginica -1.84421 0.22399 -8.234 9.35e-14 *** ## --- ## Signif. codes: 0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1 ## ## Residual standard error: 0.3008 on 146 degrees of freedom ## Multiple R-squared: 0.5335, Adjusted R-squared: 0.5239 ## F-statistic: 55.65 on 3 and 146 DF, p-value: < 2.2e-16 ``` --- ```r loadd(data, hist) data ## # A tibble: 150 x 6 ## index Sepal.Length Sepal.Width Petal.Length Petal.Width ## <dbl> <dbl> <dbl> <dbl> <dbl> ## 1 1 5.1 3.5 1.4 0.2 ## 2 2 4.9 3 1.4 0.2 ## 3 3 4.7 3.2 1.3 0.2 ## 4 4 4.6 3.1 1.5 0.2 ## 5 5 5 3.6 1.4 0.2 ## 6 6 5.4 3.9 1.7 0.4 ## 7 7 4.6 3.4 1.4 0.3 ## 8 8 5 3.4 1.5 0.2 ## 9 9 4.4 2.9 1.4 0.2 ## 10 10 4.9 3.1 1.5 0.1 ## # … with 140 more rows, and 1 more variable: Species <fct> ``` --- ```r create_plot <- function(data) { ggplot(data, aes(x = Petal.Width, fill = Species)) + geom_histogram() } plan <- drake_plan( raw_data = readxl::read_xlsx(file_in("raw-data.xlsx")), data = raw_data %>% mutate(Species = forcats::fct_inorder(Species)) %>% * select(Sepal.Length:Species), hist = create_plot(data), fit = lm(Sepal.Width ~ Petal.Width + Species, data), report = rmarkdown::render( knitr_in("report.Rmd"), output_file = file_out("report.pdf"), quiet = TRUE ) ) ``` --- <div id="htmlwidget-ac00066e7f6365b7f0d3" style="width:100%;height:504px;" class="visNetwork html-widget"></div> <script type="application/json" data-for="htmlwidget-ac00066e7f6365b7f0d3">{"x":{"nodes":{"id":["data","fit","hist","p-OJQXOLLEMF2GCLTYNRZXQ","p-OJSXA33SOQXHAZDG","raw_data","report"],"imported":[false,false,false,true,false,false,false],"label":["data","fit","hist","file raw-data.xlsx","file report.pdf","raw_data","report"],"status":["outdated","outdated","outdated","imported","outdated","up to date","outdated"],"type":["object","object","object","file","file","object","object"],"font.size":[20,20,20,20,20,20,20],"color":["#000000","#000000","#000000","#1874CD","#000000","#228B22","#000000"],"shape":["dot","dot","dot","square","square","dot","dot"],"level":[3,4,4,1,6,2,5],"x":[0,-1,1,0,0,0,0],"y":[-0.2,0.2,0.2,-1,1,-0.6,0.6]},"edges":{"from":["p-OJQXOLLEMF2GCLTYNRZXQ","raw_data","data","data","fit","hist","report"],"to":["raw_data","data","fit","hist","report","report","p-OJSXA33SOQXHAZDG"],"arrows":["to","to","to","to","to","to","to"],"smooth":[true,true,true,true,true,true,true]},"nodesToDataframe":true,"edgesToDataframe":true,"options":{"width":"100%","height":"100%","nodes":{"shape":"dot","physics":false},"manipulation":{"enabled":false},"layout":{"hierarchical":{"enabled":true,"direction":"LR"}},"edges":{"smooth":false},"physics":{"stabilization":false},"interaction":{"navigationButtons":true}},"groups":null,"width":"100%","height":null,"idselection":{"enabled":false,"style":"width: 150px; height: 26px","useLabels":true,"main":"Select by id"},"byselection":{"enabled":false,"style":"width: 150px; height: 26px","multiple":false,"hideColor":"rgba(200,200,200,0.5)"},"main":{"text":"Dependency graph","style":"font-family:Georgia, Times New Roman, Times, serif;font-weight:bold;font-size:20px;text-align:center;"},"submain":null,"footer":null,"background":"rgba(0, 0, 0, 0)","highlight":{"enabled":false,"hoverNearest":false,"degree":1,"algorithm":"all","hideColor":"rgba(200,200,200,0.5)","labelOnly":true},"collapse":{"enabled":true,"fit":false,"resetHighlight":true,"clusterOptions":null,"keepCoord":true,"labelSuffix":"(cluster)"},"legend":{"width":0.2,"useGroups":false,"position":"left","ncol":1,"stepX":100,"stepY":100,"zoom":true,"nodes":{"label":["Up to date","Outdated","Imported","Object","File"],"color":["#228B22","#000000","#1874CD","#888888","#888888"],"shape":["dot","dot","dot","dot","square"],"font.color":["black","black","black","black","black"],"font.size":[20,20,20,20,20],"id":[1,2,5,7,9]},"nodesToDataframe":true},"igraphlayout":{"type":"square"},"tooltipStay":300,"tooltipStyle":"position: fixed;visibility:hidden;padding: 5px;white-space: nowrap;font-family: verdana;font-size:14px;font-color:#000000;background-color: #f5f4ed;-moz-border-radius: 3px;-webkit-border-radius: 3px;border-radius: 3px;border: 1px solid #808074;box-shadow: 3px 3px 10px rgba(0, 0, 0, 0.2);"},"evals":[],"jsHooks":[]}</script> --- ```r make(plan) ## target data ## target fit ## target hist ## target report readd(data) ## # A tibble: 150 x 5 ## Sepal.Length Sepal.Width Petal.Length Petal.Width Species ## <dbl> <dbl> <dbl> <dbl> <fct> ## 1 5.1 3.5 1.4 0.2 setosa ## 2 4.9 3 1.4 0.2 setosa ## 3 4.7 3.2 1.3 0.2 setosa ## 4 4.6 3.1 1.5 0.2 setosa ## 5 5 3.6 1.4 0.2 setosa ## 6 5.4 3.9 1.7 0.4 setosa ## 7 4.6 3.4 1.4 0.3 setosa ## 8 5 3.4 1.5 0.2 setosa ## 9 4.4 2.9 1.4 0.2 setosa ## 10 4.9 3.1 1.5 0.1 setosa ## # … with 140 more rows ``` --- ```r readd(hist) ## `stat_bin()` using `bins = 30`. Pick better value with ## `binwidth`. ``` <!-- --> --- ```r create_plot <- function(data) { ggplot(data, aes(x = Petal.Width, fill = Species)) + * geom_histogram(binwidth = 0.25) + * theme_gray(20) } plan <- drake_plan( raw_data = readxl::read_xlsx(file_in("raw-data.xlsx")), data = raw_data %>% mutate(Species = forcats::fct_inorder(Species)) %>% select(Sepal.Length:Species), hist = create_plot(data), fit = lm(Sepal.Width ~ Petal.Width + Species, data), report = rmarkdown::render( knitr_in("report.Rmd"), output_file = file_out("report.pdf"), quiet = TRUE ) ) ``` --- .pull-left[ ```r make(plan) ## target hist ## target report ``` ] .pull-right[ <img src="index_files/figure-html/make-report-fixed-hist-1.png" width="1727" /> ] --- <div id="htmlwidget-a261f1c13aea9e511342" style="width:100%;height:504px;" class="visNetwork html-widget"></div> <script type="application/json" data-for="htmlwidget-a261f1c13aea9e511342">{"x":{"nodes":{"id":["create_plot","data","fit","hist","n-MZXXEY3BORZTUOTGMN2F62LON5ZGIZLS","n-OJSWCZDYNQ5DU4TFMFSF66DMON4A","n-OJWWC4TLMRXXO3R2HJZGK3TEMVZA","p-OJQXOLLEMF2GCLTYNRZXQ","p-OJSXA33SOQXFE3LE","p-OJSXA33SOQXHAZDG","raw_data","report"],"imported":[true,false,false,false,true,true,true,true,true,false,false,false],"label":["create_plot","data","fit","hist","forcats::fct_inorder","readxl::read_xlsx","rmarkdown::render","file raw-data.xlsx","file report.Rmd","file report.pdf","raw_data","report"],"status":["imported","up to date","up to date","up to date","imported","imported","imported","imported","imported","up to date","up to date","up to date"],"type":["function","object","object","object","function","function","function","file","file","file","object","object"],"font.size":[20,20,20,20,20,20,20,20,20,20,20,20],"color":["#1874CD","#228B22","#228B22","#228B22","#1874CD","#1874CD","#1874CD","#1874CD","#1874CD","#228B22","#228B22","#228B22"],"shape":["triangle","dot","dot","dot","triangle","triangle","triangle","square","square","square","dot","dot"],"level":[1,3,4,4,1,1,1,1,1,6,2,5],"x":[-1,-0.2,-0.2,-0.6,-0.6,-0.2,0.6,0.2,1,0.6,0,0.6],"y":[-1,-0.2,0.2,0.2,-1,-1,-1,-1,-1,1,-0.6,0.6]},"edges":{"from":["create_plot","n-OJSWCZDYNQ5DU4TFMFSF66DMON4A","p-OJQXOLLEMF2GCLTYNRZXQ","raw_data","n-MZXXEY3BORZTUOTGMN2F62LON5ZGIZLS","data","data","n-OJWWC4TLMRXXO3R2HJZGK3TEMVZA","fit","hist","p-OJSXA33SOQXFE3LE","report"],"to":["hist","raw_data","raw_data","data","data","fit","hist","report","report","report","report","p-OJSXA33SOQXHAZDG"],"arrows":["to","to","to","to","to","to","to","to","to","to","to","to"],"smooth":[true,true,true,true,true,true,true,true,true,true,true,true]},"nodesToDataframe":true,"edgesToDataframe":true,"options":{"width":"100%","height":"100%","nodes":{"shape":"dot","physics":false},"manipulation":{"enabled":false},"layout":{"hierarchical":{"enabled":true,"direction":"LR"}},"edges":{"smooth":false},"physics":{"stabilization":false},"interaction":{"navigationButtons":true}},"groups":null,"width":"100%","height":null,"idselection":{"enabled":false,"style":"width: 150px; height: 26px","useLabels":true,"main":"Select by id"},"byselection":{"enabled":false,"style":"width: 150px; height: 26px","multiple":false,"hideColor":"rgba(200,200,200,0.5)"},"main":{"text":"Dependency graph","style":"font-family:Georgia, Times New Roman, Times, serif;font-weight:bold;font-size:20px;text-align:center;"},"submain":null,"footer":null,"background":"rgba(0, 0, 0, 0)","highlight":{"enabled":false,"hoverNearest":false,"degree":1,"algorithm":"all","hideColor":"rgba(200,200,200,0.5)","labelOnly":true},"collapse":{"enabled":true,"fit":false,"resetHighlight":true,"clusterOptions":null,"keepCoord":true,"labelSuffix":"(cluster)"},"legend":{"width":0.2,"useGroups":false,"position":"left","ncol":1,"stepX":100,"stepY":100,"zoom":true,"nodes":{"label":["Up to date","Imported","Object","Function","File"],"color":["#228B22","#1874CD","#888888","#888888","#888888"],"shape":["dot","dot","dot","triangle","square"],"font.color":["black","black","black","black","black"],"font.size":[20,20,20,20,20],"id":[1,5,7,8,9]},"nodesToDataframe":true},"igraphlayout":{"type":"square"},"tooltipStay":300,"tooltipStyle":"position: fixed;visibility:hidden;padding: 5px;white-space: nowrap;font-family: verdana;font-size:14px;font-color:#000000;background-color: #f5f4ed;-moz-border-radius: 3px;-webkit-border-radius: 3px;border-radius: 3px;border: 1px solid #808074;box-shadow: 3px 3px 10px rgba(0, 0, 0, 0.2);"},"evals":[],"jsHooks":[]}</script> --- class: inverse, middle, center