From 94989636ad1a05955744bc374fd51fe57ff21f2f Mon Sep 17 00:00:00 2001
From: Alison Appling <aappling@usgs.gov>
Date: Wed, 19 Feb 2020 13:56:27 -0500
Subject: [PATCH 1/3] start on shapefile export and site_ndates map

---
 5_data_munge.yml                    | 20 +++++++++++-
 5_data_munge/src/munge_data_files.R | 48 ++++++++++++++++++++++++++++-
 2 files changed, 66 insertions(+), 2 deletions(-)

diff --git a/5_data_munge.yml b/5_data_munge.yml
index f621ec6..72d1434 100644
--- a/5_data_munge.yml
+++ b/5_data_munge.yml
@@ -3,6 +3,10 @@ target_default: 5_data_munge
 packages:
   - scipiper
   - dplyr
+  - sf
+  - ggplot2
+  - rnaturalearth
+  - rnaturalearthhires
   
 sources:
   - 5_data_munge/src/munge_data_files.R
@@ -83,4 +87,18 @@ targets:
       wqp_sites_ind = '1_wqp_pull/inout/wqp_inventory.feather.ind')
       
   5_data_munge/out/all_sites.rds:
-    command: gd_get('5_data_munge/out/all_sites.rds.ind')
\ No newline at end of file
+    command: gd_get('5_data_munge/out/all_sites.rds.ind')
+    
+  5_data_munge/out/sites_shp/sites.shp.ind:
+    command: make_sites_shp(
+      out_ind = target_name,
+      all_sites_ind = '5_data_munge/out/all_sites.rds.ind',
+      all_temps_ind = '5_data_munge/out/daily_temperatures.rds.ind')
+  
+  5_data_munge/out/sites_shp/sites.shp:
+    command: gd_get('5_data_munge/out/sites_shp/sites.shp')
+      
+  5_data_munge/out/sites_ndates.png.ind:
+    command: plot_sites_ndates(
+      out_ind = target_name,
+      sites_shp_ind = '5_data_munge/out/sites_shp/sites.shp.ind')
diff --git a/5_data_munge/src/munge_data_files.R b/5_data_munge/src/munge_data_files.R
index 75d3920..8ad9738 100644
--- a/5_data_munge/src/munge_data_files.R
+++ b/5_data_munge/src/munge_data_files.R
@@ -171,4 +171,50 @@ combine_all_sites <- function(nwis_dv_sites_ind, nwis_uv_sites_ind, wqp_sites_in
   
   saveRDS(all_sites, as_data_file(out_ind))
   gd_put(out_ind, as_data_file(out_ind))
-}
\ No newline at end of file
+}
+
+make_sites_shp <- function(all_sites_ind, all_temps_ind, out_ind) {
+  sites <- readRDS(gd_get(all_sites_ind)) # replace gd_get with sc_retrieve when pipeline is shored up
+  temps <- readRDS(gd_get(all_temps_ind)) # replace gd_get with sc_retrieve when pipeline is shored up
+  
+  temp_summary <- temps %>%
+    filter(!is.na(date), !is.na(n_obs)) %>%
+    group_by(site_id) %>%
+    summarize(n_dates = length(unique(date)),
+              n_obs = sum(n_obs, na.rm=TRUE), # why are some n_obs=NA?
+              sources=paste(sort(unique(source)), collapse=','))
+  
+  sites_sf <- sites %>%
+    left_join(temp_summary, by='site_id') %>%
+    filter(!is.na(latitude), !is.na(longitude)) %>%
+    sf::st_as_sf(coords=c('longitude','latitude'), crs=sf::st_crs('+proj=longlat +datum=WGS84'))
+  
+  if(!dir.exists(dirname(out_ind))) dir.create(dirname(out_ind))
+  sf::st_write(sites_sf, scipiper::as_data_file(out_ind), layer='sites', delete_layer=TRUE)
+  scipiper::sc_indicate(out_ind, data_file=scipiper::as_data_file(out_ind))
+  ## TO DO: gd_put instead of just indicate
+}
+
+plot_sites_ndates <- function(sites_shp_ind, out_ind) {
+  sites_sf <- sf::st_read(scipiper::as_data_file(sites_shp_ind)) # replace as_data_file with sc_retrieve when practical
+  
+  library(rnaturalearth)
+  USA <- ne_countries() %>%
+    sf::st_as_sf() %>%
+    filter(geounit == "United States of America") # to include PR, use sovereignt instead of geounit
+  
+  sites_usa <- sites_sf %>%
+    filter(!is.na(n_dates), !is.na(n_obs)) %>%
+    sample_n(1000) %>%
+    sf::st_intersection(USA) # takes a few minutes!
+  
+  g <- sites_usa %>%
+    ggplot() +
+    geom_sf(data=USA, fill=NA) +
+    geom_sf(aes(color=n_dates), size=0.2) +
+    scale_color_gradient(trans='log10', low = "#56B1F7", high = "#132B43") +
+    theme_bw()
+  
+  ggsave(scipiper::as_data_file(out_ind), plot=g, width=10, height=8)
+  ## TO DO: gd_put instead of just indicate
+}

From 8c37850bc90b562f04eccd8d3a5b8d4ac7db26a9 Mon Sep 17 00:00:00 2001
From: Alison Appling <aappling@usgs.gov>
Date: Wed, 19 Feb 2020 14:01:16 -0500
Subject: [PATCH 2/3] png retrieval target

---
 5_data_munge.yml | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/5_data_munge.yml b/5_data_munge.yml
index 72d1434..d7d4928 100644
--- a/5_data_munge.yml
+++ b/5_data_munge.yml
@@ -102,3 +102,6 @@ targets:
     command: plot_sites_ndates(
       out_ind = target_name,
       sites_shp_ind = '5_data_munge/out/sites_shp/sites.shp.ind')
+      
+  5_data_munge/out/sites_ndates.png:
+    command: gd_get('5_data_munge/out/sites_ndates.png.ind')

From ce03590aff131beda610a03d2747ec8c69805815 Mon Sep 17 00:00:00 2001
From: Alison Appling <aappling@usgs.gov>
Date: Wed, 19 Feb 2020 14:15:32 -0500
Subject: [PATCH 3/3] include PR and more sites even for this prelim PR

---
 5_data_munge/src/munge_data_files.R | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/5_data_munge/src/munge_data_files.R b/5_data_munge/src/munge_data_files.R
index 8ad9738..773ae47 100644
--- a/5_data_munge/src/munge_data_files.R
+++ b/5_data_munge/src/munge_data_files.R
@@ -201,13 +201,13 @@ plot_sites_ndates <- function(sites_shp_ind, out_ind) {
   library(rnaturalearth)
   USA <- ne_countries() %>%
     sf::st_as_sf() %>%
-    filter(geounit == "United States of America") # to include PR, use sovereignt instead of geounit
+    filter(sovereignt == "United States of America") # to exclude PR, use geounit instead of sovereignt
   
   sites_usa <- sites_sf %>%
     filter(!is.na(n_dates), !is.na(n_obs)) %>%
-    sample_n(1000) %>%
+    sample_n(10000) %>%
     sf::st_intersection(USA) # takes a few minutes!
-  
+
   g <- sites_usa %>%
     ggplot() +
     geom_sf(data=USA, fill=NA) +