Closed mfpa231 closed 1 year ago
Hello @mfpa231
Thank you for opening the issue and sharing your session info.
I am not sure what it could be. Maybe the function fails to download and get correctly the mapping data files. Could you please run this chuck of code to see if files are accessible with this R code (used in the bfs_get_base_maps()
) on your machine:
library(BFS)
library(rappdirs) # install.packages("rappdirs")
library(zip) # install.packages("zip")
# get base map files if not present in cache folder
dir <- rappdirs::user_data_dir(appname = "BFS")
path_base_map <- paste0(dir, "/base_map_", "24025646")
dir.exists(path_base_map)
dir.create(path_base_map, recursive = TRUE, showWarnings = FALSE)
BFS::bfs_download_asset(
number_asset = "24025646",
destfile = paste0(path_base_map, ".zip"))
# unzip
zip::unzip(zipfile = paste0(path_base_map, ".zip"), exdir = path_base_map)
#list all files
files_all <- list.files(path_base_map, recursive = TRUE, full.names = TRUE)
# it should show 10 files
files_all[1:10]
Thank you for your response.
I tried 5 times and each time I got the message 'R session aborted'
Strange. Please run each line of R code and let me know at which line you get the message "R session aborted"?
The code seems to work until this line: zip::unzip(zipfile = paste0(path_base_map, ".zip"), exdir = path_base_map)
library(BFS)
library(rappdirs) # install.packages("rappdirs")
library(zip) # install.packages("zip")
#>
#> Attaching package: 'zip'
#> The following objects are masked from 'package:utils':
#>
#> unzip, zip
# get base map files if not present in cache folder
dir <- rappdirs::user_data_dir(appname = "BFS")
path_base_map <- paste0(dir, "/base_map_", "24025646")
dir.exists(path_base_map)
#> [1] TRUE
dir.create(path_base_map, recursive = TRUE, showWarnings = FALSE)
BFS::bfs_download_asset(
number_asset = "24025646",
destfile = paste0(path_base_map, ".zip"))
#> [1] "/home/philipp/.local/share/BFS/base_map_24025646.zip"
# unzip
zip::unzip(zipfile = paste0(path_base_map, ".zip"), exdir = path_base_map)
#list all files
files_all <- list.files(path_base_map, recursive = TRUE, full.names = TRUE)
# it should show 10 files
files_all[1:10]
#> [1] "/home/philipp/.local/share/BFS/base_map_24025646/2023_GEOM_TK/00_TOPO/K4_flusyyyymmdd/k4flusyyyymmdd11_ch2007.cpg"
#> [2] "/home/philipp/.local/share/BFS/base_map_24025646/2023_GEOM_TK/00_TOPO/K4_flusyyyymmdd/k4flusyyyymmdd11_ch2007.dbf"
#> [3] "/home/philipp/.local/share/BFS/base_map_24025646/2023_GEOM_TK/00_TOPO/K4_flusyyyymmdd/k4flusyyyymmdd11_ch2007.prj"
#> [4] "/home/philipp/.local/share/BFS/base_map_24025646/2023_GEOM_TK/00_TOPO/K4_flusyyyymmdd/k4flusyyyymmdd11_ch2007.shp"
#> [5] "/home/philipp/.local/share/BFS/base_map_24025646/2023_GEOM_TK/00_TOPO/K4_flusyyyymmdd/k4flusyyyymmdd11_ch2007.shx"
#> [6] "/home/philipp/.local/share/BFS/base_map_24025646/2023_GEOM_TK/00_TOPO/K4_flusyyyymmdd/k4flusyyyymmdd22_ch2007.cpg"
#> [7] "/home/philipp/.local/share/BFS/base_map_24025646/2023_GEOM_TK/00_TOPO/K4_flusyyyymmdd/k4flusyyyymmdd22_ch2007.dbf"
#> [8] "/home/philipp/.local/share/BFS/base_map_24025646/2023_GEOM_TK/00_TOPO/K4_flusyyyymmdd/k4flusyyyymmdd22_ch2007.prj"
#> [9] "/home/philipp/.local/share/BFS/base_map_24025646/2023_GEOM_TK/00_TOPO/K4_flusyyyymmdd/k4flusyyyymmdd22_ch2007.shp"
#> [10] "/home/philipp/.local/share/BFS/base_map_24025646/2023_GEOM_TK/00_TOPO/K4_flusyyyymmdd/k4flusyyyymmdd22_ch2007.shx"
Created on 2023-09-13 with reprex v2.0.2
switzerland_sf <- BFS::bfs_get_base_maps(geom = "suis")
#> Error in nchar(dsn): invalid multibyte string, element 1
sessioninfo::session_info()
#> ─ Session info ───────────────────────────────────────────────────────────────
#> setting value
#> version R version 4.1.3 (2022-03-10)
#> os Ubuntu 22.04.3 LTS
#> system x86_64, linux-gnu
#> ui X11
#> language
#> collate en_US.UTF-8
#> ctype en_US.UTF-8
#> tz Europe/Zurich
#> date 2023-09-13
#> pandoc 2.9.2.1 @ /usr/bin/ (via rmarkdown)
#>
#> ─ Packages ───────────────────────────────────────────────────────────────────
#> package * version date (UTC) lib source
#> anytime 0.3.9 2020-08-27 [1] CRAN (R 4.1.3)
#> BFS 0.5.4 2023-09-13 [1] Github (lgnbhl/BFS@902c825)
#> class 7.3-20 2022-01-16 [2] CRAN (R 4.1.3)
#> classInt 0.4-10 2023-09-05 [1] CRAN (R 4.1.3)
#> cli 3.6.1 2023-03-23 [1] CRAN (R 4.1.3)
#> curl 5.0.2 2023-08-14 [1] CRAN (R 4.1.3)
#> DBI 1.1.3 2022-06-18 [1] RSPM (R 4.1.0)
#> digest 0.6.33 2023-07-07 [1] CRAN (R 4.1.3)
#> dplyr 1.1.3 2023-09-03 [1] CRAN (R 4.1.3)
#> e1071 1.7-13 2023-02-01 [1] CRAN (R 4.1.3)
#> evaluate 0.21 2023-05-05 [1] CRAN (R 4.1.3)
#> fansi 1.0.4 2023-01-22 [1] CRAN (R 4.1.3)
#> fastmap 1.1.1 2023-02-24 [1] CRAN (R 4.1.3)
#> fs 1.6.3 2023-07-20 [1] CRAN (R 4.1.3)
#> generics 0.1.3 2022-07-05 [1] CRAN (R 4.1.3)
#> glue 1.6.2 2022-02-24 [1] CRAN (R 4.1.2)
#> htmltools 0.5.6 2023-08-10 [1] CRAN (R 4.1.3)
#> httr 1.4.7 2023-08-15 [1] CRAN (R 4.1.3)
#> httr2 0.2.3 2023-05-08 [1] CRAN (R 4.1.3)
#> janitor 2.2.0 2023-02-02 [1] CRAN (R 4.1.3)
#> jsonlite 1.8.7 2023-06-29 [1] CRAN (R 4.1.3)
#> KernSmooth 2.23-20 2021-05-03 [2] CRAN (R 4.1.3)
#> knitr 1.43 2023-05-25 [1] CRAN (R 4.1.3)
#> lifecycle 1.0.3 2022-10-07 [1] CRAN (R 4.1.3)
#> lubridate 1.9.2 2023-02-10 [1] CRAN (R 4.1.3)
#> magrittr 2.0.3 2022-03-30 [1] CRAN (R 4.1.3)
#> pillar 1.9.0 2023-03-22 [1] CRAN (R 4.1.3)
#> pkgconfig 2.0.3 2019-09-22 [1] CRAN (R 4.1.2)
#> proxy 0.4-27 2022-06-09 [1] RSPM (R 4.1.0)
#> purrr 1.0.2 2023-08-10 [1] CRAN (R 4.1.3)
#> R.cache 0.16.0 2022-07-21 [1] CRAN (R 4.1.3)
#> R.methodsS3 1.8.2 2022-06-13 [1] CRAN (R 4.1.3)
#> R.oo 1.25.0 2022-06-12 [1] CRAN (R 4.1.3)
#> R.utils 2.12.2 2022-11-11 [1] CRAN (R 4.1.3)
#> R6 2.5.1 2021-08-19 [1] CRAN (R 4.1.2)
#> rappdirs 0.3.3 2021-01-31 [1] CRAN (R 4.1.2)
#> Rcpp 1.0.11 2023-07-06 [1] CRAN (R 4.1.3)
#> reprex 2.0.2 2022-08-17 [1] CRAN (R 4.1.3)
#> rlang 1.1.1 2023-04-28 [1] CRAN (R 4.1.3)
#> rmarkdown 2.24 2023-08-14 [1] CRAN (R 4.1.3)
#> sessioninfo 1.2.2 2021-12-06 [1] CRAN (R 4.1.2)
#> sf 1.0-14 2023-07-11 [1] CRAN (R 4.1.3)
#> snakecase 0.11.1 2023-08-27 [1] CRAN (R 4.1.3)
#> stringi 1.7.12 2023-01-11 [1] CRAN (R 4.1.3)
#> stringr 1.5.0 2022-12-02 [1] CRAN (R 4.1.3)
#> styler 1.10.2 2023-08-29 [1] CRAN (R 4.1.3)
#> tibble 3.2.1 2023-03-20 [1] CRAN (R 4.1.3)
#> tidyRSS 2.0.7 2023-03-05 [1] CRAN (R 4.1.3)
#> tidyselect 1.2.0 2022-10-10 [1] CRAN (R 4.1.3)
#> timechange 0.2.0 2023-01-11 [1] CRAN (R 4.1.3)
#> units 0.8-3 2023-08-10 [1] CRAN (R 4.1.3)
#> utf8 1.2.3 2023-01-31 [1] CRAN (R 4.1.3)
#> vctrs 0.6.3 2023-06-14 [1] CRAN (R 4.1.3)
#> withr 2.5.0 2022-03-03 [1] CRAN (R 4.1.3)
#> xfun 0.40 2023-08-09 [1] CRAN (R 4.1.3)
#> xml2 1.3.5 2023-07-06 [1] CRAN (R 4.1.3)
#> yaml 2.3.7 2023-01-23 [1] CRAN (R 4.1.3)
#> zip 2.3.0 2023-04-17 [1] CRAN (R 4.1.3)
#>
#> [1] /home/philipp/R/x86_64-pc-linux-gnu-library/4.1
#> [2] /opt/R/4.1.3/lib/R/library
#>
#> ──────────────────────────────────────────────────────────────────────────────
Created on 2023-09-13 with reprex v2.0.2
=> could be an encoding issue, who knows what esri allows in those shapefiles ;-) Might also be some win machine/server doing some non-UTF-8 bad things.
r$> traceback()
4: st_read.character(..., quiet = quiet, stringsAsFactors = stringsAsFactors,
as_tibble = as_tibble)
3: st_read(..., quiet = quiet, stringsAsFactors = stringsAsFactors,
as_tibble = as_tibble)
2: sf::read_sf(file_selected)
1: BFS::bfs_get_base_maps(geom = "suis")
Thank you so much @philipp-baumann !
Yes, could be an encoding issue. I will investigate that.
I have added unit tests, which confirm the bug when using bfs_get_base_maps()
on Mac and Linux: https://github.com/lgnbhl/BFS/blob/master/tests/testthat/test-bfs_get_base_maps.R
Thank you so much @philipp-baumann !
Yes, could be an encoding issue. I will investigate that.
I have added unit tests, which confirm the bug when using
bfs_get_base_maps()
on Mac and Linux: https://github.com/lgnbhl/BFS/blob/master/tests/testthat/test-bfs_get_base_maps.R
great! yeah that multibyte thing we might do some byte multiplication, but it's known shapefiles pose issues with the encoding.
For the record I tried using the options
argument when calling read_sf()
at line 86 of "bfs_get_base_maps.R" but it doesn't fix the bug on Mac and Linux:
sf::read_sf(file_selected, options = "ENCODING=UTF-8") #doesn't fix the bug
For the record I tried using the
options
argument when callingread_sf()
at line 86 of "bfs_get_base_maps.R" but it doesn't fix the bug on Mac and Linux:sf::read_sf(file_selected, options = "ENCODING=UTF-8") #doesn't fix the bug
I will check this out later today.
@lgnbhl I found the issue: it is the file path, which contains multibyte strings. When I copy the shape stuff into a clean non-multibyte path, I can read then perfectly well :-)
r$> read_sf("K4suis18480101gf_ch2007Poly.shp")
Simple feature collection with 1 feature and 2 fields
Geometry type: MULTIPOLYGON
Dimension: XY
Bounding box: xmin: 2485203 ymin: 1075283 xmax: 2834035 ymax: 1295864
Projected CRS: CH1903+ / LV95
# A tibble: 1 × 3
id name geometry
<dbl> <chr> <MULTIPOLYGON [m]>
1 1 Schweiz/Suisse/Svizzera/Svizra (((2689363 1278165, 2689175 1278875, 2687891 1280259, 26876…
r$> dir()
[1] "_pkgdown.yml" "BFS.Rproj"
[3] "CODE_OF_CONDUCT.md" "data"
[5] "data-raw" "DESCRIPTION"
[7] "docs" "K4suis18480101gf_ch2007Poly.CPG"
[9] "K4suis18480101gf_ch2007Poly.dbf" "K4suis18480101gf_ch2007Poly.prj"
[11] "K4suis18480101gf_ch2007Poly.qpj" "K4suis18480101gf_ch2007Poly.shp"
[13] "K4suis18480101gf_ch2007Poly.shp.xml" "K4suis18480101gf_ch2007Poly.shx"
[15] "LICENSE" "LICENSE.md"
[17] "man" "NAMESPACE"
[19] "NEWS.md" "pkgdown"
[21] "R" "README.md"
[23] "README.Rmd" "tests"
r$>
So we can blame Windows, as almost always.
=> regarding a fix, we need some R internal conversion of the dir tree. Cheers
I just pushed a fix by adding argument useBytes = TRUE in grep()
and encoding the selected file as "latin1" using Encoding
: https://github.com/lgnbhl/BFS/commit/203b677caacb2ba3c6e99e44f016c3f07114a8a0
This change fixes the bug for Linux. However it seems that it doesn't fix the bug when using Mac: https://github.com/lgnbhl/BFS/actions/runs/6199046399
I just pushed a fix by adding argument useBytes = TRUE in
grep()
and encoding the selected file as "latin1" usingEncoding
: 203b677This change fixes the bug for Linux. However it seems that it doesn't fix the bug when using Mac: https://github.com/lgnbhl/BFS/actions/runs/6199046399
cool, thanks! but interesting it does not work for MacOS.
The macOS runners reports a segfault:
> test_check("BFS")
*** caught segfault ***
address 0xffffffff00000000, cause 'memory not mapped'
Traceback:
1: zip::unzip(zipfile = paste0(path_base_map, ".zip"), exdir = path_base_map)
2: bfs_get_base_maps(geom = "suis")
@lgnbhl maybe I have a weird theory, but is {rappdirs} on MacOS GH runner saving into a path it should not, or respectively is not authorized to do so? hard to believe it is a {zip} bug on MacOS.
@philipp-baumann thank you so much for exploring a fix regarding the unzip()
function and {rappdirs}!
Given my limited time this weekend I explored the possibility that the issue on MacOS comes from the use of grep()
and list.files()
. I found two Stackoverflow discussions that lead me to add "ignore.case = TRUE" in all grep()
functions and using fs::dir_ls()
instead of list.files()
:
grep()
: https://stackoverflow.com/a/64651758list.files()
: https://stackoverflow.com/a/69154687Unfortunately the issue on MacOS remains. And with these two changes there is also an issue on ubuntu-latest (oldrel-1). I will continue exploring possible fixes...
Thanks again for sharing your testing regarding the unzip()
function and {rappdirs} on MacOS!
Thank you both for your time!
@philipp-baumann @mfpa231
I think I fixed the issue! 🎉
My solution is to add "junkpaths = TRUE" when calling unzip()
(equivalent to unzip -j
), so all the files are extracted in the base folder, therefore avoiding to deal with subfolder name encoding problems. The use of {fs} may also have solved file path issues on MacOS (see https://github.com/lgnbhl/BFS/issues/12#issuecomment-1722297591).
The unit tests are passed on MacOS: https://github.com/lgnbhl/BFS/actions/workflows/R-CMD-check.yaml
@mfpa231 @philipp-baumann could you please let me know if the function works on your side now with BFS version 0.5.5?
devtools::install_github("lgnbhl/BFS") # new BFS version 0.5.5
switzerland_sf <- bfs_get_base_maps(geom = "suis") # should work now
@philipp-baumann thank you so much for you time and your very valuable contributions!
I added your name as a Contributor. Is is okay for you to be named, as I will soon push this new BFS version on CRAN? https://github.com/lgnbhl/BFS/commit/e6abad2499353a112acb7ab91cabefdb9b61437f
@philipp-baumann @mfpa231
I think I fixed the issue! 🎉
My solution is to add "junkpaths = TRUE" when calling
unzip()
(equivalent tounzip -j
), so all the files are extracted in the base folder, therefore avoiding to deal with subfolder name encoding problems. The use of {fs} may also have solved file path issues on MacOS (see #12 (comment)).The unit tests are passed on MacOS: https://github.com/lgnbhl/BFS/actions/workflows/R-CMD-check.yaml
@mfpa231 @philipp-baumann could you please let me know if the function works on your side now with BFS version 0.5.5?
devtools::install_github("lgnbhl/BFS") # new BFS version 0.5.5 switzerland_sf <- bfs_get_base_maps(geom = "suis") # should work now
awesome, thanks @lgnbhl for digging deeper and resoving :-) Works for me on Ubuntu 22.04 like in the CI.
@philipp-baumann thank you so much for you time and your very valuable contributions!
I added your name as a Contributor. Is is okay for you to be named, as I will soon push this new BFS version on CRAN? e6abad2
no worries. sure you can list me as a contributor:
person("Philipp", "Baumann", , "baumann-philipp@protonmail.com", role = "ctb",
comment = c(ORCID = "0000-0002-3194-8975"))
The fix has been push to CRAN
Hello, Thank you for this great tool!
I am unable to get the Maps part to work.
For example when running : switzerland_sf <- bfs_get_base_maps(geom = "suis") I get: Error in if (nchar(dsn) < 1) stop("dsn must point to a source, not an empty string.", : missing value where TRUE/FALSE needed
This prevents the rest of the code in your example from working. Not sure if I'm missing something.
Thanks, MA
Thank for you quick reply!
Session info ─────────────────────────────────────────────────────────── setting value version R version 4.3.1 (2023-06-16) os macOS Monterey 12.6.7 system x86_64, darwin20 ui RStudio language (EN) collate en_US.UTF-8 ctype en_US.UTF-8 tz Europe/Zurich date 2023-09-13 rstudio 2023.06.0+421 Mountain Hydrangea (desktop) pandoc NA
─ Packages ─────────────────────────────────────────────────────────────── package version date (UTC) lib source anytime 0.3.9 2020-08-27 [1] CRAN (R 4.3.0) backports 1.4.1 2021-12-13 [1] CRAN (R 4.3.0) BFS 0.5.3 2023-08-19 [1] CRAN (R 4.3.0) checkmate 2.2.0 2023-04-27 [1] CRAN (R 4.3.0) class 7.3-22 2023-05-03 [1] CRAN (R 4.3.1) classInt 0.4-9 2023-02-28 [1] CRAN (R 4.3.0) cli 3.6.1 2023-03-23 [1] CRAN (R 4.3.0) colorspace 2.1-0 2023-01-23 [1] CRAN (R 4.3.0) crayon 1.5.2 2022-09-29 [1] CRAN (R 4.3.0) curl 5.0.1 2023-06-07 [1] CRAN (R 4.3.0) DBI 1.1.3 2022-06-18 [1] CRAN (R 4.3.0) dplyr 1.1.2 2023-04-20 [1] CRAN (R 4.3.0) e1071 1.7-13 2023-02-01 [1] CRAN (R 4.3.0) fansi 1.0.4 2023-01-22 [1] CRAN (R 4.3.0) generics 0.1.3 2022-07-05 [1] CRAN (R 4.3.0) ggplot2 3.4.2 2023-04-03 [1] CRAN (R 4.3.0) glue 1.6.2 2022-02-24 [1] CRAN (R 4.3.0) gtable 0.3.3 2023-03-21 [1] CRAN (R 4.3.0) httr 1.4.6 2023-05-08 [1] CRAN (R 4.3.0) httr2 0.2.3 2023-05-08 [1] CRAN (R 4.3.0) janitor 2.2.0 2023-02-02 [1] CRAN (R 4.3.0) jsonlite 1.8.5 2023-06-05 [1] CRAN (R 4.3.0) KernSmooth 2.23-21 2023-05-03 [1] CRAN (R 4.3.1) lifecycle 1.0.3 2022-10-07 [1] CRAN (R 4.3.0) lubridate 1.9.2 2023-02-10 [1] CRAN (R 4.3.0) magrittr 2.0.3 2022-03-30 [1] CRAN (R 4.3.0) munsell 0.5.0 2018-06-12 [1] CRAN (R 4.3.0) pillar 1.9.0 2023-03-22 [1] CRAN (R 4.3.0) pkgconfig 2.0.3 2019-09-22 [1] CRAN (R 4.3.0) plyr 1.8.8 2022-11-11 [1] CRAN (R 4.3.0) proxy 0.4-27 2022-06-09 [1] CRAN (R 4.3.0) purrr 1.0.1 2023-01-10 [1] CRAN (R 4.3.0) pxR 0.42.7 2022-11-23 [1] CRAN (R 4.3.0) pxweb 0.16.2 2022-10-31 [1] CRAN (R 4.3.0) R6 2.5.1 2021-08-19 [1] CRAN (R 4.3.0) rappdirs 0.3.3 2021-01-31 [1] CRAN (R 4.3.0) Rcpp 1.0.10 2023-01-22 [1] CRAN (R 4.3.0) reshape2 1.4.4 2020-04-09 [1] CRAN (R 4.3.0) RJSONIO 1.3-1.8 2023-01-31 [1] CRAN (R 4.3.0) rlang 1.1.1 2023-04-28 [1] CRAN (R 4.3.0) rstac 0.9.2-4 2023-06-14 [1] CRAN (R 4.3.0) rstudioapi 0.14 2022-08-22 [1] CRAN (R 4.3.0) rvest 1.0.3 2022-08-19 [1] CRAN (R 4.3.0) scales 1.2.1 2022-08-20 [1] CRAN (R 4.3.0) selectr 0.4-2 2019-11-20 [1] CRAN (R 4.3.0) sessioninfo 1.2.2 2021-12-06 [1] CRAN (R 4.3.0) sf 1.0-14 2023-07-11 [1] CRAN (R 4.3.0) snakecase 0.11.1 2023-08-27 [1] CRAN (R 4.3.0) stringi 1.7.12 2023-01-11 [1] CRAN (R 4.3.0) stringr 1.5.0 2022-12-02 [1] CRAN (R 4.3.0) tibble 3.2.1 2023-03-20 [1] CRAN (R 4.3.0) tidyRSS 2.0.7 2023-03-05 [1] CRAN (R 4.3.0) tidyselect 1.2.0 2022-10-10 [1] CRAN (R 4.3.0) timechange 0.2.0 2023-01-11 [1] CRAN (R 4.3.0) units 0.8-3 2023-08-10 [1] CRAN (R 4.3.0) utf8 1.2.3 2023-01-31 [1] CRAN (R 4.3.0) vctrs 0.6.3 2023-06-14 [1] CRAN (R 4.3.0) withr 2.5.0 2022-03-03 [1] CRAN (R 4.3.0) xml2 1.3.4 2023-04-27 [1] CRAN (R 4.3.0) zip 2.3.0 2023-04-17 [1] CRAN (R 4.3.0)
[1] /Library/Frameworks/R.framework/Versions/4.3-x86_64/Resources/library
─────────