diff --git a/.Rbuildignore b/.Rbuildignore new file mode 100644 index 0000000..0a619ea --- /dev/null +++ b/.Rbuildignore @@ -0,0 +1,4 @@ +^eurosat\.Rproj$ +^\.Rproj\.user$ +^README\.Rmd$ +^LICENSE\.md$ diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..cd67eac --- /dev/null +++ b/.gitignore @@ -0,0 +1 @@ +.Rproj.user diff --git a/DESCRIPTION b/DESCRIPTION new file mode 100644 index 0000000..a601b4f --- /dev/null +++ b/DESCRIPTION @@ -0,0 +1,18 @@ +Package: eurosat +Title: EuroSAT: Useful methods to handle EuroSAT (MS) dataset +Version: 0.0.1 +Authors@R: + person("Felipe", "Carlos", , "efelipecarlos@gmail.com", role = c("aut", "cre")) +Description: This R package simplifies the process of downloading and utilizing + the EuroSAT dataset, offering an easy-to-use interface that allows + users to focus on their analysis rather than data management tasks. +License: MIT + file LICENSE +Encoding: UTF-8 +Roxygen: list(markdown = TRUE) +RoxygenNote: 7.3.1 +Imports: + data.table, + fs +Suggests: + testthat (>= 3.0.0) +Config/testthat/edition: 3 diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..04fe0ce --- /dev/null +++ b/LICENSE @@ -0,0 +1,2 @@ +YEAR: 2024 +COPYRIGHT HOLDER: eurosat authors diff --git a/LICENSE.md b/LICENSE.md new file mode 100644 index 0000000..1bde30c --- /dev/null +++ b/LICENSE.md @@ -0,0 +1,21 @@ +# MIT License + +Copyright (c) 2024 eurosat authors + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/NAMESPACE b/NAMESPACE new file mode 100644 index 0000000..aadaf7a --- /dev/null +++ b/NAMESPACE @@ -0,0 +1,4 @@ +# Generated by roxygen2: do not edit by hand + +export(eurosat_download) +export(eurosat_index) diff --git a/R/config.R b/R/config.R new file mode 100644 index 0000000..e9d4a59 --- /dev/null +++ b/R/config.R @@ -0,0 +1,51 @@ +# +# Copyright (C) 2023 EuroSAT Package. +# +# EuroSAT Package is free software; you can redistribute it and/or modify it +# under the terms of the MIT License; see LICENSE file for more details. +# + +#' Get EuroSAT data directory. +#' @noRd +#' +#' @author Felipe Carlos, \email{efelipecarlos@gmail.com} +#' +#' @return Path to the EuroSAT data directory. +#' +.get_eurosat_data_dir <- function() { + fs::path(Sys.getenv("EUROSAT_DATA_DIR", "~/.local/share/eurosat")) +} + +#' Get EuroSAT data url +#' @noRd +#' +#' @author Felipe Carlos, \email{efelipecarlos@gmail.com} +#' +#' @return URL to EuroSAT (MS) file. +#' +.get_eurosat_data_url <- function() { + Sys.getenv("EUROSAT_DATA_URL", + "https://madm.dfki.de/files/sentinel/EuroSATallBands.zip") +} + +#' Get EuroSAT data filename. +#' @noRd +#' +#' @author Felipe Carlos, \email{efelipecarlos@gmail.com} +#' +#' @return EuroSAT (MS) filename. +#' +.get_eurosat_data_file <- function() { + fs::path_file(.get_eurosat_data_url()) +} + +#' Get EuroSAT files index path. +#' @noRd +#' +#' @author Felipe Carlos, \email{efelipecarlos@gmail.com} +#' +#' @return Path to EuroSAT files index file (csv format). +#' +.get_eurosat_index_file <- function() { + .get_eurosat_data_dir() / "index.csv" +} diff --git a/R/eurosat.R b/R/eurosat.R new file mode 100644 index 0000000..1628994 --- /dev/null +++ b/R/eurosat.R @@ -0,0 +1,177 @@ +# +# Copyright (C) 2023 EuroSAT Package. +# +# EuroSAT Package is free software; you can redistribute it and/or modify it +# under the terms of the MIT License; see LICENSE file for more details. +# + +#' List files in a EuroSAT data zip. +#' @noRd +#' +#' @author Felipe Carlos, \email{efelipecarlos@gmail.com} +#' +#' @param zipfile Zipfile path. +#' +#' @return Array containing all files in the zipfile. +#' +.eurosat_zip_list <- function(zipfile) { + utils::unzip(zipfile = zipfile, list = TRUE) +} + +#' Extract files from EuroSAT data zip. +#' @noRd +#' +#' @author Felipe Carlos, \email{efelipecarlos@gmail.com} +#' +#' @param zipfile Zipfile path. +#' @param output_dir Directory where the extracted content will be saved. +#' +#' @return Array containing all files in the zipfile. +#' +.eurosat_zip_extract <- function(zipfile, output_dir) { + base_extracted_dir <- output_dir / .eurosat_zip_list(zipfile)[1, 1] + + if (!.fs_check_dir(base_extracted_dir)) { + utils::unzip(zipfile = zipfile, exdir = output_dir) + } + + base_extracted_dir +} + +#' Generate file index. +#' @noRd +#' +#' @author Felipe Carlos, \email{efelipecarlos@gmail.com} +#' +#' @description This functions iterates over all files available in EuroSAT +#' folder and creates an index of it. It file is associated +#' with a ``class`` which represents the type of content +#' available in the image. The ``class`` is defined based on +#' the folder where the file is. For example, files available +#' in the folder `AnnualCrop` will be classified as ``AnnualCrop``. +#' +#' @param zipfile Zipfile path. +#' @param output_dir Directory where the extracted content will be saved. +#' +#' @return Array containing all files in the zipfile. +.eurosat_generate_index <- function(eurosat_base_data_dir) { + # listing all classes available + file_classes <- + as.character(base::lapply(fs::dir_ls(eurosat_base_data_dir), function(x) { + fs::path_file(x) + })) + + file_index <- base::lapply(file_classes, function(row) { + file_class_dir <- eurosat_base_data_dir / row + + file_class_data_files <- fs::dir_ls(file_class_dir) + + file_class_data_files <- as.data.frame(file_class_data_files) + file_class_data_files["class"] <- row + + + colnames(file_class_data_files) <- c("file", "class") + rownames(file_class_data_files) <- 1:nrow(file_class_data_files) + + file_class_data_files + }) + + data.table::as.data.table(do.call(rbind, file_index)) +} + + +#' Download EuroSAT MS data. +#' +#' @author Felipe Carlos, \email{efelipecarlos@gmail.com} +#' +#' @description +#' This function downloads, extract and index all +#' EuroSAT Dataset (MS). After use it, check the function +#' [eurosat::eurosat_index] to get the complete file index. +#' +#' To change where the data is stored, please, the env variable +#' ``EUROSAT_DATA_DIR`` to specify it. +#' +#' Also, do change the URL of the data downloaded, you can use +#' the ``EUROSAT_DATA_URL`` env variable. Of course, if you +#' change to a data with no EuroSAT MS structure, the code will +#' break. +#' +#' @references +#' Eurosat: A novel dataset and deep learning benchmark for land use and land +#' cover classification. Patrick Helber, Benjamin Bischke, Andreas Dengel, Damian +#' Borth. IEEE Journal of Selected Topics in Applied Earth Observations and +#' Remote Sensing, 2019. +#' +#' @note You only need to call this function once. After this, the data +#' downloaded will be cached in your system. Then, you can access +#' the data using the function [eurosat::eurosat_index]. +#' +#' @note To download the file, maybe is required to ignore SSL validation. For +#' this case, you can use the following code: +#' +#' ```r +#' base::options(download.file.method="curl", download.file.extra="-k -L") +#' ```` +#' +#' @seealso [eurosat::eurosat_index] to access the files index as ``data.frame``. +#' +#' @export +#' +#' @return [data.table::data.table] containing the index of EuroSAT files. +eurosat_download <- function() { + # creating output directory + eurosat_data_dir <- .fs_create_dir(.get_eurosat_data_dir()) + + # downloading output file + eurosat_output_file <- + eurosat_data_dir / fs::path_file(.get_eurosat_data_url()) + + if (!.fs_check_file(eurosat_output_file)) { + eurosat_data_url <- .get_eurosat_data_url() + + utils::download.file(eurosat_data_url, eurosat_output_file) + } + + # extracting files + eurosat_tif_dir <- + .eurosat_zip_extract(eurosat_output_file, eurosat_data_dir) + + # generating file index + eurosat_data_index <- .eurosat_generate_index(eurosat_tif_dir) + + # saving data index + eurosat_data_index_file <- eurosat_data_dir / "index.csv" + utils::write.csv2(eurosat_data_index, eurosat_data_index_file, row.names = FALSE) + + eurosat_data_index +} + + +#' Load EuroSAT MS data index. +#' +#' @author Felipe Carlos, \email{efelipecarlos@gmail.com} +#' +#' @description +#' This function loads the index of EuroSAT MS data files. To use +#' this function it is required to have all EuroSAT data downloaded. +#' +#' @references +#' Eurosat: A novel dataset and deep learning benchmark for land use and land +#' cover classification. Patrick Helber, Benjamin Bischke, Andreas Dengel, Damian +#' Borth. IEEE Journal of Selected Topics in Applied Earth Observations and +#' Remote Sensing, 2019. +#' +#' @note Make sure you have executed [eurosat::eurosat_download] before run this +#' function. +#' +#' @seealso [eurosat::eurosat_download] to download EuroSAT (MS) data. +#' +#' @export +#' +#' @return [data.table::data.table] containing the index of EuroSAT files. +eurosat_index <- function() { + eurosat_data_dir <- .get_eurosat_data_dir() / "index.csv" + + data.table::as.data.table(utils::read.csv(eurosat_data_dir, sep = ';')) +} diff --git a/R/fs.R b/R/fs.R new file mode 100644 index 0000000..d1d54dd --- /dev/null +++ b/R/fs.R @@ -0,0 +1,40 @@ +# +# Copyright (C) 2023 EuroSAT Package. +# +# EuroSAT Package is free software; you can redistribute it and/or modify it +# under the terms of the MIT License; see LICENSE file for more details. +# + +#' Create directory. +#' @noRd +#' +#' @author Felipe Carlos, \email{efelipecarlos@gmail.com} +#' +#' @return Path to the created directory. +.fs_create_dir <- function(directory) { + if (!fs::dir_exists(directory)) { + fs::dir_create(directory) + } + + directory +} + +#' Check if a given file exists. +#' @noRd +#' +#' @author Felipe Carlos, \email{efelipecarlos@gmail.com} +#' +#' @return Boolean indicating if the given file exists or not. +.fs_check_file <- function(file) { + fs::file_exists(file) +} + +#' Check if a given file exists. +#' @noRd +#' +#' @author Felipe Carlos, \email{efelipecarlos@gmail.com} +#' +#' @return Boolean indicating if the given dir exists or not. +.fs_check_dir <- function(dir) { + fs::dir_exists(dir) +} diff --git a/README.Rmd b/README.Rmd new file mode 100644 index 0000000..ea8f1d2 --- /dev/null +++ b/README.Rmd @@ -0,0 +1,71 @@ +--- +output: github_document +--- + + + +```{r, include = FALSE} +knitr::opts_chunk$set( + collapse = TRUE, + comment = "#>", + fig.path = "man/figures/README-", + out.width = "100%" +) +``` + +# eurosat 🛰️ + +The EuroSAT dataset is a comprehensive collection aimed at supporting the development and evaluation of machine learning models in land use and land cover classification tasks. Published in 2019 in the IEEE Journal of Selected Topics in Applied Earth Observations and Remote Sensing, it has become a crucial benchmark for researchers and practitioners in the field. + +This R package simplifies the process of downloading and utilizing the EuroSAT dataset, offering an easy-to-use interface that allows users to focus on their analysis rather than data management tasks. + +## Installation + +You can install the development version of `eurosat` like so: + +``` r +# install.packages("devtools") +devtools::install_github("M3nin0/eurosat") +``` + +## Using eurosat + +To download the EuroSAT dataset and create a local index, simply use: + +```{r, eval=FALSE} +library(eurosat) + +# Download the EuroSAT dataset and create an index +eurosat::eurosat_download() +``` + +## Accessing the Data + +After downloading, you can easily access the dataset through the provided index, which includes paths to the data files and their corresponding land use and land cover classes: + +```{r, eval=FALSE} +# Load the index into a data.table +index <- eurosat::eurosat_index() + +# View the first few rows of the index +head(index) +#> file class +#> +#> 1: path/to/AnnualCrop_1.tif AnnualCrop +#> 2: path/to/AnnualCrop_10.tif AnnualCrop +#> 3: path/to/AnnualCrop_100.tif AnnualCrop +#> 4: path/to/AnnualCrop_1000.tif AnnualCrop +#> 5: path/to/AnnualCrop_1001.tif AnnualCrop +``` + +## Extra configurations + +Sometimes you might need to disable SSL validation to download the EuroSAT dataset. This can be done by setting the appropriate download file method and options: + +```{r cars} +options(download.file.method="curl", download.file.extra="-k -L") +``` + +## References + +[1] Eurosat: A novel dataset and deep learning benchmark for land use and land cover classification. Patrick Helber, Benjamin Bischke, Andreas Dengel, Damian Borth. IEEE Journal of Selected Topics in Applied Earth Observations and Remote Sensing, 2019. diff --git a/README.md b/README.md new file mode 100644 index 0000000..31965c2 --- /dev/null +++ b/README.md @@ -0,0 +1,73 @@ + + + +# eurosat 🛰️ + +The EuroSAT dataset is a comprehensive collection aimed at supporting +the development and evaluation of machine learning models in land use +and land cover classification tasks. Published in 2019 in the IEEE +Journal of Selected Topics in Applied Earth Observations and Remote +Sensing, it has become a crucial benchmark for researchers and +practitioners in the field. + +This R package simplifies the process of downloading and utilizing the +EuroSAT dataset, offering an easy-to-use interface that allows users to +focus on their analysis rather than data management tasks. + +## Installation + +You can install the development version of `eurosat` like so: + +``` r +# install.packages("devtools") +devtools::install_github("M3nin0/eurosat") +``` + +## Using eurosat + +To download the EuroSAT dataset and create a local index, simply use: + +``` r +library(eurosat) + +# Download the EuroSAT dataset and create an index +eurosat::eurosat_download() +``` + +## Accessing the Data + +After downloading, you can easily access the dataset through the +provided index, which includes paths to the data files and their +corresponding land use and land cover classes: + +``` r +# Load the index into a data.table +index <- eurosat::eurosat_index() + +# View the first few rows of the index +head(index) +#> file class +#> +#> 1: path/to/AnnualCrop_1.tif AnnualCrop +#> 2: path/to/AnnualCrop_10.tif AnnualCrop +#> 3: path/to/AnnualCrop_100.tif AnnualCrop +#> 4: path/to/AnnualCrop_1000.tif AnnualCrop +#> 5: path/to/AnnualCrop_1001.tif AnnualCrop +``` + +## Extra configurations + +Sometimes you might need to disable SSL validation to download the +EuroSAT dataset. This can be done by setting the appropriate download +file method and options: + +``` r +options(download.file.method="curl", download.file.extra="-k -L") +``` + +## References + +\[1\] Eurosat: A novel dataset and deep learning benchmark for land use +and land cover classification. Patrick Helber, Benjamin Bischke, Andreas +Dengel, Damian Borth. IEEE Journal of Selected Topics in Applied Earth +Observations and Remote Sensing, 2019. diff --git a/eurosat.Rproj b/eurosat.Rproj new file mode 100644 index 0000000..69fafd4 --- /dev/null +++ b/eurosat.Rproj @@ -0,0 +1,22 @@ +Version: 1.0 + +RestoreWorkspace: No +SaveWorkspace: No +AlwaysSaveHistory: Default + +EnableCodeIndexing: Yes +UseSpacesForTab: Yes +NumSpacesForTab: 2 +Encoding: UTF-8 + +RnwWeave: Sweave +LaTeX: pdfLaTeX + +AutoAppendNewline: Yes +StripTrailingWhitespace: Yes +LineEndingConversion: Posix + +BuildType: Package +PackageUseDevtools: Yes +PackageInstallArgs: --no-multiarch --with-keep.source +PackageRoxygenize: rd,collate,namespace diff --git a/man/eurosat_download.Rd b/man/eurosat_download.Rd new file mode 100644 index 0000000..20d66f9 --- /dev/null +++ b/man/eurosat_download.Rd @@ -0,0 +1,47 @@ +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/eurosat.R +\name{eurosat_download} +\alias{eurosat_download} +\title{Download EuroSAT MS data.} +\usage{ +eurosat_download() +} +\value{ +\link[data.table:data.table]{data.table::data.table} containing the index of EuroSAT files. +} +\description{ +This function downloads, extract and index all +EuroSAT Dataset (MS). After use it, check the function +\link{eurosat_index} to get the complete file index. + +To change where the data is stored, please, the env variable +\code{EUROSAT_DATA_DIR} to specify it. + +Also, do change the URL of the data downloaded, you can use +the \code{EUROSAT_DATA_URL} env variable. Of course, if you +change to a data with no EuroSAT MS structure, the code will +break. +} +\note{ +You only need to call this function once. After this, the data +downloaded will be cached in your system. Then, you can access +the data using the function \link{eurosat_index}. + +To download the file, maybe is required to ignore SSL validation. For +this case, you can use the following code: + +\if{html}{\out{
}}\preformatted{base::options(download.file.method="curl", download.file.extra="-k -L") +}\if{html}{\out{
}} +} +\references{ +Eurosat: A novel dataset and deep learning benchmark for land use and land +cover classification. Patrick Helber, Benjamin Bischke, Andreas Dengel, Damian +Borth. IEEE Journal of Selected Topics in Applied Earth Observations and +Remote Sensing, 2019. +} +\seealso{ +\link{eurosat_index} to access the files index as \code{data.frame}. +} +\author{ +Felipe Carlos, \email{efelipecarlos@gmail.com} +} diff --git a/man/eurosat_index.Rd b/man/eurosat_index.Rd new file mode 100644 index 0000000..e4438af --- /dev/null +++ b/man/eurosat_index.Rd @@ -0,0 +1,31 @@ +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/eurosat.R +\name{eurosat_index} +\alias{eurosat_index} +\title{Load EuroSAT MS data index.} +\usage{ +eurosat_index() +} +\value{ +\link[data.table:data.table]{data.table::data.table} containing the index of EuroSAT files. +} +\description{ +This function loads the index of EuroSAT MS data files. To use +this function it is required to have all EuroSAT data downloaded. +} +\note{ +Make sure you have executed \link{eurosat_download} before run this +function. +} +\references{ +Eurosat: A novel dataset and deep learning benchmark for land use and land +cover classification. Patrick Helber, Benjamin Bischke, Andreas Dengel, Damian +Borth. IEEE Journal of Selected Topics in Applied Earth Observations and +Remote Sensing, 2019. +} +\seealso{ +\link{eurosat_download} to download EuroSAT (MS) data. +} +\author{ +Felipe Carlos, \email{efelipecarlos@gmail.com} +} diff --git a/tests/testthat.R b/tests/testthat.R new file mode 100644 index 0000000..7314dac --- /dev/null +++ b/tests/testthat.R @@ -0,0 +1,12 @@ +# This file is part of the standard setup for testthat. +# It is recommended that you do not modify it. +# +# Where should you do additional test configuration? +# Learn more about the roles of various files in: +# * https://r-pkgs.org/testing-design.html#sec-tests-files-overview +# * https://testthat.r-lib.org/articles/special-files.html + +library(testthat) +library(eurosat) + +test_check("eurosat") diff --git a/tests/testthat/test-config.R b/tests/testthat/test-config.R new file mode 100644 index 0000000..0cc6404 --- /dev/null +++ b/tests/testthat/test-config.R @@ -0,0 +1,17 @@ +# +# Copyright (C) 2023 EuroSAT Package. +# +# EuroSAT Package is free software; you can redistribute it and/or modify it +# under the terms of the MIT License; see LICENSE file for more details. +# + +test_that("(Internal) Get EuroSat Data Directory (default variables)", { + expect_equal(.get_eurosat_data_dir(), + fs::path("~/.local/share/eurosat")) +}) + +test_that("(Internal) Get EuroSat Data Directory (custom variables)", { + Sys.setenv("EUROSAT_DATA_DIR" = "/custom/dir") + + expect_equal(.get_eurosat_data_dir(), fs::path("/custom/dir")) +}) diff --git a/tests/testthat/test-fs.R b/tests/testthat/test-fs.R new file mode 100644 index 0000000..60fa273 --- /dev/null +++ b/tests/testthat/test-fs.R @@ -0,0 +1,31 @@ +# +# Copyright (C) 2023 EuroSAT Package. +# +# EuroSAT Package is free software; you can redistribute it and/or modify it +# under the terms of the MIT License; see LICENSE file for more details. +# + +test_that("(Internal) Create directory", { + tmpdir <- fs::path(tempdir()) + tmpdir_test <- tmpdir / "test" + + .fs_create_dir(tmpdir_test) + + expect_true(fs::dir_exists(tmpdir_test)) +}) + +test_that("(Internal) Check if a directory exists", { + tmpdir <- tempdir() + + fs::dir_create(tmpdir) + + expect_true(.fs_check_dir(tmpdir)) +}) + +test_that("(Internal) Check if a file exists", { + tmpfile <- tempfile() + + fs::file_create(tmpfile) + + expect_true(.fs_check_file(tmpfile)) +})