add stevenblack list support

Author: soodoku

.gitignore

@@ -13,3 +13,4 @@ docs
 *.gz
 *.gz
 rdomains_0.3.0.tar.gz
+*.csv

.lintr

@@ -28,8 +28,7 @@ Imports:
 Suggests:
     testthat,
     rmarkdown,
-    knitr (>= 1.11),
-    lintr
+    knitr (>= 1.11)
 VignetteBuilder: knitr
 License: MIT + file LICENSE
 Encoding: UTF-8

DESCRIPTION

@@ -6,10 +6,12 @@ export(dmoz_cat)
 export(get_alexa_data)
 export(get_dmoz_data)
 export(get_shalla_data)
+export(get_stevenblack_data)
 export(glm_shalla)
 export(not_news)
 export(openai_cat)
 export(shalla_cat)
+export(stevenblack_cat)
 export(uni_cat)
 export(virustotal_cat)
 importFrom(Matrix,Matrix)

NAMESPACE

@@ -18,17 +18,63 @@
 
 get_shalla_data <- function(outdir = "./", overwrite = FALSE) {
 
-  # Check if file already there
-  output_file <- paste0(outdir, "shalla_domain_category.csv")
-  if (overwrite == FALSE & file.exists(output_file)) {
-    stop("There is already a file with that name in the location.
-          Pick another name or location.")
+  # Normalize and create output directory path
+  outdir <- normalizePath(outdir, mustWork = FALSE)
+  if (!dir.exists(outdir)) {
+    tryCatch({
+      dir.create(outdir, recursive = TRUE)
+    }, error = function(e) {
+      stop("Cannot create output directory: ", outdir, "\n", 
+           "Error: ", e$message, "\n",
+           "Please check directory permissions.")
+    })
+  }
+  
+  # Use proper file path construction
+  output_file <- file.path(outdir, "shalla_domain_category.csv")
+  
+  # Check if file already exists
+  if (!overwrite && file.exists(output_file)) {
+    stop("File already exists: ", output_file, "\n",
+         "Set overwrite=TRUE to replace it or choose a different location.")
   }
 
-  tmp <- tempfile()
-  curl_download("https://raw.githubusercontent.com/themains/rdomains/master/data-raw/shallalist/accomplist/shallalist.gz", tmp)
-  gunzip(tmp, destname = output_file, overwrite = overwrite)
-  unlink(tmp, force = TRUE)
-
-  cat("Shallalist Data saved to the following destination:", outdir, "\n")
+  # Create temporary file in a writable location
+  tmp_dir <- tempdir()
+  tmp <- tempfile(tmpdir = tmp_dir, fileext = ".gz")
+  
+  tryCatch({
+    # Download file
+    cat("Downloading Shallalist data...\n")
+    curl::curl_download(
+      "https://raw.githubusercontent.com/themains/rdomains/master/data-raw/shallalist/accomplist/shallalist.gz", 
+      tmp
+    )
+    
+    # Extract to destination with proper error handling
+    R.utils::gunzip(tmp, destname = output_file, overwrite = overwrite)
+    
+    # Verify the file was created successfully
+    if (!file.exists(output_file)) {
+      stop("Failed to create output file. Please check write permissions for: ", outdir)
+    }
+    
+    cat("Shallalist data saved to:", output_file, "\n")
+    
+  }, error = function(e) {
+    # Clean up temp file on error
+    if (file.exists(tmp)) unlink(tmp, force = TRUE)
+    
+    if (grepl("permission", e$message, ignore.case = TRUE)) {
+      stop("Permission denied. Please ensure you have write access to: ", outdir, "\n",
+           "On Windows, try running R as administrator or choose a different output directory.")
+    } else {
+      stop("Error downloading or extracting Shallalist data: ", e$message)
+    }
+  })
+  
+  # Clean up temp file
+  if (file.exists(tmp)) unlink(tmp, force = TRUE)
+  
+  invisible(output_file)
 }

R/get_shalla_data.R

@@ -0,0 +1,64 @@
+#' Get Steven Black's Host List Data
+#'
+#' Downloads the latest version of Steven Black's unified hosts file.
+#' The hosts file contains domains known for serving ads, malware, and tracking.
+#'
+#' @param outdir    Optional; folder to which you want to save the file; Default is current directory
+#' @param variant   Optional; which variant to download. Options: "base", "porn", "social", "gambling", "all"
+#' @param overwrite Optional; default is FALSE. If TRUE, the file is overwritten.
+#' 
+#' @export
+#' 
+#' @references \url{https://github.com/StevenBlack/hosts}
+#' 
+#' @examples \dontrun{
+#' get_stevenblack_data()
+#' get_stevenblack_data(variant = "all")
+#' }
+
+get_stevenblack_data <- function(outdir = "./", variant = "base", overwrite = FALSE) {
+  
+  # Define available variants and their URLs
+  variants <- list(
+    base = "https://raw.githubusercontent.com/StevenBlack/hosts/master/hosts",
+    porn = "https://raw.githubusercontent.com/StevenBlack/hosts/master/alternates/porn/hosts",
+    social = "https://raw.githubusercontent.com/StevenBlack/hosts/master/alternates/social/hosts", 
+    gambling = "https://raw.githubusercontent.com/StevenBlack/hosts/master/alternates/gambling/hosts",
+    all = "https://raw.githubusercontent.com/StevenBlack/hosts/master/alternates/porn-social-gambling/hosts"
+  )
+  
+  if (!variant %in% names(variants)) {
+    stop("Invalid variant. Choose from: ", paste(names(variants), collapse = ", "))
+  }
+  
+  # Create output filename
+  output_file <- file.path(outdir, paste0("stevenblack_hosts_", variant, ".txt"))
+  
+  # Check if file already exists
+  if (!overwrite && file.exists(output_file)) {
+    stop("File already exists: ", output_file, 
+         "\nSet overwrite=TRUE to replace it.")
+  }
+  
+  # Create output directory if it doesn't exist
+  if (!dir.exists(outdir)) {
+    dir.create(outdir, recursive = TRUE)
+  }
+  
+  # Download the hosts file
+  tryCatch({
+    cat("Downloading Steven Black's hosts file (", variant, " variant)...\n")
+    curl::curl_download(variants[[variant]], output_file)
+    cat("Steven Black's hosts data saved to:", output_file, "\n")
+    
+    # Print some statistics
+    hosts_lines <- readLines(output_file, warn = FALSE)
+    blocked_count <- sum(grepl("^(0\\.0\\.0\\.0|127\\.0\\.0\\.1)\\s+", hosts_lines))
+    cat("Total blocked domains:", blocked_count, "\n")
+    
+  }, error = function(e) {
+    stop("Failed to download hosts file: ", e$message)
+  })
+  
+  invisible(output_file)
+}

R/get_stevenblack_data.R

@@ -0,0 +1,101 @@
+#' Get Category from Steven Black's Host List
+#'
+#' Classifies domains based on Steven Black's unified host list which blocks
+#' ads, malware, and tracking domains. The function checks if a domain appears
+#' in the blocklist and categorizes it accordingly.
+#' 
+#' Steven Black's host list is a consolidated list from multiple sources including
+#' adaway.org, mvps.org, malwaredomainlist.com, and someonewhocares.org.
+#' 
+#' @param domain domain names as character vector
+#' @param use_file path to a local Steven Black hosts file. If NULL, downloads from GitHub
+#' 
+#' @return data.frame with original domain name and category
+#'  
+#' @export
+#' @references \url{https://github.com/StevenBlack/hosts}
+#' 
+#' @examples \dontrun{
+#' stevenblack_cat("doubleclick.net")
+#' stevenblack_cat(c("google.com", "googleadservices.com", "malware-example.com"))
+#' }
+
+stevenblack_cat <- function(domain = NULL, use_file = NULL) {
+  
+  if (is.null(domain)) {
+    stop("Please provide at least one domain.")
+  }
+  
+  # Clean domains
+  clean_domains <- gsub("^https?://", "", domain)
+  clean_domains <- gsub("^www\\.", "", clean_domains)
+  clean_domains <- gsub("/.*$", "", clean_domains)
+  
+  # Get or load hosts data
+  if (is.null(use_file)) {
+    hosts_file <- tempfile()
+    tryCatch({
+      cat("Downloading Steven Black's hosts file...\n")
+      curl::curl_download(
+        "https://raw.githubusercontent.com/StevenBlack/hosts/master/hosts",
+        hosts_file
+      )
+    }, error = function(e) {
+      stop("Failed to download hosts file: ", e$message)
+    })
+  } else {
+    if (!file.exists(use_file)) {
+      stop("File not found: ", use_file)
+    }
+    hosts_file <- use_file
+  }
+  
+  # Read and parse hosts file
+  tryCatch({
+    hosts_lines <- readLines(hosts_file, warn = FALSE)
+  }, error = function(e) {
+    stop("Failed to read hosts file: ", e$message)
+  })
+  
+  # Extract blocked domains (lines starting with 0.0.0.0 or 127.0.0.1)
+  blocked_lines <- hosts_lines[grepl("^(0\\.0\\.0\\.0|127\\.0\\.0\\.1)\\s+", hosts_lines)]
+  
+  # Parse domain names from blocked lines
+  blocked_domains <- sub("^(0\\.0\\.0\\.0|127\\.0\\.0\\.1)\\s+", "", blocked_lines)
+  blocked_domains <- trimws(blocked_domains)
+  
+  # Remove comments and empty lines
+  blocked_domains <- blocked_domains[!grepl("^#", blocked_domains)]
+  blocked_domains <- blocked_domains[blocked_domains != ""]
+  blocked_domains <- blocked_domains[blocked_domains != "localhost"]
+  
+  # Classify domains
+  results <- data.frame(
+    domain = domain,
+    stevenblack = "safe",
+    stringsAsFactors = FALSE
+  )
+  
+  # Check each clean domain against blocklist
+  for (i in seq_along(clean_domains)) {
+    if (clean_domains[i] %in% blocked_domains) {
+      # Categorize based on common patterns
+      if (grepl("(ad|ads|doubleclick|googleadservices|googlesyndication)", clean_domains[i], ignore.case = TRUE)) {
+        results$stevenblack[i] <- "ads"
+      } else if (grepl("(malware|virus|trojan|phishing)", clean_domains[i], ignore.case = TRUE)) {
+        results$stevenblack[i] <- "malware"
+      } else if (grepl("(track|analytics|metric|stats)", clean_domains[i], ignore.case = TRUE)) {
+        results$stevenblack[i] <- "tracking"
+      } else {
+        results$stevenblack[i] <- "blocked"
+      }
+    }
+  }
+  
+  # Clean up temp file if we downloaded it
+  if (is.null(use_file)) {
+    unlink(hosts_file)
+  }
+  
+  return(results)
+}