author block-code and description added to each class

Author: jetkai

src/main/kotlin/plugin/httpclient/checkerproxy/CheckerProxy.kt

@@ -54,7 +54,7 @@ class CheckerProxy : Plugin, ProxyWebsite {
             logger.error { ex.message }
         }
 
-        return false
+        return true
     }
 
     override fun thenConnect() {

src/main/kotlin/plugin/httpclient/freeproxyapi/FreeProxyApi.kt

@@ -58,7 +58,7 @@ class FreeProxyApi : Plugin, ProxyWebsite {
             logger.error { ex.message }
         }
 
-        return false
+        return true
     }
 
     override fun thenConnect() {

src/main/kotlin/plugin/httpclient/geonode/GeoNode.kt

@@ -91,7 +91,7 @@ class GeoNode : Plugin, ProxyWebsite {
             logger.error { ex.message }
         }
 
-        return false
+        return true
     }
 
     override fun thenConnect() {

src/main/kotlin/plugin/httpclient/openproxylist/OpenProxyList.kt

@@ -57,7 +57,7 @@ class OpenProxyList : Plugin, ProxyWebsite {
             logger.error { ex.message }
         }
 
-        return false
+        return true
     }
 
     override fun thenConnect() {

src/main/kotlin/plugin/httpclient/proxyscrape/ProxyScrape.kt

@@ -59,7 +59,7 @@ class ProxyScrape : Plugin, ProxyWebsite {
             logger.error { ex.message }
         }
 
-        return false
+        return true
     }
 
     override fun thenConnect() {

src/main/kotlin/scraper/Main.kt

@@ -10,6 +10,12 @@ import java.nio.file.Path
 import kotlin.io.path.exists
 import kotlin.system.exitProcess
 
+/**
+ * Main - 12/02/2023
+ * @author Kai
+ *
+ * Description: Main class, runs the application
+ **/
 class Main {
 
     private val logger = KotlinLogging.logger { }

src/main/kotlin/scraper/net/ChromeWebDriver.kt

@@ -12,6 +12,12 @@ import org.openqa.selenium.devtools.Command
 import org.openqa.selenium.devtools.DevTools
 import org.openqa.selenium.devtools.v109.network.Network
 
+/**
+ * ChromeWebDriver - 12/02/2023
+ * @author Kai
+ *
+ * Description: Used for more complex proxy sites, ones that require JavaScript & obfuscate their data
+ **/
 class ChromeWebDriver {
 
     private val options = ChromeOptions()

src/main/kotlin/scraper/net/CoroutinesHttpClient.kt

@@ -9,14 +9,21 @@ import java.net.http.HttpResponse
 import java.net.http.HttpTimeoutException
 import java.time.Duration
 
-
+/**
+ * CoroutinesHttpClient - 12/02/2023
+ * @author Kai
+ *
+ * Description: Used for simple web calls, such as sending GET/POST data for text content (JSON etc)
+ **/
 class CoroutinesHttpClient {
 
     companion object {
 
+        val timeout : Duration = Duration.ofSeconds(15)
+
         private val httpClient : HttpClient = HttpClient.newBuilder()
-            .version(HttpClient.Version.HTTP_1_1)
-            .connectTimeout(Duration.ofSeconds(30))
+            .version(HttpClient.Version.HTTP_2)
+            .connectTimeout(timeout)
             .build()
 
     }
@@ -31,7 +38,7 @@ class CoroutinesHttpClient {
         val request : HttpRequest = if(postData == null) {
             HttpRequest.newBuilder()
                 .uri(URI.create(url))
-                .timeout(Duration.ofSeconds(30))
+                .timeout(timeout)
                 .headers(*userAgent)
                 .headers(*accept)
                 .headers(*acceptLanguage)
@@ -42,7 +49,7 @@ class CoroutinesHttpClient {
         } else {
             HttpRequest.newBuilder()
                 .uri(URI.create(url))
-                .timeout(Duration.ofSeconds(30))
+                .timeout(timeout)
                 .headers(*userAgent)
                 .headers(*accept)
                 .headers(*acceptLanguage)

src/main/kotlin/scraper/plugin/Plugin.kt

@@ -1,5 +1,11 @@
 package scraper.plugin
 
+/**
+ * Plugin - 12/02/2023
+ * @author Kai
+ *
+ * Description: Interface for Plugin, this will append the class to a List within PluginFactory
+ **/
 interface Plugin {
 
     fun register()

src/main/kotlin/scraper/plugin/PluginFactory.kt

@@ -5,6 +5,13 @@ import org.reflections.Reflections
 import scraper.plugin.hook.ProxyWebsite
 import java.lang.reflect.Modifier
 
+/**
+ * PluginFactory - 12/02/2023
+ * @author Kai
+ *
+ * Description: This is used for loading the Proxy Website Plugins from /src/main/kotlin/plugins/
+ * Makes it easier for APIs to be added / removed
+ **/
 object PluginFactory {
 
     private val logger = KotlinLogging.logger { }

src/main/kotlin/scraper/plugin/hook/ProxyWebsite.kt

@@ -2,6 +2,13 @@ package scraper.plugin.hook
 
 import scraper.util.data.ProxyData
 
+/**
+ * ProxyWebsite - 12/02/2023
+ * @author Kai
+ *
+ * Description: Interface for the Website Plugins within /src/main/kotlin/plugin/
+ * Plugins will follow this interface format
+ **/
 interface ProxyWebsite {
 
     val proxies : MutableList<ProxyData>

src/main/kotlin/scraper/util/NetworkUtils.kt

@@ -2,6 +2,12 @@ package scraper.util
 
 import java.util.regex.Pattern
 
+/**
+ * NetworkUtils - 17/02/2023
+ * @author Kai
+ *
+ * Description: Any reusable network functions are in this class
+ **/
 object NetworkUtils {
 
     fun isValidIpAddress(ipAddress : String) : Boolean {

src/main/kotlin/scraper/util/ScraperExecutor.kt

@@ -3,6 +3,12 @@ package scraper.util
 import java.util.concurrent.ExecutorService
 import java.util.concurrent.Executors
 
+/**
+ * ScraperExecutor - 17/02/2023
+ * @author Kai
+ *
+ * Description: This is used for multi-threading to speed up the application
+ **/
 object ScraperExecutor {
 
     private val PROCESSORS = Runtime.getRuntime().availableProcessors() + 1

src/main/kotlin/scraper/util/ScraperThreadFactory.kt

@@ -3,6 +3,12 @@ package scraper.util
 import java.util.concurrent.ThreadFactory
 import java.util.concurrent.atomic.AtomicInteger
 
+/**
+ * ScraperThreadFactory - 12/02/2023
+ * @author Kai
+ *
+ * Description: This is used for multi-threading to speed up the application
+ **/
 class ScraperThreadFactory(private val name : String) : ThreadFactory {
     private val threadCount = AtomicInteger()
     override fun newThread(r : Runnable): Thread {

src/main/kotlin/scraper/util/scripts/GetAllCheckerProxies.kt

@@ -18,6 +18,12 @@ import java.time.format.DateTimeFormatter
 import kotlin.io.path.exists
 import kotlin.system.exitProcess
 
+/**
+ * GetAllCheckerProxies - 17/02/2023
+ * @author Kai
+ *
+ * Description: Quick and easy script for getting all the proxies from CheckerProxy's API
+ **/
 private const val endpoint = "https://checkerproxy.net/api/archive/"
 
 private val proxies : MutableList<ProxyData> = mutableListOf()
@@ -26,7 +32,7 @@ fun main() {
     loopThroughDates()
 }
 
-fun loopThroughDates() {
+private fun loopThroughDates() {
     val format = DateTimeFormatter.ofPattern("yyyy-MM-dd")
     val fromDate = LocalDate.parse("2023-01-18", format)
     val toDate = LocalDate.parse("2023-02-17", format)