mirroring bot - 2025/08/26

Author: updating-bot

svn_trunk/src/jd/plugins/decrypter/ArchiveOrgCrawler.java

@@ -74,7 +74,7 @@
 import jd.plugins.download.HashInfo;
 import jd.plugins.hoster.ArchiveOrg;
 
-@DecrypterPlugin(revision = "$Revision: 51141 $", interfaceVersion = 2, names = { "archive.org", "subdomain.archive.org" }, urls = { "https?://(?:www\\.)?archive\\.org/((?:details|download|stream|embed)/.+|search\\?query=.+)", "https?://[^/]+\\.archive\\.org/view_archive\\.php\\?archive=[^\\&]+(?:\\&file=[^\\&]+)?" })
+@DecrypterPlugin(revision = "$Revision: 51367 $", interfaceVersion = 2, names = { "archive.org", "subdomain.archive.org" }, urls = { "https?://(?:www\\.)?archive\\.org/((?:details|download|stream|embed)/.+|search\\?query=.+)", "https?://[^/]+\\.archive\\.org/view_archive\\.php\\?archive=[^\\&]+(?:\\&file=[^\\&]+)?" })
 public class ArchiveOrgCrawler extends PluginForDecrypt {
     public ArchiveOrgCrawler(PluginWrapper wrapper) {
         super(wrapper);
@@ -367,7 +367,7 @@ private ArrayList<DownloadLink> crawlBetaSearchAPI(final String sourceurl, Strin
         do {
             query.addAndReplace("page", Integer.toString(page));
             /* This looks to be an internally used version of public crawl/search API v2 beta, see: https://archive.org/services/swagger/ */
-            brc.getPage("https://archive.org/services/search/beta/page_production/?" + query.toString());
+            brc.getPage("https://" + getHost() + "/services/search/beta/page_production/?" + query.toString());
             if (brc.getHttpConnection().getResponseCode() == 400) {
                 if (ret.size() > 0) {
                     logger.info("Stopping because: Surprisingly got http response 400 | Possibly missing items: " + (totalNumberofItems - ret.size()));
@@ -750,38 +750,11 @@ public ArrayList<DownloadLink> crawlBook(final Browser br, final String ajaxurl,
 
     private String findBookReaderURLWebsite(final Browser br) {
         String url = br.getRegex("(?:\\'|\")([^\\'\"]+BookReaderJSIA\\.php\\?[^\\'\"]+)").getMatch(0);
-        if (url != null) {
-            url = PluginJSonUtils.unescape(url);
-            return url;
-        }
-        return null;
-    }
-
-    @Deprecated
-    /** This function can parse the "track" field of json items from "/metadata/<identifier> */
-    private int[] parseAudioTrackPosition(final Object audioTrackPositionO) throws PluginException {
-        if (audioTrackPositionO == null) {
+        if (url == null) {
             return null;
-        } else if (audioTrackPositionO instanceof int[]) {
-            return (int[]) audioTrackPositionO;
-        } else if (audioTrackPositionO instanceof Number) {
-            return new int[] { ((Number) audioTrackPositionO).intValue() };
-        } else if (audioTrackPositionO instanceof String) {
-            final String string = audioTrackPositionO.toString();
-            final String xofY[] = new Regex(string, "(\\d+)\\s*/\\s*(\\d+)").getRow(0);
-            if (xofY != null) {
-                // 02/09
-                return new int[] { Integer.parseInt(xofY[0]) };
-            }
-            final String cdAndTrack[] = new Regex(string, "(\\d+)\\s*\\.\\s*(\\d+)").getRow(0);
-            if (cdAndTrack != null) {
-                // 1.01 and 3.09
-                return new int[] { Integer.parseInt(cdAndTrack[1]), Integer.parseInt(cdAndTrack[0]) };
-            }
-            return new int[] { Integer.parseInt(string) };
-        } else {
-            throw new PluginException(LinkStatus.ERROR_PLUGIN_DEFECT, "Unsupported:" + audioTrackPositionO);
         }
+        url = PluginJSonUtils.unescape(url);
+        return url;
     }
 
     /** Work in progress, see https://archive.org/metadata/<identifier> */
@@ -805,7 +778,7 @@ private ArrayList<DownloadLink> crawlMetadataJsonV2(final String identifier, fin
         final Browser brc = br.cloneBrowser();
         /* The json answer can be really big. */
         brc.setLoadLimit(Integer.MAX_VALUE);
-        brc.getPage("https://archive.org/metadata/" + Encoding.urlEncode(identifier));
+        brc.getPage("https://" + getHost() + "/metadata/" + Encoding.urlEncode(identifier));
         final Map<String, Object> root = restoreFromString(brc.getRequest().getHtmlCode(), TypeRef.MAP);
         // final Boolean is_dark = (Boolean) root.get("is_dark"); // This means that the content is offline(?)
         final List<Map<String, Object>> root_files = (List<Map<String, Object>>) root.get("files");
@@ -870,13 +843,15 @@ private ArrayList<DownloadLink> crawlMetadataJsonV2(final String identifier, fin
         DownloadLink singleDesiredFile = null;
         DownloadLink singleDesiredFile2 = null;
         final ArrayList<DownloadLink> selectedItems = new ArrayList<DownloadLink>();
+        /* FilPackage for all file items that are contained in the root of this identifiers' filesystem. */
         final FilePackage fpRoot = FilePackage.getInstance();
         fpRoot.setName(identifier);
         if (!StringUtils.isEmpty(description)) {
             fpRoot.setComment(description);
         }
         final Map<String, FilePackage> packagemap = new HashMap<String, FilePackage>();
         packagemap.put(identifier, fpRoot);
+        // final Set<ArchiveOrgType> selectedTypes = cfg.getTypesToCrawl();
         final boolean crawlOriginalFilesOnly = cfg.isFileCrawlerCrawlOnlyOriginalVersions();
         final boolean crawlMetadataFiles = cfg.isFileCrawlerCrawlMetadataFiles();
         final boolean crawlThumbnails = cfg.isFileCrawlerCrawlThumbnails();
@@ -944,7 +919,7 @@ private ArrayList<DownloadLink> crawlMetadataJsonV2(final String identifier, fin
                 filename = pathWithFilename;
             }
             final Object fileSizeO = filemap.get("size");
-            String url = "https://archive.org/download/" + identifier;
+            String url = "https://" + getHost() + "/download/" + identifier;
             if (pathWithFilename.startsWith("/")) {
                 url += URLEncode.encodeURIComponent(pathWithFilename);
             } else {
@@ -1208,7 +1183,7 @@ private ArrayList<DownloadLink> crawlMetadataJsonV2(final String identifier, fin
              * Video can't be officially downloaded but it can be streamed in segments of X seconds each -> Generate those stream-links
              */
             for (int position = 0; position < numberofVideoSegments; position++) {
-                final String directurl = "https://archive.org/download/" + identifier + "/" + identifier + ".mp4?t=" + offsetSeconds + "/" + (offsetSeconds + secondsPerSegment) + "&ignore=x.mp4";
+                final String directurl = "https://" + getHost() + "/download/" + identifier + "/" + identifier + ".mp4?t=" + offsetSeconds + "/" + (offsetSeconds + secondsPerSegment) + "&ignore=x.mp4";
                 final DownloadLink video = this.createDownloadlink(directurl);
                 video.setProperty(ArchiveOrg.PROPERTY_FILETYPE, ArchiveOrg.FILETYPE_VIDEO);
                 video.setProperty(ArchiveOrg.PROPERTY_PLAYLIST_POSITION, position);
@@ -1638,7 +1613,7 @@ private ArrayList<DownloadLink> crawlPatternSlashDownloadWebsite(final String ur
     private ArrayList<DownloadLink> crawlFiles(final String contenturl) throws Exception {
         if (br.getHttpConnection().getResponseCode() == 404) {
             throw new PluginException(LinkStatus.ERROR_FILE_NOT_FOUND);
-        } else if (br.containsHTML("(?i)>\\s*The item is not available")) {
+        } else if (br.containsHTML(">\\s*The item is not available")) {
             throw new PluginException(LinkStatus.ERROR_FILE_NOT_FOUND);
         } else if (!br.containsHTML("\"/download/")) {
             throw new PluginException(LinkStatus.ERROR_FILE_NOT_FOUND, "Maybe invalid link or nothing there to download");
@@ -1824,7 +1799,7 @@ private ArrayList<DownloadLink> crawlXML(final String contenturl, final Browser
             throw new IllegalArgumentException();
         }
         String xmlResponse = null;
-        final String xmlurl = "https://archive.org/download/" + titleSlug + "/" + titleSlug + "_files.xml";
+        final String xmlurl = "https://" + getHost() + "/download/" + titleSlug + "/" + titleSlug + "_files.xml";
         final String cacheKey = xmlurl;
         final Object lock = requestLock(cacheKey);
         try {
@@ -1869,7 +1844,7 @@ private ArrayList<DownloadLink> crawlXML(final String contenturl, final Browser
             throw new DecrypterRetryException(RetryReason.EMPTY_FOLDER, path);
         }
         logger.info("Crawling all files below path: " + path);
-        final String basePath = "https://archive.org/download/" + titleSlug;
+        final String basePath = "https://" + getHost() + "/download/" + titleSlug;
         final List<String> skippedItems = new ArrayList<String>();
         for (final String item : items) {
             /* <old_version>true</old_version> */

svn_trunk/src/jd/plugins/decrypter/GenericYetiShareFolder.java

@@ -50,7 +50,7 @@
 import jd.plugins.PluginForHost;
 import jd.plugins.components.SiteType.SiteTemplate;
 
-@DecrypterPlugin(revision = "$Revision: 51349 $", interfaceVersion = 3, names = {}, urls = {})
+@DecrypterPlugin(revision = "$Revision: 51367 $", interfaceVersion = 3, names = {}, urls = {})
 public class GenericYetiShareFolder extends PluginForDecrypt {
     public GenericYetiShareFolder(PluginWrapper wrapper) {
         super(wrapper);
@@ -98,6 +98,7 @@ public static List<String[]> getPluginDomains() {
         ret.add(new String[] { "imgcubby.com" });
         ret.add(new String[] { "vidpirate.com" });
         ret.add(new String[] { "netnaijafiles.xyz" });
+        ret.add(new String[] { "way4share.com" });
         return ret;
     }
 

svn_trunk/src/jd/plugins/decrypter/KemonoPartyCrawler.java

@@ -23,18 +23,7 @@
 import java.util.LinkedHashMap;
 import java.util.List;
 import java.util.Map;
-
-import org.appwork.net.protocol.http.HTTPConstants;
-import org.appwork.storage.TypeRef;
-import org.appwork.utils.DebugMode;
-import org.appwork.utils.Regex;
-import org.appwork.utils.StringUtils;
-import org.appwork.utils.parser.UrlQuery;
-import org.jdownloader.plugins.components.config.KemonoPartyConfig;
-import org.jdownloader.plugins.components.config.KemonoPartyConfig.TextCrawlMode;
-import org.jdownloader.plugins.components.config.KemonoPartyConfigCoomerParty;
-import org.jdownloader.plugins.config.PluginJsonConfig;
-import org.jdownloader.plugins.controller.LazyPlugin;
+import java.util.Set;
 
 import jd.PluginWrapper;
 import jd.controlling.ProgressController;
@@ -54,7 +43,19 @@
 import jd.plugins.PluginForDecrypt;
 import jd.plugins.hoster.KemonoParty;
 
-@DecrypterPlugin(revision = "$Revision: 51362 $", interfaceVersion = 3, names = {}, urls = {})
+import org.appwork.net.protocol.http.HTTPConstants;
+import org.appwork.storage.TypeRef;
+import org.appwork.utils.DebugMode;
+import org.appwork.utils.Regex;
+import org.appwork.utils.StringUtils;
+import org.appwork.utils.parser.UrlQuery;
+import org.jdownloader.plugins.components.config.KemonoPartyConfig;
+import org.jdownloader.plugins.components.config.KemonoPartyConfig.TextCrawlMode;
+import org.jdownloader.plugins.components.config.KemonoPartyConfigCoomerParty;
+import org.jdownloader.plugins.config.PluginJsonConfig;
+import org.jdownloader.plugins.controller.LazyPlugin;
+
+@DecrypterPlugin(revision = "$Revision: 51371 $", interfaceVersion = 3, names = {}, urls = {})
 public class KemonoPartyCrawler extends PluginForDecrypt {
     public KemonoPartyCrawler(PluginWrapper wrapper) {
         super(wrapper);
@@ -118,7 +119,10 @@ private String getApiBase() {
         return "https://" + getHost() + "/api/v1";
     }
 
+    private KemonoPartyConfig cfg = null;
+
     public ArrayList<DownloadLink> decryptIt(final CryptedLink param, ProgressController progress) throws Exception {
+        cfg = PluginJsonConfig.get(getConfigInterface());
         cl = param;
         if (param.getCryptedUrl().matches(TYPE_PROFILE)) {
             return this.crawlProfile(param);
@@ -130,6 +134,12 @@ public ArrayList<DownloadLink> decryptIt(final CryptedLink param, ProgressContro
         }
     }
 
+    @Override
+    public void clean() {
+        cfg = null;
+        super.clean();
+    }
+
     private ArrayList<DownloadLink> crawlProfile(final CryptedLink param) throws Exception {
         final Regex urlinfo = new Regex(param.getCryptedUrl(), TYPE_PROFILE);
         if (!urlinfo.patternFind()) {
@@ -156,8 +166,8 @@ private ArrayList<DownloadLink> crawlProfileAPI(final String service, final Stri
             throw new PluginException(LinkStatus.ERROR_PLUGIN_DEFECT);
         }
         final HashSet<String> dupes = new HashSet<String>();
-        final boolean useAdvancedDupecheck = PluginJsonConfig.get(getConfigInterface()).isEnableProfileCrawlerAdvancedDupeFiltering();
-        final boolean perPostPackageEnabled = PluginJsonConfig.get(getConfigInterface()).isPerPostURLPackageEnabled();
+        final boolean useAdvancedDupecheck = cfg.isEnableProfileCrawlerAdvancedDupeFiltering();
+        final boolean perPostPackageEnabled = cfg.isPerPostURLPackageEnabled();
         final ArrayList<DownloadLink> ret = new ArrayList<DownloadLink>();
         final FilePackage profileFilePackage = getFilePackageForProfileCrawler(service, usernameOrUserID);
         int offset = 0;
@@ -169,6 +179,7 @@ private ArrayList<DownloadLink> crawlProfileAPI(final String service, final Stri
         final int maxItemsPerPage = 50;
         int numberofContinuousPagesWithoutAnyNewItems = 0;
         final int maxPagesWithoutNewItems = 15;
+        final Set<String> retryWithSinglePostAPI = new HashSet<String>();
         do {
             getPage(br, this.getApiBase() + "/" + service + "/user/" + Encoding.urlEncode(usernameOrUserID) + "/posts?o=" + offset);
             final List<Map<String, Object>> posts = (List<Map<String, Object>>) restoreFromString(br.getRequest().getHtmlCode(), TypeRef.OBJECT);
@@ -184,6 +195,14 @@ private ArrayList<DownloadLink> crawlProfileAPI(final String service, final Stri
             final int numberofUniqueItemsOld = dupes.size();
             for (final Map<String, Object> post : posts) {
                 final ArrayList<DownloadLink> thisresults = this.crawlProcessPostAPI(post, dupes, useAdvancedDupecheck);
+                if (post.get("content") == null && StringUtils.isNotEmpty(StringUtils.valueOfOrNull(post.get("substring")))) {
+                    // posts api no longer returns full post content but only a substring, so we have to retry with post api
+                    final TextCrawlMode mode = cfg.getTextCrawlMode();
+                    if (cfg.isCrawlHttpLinksFromPostContent() || mode == TextCrawlMode.ALWAYS || (mode == TextCrawlMode.ONLY_IF_NO_MEDIA_ITEMS_ARE_FOUND && thisresults.isEmpty())) {
+                        retryWithSinglePostAPI.add(post.get("id").toString());
+                        logger.info("Need to process item:" + post.get("id") + " again due to maybe incomplete post content");
+                    }
+                }
                 for (final DownloadLink thisresult : thisresults) {
                     if (!perPostPackageEnabled) {
                         thisresult._setFilePackage(profileFilePackage);
@@ -218,6 +237,19 @@ private ArrayList<DownloadLink> crawlProfileAPI(final String service, final Stri
                 page++;
             }
         } while (!this.isAbort());
+        logger.info("Need to process " + retryWithSinglePostAPI.size() + " items again due to maybe incomplete post content");
+        while (!this.isAbort() && retryWithSinglePostAPI.size() > 0) {
+            final String nextRetryPostID = retryWithSinglePostAPI.iterator().next();
+            retryWithSinglePostAPI.remove(nextRetryPostID);
+            final ArrayList<DownloadLink> thisresults = crawlPostAPI(br, service, usernameOrUserID, nextRetryPostID);
+            for (final DownloadLink thisresult : thisresults) {
+                if (!perPostPackageEnabled) {
+                    thisresult._setFilePackage(profileFilePackage);
+                }
+                distribute(thisresult);
+            }
+            ret.addAll(thisresults);
+        }
         return ret;
     }
 
@@ -252,11 +284,11 @@ private ArrayList<DownloadLink> crawlPost(final CryptedLink param) throws Except
         final String service = urlinfo.getMatch(0);
         final String usernameOrUserID = urlinfo.getMatch(1);
         final String postID = urlinfo.getMatch(2);
-        return crawlPostAPI(param, service, usernameOrUserID, postID);
+        return crawlPostAPI(br, service, usernameOrUserID, postID);
     }
 
     /** API docs: https://kemono.su/api/schema */
-    private ArrayList<DownloadLink> crawlPostAPI(final CryptedLink param, final String service, final String userID, final String postID) throws Exception {
+    private ArrayList<DownloadLink> crawlPostAPI(final Browser br, final String service, final String userID, final String postID) throws Exception {
         if (service == null || userID == null || postID == null) {
             /* Developer mistake */
             throw new PluginException(LinkStatus.ERROR_PLUGIN_DEFECT);
@@ -282,7 +314,9 @@ private ArrayList<DownloadLink> crawlProcessPostAPI(final Map<String, Object> po
         final String postID = postmap.get("id").toString();
         final String posturl = "https://" + this.getHost() + "/" + service + "/user/" + usernameOrUserID + "/post/" + postID;
         final String postTitle = postmap.get("title").toString();
+        /* Every item has a "published" date */
         final String publishedDateStr = StringUtils.valueOfOrNull(postmap.get("published"));
+        /* Not all items have a "edited" date */
         final String editedDateStr = StringUtils.valueOfOrNull(postmap.get("edited"));
         final ArrayList<DownloadLink> kemonoResults = new ArrayList<DownloadLink>();
         int numberofResultsSimpleCount = 0;
@@ -313,7 +347,6 @@ private ArrayList<DownloadLink> crawlProcessPostAPI(final Map<String, Object> po
         final ArrayList<DownloadLink> ret = new ArrayList<DownloadLink>();
         final FilePackage postFilePackage = getFilePackageForPostCrawler(service, usernameOrUserID, postID, postTitle);
         String postTextContent = (String) postmap.get("content");
-        final KemonoPartyConfig cfg = PluginJsonConfig.get(getConfigInterface());
         if (!StringUtils.isEmpty(postTextContent)) {
             if (cfg.isCrawlHttpLinksFromPostContent()) {
                 /* Place number 1 where we can crawl external http links from */
@@ -432,15 +465,13 @@ private DownloadLink buildFileDownloadLinkAPI(final HashSet<String> dupes, final
     }
 
     private static Map<String, String> ID_TO_USERNAME = new LinkedHashMap<String, String>() {
-        protected boolean removeEldestEntry(Map.Entry<String, String> eldest) {
-            return size() > 100;
-        };
-    };
+                                                          protected boolean removeEldestEntry(Map.Entry<String, String> eldest) {
+                                                              return size() > 100;
+                                                          };
+                                                      };
 
     /**
-     * Returns userID for given username. </br>
-     * Uses API to find userID. </br>
-     * Throws Exception if it is unable to find userID.
+     * Returns userID for given username. </br> Uses API to find userID. </br> Throws Exception if it is unable to find userID.
      */
     private String findUsername(final String service, final String usernameOrUserID) throws Exception {
         synchronized (ID_TO_USERNAME) {