mirroring bot - 2024/10/30

Author: updating-bot

svn_trunk/src/jd/plugins/MultiHostHostData.java

@@ -3,10 +3,10 @@
 import java.util.ArrayList;
 import java.util.List;
 
-import org.appwork.storage.Storable;
-
 import jd.plugins.MultiHostHost.MultihosterHostStatus;
 
+import org.appwork.storage.Storable;
+
 public class MultiHostHostData implements Storable {
     private String                name;
     private List<String>          domains;
@@ -158,6 +158,9 @@ public void setUnavailableUntilTimestamp(long unavailableUntilTimestamp) {
     }
 
     public static List<MultiHostHostData> createFromMultiHostHostList(final List<MultiHostHost> sourcelist) {
+        if (sourcelist == null || sourcelist.size() == 0) {
+            return null;
+        }
         final List<MultiHostHostData> ret = new ArrayList<MultiHostHostData>();
         for (final MultiHostHost mhost : sourcelist) {
             ret.add(createFromMultiHostHost(mhost));

svn_trunk/src/jd/plugins/decrypter/MFS_ShortUrlScript.java

@@ -39,7 +39,7 @@
  *
  * @author raztoki
  */
-@DecrypterPlugin(revision = "$Revision: 50037 $", interfaceVersion = 3, names = { "gourl.us" }, urls = { "https?://(?:www\\.)?gourl\\.us/([a-zA-Z0-9_\\-]+)$" })
+@DecrypterPlugin(revision = "$Revision: 50040 $", interfaceVersion = 3, names = { "gourl.us" }, urls = { "https?://(?:www\\.)?gourl\\.us/([a-zA-Z0-9_\\-]+)$" })
 public class MFS_ShortUrlScript extends PluginForDecrypt {
     public MFS_ShortUrlScript(PluginWrapper wrapper) {
         super(wrapper);
@@ -48,14 +48,25 @@ public MFS_ShortUrlScript(PluginWrapper wrapper) {
     @Override
     public Browser createNewBrowserInstance() {
         final Browser br = super.createNewBrowserInstance();
-        br.setFollowRedirects(true);
+        br.setFollowRedirects(false);
         return br;
     }
 
     public ArrayList<DownloadLink> decryptIt(final CryptedLink param, final ProgressController progress) throws Exception {
+        final ArrayList<DownloadLink> ret = new ArrayList<DownloadLink>();
         final String parameter = param.getCryptedUrl().replaceFirst("^http://", "https://");
         // final String fuid = new Regex(parameter, this.getSupportedLinks()).getMatch(0);
         br.getPage(parameter);
+        final String redirect = br.getRedirectLocation();
+        if (redirect != null && !this.canHandle(redirect) && !redirect.contains(this.getHost() + "/")) {
+            logger.info("Found direct redirect to final link: " + redirect);
+            ret.add(createDownloadlink(redirect));
+            return ret;
+        }
+        br.setFollowRedirects(true);
+        if (redirect != null) {
+            br.followRedirect(true);
+        }
         final String errorFromURL = UrlQuery.parse(br.getURL()).get("e");
         if (errorFromURL != null) {
             throw new PluginException(LinkStatus.ERROR_FILE_NOT_FOUND);
@@ -82,7 +93,6 @@ public ArrayList<DownloadLink> decryptIt(final CryptedLink param, final Progress
                 continue;
             }
         }
-        final ArrayList<DownloadLink> ret = new ArrayList<DownloadLink>();
         final String frame = br.getRegex("<frame [^>]*src=\"(interstitualAdTop\\.php\\?url=\\d+)\"").getMatch(0);
         if (frame == null) {
             throw new PluginException(LinkStatus.ERROR_PLUGIN_DEFECT);

svn_trunk/src/jd/plugins/decrypter/MylinkLi.java

@@ -17,6 +17,7 @@
 
 import java.util.ArrayList;
 import java.util.List;
+import java.util.Locale;
 
 import javax.script.ScriptEngine;
 import javax.script.ScriptEngineManager;
@@ -30,6 +31,7 @@
 import jd.controlling.ProgressController;
 import jd.http.Browser;
 import jd.nutils.encoding.Encoding;
+import jd.parser.Regex;
 import jd.parser.html.Form;
 import jd.parser.html.InputField;
 import jd.plugins.CryptedLink;
@@ -39,7 +41,7 @@
 import jd.plugins.PluginException;
 import jd.plugins.PluginForDecrypt;
 
-@DecrypterPlugin(revision = "$Revision: 48378 $", interfaceVersion = 3, names = {}, urls = {})
+@DecrypterPlugin(revision = "$Revision: 50040 $", interfaceVersion = 3, names = {}, urls = {})
 public class MylinkLi extends PluginForDecrypt {
     public MylinkLi(PluginWrapper wrapper) {
         super(wrapper);
@@ -63,14 +65,19 @@ public String[] siteSupportedNames() {
     public static String[] getAnnotationUrls() {
         final List<String> ret = new ArrayList<String>();
         for (final String[] domains : getPluginDomains()) {
-            ret.add("https?://(?:www\\.)?" + buildHostsPatternPart(domains) + "/[A-Za-z0-9]+");
+            ret.add("https?://(?:www\\.)?" + buildHostsPatternPart(domains) + "/([A-Za-z0-9]+)");
         }
         return ret.toArray(new String[0]);
     }
 
     public ArrayList<DownloadLink> decryptIt(final CryptedLink param, ProgressController progress) throws Exception {
         /** 2023-10-19: The usage of getAndSetSpecialCookie is not needed anymore but it doesn't hurt either so I've left it in. */
         // final String contentID = new Regex(param.getCryptedUrl(), "/([A-Za-z0-9]+)$").getMatch(0);
+        final String contentID = new Regex(param.getCryptedUrl(), this.getSupportedLinks()).getMatch(0);
+        if (StringUtils.equals(contentID.toLowerCase(Locale.ENGLISH), contentID)) {
+            /* Lowercase contentID -> Invalid -> E.g. https://mylink.vc/api */
+            throw new PluginException(LinkStatus.ERROR_FILE_NOT_FOUND, "Invalid contentID");
+        }
         br.setFollowRedirects(true);
         br.getHeaders().put("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36");
         br.getPage(param.getCryptedUrl());

svn_trunk/src/jd/plugins/decrypter/NexusmodsComCrawler.java

@@ -42,7 +42,7 @@
 import jd.plugins.PluginForHost;
 import jd.utils.JDUtilities;
 
-@DecrypterPlugin(revision = "$Revision: 50039 $", interfaceVersion = 3, names = { "nexusmods.com" }, urls = { "https?://(?:www\\.)?nexusmods\\.com/(?!contents)([^/]+)/mods/(\\d+)/?" })
+@DecrypterPlugin(revision = "$Revision: 50040 $", interfaceVersion = 3, names = { "nexusmods.com" }, urls = { "https?://(?:www\\.)?nexusmods\\.com/(?!contents)([^/]+)/mods/(\\d+)/?" })
 public class NexusmodsComCrawler extends PluginForDecrypt {
     public NexusmodsComCrawler(PluginWrapper wrapper) {
         super(wrapper);
@@ -65,7 +65,7 @@ public ArrayList<DownloadLink> decryptIt(final CryptedLink param, ProgressContro
         if (apikey != null) {
             ret = crawlAPI(param, account, game_domain_name, mod_id);
         } else {
-            ret = crawlWebsite(param, account, game_domain_name, mod_id);
+            ret = crawlWebsite(param, game_domain_name, mod_id);
         }
         return ret;
     }
@@ -133,14 +133,10 @@ private ArrayList<DownloadLink> crawlAPI(final CryptedLink param, final Account
         return ret;
     }
 
-    private ArrayList<DownloadLink> crawlWebsite(final CryptedLink param, final Account account, final String game_domain_name, final String mod_id) throws Exception {
+    /** Crawls items from website (without account). */
+    private ArrayList<DownloadLink> crawlWebsite(final CryptedLink param, final String game_domain_name, final String mod_id) throws Exception {
         final String parameter = param.getCryptedUrl();
         final PluginForHost plugin = JDUtilities.getPluginForHost(this.getHost());
-        final ArrayList<DownloadLink> ret = new ArrayList<DownloadLink>();
-        if (account != null) {
-            /* Login via website */
-            ((jd.plugins.hoster.NexusmodsCom) plugin).loginWebsite(account);
-        }
         br.setFollowRedirects(true);
         ((jd.plugins.hoster.NexusmodsCom) plugin).getPage(br, parameter);
         if (jd.plugins.hoster.NexusmodsCom.isOfflineWebsite(br)) {
@@ -152,6 +148,7 @@ private ArrayList<DownloadLink> crawlWebsite(final CryptedLink param, final Acco
             logger.info("Adult content: Enable it in your account settings to be able to download such files via JD: Profile --> Settings --> Content blocking --> Show adult content");
             throw new AccountRequiredException("Adult content: Enable it in your account settings to be able to download such files via JD: Profile --> Settings --> Content blocking --> Show adult content");
         }
+        final ArrayList<DownloadLink> ret = new ArrayList<DownloadLink>();
         String mod_name = br.getRegex("<meta property=\"og:title\" content=\"([^<>\"]+)\"").getMatch(0);
         if (mod_name == null) {
             /* This should never happen */
@@ -164,61 +161,91 @@ private ArrayList<DownloadLink> crawlWebsite(final CryptedLink param, final Acco
         final Browser br2 = br.cloneBrowser();
         ((jd.plugins.hoster.NexusmodsCom) plugin).getPage(br2, "/" + game_domain_name + "/mods/" + mod_id + "?tab=files");
         final String[] categoryNames = br2.getRegex("<div class=\"file-category-header\">\\s*<h2>([^>]+)</h2>").getColumn(0);
-        final String[] downloadCategoriesHTMLs = br2.getRegex("<dd[^>]* data-id=\"\\d+\"[^>]*>.*?</div>\\s*</dt>").getColumn(-1);
-        if (downloadCategoriesHTMLs == null || downloadCategoriesHTMLs.length == 0) {
+        final String[] downloadCategoriesHTMLs = br2.getRegex("<dd[^>]*data-id=\"\\d+\"[^>]*>.*?</div>\\s*</dt>").getColumn(-1);
+        String description = br2.getRegex("<meta name=\"description\" content=\"([^\"]+)").getMatch(0);
+        if (downloadCategoriesHTMLs != null && downloadCategoriesHTMLs.length > 0) {
+            int index = -1;
+            for (final String downnloadTypeHTML : downloadCategoriesHTMLs) {
+                index++;
+                String category_name = null;
+                if (categoryNames != null && categoryNames.length == downloadCategoriesHTMLs.length) {
+                    category_name = categoryNames[index];
+                }
+                category_name = Encoding.htmlDecode(category_name).trim();
+                ret.addAll(websiteCrawlFiles(downnloadTypeHTML, category_name, index, mod_id, game_id, game_domain_name, mod_name));
+            }
+        } else {
+            /* Fallback: Don't care about the categories, just find the files */
+            logger.warning("Failed to find categories");
+            ret.addAll(websiteCrawlFiles(br2.getRequest().getHtmlCode(), null, 0, mod_id, game_id, game_domain_name, mod_name));
+        }
+        if (ret.isEmpty()) {
+            /*
+             * Effectively, items are only downloadable via (paid) account so if the crawler fails, let's suppress that and instead ask the
+             * user to retry again with account -> User can then use the API so then the crawler will work fine.
+             */
+            // throw new PluginException(LinkStatus.ERROR_PLUGIN_DEFECT, "Failed to find game_id");
+            throw new AccountRequiredException();
+        }
+        if (description != null) {
+            /* Set description as package comment. */
+            description = Encoding.htmlDecode(description).trim();
+            for (final DownloadLink link : ret) {
+                if (link.getFilePackage() == null) {
+                    continue;
+                }
+                link.getFilePackage().setComment(description);
+            }
+        }
+        return ret;
+    }
+
+    private List<DownloadLink> websiteCrawlFiles(final String html_source, final String category_name, final int categoryIndex, final String mod_id, final String game_id, final String game_domain_name, final String mod_name) throws PluginException {
+        final ArrayList<DownloadLink> ret = new ArrayList<DownloadLink>();
+        final FilePackage fp = FilePackage.getInstance();
+        if (category_name != null) {
+            fp.setName(game_domain_name + " - " + mod_name + " - " + category_name);
+        } else {
+            fp.setName(game_domain_name + " - " + mod_name);
+        }
+        final String currentPath = game_domain_name + "/" + mod_name + "/" + category_name;
+        final String[] htmls = new Regex(html_source, "<dt id=\"file-expander-header-\\d+\".*?/use>\\s*</svg>\\s*</div>").getColumn(-1);
+        if (htmls == null || htmls.length == 0) {
             throw new PluginException(LinkStatus.ERROR_PLUGIN_DEFECT);
         }
-        int index = -1;
-        for (final String downnloadTypeHTML : downloadCategoriesHTMLs) {
-            index++;
-            String category_name = null;
-            if (categoryNames != null && categoryNames.length == downloadCategoriesHTMLs.length) {
-                category_name = categoryNames[index];
+        for (final String html : htmls) {
+            String file_id = new Regex(html, "\\?id=(\\d+)").getMatch(0);
+            if (file_id == null) {
+                file_id = new Regex(html, "data-id=\"(\\d+)\"").getMatch(0);
             }
-            if (category_name == null) {
-                /* Fallback */
-                category_name = "Unknown_category_" + (index + 1);
+            if (file_id == null) {
+                logger.info("file_id is null");
+                continue;
             }
-            final FilePackage fp = FilePackage.getInstance();
-            fp.setName(game_domain_name + " - " + mod_name + " - " + category_name);
-            category_name = Encoding.htmlDecode(category_name).trim();
-            final String currentPath = game_domain_name + "/" + mod_name + "/" + category_name;
-            final String[] htmls = new Regex(downnloadTypeHTML, "<dt id=\"file-expander-header-\\d+\".*?/use>\\s*</svg>\\s*</div>").getColumn(-1);
-            if (htmls == null || htmls.length == 0) {
-                throw new PluginException(LinkStatus.ERROR_PLUGIN_DEFECT);
+            final String filename = new Regex(html, "data-url=\"([^<>\"]+)\"").getMatch(0);
+            final DownloadLink link = createDownloadlink(generatePluginPatternMatcher(file_id, game_id));
+            link.setContentUrl(generateContentURL(game_domain_name, mod_id, file_id));
+            final String filesizeStr = new Regex(html, ">\\s*File\\s*size\\s*</div>.*?\"stat\"\\s*>\\s*([0-9\\.TKGMB]+)").getMatch(0);
+            if (filesizeStr != null) {
+                link.setDownloadSize(SizeFormatter.getSize(filesizeStr));
             }
-            for (final String html : htmls) {
-                String file_id = new Regex(html, "\\?id=(\\d+)").getMatch(0);
-                if (file_id == null) {
-                    file_id = new Regex(html, "data-id=\"(\\d+)\"").getMatch(0);
-                }
-                if (file_id == null) {
-                    logger.info("file_id is null");
-                    continue;
-                }
-                final String filename = new Regex(html, "data-url=\"([^<>\"]+)\"").getMatch(0);
-                final DownloadLink link = createDownloadlink(generatePluginPatternMatcher(file_id, game_id));
-                link.setContentUrl(generateContentURL(game_domain_name, mod_id, file_id));
-                final String filesizeStr = new Regex(html, ">\\s*File\\s*size\\s*</div>.*?\"stat\"\\s*>\\s*([0-9\\.TKGMB]+)").getMatch(0);
-                if (filesizeStr != null) {
-                    link.setDownloadSize(SizeFormatter.getSize(filesizeStr));
-                }
-                if (filename != null) {
-                    link.setName(file_id + "_" + Encoding.htmlOnlyDecode(filename));
-                } else {
-                    link.setName(file_id);
-                }
-                link.setAvailable(true);
-                link.setMimeHint(CompiledFiletypeFilter.ArchiveExtensions.ZIP);
-                link.setRelativeDownloadFolderPath(currentPath);
-                link._setFilePackage(fp);
-                /* Important! These properties are especially required for all API requests! */
-                link.setProperty("game_domain_name", game_domain_name);
-                link.setProperty("mod_id", mod_id);
+            if (filename != null) {
+                link.setName(file_id + "_" + Encoding.htmlOnlyDecode(filename));
+            } else {
+                link.setName(file_id);
+            }
+            link.setAvailable(true);
+            link.setMimeHint(CompiledFiletypeFilter.ArchiveExtensions.ZIP);
+            link.setRelativeDownloadFolderPath(currentPath);
+            link._setFilePackage(fp);
+            /* Important! These properties are especially required for all API requests! */
+            link.setProperty("game_domain_name", game_domain_name);
+            link.setProperty("mod_id", mod_id);
+            if (category_name != null) {
                 /* Every category goes into a subfolder */
                 link.setRelativeDownloadFolderPath(game_domain_name + "/" + category_name);
-                ret.add(link);
             }
+            ret.add(link);
         }
         return ret;
     }

svn_trunk/src/jd/plugins/decrypter/NicoVideoJpPlaylist.java

@@ -20,48 +20,48 @@
 import jd.PluginWrapper;
 import jd.controlling.ProgressController;
 import jd.nutils.encoding.Encoding;
+import jd.plugins.AccountRequiredException;
 import jd.plugins.CryptedLink;
 import jd.plugins.DecrypterPlugin;
 import jd.plugins.DownloadLink;
 import jd.plugins.FilePackage;
+import jd.plugins.LinkStatus;
+import jd.plugins.PluginException;
 import jd.plugins.PluginForDecrypt;
 
-@DecrypterPlugin(revision = "$Revision: 43195 $", interfaceVersion = 2, names = { "nicovideo.jp" }, urls = { "https?://(?:www\\.)?nicovideo\\.jp/(?:user/\\d+/)?mylist/\\d+" })
+@DecrypterPlugin(revision = "$Revision: 50041 $", interfaceVersion = 2, names = { "nicovideo.jp" }, urls = { "https?://(?:www\\.)?nicovideo\\.jp/(?:user/\\d+/)?mylist/\\d+" })
 public class NicoVideoJpPlaylist extends PluginForDecrypt {
     public NicoVideoJpPlaylist(PluginWrapper wrapper) {
         super(wrapper);
     }
 
     public ArrayList<DownloadLink> decryptIt(CryptedLink param, ProgressController progress) throws Exception {
-        ArrayList<DownloadLink> decryptedLinks = new ArrayList<DownloadLink>();
+        ArrayList<DownloadLink> ret = new ArrayList<DownloadLink>();
         final String parameter = param.toString();
         br.setFollowRedirects(true);
         br.getPage(parameter);
-        if (br.containsHTML(">This My List is set as private")) {
-            logger.info("Private playlist, cannot decrypt: " + parameter);
-            return decryptedLinks;
+        if (br.containsHTML(">\\s*This My List is set as private")) {
+            throw new AccountRequiredException();
         } else if (br.getHttpConnection().getResponseCode() == 404) {
-            decryptedLinks.add(this.createOfflinelink(parameter));
-            return decryptedLinks;
+            throw new PluginException(LinkStatus.ERROR_FILE_NOT_FOUND);
         }
         final String fpName = br.getRegex("MylistGroup\\.preloadSingle\\(\\d+, \\{.*?name: \"([^<>\"]*?)\"").getMatch(0);
         final String[] videoids = br.getRegex("nicovideo\\.jp..watch..(([a-z]{2})?\\d+)").getColumn(0);
         if (videoids == null || videoids.length == 0) {
-            logger.warning("Decrypter broken for link: " + parameter);
-            return null;
+            throw new PluginException(LinkStatus.ERROR_PLUGIN_DEFECT);
         }
         for (String singleLink : videoids) {
-            decryptedLinks.add(createDownloadlink("https://www.nicovideo.jp/watch/" + singleLink));
+            ret.add(createDownloadlink("https://www.nicovideo.jp/watch/" + singleLink));
         }
         if (fpName != null) {
             final FilePackage fp = FilePackage.getInstance();
-            fp.setName(Encoding.htmlDecode(fpName.trim()));
-            fp.addLinks(decryptedLinks);
+            fp.setName(Encoding.htmlDecode(fpName).trim());
+            fp.addLinks(ret);
         }
-        return decryptedLinks;
+        return ret;
     }
 
-    /* NO OVERRIDE!! */
+    @Override
     public boolean hasCaptcha(CryptedLink link, jd.plugins.Account acc) {
         return false;
     }