Merge pull request #588 from machawk1/update-heritrix-3.7

Update WAIL to use Heritrix 3.7

Author: machawk1

.codeclimate.yml

@@ -7,6 +7,6 @@ exclude_paths:
 - "build/*"
 - "support/*"
 - "WAIL.spec"
-- "bundledApps/heritrix-3.2.0/*"
+- "bundledApps/heritrix-3.4.0-20240909/*"
 - "bundledApps/html/*"
 - "bundledApps/tomcat/*"

.gitignore

@@ -24,6 +24,7 @@ WAIL.spec
 .idea
 bundledApps/tomcat/logs
 config/path-index.txt
+archiveIndexes/*
 bundledApps/tomcat/work
 jobs/
 !bundledApps/memgator*

README.md

@@ -5,7 +5,7 @@
 
 Web Archiving Integration Layer (WAIL) is a graphical user interface (GUI) atop multiple web archiving tools intended to be used as an easy way for anyone to preserve and replay web pages.
 
-Tools included and accessible through the GUI are <a href="https://github.com/internetarchive/heritrix3">Heritrix 3.2.0</a> and <a href="https://github.com/iipc/openwayback">OpenWayback 2.4.0</a>. Support packages include Apache Tomcat, <a href="https://github.com/pyinstaller/pyinstaller/">pyinstaller</a>, and <a href="https://github.com/oduwsdl/memgator">MemGator</a>.
+Tools included and accessible through the GUI are <a href="https://github.com/internetarchive/heritrix3">Heritrix 3.4.0-20240909</a> and <a href="https://github.com/iipc/openwayback">OpenWayback 2.4.0</a>. Support packages include Apache Tomcat, <a href="https://github.com/pyinstaller/pyinstaller/">pyinstaller</a>, and <a href="https://github.com/oduwsdl/memgator">MemGator</a>.
 
 WAIL is written in Python and compiled to a native executable using <a href="http://www.pyinstaller.org/">PyInstaller</a>.
 

build/Info.plist

@@ -3,7 +3,7 @@
 <plist version="1.0">
 <dict>
 	<key>CFBundleShortVersionString</key>
-	<string>2020.03.20</string>
+	<string>2024.10.03</string>
 	<key>NSHumanReadableCopyright</key>
 	<string>Copyright © Mat Kelly - Web Archiving Integration Layer (WAIL)</string>
 	<key>CFBundleExecutable</key>

bundledApps/HeritrixJob.py

@@ -0,0 +1 @@
+legal/java.base/LICENSE

bundledApps/Java/JavaVirtualMachines/jdk17.0.11.jdk/Contents/CodeResources

@@ -0,0 +1,6 @@
+Thank you for using the Oracle JDK.
+The license for this software can be found in the LICENSE file.
+
+Information on installing, configuring, and running this program is available on https://java.com/readme
+
+Documentation on the Java SE Platform can be found on https://docs.oracle.com/java

bundledApps/Java/JavaVirtualMachines/jdk17.0.11.jdk/Contents/Home/LICENSE

@@ -0,0 +1,63 @@
+############################################################
+#  	Default Logging Configuration File
+#
+# You can use a different file by specifying a filename
+# with the java.util.logging.config.file system property.
+# For example, java -Djava.util.logging.config.file=myfile
+############################################################
+
+############################################################
+#  	Global properties
+############################################################
+
+# "handlers" specifies a comma-separated list of log Handler
+# classes.  These handlers will be installed during VM startup.
+# Note that these classes must be on the system classpath.
+# By default we only configure a ConsoleHandler, which will only
+# show messages at the INFO and above levels.
+handlers= java.util.logging.ConsoleHandler
+
+# To also add the FileHandler, use the following line instead.
+#handlers= java.util.logging.FileHandler, java.util.logging.ConsoleHandler
+
+# Default global logging level.
+# This specifies which kinds of events are logged across
+# all loggers.  For any given facility this global level
+# can be overridden by a facility-specific level
+# Note that the ConsoleHandler also has a separate level
+# setting to limit messages printed to the console.
+.level= INFO
+
+############################################################
+# Handler specific properties.
+# Describes specific configuration info for Handlers.
+############################################################
+
+# default file output is in user's home directory.
+java.util.logging.FileHandler.pattern = %h/java%u.log
+java.util.logging.FileHandler.limit = 50000
+java.util.logging.FileHandler.count = 1
+# Default number of locks FileHandler can obtain synchronously.
+# This specifies maximum number of attempts to obtain lock file by FileHandler
+# implemented by incrementing the unique field %u as per FileHandler API documentation.
+java.util.logging.FileHandler.maxLocks = 100
+java.util.logging.FileHandler.formatter = java.util.logging.XMLFormatter
+
+# Limit the messages that are printed on the console to INFO and above.
+java.util.logging.ConsoleHandler.level = INFO
+java.util.logging.ConsoleHandler.formatter = java.util.logging.SimpleFormatter
+
+# Example to customize the SimpleFormatter output format
+# to print one-line log message like this:
+#     <level>: <log message> [<date/time>]
+#
+# java.util.logging.SimpleFormatter.format=%4$s: %5$s [%1$tc]%n
+
+############################################################
+# Facility-specific properties.
+# Provides extra control for each logger.
+############################################################
+
+# For example, set the com.xyz.foo logger to only log SEVERE
+# messages:
+# com.xyz.foo.level = SEVERE

bundledApps/Java/JavaVirtualMachines/jdk17.0.11.jdk/Contents/Home/README

@@ -0,0 +1,79 @@
+######################################################################
+#     Default Access Control File for Remote JMX(TM) Monitoring
+######################################################################
+#
+# Access control file for Remote JMX API access to monitoring.
+# This file defines the allowed access for different roles.  The
+# password file (jmxremote.password by default) defines the roles and their
+# passwords.  To be functional, a role must have an entry in
+# both the password and the access files.
+#
+# The default location of this file is $JRE/conf/management/jmxremote.access
+# You can specify an alternate location by specifying a property in
+# the management config file $JRE/conf/management/management.properties
+# (See that file for details)
+#
+# The file format for password and access files is syntactically the same
+# as the Properties file format.  The syntax is described in the Javadoc
+# for java.util.Properties.load.
+# A typical access file has multiple lines, where each line is blank,
+# a comment (like this one), or an access control entry.
+#
+# An access control entry consists of a role name, and an
+# associated access level.  The role name is any string that does not
+# itself contain spaces or tabs.  It corresponds to an entry in the
+# password file (jmxremote.password).  The access level is one of the
+# following:
+#       "readonly" grants access to read attributes of MBeans.
+#                   For monitoring, this means that a remote client in this
+#                   role can read measurements but cannot perform any action
+#                   that changes the environment of the running program.
+#       "readwrite" grants access to read and write attributes of MBeans,
+#                   to invoke operations on them, and optionally
+#                   to create or remove them. This access should be granted
+#                   only to trusted clients, since they can potentially
+#                   interfere with the smooth operation of a running program.
+#
+# The "readwrite" access level can optionally be followed by the "create" and/or
+# "unregister" keywords.  The "unregister" keyword grants access to unregister
+# (delete) MBeans.  The "create" keyword grants access to create MBeans of a
+# particular class or of any class matching a particular pattern.  Access
+# should only be granted to create MBeans of known and trusted classes.
+#
+# For example, the following entry would grant readwrite access
+# to "controlRole", as well as access to create MBeans of the class
+# javax.management.monitor.CounterMonitor and to unregister any MBean:
+#  controlRole readwrite \
+#              create javax.management.monitor.CounterMonitorMBean \
+#              unregister
+# or equivalently:
+#  controlRole readwrite unregister create javax.management.monitor.CounterMBean
+#
+# The following entry would grant readwrite access as well as access to create
+# MBeans of any class in the packages javax.management.monitor and
+# javax.management.timer:
+#  controlRole readwrite \
+#              create javax.management.monitor.*,javax.management.timer.* \
+#              unregister
+#
+# The \ character is defined in the Properties file syntax to allow continuation
+# lines as shown here.  A * in a class pattern matches a sequence of characters
+# other than dot (.), so javax.management.monitor.* matches
+# javax.management.monitor.CounterMonitor but not
+# javax.management.monitor.foo.Bar.
+#
+# A given role should have at most one entry in this file.  If a role
+# has no entry, it has no access.
+# If multiple entries are found for the same role name, then the last
+# access entry is used.
+#
+#
+# Default access control entries:
+# o The "monitorRole" role has readonly access.
+# o The "controlRole" role has readwrite access and can create the standard
+#   Timer and Monitor MBeans defined by the JMX API.
+
+monitorRole   readonly
+controlRole   readwrite \
+              create javax.management.monitor.*,javax.management.timer.* \
+              unregister

bundledApps/Java/JavaVirtualMachines/jdk17.0.11.jdk/Contents/Home/bin/jar

@@ -0,0 +1,115 @@
+# ----------------------------------------------------------------------
+#           Template for jmxremote.password
+#
+# o Copy this template to jmxremote.password
+# o Set the user/password entries in jmxremote.password
+# o Change the permission of jmxremote.password to be accessible
+#   only by the owner.
+# o The jmxremote.passwords file will be re-written by the server
+#   to replace all plain text passwords with hashed passwords when
+#   the file is read by the server.
+#
+
+##############################################################
+#        Password File for Remote JMX Monitoring
+##############################################################
+#
+# Password file for Remote JMX API access to monitoring.  This
+# file defines the different roles and their passwords.  The access
+# control file (jmxremote.access by default) defines the allowed
+# access for each role.  To be functional, a role must have an entry
+# in both the password and the access files.
+#
+# Default location of this file is $JRE/conf/management/jmxremote.password
+# You can specify an alternate location by specifying a property in
+# the management config file $JRE/conf/management/management.properties
+# or by specifying a system property (See that file for details).
+
+##############################################################
+#    File format of the jmxremote.password file
+##############################################################
+#
+# The file contains multiple lines where each line is blank,
+# a comment (like this one), or a password entry.
+#
+# password entry follows the below syntax
+#   role_name W [clearPassword|hashedPassword]
+#
+# role_name is any string that does not itself contain spaces or tabs.
+# W = spaces or tabs
+#
+# Passwords can be specified via clear text or via a hash. Clear text password
+# is any string that does not contain spaces or tabs. Hashed passwords must
+# follow the below format.
+# hashedPassword = base64_encoded_64_byte_salt W base64_encoded_hash W hash_algorithm
+# where,
+#   base64_encoded_64_byte_salt = 64 byte random salt
+#   base64_encoded_hash = Hash_algorithm(password + salt)
+#   W = spaces or tabs
+#   hash_algorithm = Algorithm string specified using the format below
+#       https://docs.oracle.com/javase/9/docs/specs/security/standard-names.html#messagedigest-algorithms
+#       This is an optional field. If not specified, SHA3-512 will be assumed.
+#
+# If passwords are in clear, they will be overwritten by their hash if all of
+# the below criteria are met.
+#   * com.sun.management.jmxremote.password.toHashes property is set to true in
+#     management.properties file
+#   * the password file is writable
+#   * the system security policy allows writing into the password file, if a
+#     security manager is configured
+#
+# In order to change the password for a role, replace the hashed password entry
+# with a new clear text password or a new hashed password. If the new password
+# is in clear, it will be replaced with its hash when a new login attempt is made.
+#
+# A given role should have at most one entry in this file.  If a role
+# has no entry, it has no access.
+# If multiple entries are found for the same role name, then the last one
+# is used.
+#
+# A user generated hashed password file can also be used instead of clear-text
+# password file. If generated by the user, hashed passwords must follow the
+# format specified above.
+#
+# Caution: It is recommended not to edit the password file while the
+# agent is running, as edits could be lost if a client connection triggers the
+# hashing of the password file at the same time that the file is externally modified.
+# The integrity of the file is guaranteed, but any external edits made to the
+# file during the short period between the time that the agent reads the file
+# and the time that it writes it back might get lost
+
+##############################################################
+#    File permissions of the jmxremote.password file
+##############################################################
+#       This file must be made accessible by ONLY the owner,
+#       otherwise the program will exit with an error.
+#
+# In a typical installation, this file can be accessed by anybody on the
+# local machine, and possibly by people on other machines.
+# For security, you should either restrict the access to this file except for owner,
+# or specify another, less accessible file in the management config file
+# as described above.
+#
+# In order to prevent inadverent edits to the password file in the 
+# production environment, it is recommended to deploy a read-only 
+# hashed password file. The hashed entries for clear passwords can be generated 
+# in advance by running the JMX agent.
+#
+
+##############################################################
+#    Sample of the jmxremote.password file
+##############################################################
+# Following are two commented-out entries.  The "monitorRole" role has
+# password "QED".  The "controlRole" role has password "R&D". This is an example
+# of specifying passwords in the clear
+#
+#   monitorRole  QED
+#   controlRole  R&D
+# 
+# Once a login attempt is made, passwords will be hashed and the file will have 
+# below entries with clear passwords overwritten by their respective 
+# SHA3-512 hash
+#
+#   monitorRole trilby APzBTt34rV2l+OMbuvbnOQ4si8UZmfRCVbIY1+fAofV5CkQzXS/FDMGteQQk/R3q1wtt104qImzJEA7gCwl6dw== 4EeTdSJ7X6Imu0Mb+dWqIns7a7QPIBoM3NB/XlpMQSPSicE7PnlALVWn2pBY3Q3pGDHyAb32Hd8GUToQbUhAjA== SHA3-512
+#   controlRole roHEJSbRqSSTII4Z4+NOCV2OJaZVQ/dw153Fy2u4ILDP9XiZ426GwzCzc3RtpoqNMwqYIcfdd74xWXSMrWtGaA== w9qDsekgKn0WOVJycDyU0kLBa081zbStcCjUAVEqlfon5Sgx7XHtaodbmzpLegA1jT7Ag36T0zHaEWRHJe2fdA== SHA3-512
+#