Fix indexing (#13498)

* Add support for short output for BibEntry

* Add trace logging for entry addition

* Fix maxlength handling

* Refine coment

* Use LatexToUnicodeAdapter

* Fix exception on empty field when indexing

* Use @nonnull annotation

* Add PR number

* Apply suggestion from @calixtus

Co-authored-by: Carl Christian Snethlage <50491877+calixtus@users.noreply.github.com>

* Fix checkstyle

* Remove CHANGELOG entry

---------

Co-authored-by: Carl Christian Snethlage <50491877+calixtus@users.noreply.github.com>

Author: koppor

jablib/src/main/java/org/jabref/logic/search/indexing/BibFieldsIndexer.java

@@ -11,7 +11,6 @@
 import java.util.regex.Pattern;
 
 import org.jabref.logic.l10n.Localization;
-import org.jabref.logic.layout.format.LatexToUnicodeFormatter;
 import org.jabref.logic.util.BackgroundTask;
 import org.jabref.logic.util.HeadlessExecutorService;
 import org.jabref.model.database.BibDatabaseContext;
@@ -23,6 +22,7 @@
 import org.jabref.model.entry.field.FieldProperty;
 import org.jabref.model.entry.field.StandardField;
 import org.jabref.model.search.PostgreConstants;
+import org.jabref.model.strings.LatexToUnicodeAdapter;
 
 import io.github.thibaultmeyer.cuid.CUID;
 import org.slf4j.Logger;
@@ -37,7 +37,7 @@
 
 public class BibFieldsIndexer {
     private static final Logger LOGGER = LoggerFactory.getLogger(BibFieldsIndexer.class);
-    private static final LatexToUnicodeFormatter LATEX_TO_UNICODE_FORMATTER = new LatexToUnicodeFormatter();
+
     private static final Pattern GROUPS_SEPARATOR_REGEX = Pattern.compile("\s*,\s*");
     private static final Set<Field> DATE_FIELDS = Set.of(StandardField.DATE, StandardField.YEAR, StandardField.MONTH, StandardField.DAY);
 
@@ -209,20 +209,21 @@ private void addToIndex(BibEntry bibEntry) {
         try (PreparedStatement preparedStatement = connection.prepareStatement(insertFieldQuery);
              PreparedStatement preparedStatementSplitValues = connection.prepareStatement(insertIntoSplitTable)) {
             String entryId = bibEntry.getId();
+            LOGGER.atTrace().setMessage("Adding entry {}").addArgument(() -> bibEntry.getKeyAuthorTitleYear()).log();
             for (Map.Entry<Field, String> fieldPair : bibEntry.getFieldMap().entrySet()) {
                 Field field = fieldPair.getKey();
                 String value = fieldPair.getValue();
 
-                // If a field exists, there also exists a resolved field latex free.
-                // We add a `.orElse("")` only because there could be some flaw in the future in the code - and we want to have search working even if the flaws are present.
-                // To uncover these flaws, we add the "assert" statement.
-                // One potential future flaw is that the bibEntry is modified concurrently and the field being deleted.
                 // Skip indexing of date-related fields separately to ensure proper handling later in the process.
                 if (!DATE_FIELDS.contains(field)) {
-                    Optional<String> resolvedFieldLatexFree = bibEntry.getResolvedFieldOrAliasLatexFree(field, this.databaseContext.getDatabase());
-                    assert resolvedFieldLatexFree.isPresent();
-                    addBatch(preparedStatement, entryId, field, value, resolvedFieldLatexFree.orElse(""));
+                    // If a field exists, there also exists a resolved field latex free.
+                    // Only exception: If the content of the field is empty, then the resolved field is also empty. Example: `series = {{}}`.
+                    String resolvedFieldLatexFree = bibEntry.getResolvedFieldOrAliasLatexFree(field, this.databaseContext.getDatabase()).orElse("");
+
+                    // One potential future flaw is that the bibEntry is modified concurrently and the field being deleted.
+                    addBatch(preparedStatement, entryId, field, value, resolvedFieldLatexFree);
                 }
+
                 // region Handling of known multi-value fields
                 // split and convert to Unicode
                 if (field.getProperties().contains(FieldProperty.PERSON_NAMES)) {
@@ -457,7 +458,7 @@ private static void addAuthors(String value, PreparedStatement preparedStatement
     }
 
     private static void addBatch(PreparedStatement preparedStatement, String entryId, Field field, String value) {
-        addBatch(preparedStatement, entryId, field, value, LATEX_TO_UNICODE_FORMATTER.format(value));
+        addBatch(preparedStatement, entryId, field, value, LatexToUnicodeAdapter.format(value));
     }
 
     /**

jablib/src/main/java/org/jabref/model/database/BibDatabase.java

@@ -41,6 +41,7 @@
 
 import com.google.common.eventbus.EventBus;
 import com.google.common.eventbus.Subscribe;
+import org.jspecify.annotations.NonNull;
 import org.jspecify.annotations.Nullable;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -419,8 +420,7 @@ public synchronized boolean hasStringByName(String label) {
      * Resolves any references to strings contained in this field content,
      * if possible.
      */
-    public String resolveForStrings(String content) {
-        Objects.requireNonNull(content, "Content for resolveForStrings must not be null.");
+    public String resolveForStrings(@NonNull String content) {
         return resolveContent(content, new HashSet<>(), new HashSet<>());
     }
 

jablib/src/main/java/org/jabref/model/entry/BibEntry.java

@@ -717,7 +717,7 @@ public Object clone() {
     /// as is. In case the JabRef "hack" for distinguishing "field = value" and "field = {value}" (in .bib files) is
     /// used, it is output as "field = {#value#}", which may cause headaches in debugging.
     ///
-    /// Alternative for some more readable output: [#getAuthorTitleYear(int)]
+    /// Alternative for some more readable output: [#getAuthorTitleYear(int)] or [#getKeyAuthorTitleYear(int)].
     ///
     /// @return A user-readable string NOT A VALID BibTeX string
     @Override
@@ -729,6 +729,10 @@ public String getAuthorTitleYear() {
         return getAuthorTitleYear(0);
     }
 
+    public String getKeyAuthorTitleYear() {
+        return getKeyAuthorTitleYear(0);
+    }
+
     /**
      * Creates a short textual description of the entry in the format: <code>Author1, Author2: Title (Year)</code>
      *
@@ -754,12 +758,14 @@ public String getAuthorTitleYear(int maxCharacters) {
                 .append("\" (")
                 .append(yearField)
                 .append(')');
-        String text = textBuilder.toString();
 
-        if ((maxCharacters <= 0) || (text.length() <= maxCharacters)) {
-            return text;
-        }
-        return text.substring(0, maxCharacters + 1) + "...";
+        return StringUtil.limitStringLength(textBuilder.toString(), maxCharacters);
+    }
+
+    public String getKeyAuthorTitleYear(int maxCharacters) {
+        String citationKey = getCitationKey().orElse("N/A");
+        String result = citationKey + " - " + getAuthorTitleYear(0);
+        return StringUtil.limitStringLength(result, maxCharacters);
     }
 
     /**

jablib/src/main/java/org/jabref/model/strings/LatexToUnicodeAdapter.java

@@ -1,12 +1,12 @@
 package org.jabref.model.strings;
 
 import java.text.Normalizer;
-import java.util.Objects;
 import java.util.Optional;
 import java.util.regex.Pattern;
 
 import com.github.tomtung.latex2unicode.LaTeX2Unicode;
 import fastparse.Parsed;
+import org.jspecify.annotations.NonNull;
 
 /**
  * Adapter class for the latex2unicode lib. This is an alternative to our LatexToUnicode class
@@ -25,8 +25,7 @@ public class LatexToUnicodeAdapter {
      * @param inField a String containing LaTeX
      * @return a String with LaTeX resolved into Unicode, or the original String if the LaTeX could not be parsed
      */
-    public static String format(String inField) {
-        Objects.requireNonNull(inField);
+    public static String format(@NonNull String inField) {
         return parse(inField).orElse(Normalizer.normalize(inField, Normalizer.Form.NFC));
     }
 
@@ -36,8 +35,7 @@ public static String format(String inField) {
      * @param inField a String containing LaTeX
      * @return an {@code Optional<String>} with LaTeX resolved into Unicode or {@code empty} on failure.
      */
-    public static Optional<String> parse(String inField) {
-        Objects.requireNonNull(inField);
+    public static Optional<String> parse(@NonNull String inField) {
         String toFormat = UNDERSCORE_MATCHER.matcher(inField).replaceAll(REPLACEMENT_CHAR);
         Parsed<String> parsingResult = LaTeX2Unicode.parse(toFormat);
         if (parsingResult instanceof Parsed.Success) {

jablib/src/main/java/org/jabref/model/strings/StringUtil.java

@@ -569,16 +569,21 @@ public static List<String> tokenizeToList(String buf, String delimstr) {
         return list;
     }
 
+    /// Limits the length of a string to a maximum length.
+    ///
+    /// Note the implementation is different from [StringUtils.substring](https://commons.apache.org/proper/commons-lang/javadocs/api-2.6/org/apache/commons/lang/StringUtils.html#substring%28java.lang.String,%20int,%20int%29), because it accepts parameters smaller than 4.
+    ///
+    /// @param maxLength the maximum length of the string - <= 0 means no limit
     public static String limitStringLength(String s, int maxLength) {
         if (s == null) {
             return "";
         }
 
-        if (s.length() <= maxLength) {
+        if (maxLength <= 0 || s.length() <= maxLength) {
             return s;
         }
 
-        return s.substring(0, maxLength - 3) + "...";
+        return s.substring(0, Math.max(0, maxLength - 3)) + "...";
     }
 
     /**

jablib/src/test/java/org/jabref/model/strings/StringUtilTest.java

@@ -15,6 +15,7 @@
 import org.junit.jupiter.api.Test;
 import org.junit.jupiter.params.ParameterizedTest;
 import org.junit.jupiter.params.provider.Arguments;
+import org.junit.jupiter.params.provider.CsvSource;
 import org.junit.jupiter.params.provider.MethodSource;
 
 import static org.junit.jupiter.api.Assertions.assertArrayEquals;
@@ -294,22 +295,22 @@ void intValueOfWithNullExceptionfIfStringEmpty() {
         assertEquals(Optional.empty(), StringUtil.intValueOfOptional(""));
     }
 
-    @Test
-    void limitStringLengthShort() {
-        assertEquals("Test", StringUtil.limitStringLength("Test", 20));
+    @ParameterizedTest
+    @CsvSource({
+            "'Test', 'Test', 20",
+            "'...', 'Test', 3",
+            "'TestTes...', 'TestTestTestTestTest', 10",
+            "'', , 10"
+    })
+    void limitStringLength(String expected, String input, int maxLength) {
+        assertEquals(expected, StringUtil.limitStringLength(input, maxLength));
     }
 
     @Test
     void limitStringLengthLimiting() {
-        assertEquals("TestTes...", StringUtil.limitStringLength("TestTestTestTestTest", 10));
         assertEquals(10, StringUtil.limitStringLength("TestTestTestTestTest", 10).length());
     }
 
-    @Test
-    void limitStringLengthNullInput() {
-        assertEquals("", StringUtil.limitStringLength(null, 10));
-    }
-
     @Test
     void replaceSpecialCharacters() {
         assertEquals("Hallo Arger", StringUtil.replaceSpecialCharacters("Hallo Arger"));