⬆️ Upgrade GPT 3.5 Turbo Model (#398)

gpt-3.5-turbo-0613 has been removed, so replace it with new models.

* Updated failed authentication logging codemod to use new models and
`SarifToLLMForMultiOutcomeCodemod`.
* Added a `Model` type to describe GPT models and colocates
model-specific logic such as token counting.

/close #work

Author: gilday

core-codemods/src/main/java/io/codemodder/codemods/LogFailedLoginCodemod.java

@@ -1,14 +1,11 @@
 package io.codemodder.codemods;
 
-import static io.codemodder.CodemodResources.getClassResourceAsString;
-
-import com.contrastsecurity.sarif.Result;
-import com.github.difflib.patch.AbstractDelta;
-import com.github.difflib.patch.InsertDelta;
-import com.github.difflib.patch.Patch;
 import io.codemodder.*;
+import io.codemodder.plugins.llm.CodeChangingLLMRemediationOutcome;
+import io.codemodder.plugins.llm.NoActionLLMRemediationOutcome;
 import io.codemodder.plugins.llm.OpenAIService;
-import io.codemodder.plugins.llm.SarifToLLMForBinaryVerificationAndFixingCodemod;
+import io.codemodder.plugins.llm.SarifToLLMForMultiOutcomeCodemod;
+import io.codemodder.plugins.llm.StandardModel;
 import io.codemodder.providers.sarif.semgrep.SemgrepScan;
 import java.util.List;
 import javax.inject.Inject;
@@ -17,39 +14,68 @@
     id = "pixee:java/log-failed-login",
     importance = Importance.HIGH,
     reviewGuidance = ReviewGuidance.MERGE_AFTER_REVIEW)
-public final class LogFailedLoginCodemod extends SarifToLLMForBinaryVerificationAndFixingCodemod {
+public final class LogFailedLoginCodemod extends SarifToLLMForMultiOutcomeCodemod {
 
   @Inject
   public LogFailedLoginCodemod(
       @SemgrepScan(ruleId = "log-failed-login") final RuleSarif sarif, final OpenAIService openAI) {
-    super(sarif, openAI);
+    super(
+        sarif,
+        openAI,
+        List.of(
+            new NoActionLLMRemediationOutcome(
+                "logs_failed_login_with_logger",
+                """
+                  The code uses a logger to log a message that indicates a failed login attempt.
+                  The message is logged at the INFO or higher level.
+                  """
+                    .replace('\n', ' ')),
+            new NoActionLLMRemediationOutcome(
+                "logs_failed_login_with_console",
+                """
+                  The code sends a message to the console that indicates a failed login attempt.
+                  The code may output this message to either System.out or System.err.
+                  """
+                    .replace('\n', ' ')),
+            new NoActionLLMRemediationOutcome(
+                "throws_exception",
+                """
+                  The code throws an exception that indicates a failed login attempt.
+                  Throwing such an exception is a reasonable alternative to logging the failed login attempt.
+                  When the username for the failed login is in-scope, the exception message includes the username.
+                  """
+                    .replace('\n', ' ')),
+            new NoActionLLMRemediationOutcome(
+                "no_authentication",
+                """
+                  The login validation fails because the request lacks credentials to validate. This is not considered a failed login attempt that requires auditing.
+                  """
+                    .replace('\n', ' ')),
+            new CodeChangingLLMRemediationOutcome(
+                "add_missing_logging",
+                """
+                  None of the other outcomes apply.
+                  The code that validates the login credentials does not log a message when the login attempt fails,
+                  NOR does it throw an exception that reasonably indicates a failed login attempt and includes the username in the exception message.
+                  """
+                    .replace('\n', ' '),
+                """
+                  Immediately following the login failure, add precisely one statement to log the failed login attempt at the INFO level.
+                  If the username for the failed login is in scope, the new log message references the username.
+                  Add exactly one such log statement! Exactly one!
+                  The new log statement is consistent with the rest of the code with respect to formatting, braces, casing, etc.
+                  When no logger is in scope, the new code emits a log message to the console.
+                  """
+                    .replace('\n', ' '))),
+        StandardModel.GPT_4O,
+        StandardModel.GPT_4);
   }
 
   @Override
-  protected String getThreatPrompt(
-      final CodemodInvocationContext context, final List<Result> results) {
-    return getClassResourceAsString(getClass(), "threat_prompt.txt");
-  }
-
-  @Override
-  protected String getFixPrompt() {
-    return getClassResourceAsString(getClass(), "fix_prompt.txt");
-  }
-
-  @Override
-  protected boolean isPatchExpected(Patch<String> patch) {
-    // This codemod should make two or fewer modifications.
-    if (patch.getDeltas().size() > 2) {
-      return false;
-    }
-
-    // This codemod should only insert lines.
-    for (AbstractDelta<String> delta : patch.getDeltas()) {
-      if (!(delta instanceof InsertDelta<String>)) {
-        return false;
-      }
-    }
-
-    return true;
+  protected String getThreatPrompt() {
+    return """
+            The tool has cited an authentication check that does not include a means for auditing failed login attempt.
+            """
+        .replace('\n', ' ');
   }
 }

core-codemods/src/main/resources/io/codemodder/codemods/LogFailedLoginCodemod/fix_prompt.txt

@@ -5,12 +5,45 @@
 import io.codemodder.testutils.Metadata;
 import org.junit.jupiter.api.Disabled;
 
+/**
+ * Tests for the {@link LogFailedLoginCodemod}.
+ *
+ * <p>Test cases that should not have code changes:
+ *
+ * <dl>
+ *   <dt>safe/AuthProvider.java.before
+ *   <dd>Describes a type that performs authentication, but no authentication implemented here.
+ *   <dt>safe/JaaSAuthenticationBroker.java.before
+ *   <dd>Throws exceptions that indicate failed login attempts.
+ *   <dt>safe/LoginServlet.java.before
+ *   <dd>logs authentication failures at the WARN level.
+ *   <dt>safe/Main.java.before
+ *   <dd>Logs a message when authentication fails.
+ *   <dt>safe/MainPrint.before
+ *   <dd>prints to the console when a login attempt fails.
+ *   <dt>safe/Queue.java.before
+ *   <dd>is too large to be analyzed.
+ * </dl>
+ *
+ * Test cases that should have code changes:
+ *
+ * <dl>
+ *   <dt>unsafe/LoginServlet.java.before
+ *   <dd>lacks a log statement before returning unauthorized response
+ *   <dt>unsafe/LoginValidate.java.before
+ *   <dd>lacks a print statement before redirecting to error page.
+ *   <dt>unsafe/MainFame.java.before
+ *   <dd>lacks a log statement before showing the dialog.
+ *   <dt>unsafe/SaltedHashLoginModule
+ *   <dd>lacks a log statement before returning the authenticated decision. That is the correct
+ *       place to log, because it has the username in scope.
+ */
 @Metadata(
     codemodType = LogFailedLoginCodemod.class,
     testResourceDir = "log-failed-login",
     dependencies = {})
 @OpenAIIntegrationTest
-@Disabled("codemod is in disrepair")
+@Disabled("codemod in disrepair - behavior is too indeterminate")
 public final class LogFailedLoginCodemodTest implements LLMVerifyingCodemodTestMixin {
 
   @Override

core-codemods/src/main/resources/io/codemodder/codemods/LogFailedLoginCodemod/threat_prompt.txt

@@ -154,7 +154,7 @@ ResultSet rs = stmt.executeQuery(sql);
                     this.dispose();
             }
             else{
-                    Logger.getLogger(MainFrame.class.getName()).log(Level.WARNING, "Failed login attempt for user: " + uname1);
+                    Logger.getLogger(MainFrame.class.getName()).log(Level.INFO, "Failed login attempt for user: " + uname1);
                     JOptionPane.showMessageDialog(null, "Incorrect Username Or Password", "Login Failed", 2);
                 }
 
@@ -211,6 +211,3 @@ ResultSet rs = stmt.executeQuery(sql);
     private javax.swing.JTextField upass;
     // End of variables declaration//GEN-END:variables
 }
-
-
-