From ec01b8a27e687a7b22d349c96a66646f1957b1f7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9ophas=20Fournier?= Date: Mon, 19 May 2025 15:46:15 +0200 Subject: [PATCH 1/3] GCI96 avoidIterativeMatrixOperations #Python #DLG #Build Co-authored-by: DataLabGroupe-CreditAgricole --- CHANGELOG.md | 2 + .../python/PythonRuleRepository.java | 3 +- .../AvoidIterativeMatrixOperations.java | 154 ++++++++++++++++++ .../AvoidIterativeMatrixOperationsTest.java | 29 ++++ .../checks/avoidIterativeMatrixOperations.py | 43 +++++ 5 files changed, 230 insertions(+), 1 deletion(-) create mode 100644 src/main/java/org/greencodeinitiative/creedengo/python/checks/AvoidIterativeMatrixOperations.java create mode 100644 src/test/java/org/greencodeinitiative/creedengo/python/checks/AvoidIterativeMatrixOperationsTest.java create mode 100644 src/test/resources/checks/avoidIterativeMatrixOperations.py diff --git a/CHANGELOG.md b/CHANGELOG.md index bea9b72..a22e08a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added +- Add rule GCI96 avoidIterativeMatrixOperations + ### Changed - compatibility updates for SonarQube 25.5.0 diff --git a/src/main/java/org/greencodeinitiative/creedengo/python/PythonRuleRepository.java b/src/main/java/org/greencodeinitiative/creedengo/python/PythonRuleRepository.java index c385979..d8f70ea 100644 --- a/src/main/java/org/greencodeinitiative/creedengo/python/PythonRuleRepository.java +++ b/src/main/java/org/greencodeinitiative/creedengo/python/PythonRuleRepository.java @@ -40,7 +40,8 @@ public class PythonRuleRepository implements RulesDefinition, PythonCustomRuleRe AvoidFullSQLRequest.class, AvoidListComprehensionInIterations.class, DetectUnoptimizedImageFormat.class, - AvoidMultipleIfElseStatementCheck.class + AvoidMultipleIfElseStatementCheck.class, + avoidIterativeMatrixOperations.class ); public static final String LANGUAGE = "py"; diff --git a/src/main/java/org/greencodeinitiative/creedengo/python/checks/AvoidIterativeMatrixOperations.java b/src/main/java/org/greencodeinitiative/creedengo/python/checks/AvoidIterativeMatrixOperations.java new file mode 100644 index 0000000..0cb9602 --- /dev/null +++ b/src/main/java/org/greencodeinitiative/creedengo/python/checks/AvoidIterativeMatrixOperations.java @@ -0,0 +1,154 @@ +/* + * creedengo - Python language - Provides rules to reduce the environmental footprint of your Python programs + * Copyright © 2024 Green Code Initiative (https://green-code-initiative.org) + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.greencodeinitiative.creedengo.python.checks; + + +import org.sonar.check.Rule; +import org.sonar.plugins.python.api.PythonSubscriptionCheck; +import org.sonar.plugins.python.api.SubscriptionContext; +import org.sonar.plugins.python.api.tree.Expression; +import org.sonar.plugins.python.api.tree.BinaryExpression; +import org.sonar.plugins.python.api.tree.CompoundAssignmentStatement; +import org.sonar.plugins.python.api.tree.ForStatement; +import org.sonar.plugins.python.api.tree.Tree; +import org.sonar.plugins.python.api.tree.Statement; +import org.sonar.plugins.python.api.tree.SubscriptionExpression; +import org.sonar.plugins.python.api.tree.AssignmentStatement; + +import java.util.List; + +@Rule(key = "GCI96") + +public class AvoidIterativeMatrixOperations extends PythonSubscriptionCheck { + + private static final String DESCRIPTION = "Avoid iterative matrix operations, use numpy dot or outer function instead"; + + @Override + public void initialize(Context context) { + context.registerSyntaxNodeConsumer(Tree.Kind.FOR_STMT, this::visitForStatement); + } + + private void visitForStatement(SubscriptionContext context) { + ForStatement forStatement = (ForStatement) context.syntaxNode(); + if (isDotProduct(forStatement) || isOuterProduct(forStatement) || isMatrixDotProduct(forStatement)) { + context.addIssue(forStatement.firstToken(), DESCRIPTION); + } + } + + + private boolean isDotProduct(ForStatement forStatement) { + List statements = forStatement.body().statements(); + for (Statement stmt : statements) { + if (stmt.is(Tree.Kind.COMPOUND_ASSIGNMENT)) { + CompoundAssignmentStatement assign = (CompoundAssignmentStatement) stmt; + Expression lhsExpression = assign.lhsExpression(); + if (assign.compoundAssignmentToken().value().equals("+=") + && isMultiplicationOfIndexedElements(assign.rhsExpression(),false) + && !isDoubleSubscription(lhsExpression)) { + System.out.println("Dot product found"); + return true; + } + } + } + return false; + } + + private boolean isOuterProduct(ForStatement outerForStatement) { + List outerStatements = outerForStatement.body().statements(); + for (Statement outerStatement : outerStatements) { + if (outerStatement.is(Tree.Kind.FOR_STMT)) { + ForStatement innerForStatement = (ForStatement) outerStatement; + List innerStatements = innerForStatement.body().statements(); + for (Statement innermostStatement : innerStatements) { + if (isOuterProductOperation(innermostStatement)) { + System.out.println("Outer product found"); + return true; + } + } + } + } + return false; + + } + private boolean isOuterProductOperation(Statement statement) { + if (statement.is(Tree.Kind.ASSIGNMENT_STMT)) { + AssignmentStatement assignmentStmt = (AssignmentStatement) statement; + Expression lhsExpression = assignmentStmt.lhsExpressions().get(0).expressions().get(0); + + if (isDoubleSubscription(lhsExpression)) { + Expression rhsExpression = assignmentStmt.assignedValue(); + if (rhsExpression.is(Tree.Kind.MULTIPLICATION)) { + return isMultiplicationOfIndexedElements(rhsExpression,false); + } + } + } + return false;} + + private boolean isMatrixDotProduct(ForStatement outerForStatement) { + List outerStatements = outerForStatement.body().statements(); + for (Statement outerStatement : outerStatements) { + if (outerStatement.is(Tree.Kind.FOR_STMT)) { + ForStatement middleForStatement = (ForStatement) outerStatement; + List middleStatements = middleForStatement.body().statements(); + for (Statement middleStatement : middleStatements) { + if (middleStatement.is(Tree.Kind.FOR_STMT)) { + ForStatement innerForStatement = (ForStatement) middleStatement; + List innerStatements = innerForStatement.body().statements(); + for (Statement innermostStatement : innerStatements) { + if (isMatrixDotProductOperation(innermostStatement)) { + System.out.println("Matrix dot product found"); + return true; + } + } + } + } + } + } + return false; + } + + private boolean isMatrixDotProductOperation(Statement statement) { + if (statement.is(Tree.Kind.COMPOUND_ASSIGNMENT)) { + CompoundAssignmentStatement compoundStatement = (CompoundAssignmentStatement) statement; + String operator = compoundStatement.compoundAssignmentToken().value(); + if (operator.equals("+=")) { + if (isDoubleSubscription(compoundStatement.lhsExpression())) { + Expression rhsExpression = compoundStatement.rhsExpression(); + return isMultiplicationOfIndexedElements(rhsExpression, true); + } + } + } + return false; + } + + private boolean isMultiplicationOfIndexedElements(Expression expr, boolean matrixOps) { + if (expr.is(Tree.Kind.MULTIPLICATION)) { + BinaryExpression bin = (BinaryExpression) expr; + if (matrixOps) { + return isDoubleSubscription(bin.leftOperand()) && isDoubleSubscription(bin.rightOperand()); + } else { + return bin.leftOperand().is(Tree.Kind.SUBSCRIPTION) && bin.rightOperand().is(Tree.Kind.SUBSCRIPTION); + } + } + return false; + } + + private boolean isDoubleSubscription(Expression expr) { + return expr.is(Tree.Kind.SUBSCRIPTION) && ((SubscriptionExpression) expr).object().is(Tree.Kind.SUBSCRIPTION); + } +} \ No newline at end of file diff --git a/src/test/java/org/greencodeinitiative/creedengo/python/checks/AvoidIterativeMatrixOperationsTest.java b/src/test/java/org/greencodeinitiative/creedengo/python/checks/AvoidIterativeMatrixOperationsTest.java new file mode 100644 index 0000000..ecf2ba5 --- /dev/null +++ b/src/test/java/org/greencodeinitiative/creedengo/python/checks/AvoidIterativeMatrixOperationsTest.java @@ -0,0 +1,29 @@ +/* + * creedengo - Python language - Provides rules to reduce the environmental footprint of your Python programs + * Copyright © 2024 Green Code Initiative (https://green-code-initiative.org) + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.greencodeinitiative.creedengo.python.checks; + +import org.junit.Test; +import org.sonar.python.checks.utils.PythonCheckVerifier; + +public class AvoidIterativeMatrixOperationsTest { + + @Test + public void test() { + PythonCheckVerifier.verify("src/test/resources/checks/avoidIterativeMatrixOperations.py", new AvoidIterativeMatrixOperations()); + } +} diff --git a/src/test/resources/checks/avoidIterativeMatrixOperations.py b/src/test/resources/checks/avoidIterativeMatrixOperations.py new file mode 100644 index 0000000..dd18422 --- /dev/null +++ b/src/test/resources/checks/avoidIterativeMatrixOperations.py @@ -0,0 +1,43 @@ +import numpy as np + +# Test 1: Simple dot product +a = [1, 2, 3, 4] +b = [2, 3, 4, 5] + +dot = 0 +for i in range(len(a)): # Noncompliant {{Avoid iterative matrix operations, use numpy dot or outer function instead}} + dot += a[i] * b[i] + +dot_numpy = np.dot(a, b) # Compliant + +# Test 2: Matrix dot product +A = [[1, 2], [3, 4]] +B = [[5, 6], [7, 8]] + +def iterative_matrix_product(A, B): + results = [[0 for _ in range(len(B[0]))] for _ in range(len(A))] + + for i in range(len(A)): # Noncompliant {{Avoid iterative matrix operations, use numpy dot or outer function instead}} + for j in range(len(B[0])): + for k in range(len(B)): + results[i][j] += A[i][k] * B[k][j] + + return results + +results = iterative_matrix_product(A, B) + + +results_numpy = np.dot(A, B) # Compliant + +# Test 3: Outer product +x = np.random.rand(100) +y = np.random.rand(100) + + +o = np.zeros((len(x), len(y))) +for i in range(len(x)): # Noncompliant {{Avoid iterative matrix operations, use numpy dot or outer function instead}} + for j in range(len(y)): + o[i][j] = x[i] * y[j] + + +outer_numpy = np.outer(x, y) # Compliant From dbc5306124f12323adc732d7c3ead9324dfde633 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9ophas=20Fournier?= Date: Tue, 27 May 2025 09:52:39 +0200 Subject: [PATCH 2/3] GCI96 avoidIterativeMatrixOperations slight fix Co-authored-by: DataLabGroupe-CreditAgricole --- .../creedengo/python/PythonRuleRepository.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/org/greencodeinitiative/creedengo/python/PythonRuleRepository.java b/src/main/java/org/greencodeinitiative/creedengo/python/PythonRuleRepository.java index d8f70ea..f83a8ff 100644 --- a/src/main/java/org/greencodeinitiative/creedengo/python/PythonRuleRepository.java +++ b/src/main/java/org/greencodeinitiative/creedengo/python/PythonRuleRepository.java @@ -41,7 +41,7 @@ public class PythonRuleRepository implements RulesDefinition, PythonCustomRuleRe AvoidListComprehensionInIterations.class, DetectUnoptimizedImageFormat.class, AvoidMultipleIfElseStatementCheck.class, - avoidIterativeMatrixOperations.class + AvoidIterativeMatrixOperations.class ); public static final String LANGUAGE = "py"; From e1c193f849811ab4692ee5bfd525265e8168ee29 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9ophas=20Fournier?= Date: Tue, 27 May 2025 10:12:15 +0200 Subject: [PATCH 3/3] GCI96 avoidIterativeMatrixOperations add multiple tests Co-authored-by: DataLabGroupe-CreditAgricole --- .../AvoidIterativeMatrixOperations.java | 35 ++++--- .../checks/avoidIterativeMatrixOperations.py | 92 +++++++++++++++++-- 2 files changed, 109 insertions(+), 18 deletions(-) diff --git a/src/main/java/org/greencodeinitiative/creedengo/python/checks/AvoidIterativeMatrixOperations.java b/src/main/java/org/greencodeinitiative/creedengo/python/checks/AvoidIterativeMatrixOperations.java index 0cb9602..073bc5d 100644 --- a/src/main/java/org/greencodeinitiative/creedengo/python/checks/AvoidIterativeMatrixOperations.java +++ b/src/main/java/org/greencodeinitiative/creedengo/python/checks/AvoidIterativeMatrixOperations.java @@ -86,18 +86,31 @@ private boolean isOuterProduct(ForStatement outerForStatement) { } private boolean isOuterProductOperation(Statement statement) { - if (statement.is(Tree.Kind.ASSIGNMENT_STMT)) { - AssignmentStatement assignmentStmt = (AssignmentStatement) statement; - Expression lhsExpression = assignmentStmt.lhsExpressions().get(0).expressions().get(0); - - if (isDoubleSubscription(lhsExpression)) { - Expression rhsExpression = assignmentStmt.assignedValue(); - if (rhsExpression.is(Tree.Kind.MULTIPLICATION)) { - return isMultiplicationOfIndexedElements(rhsExpression,false); - } - } + if (statement.is(Tree.Kind.ASSIGNMENT_STMT)) { + AssignmentStatement assignmentStmt = (AssignmentStatement) statement; + Expression lhsExpression = assignmentStmt.lhsExpressions().get(0).expressions().get(0); + + if (isDoubleSubscription(lhsExpression)) { + Expression rhsExpression = assignmentStmt.assignedValue(); + return containsMultiplicationOfIndexedElements(rhsExpression, false); } - return false;} + } + return false; +} + +private boolean containsMultiplicationOfIndexedElements(Expression expr, boolean matrixOps) { + if (isMultiplicationOfIndexedElements(expr, matrixOps)) { + return true; + } + + if (expr instanceof BinaryExpression) { + BinaryExpression bin = (BinaryExpression) expr; + return containsMultiplicationOfIndexedElements(bin.leftOperand(), matrixOps) + || containsMultiplicationOfIndexedElements(bin.rightOperand(), matrixOps); + } + + return false; +} private boolean isMatrixDotProduct(ForStatement outerForStatement) { List outerStatements = outerForStatement.body().statements(); diff --git a/src/test/resources/checks/avoidIterativeMatrixOperations.py b/src/test/resources/checks/avoidIterativeMatrixOperations.py index dd18422..b52709e 100644 --- a/src/test/resources/checks/avoidIterativeMatrixOperations.py +++ b/src/test/resources/checks/avoidIterativeMatrixOperations.py @@ -6,38 +6,116 @@ dot = 0 for i in range(len(a)): # Noncompliant {{Avoid iterative matrix operations, use numpy dot or outer function instead}} - dot += a[i] * b[i] + dot += a[i] * b[i] dot_numpy = np.dot(a, b) # Compliant # Test 2: Matrix dot product A = [[1, 2], [3, 4]] -B = [[5, 6], [7, 8]] +B = [[5, 6], [7, 8]] def iterative_matrix_product(A, B): results = [[0 for _ in range(len(B[0]))] for _ in range(len(A))] - + for i in range(len(A)): # Noncompliant {{Avoid iterative matrix operations, use numpy dot or outer function instead}} for j in range(len(B[0])): for k in range(len(B)): results[i][j] += A[i][k] * B[k][j] - + return results results = iterative_matrix_product(A, B) - results_numpy = np.dot(A, B) # Compliant # Test 3: Outer product x = np.random.rand(100) y = np.random.rand(100) - o = np.zeros((len(x), len(y))) for i in range(len(x)): # Noncompliant {{Avoid iterative matrix operations, use numpy dot or outer function instead}} for j in range(len(y)): o[i][j] = x[i] * y[j] - outer_numpy = np.outer(x, y) # Compliant + +# Test 4: Dot product with different variable names +vec1 = [1, 2, 3] +vec2 = [4, 5, 6] +res = 0 +for idx in range(3): # Noncompliant {{Avoid iterative matrix operations, use numpy dot or outer function instead}} + res += vec1[idx] * vec2[idx] + +# Test 5: False positive - scalar addition in loop +total = 0 +for i in range(10): + total += i # Compliant + +# Test 6: False positive - unrelated list indexing +c = [10, 20, 30] +d = [5, 6, 7] +e = [] +for i in range(len(c)): + e.append(c[i] + d[i]) # Compliant + +# Test 7: Dot product in list comprehension (should be compliant) +dp = sum([a[i] * b[i] for i in range(len(a))]) # Compliant + +# Test 8: Double subscription but not matrix op +m = [[1, 2], [3, 4]] +n = [[5, 6], [7, 8]] +for i in range(len(m)): + for j in range(len(n)): + print(m[i][j] + n[i][j]) # Compliant + +# Test 9: Outer product with extra operation +x = [1, 2] +y = [3, 4] +result = [[0]*len(y) for _ in range(len(x))] +for i in range(len(x)): # Noncompliant {{Avoid iterative matrix operations, use numpy dot or outer function instead}} + for j in range(len(y)): + result[i][j] = x[i] * y[j] + 1 + +# Test 9: Outer product with extra operation +x = [1, 2] +y = [3, 4] +result = [[0]*len(y) for _ in range(len(x))] +for i in range(len(x)): # Noncompliant {{Avoid iterative matrix operations, use numpy dot or outer function instead}} + for j in range(len(y)): + result[i][j] = x[i] * y[j] -10 + +# Test 10: 3-level nested matrix product with aliases +X = [[1, 2], [3, 4]] +Y = [[5, 6], [7, 8]] +Z = [[0, 0], [0, 0]] +for r in range(2): # Noncompliant {{Avoid iterative matrix operations, use numpy dot or outer function instead}} + for c in range(2): + for t in range(2): + Z[r][c] += X[r][t] * Y[t][c] + +# Test 11: False positive - nested loops without multiplication +total = 0 +for i in range(10): + for j in range(5): + total += i + j # Compliant + +# Test 12: Matrix dot with transpose +M1 = [[1, 2], [3, 4]] +M2 = [[5, 7], [6, 8]] +out = [[0 for _ in range(len(M2))] for _ in range(len(M1))] +for i in range(len(M1)): # Noncompliant {{Avoid iterative matrix operations, use numpy dot or outer function instead}} + for j in range(len(M2)): + for k in range(len(M1[0])): + out[i][j] += M1[i][k] * M2[j][k] # Transposed multiplication + +# Test 13: Outer product with offset indexing (still counts) +x = [1, 2] +y = [3, 4] +res = [[0, 0], [0, 0]] +for i in range(2): # Noncompliant {{Avoid iterative matrix operations, use numpy dot or outer function instead}} + for j in range(2): + res[i][j] = x[i] * y[j] + + +# Test 15: Matrix dot using zip (compliant) +res = sum(i * j for i, j in zip(a, b)) # Compliant